diff --git a/.gitignore b/.gitignore
index 53591c3c..69202bde 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,8 @@
# ide
.idea/
+dist
debug.log
+.DS_Store
*.thread.yml
# Byte-compiled / optimized / DLL files
__pycache__/
@@ -103,7 +105,7 @@ ipython_config.py
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
-#poetry.lock
+poetry.lock
# pdm
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
@@ -163,4 +165,4 @@ cython_debug/
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
#.idea/
-.vscode/
\ No newline at end of file
+.vscode/
diff --git a/README.md b/README.md
index 699965ed..e41e620a 100644
--- a/README.md
+++ b/README.md
@@ -1,246 +1,205 @@
-# GhostOS framework
+# GhostOS
-
-
-
GhostOS framework
-
` marker.
+
+```python
+
+#
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+ from ghostos.core.moss import MossPrompter
+
+
+# The code defined here will execute normally but will not be submitted to the LLM.
+# This code is typically used to define the logic within the lifecycle of MossCompiler/Runtime operations.
+# Shielding these logics helps the LLM to focus more.
+
+def __moss_module_prompt__(prompter: "MossPrompter") -> str:
+ ...
+
+#
+```
+
+#### Code Reflection
+
+We utilize reflection mechanisms to automatically generate Prompts from code information and provide them to the Large
+Language Model.
+The basic idea is similar to how programmers view reference libraries, only allowing the LLM to see the minimal amount
+of information it cares about, mainly the definitions of classes and functions along with key variables.
+Instead of directly providing all the source code to the model.
+
+#### Default Reflection Pattern
+
+`MossRuntime` reflects variables imported into the current Python module and generates their Prompts according to
+certain rules.
+The current rules are as follows:
+
+* Function & Method: Only reflect the function name + doc
+* Abstract class: Reflect the source code
+* pydantic.BaseModel: Reflect the source code
+
+Additionally, any class that implements `ghostos.prompter.PromptAbleClass` will use its `__class_prompt__` method to
+generate the reflection result.
+
+#### Custom Attr Prompt
+
+If the target Python module file defines the magic method `__moss_attr_prompts__`, it will use the provided results to
+override the automatically reflected results.
+
+```python
+def __moss_attr_prompts__() -> "AttrPrompts":
+ yield "key", "prompt"
+```
+
+If the returned prompt is empty, then ignore it to the LLM.
+
+### Runtime Execution
+
+Based on `MossRuntime`, you can execute the code generated by the Large Language Model directly within a temporarily
+compiled module. The benefits of doing this are:
+
+1. The LLM does not need to import all libraries, saving the overhead of tokens.
+2. Accelerate the generation speed, expecting to surpass the output of JSON schema in many cases.
+3. Avoid pollution of the context module by code generated by the Large Language Model.
+4. Compared to executing code in Jupyter or a sandbox, temporarily compiling a module aims to achieve a "minimum context
+ unit."
+
+The basic principle is to use the current module as the context to compile and execute the code generated by the Large
+Language Model. The internal logic is as follows:
+
+```python
+import ghostos.core.moss
+
+runtime: ghostos.core.moss.MossRuntime = ...
+pycontext = runtime.dump_pycontext()
+local_values = runtime.locals()
+
+generated_code: str = ...
+
+filename = pycontext.module if pycontext.module is not None else ""
+compiled = compile(generated_code, filename=filename, mode='exec')
+# 直接编译
+exec(compiled, local_values)
+```
+
+We can request that the code generated by the Large Language Model be a main function. After MossRuntime compiles the
+code, we can immediately execute this function.
+
+```python
+import ghostos.core.moss
+
+runtime: ghostos.core.moss.MossRuntime = ...
+# 包含 main 函数的代码
+generated_code: str = ...
+
+with runtime:
+ result = runtime.execute(target="main", code=generated_code, local_args=["foo", "bar"])
+
+ # 执行过程中的 std output
+ std_output = runtime.dump_std_output()
+ # 获取变更过的 pycontext
+ pycontext = runtime.dump_pycontext()
+```
+
+### Custom Lifecycle functions
+
+`MossRuntime`, during its lifecycle, attempts to locate and execute magic methods within the compiled modules. All magic
+methods are defined
+in [ghostos.core.moss.lifecycle](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/core/moss/lifecycle.py). For details,
+please refer to the file. The main methods include:
+
+```python
+__all__ = [
+ '__moss_compile__', # prepare moss compiler, handle dependencies register
+ '__moss_compiled__', # when moss instance is compiled
+ '__moss_attr_prompts__', # generate custom local attr prompt
+ '__moss_module_prompt__', # define module prompt
+ '__moss_exec__', # execute the generated code attach to the module
+]
+```
+
+### Moss 类
+
+In the target module compiled by `MossCompiler`, you can define a class named `Moss` that inherits
+from `ghostos.core.moss.Moss`. This allows it to receive key dependency injections during its lifecycle, achieving a
+what-you-see-is-what-you-get (WYSIWYG) effect.
+
+The `Moss` class serves two purposes:
+
+1. Automated Dependency Injection: Abstract classes mounted on Moss will receive dependency injection from the IoC
+ container.
+2. Managing Persistent Context: Data objects on the Moss class will be automatically stored in `PyContext`.
+
+The existence of this class is default; even if you do not define it, an instance named `moss` will be generated in the
+compiled temporary module. The `moss` instance can be passed to functions in code generated by the Large Language Model.
+
+For example, regarding context:
+
+```python
+from abc import ABC
+from ghostos.core.moss import Moss as Parent
+
+
+class Foo(ABC):
+ ...
+
+
+class Moss(Parent):
+ int_val: int = 0
+
+ foo: Foo # the abstract class bound to Moss will automatically get injection from MossRuntime.container()
+```
+
+The LLM generated code are:
+
+```python
+# 大模型生成的 main 函数
+def main(moss) -> int:
+ moss.int_var = 123
+ return moss.int_var
+```
+
+Executing this function will change the value of `Moss.int_val` to `123` in the future.
+
+The purpose of this is to manage the context in a WYSIWYG manner. There are several default rules:
+
+1. Variable Storage: All variables bound to the `Moss` instance, including those of type `pydantic.BaseModel`
+ and `int | str | float | bool`, will be automatically stored in `PyContext`.
+2. Abstract Class Dependency Injection: Any class mounted on `Moss` will automatically attempt to inject instances using
+ the IoC Container.
+3. Lifecycle Management: If a class implements `ghostos.core.moss.Injection`, its `on_injection` and `on_destroy`
+ methods will be automatically called when injected into the `moss` instance.
+4. Defining a `Moss` class will not pollute or disrupt the original functionality of the target file.
+
+You can also use `MossRuntime` to obtain all the injection results for the `Moss` class.
+
+```python
+from ghostos.core.moss import Moss, MossRuntime
+
+runtime: MossRuntime = ...
+
+moss_class = runtime.moss_type()
+assert issubclass(moss_class, Moss)
+
+moss_instance = runtime.moss()
+assert isinstance(moss_instance, moss_class)
+
+injections = runtime.moss_injections()
+```
+
+## Examples
+
+Baseline test of `MOSS` located at:
+[ghostos.core.moss.examples](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/core/moss/examples)
+
+## MOSS TestSuite
+
+All source files that can be compiled by `MossCompiler` are also referred to as `MOSS files`.
+
+In these files, the functions, variables, and classes defined can be unit tested, but runtime dependency injection
+requires the construction of a test suite.
+
+`GhostOS` provides a default suite called `ghostos.core.moss.testsuite.MossTextSuite`. For more details, please refer to
+the code.
diff --git a/docs/en/concepts/prompter.md b/docs/en/concepts/prompter.md
new file mode 100644
index 00000000..59d6d16c
--- /dev/null
+++ b/docs/en/concepts/prompter.md
@@ -0,0 +1,62 @@
+# Prompter
+
+Providing prompts to large language models will inevitably move towards structuring and modularization. Traditional
+template languages cannot handle the complexity of Prompt Engineering.
+
+The author believes that Prompt Engineering should use something similar to the front-end interface's `DOM` (Document
+Object Model) to build, possibly called `POM` (Prompt Object Model).
+
+The System Prompt generated by LLM is essentially a `POM Tree`, which can assemble various context-related data objects
+into a prompt.
+
+Some foreseeable benefits of `POM`:
+
+1. Nodes can be encapsulated with data structures, making them easy to reuse in other projects.
+2. Complex UI interfaces can be mapped to `POM Tree`, automatically providing LLM with additional visual object
+ information.
+3. `POM Tree` can be rendered on the front end, making it easy for humans to manage sufficiently complex contexts, even
+ with visual rendering.
+4. `POM Tree` can be programmed, so it can be generated autonomously by Meta-Agents.
+5. Tokens can be prioritized and pruned specifically for `POM Tree`.
+
+The implementation of this technology is not a goal of `GhostOS` itself, but since the open-source community has not yet
+provided a mature `Prompt Object Model` implementation, the author has implemented a simplified version.
+
+详见: [ghostos.prompter](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/prompter.py)
+
+Take [MossAgent](/en/usages/moss_agent.md) as example:
+
+```python
+ def _get_instruction_prompter(self, session: Session, runtime: MossRuntime) -> Prompter:
+ agent = self.ghost
+ return TextPrmt().with_children(
+ # system meta prompt
+ TextPrmt(
+ title="Meta Instruction",
+ content=AGENT_META_INTRODUCTION,
+ ).with_children(
+ TextPrmt(title="GhostOS", content=GHOSTOS_INTRODUCTION),
+ TextPrmt(title="MOSS", content=MOSS_INTRODUCTION),
+ # code context
+ get_moss_context_prompter("Code Context", runtime),
+ ),
+
+ # agent prompt
+ TextPrmt(
+ title="Agent Info",
+ content="The Agent info about who you are and what you are doing: ",
+ ).with_children(
+ get_agent_identity("Identity", agent.__identifier__()),
+ TextPrmt(title="Persona", content=self._get_agent_persona(session, runtime)),
+ TextPrmt(title="Instruction", content=self._get_agent_instruction(session, runtime)),
+ ),
+
+ # context prompt
+ TextPrmt(
+ title="Context",
+ content="",
+ ).with_children(
+ self._get_context_prompter(session),
+ )
+ )
+```
\ No newline at end of file
diff --git a/docs/en/frameworks/eventbus.md b/docs/en/frameworks/eventbus.md
new file mode 100644
index 00000000..ff52215c
--- /dev/null
+++ b/docs/en/frameworks/eventbus.md
@@ -0,0 +1,47 @@
+# EventBus
+
+`GhostOS` manages event communication between Agents, between Agents and the external world, and within Agents
+themselves through the [EventBus](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/core/runtime/events.py) class.
+
+Based on the event bus, we can implement a fully asynchronous Agent. Taking the example of a time-consuming network
+car-hailing:
+
+1. The user converses with the main Agent, requesting that the Agent hail a car.
+2. The Agent calls upon a sub-Agent with car-hailing capabilities to execute the task.
+3. The Agent continues to converse with the user.
+4. The Agent can inquire about the task execution status from the sub-Agent at any time.
+5. After the sub-Agent hails a car, it notifies the main Agent through an Event.
+
+The event bus maintains the Event Loop for all Agents, thereby achieving fully asynchronous communication.
+
+In addition to communication between Agents, communication between external systems and Agents also needs to go through
+the EventBus. However, the `ghostos.abcd.Conversation` abstraction includes the relevant interfaces for this purpose.
+Communication from external systems can include:
+
+* Events that occur in the environment
+* Scheduled tasks
+* Asynchronous callbacks from interfaces
+
+# Event Object
+
+GhostOS
+中的事件对象定义在 [ghostos.core.runtime.events.Event](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/core/runtime/events.py).
+相关 API 详见代码.
+
+# EventBus Registrar
+
+As the base class, EventBus is registered in `ghostos.bootstrap.app_container`. By simply changing the Provider
+registered with EventBus, you can modify its implementation. For more details, see the relevant section on the
+Container.
+
+# Current Implementation
+
+The `EventBus` can be implemented using various technologies, including file-based, relational database-based, and Redis
+or other KV storage-based implementations to achieve an event bus for distributed systems.
+
+Since `GhostOS` lacks development resources, the current implementation is a memory-based dictionary. For more details,
+see [MemEventBusImpl](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/framework/eventbuses/memimpl.py). This means that
+shutting down a running program will result in the loss of events.
+
+In the future, it is hoped that the default implementation of EventBus can be made configurable, allowing users to
+choose between several out-of-the-box solutions such as `file`, `redis`, `mysql` through configuration options.
\ No newline at end of file
diff --git a/docs/en/frameworks/llms.md b/docs/en/frameworks/llms.md
new file mode 100644
index 00000000..81924c7f
--- /dev/null
+++ b/docs/en/frameworks/llms.md
@@ -0,0 +1,8 @@
+# LLMs
+
+GhostOS encapsulates the invocation of Large Language Models within its own abstraction.
+
+For more details, see [ghostos/core/llms](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/core/llms/abcd.py).
+
+> The encapsulation of related capabilities are not yet test enough.
+> More documentation will be enriched when there is time.
\ No newline at end of file
diff --git a/docs/en/frameworks/messages.md b/docs/en/frameworks/messages.md
new file mode 100644
index 00000000..74989c92
--- /dev/null
+++ b/docs/en/frameworks/messages.md
@@ -0,0 +1,44 @@
+# Messages
+
+One of the design goals of `GhostOS` is to implement a fully asynchronous intelligent agent cluster on the server side.
+Therefore, the transmission and storage of historical messages cannot be limited to the client side; they also need to
+be handled on the server side.
+
+To address issues such as streaming message protocol, model compatibility, storage, and reading; `GhostOS` has designed
+its own message container.
+For more details, see [ghostos.core.messages](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/core/messages/message.py)
+
+At present, there is no energy to introduce all the details, so I will focus on introducing a few key concepts:
+
+## Variable Message
+
+`GhostOS` agents are driven by code, so they can transmit various runtime variables in the form of `VariableMessage`,
+including:
+
+1. Passed to the client side, such as streamlit
+2. Transmitted to other Agents
+
+In the historical records, the LLM can see the `vid` parameter of the variable,
+and the corresponding variable can be obtained using
+the [ghostos/contracts/variables](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/contracts/variables.py) library.
+This enables interaction based on variables.
+
+Examples:
+
+1. An Agent transmits its own variables to another Agent.
+2. An Agent sends a variable of a certain data structure to the client, which then renders it on its own.
+3. The client side can send variables in the form of messages, and the Agent can retrieve the variable data structure in
+ the code and manipulate it.
+4. An Agent can manipulate variables seen in the historical context.
+
+## Audio & Image Message
+
+In `GhostOS`, images and audio messages in the historical records are stored in centralized storage, with the message ID
+serving as the storage ID for both images and audio. For more details,
+see [ghostos/contracts/assets](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/contracts/assets.py).
+
+It is expected that in the future, Audio and Image will also support `variable type messages`, allowing large language
+models to manipulate them through code.
+
+For example, a large language model without image recognition capabilities can call another model capable of image
+recognition to assist it in reading images through code.
\ No newline at end of file
diff --git a/docs/en/frameworks/tasks.md b/docs/en/frameworks/tasks.md
new file mode 100644
index 00000000..648fb30c
--- /dev/null
+++ b/docs/en/frameworks/tasks.md
@@ -0,0 +1,154 @@
+# Tasks
+
+One of the core features of `GhostOS` is its fully asynchronous Multi-Agent architecture.
+
+Each running Agent (Ghost) is considered a `minimal stateful unit`, capable of scheduling the execution status of a
+task:
+
+```python
+
+class TaskState(str, Enum):
+ """ runtime state of the task. """
+
+ NEW = "new"
+ """the task is yet created"""
+
+ RUNNING = "running"
+ """the task is running"""
+
+ WAITING = "waiting"
+ """the task needs more inputs"""
+
+ # QUEUED = "queued"
+ # """the task is queued to run"""
+
+ CANCELLED = "cancelled"
+ """the task is canceled"""
+
+ FAILED = "failed"
+ """the task is failed due to an exception"""
+
+ FINISHED = "finished"
+ """the task is finished"""
+```
+
+Agent 可以直接使用 [MOSS](/en/concepts/moss_protocol.md) 提供的类库来操作自身的状态:
+
+```python
+
+class Taskflow(Prompter, ABC):
+ """
+ default operations
+ """
+ MessageKind = Union[str, Message, Any]
+ """message kind shall be string or serializable object"""
+
+ # --- 基本操作 --- #
+ @abstractmethod
+ def finish(self, status: str = "", *replies: MessageKind) -> Operator:
+ """
+ finish self task
+ :param status: describe status of the task
+ :param replies: replies to parent task or user
+ """
+ pass
+
+ @abstractmethod
+ def fail(self, reason: str = "", *replies: MessageKind) -> Operator:
+ """
+ self task failed.
+ :param reason: describe status of the task
+ :param replies: replies to parent task or user
+ """
+ pass
+
+ @abstractmethod
+ def wait(self, status: str = "", *replies: MessageKind) -> Operator:
+ """
+ wait for the parent task or user to provide more information or further instruction.
+ :param status: describe current status
+ :param replies: question, inform or
+ """
+ pass
+
+ @abstractmethod
+ def think(self, *messages: MessageKind, instruction: str = "", sync: bool = False) -> Operator:
+ """
+ start next round thinking on messages
+ :param messages: observe target
+ :param instruction: instruction when receive the observation.
+ :param sync: if True, observe immediately, otherwise check other event first
+ :return:
+ """
+ pass
+
+ @abstractmethod
+ def observe(self, **kwargs) -> Operator:
+ """
+ observe values
+ :param kwargs:
+ :return:
+ """
+
+ @abstractmethod
+ def error(self, *messages: MessageKind) -> Operator:
+ pass
+
+
+class Subtasks(Prompter, ABC):
+ """
+ library that can handle async subtasks by other ghost instance.
+ """
+ MessageKind = Union[str, Message, Any]
+ """message kind shall be string or serializable object"""
+
+ @abstractmethod
+ def cancel(self, name: str, reason: str = "") -> None:
+ """
+ cancel an exists subtask
+ :param name: name of the task
+ :param reason: the reason to cancel it
+ :return:
+ """
+ pass
+
+ @abstractmethod
+ def send(
+ self,
+ name: str,
+ *messages: MessageKind,
+ ctx: Optional[Ghost.ContextType] = None,
+ ) -> None:
+ """
+ send message to an existing subtask
+ :param name: name of the subtask
+ :param messages: the messages to the subtask
+ :param ctx: if given, update the ghost context of the task
+ :return:
+ """
+ pass
+
+ @abstractmethod
+ def create(
+ self,
+ ghost: Ghost,
+ instruction: str = "",
+ ctx: Optional[Ghost.ContextType] = None,
+ task_name: Optional[str] = None,
+ task_description: Optional[str] = None,
+ ) -> None:
+ """
+ create subtask from a ghost instance
+ :param ghost: the ghost instance that handle the task
+ :param instruction: instruction to the ghost
+ :param ctx: the context that the ghost instance needed
+ :param task_name: if not given, use the ghost's name as the task name
+ :param task_description: if not given, use the ghost's description as the task description
+ """
+ pass
+```
+
+This allows multiple Agents to communicate and interact with each other on a `Task` basis.
+
+For more details on the implementation,
+see [ghostos.core.runtime.tasks](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/core/runtime/tasks.py).
\ No newline at end of file
diff --git a/docs/en/frameworks/threads.md b/docs/en/frameworks/threads.md
new file mode 100644
index 00000000..ffd77cd4
--- /dev/null
+++ b/docs/en/frameworks/threads.md
@@ -0,0 +1,12 @@
+# Threads
+
+In order to achieve fully asynchronous Multi-Agent functionality, `GhostOS` needs to manage the context of all Agents
+itself.
+
+Therefore, `GhostOS` has implemented an infrastructure similar to
+the [OpenAI Assistant](https://platform.openai.com/docs/api-reference/assistants).
+
+The historical messages generated by Agents are stored and used with the `GoThreadInfo` structure.
+
+Due to limited personal development capacity, please refer to the detailed implementation
+at [ghostos.core.runtime.threads](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/core/runtime/threads.py).
\ No newline at end of file
diff --git a/docs/en/getting_started/chat_with_ghost.md b/docs/en/getting_started/chat_with_ghost.md
new file mode 100644
index 00000000..f75ad4b3
--- /dev/null
+++ b/docs/en/getting_started/chat_with_ghost.md
@@ -0,0 +1,62 @@
+# Chat
+
+## Quick start
+
+`GhostOS` use [Streamlit](https://streamlit.io/) to offer open-box Agent UI.
+
+run:
+
+```bash
+ghostos web python_modulename_or_filename
+```
+
+can convert a python module or file into a streamlit Agent.
+
+The system default testing Agent is:
+
+```bash
+# start chatbot
+ghostos web ghostos.demo.agents.jojo
+```
+
+You can launch a standalone Python file as an Agent to interpret file content and call the file's relevant
+methods. `GhostOS` will automatically reflect the code to generate the context visible to the Agent.
+
+
+
+## Realtime Chat
+
+`GhostOS` Implements [OpenAI Realtime Beta](https://platform.openai.com/docs/api-reference/realtime).
+
+To use it, you need to install the relevant libraries first:
+
+```bash
+pip install ghostos[realtime]
+```
+
+For configuration details of the real-time model, see [configuration](./configuration.md).
+
+> There are still many bugs and experience issues with the current real-time model.
+> After all, it is still a personal project, so...
+
+## Runtime files
+
+When you converse with an agent using `GhostOS`, the system generates various runtime files, such as:
+
+* thread: stores historical messages.
+* task: stores the state of the conversation state machine.
+* images and audio: images and audio from the process.
+* logs: runtime logs.
+
+All these runtime files are saved in
+the [\[workspace\]/runtime](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/app/runtime) directory.
+
+If you need to clear them, please run:
+
+```bash
+ghostos clear-runtime
+```
+
+## Create Your Agent
+
+see [Usage](/en/usages/moss_agent.md)
\ No newline at end of file
diff --git a/docs/en/getting_started/configuration.md b/docs/en/getting_started/configuration.md
new file mode 100644
index 00000000..ba453948
--- /dev/null
+++ b/docs/en/getting_started/configuration.md
@@ -0,0 +1,61 @@
+# Configuration
+
+Configuration files that `GhostOS` relies on are stored within the workspace.
+
+Running `ghostos init` creates a workspace in the current directory.
+
+The default workspace is located at: [ghostos/app](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/app).
+
+The default location for configuration files is
+at: [ghostos/app/configs](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/app/configs).
+
+## Edit in UI
+
+`GhostOS` default configuration items are defined through `pydantic.BaseModel`, hence it can automatically
+generate `JSON Schema`.
+
+The author has developed the
+repository [streamlit-react-jsonschema](https://github.com/ghost-in-moss/streamlit-react-jsonschema), which is based
+on [react-jsonschema-form](https://react-jsonschema-form.readthedocs.io/) for automated form rendering.
+
+Running `ghostos config` opens a streamlit page where you can visually configure options.
+
+> Since I don't have much time for testing, directly modifying the target configuration file is more reliable and
+> secure.
+
+## LLM Config
+
+`GhostOS` encapsulates its
+own [LLMs Interface](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/core/llms/abcd.py).
+
+For related configuration items,
+see [LLMsConfig](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/core/llms/configs.py).
+
+The configuration file is located
+at [\[workspace\]/configs/llms_conf.yml](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/app/configs/llms_conf.yml).
+
+The currently tested model services include:
+
+- OpenAI
+- Moonshot
+
+Most Agents that do not specify model configurations directly will use the `LLMConfigs.default` model.
+
+## Realtime Beta Config
+
+`GhostOS` support [OpenAI Realtime Beta](https://platform.openai.com/docs/api-reference/realtime).
+
+Configuration Model of
+it: [OpenAIRealtimeAppConfig](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/frameworks/openai_realtime/configs.py).
+
+The config file
+is [\[workspace\]/configs/openai_realtime_config.yml](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/app/configs/openai_realtime_config.yml).
+
+## Streamlit Config
+
+The
+file [ghostos/app/.streamlit/config.toml](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/.streamlit/config.toml)
+is the Streamlit configuration that is read when running `ghostos web`.
+
+For details on how to modify them,
+see [streamlit configuration](https://docs.streamlit.io/en/latest/advanced-features/configuration.html).
\ No newline at end of file
diff --git a/docs/en/getting_started/installation.md b/docs/en/getting_started/installation.md
new file mode 100644
index 00000000..a37821bf
--- /dev/null
+++ b/docs/en/getting_started/installation.md
@@ -0,0 +1,76 @@
+# Installation
+
+> GhostOS is still an experimental AI project, and it is strongly recommended to install it in a container like Docker
+> rather than executing it locally.
+
+## PIP install
+
+```bash
+pip install ghostos
+```
+
+Initialize the `workspace` (default `app`), all runtime files of the current version will be stored in the directory.
+
+```bash
+ghostos init
+```
+
+Configure the Large Language Model. By default, use OpenAI's `gpt-4o`, requiring the environment
+variable `OPENAI_API_KEY` to exist.
+Alternatively, run the `streamlit` editing interface:
+
+```bash
+ghostos config
+```
+
+Test the default agent:
+
+```bash
+# run an agent with python filename or modulename
+ghostos web ghostos.demo.agents.jojo
+```
+
+Or turn a local Python file into an Agent that can be instructed to call functions or methods within the file through
+natural language dialogue.
+
+```bash
+ghostos web [my_path_file_path]
+```
+
+## Extra
+
+Install extra dependencies:
+
+```bash
+pip install ghostos[sphero] # 安装 sphero 类库
+pip install ghostos[realtime] # 安装 realtime 相关类库. pyaudio 和 websockets
+```
+
+## Workspace
+
+`GhostOS` is currently using local files to store runtime data, so it's necessary to initialize a workspace.
+
+Running `ghostos init` can be scripted to copy the workspace to the current directory.
+
+Data generated by `GhostOS` during operation will be stored in this directory. When you need to clear historical data,
+please execute:
+
+```bash
+ghostos clear-runtime
+```
+
+## Env
+
+`GhostOS` relies on various model `access tokens`, which are read from environment variables by default.
+
+There are two ways to define these environment variables:
+
+Using a `.env` file (automatically read through `dotenv`)
+
+```bash
+
+copy [workspace]/.example.env [workspace]/.env
+vim [workspace]/.env
+```
+
+More details: [.example.env](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/app/.example.env)
\ No newline at end of file
diff --git a/docs/en/getting_started/scripts.md b/docs/en/getting_started/scripts.md
new file mode 100644
index 00000000..d0680e36
--- /dev/null
+++ b/docs/en/getting_started/scripts.md
@@ -0,0 +1,40 @@
+# Scripts
+
+`GhostOS` has some cli tools:
+
+```bash
+$ ghostos help
+
+Commands:
+ clear-runtime clear workspace runtime files
+ config config the ghostos in streamlit web app
+ console turn a python file or module into a console agent
+ docs See GhostOS Docs
+ help Print this help message.
+ init init ghostos workspace
+ web turn a python file or module into a streamlit web agent
+```
+
+Brief introductions to the commands:
+
+`ghostos config`: This command utilizes a Streamlit interface to modify configuration settings.
+
+`ghostos init`: Initializes a workspace in the current directory.
+
+`ghostos web`: Launches an agent conversation interface implemented with Streamlit, based on a Python file or module.
+
+`ghostos console`: Starts the agent in the command line, primarily used for debugging purposes.
+
+## Developing Scripts
+
+Here are descriptions of the additional command-line tools under development:
+
+`ghostos main`: Launches the official GhostOS agent, which serves as an introduction to everything about GhostOS.
+
+`ghostos meta`: Utilizes a meta agent to edit a Python file, enabling the implementation
+of [MossAgent](/en/usages/moss_agent.md).
+
+`ghostos edit`: Employs an edit agent to edit any file, leveraging context to modify file contents.
+
+`ghostos script`: Executes various automation scripts based on Large Language Models (LLMs), with the capability to
+continuously add related scripts to the local environment.
\ No newline at end of file
diff --git a/docs/en/libraries/libraries.md b/docs/en/libraries/libraries.md
new file mode 100644
index 00000000..586d850d
--- /dev/null
+++ b/docs/en/libraries/libraries.md
@@ -0,0 +1,102 @@
+# Libraries
+
+For traditional agents based on `JSON Schema Function Call`, their interaction object is `Tool`. In contrast, `GhostOS`
+provides Turing-complete code to the Agent, making the interaction object `Library`.
+
+The `libraries` here are not for developers but for Large Language Models (LLMs).
+
+There are three steps for LLMs to use Libraries:
+
+1. Define and implement a Library.
+2. Register the abstract and implementation of the Library to the IoC (Inversion of Control) Container.
+3. Bind the Library to the Moss class.
+
+## Code As Prompt
+
+The core concept here is `Code As Prompt`, which means that while writing code, you are also defining the prompt. Taking
+multi-task scheduling as an example:
+
+```python
+class Subtasks(Prompter, ABC):
+ """
+ library that can handle async subtasks by other ghost instance.
+ """
+ MessageKind = Union[str, Message, Any]
+ """message kind shall be string or serializable object"""
+
+ @abstractmethod
+ def cancel(self, name: str, reason: str = "") -> None:
+ """
+ cancel an exists subtask
+ :param name: name of the task
+ :param reason: the reason to cancel it
+ :return:
+ """
+ pass
+
+ @abstractmethod
+ def send(
+ self,
+ name: str,
+ *messages: MessageKind,
+ ctx: Optional[Ghost.ContextType] = None,
+ ) -> None:
+ """
+ send message to an existing subtask
+ :param name: name of the subtask
+ :param messages: the messages to the subtask
+ :param ctx: if given, update the ghost context of the task
+ :return:
+ """
+ pass
+
+ @abstractmethod
+ def create(
+ self,
+ ghost: Ghost,
+ instruction: str = "",
+ ctx: Optional[Ghost.ContextType] = None,
+ task_name: Optional[str] = None,
+ task_description: Optional[str] = None,
+ ) -> None:
+ """
+ create subtask from a ghost instance
+ :param ghost: the ghost instance that handle the task
+ :param instruction: instruction to the ghost
+ :param ctx: the context that the ghost instance needed
+ :param task_name: if not given, use the ghost's name as the task name
+ :param task_description: if not given, use the ghost's description as the task description
+ """
+ pass
+```
+
+Bind it to the moss class that the Agent can see:
+
+```python
+from ghostos.abcd import Subtasks
+from ghostos.core.moss import Moss as Parent
+
+
+class Moss(Parent):
+ subtasks: Subtasks
+ """manager your multi-agent tasks"""
+```
+
+* The source code of the class will be automatically reflected to the Prompt, allowing the Large Language Model to see
+ it.
+* The implementation of this library will be automatically injected into the `Moss` instance, and the Large Language
+ Model can use the generated code to call it.
+
+For more specific usage, please refer to [MossAgent](/en/usages/moss_agent.md).
+
+We hope that tools provide to the Large Language Models in the future, should be based on industry-standard protocols
+similar
+to
+the [MOSS Protocol](/en/concepts/moss_protocol.md), and be developed and shared in the form of code
+repositories.
+
+## Developing Libraries
+
+The libraries that come with GhostOS out of the box are still under development and testing.
+These tools are expected to be placed
+in [ghostos/libraries](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/libraries).
\ No newline at end of file
diff --git a/docs/en/usages/chatbot.md b/docs/en/usages/chatbot.md
new file mode 100644
index 00000000..a7e5134f
--- /dev/null
+++ b/docs/en/usages/chatbot.md
@@ -0,0 +1,35 @@
+# ChatBot
+
+A conversational robot implemented based on LLM is a baseline testing object during the development of `GhostOS` due to
+its simplicity.
+It is the simplest implementation of [Ghost](/en/usages/ghost.md).
+
+To create your own conversational robot, you can refer to the file
+[ghostos/demo/agents/jojo.py](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/demo/agents/jojo.py).
+
+```python
+from ghostos.ghosts.chatbot import Chatbot
+
+# the __ghost__ magic attr define a ghost instance
+# so the script `ghostos web` or `ghostos console` can detect it
+# and run agent application with this ghost.
+__ghost__ = Chatbot(
+ name="jojo",
+ description="a chatbot for baseline test",
+ persona="you are an LLM-driven cute girl, named jojo",
+ instruction="remember talk to user with user's language."
+)
+```
+
+To define your own chatbot, simply create a similar Python file and run the `ghostos web` command to invoke it.
+
+The interface generated by Streamlit can be directly modified by the `settings` option to change Ghost's configuration.
+The changes will be saved to a local file `.ghosts.yml`, so when `ghostos web` starts, it will prioritize reading the
+configuration in `.ghosts.yml`.
+
+The meta-agent for generating chatbots through dialogue is currently in testing, and will be released in the next few
+versions.
+
+The core Agent design of `GhostOS` is fully code-interactive [MossAgent](/en/usages/moss_agent.md), see the
+documentation for
+details.
\ No newline at end of file
diff --git a/docs/en/usages/ghost.md b/docs/en/usages/ghost.md
new file mode 100644
index 00000000..f6ae3506
--- /dev/null
+++ b/docs/en/usages/ghost.md
@@ -0,0 +1,254 @@
+# Ghost
+
+"Ghost" is the "minimum stateful unit" of an LLM-driven unit.
+The term comes from [Ghost In the Shell](https://en.wikipedia.org/wiki/Ghost_in_the_Shell).
+
+In the architectural design of "GhostOS", an intelligent agent swarm is composed of many "Ghost" units, each with its
+own state, memory, and context (Session);
+and they can communicate fully asynchronously through the "EventBus".
+
+
+
+## Why the word `Ghost` instead of `Agent`
+
+In the author's architectural vision, an `Agent` is a robot or interaction interface (like IM) for users, possessing a
+physical form (also known as Shell).
+
+However, within a single shell, there may be a Multi-Agent (or Multi-Ghost) swarm running, which serves the following
+purposes:
+
+* Parallel execution of multiple tasks.
+* Simulation of different roles for thought experimentation.
+* Asynchronous executions of long-duration tasks.
+
+Let's take a simple example:
+
+1. The `Agent` for user conversation, by default, runs the fast `gpt-4o` model to control the dialogue.
+2. When the user asks a complex question, the main ghost calls `gpt-o3` to run a 30-second thought process.
+3. During these 30 seconds, the main agent does not block but continues to converse with the user.
+4. After 30 seconds, the main agent receives the asynchronous callback result and informs the user.
+
+In this example, parallel execution and the event bus are the most critical features. Therefore, a Ghost can be:
+
+* An Agent for user conversation.
+* A clone opened by the main Agent, using different models, focused on a specific task or thought.
+* A workflow running in the background.
+* An automated robot running in the background.
+* An independent script.
+* A component of an embodied intelligent agent that can execute natural language commands.
+* A background program that reflects on its own operational effectiveness.
+
+Their common characteristics are:
+
+* Driven by large language models.
+* Possessing the ability to run in multiple rounds.
+* Similar to operating system threads, they are the smallest synchronous running units with independent contexts.
+
+## Ghost Driver
+
+In `GhostOS`, the prototype of a `Ghost` needs to be defined through at least two classes.
+
+[Ghost](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/abcd/concepts.py):
+
+```python
+
+class Ghost(Identical, EntityClass, ABC):
+ """
+ the class defines the model of a kind of ghosts.
+ four parts included:
+ 1. configuration of the Ghost, which is Ghost.__init__. we can predefine many ghost instance for special scenes.
+ 2. context is always passed by the Caller of a ghost instance. each ghost class has it defined context model.
+ 3. goal is the static output (other than conversation messages) of a ghost instance.
+ 4. driver is
+ """
+
+ ArtifactType: ClassVar[Optional[Type]] = None
+ """ the model of the ghost's artifact, is completing during runtime"""
+
+ ContextType: ClassVar[Optional[Type[ContextType]]] = None
+ """ the model of the ghost's context, is completing during runtime'"""
+
+ DriverType: ClassVar[Optional[Type[GhostDriver]]] = None
+ """ separate ghost's methods to the driver class, make sure the ghost is simple and clear to other ghost"""
+
+```
+
+[GhostDriver](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/abcd/concepts.py) :
+
+```python
+
+class GhostDriver(Generic[G], ABC):
+ """
+ Ghost class is supposed to be a data class without complex methods definitions.
+ so it seems much clear when prompt to the LLM or user-level developer.
+ when LLM is creating a ghost class or instance, we expect it only see the code we want it to see,
+ without knowing the details codes of it, for safety / fewer tokens / more focus or other reasons.
+
+ so the methods of the ghost class defined in this class.
+ only core developers should know details about it.
+ """
+
+ def __init__(self, ghost: G) -> None:
+ self.ghost = ghost
+
+ def make_task_id(self, parent_scope: Scope) -> str:
+ """
+ generate unique instance id (task id) of the ghost instance.
+ """
+ pass
+
+ @abstractmethod
+ def get_artifact(self, session: Session) -> Optional[G.ArtifactType]:
+ """
+ generate the ghost goal from session_state
+ may be the Goal Model is a SessionStateValue that bind to it.
+
+ The AI behind a ghost is not supposed to operate the session object,
+ but work on the goal through functions or Moss Injections.
+ """
+ pass
+
+ @abstractmethod
+ def get_instructions(self, session: Session) -> str:
+ """
+ get system instructions of the ghost.
+ usually used in client side.
+ """
+ pass
+
+ @abstractmethod
+ def actions(self, session: Session) -> List[Action]:
+ """
+ return actions that react to the streaming output of llm
+ """
+ pass
+
+ @abstractmethod
+ def providers(self) -> Iterable[Provider]:
+ """
+ ghost return conversation level container providers.
+ the provider that is not singleton will bind to session also.
+ """
+ pass
+
+ @abstractmethod
+ def parse_event(
+ self,
+ session: Session,
+ event: Event,
+ ) -> Union[Event, None]:
+ """
+ intercept the ghost event
+ :returns: if None, the event will be ignored
+ """
+ pass
+
+ @abstractmethod
+ def on_creating(self, session: Session) -> None:
+ """
+ when the ghost task is created first time.
+ this method can initialize the thread, pycontext etc.
+ """
+ pass
+
+ @abstractmethod
+ def on_event(self, session: Session, event: Event) -> Union[Operator, None]:
+ """
+ all the state machine is only handling session event with the predefined operators.
+ """
+ pass
+
+ @abstractmethod
+ def truncate(self, session: Session) -> GoThreadInfo:
+ """
+ truncate the history messages in the thread
+ """
+ pass
+
+```
+
+The motivation for this approach is that `GhostOS` employs the `Code As Prompt` concept to directly reflect code into
+prompts that the Large Language Model perceives. Within the Multi-Agent architecture, the detailed code of `GhostDriver`
+is unnecessary for the Agents that utilize it. By separating the data structure-focused `Ghost` from the
+logic-focused `GhostDriver`, it facilitates a more straightforward understanding for other Agents on how to construct an
+Agent.
+
+## Ghost Context
+
+The [Context](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/abcd/concepts.py) can be understood as the
+input parameters for the `Ghost` runtime. It accepts strongly-typed data structures and generates system prompts for the
+Large Language Model to understand the context. At the same time, the Large Language Model can manipulate the context as
+a variable.
+
+The Context implements [Prompter](/en/concepts/prompter.md), which is essentially a `Prompt Object Model` similar
+to `DOM`. It requires strongly-typed parameters to reflect system instructions as part of the prompt submitted to the
+LLM.
+
+The Context is typically used to implement:
+
+* The state of an embodied agent's own body and recognition of the surrounding environment.
+* The state of AI applications on the edge side (such as IDEs) and synchronize cognition with users.
+* Dynamically changing input data parameters, such as what an AI operator sees on a monitoring panel.
+
+Passed as input to the conversation:
+
+```python
+from pydantic import Field
+from ghostos.abcd import Context, Conversation, Ghost
+
+
+class ProjectContext(Context):
+ directory: str = Field(description="the root directory of a project")
+
+
+project_agent: Ghost = ...
+project_context: ProjectContext = ...
+conversation: Conversation = ...
+
+conversation.talk("your query to edit the project", project_context)
+```
+
+If necessary, MossAgent can manipulate a Context through `Moss`:
+
+```python
+from ghostos.abcd import Context
+from ghostos.core.moss import Moss as Parent
+from pydantic import Field
+
+
+class ProjectContext(Context):
+ directory: str = Field(description="the root directory of a project")
+
+
+class Moss(Parent):
+ # the moss agent can operate this ctx instance.
+ ctx: ProjectContext
+```
+
+## Ghost Artifact
+
+The [Artifact](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/abcd/concepts.py) can be understood as the
+output parameter of `Ghost` at runtime. However, this output parameter is subject to continuous changes.
+
+Through `Conversation`, you can obtain the `Artifact` object of `Ghost` at runtime for rendering on the client side.
+
+```python
+from ghostos.abcd import Conversation, Ghost, Shell
+
+my_ghost: Ghost = ...
+shell: Shell = ...
+conversation: Conversation = shell.sync(ghost=my_ghost)
+
+conversation.talk(...)
+
+# get artifact
+artifact = conversation.get_artifact()
+```
+
+## ChatBot and MossAgent
+
+[Chatbot](chatbot.md) and [MossAgent](moss_agent.md) are the basic implementations of Ghost in the `GhostOS` project.
+
+## More Ghosts
+
+developing...
\ No newline at end of file
diff --git a/evaluation/adas/__init__.py b/docs/en/usages/ghost_func.md
similarity index 100%
rename from evaluation/adas/__init__.py
rename to docs/en/usages/ghost_func.md
diff --git a/docs/en/usages/moss_agent.md b/docs/en/usages/moss_agent.md
new file mode 100644
index 00000000..0d1121cf
--- /dev/null
+++ b/docs/en/usages/moss_agent.md
@@ -0,0 +1,423 @@
+# Moss Agent
+
+`MossAgent` is the most fundamental Agent unit in the `GhostOS` project. It uses
+the [MOSS Protocol](/en/concepts/moss_protocol.md) to provide a code interaction interface, allowing Large
+Language Models to generate code to drive their own behavior.
+
+## Simplest Example
+
+create file `foo.py`:
+
+```python
+
+def plus(a: int, b: int) -> int:
+ return a + b
+```
+
+run `ghostos web foo.py`, and ask the agent to call `plus` function.
+
+## Run Agent
+
+Running the command `ghostos web [python_modulename_or_filename]` can directly turn a Python file into an Agent and run
+it with Streamlit.
+
+For example:
+
+```bash
+ghostos web ghostos/demo/agents/jojo.py
+# or
+ghostos web ghostos.demo.agents.jojo
+```
+
+When the command is executed, if the target file does not have a `__ghost__` attribute, it will reflect the target file
+and generate an instance
+of [MossAgent](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/ghosts/moss_agent/agent.py). This Agent can
+call the functions and classes provided by the target file to perform tasks you propose in natural language.
+
+Here is the source code:
+
+```python
+class MossAgent(ModelEntity, Agent):
+ """
+ Basic Agent that turn a python module into a conversational agent.
+ """
+
+ """ subclass of MossAgent could have a GoalType, default is None"""
+
+ moss_module: str = Field(description="Moss module name for the agent")
+ persona: str = Field(description="Persona for the agent, if not given, use global persona")
+ instructions: str = Field(description="The instruction that the agent should follow")
+
+ # optional configs
+ name: str = Field(default="", description="name of the agent")
+ description: str = Field(default="", description="description of the agent")
+ code: Optional[str] = Field(default=None, description="code override the module")
+ compile_module: Optional[str] = Field(None, description="Compile module name for the agent")
+ llm_api: str = Field(default="", description="name of the llm api, if none, use default one")
+ truncate_at_turns: int = Field(default=40, description="when history turns reach the point, truncate")
+ truncate_to_turns: int = Field(default=20, description="when truncate the history, left turns")
+```
+
+You can also manually define a `__ghost__` instance:
+
+```python
+
+# the python module codes
+...
+
+#
+# add and agent definition manually at the tail of the file.
+from ghostos.ghosts.moss_agent import MossAgent
+
+__ghost__ = MossAgent(
+ moss_module=__name__,
+ name="agent name",
+ description="agent desc",
+ persona="persona",
+ instructions="system instructions",
+ # use llms model defined at app/configs/llms_conf.yml
+ llm_api="moonshot-v1-128k",
+)
+
+#
+```
+
+> Normally, a Python file can be started as an agent without any modifications. For example, a unit test file.Normally,
+> a Python file can be started as an agent without any modifications. For example, a unit test file.
+
+## Code As Prompt
+
+MossAgent will automatically reflect the target Python module into a Prompt, providing it to the large model.
+To see the detailed prompt, you can use `ghostos web` to generate the `instructions` button on the interface to view its
+system instruction.
+
+The default reflection principle can be found in [MOSS Protocol](/en/concepts/moss_protocol.md). In short:
+
+1. Referenced functions will automatically reflect the function name + doc
+2. Abstract classes will reflect the source code
+
+The large model will call a tool named `moss` based on the instruction to generate code.
+The generated code will be executed in the temporary module compiled
+by [Moss](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/core/moss/abcd.py).
+
+Source
+Code: [MossAction](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/ghosts/moss_agent/agent.py#MossAction).
+
+If some code you want hide from LLM, use `# ` and `# ` markers:
+
+```python
+
+#
+...
+# the code here is not visible to llm
+#
+```
+
+If the results of automatic reflection are not satisfactory, you can also manually define it through the magic
+method `__moss_attr_prompts__`.
+
+```python
+from foo import Foo
+
+
+#
+
+def __moss_attr_prompts__():
+ """
+ :return: Iterable[Tuple[attr_name: str, attr_prompt: str]]
+ """
+ yield "Foo", "" # if the prompt is empty, won't prompt it to llm
+#
+```
+
+## Magic lifecycle functions
+
+The `MossAgent` uses magic methods within various files to define its special operational logic.
+The benefits of this approach are, first, to simplify the use for developers; and second, for the Meta-Agent, to reduce
+the workload when creating an Agent.
+
+All lifecycle methods can be found in the following three files:
+
+- [for developer](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/ghosts/moss_agent/for_developer.py):
+ Lifecycle management for developers.
+ - `__moss_agent_providers__`
+ - `__shell_providers__`
+ - `__moss_agent_creating__`
+ - `__moss_agent_truncate__`
+ - `__moss_agent_parse_event__`
+ - `__moss_agent_injections__`
+ - `__moss_agent_on_[event_type]__`:
+- [for meta ai](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/ghosts/moss_agent/for_meta_ai.py): for
+ developer and Meta AI
+ - `__moss_agent_artifact__`
+ - `__moss_agent_actions__`
+ - `__moss_agent_thought__`
+ - `__moss_agent_instruction__`
+ - `__moss_agent_persona__`
+- [moss lifecycle](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/core/moss/lifecycle.py)
+
+Copy these methods into the current file to activate custom magic methods.
+All these magic methods are **optional**. If they can solve the problem, then you can use them.
+
+If all magic methods are insufficient, then the best approach is to implement your own `Ghost` and `GhostDriver`
+classes,
+see [concepts.py](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/abcd/concepts.py).
+
+## Define Moss Class
+
+Usually importing classes and methods is sufficient for an Agent to operate. However, there are two situations where you
+need to introduce the `Moss` class. (Model-oriented Operating System Simulator):
+
+1. `Context Manage`: Wish to define variables that can be changed continuously in multi-turn conversations.
+2. `Runtime Injection`: use [IoC Container](/en/concepts/ioc_container.md) for dependencies injections.
+
+Define a Moss class in the target module:
+
+```python
+from ghostos.core.moss import Moss as Parent
+
+
+class Moss(Parent):
+ ...
+ pass
+```
+
+Whether this class is defined or not, a `moss` object will be generated during the execution of MossAgent. The code
+written for MossAgent also uses it, with a prompt as follows:
+
+(Will be continuously optimized):
+
+```markdown
+You are able to call the `moss` tool, generate code to fulfill your will.
+the python code you generated, must include a `run` function, follow the pattern:
+
+\```python
+def run(moss: Moss):
+"""
+:param moss: instance of the class `Moss`, the properties on it will be injected with runtime implementations.
+:return: Optional[Operator]
+if return None, the outer system will perform default action, or observe the values you printed.
+Otherwise, the outer system will execute the operator.
+You shall only return operator by the libraries provided on `moss`.
+"""
+\```
+```
+
+详见 [instructions](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/ghosts/moss_agent/instructions.py)
+
+### Define Variables On Moss
+
+The `str`, `float`, `int`, `bool`, `str`, and `pydantic.BaseModel` types mounted on the Moss class will be automatically
+saved, so MossAgent can directly use them as variables.
+
+Note that these variable types must be serializable. For example:
+
+```python
+from ghostos.core.moss import Moss as Parent
+from pydantic import BaseModel, Field
+
+
+class YourVariables(BaseModel):
+ variables: dict = Field(default_factory=dict, description="you can manage your variables here")
+
+
+# 名为 Moss 的类是一个特殊的类.
+class Moss(Parent):
+ vars: YourVariables = YourVariables()
+```
+
+Furthermore, if the mounted data object
+implements [ghostos.prompter.Prompter](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/prompter.py),
+MossAgent will automatically generate a prompt in the system instruction to be provided to the large model.
+
+For more information on this logic, see the `ghostos.ghosts.moss_agent.instructions.get_moss_context_prompter` function.
+
+### Runtime Injection
+
+The `abstract class` mounted on the Moss class will automatically perform dependency injection from
+the [IoC Container](/en/concepts/ioc_container.md). There are three ways to provide implementations for these
+abstract classes:
+
+- Pass in instances at definition:
+
+```python
+from ghostos.core.moss import Moss as Parent
+
+
+class Foo:
+ ...
+ pass
+
+
+# 名为 Moss 的类是一个特殊的类.
+class Moss(Parent):
+ foo: Foo = Foo()
+```
+
+- Through the magic method `__moss_agent_injections__`, manually define the instances to be injected.
+
+```python
+from ghostos.core.moss import Moss as Parent
+from foo import Foo
+
+
+class Moss(Parent):
+ foo: Foo
+
+
+#
+# the code in moss-hide is invisible to llm
+
+def __moss_agent_injections__(agent, session) -> Dict[str, Any]:
+ """
+ manually define some of the injections to the Moss Class.
+ if a property of Moss is not injected here, the session container will inject it by typehint.
+ """
+ from foo.impl import FooImpl
+ return {
+ "foo": FooImpl(...)
+ }
+#
+```
+
+The third method is to register dependency implementations in the [IoC Container](/en/concepts/ioc_container.md).
+The `Moss` class will perform type analysis upon instantiation and automatically perform dependency injection. There are
+several ways to register dependencies:
+
+## Register dependencies
+
+`GhostOS` isolates dependencies at different levels during runtime through an inheritable `IoC Container Tree`. The
+system has the following default container levels:
+
+- [App Root Container](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/bootstrap.py): Unique container for
+ the process
+- `GhostOS.container`: Unique container for the process, essentially the same as the App Root Container.
+- `Shell.container`: A container shared by all ghosts running in parallel within the same process. Typically used to
+ launch singletons related to the body.
+- `Conversation.container`: Dependencies owned by a single Ghost.
+- `MossRuntime.container`: A temporary container generated each time `MossRuntime` is compiled. Used to
+ register `MossRuntime` itself.
+
+During the runtime of `MossAgent`, dependency injection is performed by `MossRuntime.container`, so it inherits
+registered dependencies from each parent container and can also override them.
+
+Some dependencies provided by the `GhostOS` system are as follows:
+
+- [LoggerItf](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/contracts/logger.py)
+- [Configs](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/contracts/configs.py)
+- [Workspace](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/contracts/workspace.py)
+- [Variables](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/contracts/variables.py)
+- [LLMs](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/core/llms/llms.py)
+- [Assets](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/contracts/assets.py)
+- [GhostOS](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/abcd/concepts.py)
+- [Shell](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/abcd/concepts.py)
+- [Conversation](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/abcd/concepts.py)
+- [Session](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/abcd/concepts.py)
+- [Scope](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/abcd/concepts.py)
+- [Ghost](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/abcd/concepts.py)
+- [MossCompiler](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/core/moss/abcd.py)
+- [Tasks](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/core/runtime/tasks.py)
+- [Threads](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/core/runtime/threads.py)
+- [EventBus](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/core/runtime/events.py)
+
+More system-level bindings can be debugged by calling `Container.contracts(recursively=True)`.
+
+### Register MossAgent dependencies
+
+The simplest method is to define dependencies directly in a Python file using magic methods:
+
+```python
+
+#
+
+def __moss_agent_providers__(agent: A) -> Iterable[Provider]:
+ """
+ return conversation level providers that specially required by the Agent.
+ the conversation container will automatically register the providers and run them.
+
+ :param agent: the moss agent instance.
+ :return: providers that register to the session container.
+ """
+ return []
+
+#
+```
+
+These dependencies will be registered when the `Conversation` is created.
+
+### Register root dependencies
+
+Modifying the global container, or creating your own container, both can register services in the process:
+
+```python
+from ghostos.bootstrap import reset, make_app_container
+
+# 定义新的全局容器
+new_root_container = make_app_container(...)
+
+# 重置 ghostos.bootstrap.app_container
+reset(new_root_container)
+```
+
+This way, you can register process-level dependencies that take effect for all containers.
+
+### Register Shell dependencies
+
+Dependencies can be registered when Shell is launched. A process may repeatedly start multiple Shells, hence Shell has a
+separate isolation level.
+
+The simplest way is to register at the start of the life cycle when the shell is launched:
+
+```python
+from ghostos.bootstrap import get_ghostos
+
+ghostos = get_ghostos()
+
+# register shell level providers at when shell is creating
+shell = ghostos.create_shell("shell name", providers=[...])
+```
+
+对于使用 `ghostos web` 或 `ghostos console` 启动的 python 文件, 也可以简单注册在文件的魔术方法内:
+
+```python
+#
+
+def __shell_providers__() -> Iterable[Provider]:
+ """
+ return shell level providers that specially required by the Agent.
+ if the shell is running by `ghostos web` or `ghostos console`,
+ the script will detect the __shell_providers__ attribute and register them into shell level container.
+
+ You can consider the Shell is the body of an agent.
+ So shell level providers usually register the body parts singletons, bootstrap them and register shutdown functions.
+ """
+ return []
+#
+```
+
+For Python files launched with `ghostos web` or `ghostos console`, they can also be simply registered within the magic
+method of the file:
+
+## Register Conversation dependencies
+
+The `__moss_agent_providers__` magic method can usually handle the registration of dependencies for Conversation.
+However,
+if manual registration is needed, it should be done when creating the Conversation:
+
+```python
+from ghostos.abcd import Shell, Conversation, Ghost
+
+shell: Shell = ...
+my_ghost: Ghost = ...
+
+conversation = shell.sync(my_ghost)
+
+# register here. usually not necessary
+conversation.container().register(...)
+
+```
+
+## Meta-Agent
+
+`GhostOS` will provide a `MossAgent` to generate other `MossAgent`s, which is the Meta-Agent.
+Currently, it is still in development and testing.
\ No newline at end of file
diff --git a/docs/index.html b/docs/index.html
new file mode 100644
index 00000000..70854470
--- /dev/null
+++ b/docs/index.html
@@ -0,0 +1,40 @@
+
+
+
+ GhostOS Document
+
+
+
+
+
+
+
+
+
diff --git a/docs/zh-cn/README.md b/docs/zh-cn/README.md
new file mode 100644
index 00000000..dc3d6f6e
--- /dev/null
+++ b/docs/zh-cn/README.md
@@ -0,0 +1,175 @@
+# GhostOS
+
+> The AI `Ghosts` wonder in the `Shells`.
+
+* [中文文档](/zh-cn/README.md)
+* [Documents](/en/README.md)
+* [Discord Server](https://discord.gg/NG6VKwd5jV)
+
+## Example
+
+使用 Python 代码 [SpheroBoltGPT](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/demo/sphero/bolt_gpt.py),
+定义了一个以 [SpheroBolt](https://sphero.com/products/sphero-bolt) 玩具为躯体的智能机器人.
+
+如果你有 SpheroBolt, 运行 `ghostos web ghostos.demo.sphero.bolt_gpt` 可以启动这个机器人:
+
+
+
+Demo 中初步实现的功能:
+
+1. 实时语音对话.
+2. 控制身体运动, 控制 8*8 led matrix 绘画图形.
+3. 通过自然语言对话, 学习包含动作和动画的技能.
+4. 在对话中通过动作表达情绪.
+
+## Introduce
+
+`GhostOS` 是一个 AI Agent 框架, 旨在用图灵完备的代码交互界面 ([Moss Protocol](/zh-cn/concepts/moss_protocol.md)) 取代
+JSON Schema,
+成为 LLM 和 Agent 系统能力交互的核心方式.
+详见: [MOSS: Enabling Code-Driven Evolution and Context Management for AI Agents](https://arxiv.org/abs/2409.16120)
+
+预期通过代码调用的对象包括`工具`, `人格`, `智能体集群`, `工作流`, `思维`, `规划`, `知识` 和 `记忆`.
+从而使一个 Meta-Agent 能用代码生成和项目管理的方式, 变成一个可以持续学习成长的智能体.
+
+而这样用代码仓库实现的智能体, 又能以仓库的形式分享和安装.
+
+`GhostOS` 还在早期验证的阶段, 当前版本主要实现开箱即用的能力, 包括:
+
+- [x] 将各种 python 脚本直接变成对话 Agent
+- [x] Agent 拥有基于 [Streamlit Web](https://streamlit.io/) 实现的界面.
+- [x] 支持 `OpenAI`, `Moonshot` 等模型
+- [x] 支持 OpenAI 的视觉能力 ([OpenAI vision](https://platform.openai.com/docs/guides/vision))
+- [x] 支持 OpenAI 实时语音对话 ([OpenAI Realtime Beta](https://platform.openai.com/docs/guides/realtime))
+
+## Quick Start
+
+> `GhostOS` 仍然是一个验证中的 AI 项目, 强烈建议安装到 docker 之类的容器中, 而不在本地执行.
+
+安装 `GhostOS`:
+
+```bash
+pip install ghostos
+```
+
+初始化 `workspace` (默认 `app`), 当前版本的运行时文件都会存入目录.
+
+```bash
+ghostos init
+```
+
+配置大模型. 默认使用 OpenAI `gpt-4o`, 要求环境变量存在 `OPENAI_API_KEY`.
+或者运行 `streamlit` 编辑界面:
+
+```bash
+ghostos config
+```
+
+测试运行自带的 agent:
+
+```bash
+# run an agent with python filename or modulename
+ghostos web ghostos.demo.agents.jojo
+```
+
+或者将本地的 Python 文件变成一个 Agent, 可以通过自然语言对话要求它调用文件中的函数或方法:
+
+```bash
+ghostos web [my_path_file_path]
+```
+
+安装更多关联依赖:
+
+```bash
+pip install ghostos[sphero] # 安装 sphero 类库
+pip install ghostos[realtime] # 安装 realtime 相关类库. pyaudio 和 websockets
+```
+
+可以通过创建本地 python 文件, 定义出自己的 Agents. 详情请见:
+
+* [Chatbot](/zh-cn/usages/chatbot.md): 极简的对话机器人
+* [MossAgent](/zh-cn/usages/moss_agent.md): 能使用 python 的 agent
+
+## Use In Python
+
+```python
+from ghostos.bootstrap import make_app_container, get_ghostos
+from ghostos.ghosts.chatbot import Chatbot
+
+# create your own root ioc container.
+# register or replace the dependencies by IoC service providers.
+container = make_app_container(...)
+
+# fetch the GhostOS instance.
+ghostos = get_ghostos(container)
+
+# Create a shell instance, which managing sessions that keep AI Ghost inside it.
+# and initialize the shell level dependency providers.
+shell = ghostos.create_shell("your robot shell")
+# Shell can handle parallel ghosts running, and communicate them through an EventBus.
+# So the Multi-Agent swarm in GhostOS is asynchronous.
+shell.background_run() # Optional
+
+# need an instance implements `ghostos.abcd.Ghost` interface.
+my_chatbot: Chatbot = ...
+
+# use Shell to create a synchronous conversation channel with the Ghost.
+conversation = shell.sync(my_chatbot)
+
+# use the conversation channel to talk
+event, receiver = conversation.talk("hello?")
+with receiver:
+ for chunk in receiver.recv():
+ print(chunk.content)
+```
+
+## Developing Features
+
+* [ ] 开箱即用的 Agent 能力类库.
+* [ ] 变量类型消息与 Streamlit 渲染.
+* [ ] 异步的 Multi-Agent.
+* [ ] 长程任务规划与执行.
+* [ ] 原子化的思维能力.
+* [ ] 树形项目的自动执行与管理.
+* [ ] 框架可配置化的组件.
+* [ ] 玩具级具身智能的实验.
+
+> `GhostOS` 作为一个个人项目, 目前没有精力用于完善文档, 存储模块, 稳定性或安全性等问题.
+>
+> 项目的迭代将长时间聚焦于验证 `代码驱动的具身智能体`, `代码驱动的思维能力`, `代码驱动的学习与成长` 三个方向.
+> 并完善开箱即用的 agent 相关能力.
+
+# So What is GhostOS?
+
+`GhostOS` 这个项目是作者用来做 AI 应用探索而开发的. 基本思路如下:
+
+AI Agent 技术有两条并行的演进路径, 一种是模型自身能力的完善, 一种则是 Agent 工程框架的进化.
+Agent 框架的生产力水平, 决定了 AI 模型在应用场景落地的可行性.
+
+`GhostOS` 把 Agent 的能力从代码反射成 Prompt, 提供给大模型, 大模型生成的代码直接在环境中运行.
+通过一个图灵完备的编程语言界面, 大模型可以解决包括计算, 调用工具, 身体控制, 人格切换, 思维范式, 状态调度, Multi-Agent,
+记忆与召回等一切动作.
+
+这会比基于 json schema 等方式的交互能力更强, 开销更小.
+在这过程中生成的交互数据, 又可以用于模型的 post-training 或强化学习, 从而不断优化效果.
+
+AI Agent 本身也是使用代码实现的. 所以大模型驱动的 Meta-Agent 实现其它的 Agent, 可以还原为一个编程问题.
+理想情况下, 大模型驱动的 Meta-Agent 可以通过编写代码, 编写自己的工具, 用数据结构定义的记忆和思维链, 乃至于生成其它的
+Agent.
+
+
+
+进一步, 大多数有严谨步骤的复杂任务, 都可以用树或者图的数据结构描述.
+用 json 之类的方式构建一个结构嵌套的图或者树非常困难, 而用编程语言是最高效的.
+大模型可以把对话学习到的成果沉淀成代码中的节点, 再将它们规划成树或者图, 从而执行足够复杂的任务.
+
+这样, AI Agent 可以把自然语言教学习得的知识和能力, 以文件和代码的形式存储, 从而自我进化. 这是模型迭代之外的进化之路.
+
+基于以上思路, `GhostOS` 希望把 Agent 集群变成一个通过代码构建出来的项目. Agent 又不断把新的知识和能力用代码形式沉淀,
+丰富这个项目.
+最终 Agent 项目可以用仓库的形式复制, 分享或部署. 形成一种可基于代码自我进化, 持续学习的智能体集群.
+在这种新生产力形态中, 用纯代码交互是最关键的一步.
+
+作者最大的目标不是 `GhostOS` 本身, 而是验证和推动这种代码交互的设计与应用. 希望有一天行业里的 Agent, 思维范式, 躯体和工具,
+都可以基于相同的编程语言协议设计, 实现跨项目通用.
+
diff --git a/docs/zh-cn/_navbar.md b/docs/zh-cn/_navbar.md
new file mode 100644
index 00000000..93f37fe8
--- /dev/null
+++ b/docs/zh-cn/_navbar.md
@@ -0,0 +1,2 @@
+* [中文文档](/zh-cn/)
+* [En](/en/)
diff --git a/docs/zh-cn/_sidebar.md b/docs/zh-cn/_sidebar.md
new file mode 100644
index 00000000..871efd0e
--- /dev/null
+++ b/docs/zh-cn/_sidebar.md
@@ -0,0 +1,23 @@
+- [Home](/zh-cn/)
+- Getting Started
+ - [Install](/zh-cn/getting_started/installation.md)
+ - [Configuration](/zh-cn/getting_started/configuration.md)
+ - [Chat](/zh-cn/getting_started/chat_with_ghost.md)
+ - [Scripts](/zh-cn/getting_started/scripts.md)
+- Concepts
+ - [Architecture](/zh-cn/concepts/abcd.md)
+ - [MOSS Protocol](/zh-cn/concepts/moss_protocol.md)
+ - [IoC Container](/zh-cn/concepts/ioc_container.md)
+ - [EntityMeta](/zh-cn/concepts/entity_meta.md)
+ - [Prompter](/zh-cn/concepts/prompter.md)
+- Usages
+ - [Ghost](/zh-cn/usages/ghost.md)
+ - [Chatbot](/zh-cn/usages/chatbot.md)
+ - [MossAgent](/zh-cn/usages/moss_agent.md)
+- [Libraries](/zh-cn/libraries/libraries.md)
+- Frameworks
+ - [Messages](/zh-cn/frameworks/messages.md)
+ - [LLMs](/zh-cn/frameworks/llms.md)
+ - [EventBus](/zh-cn/frameworks/eventbus.md)
+ - [Tasks](/zh-cn/frameworks/tasks.md)
+ - [Threads](/zh-cn/frameworks/threads.md)
\ No newline at end of file
diff --git a/docs/zh-cn/concepts/abcd.md b/docs/zh-cn/concepts/abcd.md
new file mode 100644
index 00000000..c502541d
--- /dev/null
+++ b/docs/zh-cn/concepts/abcd.md
@@ -0,0 +1,46 @@
+# 抽象设计
+
+`GhostOS` 的抽象设计遵循了 `面向接口编程` 这一原则.
+所有模块使用了抽象类来设计, 通过 [IoC Container](/zh-cn/concepts/ioc_container.md) 来组装实现.
+
+
+
+这些抽象基本的相互关系, 和使用逻辑如下:
+
+```python
+from ghostos.abcd import GhostOS, Shell, Conversation, Ghost
+from ghostos.container import Container
+from ghostos.bootstrap import make_app_container, get_ghostos
+
+# create your own root ioc container.
+# register or replace the dependencies by IoC service providers.
+container: Container = make_app_container(...)
+
+# fetch the GhostOS instance.
+ghostos: GhostOS = get_ghostos(container)
+
+# Create a shell instance, which managing sessions that keep AI Ghost inside it.
+# and initialize the shell level dependency providers.
+shell: Shell = ghostos.create_shell("your robot shell")
+# Shell can handle parallel ghosts running, and communicate them through an EventBus.
+# So the Multi-Agent swarm in GhostOS is asynchronous.
+shell.background_run() # Optional
+
+# need an instance implements `ghostos.abcd.Ghost` interface.
+my_chatbot: Ghost = ...
+
+# use Shell to create a synchronous conversation channel with the Ghost.
+conversation: Conversation = shell.sync(my_chatbot)
+
+# use the conversation channel to talk
+event, receiver = conversation.talk("hello?")
+with receiver:
+ for chunk in receiver.recv():
+ print(chunk.content)
+
+```
+
+
+详细内容请直接看代码 [ghostos.abcd.concepts](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/abcd/concepts.py)
+
+> 抽象设计的相关介绍非常繁杂, 有精力再完成文档.
\ No newline at end of file
diff --git a/docs/zh-cn/concepts/entity_meta.md b/docs/zh-cn/concepts/entity_meta.md
new file mode 100644
index 00000000..bab21a9a
--- /dev/null
+++ b/docs/zh-cn/concepts/entity_meta.md
@@ -0,0 +1,15 @@
+# Entity Meta
+
+基于代码驱动的 AI Agent 需要在运行过程中将各种数据进行存储, 同时能在后续运行中还原变量.
+考虑到 `分布式系统` 或是 `可中断 Agent`, 这些数据需要有长期存储的方案.
+
+`GhostOS` 以 `pickle` 和 `pydantic` 为基础,
+实现了 [EntityMeta](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/entity.py).
+
+它旨在:
+1. 把绝大部分可存取的 python 数据结构进行序列化和反序列化
+2. 并提供通用的容器和 API 来操作数据
+3. 尽可能保证数据的可读性.
+
+更多技术细节详见代码.
+
diff --git a/docs/zh-cn/concepts/ioc_container.md b/docs/zh-cn/concepts/ioc_container.md
new file mode 100644
index 00000000..b7352ca0
--- /dev/null
+++ b/docs/zh-cn/concepts/ioc_container.md
@@ -0,0 +1,289 @@
+# IoC Container
+
+`GhostOS` 遵循 `面向接口编程` 的思路构建项目.
+大多数模块分为 `interface` 与 `implementation`,
+通过 [IoC Container](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/container.py) (控制反转容器) 来注册与获取实现.
+
+关于 IoC 详见: [Inverse of Control](https://en.wikipedia.org/wiki/Inversion_of_control)
+
+## Why?
+
+在 `Java` 和 `PHP` 的项目中, `IoC Contaienr` 是被大规模使用的. 比如:
+
+* [Java Spring](https://docs.spring.io/spring-framework/docs/3.2.x/spring-framework-reference/html/beans.html)
+* [PHP Laravel](https://laravel.com/docs/11.x/container)
+
+但 Python 项目中很少使用, 通常用单例和工厂方法替代.
+
+`GhostOS` 引入 `IoC Container`, 最核心的动机是实现`面向接口编程` 和 `运行时依赖注入`. 我们以 SpheroBoltGPT 为例:
+
+```python
+from ghostos.prototypes.spherogpt.bolt import (
+ RollFunc,
+ Ball,
+ Move,
+ LedMatrix,
+ Animation,
+)
+from ghostos.core.moss import Moss as Parent
+
+
+class Moss(Parent):
+ body: Ball
+ """your sphero ball body"""
+
+ face: LedMatrix
+ """you 8*8 led matrix face"""
+```
+
+这部分代码会被自动反射成 prompt 提供给大模型. 但其中的 `Ball` 和 `LedMatrix` 在项目正式启动前都不应该实例化.
+尤其是当一个 Meta-Agent 需要分析这段代码时, 它不应该在阅读代码时导致创建和 Sphero Bolt 的连接.
+
+所以 `Ball` 和 `LedMatrix` 可以用抽象来设计:
+
+```python
+class Ball(ABC):
+ """
+ Sphero bolt body (which is a rolling ball) control interface.
+ """
+
+ @abstractmethod
+ def new_move(
+ self,
+ *,
+ run: bool = False,
+ animation: Optional[Animation] = None,
+ ) -> Move:
+ """
+ create a new Move instance, to define a sequence of movements.
+ :param run: run immediately if True, otherwise the move will not execute until run it.
+ :param animation: if animation is not none, it will be played while run the move.
+ """
+ pass
+
+ @abstractmethod
+ def run(self, move: Move, stop_at_first: bool = True) -> None:
+ """
+ run the bolt ball movement
+ :param move: the Move instance that defined the movements by calling it methods one by one.
+ :param stop_at_first: shall stop any movement of the ball before executing the new move?
+ """
+ pass
+```
+
+而真正的实例, 只在项目运行时才通过 container 注入:
+
+
+
+## Basic Usage
+
+```python
+from abc import ABC, abstractmethod
+from typing import Type
+from ghostos.container import Container, Provider
+
+
+def test_container_baseline():
+ class Abstract(ABC):
+ @abstractmethod
+ def foo(self) -> int:
+ pass
+
+ class Foo(Abstract):
+ count = 0
+
+ def foo(self) -> int:
+ self.count += 1
+ return self.count
+
+ container = Container()
+
+ # set instance
+ foo = Foo()
+ container.set(Foo, foo)
+ assert container.get(Foo) is foo
+```
+
+## Provider
+
+通过 `Container.set` 方法注册的实现是单例. 在面向组合的场景中, 需要用 `工厂方法` 来获取依赖生成实例.
+这时可以使用 `ghostos.container.Provider`:
+
+```python
+from abc import ABC, abstractmethod
+from typing import Type
+from ghostos.container import Container, Provider
+
+
+def test_container_baseline():
+ class Abstract(ABC):
+ @abstractmethod
+ def foo(self) -> int:
+ pass
+
+ class Foo(Abstract):
+ def __init__(self, count):
+ self.count = count
+
+ def foo(self) -> int:
+ return self.count
+
+ class FooProvider(Provider):
+
+ def singleton(self) -> bool:
+ return True
+
+ def contract(self) -> Type[Abstract]:
+ return Abstract
+
+ def factory(self, con: Container) -> Abstract:
+ # get dependencies from con
+ count = con.get("count")
+ return Foo(count)
+
+ # register
+ container = Container()
+ container.set("count", 123)
+ container.register(FooProvider())
+
+ # get instance
+ foo = container.force_fetch(Abstract)
+ assert isinstance(foo, Foo)
+ assert foo.foo() is 123
+```
+
+此外有语法糖 `ghostos.container.provide` 可以方便封装一个工厂方法为 Provider.
+
+```python
+from abc import ABC, abstractmethod
+from ghostos.container import Container, provide
+
+
+class Abstract(ABC):
+ @abstractmethod
+ def foo(self) -> int:
+ pass
+
+
+class Foo(Abstract):
+ def __init__(self, count):
+ self.count = count
+
+ def foo(self) -> int:
+ return self.count
+
+
+@provide(Abstract, singleton=True)
+def foo_factory(self, con: Container) -> Abstract:
+ # get dependencies from con
+ count = con.get("count")
+ return Foo(count)
+
+
+# register
+container = Container()
+container.set("count", 123)
+container.register(foo_factory)
+
+# get instance
+foo = container.force_fetch(Abstract)
+assert isinstance(foo, Foo)
+assert foo.foo() is 123
+```
+
+## Inheritance
+
+`Container` 是可以嵌套的:
+
+```python
+from ghostos.container import Container
+
+container = Container(name="parent")
+container.set("foo", "foo")
+
+child_container = Container(parent=container, name="child")
+assert child_container.get("foo") == "foo"
+```
+
+当一个后代 Container 查找一个注册的依赖时, 如果没找到, 它会递归地去父级 Container 中查找.
+
+此外 `Provider` 也有继承机制:
+
+```python
+from ghostos.container import Provider
+
+
+class MyProvider(Provider):
+
+ def inheritable(self) -> bool:
+ return not self.singleton()
+```
+
+所有在父级 container 中注册的 `inheritable provider` 也会自动注册到 子级 container.
+
+## Bootstrap and Shutdown
+
+Container 同时可以作为启动和关闭运行的容器.
+
+```python
+from ghostos.container import Bootstrapper, Container
+
+container = Container()
+
+
+class MyBootstrapper(Bootstrapper):
+ def bootstrap(self, container: Container) -> None:
+ # do something
+ ...
+
+
+# start all the bootstrapper
+container.bootstrap()
+```
+
+`Bootstrapper` 也可以用 `ghostos.container.BootstrapProvider` 来定义.
+
+同样的, Container 可以用 `Container.add_shutdown` 注册关闭事件, 调用 `Container.shutdown` 时会依次执行它们.
+我们以 `SpheroRuntime` 为例, 它需要全局运行, 作为 [SpheroBolt](https://sphero.com/products/sphero-bolt) 的驱动.
+
+```python
+
+
+class SpheroBoltProvider(BootstrapProvider):
+ """
+ Sphero Bolt Provider interface
+ """
+ ...
+
+ @staticmethod
+ def bootstrap(container: Container) -> None:
+ # get singleton
+ sphero_bolt = container.force_fetch(SpheroBolt)
+ if isinstance(sphero_bolt, SpheroBoltImpl):
+ # register shutdown method
+ container.add_shutdown(sphero_bolt.destroy)
+ # bootstrap sphero bolt
+ sphero_bolt.bootstrap()
+
+```
+
+## Container Tree
+
+在 `GhostOS` 中, 有不同层级的 Container, 每个 Container 继承自父级 Container, 又管理一个独立的依赖关系.
+
+* 当子级 Container 注册依赖时, 不会污染父级或兄弟级 Container.
+* 当子级 Container 销毁的时候, 并不会影响父级或兄弟级 Container.
+
+这样 `Container` 类似于 Python `contextvar`, 可以管理一个独立的运行上下文.
+
+`GhostOS` 里的 Container 继承层级关系如下:
+
+* Root level: [ghostos.bootstrap.app_container](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/bootstrap.py)
+* GhostOS level: [ghostos.abcd.GhostOS:container](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/abcd/concepts.py)
+* Shell level: [ghostos.abcd.Shell:container](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/abcd/concepts.py)
+* Conversation
+ level: [ghostos.abcd.Conversation:container](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/abcd/concepts.py)
+* Moss
+ level: [ghostos.core.moss.MossRuntime:container](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/core/moss/abcd.py)
+
+在正确的层级注册 Container 和 Provider, 可以用来管理需要继承或隔离的依赖关系.
\ No newline at end of file
diff --git a/docs/zh-cn/concepts/moss_protocol.md b/docs/zh-cn/concepts/moss_protocol.md
new file mode 100644
index 00000000..913bf69a
--- /dev/null
+++ b/docs/zh-cn/concepts/moss_protocol.md
@@ -0,0 +1,354 @@
+# MOSS Protocol
+
+当前主流 AI Agent 的框架使用 `JSON Schema Function Call` 为代表的手段来操作系统提供的能力.
+也有越来越多的框架开始直接用模型生成的代码来驱动, 代表性的是 OpenInterpreter.
+
+`GhostOS` 项目设想, 未来的 AI Agent 与外部系统主要的交互手段是基于代码化的协议, 这包含四个方面:
+
+* `Code As Prompt`: 系统通过一系列的规则, 将代码直接反射成 Prompt 提供给大模型, 让大模型直接调用.
+* `Code Interpreter`: 系统将大模型生成的代码直接在环境里执行, 用来驱动系统行为.
+* `Runtime Injection`: 系统将运行时生成的各种实例, 注入到上下文中.
+* `Context Manager`: 系统管理多轮对话中各种变量的存储, 使用和回收.
+
+这整套方案在 `GhostOS` 里定义为 `MOSS` 协议, 全称是 `Model-oriented Operating System Simulator` (
+面向模型的操作系统模拟器).
+
+## MOSS
+
+MOSS 的实现 [ghostos.core.moss](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/core/moss)
+是一个可以脱离 `GhostOS` 项目使用的 python 模块.
+
+### Purpose
+
+`MOSS` 的设计目标, 是让人类工程师和大模型一样, 所见即所得地阅读一个代码上下文.
+我们以 `SpheroBoltGPT` (用代码驱动玩具 SpheroBolt) 为例:
+
+```python
+from ghostos.prototypes.spherogpt.bolt import (
+ RollFunc,
+ Ball,
+ Move,
+ LedMatrix,
+ Animation,
+)
+from ghostos.core.moss import Moss as Parent
+
+
+class Moss(Parent):
+ body: Ball
+ """your sphero ball body"""
+
+ face: LedMatrix
+ """you 8*8 led matrix face"""
+
+
+```
+这段代码就定义出了一个用来控制 Sphero Bolt 的 python 上下文.
+
+大模型和人类工程师阅读这份代码, 看到可以通过 `moss.body` 或 `moss.face` 驱动 SpheroBolt 的行为.
+代码中被引用的 `RollFunc`, `Ball`, `Move` 等类库, 会自动反射成 Prompt, 和源码一起提交给 LLM, 用来生成控制代码.
+
+这样可以要求 LLM 生成一个函数:
+
+```python
+def run(moss: Moss):
+ # body spin 360 degree in 1 second.
+ moss.body.new_move(True).spin(360, 1)
+```
+
+`MossRuntime` 会将这个函数编译到当前模块, 然后运行其中的 `run` 函数. 当这个代码执行时, 就真实调用了 SpheroBoltGPT 的 `body` 对象.
+
+### Abstract Classes
+
+`MOSS` 实现的核心是三个类:
+
+* [MossCompiler](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/core/moss/abcd.py): 编译任何 python module, 生成一个可供解析的临时模块.
+* [MossPrompter](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/core/moss/abcd.py): 反射 python module, 用来生成大模型看到的 Prompt.
+* [MossRuntime](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/core/moss/abcd.py): 在编译的临时 module 中, 执行大模型生成的代码.
+
+
+
+### Get MossCompiler
+
+`MossCompiler` 注册到了 [IoC Container](/zh-cn/concepts/ioc_container.md) 中. 要获取它的实例可以:
+
+```python
+from ghostos.bootstrap import get_container
+from ghostos.core.moss import MossCompiler
+
+compiler = get_container().force_fetch(MossCompiler)
+```
+
+### PyContext
+
+`MossCompiler` 使用 [PyContext](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/core/moss/pycontext.py) 数据结构来管理一个可持久化的上下文.
+它可以用来存储运行时定义, 修改过的变量; 也可以管理对 python 代码的直接修改, 用于下一次运行.
+
+每个 `MossCompiler` 都会继承一个独立的 IoC Container, 因此可以使用它进行依赖注入的注册:
+
+```python
+from ghostos.core.moss import MossCompiler
+from ghostos.container import Provider
+
+compiler: MossCompiler = ...
+
+
+class Foo:
+ ...
+
+
+f: Foo = ...
+
+some_provider: Provider = ...
+
+compiler.bind(Foo, f) # 绑定到 compiler.container()
+compiler.register(some_provider) # 注册 provider 到 compiler.container()
+
+attr_value = ...
+
+compiler.with_locals(attr_name=attr_value) # 在目标 python module 注入一个本地变量 attr_name
+```
+
+### Compile Runtime
+
+使用 `MossCompiler` 可以基于 PyContext 或 python modulename 来编译一个临时的 module.
+
+```python
+from ghostos.bootstrap import get_container
+from ghostos.core.moss import MossCompiler, PyContext
+
+pycontext_instance: PyContext = ...
+compiler = get_container().force_fetch(MossCompiler)
+
+# join python context to the compiler
+compiler.join_context(pycontext_instance)
+
+runtime = compiler.compile(None)
+```
+
+### Get Compiled Module
+
+可以从 MossRuntime 中获取被编译的临时模块:
+
+```python
+from types import ModuleType
+from ghostos.core.moss import MossRuntime
+
+runtime: MossRuntime = ...
+
+module: ModuleType = runtime.module()
+```
+
+### Moss Prompter
+
+使用 `MossRuntime` 可以得到一个 `MossPrompter`, 用来生成大模型的 Prompt:
+
+```python
+from ghostos.core.moss import MossRuntime
+
+runtime: MossRuntime = ...
+
+with runtime:
+ prompter = runtime.prompter()
+ prompt = prompter.dump_module_prompt() # 获取模块完整的 Prompt
+
+ # prompt 由以下部分构成:
+
+ # 1. 编译模块的源码
+ code = prompter.pycontext_code() # 获取模块的源码
+
+ for attr_name, attr_prompt in prompter.reflect_module_attr():
+ # 获取编译模块中, 各个属性的 prompt. 默认情况只会反射从其它模块 import 的属性.
+ print(attr_name, attr_prompt)
+
+ # 2. 被引用的变量反射出来的 prompt.
+ attr_prompt = prompter.dump_attrs_prompt()
+```
+
+#### Hide Code to LLM
+
+被 `MossCompiler` 编译的 module 其源码会全部提供给大模型. 如果想要隐藏一部分代码的话, 可以通过 `# ` 标记:
+
+```python
+
+#
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+ from ghostos.core.moss import MossPrompter
+
+
+# 这里定义的代码会正常执行, 但不会提交给 LLM
+# 这些代码通常用来定义 MossCompiler/Runtime 运行生命周期里的逻辑
+# 大模型不需要知道这些逻辑, 屏蔽它们有利于大模型更专注.
+
+def __moss_module_prompt__(prompter: "MossPrompter") -> str:
+ ...
+
+#
+```
+
+#### Code Reflection
+
+我们使用反射机制将代码的讯息自动生成 Prompt, 并提供给大模型.
+基本思路与程序员看引用库的心态一样, 只让 LLM 看到它关心的最少量讯息, 主要是类与函数的定义以及关键的变量.
+而不是直接把所有源码提供给模型.
+
+#### Default Reflection Pattern
+
+`MossRuntime` 会反射当前 Python module 里 import 进来的变量, 按一定规则生成它们的 Prompt.
+当前规则如下:
+
+* Function & Method: 只反射 函数名 + doc
+* Abstract class: 反射源码
+* pydantic.BaseModel: 反射源码
+
+此外, 任何一个类如果实现了 `ghostos.prompter.PromptAbleClass`, 会使用它的 `__class_prompt__` 方法生成反射结果.
+
+#### Custom Attr Prompt
+
+如果目标 python module 文件中定义了魔术方法 `__moss_attr_prompts__`, 会使用它提供的结果覆盖掉自动反射的结果.
+
+```python
+def __moss_attr_prompts__() -> "AttrPrompts":
+ yield "key", "prompt"
+```
+
+如果返回的 Prompt 为空, 则不会对大模型展示.
+
+### Runtime Execution
+
+基于 `MossRuntime`, 可以将大模型生成代码直接在编译出来的临时 Module 里执行. 这么做的好处是:
+
+1. LLM 不需要 import 所有的类库, 节省 tokens 的开销
+2. 加快生成的速度, 期望许多时候超过 json schema 的输出.
+3. 避免大模型生成代码对上下文模块的污染.
+4. 相比 Jupyter 或沙盒执行代码, 临时编译 module 是想获得一个 "最小上下文单元".
+
+基本原理就是使用当前模块作为上下文, 编译执行大模型生成的代码. 内部逻辑如下:
+
+```python
+import ghostos.core.moss
+
+runtime: ghostos.core.moss.MossRuntime = ...
+pycontext = runtime.dump_pycontext()
+local_values = runtime.locals()
+
+generated_code: str = ...
+
+filename = pycontext.module if pycontext.module is not None else ""
+compiled = compile(generated_code, filename=filename, mode='exec')
+# 直接编译
+exec(compiled, local_values)
+```
+
+我们可以要求大模型生成的代码是一个 `main` 函数, 我们用 `MossRuntime` 编译完代码后可以立刻执行这个函数:
+
+```python
+import ghostos.core.moss
+
+runtime: ghostos.core.moss.MossRuntime = ...
+# 包含 main 函数的代码
+generated_code: str = ...
+
+with runtime:
+ result = runtime.execute(target="main", code=generated_code, local_args=["foo", "bar"])
+
+ # 执行过程中的 std output
+ std_output = runtime.dump_std_output()
+ # 获取变更过的 pycontext
+ pycontext = runtime.dump_pycontext()
+```
+
+### Custom Lifecycle functions
+
+`MossRuntime` 在运行的生命周期中, 会尝试寻找编译模块里的魔术方法并执行.
+所有的魔术方法都定义在 [ghostos.core.moss.lifecycle](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/core/moss/lifecycle.py) 中. 详情请查看文件.
+主要有以下几个方法:
+
+```python
+__all__ = [
+ '__moss_compile__', # prepare moss compiler, handle dependencies register
+ '__moss_compiled__', # when moss instance is compiled
+ '__moss_attr_prompts__', # generate custom local attr prompt
+ '__moss_module_prompt__', # define module prompt
+ '__moss_exec__', # execute the generated code attach to the module
+]
+```
+
+### Moss 类
+
+在使用 `MossCompiler` 编译的目标模块中, 可以定义一个名为 `Moss` 的类, 它需要继承自 `ghostos.core.moss.Moss`,
+这样它就可以在生命周期中得到关键的依赖注入, 达到所见即所得的效果.
+
+`Moss` 类的作用有两个:
+- 自动化依赖注入: 挂载到 Moss 上的抽象类都会获得 IoC 容器的依赖注入.
+- 管理持久上下文: Moss 类上的数据对象会自动存储到 `PyContext`
+
+这个类的存在是默认的, 即便没有定义它, 也会生成一个名为 `moss` 的实例到编译的临时模块中. `moss` 实例可以传递给大模型生成代码里的函数.
+
+比如上下文:
+
+```python
+from abc import ABC
+from ghostos.core.moss import Moss as Parent
+
+class Foo(ABC):
+ ...
+
+class Moss(Parent):
+ int_val: int = 0
+
+ foo: Foo # the abstract class bound to Moss will automatically get injection from MossRuntime.container()
+```
+
+大模型生成的代码:
+
+```python
+# 大模型生成的 main 函数
+def main(moss) -> int:
+ moss.int_var = 123
+ return moss.int_var
+```
+
+执行这个函数, 未来 `Moss.int_val` 的值就会变成 `123`.
+
+这么做的目的是所见即所得的上下文管理. 主要有以下默认规则:
+
+1. 变量存储: 所有绑定到 `Moss` 实例上的 `pydantic.BaseModel` 和 `int | str | float | bool` 等变量, 会自动存储到
+ pycontext.
+2. 抽象类依赖注入: 所以在 `Moss` 上挂载的类, 会自动用 IoC Container 尝试注入实例
+3. 生命周期管理: 如果一个实现了 `ghostos.core.moss.Injection` 的类, 在注入到 `moss` 实例时,
+ 会自动调用它的 `on_injection` 与 `on_destroy` 方法.
+4. 定义一个 `Moss` 类, 并不会污染破坏目标文件的原有功能.
+
+也可以使用 `MossRuntime` 来获取所有对 `Moss` 类的注入结果:
+
+```python
+from ghostos.core.moss import Moss, MossRuntime
+
+runtime: MossRuntime = ...
+
+moss_class = runtime.moss_type()
+assert issubclass(moss_class, Moss)
+
+moss_instance = runtime.moss()
+assert isinstance(moss_instance, moss_class)
+
+injections = runtime.moss_injections()
+```
+
+## Examples
+
+关于 `MOSS` 的基线测试用例在 [ghostos.core.moss.examples](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/core/moss/examples)
+可以参考其中的测试代码理解它的原理.
+
+## MOSS TestSuite
+
+所有可以被 `MossCompiler` 编译的源码文件, 这里也称之为 `MOSS 文件`.
+
+在文件中定义出来的函数, 变量和类是可以单元测试的, 但运行时依赖注入则需要构建测试套件.
+
+`GhostOS` 提供了一个默认的套件 `ghostos.core.moss.testsuite.MossTextSuite`, 详情请见代码.
+
+
diff --git a/docs/zh-cn/concepts/prompter.md b/docs/zh-cn/concepts/prompter.md
new file mode 100644
index 00000000..2422f108
--- /dev/null
+++ b/docs/zh-cn/concepts/prompter.md
@@ -0,0 +1,58 @@
+# Prompter
+
+向大模型提供 Prompt 的工作必然走向结构化和模块化. 传统的模板语言无法承载 Prompt Engineering 的复杂性.
+
+作者认为 Prompt Engineering 应该使用类似前端界面的 `DOM` (Document Object Model) 来构建,
+可能叫做 `POM` (Prompt Object Model).
+
+对 LLM 生成的 System Prompt 本质上是一个 `POM Tree`, 它可以将上下文相关的各种数据对象组装成 Prompt.
+
+`POM` 的一些可预见的好处:
+
+1. 节点可以用数据结构来封装, 方便被其它项目复用.
+2. 可以将复杂的 UI 界面映射成 `POM Tree`, 为 LLM 自动提供视觉对象的额外讯息.
+3. `POM Tree` 可以做前端渲染, 方便人类管理足够复杂的上下文, 甚至可视化渲染.
+4. `POM Tree` 可以编程, 所以可以由 Meta-Agent 自主生成.
+5. 可以针对 `POM Tree` 进行 tokens 的优先级裁剪.
+
+这个技术实现不是 `GhostOS` 自身的目标, 但由于开源社区还没提供成熟的 `Prompt Object Model` 实现, 因此作者先实现了一个简单版.
+
+详见: [ghostos.prompter](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/prompter.py)
+
+
+以 [MossAgent](/zh-cn/usages/moss_agent.md) 为例, 它默认的 Prompter 是如下结构:
+
+```python
+ def _get_instruction_prompter(self, session: Session, runtime: MossRuntime) -> Prompter:
+ agent = self.ghost
+ return TextPrmt().with_children(
+ # system meta prompt
+ TextPrmt(
+ title="Meta Instruction",
+ content=AGENT_META_INTRODUCTION,
+ ).with_children(
+ TextPrmt(title="GhostOS", content=GHOSTOS_INTRODUCTION),
+ TextPrmt(title="MOSS", content=MOSS_INTRODUCTION),
+ # code context
+ get_moss_context_prompter("Code Context", runtime),
+ ),
+
+ # agent prompt
+ TextPrmt(
+ title="Agent Info",
+ content="The Agent info about who you are and what you are doing: ",
+ ).with_children(
+ get_agent_identity("Identity", agent.__identifier__()),
+ TextPrmt(title="Persona", content=self._get_agent_persona(session, runtime)),
+ TextPrmt(title="Instruction", content=self._get_agent_instruction(session, runtime)),
+ ),
+
+ # context prompt
+ TextPrmt(
+ title="Context",
+ content="",
+ ).with_children(
+ self._get_context_prompter(session),
+ )
+ )
+```
\ No newline at end of file
diff --git a/docs/zh-cn/frameworks/eventbus.md b/docs/zh-cn/frameworks/eventbus.md
new file mode 100644
index 00000000..d4a22d84
--- /dev/null
+++ b/docs/zh-cn/frameworks/eventbus.md
@@ -0,0 +1,44 @@
+# EventBus
+
+`GhostOS` 通过 [EventBus](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/core/runtime/events.py) 类来管理 Agent 之间,
+Agent 和外部世界, Agent 自身的事件通讯.
+
+基于事件总线, 我们可以实现一个全异步的 Agent. 以耗时较长的网络叫车为例子:
+
+1. 用户和主 Agent 对话, 要求 Agent 叫车.
+2. Agent 调用拥有叫车能力的子 Agent, 让它去执行任务.
+3. Agent 继续和用户对话.
+4. Agent 可以随时询问子 Agent 任务执行情况.
+5. 子 Agent 打到车后, 通过 Event 通知主 Agent.
+
+事件总线维持所有 Agent 的 Event Loop, 从而实现了全异步的通讯.
+
+除了 Agent 之间的通讯外, 外部系统和 Agent 的通讯也需要通过 EventBus. 不过在 `ghostos.abcd.Conversation` 抽象中内置了相关接口.
+外部系统的通讯可以包括:
+
+* 环境中发生的事件
+* 定时任务
+* 接口的异步回调
+
+# Event 对象的设计
+
+GhostOS
+中的事件对象定义在 [ghostos.core.runtime.events.Event](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/core/runtime/events.py).
+相关 API 详见代码.
+
+# EventBus 的注册
+
+作为基础类, EventBus 在 `ghostos.bootstrap.app_container` 中注册.
+只需要变更 EventBus 注册的 Provider, 就可以修改它的实现. 详见 Container 相关章节.
+
+# EventBus 的实现
+
+`EventBus` 可以有各种技术实现, 包括基于文件的, 基于关系型数据库的, 基于 Redis 等 KV 存储的. 从而实现分布式系统的事件总线.
+
+由于 `GhostOS` 没有开发人力, 目前的实现是基于内存的 dict.
+详见 [MemEventBusImpl](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/framework/eventbuses/memimpl.py).
+这意味着关闭运行中的程序, 就会导致事件丢失.
+
+# EventBus 的配置化
+
+未来希望将 EventBus 的系统默认实现变成可配置的, 用户可以通过配置项选择 `file`, `redis`, `mysql` 等几种开箱自带方案.
\ No newline at end of file
diff --git a/docs/zh-cn/frameworks/llms.md b/docs/zh-cn/frameworks/llms.md
new file mode 100644
index 00000000..feddae2f
--- /dev/null
+++ b/docs/zh-cn/frameworks/llms.md
@@ -0,0 +1,8 @@
+# LLMs
+
+`GhostOS` 将大模型调用封装到自己的抽象中.
+
+详见 [ghostos/core/llms](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/core/llms/abcd.py).
+
+> 相关能力封装和测试尚不够完整.
+> 有时间再丰富文档了.
\ No newline at end of file
diff --git a/docs/zh-cn/frameworks/messages.md b/docs/zh-cn/frameworks/messages.md
new file mode 100644
index 00000000..848f59e9
--- /dev/null
+++ b/docs/zh-cn/frameworks/messages.md
@@ -0,0 +1,38 @@
+# Messages
+
+`GhostOS` 的设计目标之一, 是实现服务端全异步的智能体集群.
+因此历史消息的传输和存取不能仅仅在客户端, 还需要在服务端.
+
+为了解决消息协议的流式传输, 模型兼容, 存储与读取等问题; `GhostOS` 设计了自己的消息容器.
+详见 [ghostos.core.messages](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/core/messages/message.py)
+
+目前没有精力介绍所有的细节, 重点介绍几个关键概念:
+
+
+## Variable Message
+
+`GhostOS` 的 agent 使用代码驱动, 所以它可以把各种运行时的变量通过 `VariableMessage` 形式传输, 包括:
+
+1. 传入给端侧, 比如 streamlit
+2. 传输给其它的 Agent
+
+在历史记录中, LLM 可以看到变量的 `vid` 参数,
+使用 [ghostos/contracts/variables](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/contracts/variables.py) 库可以获取对应的变量.
+从而可以实现基于变量的交互.
+
+举例:
+
+1. Agent 将自己的变量, 传输给另一个 Agent.
+2. Agent 将某个数据结构的变量发送给端, 端侧自行渲染.
+3. 端侧可以将变量以消息方式发送, 而 Agent 可以在代码中获取变量数据结构, 并操作它.
+4. Agent 可以操作历史上下文中看到的变量.
+
+## Audio & Image Message
+
+`GhostOS` 历史消息中的图片和音频都会使用中心化的存储, 消息 id 就是 图片 & 音频 的存储 id.
+详见 [ghostos/contracts/assets](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/contracts/assets.py).
+
+预期未来 Audio 和 Image 也支持 `变量类型消息`, 从而大模型可以用代码来操作它们.
+
+比如一个没有识图能力的大模型, 通过代码调用另一个可识图的模型来帮助自己阅读图片.
+
diff --git a/docs/zh-cn/frameworks/tasks.md b/docs/zh-cn/frameworks/tasks.md
new file mode 100644
index 00000000..9e52587b
--- /dev/null
+++ b/docs/zh-cn/frameworks/tasks.md
@@ -0,0 +1,152 @@
+# Tasks
+
+`GhostOS` 核心 feature 之一是全异步的 Multi-Agent 架构.
+
+每个运行中的 Agent (Ghost) 被视作一个 `最小有状态单元`, 它可以调度一个任务的运行状态:
+
+```python
+
+class TaskState(str, Enum):
+ """ runtime state of the task. """
+
+ NEW = "new"
+ """the task is yet created"""
+
+ RUNNING = "running"
+ """the task is running"""
+
+ WAITING = "waiting"
+ """the task needs more inputs"""
+
+ # QUEUED = "queued"
+ # """the task is queued to run"""
+
+ CANCELLED = "cancelled"
+ """the task is canceled"""
+
+ FAILED = "failed"
+ """the task is failed due to an exception"""
+
+ FINISHED = "finished"
+ """the task is finished"""
+```
+
+Agent 可以直接使用 [MOSS](/zh-cn/concepts/moss_protocol.md) 提供的类库来操作自身的状态:
+
+```python
+
+class Taskflow(Prompter, ABC):
+ """
+ default operations
+ """
+ MessageKind = Union[str, Message, Any]
+ """message kind shall be string or serializable object"""
+
+ # --- 基本操作 --- #
+ @abstractmethod
+ def finish(self, status: str = "", *replies: MessageKind) -> Operator:
+ """
+ finish self task
+ :param status: describe status of the task
+ :param replies: replies to parent task or user
+ """
+ pass
+
+ @abstractmethod
+ def fail(self, reason: str = "", *replies: MessageKind) -> Operator:
+ """
+ self task failed.
+ :param reason: describe status of the task
+ :param replies: replies to parent task or user
+ """
+ pass
+
+ @abstractmethod
+ def wait(self, status: str = "", *replies: MessageKind) -> Operator:
+ """
+ wait for the parent task or user to provide more information or further instruction.
+ :param status: describe current status
+ :param replies: question, inform or
+ """
+ pass
+
+ @abstractmethod
+ def think(self, *messages: MessageKind, instruction: str = "", sync: bool = False) -> Operator:
+ """
+ start next round thinking on messages
+ :param messages: observe target
+ :param instruction: instruction when receive the observation.
+ :param sync: if True, observe immediately, otherwise check other event first
+ :return:
+ """
+ pass
+
+ @abstractmethod
+ def observe(self, **kwargs) -> Operator:
+ """
+ observe values
+ :param kwargs:
+ :return:
+ """
+
+ @abstractmethod
+ def error(self, *messages: MessageKind) -> Operator:
+ pass
+
+
+class Subtasks(Prompter, ABC):
+ """
+ library that can handle async subtasks by other ghost instance.
+ """
+ MessageKind = Union[str, Message, Any]
+ """message kind shall be string or serializable object"""
+
+ @abstractmethod
+ def cancel(self, name: str, reason: str = "") -> None:
+ """
+ cancel an exists subtask
+ :param name: name of the task
+ :param reason: the reason to cancel it
+ :return:
+ """
+ pass
+
+ @abstractmethod
+ def send(
+ self,
+ name: str,
+ *messages: MessageKind,
+ ctx: Optional[Ghost.ContextType] = None,
+ ) -> None:
+ """
+ send message to an existing subtask
+ :param name: name of the subtask
+ :param messages: the messages to the subtask
+ :param ctx: if given, update the ghost context of the task
+ :return:
+ """
+ pass
+
+ @abstractmethod
+ def create(
+ self,
+ ghost: Ghost,
+ instruction: str = "",
+ ctx: Optional[Ghost.ContextType] = None,
+ task_name: Optional[str] = None,
+ task_description: Optional[str] = None,
+ ) -> None:
+ """
+ create subtask from a ghost instance
+ :param ghost: the ghost instance that handle the task
+ :param instruction: instruction to the ghost
+ :param ctx: the context that the ghost instance needed
+ :param task_name: if not given, use the ghost's name as the task name
+ :param task_description: if not given, use the ghost's description as the task description
+ """
+ pass
+```
+
+从而让多个 Agent 之间以 `Task` 为单位进行交流和互动.
+
+相关实现详见 [ghostos.core.runtime.tasks](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/core/runtime/tasks.py)
\ No newline at end of file
diff --git a/docs/zh-cn/frameworks/threads.md b/docs/zh-cn/frameworks/threads.md
new file mode 100644
index 00000000..80923bd8
--- /dev/null
+++ b/docs/zh-cn/frameworks/threads.md
@@ -0,0 +1,10 @@
+# Threads
+
+`GhostOS` 为了实现全异步的 Multi-Agent, 需要自己管理所有 Agent 的上下文.
+
+所以 `GhostOS` 实现了一个类似 [OpenAI Assistant](https://platform.openai.com/docs/api-reference/assistants) 的基建.
+
+Agent 生成的历史消息, 会用 `GoThreadInfo` 结构存储和使用.
+
+由于个人开发精力有限,
+详细实现请看[ghostos.core.runtime.threads](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/core/runtime/threads.py)
\ No newline at end of file
diff --git a/docs/zh-cn/getting_started/chat_with_ghost.md b/docs/zh-cn/getting_started/chat_with_ghost.md
new file mode 100644
index 00000000..d3a10b7c
--- /dev/null
+++ b/docs/zh-cn/getting_started/chat_with_ghost.md
@@ -0,0 +1,61 @@
+# Chat
+
+## Quick start
+
+`GhostOS` 选择用 [Streamlit](https://streamlit.io/) 提供开箱即用的 Agent.
+
+运行:
+
+```bash
+ghostos web python_modulename_or_filename
+```
+
+可以将一个 python 文件启动为一个 streamlit 的 agent.
+
+当前版本系统用于测试的 Agent 有:
+
+```bash
+# start chatbot
+ghostos web ghostos.demo.agents.jojo
+```
+
+你可以启动一个可独立运行的 python 文件作为 Agent, 让它解释文件内容, 调用文件的相关方法.
+`GhostOS` 会自动反射代码, 生成 Agent 可以看到的上下文.
+
+
+
+## Realtime Chat
+
+`GhostOS` 实现了 [OpenAI Realtime Beta](https://platform.openai.com/docs/api-reference/realtime).
+
+想要使用它, 需要先安装相关库:
+
+```bash
+pip install ghostos[realtime]
+```
+
+对 realtime 模型的配置详见 [configuration](./configuration.md).
+
+> 目前的 realtime 还有很多 bug 和体验问题.
+> 毕竟还是个人项目, 将就着用吧...
+
+## Runtime files
+
+当你使用 `GhostOS` 与 agent 对话时, 系统会生成各种运行时文件, 比如:
+
+* thread: 存储历史消息.
+* task: 存储对话状态机的状态.
+* images and audio: 过程中的图片和音频.
+* logs: 运行时日志.
+
+所有这类运行时文件, 都保存在 [\[workspace\]/runtime](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/app/runtime) 目录.
+
+如果需要清空它们, 请运行:
+
+```bash
+ghostos clear-runtime
+```
+
+## Create Your Agent
+
+想要创建自己的 agent, 详见 [Usage](/zh-cn/usages/moss_agent)
\ No newline at end of file
diff --git a/docs/zh-cn/getting_started/configuration.md b/docs/zh-cn/getting_started/configuration.md
new file mode 100644
index 00000000..b44cbba9
--- /dev/null
+++ b/docs/zh-cn/getting_started/configuration.md
@@ -0,0 +1,45 @@
+# Configuration
+
+`GhostOS` 依赖的配置文件都会存在 workspace 中.
+运行 `ghostos init` 可以在当前目录创建 workspace.
+系统默认的 workspace 在: [ghostos/app](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/app).
+
+而配置文件默认地址在 [ghostos/app/configs](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/app/configs).
+
+## Edit in UI
+
+`GhostOS` 默认的配置项都是通过 `pydantic.BaseModel` 定义的, 所以可以自动生成 `JSON Schema`.
+作者开发了仓库 [streamlit-react-jsonschema](https://github.com/ghost-in-moss/streamlit-react-jsonschema),
+基于 [react-jsonschema-form](https://react-jsonschema-form.readthedocs.io/) 自动化渲染表单.
+
+运行 `ghostos config` 会打开一个 streamlit 的页面, 可以可视化配置选项.
+
+> 由于我没有多少时间做测试, 所以直接修改目标配置文件更加安全可靠.
+
+## LLM Config
+
+`GhostOS` 封装了自己的 [LLMs](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/core/llms/abcd.py).
+相关配置项详见 [LLMsConfig](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/core/llms/configs.py).
+
+配置文件在 [\[workspace\]/configs/llms_conf.yml](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/app/configs/llms_conf.yml).
+
+目前经过反复测试的模型服务有:
+
+- OpenAI
+- Moonshot
+
+大多数没有直接指定模型配置的 Agent, 会直接使用这里 `LLMConfigs.default` 的模型配置项.
+
+## Realtime Beta Config
+
+`GhostOS` 支持了 [OpenAI Realtime Beta](https://platform.openai.com/docs/api-reference/realtime).
+相关配置项详见 [LLMsConfig](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/frameworks/openai_realtime/configs.py).
+
+配置文件在 [\[workspace\]/configs/openai_realtime_config.yml](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/app/configs/openai_realtime_config.yml).
+
+## Streamlit Config
+
+文件 [ghostos/app/.streamlit/config.toml](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/.streamlit/config.toml)
+是运行 `ghostos web` 时读取的 streamlit 配置项.
+
+修改它们的方式详见 [streamlit configuration](https://docs.streamlit.io/develop/concepts/configuration).
\ No newline at end of file
diff --git a/docs/zh-cn/getting_started/installation.md b/docs/zh-cn/getting_started/installation.md
new file mode 100644
index 00000000..e44d1a6a
--- /dev/null
+++ b/docs/zh-cn/getting_started/installation.md
@@ -0,0 +1,71 @@
+# Installation
+
+> `GhostOS` 仍然是一个验证中的 AI 项目, 强烈建议安装到 docker 之类的容器中, 而不在本地执行.
+
+## PIP install
+
+```bash
+pip install ghostos
+```
+
+初始化 `workspace` (默认 `app`), 当前版本的运行时文件都会存入目录.
+
+```bash
+ghostos init
+```
+
+配置大模型. 默认使用 OpenAI `gpt-4o`, 要求环境变量存在 `OPENAI_API_KEY`.
+或者运行 `streamlit` 编辑界面:
+
+```bash
+ghostos config
+```
+
+测试运行自带的 agent:
+
+```bash
+# run an agent with python filename or modulename
+ghostos web ghostos.demo.agents.jojo
+```
+
+或者将本地的 Python 文件变成一个 Agent, 可以通过自然语言对话要求它调用文件中的函数或方法:
+
+```bash
+ghostos web [my_path_file_path]
+```
+
+## Extra
+
+安装关联依赖:
+
+```bash
+pip install ghostos[sphero] # 安装 sphero 类库
+pip install ghostos[realtime] # 安装 realtime 相关类库. pyaudio 和 websockets
+```
+
+## Workspace
+
+`GhostOS` 当前版本使用本地文件来存运行时数据. 所以需要初始化一个 workspace.
+
+运行 `ghostos init` 可以用脚本复制 workspace 到当前目录.
+
+`GhostOS` 在运行中产生的数据会存放到这个目录下. 当需要清除历史数据时, 请执行:
+
+```bash
+ghostos clear-runtime
+```
+
+## Env
+
+`GhostOS` 依赖各种模型的 `access token`, 默认是从环境变量中读取.
+定义这些环境变量有两种方法:
+
+- 使用 `.env` 文件 (自动通过 `dotenv` 读取)
+
+```bash
+
+copy [workspace]/.example.env [workspace]/.env
+vim [workspace]/.env
+```
+
+配置项详见 [.example.env](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/app/.example.env)
\ No newline at end of file
diff --git a/docs/zh-cn/getting_started/scripts.md b/docs/zh-cn/getting_started/scripts.md
new file mode 100644
index 00000000..6b17ce3e
--- /dev/null
+++ b/docs/zh-cn/getting_started/scripts.md
@@ -0,0 +1,38 @@
+# Scripts
+
+`GhostOS` 自带了部分命令行工具:
+
+```bash
+$ ghostos help
+
+Commands:
+ clear-runtime clear workspace runtime files
+ config config the ghostos in streamlit web app
+ console turn a python file or module into a console agent
+ docs See GhostOS Docs
+ help Print this help message.
+ init init ghostos workspace
+ web turn a python file or module into a streamlit web agent
+```
+
+简单介绍几个命令:
+
+`ghostos config` : 使用 streamlit 界面修改配置项.
+
+`ghostos init` : 初始化 workspace 到当前目录.
+
+`ghostos web` : 基于 python 文件或者模块, 启动一个 streamlit 实现的 agent 对话界面.
+
+`ghostos console` : 用命令行启动 agent, 主要用来 debug.
+
+## Developing Scripts
+
+更多的命令行工具还在开发中. 预期有以下几个:
+
+`ghostos main` : 启动 ghostos 的官方 agent, 用来介绍 ghostos 的一切.
+
+`ghostos meta` : 使用 meta agent 编辑一个 python 文件, 可以实现 [MossAgent](/zh-cn/usages/moss_agent.md)
+
+`ghostos edit` : 使用 edit agent 编辑任何一个文件, 结合上下文, 可以修改文件内容.
+
+`ghostos script` : 运行各种基于 LLM 实现的自动化脚本, 可以不断增加相关脚本到本地.
\ No newline at end of file
diff --git a/docs/zh-cn/libraries/libraries.md b/docs/zh-cn/libraries/libraries.md
new file mode 100644
index 00000000..e65cef1b
--- /dev/null
+++ b/docs/zh-cn/libraries/libraries.md
@@ -0,0 +1,95 @@
+# Libraries
+
+对于传统基于 `JSON Schema Function Call` 实现的 Agent 而言, 它的交互对象是 `Tool`.
+而 `GhostOS` 以图灵完备的代码提供给 Agent, 所以 Agent 的交互对象是 `Library`.
+
+这里的 `libraries` 不是给开发者用的, 而是给大模型用的.
+
+让 LLM 使用 Library 有三个步骤:
+
+1. 定义和实现一个 Library.
+2. 将 Library 抽象和实现注册到 IoC Container.
+3. 将 Library 绑定到 Moss 类上.
+
+## Code As Prompt
+
+这其中一个核心的概念是 `Code As Prompt`, 写代码的同时就在定义 Prompt. 我们以多任务调度为例:
+
+```python
+class Subtasks(Prompter, ABC):
+ """
+ library that can handle async subtasks by other ghost instance.
+ """
+ MessageKind = Union[str, Message, Any]
+ """message kind shall be string or serializable object"""
+
+ @abstractmethod
+ def cancel(self, name: str, reason: str = "") -> None:
+ """
+ cancel an exists subtask
+ :param name: name of the task
+ :param reason: the reason to cancel it
+ :return:
+ """
+ pass
+
+ @abstractmethod
+ def send(
+ self,
+ name: str,
+ *messages: MessageKind,
+ ctx: Optional[Ghost.ContextType] = None,
+ ) -> None:
+ """
+ send message to an existing subtask
+ :param name: name of the subtask
+ :param messages: the messages to the subtask
+ :param ctx: if given, update the ghost context of the task
+ :return:
+ """
+ pass
+
+ @abstractmethod
+ def create(
+ self,
+ ghost: Ghost,
+ instruction: str = "",
+ ctx: Optional[Ghost.ContextType] = None,
+ task_name: Optional[str] = None,
+ task_description: Optional[str] = None,
+ ) -> None:
+ """
+ create subtask from a ghost instance
+ :param ghost: the ghost instance that handle the task
+ :param instruction: instruction to the ghost
+ :param ctx: the context that the ghost instance needed
+ :param task_name: if not given, use the ghost's name as the task name
+ :param task_description: if not given, use the ghost's description as the task description
+ """
+ pass
+```
+
+将它绑定到 Agent 可以看到的 moss 文件上:
+
+```python
+from ghostos.abcd import Subtasks
+from ghostos.core.moss import Moss as Parent
+
+class Moss(Parent):
+
+ subtasks: Subtasks
+ """manager your multi-agent tasks"""
+```
+
+* 类的源码会自动反射到 Prompt, 让大模型看到.
+* 这个库的实现会自动注入到 `Moss` 实例上, 大模型可以用生成的代码调用它.
+
+更具体的用法, 请看 [MossAgent](/zh-cn/usages/moss_agent.md).
+
+我们预期基于 [MOSS Protocol](/zh-cn/concepts/moss_protocol.md) 类似的行业标准化协议, 未来大模型使用的工具, 会单纯以代码仓库的形式开发和分享.
+
+
+## Developing Libraries
+
+`GhostOS` 开箱自带的 libraries 还在开发和测试中.
+这些工具预期都会放入 [ghostos/libraries]((https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/libraries))
\ No newline at end of file
diff --git a/docs/zh-cn/usages/chatbot.md b/docs/zh-cn/usages/chatbot.md
new file mode 100644
index 00000000..c729c36f
--- /dev/null
+++ b/docs/zh-cn/usages/chatbot.md
@@ -0,0 +1,30 @@
+# ChatBot
+
+基于 LLM 实现的对话机器人, 由于非常简单, 是 `GhostOS` 开发时的基线测试对象.
+它是 [Ghost](/zh-cn/usages/ghost.md) 一个最简单的实现.
+
+要创建属于自己的对话机器人, 可以参考文件
+[ghostos/demo/agents/jojo.py](https://github.com/ghost-in-moss/GhostOS/ghostos/demo/agents/jojo.py):
+
+```python
+from ghostos.ghosts.chatbot import Chatbot
+
+# the __ghost__ magic attr define a ghost instance
+# so the script `ghostos web` or `ghostos console` can detect it
+# and run agent application with this ghost.
+__ghost__ = Chatbot(
+ name="jojo",
+ description="a chatbot for baseline test",
+ persona="you are an LLM-driven cute girl, named jojo",
+ instruction="remember talk to user with user's language."
+)
+```
+
+想要定义自己的 chatbot, 只需要创建一个类似的 python 文件, 然后运行 `ghostos web` 命令调用它.
+
+> streamlit 生成的界面可以直接通过 `settings` 选项修改 ghost 的配置.
+> 修改结果会保存到一个本地文件 `.ghosts.yml` 中, 这样 `ghostos web` 启动时优先读取 `.ghosts.yml` 中的配置.
+
+通过对话生成 chatbot 的 meta-agent 正在测试中, 未来几个版本会放出.
+
+`GhostOS` 核心的 Agent 设计是全代码交互的 [MossAgent](/zh-cn/usages/moss_agent.md), 详见文档.
\ No newline at end of file
diff --git a/docs/zh-cn/usages/ghost.md b/docs/zh-cn/usages/ghost.md
new file mode 100644
index 00000000..569e8f49
--- /dev/null
+++ b/docs/zh-cn/usages/ghost.md
@@ -0,0 +1,250 @@
+# Ghost
+
+`Ghost` 是 `GhostOS` 的名字来源, 是智能体的 `最小有状态单元`.
+这个词来自于 [Ghost In the Shell](https://en.wikipedia.org/wiki/Ghost_in_the_Shell).
+
+在 `GhostOS` 的架构设计中, 一个智能体集群由许多个 `Ghost` 单元构成, 每个 `Ghost` 拥有自身的状态, 记忆和上下文 (Session),
+它们之间可以通过 `EventBus` 进行全异步的通讯.
+
+
+
+## Why the word `Ghost` instead of `Agent`
+
+在作者的架构设想中, `Agent` 是对于用户而言的单一实体或交互界面.
+它是拥有躯体 (也就是 Shell) 的.
+
+但在一个躯体内运行的, 可能是一个 Multi-Agent (或 Multi-Ghost) 集群, 用于:
+
+* 并行执行多个任务.
+* 模拟不同的角色进行思维推演.
+* 异步地解决长耗时任务.
+
+我们举一个简单的例子:
+
+1. 用户对话的 `Agent`, 默认运行快速的 `gpt-4o` 模型来控制对话.
+2. 当用户问到复杂问题时, 主 ghost 调用 `gpt-o3` 运行了一个长达 30秒的思考过程.
+3. 在这 30 秒内, 主 agent 并未阻塞, 而是继续和用户对话.
+4. 30 秒后, 主 agent 拿到异步回调的结果, 并告知用户.
+
+在这个例子中, 并行执行和事件总线是最关键的功能. 所以 Ghost 可以是:
+
+* 用户对话的一个 Agent
+* 主 Agent 开启的一个分身, 使用不同的模型, 专注于某个任务或思考
+* 后台运行的一个 workflow
+* 后台运行的自动机器人
+* 一个独立的脚本
+* 具身智能体的某一个可以执行自然语言命令的组件
+* 对自身运行效果进行反思的后台程序
+
+它们共同的特点是:
+
+* 大模型驱动
+* 拥有多轮运行能力
+* 类似操作系统的 thread, 是最小的同步运行单元, 拥有独立的上下文
+
+## Ghost Driver
+
+在 `GhostOS` 中, `Ghost` 需要通过至少两个类来定义其原型.
+
+[Ghost](https://github.com/ghost-in-moss/GhostOS/ghostos/abcd/concepts.py) 类:
+
+```python
+
+class Ghost(Identical, EntityClass, ABC):
+ """
+ the class defines the model of a kind of ghosts.
+ four parts included:
+ 1. configuration of the Ghost, which is Ghost.__init__. we can predefine many ghost instance for special scenes.
+ 2. context is always passed by the Caller of a ghost instance. each ghost class has it defined context model.
+ 3. goal is the static output (other than conversation messages) of a ghost instance.
+ 4. driver is
+ """
+
+ ArtifactType: ClassVar[Optional[Type]] = None
+ """ the model of the ghost's artifact, is completing during runtime"""
+
+ ContextType: ClassVar[Optional[Type[ContextType]]] = None
+ """ the model of the ghost's context, is completing during runtime'"""
+
+ DriverType: ClassVar[Optional[Type[GhostDriver]]] = None
+ """ separate ghost's methods to the driver class, make sure the ghost is simple and clear to other ghost"""
+
+```
+
+[GhostDriver](https://github.com/ghost-in-moss/GhostOS/ghostos/abcd/concepts.py) 类:
+
+```python
+
+class GhostDriver(Generic[G], ABC):
+ """
+ Ghost class is supposed to be a data class without complex methods definitions.
+ so it seems much clear when prompt to the LLM or user-level developer.
+ when LLM is creating a ghost class or instance, we expect it only see the code we want it to see,
+ without knowing the details codes of it, for safety / fewer tokens / more focus or other reasons.
+
+ so the methods of the ghost class defined in this class.
+ only core developers should know details about it.
+ """
+
+ def __init__(self, ghost: G) -> None:
+ self.ghost = ghost
+
+ def make_task_id(self, parent_scope: Scope) -> str:
+ """
+ generate unique instance id (task id) of the ghost instance.
+ """
+ pass
+
+ @abstractmethod
+ def get_artifact(self, session: Session) -> Optional[G.ArtifactType]:
+ """
+ generate the ghost goal from session_state
+ may be the Goal Model is a SessionStateValue that bind to it.
+
+ The AI behind a ghost is not supposed to operate the session object,
+ but work on the goal through functions or Moss Injections.
+ """
+ pass
+
+ @abstractmethod
+ def get_instructions(self, session: Session) -> str:
+ """
+ get system instructions of the ghost.
+ usually used in client side.
+ """
+ pass
+
+ @abstractmethod
+ def actions(self, session: Session) -> List[Action]:
+ """
+ return actions that react to the streaming output of llm
+ """
+ pass
+
+ @abstractmethod
+ def providers(self) -> Iterable[Provider]:
+ """
+ ghost return conversation level container providers.
+ the provider that is not singleton will bind to session also.
+ """
+ pass
+
+ @abstractmethod
+ def parse_event(
+ self,
+ session: Session,
+ event: Event,
+ ) -> Union[Event, None]:
+ """
+ intercept the ghost event
+ :returns: if None, the event will be ignored
+ """
+ pass
+
+ @abstractmethod
+ def on_creating(self, session: Session) -> None:
+ """
+ when the ghost task is created first time.
+ this method can initialize the thread, pycontext etc.
+ """
+ pass
+
+ @abstractmethod
+ def on_event(self, session: Session, event: Event) -> Union[Operator, None]:
+ """
+ all the state machine is only handling session event with the predefined operators.
+ """
+ pass
+
+ @abstractmethod
+ def truncate(self, session: Session) -> GoThreadInfo:
+ """
+ truncate the history messages in the thread
+ """
+ pass
+
+```
+
+这么做的动机是, `GhostOS` 使用 `Code As Prompt` 的思路直接将代码反射成大模型看到的 Prompt.
+在 Mutli-Agent 架构中, `GhostDriver` 的细节代码对于使用它的 Agent 而言是不必要的.
+将数据结构为主的 `Ghost` 和运行逻辑为主的 `GhostDriver` 进行拆分, 有利于其它 Agent 更简洁地理解如何构建一个 Agent.
+
+## Ghost Context
+
+[Context](https://github.com/ghost-in-moss/GhostOS/ghostos/abcd/concepts.py) 可以理解为 `Ghost` 运行时的入参.
+它可以接受强类型的数据结构, 同时生成 system prompt 提供给大模型, 用来理解上下文. 同时大模型可以将 context 作为一个变量来操作.
+
+Context 实现了 [Prompter](../concepts/prompter.md), 它本质上是类似 `DOM` 的 `Prompt Object Model` 数据结构,
+需要用强类型的参数反射出 system instruction, 作为 prompt 的一部分提交给 LLM.
+
+Context 通常用来实现:
+
+* 具身智能体自己身体的状态, 对周围环境的识别
+* AI 应用在端侧 (比如 IDE) 的状态, 和用户同步认知.
+* 某些动态变更的入参数据, 比如一个 AI 运维看到的监控面板.
+
+作为入参传递给对话:
+
+```python
+from pydantic import Field
+from ghostos.abcd import Context, Conversation, Ghost
+
+
+class ProjectContext(Context):
+ directory: str = Field(description="the root directory of a project")
+
+
+project_agent: Ghost = ...
+project_context: ProjectContext = ...
+conversation: Conversation = ...
+
+conversation.talk("your query to edit the project", project_context)
+```
+
+有必要的话, 可以让 MossAgent 通过 `Moss` 去操作一个 Context:
+
+```python
+from ghostos.abcd import Context
+from ghostos.core.moss import Moss as Parent
+from pydantic import Field
+
+
+class ProjectContext(Context):
+ directory: str = Field(description="the root directory of a project")
+
+
+class Moss(Parent):
+ # the moss agent can operate this ctx instance.
+ ctx: ProjectContext
+```
+
+## Ghost Artifact
+
+[Artifact](https://github.com/ghost-in-moss/GhostOS/ghostos/abcd/concepts.py) 可以理解为 `Ghost` 运行时的出参.
+只不过这个出参是可以一直变动的.
+
+通过 `Conversation` 可以拿到 `Ghost` 运行时的 `Artifact` 对象, 用于端侧的渲染:
+
+```python
+from ghostos.abcd import Conversation, Ghost, Shell
+
+my_ghost: Ghost = ...
+shell: Shell = ...
+conversation: Conversation = shell.sync(ghost=my_ghost)
+
+conversation.talk(...)
+
+# get artifact
+artifact = conversation.get_artifact()
+```
+
+在 `MossAgent` 中, 可以将 `Artifact` 挂载在 `Moss` 对象上让大模型操作.
+
+## ChatBot and MossAgent
+
+[Chatbot](/zh-cn/usages/chatbot.md) 和 [MossAgent](/zh-cn/usages/moss_agent.md) 是 `GhostOS` 项目中对 Ghost
+的基础实现. 详见相关文档.
+
+## More Ghosts
+
+developing...
\ No newline at end of file
diff --git a/evaluation/swe_bench_lite/__init__.py b/docs/zh-cn/usages/ghost_func.md
similarity index 100%
rename from evaluation/swe_bench_lite/__init__.py
rename to docs/zh-cn/usages/ghost_func.md
diff --git a/docs/zh-cn/usages/moss_agent.md b/docs/zh-cn/usages/moss_agent.md
new file mode 100644
index 00000000..67ad7d41
--- /dev/null
+++ b/docs/zh-cn/usages/moss_agent.md
@@ -0,0 +1,405 @@
+# Moss Agent
+
+`MossAgent` 是 `GhostOS` 项目最基本的 Agent 单元. 它使用 [MOSS Protocol](/zh-cn/concepts/moss_protocol.md) 提供代码交互界面,
+让 LLM 可以生成代码来驱动自己的行为.
+
+## Simplest Example
+
+创建文件 `foo.py`:
+
+```python
+
+def plus(a: int, b: int) -> int:
+ return a + b
+```
+
+执行 `ghostos web foo.py`, 要求 agent 调用 `plus` 方法.
+
+## Run Agent
+
+运行命令 `ghostos web [python_modulename_or_filename]` 可以将 python 文件直接变成 Agent, 并用 streamlit 运行.
+
+例如:
+
+```bash
+ghostos web ghostos/demo/agents/jojo.py
+# or
+ghostos web ghostos.demo.agents.jojo
+```
+
+命令执行时, 如果目标文件不存在 `__ghost__` 属性, 则会反射目标文件,
+生成 [MossAgent](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/ghosts/moss_agent/agent.py) 实例.
+这个 Agent 可以调用目标文件提供的函数和类, 执行你用自然语言提出的任务.
+
+源码如下:
+
+```python
+class MossAgent(ModelEntity, Agent):
+ """
+ Basic Agent that turn a python module into a conversational agent.
+ """
+
+ """ subclass of MossAgent could have a GoalType, default is None"""
+
+ moss_module: str = Field(description="Moss module name for the agent")
+ persona: str = Field(description="Persona for the agent, if not given, use global persona")
+ instructions: str = Field(description="The instruction that the agent should follow")
+
+ # optional configs
+ name: str = Field(default="", description="name of the agent")
+ description: str = Field(default="", description="description of the agent")
+ code: Optional[str] = Field(default=None, description="code override the module")
+ compile_module: Optional[str] = Field(None, description="Compile module name for the agent")
+ llm_api: str = Field(default="", description="name of the llm api, if none, use default one")
+ truncate_at_turns: int = Field(default=40, description="when history turns reach the point, truncate")
+ truncate_to_turns: int = Field(default=20, description="when truncate the history, left turns")
+```
+
+你也可以在目标文件里手动定义一个 `__ghost__` 对象, 方便定义详细的 instructions:
+
+```python
+
+# the python module codes
+...
+
+#
+# add and agent definition manually at the tail of the file.
+from ghostos.ghosts.moss_agent import MossAgent
+
+__ghost__ = MossAgent(
+ moss_module=__name__,
+ name="agent name",
+ description="agent desc",
+ persona="persona",
+ instructions="system instructions",
+ # use llms model defined at app/configs/llms_conf.yml
+ llm_api="moonshot-v1-128k",
+)
+
+#
+```
+
+> 通常一个 python 文件不做改动也是可以直接作为 agent 启动的.
+> 比如单元测试文件.
+
+## Code As Prompt
+
+MossAgent 会自动将目标 Python 模块反射成 Prompt, 提供给大模型.
+想要看到详细的 prompt, 可以用 `ghostos web` 生成界面上的 `instructions` 按钮查看它的 system instruction.
+
+默认的反射原理, 请看 [MOSS Protocol](/zh-cn/concepts/moss_protocol.md). 简单来说:
+
+1. 引用的函数会自动反射出函数名 + doc
+2. 抽象类反射出源代码
+
+大模型会根据 instruction, 调用名为 `moss` 的工具, 生成代码.
+生成的代码会在 [Moss](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/core/moss/abcd.py) 编译的临时模块中执行.
+
+源码请看 [MossAction](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/ghosts/moss_agent/agent.py#MossAction).
+
+如果一部分源码不想让 LLM 看到, 可以使用`# ` 和 `# ` 标记:
+
+```python
+
+#
+...
+# the code here is not visible to llm
+#
+```
+
+如果自动反射的结果不让人满意, 也可以通过魔术方法 `__moss_attr_prompts__` 手动定义:
+
+```python
+from foo import Foo
+
+
+#
+
+def __moss_attr_prompts__():
+ """
+ :return: Iterable[Tuple[attr_name: str, attr_prompt: str]]
+ """
+ yield "Foo", "" # if the prompt is empty, won't prompt it to llm
+#
+```
+
+[MOSS Protocol](/zh-cn/concepts/moss_protocol.md) 系统默认的魔术方法在
+[lifecycle](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/core/moss/lifecycle.py).
+
+## Magic lifecycle functions
+
+`MossAgent` 使用各种文件内的魔术方法来定义其特殊的运行逻辑.
+这种做法的好处第一是简化开发者使用; 第二则是对于 Meta-Agent 来说, 简化创建 Agent 时的工作量.
+
+所有的生命周期方法可以查看以下三个文件:
+
+- [for developer](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/ghosts/moss_agent/for_developer.py): 面向开发者的生命周期管理
+ - `__moss_agent_providers__`
+ - `__shell_providers__`
+ - `__moss_agent_creating__`
+ - `__moss_agent_truncate__`: 按需压缩历史消息
+ - `__moss_agent_parse_event__`: 事件拦截
+ - `__moss_agent_injections__`: 手动依赖注入
+ - `__moss_agent_on_[event_type]__`: 自定义事件处理
+- [for meta ai](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/ghosts/moss_agent/for_meta_ai.py): 面向开发者和 AI
+ 的生命周期管理
+ - `__moss_agent_artifact__`: 定义 agent 的输出
+ - `__moss_agent_actions__`: 定义 moss 之外的工具
+ - `__moss_agent_thought__`: 定义思维链
+ - `__moss_agent_instruction__`: 定义 instruction 获取方法
+ - `__moss_agent_persona__`: 定义 persona 获取方法
+- [moss lifecycle](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/core/moss/lifecycle.py): Moss 库的生命周期方法.
+
+复制这些方法到当前文件, 既可以生效自定义魔术方法.
+所有这些魔术方法都是 `可选的`. 如果能用来解决问题, 则可以使用它们.
+
+如果一切魔术方法都不够用, 那么最好的办法是自己实现 `Ghost` 和 `GhostDriver` 类,
+详见 [concepts.py](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/abcd/concepts.py)
+
+## Define Moss Class
+
+通常 import 类和方法足以让 Agent 运行. 但有两种情况需要引入 `Moss` 类 (Model-oriented Operating System Simulator):
+
+1. `Context Manage`: 希望定义在多轮对话中可以持续变更的变量
+2. `Runtime Injection`: 通过 [IoC Container](/zh-cn/concepts/ioc_container.md) 进行依赖注入.
+
+在目标文件中定义一个 Moss 类:
+
+```python
+from ghostos.core.moss import Moss as Parent
+
+
+# 名为 Moss 的类是一个特殊的类.
+class Moss(Parent):
+ ...
+ pass
+```
+
+是否定义这个类, 都会在 MossAgent 运行时生成一个 `moss` 对象. 而 MossAgent 撰写的代码也是使用它, prompt 如下 (
+会不断优化) :
+
+```markdown
+You are able to call the `moss` tool, generate code to fulfill your will.
+the python code you generated, must include a `run` function, follow the pattern:
+
+\```python
+def run(moss: Moss):
+ """
+ :param moss: instance of the class `Moss`, the properties on it will be injected with runtime implementations.
+ :return: Optional[Operator]
+ if return None, the outer system will perform default action, or observe the values you printed.
+ Otherwise, the outer system will execute the operator.
+ You shall only return operator by the libraries provided on `moss`.
+ """
+\```
+```
+
+详见 [instructions](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/ghosts/moss_agent/instructions.py)
+
+### Define Variables On Moss
+
+Moss 类上挂载的 `str`, `float`, `int`, `bool`, `str` 和 `pydantic.BaseModel` 类型数据会自动保存,
+因此 MossAgent 可以直接使用他们做变量.
+
+注意这些变量类型必须保证可以序列化. 举例:
+
+```python
+from ghostos.core.moss import Moss as Parent
+from pydantic import BaseModel, Field
+
+
+class YourVariables(BaseModel):
+ variables: dict = Field(default_factory=dict, description="you can manage your variables here")
+
+
+# 名为 Moss 的类是一个特殊的类.
+class Moss(Parent):
+ vars: YourVariables = YourVariables()
+```
+
+进一步的,
+如果挂载的数据对象实现了 [ghostos.prompter.Prompter](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/prompter.py),
+MossAgent 会自动在 system instruction 中生成 prompt 提供给大模型.
+
+相关逻辑详见 `ghostos.ghosts.moss_agent.instructions.get_moss_context_prompter` 函数.
+
+### Runtime Injection
+
+Moss 类上挂载的 `abstract class` 会自动从 [IoC Container](/zh-cn/concepts/ioc_container.md) 进行依赖注入.
+为这些抽象类提供实现有三种方法:
+
+- 定义时传入实例:
+
+```python
+from ghostos.core.moss import Moss as Parent
+
+
+class Foo:
+ ...
+ pass
+
+
+# 名为 Moss 的类是一个特殊的类.
+class Moss(Parent):
+ foo: Foo = Foo()
+```
+
+- 通过魔术方法 `__moss_agent_injections__`, 手动定义注入的实例
+
+```python
+from ghostos.core.moss import Moss as Parent
+from foo import Foo
+
+
+class Moss(Parent):
+ foo: Foo
+
+
+#
+# the code in moss-hide is invisible to llm
+
+def __moss_agent_injections__(agent, session) -> Dict[str, Any]:
+ """
+ manually define some of the injections to the Moss Class.
+ if a property of Moss is not injected here, the session container will inject it by typehint.
+ """
+ from foo.impl import FooImpl
+ return {
+ "foo": FooImpl(...)
+ }
+#
+```
+
+第三种方法是在 [IoC Container](/zh-cn/concepts/ioc_container.md) 中注册依赖实现.
+`Moss` 在实例化时会根据类型分析, 自动做依赖注入.
+有以下几个方法进行依赖注册.
+
+## Register dependencies
+
+`GhostOS` 在运行时通过可继承的 `IoC Container Tree` 来隔离不同级别的依赖. 系统默认存在的容器有以下几个级别:
+
+- [App Root Container](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/bootstrap.py) : 进程唯一容器
+- `GhostOS.container` : 进程唯一容器, 基本和 App Root Container 相等.
+- `Shell.container` : 对于同一个进程内, 所有平行运行的 Ghost 共享的容器. 通常用来启动和躯体相关的单例.
+- `Conversation.container`: 对于单个 Ghost 拥有的依赖.
+- `MossRuntime.container`: 每次 MossRuntime 被编译时, 生成的临时容器. 用来注册 `MossRuntime` 自身.
+
+在 `MossAgent` 运行时进行依赖注入的是 `MossRuntime.container`, 因此它会继承每一层父容器的注册依赖, 也可以重写它们.
+
+部分 `GhostOS` 系统提供的依赖如下:
+
+- [LoggerItf](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/contracts/logger.py): 日志
+- [Configs](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/contracts/configs.py): 配置文件
+- [Workspace](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/contracts/workspace.py): 工作区
+- [Variables](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/contracts/variables.py): 持久化变量存储
+- [LLMs](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/core/llms/llms.py): 大模型
+- [Assets](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/contracts/assets.py): 图片和音频
+- [GhostOS](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/abcd/concepts.py): GhostOS 自身
+- [Shell](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/abcd/concepts.py): 运行时生成的 Shell
+- [Conversation](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/abcd/concepts.py): 运行时生成的 conversation
+- [Session](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/abcd/concepts.py): 运行时生成的 Session, 管理主要的 API
+- [Scope](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/abcd/concepts.py): 当前对话的座标.
+- [Ghost](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/abcd/concepts.py): 当前 Agent 自身
+- [MossCompiler](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/core/moss/abcd.py): moss 编译器
+- [Tasks](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/core/runtime/tasks.py): 任务存储
+- [Threads](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/core/runtime/threads.py): 历史消息存储
+- [EventBus](https://github.com/ghost-in-moss/GhostOS/tree/main/ghostos/core/runtime/events.py): 事件总线.
+
+更多系统级绑定可调用 `Container.contracts(recursively=True)` 来调试.
+
+### MossAgent dependencies
+
+最简单的方法, 是在 Python 文件里直接用魔术方法定义依赖:
+
+```python
+
+#
+
+def __moss_agent_providers__(agent: A) -> Iterable[Provider]:
+ """
+ return conversation level providers that specially required by the Agent.
+ the conversation container will automatically register the providers and run them.
+
+ :param agent: the moss agent instance.
+ :return: providers that register to the session container.
+ """
+ return []
+
+#
+```
+
+这些依赖会在 `Conversation` 创建时进行注册.
+
+### Register root dependencies
+
+修改全局容器, 或者创建自己的容器, 都可以在过程中注册服务:
+
+```python
+from ghostos.bootstrap import reset, make_app_container
+
+# 定义新的全局容器
+new_root_container = make_app_container(...)
+
+# 重置 ghostos.bootstrap.app_container
+reset(new_root_container)
+```
+
+这样可以注册进程级别的依赖, 对所有容器生效.
+
+### Register Shell dependencies
+
+在 Shell 启动时可以注册它的依赖. 一个进程可能会反复开启多个 Shell, 因此 Shell 有单独的隔离级别.
+
+最简单的是在生命周期启动 shell 时注册:
+
+```python
+from ghostos.bootstrap import get_ghostos
+
+ghostos = get_ghostos()
+
+# register shell level providers at when shell is creating
+shell = ghostos.create_shell("shell name", providers=[...])
+```
+
+对于使用 `ghostos web` 或 `ghostos console` 启动的 python 文件, 也可以简单注册在文件的魔术方法内:
+
+```python
+#
+
+def __shell_providers__() -> Iterable[Provider]:
+ """
+ return shell level providers that specially required by the Agent.
+ if the shell is running by `ghostos web` or `ghostos console`,
+ the script will detect the __shell_providers__ attribute and register them into shell level container.
+
+ You can consider the Shell is the body of an agent.
+ So shell level providers usually register the body parts singletons, bootstrap them and register shutdown functions.
+ """
+ return []
+#
+```
+
+这个方法里定义的 providers, 会在运行 `ghostos web` 时自动加载到 shell 中.
+
+## Register Conversation dependencies
+
+通常 `__moss_agent_providers__` 魔术方法可以完成 Conversation 依赖的注册.
+但如果还需要手动的话, 则应该在创建 Conversation 时注册:
+
+```python
+from ghostos.abcd import Shell, Conversation, Ghost
+
+shell: Shell = ...
+my_ghost: Ghost = ...
+
+conversation = shell.sync(my_ghost)
+
+# register here. usually not necessary
+conversation.container().register(...)
+
+```
+
+## Meta-Agent
+
+`GhostOS` 会提供一个 `MossAgent` 用来生成其它的 `MossAgent`, 也就是 Meta-Agent.
+目前还在开发测试中.
\ No newline at end of file
diff --git a/evaluation/swe_bench_lite/ai_funcs/exploration_project.py b/evaluation/swe_bench_lite/ai_funcs/exploration_project.py
deleted file mode 100644
index c956953d..00000000
--- a/evaluation/swe_bench_lite/ai_funcs/exploration_project.py
+++ /dev/null
@@ -1,50 +0,0 @@
-from typing import Optional, List, Any
-
-from ghostos.core.aifunc import AIFunc, AIFuncResult
-from pydantic import BaseModel, Field
-from evaluation.swe_bench_lite.tools.file_reader import FileReader
-from evaluation.swe_bench_lite.ai_funcs.swe_task import SWEDebugTaskCtx
-
-from ghostos.core.moss.decorators import cls_source_code
-
-@cls_source_code()
-class CulpritFilePart(BaseModel):
- file_path: str = Field(..., description="The path of the culprit file")
- culprit_reason: str = Field(..., description="The reason why the part is the culprit")
- culprit_line_start: int = Field(..., description="The start line of the culprit")
- culprit_line_end: int = Field(..., description="The end line of the culprit")
- confidence_percentage: int = Field(..., description="The confidence percentage of the culprit")
-
-
-@cls_source_code()
-class ExplorationProjectAIFuncResult(AIFuncResult):
- found_culprits: Optional[List[CulpritFilePart]] = Field(..., description="The final culprit file parts, it can be empty if not found")
- cost_steps: int = Field(..., description="The cost steps to find the culprit files")
- confidence_percentage_requirement: int = Field(..., description="The requirement of least confidence percentage of the culprit file part")
- confidence_percentage_of_current_plan: int = Field(..., description="The confidence percentage of the current plan")
-
-
-__result_type__ = ExplorationProjectAIFuncResult
-
-file_read_func = FileReader.read_file
-
-class ExplorationProjectAIFunc(AIFunc):
- """
- write a plan (contains AI functions) to explore & exploit the target repository, to localize the issue files
- It should be a plan with loop or MCTS exploration algorithm that can be executed by AI functions
- These variables must be filled with value
- """
- max_steps: int = Field(default=20, description="the expectation max steps to localize the issue files")
- cur_step: int = Field(default=0, description="the current step of the exploration")
- thoughts: str = Field(default="", description="the brief plan of the exploration")
- debug_task_ctx: Any = Field(..., description="the debug task context")
- # parent_plan: Optional["ExplorationProjectAIFunc"] = Field(default=None, description="the parent plan of the exploration")
-
-#
-
-
-def __aifunc_instruction__(fn: ExplorationProjectAIFunc) -> str:
- return (f"ExplorationProjectAIFunc input vals: max_steps: {fn.max_steps}, cur_step: {fn.cur_step}, "
- f"thoughts: {fn.thoughts}, debug_task_ctx: {fn.debug_task_ctx}")
-
-#
diff --git a/evaluation/swe_bench_lite/ai_funcs/file_explorer.py b/evaluation/swe_bench_lite/ai_funcs/file_explorer.py
deleted file mode 100644
index c194c39a..00000000
--- a/evaluation/swe_bench_lite/ai_funcs/file_explorer.py
+++ /dev/null
@@ -1,43 +0,0 @@
-from typing import Optional, List, Any
-
-from ghostos.core.aifunc import AIFuncCtx
-from ghostos.core.moss import Moss as Parent
-from ghostos.core.aifunc import AIFunc, AIFuncResult
-from pydantic import BaseModel, Field
-from evaluation.swe_bench_lite.tools.file_content_operations import FileContentOperations
-from evaluation.swe_bench_lite.ai_funcs.swe_task_manager import SWEDebugTaskCtx
-from evaluation.swe_bench_lite.base.culprit_file_part import CulpritFilePart
-
-from ghostos.core.moss.decorators import cls_source_code
-
-
-@cls_source_code()
-class FileExplorerAIFuncResult(AIFuncResult):
- found_culprits: Optional[List[CulpritFilePart]] = Field(..., description="The final culprit file parts, it can be empty if not found")
- confidence_percentage: Optional[int] = Field(..., description="The confidence percentage of the found_culprits is the true culprit")
- file_outline: Optional[str] = Field(default="", description="the important key information or clue, or summarize of the file")
-
-
-class FileExplorerAIFunc(AIFunc):
- """
- explore & exploit the target file by reading and reasoning the file content
- """
- cur_object: str = Field(default="", description="current object to explore")
-
- debug_task_ctx: Any = Field(..., description="the debug task context")
- # parent_plan: Optional["ExplorationProjectAIFunc"] = Field(default=None, description="the parent plan of the exploration")
-
- file_content: str = Field(description="the content of the file")
-
-
-class Moss(Parent):
-
- ai_func_ctx: AIFuncCtx
- """useful to run AIFunc"""
-
-#
-
-def __aifunc_instruction__(fn: FileExplorerAIFunc) -> str:
- return (f"Your current task is {fn.cur_object}, you should use the read_file method at first, culprit parts might exist in this then thought step by step to find clues about the issue. content of file: {fn.file_content}")
-
-#
diff --git a/evaluation/swe_bench_lite/ai_funcs/project_explorer.py b/evaluation/swe_bench_lite/ai_funcs/project_explorer.py
deleted file mode 100644
index 53272511..00000000
--- a/evaluation/swe_bench_lite/ai_funcs/project_explorer.py
+++ /dev/null
@@ -1,49 +0,0 @@
-from typing import Optional, List, Any
-
-from ghostos.core.aifunc import AIFuncCtx
-from ghostos.core.moss import Moss as Parent
-from ghostos.core.aifunc import AIFunc, AIFuncResult
-from pydantic import BaseModel, Field
-from evaluation.swe_bench_lite.tools.file_content_operations import FileContentOperations
-from evaluation.swe_bench_lite.ai_funcs.swe_task_manager import SWEDebugTaskCtx
-from evaluation.swe_bench_lite.ai_funcs.file_explorer import FileExplorerAIFunc, FileExplorerAIFuncResult
-from evaluation.swe_bench_lite.base.culprit_file_part import CulpritFilePart
-
-from ghostos.core.moss.decorators import cls_source_code
-
-
-@cls_source_code()
-class ExplorationProjectAIFuncResult(AIFuncResult):
- found_culprits: Optional[List[CulpritFilePart]] = Field(..., description="The final culprit file parts, it can be empty if not found")
- cost_steps: int = Field(..., description="The cost steps to find the culprit files")
- confidence_percentage_requirement: int = Field(..., description="The requirement of least confidence percentage of the culprit file part")
- confidence_percentage_of_current_plan: int = Field(..., description="The confidence percentage of the current plan")
-
-
-class ExplorationProjectAIFunc(AIFunc):
- """
- write a plan (contains AI functions) to explore & exploit the target repository, to localize the issue files
- It should be a plan with loop or MCTS exploration algorithm that can be executed by AI functions
- These variables must be filled with value
- """
- max_steps: int = Field(default=20, description="the expectation max steps to localize the issue files")
- cur_step: int = Field(default=0, description="the current step of the exploration")
- thoughts: str = Field(default="", description="the brief plan of the exploration")
- debug_task_ctx: Any = Field(..., description="the debug task context")
- # parent_plan: Optional["ExplorationProjectAIFunc"] = Field(default=None, description="the parent plan of the exploration")
-
-
-class Moss(Parent):
-
- ai_func_ctx: AIFuncCtx
- """useful to run AIFunc"""
-
-
-#
-
-def __aifunc_instruction__(fn: ExplorationProjectAIFunc) -> str:
- return (f"ExplorationProjectAIFunc input vals: max_steps: {fn.max_steps}, cur_step: {fn.cur_step}, "
- f"thoughts: {fn.thoughts}, debug_task_ctx: {fn.debug_task_ctx}. You should return an ExplorationProjectAIFuncResult object。"
- f"Before you return the culprit file parts, you should use the read_file method or FileExplorerAIFunc, you can also using multi-run or MCTS to explore the key directory. ")
-
-#
diff --git a/evaluation/swe_bench_lite/ai_funcs/swe_task.py b/evaluation/swe_bench_lite/ai_funcs/swe_task.py
deleted file mode 100644
index 5a6c8e8a..00000000
--- a/evaluation/swe_bench_lite/ai_funcs/swe_task.py
+++ /dev/null
@@ -1,120 +0,0 @@
-from typing import Optional, List
-from ghostos.core.aifunc import AIFunc, AIFuncResult
-from pydantic import BaseModel, Field
-from ghostos.core.moss import Moss
-from ghostos.core.moss import cls_source_code
-
-import logging
-import json
-import re
-
-# 定义正则表达式
-pattern = r'(\w+)\s\(([\w\.]+)\)'
-# 编译正则表达式
-compiled_pattern = re.compile(pattern)
-
-def extract_info(text):
- match = compiled_pattern.search(text)
- if match:
- method_name, class_path = match.groups()
- return method_name, class_path
- return None
-
-
-class UnitTestInfo(BaseModel):
- test_method_name: str = Field(..., description="The name of the test method")
- test_class_name: str = Field(..., description="The name of the test class from repository content root")
-
-SEP = "\n====================================\n"
-
-@cls_source_code()
-class SWEDebugTaskCtx(BaseModel):
- workspace_path: str = Field(..., description="The path of the root workspace")
- repo: str = Field(..., description="The target repository to debug")
- instance_id: str = Field(..., description="The id of the debug task")
- base_commit: str = Field(..., description="The base commit of the repository to debug")
- issue_info: str = Field(..., description="The issue statement of the bug")
- supplementary_issue_info: str = Field(..., description="The discussion and supplementary text of the bug")
- passed_tests: List[UnitTestInfo] = Field(..., description="The list of passed unit tests before the fix")
- # environment_setup_commit: str = Field(..., description="The commit used to environment setup")
-
- def __str__(self):
- ret = (f"SWEDebugTaskCtx(workspace_path={self.workspace_path}, repo={self.repo}, instance_id={self.instance_id}, "
- f"base_commit={self.base_commit}, passed_tests={self.passed_tests}, issue_info: {SEP}{repr(self.issue_info)} ")
- if len(self.supplementary_issue_info) > 0:
- ret += f", supplementary_issue_info: {SEP}{repr(self.supplementary_issue_info)} )"
- else:
- ret += ")"
- return ret
-
-
-
-def _get_method_name_and_class_from_str(s: str) -> (str, str):
- info = extract_info(s)
- if info:
- return info[0], info[1]
- return "", ""
-
-def get_swe_debug_task_ctx(task_json_path: str="/home/llm/Project/PythonProjects/ghostos/evaluation/swe_bench_lite/django_15347.json",
- workspace_path: str="/home/llm/Project/PythonProjects/workspace") -> SWEDebugTaskCtx:
- with open(task_json_path, 'r') as f:
- task_json = json.load(f)
-
- try:
- # parse the task json to get the task context
- repo_name = task_json['repo'].split('/')[-1]
- logging.info(f"get swe debug task, repo_name: {repo_name}")
-
- instance_id = task_json['instance_id']
- logging.info(f"get swe debug task, instance_id: {instance_id}")
-
- passed_tests = []
- for test_info in task_json['PASS_TO_PASS']:
- method_name, class_path = _get_method_name_and_class_from_str(test_info)
- if len(method_name) > 0:
- passed_tests.append(UnitTestInfo(test_method_name=method_name, test_class_name=class_path))
- logging.info(f"get swe debug task, passed_test: {method_name} in {class_path}")
-
- task_ctx = SWEDebugTaskCtx(
- workspace_path=workspace_path,
- repo=repo_name,
- instance_id=instance_id,
- base_commit=task_json['base_commit'],
- issue_info=task_json['problem_statement'],
- supplementary_issue_info=task_json['hints_text'],
- passed_tests=passed_tests
- )
- except Exception as e:
- logging.error(f"Failed to get task context: {e}")
- raise e
-
- return task_ctx
-
-
-class SWETaskAIFuncResult(AIFuncResult):
- """
- news result
- """
- debug_task_ctx: SWEDebugTaskCtx = Field(..., description="the detailed information of the swe debug task")
-
-
-
-__result_type__ = SWETaskAIFuncResult
-
-class SWETaskAIFunc(AIFunc):
- """
- get detailed information about the swe debug task
- """
- instruction: str = Field(description="the instruction of the task, to get the detailed information of the swe debug task")
-
-
-#
-
-def __aifunc_instruction__(fn: SWETaskAIFunc) -> str:
- return fn.instruction
-
-
-example = SWETaskAIFunc(instruction="Fetch the metadata of detailed swe debug task information")
-
-#
-
diff --git a/evaluation/swe_bench_lite/ai_funcs/swe_task_manager.py b/evaluation/swe_bench_lite/ai_funcs/swe_task_manager.py
deleted file mode 100644
index de9ebd09..00000000
--- a/evaluation/swe_bench_lite/ai_funcs/swe_task_manager.py
+++ /dev/null
@@ -1,116 +0,0 @@
-from typing import Optional, List
-from ghostos.core.aifunc import AIFunc, AIFuncResult
-from pydantic import BaseModel, Field
-from ghostos.core.moss import Moss
-from ghostos.core.moss.decorators import cls_source_code
-
-import logging
-import json
-import re
-
-# 定义正则表达式
-pattern = r'(\w+)\s\(([\w\.]+)\)'
-# 编译正则表达式
-compiled_pattern = re.compile(pattern)
-
-def extract_info(text):
- match = compiled_pattern.search(text)
- if match:
- method_name, class_path = match.groups()
- return method_name, class_path
- return None
-
-
-class UnitTestInfo(BaseModel):
- test_method_name: str = Field(..., description="The name of the test method")
- test_class_name: str = Field(..., description="The name of the test class from repository content root")
-
-SEP = "\n====================================\n"
-
-@cls_source_code()
-class SWEDebugTaskCtx(BaseModel):
- workspace_path: str = Field(..., description="The path of the root workspace")
- repo: str = Field(..., description="The target repository to debug")
- instance_id: str = Field(..., description="The id of the debug task")
- base_commit: str = Field(..., description="The base commit of the repository to debug")
- issue_info: str = Field(..., description="The issue statement of the bug")
- supplementary_issue_info: str = Field(..., description="The discussion and supplementary text of the bug")
- passed_tests: List[UnitTestInfo] = Field(..., description="The list of passed unit tests before the fix")
- # environment_setup_commit: str = Field(..., description="The commit used to environment setup")
-
- def __str__(self):
- ret = (f"SWEDebugTaskCtx(workspace_path={self.workspace_path}, repo={self.repo}, instance_id={self.instance_id}, "
- f"base_commit={self.base_commit}, passed_tests={self.passed_tests}, issue_info: {SEP}{repr(self.issue_info)} ")
- if len(self.supplementary_issue_info) > 0:
- ret += f", supplementary_issue_info: {SEP}{repr(self.supplementary_issue_info)} )"
- else:
- ret += ")"
- return ret
-
-
-
-def _get_method_name_and_class_from_str(s: str) -> (str, str):
- info = extract_info(s)
- if info:
- return info[0], info[1]
- return "", ""
-
-def get_swe_debug_task_ctx(task_json_path: str, workspace_path: str) -> SWEDebugTaskCtx:
- with open(task_json_path, 'r') as f:
- task_json = json.load(f)
-
- try:
- # parse the task json to get the task context
- repo_name = task_json['repo'].split('/')[-1]
- logging.info(f"get swe debug task, repo_name: {repo_name}")
-
- instance_id = task_json['instance_id']
- logging.info(f"get swe debug task, instance_id: {instance_id}")
-
- passed_tests = []
- for test_info in task_json['PASS_TO_PASS']:
- method_name, class_path = _get_method_name_and_class_from_str(test_info)
- if len(method_name) > 0:
- passed_tests.append(UnitTestInfo(test_method_name=method_name, test_class_name=class_path))
- logging.info(f"get swe debug task, passed_test: {method_name} in {class_path}")
-
- task_ctx = SWEDebugTaskCtx(
- workspace_path=workspace_path,
- repo=repo_name,
- instance_id=instance_id,
- base_commit=task_json['base_commit'],
- issue_info=task_json['problem_statement'],
- supplementary_issue_info=task_json['hints_text'],
- passed_tests=passed_tests
- )
- except Exception as e:
- logging.error(f"Failed to get task context: {e}")
- raise e
-
- return task_ctx
-
-
-class SWETaskAIFuncResult(AIFuncResult):
- """
- news result
- """
- debug_task_ctx: SWEDebugTaskCtx = Field(..., description="the detailed information of the swe debug task")
-
-
-
-class SWETaskAIFunc(AIFunc):
- """
- get detailed information about the swe debug task
- """
- instruction: str = Field(description="the instruction of the task, to get the detailed information of the swe debug task")
- task_json_path: str = Field(description="the path of the task json file")
- workspace_path: str = Field(description="the path of the workspace")
-
-
-#
-
-def __aifunc_instruction__(fn: SWETaskAIFunc) -> str:
- return f"instruction: {fn.instruction} task_json_path: {fn.task_json_path}, workspace_path: {fn.workspace_path}"
-
-#
-
diff --git a/evaluation/swe_bench_lite/base/culprit_file_part.py b/evaluation/swe_bench_lite/base/culprit_file_part.py
deleted file mode 100644
index 0580b794..00000000
--- a/evaluation/swe_bench_lite/base/culprit_file_part.py
+++ /dev/null
@@ -1,11 +0,0 @@
-from ghostos.core.moss.decorators import cls_source_code
-from pydantic import BaseModel, Field
-
-
-@cls_source_code()
-class CulpritFilePart(BaseModel):
- file_path: str = Field(..., description="The path of the culprit file")
- culprit_reason: str = Field(..., description="The reason why the part is the culprit")
- culprit_line_start: int = Field(..., description="The start line of the culprit")
- culprit_line_end: int = Field(..., description="The end line of the culprit")
- confidence_percentage: int = Field(..., description="The confidence percentage of the culprit")
diff --git a/evaluation/swe_bench_lite/debug_localization.py b/evaluation/swe_bench_lite/debug_localization.py
deleted file mode 100644
index 0e6e45e4..00000000
--- a/evaluation/swe_bench_lite/debug_localization.py
+++ /dev/null
@@ -1,51 +0,0 @@
-from typing import Optional, List
-from ghostos.core.aifunc import AIFunc, AIFuncResult, AIFuncCtx
-from ghostos.core.moss import Moss as Parent
-from pydantic import Field
-from evaluation.swe_bench_lite.ai_funcs.swe_task_manager import SWETaskAIFunc, SWETaskAIFuncResult, SWEDebugTaskCtx
-from evaluation.swe_bench_lite.ai_funcs.project_explorer import ExplorationProjectAIFunc, ExplorationProjectAIFuncResult
-from evaluation.swe_bench_lite.tools.repo_context_manager import RepositoryContextManager, PrepareRepositoryResult
-from evaluation.swe_bench_lite.tools.directory_explorer import DirectoryExplorer
-
-
-
-class AgentFn(AIFunc):
- """
- AIFunc that act like an agent
- """
- request: str = Field(default="", description="raw request for the agent")
-
-
-class AgentFnResult(AIFuncResult):
- """
- the result that follow the agent request
- """
- issue_files: List[str] = Field(default=[], description="the file paths that caused the issue")
- err: Optional[str] = Field(default=None, description="error message")
-
-
-class Moss(Parent):
-
- ai_func_ctx: AIFuncCtx
- """useful to run AIFunc"""
-
-
-#
-
-
-def __aifunc_instruction__(fn: AgentFn) -> str:
- return fn.request
-
-
-example = AgentFn(
- request="Your task is localization issue files in a repository. "
- "First get the information of the swe bench task"
- "Then using prepare the environment to debug the repository. "
- "Then localize the file caused the issue (not mock, it might be a Localization(exploration and exploitation) AIFunc). "
- "If you realize some steps needs to utilizing AI to plan or implementation, utilize the AIFunc. "
- "Task json file path: /home/llm/Project/PythonProjects/GhostOS/evaluation/swe_bench_lite/django_15347.json"
- "workspace path: /home/llm/Project/PythonProjects/workspace/django"
- # "You can create AIFunc by definition class outside of the `def main(moss)`"
-)
-
-#
diff --git a/evaluation/swe_bench_lite/django_15347.json b/evaluation/swe_bench_lite/django_15347.json
deleted file mode 100644
index 4ba23db2..00000000
--- a/evaluation/swe_bench_lite/django_15347.json
+++ /dev/null
@@ -1,14 +0,0 @@
-{
- "repo": "django/django",
- "instance_id": "django__django-15347",
- "base_commit": "7c4f3965098baad2396e24501e09237425a7bd6f",
- "patch": "diff --git a/django/contrib/messages/storage/cookie.py b/django/contrib/messages/storage/cookie.py\n--- a/django/contrib/messages/storage/cookie.py\n+++ b/django/contrib/messages/storage/cookie.py\n@@ -19,7 +19,7 @@ def default(self, obj):\n # Using 0/1 here instead of False/True to produce more compact json\n is_safedata = 1 if isinstance(obj.message, SafeData) else 0\n message = [self.message_key, is_safedata, obj.level, obj.message]\n- if obj.extra_tags:\n+ if obj.extra_tags is not None:\n message.append(obj.extra_tags)\n return message\n return super().default(obj)\n",
- "test_patch": "diff --git a/tests/messages_tests/test_cookie.py b/tests/messages_tests/test_cookie.py\n--- a/tests/messages_tests/test_cookie.py\n+++ b/tests/messages_tests/test_cookie.py\n@@ -52,6 +52,12 @@ class CookieTests(BaseTests, SimpleTestCase):\n def stored_messages_count(self, storage, response):\n return stored_cookie_messages_count(storage, response)\n \n+ def encode_decode(self, *args, **kwargs):\n+ storage = self.get_storage()\n+ message = Message(constants.DEBUG, *args, **kwargs)\n+ encoded = storage._encode(message)\n+ return storage._decode(encoded)\n+\n def test_get(self):\n storage = self.storage_class(self.get_request())\n # Set initial data.\n@@ -168,12 +174,23 @@ def test_safedata(self):\n A message containing SafeData is keeping its safe status when\n retrieved from the message storage.\n \"\"\"\n- def encode_decode(data):\n- message = Message(constants.DEBUG, data)\n- encoded = storage._encode(message)\n- decoded = storage._decode(encoded)\n- return decoded.message\n+ self.assertIsInstance(\n+ self.encode_decode(mark_safe('Hello Django! ')).message,\n+ SafeData,\n+ )\n+ self.assertNotIsInstance(\n+ self.encode_decode('Hello Django! ').message,\n+ SafeData,\n+ )\n \n- storage = self.get_storage()\n- self.assertIsInstance(encode_decode(mark_safe(\"Hello Django! \")), SafeData)\n- self.assertNotIsInstance(encode_decode(\"Hello Django! \"), SafeData)\n+ def test_extra_tags(self):\n+ \"\"\"\n+ A message's extra_tags attribute is correctly preserved when retrieved\n+ from the message storage.\n+ \"\"\"\n+ for extra_tags in ['', None, 'some tags']:\n+ with self.subTest(extra_tags=extra_tags):\n+ self.assertEqual(\n+ self.encode_decode('message', extra_tags=extra_tags).extra_tags,\n+ extra_tags,\n+ )\n",
- "problem_statement": "Messages framework incorrectly serializes/deserializes extra_tags when it's an empty string\nDescription\n\t\nWhen a message is serialised and then deserialised with any of the built in storage backends, then extra_tags==\"\" is converted to extra_tags==None. This is because MessageEncoder checks for the truthyness of extra_tags rather than checking it is not None.\nTo replicate this bug\n>>> from django.conf import settings\n>>> settings.configure() # Just to allow the following import\n>>> from django.contrib.messages.storage.base import Message\n>>> from django.contrib.messages.storage.cookie import MessageEncoder, MessageDecoder\n>>> original_message = Message(10, \"Here is a message\", extra_tags=\"\")\n>>> encoded_message = MessageEncoder().encode(original_message)\n>>> decoded_message = MessageDecoder().decode(encoded_message)\n>>> original_message.extra_tags == \"\"\nTrue\n>>> decoded_message.extra_tags is None\nTrue\nEffect of the bug in application behaviour\nThis error occurred in the wild with a template tag similar to the following:\n{% if x not in message.extra_tags %}\nWhen the message was displayed as part of a redirect, it had been serialised and deserialized which meant that extra_tags was None instead of the empty string. This caused an error.\nIt's important to note that this bug affects all of the standard API (messages.debug, messages.info etc. all have a default value of extra_tags equal to \"\").\n",
- "hints_text": "",
- "created_at": "2022-01-22T01:56:48Z",
- "version": "4.1",
- "FAIL_TO_PASS": "[\"A message's extra_tags attribute is correctly preserved when retrieved\"]",
- "PASS_TO_PASS": "[\"test_add (messages_tests.test_cookie.CookieTests)\", \"test_add_lazy_translation (messages_tests.test_cookie.CookieTests)\", \"test_add_update (messages_tests.test_cookie.CookieTests)\", \"test_context_processor_message_levels (messages_tests.test_cookie.CookieTests)\", \"CookieStorage honors SESSION_COOKIE_DOMAIN, SESSION_COOKIE_SECURE, and\", \"test_custom_tags (messages_tests.test_cookie.CookieTests)\", \"test_default_level (messages_tests.test_cookie.CookieTests)\", \"test_existing_add (messages_tests.test_cookie.CookieTests)\", \"test_existing_add_read_update (messages_tests.test_cookie.CookieTests)\", \"Reading the existing storage doesn't cause the data to be lost.\", \"test_existing_read_add_update (messages_tests.test_cookie.CookieTests)\", \"With the message middleware enabled, messages are properly stored and\", \"test_get (messages_tests.test_cookie.CookieTests)\", \"test_get_bad_cookie (messages_tests.test_cookie.CookieTests)\", \"test_high_level (messages_tests.test_cookie.CookieTests)\", \"A complex nested data structure containing Message\", \"test_level_tag (messages_tests.test_cookie.CookieTests)\", \"test_low_level (messages_tests.test_cookie.CookieTests)\", \"If the data exceeds what is allowed in a cookie, older messages are\", \"test_message_rfc6265 (messages_tests.test_cookie.CookieTests)\", \"When the middleware is disabled, an exception is raised when one\", \"When the middleware is disabled, an exception is not raised\", \"Messages persist properly when multiple POSTs are made before a GET.\", \"test_no_update (messages_tests.test_cookie.CookieTests)\", \"test_repr (messages_tests.test_cookie.CookieTests)\", \"A message containing SafeData is keeping its safe status when\", \"test_settings_level (messages_tests.test_cookie.CookieTests)\", \"test_tags (messages_tests.test_cookie.CookieTests)\", \"test_with_template_response (messages_tests.test_cookie.CookieTests)\"]",
- "environment_setup_commit": "647480166bfe7532e8c471fef0146e3a17e6c0c9"
-}
\ No newline at end of file
diff --git a/evaluation/swe_bench_lite/localization.py b/evaluation/swe_bench_lite/localization.py
deleted file mode 100644
index f9f5074e..00000000
--- a/evaluation/swe_bench_lite/localization.py
+++ /dev/null
@@ -1,49 +0,0 @@
-from typing import Optional, List
-from ghostos.core.aifunc import AIFunc, AIFuncResult, AIFuncCtx
-from ghostos.core.moss import Moss as Parent
-from pydantic import Field
-from evaluation.swe_bench_lite.ai_funcs.swe_task import SWETaskAIFunc, SWETaskAIFuncResult, SWEDebugTaskCtx
-from evaluation.swe_bench_lite.ai_funcs.exploration_project import ExplorationProjectAIFunc, ExplorationProjectAIFuncResult
-from evaluation.swe_bench_lite.tools.environment_prepare import prepare_repository_for_debug, reset_repository_after_debug
-
-class AgentFn(AIFunc):
- """
- AIFunc that act like an agent
- """
- request: str = Field(description="raw request for the agent")
-
-
-class AgentFnResult(AIFuncResult):
- """
- the result that follow the agent request
- """
- issue_files: List[str] = Field(description="the file paths that caused the issue")
- err: Optional[str] = Field(default=None, description="error message")
-
-
-__result_type__ = AgentFnResult
-
-
-class Moss(Parent):
-
- ai_func_ctx: AIFuncCtx
- """useful to run AIFunc"""
-
-
-#
-
-
-def __aifunc_instruction__(fn: AgentFn) -> str:
- return fn.request
-
-
-example = AgentFn(
- request="Your task is localization issue files in a repository. "
- "First get the information of the swe bench task"
- "Then using prepare the environment to debug the repository. "
- "Then localize the file caused the issue (not mock, it might be a Localization(exploration and exploitation) AIFunc). "
- "If you realize some steps needs to utilizing AI to plan or implementation, utilize the AIFunc. "
- # "You can create AIFunc by definition class outside of the `def main(moss)`"
-)
-
-#
diff --git a/evaluation/swe_bench_lite/tools/directory_explorer.py b/evaluation/swe_bench_lite/tools/directory_explorer.py
deleted file mode 100644
index 18fd5760..00000000
--- a/evaluation/swe_bench_lite/tools/directory_explorer.py
+++ /dev/null
@@ -1,114 +0,0 @@
-import os
-import logging
-from typing import List
-from ghostos.core.moss.decorators import cls_outline
-
-
-@cls_outline()
-class DirectoryExplorer:
- """
- Must use this class when you want to explore directory in file system
- """
-
- def __init__(self, workspace_root: str):
- """
- :param workspace_root: the root path of the workspace, all operations will be restricted within this root
- """
- self.workspace_root = workspace_root
-
-
- def is_path_within_workspace(self, abs_file_path: str) -> bool:
- """
- Check if the given absolute file path is within the workspace root.
- """
- ret = abs_file_path.startswith(self.workspace_root)
- if not ret:
- logging.warning(f"#DirectoryExplorer: The given absolute file path is not within the workspace root: {abs_file_path}")
- return ret
-
- def tree(self, directory, expand_depth=1, max_show_items=10, file_extensions_whitelist=None) -> str:
- """
- Efficient for explore directory structure. More token-efficient than 'tree' or 'os.listdir()'
-
- :param directory: The target directory path to explore
- :param expand_depth: Controls the depth of directory expansion, -1 means expand all levels
- :param max_show_items: Maximum number of items to display
- :param file_extensions_whitelist: List of file extensions to display, '*' means display all files. directories are always displayed
- :return: A string representation of the directory structure
- """
- if not self.is_path_within_workspace(directory):
- return f"Error: The directory {directory} is not within the workspace."
-
- result = [f"structure of {directory}:"]
- total_items = [0]
-
- def tree_inner(directory, expand_depth, indent, max_show_items, file_extensions_whitelist, current_item_count=[0]):
- if not self.is_path_within_workspace(directory):
- return
-
- if expand_depth == 0 or current_item_count[0] >= max_show_items:
- return
-
- if file_extensions_whitelist is None:
- file_extensions_whitelist = ['*']
-
- # Directories to exclude from the output
- exclude_directories = {'.git', '.idea', '__pycache__', '.pytest_cache', '.github', '.gitignore', '.gitattributes',
- '.tx', 'LICENSE', 'LICENSE.python', 'AUTHORS', 'CONTRIBUTING.rst'}
-
- try:
- items = os.listdir(directory)
- items.sort()
- except Exception as e:
- print(f"{indent}Error accessing directory {directory}: {e}")
- return
-
- # Exclude the directories specified in exclude_directories
- items = [item for item in items if item not in exclude_directories]
-
- for item in items:
- total_items[0] += 1
- if current_item_count[0] >= max_show_items:
- continue
-
- path = os.path.join(directory, item)
- if not self.is_path_within_workspace(path):
- continue
-
- if os.path.isdir(path):
- result.append(f"{indent}{item}/")
- current_item_count[0] += 1
- tree_inner(path, expand_depth - 1, indent + " ", max_show_items, file_extensions_whitelist, current_item_count)
- else:
- if '*' in file_extensions_whitelist or any(item.endswith(ext) for ext in file_extensions_whitelist):
- result.append(f"{indent}{item}")
- current_item_count[0] += 1
-
- tree_inner(directory, expand_depth, "", max_show_items, file_extensions_whitelist)
-
- if total_items[0] > max_show_items:
- result.append(f"... {total_items[0] - max_show_items} more items")
-
- return "\n".join(result)
-
- def list_files_in_dir(self, directory: str) -> List[str]:
- """
- List all files in the specified directory (excluding subdirectories).
-
- :param directory: Path to the directory to list files from
- :return: List of all files in the directory
- """
- if not self.is_path_within_workspace(directory):
- return []
- return [f for f in os.listdir(directory) if os.path.isfile(os.path.join(directory, f)) and self.is_path_within_workspace(os.path.join(directory, f))]
-
- def list_dirs_in_dir(self, directory: str) -> List[str]:
- """
- List all subdirectories in the specified directory.
-
- :param directory: Path to the directory to list subdirectories from
- :return: List of all subdirectories in the directory
- """
- if not self.is_path_within_workspace(directory):
- return []
- return [d for d in os.listdir(directory) if os.path.isdir(os.path.join(directory, d)) and self.is_path_within_workspace(os.path.join(directory, d))]
diff --git a/evaluation/swe_bench_lite/tools/environment_prepare.py b/evaluation/swe_bench_lite/tools/environment_prepare.py
deleted file mode 100644
index 8eb66886..00000000
--- a/evaluation/swe_bench_lite/tools/environment_prepare.py
+++ /dev/null
@@ -1,109 +0,0 @@
-import os
-import logging
-
-from typing import Optional, List, Tuple, Dict
-from enum import Enum
-
-from evaluation.swe_bench_lite.ai_funcs.swe_task import SWEDebugTaskCtx
-from ghostos.core.moss.decorators import cls_definition, cls_source_code
-
-
-# create a error reason enum for prepare_repository_for_debug
-@cls_definition()
-class PrepareRepositoryResult(Enum):
- PREPARE_SUCCESS = "Prepare repository successfully"
- REPO_NOT_FOUND = "Repository not found"
- REPO_NOT_DIR = "Repository is not a directory"
- REPO_CHECKOUT_FAILED = "Failed to checkout to base commit"
- REPO_CHECKOUT_BRANCH_FAILED = "Failed to checkout to a new branch"
- REPO_BRANCH_DEL_FAILED = "Failed to delete the new branch"
-
-
-def prepare_repository_for_debug(swe_debug_task_ctx: SWEDebugTaskCtx) -> PrepareRepositoryResult:
- """
- Prepare the repository for debugging (contains git operations)
- But notice after debugging work, you MUST invoke reset_repository_after_debug to reset the repository
- :param swe_debug_task_ctx: the debug task context
- :return: the result of the preparation
- """
- target_path = os.path.join(swe_debug_task_ctx.workspace_path, swe_debug_task_ctx.repo)
- if not os.path.exists(target_path):
- logging.error(f"Repository not found: {target_path}")
- return PrepareRepositoryResult.REPO_NOT_FOUND
- if not os.path.isdir(target_path):
- logging.error(f"Repository is not a directory: {target_path}")
- return PrepareRepositoryResult.REPO_NOT_DIR
-
- os.chdir(target_path)
- logging.info(f"cd to the target repository: {target_path} succeed")
-
- # discard all changes
- discard_cmd = "git checkout -- ."
- if os.system(discard_cmd) != 0:
- logging.error(f"Failed to discard all changes at initialize")
- return PrepareRepositoryResult.REPO_CHECKOUT_FAILED
-
- # checkout to main branch
- checkout_main_cmd = "git checkout main"
- if os.system(checkout_main_cmd) != 0:
- logging.error(f"Failed to checkout to main branch")
- return PrepareRepositoryResult.REPO_CHECKOUT_FAILED
-
- # if target branch exist, delete it first
- delete_branch_cmd = f"git branch -D {swe_debug_task_ctx.instance_id}"
- if os.system(delete_branch_cmd) != 0:
- logging.error(f"Failed to delete the target branch: {swe_debug_task_ctx.instance_id}")
- return PrepareRepositoryResult.REPO_BRANCH_DEL_FAILED
-
- # checkout to a new branch
- checkout_branch_cmd = f"git checkout -b {swe_debug_task_ctx.instance_id}"
- if os.system(checkout_branch_cmd) != 0:
- logging.error(f"Failed to checkout to a new branch: {swe_debug_task_ctx.instance_id}")
- return PrepareRepositoryResult.REPO_CHECKOUT_BRANCH_FAILED
-
- # checkout to the base commit
- checkout_cmd = f"git checkout {swe_debug_task_ctx.base_commit}"
- if os.system(checkout_cmd) != 0:
- logging.error(f"Failed to checkout to base commit: {swe_debug_task_ctx.base_commit}")
- return PrepareRepositoryResult.REPO_CHECKOUT_FAILED
-
- logging.info(f"Prepare repository for debug succeed")
- return PrepareRepositoryResult.PREPARE_SUCCESS
-
-
-
-def reset_repository_after_debug(swe_debug_task_ctx: SWEDebugTaskCtx) -> None:
- """
- Reset the repository after debugging
- :param swe_debug_task_ctx: the debug task context
- """
- target_path = os.path.join(swe_debug_task_ctx.workspace_path, swe_debug_task_ctx.repo)
- if not os.path.exists(target_path) or not os.path.isdir(target_path):
- logging.error(f"reset_repository_after_debug Repository not found or not a directory: {target_path}")
- return
-
- # cd the target repository
- os.chdir(target_path)
- logging.info(f"reset_repository_after_debug cd to the target repository: {target_path} succeed")
-
- # discard all changes
- discard_cmd = "git checkout -- ."
- if os.system(discard_cmd) != 0:
- logging.error(f"Failed to discard all changes")
- return
-
- # checkout back to main
- checkout_main_cmd = "git checkout main"
- if os.system(checkout_main_cmd) != 0:
- logging.error(f"Failed to checkout back to main")
- return
-
- # delete the new branch
- delete_branch_cmd = f"git branch -D {swe_debug_task_ctx.instance_id}"
- if os.system(delete_branch_cmd) != 0:
- logging.error(f"Failed to delete the new branch: {swe_debug_task_ctx.instance_id}")
- return
-
- logging.info(f"Reset repository after debug succeed")
-
-
diff --git a/evaluation/swe_bench_lite/tools/file_content_operations.py b/evaluation/swe_bench_lite/tools/file_content_operations.py
deleted file mode 100644
index eaa2c0bd..00000000
--- a/evaluation/swe_bench_lite/tools/file_content_operations.py
+++ /dev/null
@@ -1,145 +0,0 @@
-import os
-import math
-import logging
-from typing import List
-from ghostos.core.moss.decorators import cls_outline
-
-
-@cls_outline()
-class FileContentOperations:
- """
- Must use this class when you want to read/find/write file in file system
- """
- max_line_of_file = 20000
-
- def __init__(self, workspace_root: str):
- """
- :param workspace_root: the root path of the workspace, all operations will be restricted within this root
- """
- self.workspace_root = workspace_root
-
- def is_path_within_workspace(self, abs_file_path: str) -> bool:
- """
- Check if the given absolute file path is within the workspace root.
- """
- ret = abs_file_path.startswith(self.workspace_root)
- if not ret:
- logging.warning(f"#FileContentOperations: The given absolute file path is not within the workspace root: {abs_file_path}")
- return ret
-
- @staticmethod
- def __get_digit_size(number: int) -> int:
- ret = 0
- while number >= 1:
- number /= 10
- ret += 1
- return ret
-
- def read_file(self, abs_path, page_number=1, page_size=500) -> str:
- """
- Read the file content with page number and page size(page_number is from 1 to n)
- """
-
- if not self.is_path_within_workspace(abs_path):
- return f"It must be a valid file path within the workspace: {self.workspace_root}"
-
- is_valid_file_path = os.path.exists(abs_path) and os.path.isfile(abs_path)
- if not is_valid_file_path:
- print(f"Path exists: {os.path.exists(abs_path)}") # Debug print
- print(f"Is file: {os.path.isfile(abs_path)}") # Debug print
-
- return "It's not a valid file path"
-
- with open(abs_path, "r") as f:
- lines = f.readlines()
-
- if len(lines) > FileContentOperations.max_line_of_file:
- return f"The number of line {len(lines)} exceeded our limit: {FileContentOperations.max_line_of_file}"
-
- digit_size = FileContentOperations.__get_digit_size(len(lines))
-
- page_numbers = math.ceil(len(lines) / page_size)
- if page_numbers < page_number:
- return f"page_number: {page_number} outbound the max page number ({page_numbers}) of this file "
- output_lines = []
- for i in range((page_number - 1) * page_size, min(page_number * page_size, len(lines))):
- output_lines.append(f'{i:0{digit_size}}|' + lines[i].rstrip())
-
- last_sentence = f"[Showing page {page_number}/{page_numbers} , specify the page_number to see more content in this file]"
- output_lines.append(last_sentence)
-
- return '\n'.join(output_lines)
-
- def write_file(self, abs_path: str, content: str) -> str:
- if not self.is_path_within_workspace(abs_path):
- return f"It must be a valid file path within the workspace: {self.workspace_root}"
-
- with open(abs_path, "w") as f:
- f.write(content)
-
- return ""
-
-
- def find_line_numbers_containing_string(self, abs_filepath: str, search_string: str) -> List[int]:
- """
- Find the line numbers of the specific string in the file
- """
- if not self.is_path_within_workspace(abs_filepath):
- return []
-
- with open(abs_filepath, 'r') as file:
- lines = file.readlines()
- return [i + 1 for i, line in enumerate(lines) if search_string in line]
-
- def find_files_containing_string_in_directory(self, directory: str, search_string: str) -> List[str]:
- """
- Find the file paths that contain the specific string in the directory
- """
- if not self.is_path_within_workspace(directory):
- return []
-
- file_paths = []
- for root, dirs, files in os.walk(directory):
- for file in files:
- abs_filepath = os.path.join(root, file)
- if self.is_text_file(abs_filepath):
- try:
- with open(abs_filepath, 'r', encoding='utf-8') as f:
- if search_string in f.read():
- file_paths.append(abs_filepath)
- except UnicodeDecodeError:
- # Skip files that can't be decoded with UTF-8
- continue
-
- return file_paths
-
- def is_text_file(self, filepath: str) -> bool:
- """
- Check if a file is likely to be a text file based on its content and extension
- """
- # Check file extension first
- text_extensions = {'.txt', '.py', '.js', '.html', '.css', '.json', '.xml', '.yml', '.yaml', '.md',
- '.go', '.java', '.c', '.cpp', '.h', '.hpp', '.rs', '.rb', '.php', '.ts',
- '.scala', '.kt', '.swift', '.m', '.sh', '.bat', '.ps1', '.sql', '.r', '.pl',
- '.cfg', '.ini', '.conf', '.toml', '.rst', '.tex', '.log', '.gitignore',
- '.env', '.properties', '.gradle', '.pom', '.sbt', '.dockerfile', '.makefile'}
-
- if os.path.splitext(filepath)[1].lower() in text_extensions:
- return True
-
- # Check if the file is a markdown or restructured text file without extension
- if os.path.basename(filepath).lower() in {'readme', 'license', 'authors', 'contributing'}:
- return True
-
- try:
- with open(filepath, 'rb') as f:
- return not self.is_binary_string(f.read(1024))
- except IOError:
- return False
-
- def is_binary_string(self, bytes_to_check: bytes) -> bool:
- """
- Check if a byte string is likely to be binary
- """
- textchars = bytearray({7,8,9,10,12,13,27} | set(range(0x20, 0x100)) - {0x7f})
- return bool(bytes_to_check.translate(None, textchars))
diff --git a/evaluation/swe_bench_lite/tools/file_reader.py b/evaluation/swe_bench_lite/tools/file_reader.py
deleted file mode 100644
index a2ea2c9c..00000000
--- a/evaluation/swe_bench_lite/tools/file_reader.py
+++ /dev/null
@@ -1,55 +0,0 @@
-import os
-import math
-from ghostos.core.moss.decorators import cls_definition
-
-
-@cls_definition()
-class FileReader:
- """
- Must use this when you want to read source file in file system
- """
- max_line_of_file = 20000
-
- @staticmethod
- def __get_digit_size(number: int) -> int:
- ret = 0
- while number >= 1:
- number /= 10
- ret += 1
- return ret
-
- @staticmethod
- def read_file(path, page_number=1, page_size=500) -> str:
- """
- page_number is from 1 to n
- """
- abs_path = os.path.abspath(path)
- print(f"Current working directory: {os.getcwd()}") # Debug print
- print(f"Checking file path: {path}") # Debug print
- print(f"Absolute file path: {abs_path}") # Debug print
- is_valid_file_path = os.path.exists(abs_path) and os.path.isfile(abs_path)
- if not is_valid_file_path:
- print(f"Path exists: {os.path.exists(abs_path)}") # Debug print
- print(f"Is file: {os.path.isfile(abs_path)}") # Debug print
-
- return "It's not a valid file path"
-
- with open(abs_path, "r") as f:
- lines = f.readlines()
-
- if len(lines) > FileReader.max_line_of_file:
- return f"The number of line {len(lines)} exceeded our limit: {FileReader.max_line_of_file}"
-
- digit_size = FileReader.__get_digit_size(len(lines))
-
- page_numbers = math.ceil(len(lines) / page_size)
- if page_numbers < page_number:
- return f"page_number: {page_number} outbound the max page number ({page_numbers}) of this file "
- output_lines = []
- for i in range((page_number - 1) * page_size, min(page_number * page_size, len(lines))):
- output_lines.append(f'{i:0{digit_size}}|' + lines[i].rstrip())
-
- last_sentence = f"[Showing page {page_number}/{page_numbers} , specify the page_number to see more content in this file]"
- output_lines.append(last_sentence)
-
- return '\n'.join(output_lines)
diff --git a/evaluation/swe_bench_lite/tools/graph_searcher.py b/evaluation/swe_bench_lite/tools/graph_searcher.py
deleted file mode 100644
index 8f838ba1..00000000
--- a/evaluation/swe_bench_lite/tools/graph_searcher.py
+++ /dev/null
@@ -1,47 +0,0 @@
-import networkx as nx
-
-class RepoSearcher:
- def __init__(self, graph):
- self.graph = graph
-
- def one_hop_neighbors(self, query):
- # get one-hop neighbors from networkx graph
- return list(self.graph.neighbors(query))
-
- def two_hop_neighbors(self, query):
- # get two-hop neighbors from networkx graph
- one_hop = self.one_hop_neighbors(query)
- two_hop = []
- for node in one_hop:
- two_hop.extend(self.one_hop_neighbors(node))
- return list(set(two_hop))
-
- def dfs(self, query, depth):
- # perform depth-first search on networkx graph
- visited = []
- stack = [(query, 0)]
- while stack:
- node, level = stack.pop()
- if node not in visited:
- visited.append(node)
- if level < depth:
- stack.extend(
- [(n, level + 1) for n in self.one_hop_neighbors(node)]
- )
- return visited
-
- def bfs(self, query, depth):
- # perform breadth-first search on networkx graph
- visited = []
- queue = [(query, 0)]
- while queue:
- node, level = queue.pop(0)
- if node not in visited:
- visited.append(node)
- if level < depth:
- queue.extend(
- [(n, level + 1) for n in self.one_hop_neighbors(node)]
- )
- return visited
-
-
diff --git a/evaluation/swe_bench_lite/tools/repo_code_navigator.py b/evaluation/swe_bench_lite/tools/repo_code_navigator.py
deleted file mode 100644
index 2d052617..00000000
--- a/evaluation/swe_bench_lite/tools/repo_code_navigator.py
+++ /dev/null
@@ -1,307 +0,0 @@
-import os
-import tree_sitter
-from pydantic import BaseModel, Field
-from typing import List, Optional
-from tree_sitter import Language, Parser
-from tree_sitter_languages import get_language, get_parser
-import pylint.lint
-from pylint.reporters.text import TextReporter
-from io import StringIO
-
-
-_PythonParser = get_parser('python')
-
-
-class CodeLocation(BaseModel):
- file_path: str = Field(
- description="The full path to the file where the definition is found"
- )
- line_number: int = Field(
- description="The line number where the definition starts (1-indexed)"
- )
- column_number: int = Field(
- description="The column number where the definition starts (1-indexed)"
- )
- context: List[str] = Field(
- description="A list of strings representing the lines of code around the definition, typically including a few lines before and after for context"
- )
-
- def __str__(self):
- context_str = '\n'.join(f" {line.rstrip()}" for line in self.context)
- return f"Definition found:\n" \
- f" File: {self.file_path}\n" \
- f" Line: {self.line_number}, Column: {self.column_number}\n" \
- f" Context:\n{context_str}"
-
-
-
-class CodeReference(BaseModel):
- file_path: str = Field(
- description="The full path to the file where the reference is found"
- )
- line_number: int = Field(
- description="The line number where the reference is found (1-indexed)"
- )
- column_number: int = Field(
- description="The column number where the reference starts (1-indexed)"
- )
- context: str = Field(
- description="The line of code containing the reference"
- )
-
- def __str__(self):
- return f"Reference found:\n" \
- f" File: {self.file_path}\n" \
- f" Line: {self.line_number}, Column: {self.column_number}\n" \
- f" Context: {self.context.strip()}"
-
-
-
-class RepositoryCodeNavigator:
- def __init__(self, repo_path):
- self.parser = _PythonParser
- self.repo_path = os.path.abspath(repo_path)
- self.file_trees = {}
- self._parse_repository()
-
- def _parse_repository(self):
- for root, _, files in os.walk(self.repo_path):
- for file in files:
- if file.endswith('.py'):
- file_path = os.path.relpath(os.path.join(root, file), self.repo_path)
- with open(os.path.join(self.repo_path, file_path), 'r') as f:
- code = f.read()
- tree = self.parser.parse(bytes(code, 'utf8'))
- self.file_trees[file_path] = tree
-
- def go_to_definition(self, file_path, line_number, target_string) -> Optional[CodeLocation]:
- # Convert relative path to absolute path
- abs_file_path = os.path.join(self.repo_path, file_path)
-
- # First, try to find the definition in the current file
- definition = self._find_definition_in_file(abs_file_path, line_number, target_string)
- if definition:
- # Convert absolute path back to relative path in the result
- definition.file_path = os.path.relpath(definition.file_path, self.repo_path)
- return definition
-
- # If not found, search in all files
- for rel_path, tree in self.file_trees.items():
- definition = self._find_definition_in_tree(tree, target_string)
- if definition:
- # Path is already relative in this case
- return definition
-
- return None
-
- def find_references(self, file_path, line_number, target_string) -> List[CodeReference]:
- """
- TODO: IDE的find_usages功能并不需要每次都遍历所有文件，只需要遍历与当前文件相关的文件即可。可以通过文件的import关系来确定。
- """
- references = []
-
- # Search in all files
- for rel_path, tree in self.file_trees.items():
- # Read the content of the file
- with open(os.path.join(self.repo_path, rel_path), 'r') as file:
- content = file.read()
-
- root_node = tree.root_node
- cursor = root_node.walk()
-
- reached_root = False
- while not reached_root:
- if cursor.node.type == 'identifier' and cursor.node.text.decode('utf8') == target_string:
- start_line = cursor.node.start_point[0] + 1
- start_column = cursor.node.start_point[1] + 1
- context_line = content.splitlines()[start_line - 1]
- references.append(CodeReference(
- file_path=rel_path, # Use relative path
- line_number=start_line,
- column_number=start_column,
- context=context_line
- ))
-
- if not cursor.goto_first_child():
- while not cursor.goto_next_sibling():
- if not cursor.goto_parent():
- reached_root = True
- break
- return references
-
- def find_implementations(self, target_string: str) -> List[CodeLocation]:
- implementations = []
- for rel_path, tree in self.file_trees.items():
- root_node = tree.root_node
- implementation_nodes = self._find_implementation_nodes(root_node, target_string)
-
- if implementation_nodes:
- file_path = os.path.join(self.repo_path, rel_path)
- with open(file_path, 'r') as file:
- content = file.read()
- lines = content.splitlines()
-
- for node in implementation_nodes:
- start_line = node.start_point[0] + 1
- start_column = node.start_point[1] + 1
-
- context_start = max(0, start_line - 3)
- context_end = min(len(lines), start_line + 4)
- context = lines[context_start:context_end]
-
- implementations.append(CodeLocation(
- file_path=rel_path,
- line_number=start_line,
- column_number=start_column,
- context=context
- ))
-
- return implementations
-
- def _find_implementation_nodes(self, root_node, target_string):
- implementation_nodes = []
- cursor = root_node.walk()
-
- reached_root = False
- while not reached_root:
- if cursor.node.type in ['function_definition', 'class_definition', 'method_definition']:
- name_node = cursor.node.child_by_field_name('name')
- if name_node and name_node.text.decode('utf8') == target_string:
- implementation_nodes.append(cursor.node)
-
- if not cursor.goto_first_child():
- while not cursor.goto_next_sibling():
- if not cursor.goto_parent():
- reached_root = True
- break
-
- return implementation_nodes
-
- def _find_definition_in_file(self, file_path, line_number, target_string):
- with open(file_path, 'r') as file:
- content = file.read()
-
- tree = self.parser.parse(bytes(content, 'utf8'))
- root_node = tree.root_node
-
- target_node = self._find_node_by_line_and_string(root_node, line_number, target_string)
-
- if target_node:
- start_line = target_node.start_point[0]
- end_line = target_node.end_point[0]
-
- # Capture more lines before and after for context
- context_start = max(0, start_line - 3)
- context_end = min(len(content.splitlines()), end_line + 4)
-
- context_lines = content.splitlines()[context_start:context_end]
-
- return CodeLocation(
- file_path=file_path,
- line_number=start_line + 1,
- column_number=target_node.start_point[1] + 1,
- context=context_lines
- )
-
- return None
-
- def _find_definition_in_tree(self, tree, target_string) -> Optional[CodeLocation]:
- root_node = tree.root_node
- definition_node = self._find_definition(root_node, target_string)
- if definition_node:
- file_path = next((path for path, t in self.file_trees.items() if t == tree), None)
- if file_path:
- with open(os.path.join(self.repo_path, file_path), 'r') as file:
- lines = file.readlines()
- def_line = definition_node.start_point[0] + 1
- def_column = definition_node.start_point[1] + 1
- context = lines[max(0, def_line-3):def_line+2]
- return CodeLocation(
- file_path=file_path, # This is already a relative path
- line_number=def_line,
- column_number=def_column,
- context=context
- )
- return None
-
- def _find_node_by_line_and_string(self, root_node, line_number, target_string):
- for node in root_node.children:
- if node.start_point[0] + 1 <= line_number <= node.end_point[0] + 1:
- cursor = node.walk()
- reached_end = False
- while not reached_end:
- current_node = cursor.node
- if current_node.type in ['function_definition', 'class_definition', 'method_definition'] and \
- current_node.child_by_field_name('name').text.decode('utf8') == target_string:
- return current_node
- if not cursor.goto_first_child():
- while not cursor.goto_next_sibling():
- if not cursor.goto_parent():
- reached_end = True
- break
- return None
-
- def _find_definition_from_node(self, start_node, target_string):
- current = start_node
- while current.parent:
- current = current.parent
- if current.type in ['function_definition', 'class_definition', 'assignment', 'expression_statement']:
- # 检查是否是目标的定义
- if self._is_definition_of(current, target_string):
- return current
- return None
-
- def _is_definition_of(self, node, target_string):
- if node.type in ['function_definition', 'class_definition']:
- name_node = node.child_by_field_name('name')
- return name_node and name_node.text.decode('utf8') == target_string
- elif node.type == 'assignment':
- left_side = node.child_by_field_name('left')
- if left_side:
- if left_side.type == 'identifier':
- return left_side.text.decode('utf8') == target_string
- elif left_side.type == 'pattern_list':
- # Handle multiple assignments
- for child in left_side.children:
- if child.type == 'identifier' and child.text.decode('utf8') == target_string:
- return True
- elif node.type == 'expression_statement':
- child = node.child_by_field_name('expression')
- if child and child.type == 'assignment':
- return self._is_definition_of(child, target_string)
- elif node.type == 'identifier' and node.text.decode('utf8') == target_string:
- # Check if the identifier is part of an assignment
- parent = node.parent
- if parent and parent.type == 'assignment':
- return True
- return False
-
- def _find_definition(self, root_node, target_string):
- cursor = root_node.walk()
-
- reached_root = False
- while not reached_root:
- if cursor.node.type in ['function_definition', 'class_definition', 'assignment', 'expression_statement']:
- if self._is_definition_of(cursor.node, target_string):
- return cursor.node
-
- if not cursor.goto_first_child():
- while not cursor.goto_next_sibling():
- if not cursor.goto_parent():
- reached_root = True
- break
- return None
-
-
-
-# 使用示例
-if __name__ == "__main__":
- repo_path = '/home/llm/Project/PythonProjects/auto-code-rover'
- helper = RepositoryCodeNavigator(repo_path)
-
- file_path = 'app/api/manage.py' # Now using relative path
- target_string = 'SearchManager'
-
- definition = helper.go_to_definition(file_path, 81, target_string)
- print(f"Definition: {definition}")
-
diff --git a/evaluation/swe_bench_lite/tools/repo_context_manager.py b/evaluation/swe_bench_lite/tools/repo_context_manager.py
deleted file mode 100644
index c65a9ec4..00000000
--- a/evaluation/swe_bench_lite/tools/repo_context_manager.py
+++ /dev/null
@@ -1,118 +0,0 @@
-import os
-import logging
-from typing import Optional, List, Tuple, Dict
-from enum import Enum
-import atexit
-import subprocess
-from evaluation.swe_bench_lite.ai_funcs.swe_task_manager import SWEDebugTaskCtx
-from ghostos.core.moss.decorators import cls_definition, cls_source_code, cls_outline
-
-
-@cls_source_code()
-class PrepareRepositoryResult(Enum):
- PREPARE_SUCCESS = "Prepare repository successfully"
- REPO_NOT_FOUND = "Repository not found"
- REPO_NOT_DIR = "Repository is not a directory"
- REPO_CHECKOUT_FAILED = "Failed to checkout to base commit"
- REPO_CHECKOUT_BRANCH_FAILED = "Failed to checkout to a new branch"
- REPO_BRANCH_DEL_FAILED = "Failed to delete the new branch"
-
-
-@cls_outline()
-class RepositoryContextManager:
- """
- This class manages the repository context for debugging.
- You should call prepare_repository_for_debug() before your business logic, and call reset_repository_after_debug() after you're ALL done.
- """
- def __init__(self, debug_task_ctx: SWEDebugTaskCtx):
- self.debug_task_ctx = debug_task_ctx
- atexit.register(self.reset_repository_after_debug)
-
- def prepare_repository_for_debug(self) -> PrepareRepositoryResult:
- """
- Prepare the repository for debugging (contains git operations)
- :return: the result of the preparation
- """
- target_path = os.path.join(self.debug_task_ctx.workspace_path, self.debug_task_ctx.repo)
- if not os.path.exists(target_path):
- logging.error(f"Repository not found: {target_path}")
- return PrepareRepositoryResult.REPO_NOT_FOUND
- if not os.path.isdir(target_path):
- logging.error(f"Repository is not a directory: {target_path}")
- return PrepareRepositoryResult.REPO_NOT_DIR
-
- os.chdir(target_path)
- logging.info(f"cd to the target repository: {target_path} succeed")
-
- # discard all changes
- discard_cmd = "git checkout -- ."
- if os.system(discard_cmd) != 0:
- logging.error(f"Failed to discard all changes at initialize")
- return PrepareRepositoryResult.REPO_CHECKOUT_FAILED
-
- # checkout to main branch
- checkout_main_cmd = "git checkout main"
- if os.system(checkout_main_cmd) != 0:
- logging.error(f"Failed to checkout to main branch")
- return PrepareRepositoryResult.REPO_CHECKOUT_FAILED
-
- # Check if the target branch exists
- check_branch_cmd = f"git branch --list {self.debug_task_ctx.instance_id}"
- #If the stripped output is non-empty (truthy), it means the branch exists.
- if subprocess.run(check_branch_cmd, shell=True, capture_output=True, text=True).stdout.strip():
- # If the branch exists, delete it
- delete_branch_cmd = f"git branch -D {self.debug_task_ctx.instance_id}"
- if os.system(delete_branch_cmd) != 0:
- logging.error(f"Failed to delete the target branch: {self.debug_task_ctx.instance_id}")
- return PrepareRepositoryResult.REPO_BRANCH_DEL_FAILED
-
- # checkout to a new branch
- checkout_branch_cmd = f"git checkout -b {self.debug_task_ctx.instance_id}"
- if os.system(checkout_branch_cmd) != 0:
- logging.error(f"Failed to checkout to a new branch: {self.debug_task_ctx.instance_id}")
- return PrepareRepositoryResult.REPO_CHECKOUT_BRANCH_FAILED
-
- # checkout to the base commit
- checkout_cmd = f"git checkout {self.debug_task_ctx.base_commit}"
- if os.system(checkout_cmd) != 0:
- logging.error(f"Failed to checkout to base commit: {self.debug_task_ctx.base_commit}")
- return PrepareRepositoryResult.REPO_CHECKOUT_FAILED
-
- logging.info(f"Prepare repository for debug succeed")
- return PrepareRepositoryResult.PREPARE_SUCCESS
-
- def reset_repository_after_debug(self) -> None:
- """
- Reset the repository after debugging
- """
- logging.info(f"^^^^^^^^^^^^^^^^^^^^^^^^Reset repository after debug^^^^^^^^^^^^^^^^^^^^^^^^")
- target_path = os.path.join(self.debug_task_ctx.workspace_path, self.debug_task_ctx.repo)
- if not os.path.exists(target_path) or not os.path.isdir(target_path):
- logging.error(f"reset_repository_after_debug Repository not found or not a directory: {target_path}")
- return
-
- # cd the target repository
- os.chdir(target_path)
- logging.info(f"reset_repository_after_debug cd to the target repository: {target_path} succeed")
-
- # discard all changes
- discard_cmd = "git checkout -- ."
- if os.system(discard_cmd) != 0:
- logging.error(f"Failed to discard all changes")
- return
-
- # checkout back to main
- checkout_main_cmd = "git checkout main"
- if os.system(checkout_main_cmd) != 0:
- logging.error(f"Failed to checkout back to main")
- return
-
- # delete the new branch
- delete_branch_cmd = f"git branch -D {self.debug_task_ctx.instance_id}"
- if os.system(delete_branch_cmd) != 0:
- logging.error(f"Failed to delete the new branch: {self.debug_task_ctx.instance_id}")
- return
-
- logging.info(f"Reset repository after debug succeed")
- atexit.unregister(self.reset_repository_after_debug) # Unregister after execution
-
diff --git a/evaluation/swe_bench_lite/tools/repo_graph.py b/evaluation/swe_bench_lite/tools/repo_graph.py
deleted file mode 100644
index e8402e48..00000000
--- a/evaluation/swe_bench_lite/tools/repo_graph.py
+++ /dev/null
@@ -1,586 +0,0 @@
-# This file is adapted from the following sources:
-# RepoMap: https://github.com/paul-gauthier/aider/blob/main/aider/repomap.py
-# Agentless: https://github.com/OpenAutoCoder/Agentless/blob/main/get_repo_structure/get_repo_structure.py
-# grep-ast: https://github.com/paul-gauthier/grep-ast
-
-import colorsys
-import os
-import random
-import re
-import warnings
-from collections import Counter, defaultdict, namedtuple
-from pathlib import Path
-import builtins
-import inspect
-import networkx as nx
-from grep_ast import TreeContext, filename_to_lang
-from pygments.lexers import guess_lexer_for_filename
-from pygments.token import Token
-from pygments.util import ClassNotFound
-from tqdm import tqdm
-import ast
-import pickle
-import json
-from copy import deepcopy
-from evaluation.swe_bench_lite.tools.utils import create_structure
-
-# tree_sitter is throwing a FutureWarning
-warnings.simplefilter("ignore", category=FutureWarning)
-from tree_sitter_languages import get_language, get_parser
-
-# relative path, full path, line numbers(start, end), name, kind, category, info(source code)
-Tag = namedtuple("Tag", "rel_fname fname line name kind category info".split())
-
-
-class CodeGraph:
-
- warned_files = set()
-
- def __init__(
- self,
- map_tokens=1024,
- root=None,
- main_model=None,
- io=None,
- repo_content_prefix=None,
- max_context_window=None,
- ):
- self.io = io
- if not root:
- root = os.getcwd()
- self.root = root
- self.max_map_tokens = map_tokens
- self.max_context_window = max_context_window
- self.repo_content_prefix = repo_content_prefix
- self.structure = create_structure(self.root)
-
- def get_code_graph(self, other_files, mentioned_fnames=None):
- if self.max_map_tokens <= 0:
- return
- if not other_files:
- return
- if not mentioned_fnames:
- mentioned_fnames = set()
-
- max_map_tokens = self.max_map_tokens
-
- # With no files in the chat, give a bigger view of the entire repo
- MUL = 16
- padding = 4096
- if max_map_tokens and self.max_context_window:
- target = min(max_map_tokens * MUL, self.max_context_window - padding)
- else:
- target = 0
-
- tags = self.get_tag_files(other_files, mentioned_fnames)
- code_graph = self.tag_to_graph(tags)
-
- return tags, code_graph
-
- def get_tag_files(self, other_files, mentioned_fnames=None):
- try:
- tags = self.get_ranked_tags(other_files, mentioned_fnames)
- return tags
- except RecursionError:
- self.io.tool_error("Disabling code graph, git repo too large?")
- self.max_map_tokens = 0
- return
-
- def tag_to_graph(self, tags):
-
- G = nx.MultiDiGraph()
- for tag in tags:
- G.add_node(tag.name, category=tag.category, info=tag.info, fname=tag.fname, line=tag.line, kind=tag.kind)
-
- for tag in tags:
- if tag.category == 'class':
- class_funcs = tag.info.split('\t')
- for f in class_funcs:
- G.add_edge(tag.name, f.strip())
-
- tags_ref = [tag for tag in tags if tag.kind == 'ref']
- tags_def = [tag for tag in tags if tag.kind == 'def']
- for tag in tags_ref:
- for tag_def in tags_def:
- if tag.name == tag_def.name:
- G.add_edge(tag.name, tag_def.name)
- return G
-
- def get_rel_fname(self, fname):
- return os.path.relpath(fname, self.root)
-
- def split_path(self, path):
- path = os.path.relpath(path, self.root)
- return [path + ":"]
-
- def get_mtime(self, fname):
- try:
- return os.path.getmtime(fname)
- except FileNotFoundError:
- self.io.tool_error(f"File not found error: {fname}")
-
- def get_class_functions(self, tree, class_name):
- class_functions = []
-
- for node in ast.walk(tree):
- if isinstance(node, ast.ClassDef) and node.name == class_name:
- for item in node.body:
- if isinstance(item, ast.FunctionDef):
- class_functions.append(item.name)
-
- return class_functions
-
- def get_func_block(self, first_line, code_block):
- first_line_escaped = re.escape(first_line)
- pattern = re.compile(rf'({first_line_escaped}.*?)(?=(^\S|\Z))', re.DOTALL | re.MULTILINE)
- match = pattern.search(code_block)
-
- return match.group(0) if match else None
-
- def std_proj_funcs(self, code, fname):
- """
- write a function to analyze the *import* part of a py file.
- Input: code for fname
- output: [standard functions]
- please note that the project_dependent libraries should have specific project names.
- """
- std_libs = []
- std_funcs = []
- tree = ast.parse(code)
- codelines = code.split('\n')
-
- for node in ast.walk(tree):
- if isinstance(node, ast.Import):
- # identify the import statement
- import_statement = codelines[node.lineno-1]
- for alias in node.names:
- import_name = alias.name.split('.')[0]
- if import_name in fname:
- continue
- else:
- # execute the import statement to find callable functions
- import_statement = import_statement.strip()
- try:
- exec(import_statement)
- except:
- continue
- std_libs.append(alias.name)
- eval_name = alias.name if alias.asname is None else alias.asname
- std_funcs.extend([name for name, member in inspect.getmembers(eval(eval_name)) if callable(member)])
-
- if isinstance(node, ast.ImportFrom):
- # execute the import statement
- import_statement = codelines[node.lineno-1]
- if node.module is None:
- continue
- module_name = node.module.split('.')[0]
- if module_name in fname:
- continue
- else:
- # handle imports with parentheses
- if "(" in import_statement:
- for ln in range(node.lineno-1, len(codelines)):
- if ")" in codelines[ln]:
- code_num = ln
- break
- import_statement = '\n'.join(codelines[node.lineno-1:code_num+1])
- import_statement = import_statement.strip()
- try:
- exec(import_statement)
- except:
- continue
- for alias in node.names:
- std_libs.append(alias.name)
- eval_name = alias.name if alias.asname is None else alias.asname
- if eval_name == "*":
- continue
- std_funcs.extend([name for name, member in inspect.getmembers(eval(eval_name)) if callable(member)])
- return std_funcs, std_libs
-
-
- def get_tags(self, fname, rel_fname):
- # Check if the file is in the cache and if the modification time has not changed
- file_mtime = self.get_mtime(fname)
- if file_mtime is None:
- return []
- # miss!
- data = list(self.get_tags_raw(fname, rel_fname))
- return data
-
- def get_tags_raw(self, fname, rel_fname):
- ref_fname_lst = rel_fname.split('/')
- s = deepcopy(self.structure)
- for fname_part in ref_fname_lst:
- s = s[fname_part]
- structure_classes = {item['name']: item for item in s['classes']}
- structure_functions = {item['name']: item for item in s['functions']}
- structure_class_methods = dict()
- for cls in s['classes']:
- for item in cls['methods']:
- structure_class_methods[item['name']] = item
- structure_all_funcs = {**structure_functions, **structure_class_methods}
-
- lang = filename_to_lang(fname)
- if not lang:
- return
- language = get_language(lang)
- parser = get_parser(lang)
-
- # Load the tags queries
- try:
- # scm_fname = resources.files(__package__).joinpath(
- # "/shared/data3/siruo2/SWE-agent/sweagent/environment/queries", f"tree-sitter-{lang}-tags.scm")
- scm_fname = """
- (class_definition
- name: (identifier) @name.definition.class) @definition.class
-
- (function_definition
- name: (identifier) @name.definition.function) @definition.function
-
- (call
- function: [
- (identifier) @name.reference.call
- (attribute
- attribute: (identifier) @name.reference.call)
- ]) @reference.call
- """
- except KeyError:
- return
- query_scm = scm_fname
- # if not query_scm.exists():
- # return
- # query_scm = query_scm.read_text()
-
- with open(str(fname), "r", encoding='utf-8') as f:
- code = f.read()
- with open(str(fname), "r", encoding='utf-8') as f:
- codelines = f.readlines()
-
- # hard-coded edge cases
- code = code.replace('\ufeff', '')
- code = code.replace('constants.False', '_False')
- code = code.replace('constants.True', '_True')
- code = code.replace("False", "_False")
- code = code.replace("True", "_True")
- code = code.replace("DOMAIN\\username", "DOMAIN\\\\username")
- code = code.replace("Error, ", "Error as ")
- code = code.replace('Exception, ', 'Exception as ')
- code = code.replace("print ", "yield ")
- pattern = r'except\s+\(([^,]+)\s+as\s+([^)]+)\):'
- # Replace 'as' with ','
- code = re.sub(pattern, r'except (\1, \2):', code)
- code = code.replace("raise AttributeError as aname", "raise AttributeError")
-
- # code = self.io.read_text(fname)
- if not code:
- return
- tree = parser.parse(bytes(code, "utf-8"))
- try:
- tree_ast = ast.parse(code)
- except:
- tree_ast = None
-
- # functions from third-party libs or default libs
- try:
- std_funcs, std_libs = self.std_proj_funcs(code, fname)
- except:
- std_funcs, std_libs = [], []
-
- # functions from builtins
- builtins_funs = [name for name in dir(builtins)]
- builtins_funs += dir(list)
- builtins_funs += dir(dict)
- builtins_funs += dir(set)
- builtins_funs += dir(str)
- builtins_funs += dir(tuple)
-
- # Run the tags queries
- query = language.query(query_scm)
- captures = query.captures(tree.root_node)
- captures = list(captures)
-
- saw = set()
- for node, tag in captures:
- if tag.startswith("name.definition."):
- kind = "def"
- elif tag.startswith("name.reference."):
- kind = "ref"
- else:
- continue
-
- saw.add(kind)
- cur_cdl = codelines[node.start_point[0]]
- category = 'class' if 'class ' in cur_cdl else 'function'
- tag_name = node.text.decode("utf-8")
-
- # we only want to consider project-dependent functions
- if tag_name in std_funcs:
- continue
- elif tag_name in std_libs:
- continue
- elif tag_name in builtins_funs:
- continue
-
- if category == 'class':
- # try:
- # class_functions = self.get_class_functions(tree_ast, tag_name)
- # except:
- # class_functions = "None"
- class_functions = [item['name'] for item in structure_classes[tag_name]['methods']]
- if kind == 'def':
- line_nums = [structure_classes[tag_name]['start_line'], structure_classes[tag_name]['end_line']]
- else:
- line_nums = [node.start_point[0], node.end_point[0]]
- result = Tag(
- rel_fname=rel_fname,
- fname=fname,
- name=tag_name,
- kind=kind,
- category=category,
- info='\n'.join(class_functions), # list unhashable, use string instead
- line=line_nums,
- )
-
- elif category == 'function':
-
- if kind == 'def':
- # func_block = self.get_func_block(cur_cdl, code)
- # cur_cdl =func_block
- cur_cdl = '\n'.join(structure_all_funcs[tag_name]['text'])
- line_nums = [structure_all_funcs[tag_name]['start_line'], structure_all_funcs[tag_name]['end_line']]
- else:
- line_nums = [node.start_point[0], node.end_point[0]]
-
- result = Tag(
- rel_fname=rel_fname,
- fname=fname,
- name=tag_name,
- kind=kind,
- category=category,
- info=cur_cdl,
- line=line_nums,
- )
-
- yield result
-
- if "ref" in saw:
- return
- if "def" not in saw:
- return
-
- # We saw defs, without any refs
- # Some tags files only provide defs (cpp, for example)
- # Use pygments to backfill refs
-
- try:
- lexer = guess_lexer_for_filename(fname, code)
- except ClassNotFound:
- return
-
- tokens = list(lexer.get_tokens(code))
- tokens = [token[1] for token in tokens if token[0] in Token.Name]
-
- for token in tokens:
- yield Tag(
- rel_fname=rel_fname,
- fname=fname,
- name=token,
- kind="ref",
- line=-1,
- category='function',
- info='none',
- )
-
- def get_ranked_tags(self, other_fnames, mentioned_fnames):
- # defines = defaultdict(set)
- # references = defaultdict(list)
- # definitions = defaultdict(set)
-
- tags_of_files = list()
-
- personalization = dict()
-
- fnames = set(other_fnames)
- # chat_rel_fnames = set()
-
- fnames = sorted(fnames)
-
- # Default personalization for unspecified files is 1/num_nodes
- # https://networkx.org/documentation/stable/_modules/networkx/algorithms/link_analysis/pagerank_alg.html#pagerank
- personalize = 10 / len(fnames)
-
- for fname in tqdm(fnames):
- if not Path(fname).is_file():
- if fname not in self.warned_files:
- if Path(fname).exists():
- self.io.tool_error(
- f"Code graph can't include {fname}, it is not a normal file"
- )
- else:
- self.io.tool_error(f"Code graph can't include {fname}, it no longer exists")
-
- self.warned_files.add(fname)
- continue
-
- # dump(fname)
- rel_fname = self.get_rel_fname(fname)
-
- # if fname in chat_fnames:
- # personalization[rel_fname] = personalize
- # chat_rel_fnames.add(rel_fname)
-
- if fname in mentioned_fnames:
- personalization[rel_fname] = personalize
-
- tags = list(self.get_tags(fname, rel_fname))
-
- tags_of_files.extend(tags)
-
- if tags is None:
- continue
-
- return tags_of_files
-
-
- def render_tree(self, abs_fname, rel_fname, lois):
- key = (rel_fname, tuple(sorted(lois)))
-
- if key in self.tree_cache:
- return self.tree_cache[key]
-
- # code = self.io.read_text(abs_fname) or ""
- with open(str(abs_fname), "r", encoding='utf-8') as f:
- code = f.read() or ""
-
- if not code.endswith("\n"):
- code += "\n"
-
- context = TreeContext(
- rel_fname,
- code,
- color=False,
- line_number=False,
- child_context=False,
- last_line=False,
- margin=0,
- mark_lois=False,
- loi_pad=0,
- # header_max=30,
- show_top_of_file_parent_scope=False,
- )
-
- context.add_lines_of_interest(lois)
- context.add_context()
- res = context.format()
- self.tree_cache[key] = res
- return res
-
- def to_tree(self, tags, chat_rel_fnames):
- if not tags:
- return ""
-
- tags = [tag for tag in tags if tag[0] not in chat_rel_fnames]
- tags = sorted(tags)
-
- cur_fname = None
- cur_abs_fname = None
- lois = None
- output = ""
-
- # add a bogus tag at the end so we trip the this_fname != cur_fname...
- dummy_tag = (None,)
- for tag in tags + [dummy_tag]:
- this_rel_fname = tag[0]
-
- # ... here ... to output the final real entry in the list
- if this_rel_fname != cur_fname:
- if lois is not None:
- output += "\n"
- output += cur_fname + ":\n"
- output += self.render_tree(cur_abs_fname, cur_fname, lois)
- lois = None
- elif cur_fname:
- output += "\n" + cur_fname + "\n"
- if type(tag) is Tag:
- lois = []
- cur_abs_fname = tag.fname
- cur_fname = this_rel_fname
-
- if lois is not None:
- lois.append(tag.line)
-
- # truncate long lines, in case we get minified js or something else crazy
- output = "\n".join([line[:100] for line in output.splitlines()]) + "\n"
-
- return output
-
-
- def find_src_files(self, directory):
- if not os.path.isdir(directory):
- return [directory]
-
- src_files = []
- for root, dirs, files in os.walk(directory):
- for file in files:
- src_files.append(os.path.join(root, file))
- return src_files
-
-
- def find_files(self, dir):
- chat_fnames = []
-
- for fname in dir:
- if Path(fname).is_dir():
- chat_fnames += self.find_src_files(fname)
- else:
- chat_fnames.append(fname)
-
- chat_fnames_new = []
- for item in chat_fnames:
- # filter out non-python files
- if not item.endswith('.py'):
- continue
- else:
- chat_fnames_new.append(item)
-
- return chat_fnames_new
-
-
-def get_random_color():
- hue = random.random()
- r, g, b = [int(x * 255) for x in colorsys.hsv_to_rgb(hue, 1, 0.75)]
- res = f"#{r:02x}{g:02x}{b:02x}"
- return res
-
-
-if __name__ == "__main__":
-
- # dir_name = sys.argv[1]
- dir_name = "/home/llm/Project/PythonProjects/GhostOS"
- code_graph = CodeGraph(root=dir_name)
- chat_fnames_new = code_graph.find_files([dir_name])
-
- tags, G = code_graph.get_code_graph(chat_fnames_new)
-
- print("---------------------------------")
- print(f"🏅 Successfully constructed the code graph for repo directory {dir_name}")
- print(f" Number of nodes: {len(G.nodes)}")
- print(f" Number of edges: {len(G.edges)}")
- print("---------------------------------")
-
- with open(f'{os.getcwd()}/graph.pkl', 'wb') as f:
- pickle.dump(G, f)
-
- for tag in tags:
- with open(f'{os.getcwd()}/tags.json', 'a+') as f:
- line = json.dumps({
- "fname": tag.fname,
- 'rel_fname': tag.rel_fname,
- 'line': tag.line,
- 'name': tag.name,
- 'kind': tag.kind,
- 'category': tag.category,
- 'info': tag.info,
- })
- f.write(line+'\n')
- print(f"🏅 Successfully cached code graph and node tags in directory ''{os.getcwd()}''")
diff --git a/evaluation/swe_bench_lite/tools/utils.py b/evaluation/swe_bench_lite/tools/utils.py
deleted file mode 100644
index 754b1a5d..00000000
--- a/evaluation/swe_bench_lite/tools/utils.py
+++ /dev/null
@@ -1,102 +0,0 @@
-import os
-import ast
-
-
-def create_structure(directory_path):
- """Create the structure of the repository directory by parsing Python files.
- :param directory_path: Path to the repository directory.
- :return: A dictionary representing the structure.
- """
- structure = {}
-
- for root, _, files in os.walk(directory_path):
- repo_name = os.path.basename(directory_path)
- relative_root = os.path.relpath(root, directory_path)
- if relative_root == ".":
- relative_root = repo_name
- curr_struct = structure
- for part in relative_root.split(os.sep):
- if part not in curr_struct:
- curr_struct[part] = {}
- curr_struct = curr_struct[part]
- for file_name in files:
- if file_name.endswith(".py"):
- file_path = os.path.join(root, file_name)
- class_info, function_names, file_lines = parse_python_file(file_path)
- curr_struct[file_name] = {
- "classes": class_info,
- "functions": function_names,
- "text": file_lines,
- }
- else:
- curr_struct[file_name] = {}
-
- return structure
-
-def parse_python_file(file_path, file_content=None):
- """Parse a Python file to extract class and function definitions with their line numbers.
- :param file_path: Path to the Python file.
- :return: Class names, function names, and file contents
- """
- if file_content is None:
- try:
- with open(file_path, "r") as file:
- file_content = file.read()
- parsed_data = ast.parse(file_content)
- except Exception as e: # Catch all types of exceptions
- print(f"Error in file {file_path}: {e}")
- return [], [], ""
- else:
- try:
- parsed_data = ast.parse(file_content)
- except Exception as e: # Catch all types of exceptions
- print(f"Error in file {file_path}: {e}")
- return [], [], ""
-
- class_info = []
- function_names = []
- class_methods = set()
-
- for node in ast.walk(parsed_data):
- if isinstance(node, ast.ClassDef):
- methods = []
- for n in node.body:
- if isinstance(n, ast.FunctionDef):
- methods.append(
- {
- "name": n.name,
- "start_line": n.lineno,
- "end_line": n.end_lineno,
- "text": file_content.splitlines()[
- n.lineno - 1 : n.end_lineno
- ],
- }
- )
- class_methods.add(n.name)
- class_info.append(
- {
- "name": node.name,
- "start_line": node.lineno,
- "end_line": node.end_lineno,
- "text": file_content.splitlines()[
- node.lineno - 1 : node.end_lineno
- ],
- "methods": methods,
- }
- )
- elif isinstance(node, ast.FunctionDef) and not isinstance(
- node, ast.AsyncFunctionDef
- ):
- if node.name not in class_methods:
- function_names.append(
- {
- "name": node.name,
- "start_line": node.lineno,
- "end_line": node.end_lineno,
- "text": file_content.splitlines()[
- node.lineno - 1 : node.end_lineno
- ],
- }
- )
-
- return class_info, function_names, file_content.splitlines()
\ No newline at end of file
diff --git a/ghostos/abc.py b/ghostos/abc.py
deleted file mode 100644
index ed2ac849..00000000
--- a/ghostos/abc.py
+++ /dev/null
@@ -1,67 +0,0 @@
-from abc import ABC, abstractmethod
-from pydantic import BaseModel, Field
-from ghostos.helpers import generate_import_path
-
-
-class Descriptive(ABC):
-
- @abstractmethod
- def get_description(self) -> str:
- pass
-
-
-class Identifier(BaseModel):
- id: str = Field(default="", description="Unique identifier")
- name: str = Field(default="", description="Name of the object")
- description: str = Field(default="", description="Description of the object")
-
-
-class Identifiable(ABC):
- """
- 描述一个可识别的对象.
- """
-
- @abstractmethod
- def identifier(self) -> Identifier:
- pass
-
-
-class IdentifiableClass(ABC):
-
- @classmethod
- @abstractmethod
- def class_identifier(cls) -> Identifier:
- pass
-
-
-def describe_class(cls: type) -> Identifier:
- """
- 一个默认的用来描述类的方法.
- :param cls: 目标类.
- :return: 返回一个 identifier.
- """
- if issubclass(cls, IdentifiableClass):
- return cls.class_identifier()
- id_ = generate_import_path(cls)
- name = cls.__name__
- desc = cls.__doc__
- return Identifier(id=id_, name=name, description=desc)
-
-
-class PromptAble(ABC):
- """
- 拥有 __prompt__ 方法的类.
- 这里只是一个示范, 并不需要真正继承这个类, 只需要有 __prompt__ 方法或属性.
- """
-
- @abstractmethod
- def __prompt__(self) -> str:
- pass
-
-
-class PromptAbleClass(ABC):
-
- @classmethod
- @abstractmethod
- def __class_prompt__(cls) -> str:
- pass
diff --git a/ghostos/abcd/__init__.py b/ghostos/abcd/__init__.py
new file mode 100644
index 00000000..48395b24
--- /dev/null
+++ b/ghostos/abcd/__init__.py
@@ -0,0 +1,14 @@
+from ghostos.abcd.concepts import (
+ GhostOS, Ghost, GhostDriver, Shell,
+ Operator, Action,
+ Session, Scope, StateValue, Messenger,
+ Background, Conversation,
+ Context,
+ Taskflow, Subtasks,
+)
+from ghostos.abcd.ghosts import Agent
+from ghostos.abcd.utils import (
+ get_ghost_driver_type, get_ghost_driver, is_ghost,
+ run_session_event, fire_session_event,
+)
+from ghostos.abcd.thoughts import Thought, LLMThought, ChainOfThoughts
diff --git a/ghostos/abcd/concepts.py b/ghostos/abcd/concepts.py
new file mode 100644
index 00000000..83a0d0ba
--- /dev/null
+++ b/ghostos/abcd/concepts.py
@@ -0,0 +1,847 @@
+from __future__ import annotations
+from typing import (
+ Type, Generic, Protocol, ClassVar, TypeVar, Callable,
+ Tuple, Optional, Iterable, List, Self, Union, Dict, Any
+)
+
+from abc import ABC, abstractmethod
+from ghostos.identifier import Identical
+from ghostos.entity import EntityType, EntityClass
+from ghostos.prompter import Prompter, DataPrompter, DataPrompterDriver
+from ghostos.core.runtime import (
+ TaskState,
+)
+from ghostos.core.runtime.events import Event
+from ghostos.core.runtime.tasks import GoTaskStruct, TaskBrief
+from ghostos.core.runtime.threads import GoThreadInfo
+from ghostos.core.llms import PromptPipe, Prompt, LLMFunc
+from ghostos.core.messages import MessageKind, Message, Stream, FunctionCaller, Payload, Receiver, Role
+from ghostos.contracts.logger import LoggerItf
+from ghostos.container import Container, Provider
+from ghostos.identifier import get_identifier
+from pydantic import BaseModel
+
+"""
+# Core Concepts of GhostOS framework.
+
+The word `Ghost` is picked from `Ghost In the Shell` movie.
+The Ghost can perform as both conversational object or an async function.
+Ghost is the abstract of atomic state machine unit in the GhostOS.
+
+for example, llm-based `Agent` is a state machine, an implementation of Ghost in GhostOS.
+
+Why Agent is a state machine?
+1. Agent receives an event at a time, not parallel, or face brain split.
+2. Agent keep it state in the system prompt and messages, by nature language.
+3. Agent take actions that matching expectation.
+So Agent is an AI-State-Machine, defined from prompt, not code; executed by Model, not Interpreter.
+
+About the Ghost Abstract:
+1. it is a class.
+2. the ghost class can construct ghost instance.
+3. any ghost instance can run as a conversational task
+4. a conversational task runs in turns, receiving event and replying messages.
+5. the conversational task is stateful, accept one event at a time.
+6. the conversational task reach the end when it is canceled, done or failed
+7. all the ghost has a Goal model to describe its current achievement.
+8. The Ghost Class shall be simple and clear to the AI models, when they are creating ghosts themselves.
+
+and the Most valuable features about ghost are:
+1. ghosts shall be fractal, can be called by other ghosts.
+2. ghost shall be defined by code, which can be generated by meta-agents.
+"""
+
+__all__ = (
+ "Ghost", "GhostDriver", "GhostOS", "Shell", "Conversation", "Background",
+ "Operator", "Action",
+ "Session", "Messenger", "StateValue", "Scope",
+ "Taskflow", "Subtasks",
+ "Context",
+)
+
+
+class Ghost(Identical, EntityClass, ABC):
+ """
+ the class defines the model of a kind of ghosts.
+ four parts included:
+ 1. configuration of the Ghost, which is Ghost.__init__. we can predefine many ghost instance for special scenes.
+ 2. context is always passed by the Caller of a ghost instance. each ghost class has it defined context model.
+ 3. goal is the static output (other than conversation messages) of a ghost instance.
+ 4. driver is
+ """
+
+ ArtifactType: ClassVar[Optional[Type]] = None
+ """ the model of the ghost's artifact, is completing during runtime"""
+
+ ContextType: ClassVar[Optional[Type[ContextType]]] = None
+ """ the model of the ghost's context, is completing during runtime'"""
+
+ DriverType: ClassVar[Optional[Type[GhostDriver]]] = None
+ """ separate ghost's methods to the driver class, make sure the ghost is simple and clear to other ghost"""
+
+
+G = TypeVar("G", bound=Ghost)
+
+
+class GhostDriver(Generic[G], ABC):
+ """
+ Ghost class is supposed to be a data class without complex methods definitions.
+ so it seems much clear when prompt to the LLM or user-level developer.
+ when LLM is creating a ghost class or instance, we expect it only see the code we want it to see,
+ without knowing the details codes of it, for safety / fewer tokens / more focus or other reasons.
+
+ so the methods of the ghost class defined in this class.
+ only core developers should know details about it.
+ """
+
+ def __init__(self, ghost: G) -> None:
+ self.ghost = ghost
+
+ def make_task_id(self, parent_scope: Scope) -> str:
+ """
+ generate unique instance id (task id) of the ghost instance.
+ """
+ from ghostos.helpers import md5
+ id_ = get_identifier(self.ghost)
+ if id_.id:
+ # if ghost instance has id, it is unique in process.
+ scope_ids = f"{parent_scope.process_id}-{id_.id}"
+ else:
+ # if ghost do not have id, it is unique to parent by name
+ scope_ids = f"{parent_scope.process_id}-{parent_scope.task_id}-{id_.name}"
+ # the task id point to a unique entity
+ return md5(scope_ids)
+
+ @abstractmethod
+ def get_artifact(self, session: Session) -> Optional[G.ArtifactType]:
+ """
+ generate the ghost goal from session_state
+ may be the Goal Model is a SessionStateValue that bind to it.
+
+ The AI behind a ghost is not supposed to operate the session object,
+ but work on the goal through functions or Moss Injections.
+ """
+ pass
+
+ @abstractmethod
+ def get_instructions(self, session: Session) -> str:
+ """
+ get system instructions of the ghost.
+ usually used in client side.
+ """
+ pass
+
+ @abstractmethod
+ def actions(self, session: Session) -> List[Action]:
+ """
+ return actions that react to the streaming output of llm
+ """
+ pass
+
+ @abstractmethod
+ def providers(self) -> Iterable[Provider]:
+ """
+ ghost return conversation level container providers.
+ the provider that is not singleton will bind to session also.
+ """
+ pass
+
+ @abstractmethod
+ def parse_event(
+ self,
+ session: Session,
+ event: Event,
+ ) -> Union[Event, None]:
+ """
+ intercept the ghost event
+ :returns: if None, the event will be ignored
+ """
+ pass
+
+ @abstractmethod
+ def on_creating(self, session: Session) -> None:
+ """
+ when the ghost task is created first time.
+ this method can initialize the thread, pycontext etc.
+ """
+ pass
+
+ @abstractmethod
+ def on_event(self, session: Session, event: Event) -> Union[Operator, None]:
+ """
+ all the state machine is only handling session event with the predefined operators.
+ """
+ pass
+
+ @abstractmethod
+ def truncate(self, session: Session) -> GoThreadInfo:
+ """
+ truncate the history messages in the thread
+ """
+ pass
+
+
+class Context(Payload, DataPrompter, ABC):
+ """
+ context prompter that generate prompt to provide information
+ the modeling defines strong-typed configuration to generate prompt.
+ """
+ key = "ghostos_context"
+
+ __driver__: Optional[Type[ContextDriver]] = None
+
+
+class ContextDriver(DataPrompterDriver, ABC):
+ """
+ the context driver is separated from context data.
+ LLM see
+ """
+ pass
+
+
+class Operator(ABC):
+ """
+ Operator to operating the GhostOS through the Session encapsulation.
+
+ The Operator is just like the primitives of any coding language.
+ for example, GhostOS have some operators work like python's `return`, `yield`, `await` .
+
+ I'm not capable to develop a real OS or a new coding language for AI,
+ GhostOS is built above python with the additional complexities.
+
+ Operators should be predefined, offer to user-level developer, or AI-models.
+ """
+
+ @abstractmethod
+ def run(self, session: Session) -> Union[Operator, None]:
+ """
+ :return: None means stop the loop, otherwise keep going.
+
+ operator returns an operator is a way to encapsulate repetitive codes.
+ """
+ pass
+
+ @abstractmethod
+ def destroy(self):
+ """
+ Python gc is not trust-worthy
+ Especially A keep B, B keep C, C keep A, father and child keep each other.
+ I prefer to del the object attributes in the end of the object lifecycle.
+ """
+ pass
+
+
+class Action(PromptPipe, ABC):
+ @abstractmethod
+ def name(self) -> str:
+ pass
+
+ @abstractmethod
+ def as_function(self) -> Optional[LLMFunc]:
+ pass
+
+ @abstractmethod
+ def update_prompt(self, prompt: Prompt) -> Prompt:
+ pass
+
+ @abstractmethod
+ def run(self, session: Session, caller: FunctionCaller) -> Union[Operator, None]:
+ pass
+
+
+class GhostOS(Protocol):
+
+ @abstractmethod
+ def container(self) -> Container:
+ """
+ root container for GhostOS
+ """
+ pass
+
+ @abstractmethod
+ def create_shell(
+ self,
+ name: str,
+ *,
+ shell_id: str = "",
+ providers: Optional[List[Provider]] = None,
+ process_id: Optional[str] = None,
+ ) -> Shell:
+ pass
+
+
+class Background(ABC):
+
+ @abstractmethod
+ def on_error(self, error: Exception) -> bool:
+ pass
+
+ @abstractmethod
+ def on_event(self, event: Event, messages: List[Message]) -> None:
+ pass
+
+ @abstractmethod
+ def alive(self) -> bool:
+ pass
+
+ @abstractmethod
+ def halt(self) -> int:
+ pass
+
+
+class Shell(ABC):
+
+ @abstractmethod
+ def container(self) -> Container:
+ """
+ root container for GhostOS
+ """
+ pass
+
+ @abstractmethod
+ def send_event(self, event: Event) -> None:
+ """
+ send an event into the loop.
+ the event always has a task_id, so the task shall be created first.
+ """
+ pass
+
+ @abstractmethod
+ def sync(
+ self,
+ ghost: G,
+ context: Optional[G.ContextType] = None,
+ username: str = "",
+ user_role: str = Role.USER.value,
+ force: bool = False,
+ ) -> Conversation[G]:
+ """
+ create a top-level conversation with a ghost.
+ top-level means task depth is 0.
+ So it never locked until the conversation is created.
+ if force is True, the conversation will seize the task locker anyway.
+ """
+ pass
+
+ @abstractmethod
+ def call(
+ self,
+ ghost: G,
+ context: Optional[G.ContextType] = None,
+ instructions: Optional[Iterable[Message]] = None,
+ timeout: float = 0.0,
+ stream: Optional[Stream] = None,
+ ) -> Tuple[Union[G.ArtifactType, None], TaskState]:
+ """
+ run a ghost task until it stopped,
+ """
+ pass
+
+ @abstractmethod
+ def run_background_event(self, background: Optional[Background] = None) -> Union[Event, None]:
+ """
+ run the event loop for the ghosts in the Shell.
+ 1. pop task notification.
+ 2. try to converse the task
+ 3. if failed, pop another task notification.
+ 4. if success, pop task event and handle it until no event found.
+ 5. send a task notification after handling, make sure someone check the task events are empty.
+ only the tasks that depth > 0 have notifications.
+ background run itself is blocking method, run it in a separate thread for parallel execution.
+ :return: the handled event
+ """
+ pass
+
+ @abstractmethod
+ def background_run(self, worker: int = 4, background: Optional[Background] = None) -> None:
+ pass
+
+ @abstractmethod
+ def submit(self, caller: Callable, *args, **kwargs):
+ pass
+
+ @abstractmethod
+ def close(self):
+ pass
+
+ @abstractmethod
+ def closed(self) -> bool:
+ pass
+
+
+class Conversation(Protocol[G]):
+ """
+ interface for operate on synchronized (task is locked) ghost
+ """
+ task_id: str
+
+ scope: Scope
+
+ logger: LoggerItf
+
+ @abstractmethod
+ def container(self) -> Container:
+ """
+ root container for GhostOS
+ """
+ pass
+
+ @abstractmethod
+ def get_task(self) -> GoTaskStruct:
+ pass
+
+ @abstractmethod
+ def get_thread(self, truncated: bool = False) -> GoThreadInfo:
+ pass
+
+ @abstractmethod
+ def update_thread(self, thread: GoThreadInfo) -> None:
+ pass
+
+ @abstractmethod
+ def get_ghost(self) -> G:
+ pass
+
+ def get_ghost_driver(self) -> GhostDriver[G]:
+ from ghostos.abcd.utils import get_ghost_driver
+ ghost = self.get_ghost()
+ return get_ghost_driver(ghost)
+
+ @abstractmethod
+ def get_context(self) -> G.ContextType:
+ pass
+
+ @abstractmethod
+ def get_instructions(self) -> str:
+ pass
+
+ @abstractmethod
+ def get_functions(self) -> List[LLMFunc]:
+ pass
+
+ @abstractmethod
+ def get_artifact(self) -> Tuple[Union[G.ArtifactType, None], TaskState]:
+ pass
+
+ @abstractmethod
+ def refresh(self) -> bool:
+ pass
+
+ @abstractmethod
+ def talk(self, query: str, user_name: str = "", context: Optional[G.ContextType] = None) -> Tuple[Event, Receiver]:
+ pass
+
+ @abstractmethod
+ def update_context(self, context: Context) -> None:
+ pass
+
+ @abstractmethod
+ def respond(
+ self,
+ inputs: Iterable[MessageKind],
+ context: Optional[G.ContextType] = None,
+ streaming: bool = True,
+ ) -> Tuple[Event, Receiver]:
+ """
+ create response immediately by inputs. the inputs will change to event.
+ """
+ pass
+
+ @abstractmethod
+ def respond_event(self, event: Event, streaming: bool = True) -> Receiver:
+ """
+ create response to the event immediately
+ """
+ pass
+
+ @abstractmethod
+ def pop_event(self) -> Optional[Event]:
+ """
+ pop event of the current task
+ """
+ pass
+
+ @abstractmethod
+ def send_event(self, event: Event) -> None:
+ pass
+
+ @abstractmethod
+ def fail(self, error: Exception) -> bool:
+ """
+ exception occur
+ :return: catch the exception or not
+ """
+ pass
+
+ @abstractmethod
+ def close(self):
+ """
+ close the conversation
+ """
+ pass
+
+ @abstractmethod
+ def available(self) -> bool:
+ """
+ beside closed, conversation may be executing, that means it is not available.
+ """
+ pass
+
+ @abstractmethod
+ def is_closed(self) -> bool:
+ """
+ closed
+ """
+ pass
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ if self.is_closed():
+ return
+ if exc_val is not None:
+ return self.fail(exc_val)
+ else:
+ self.close()
+ return None
+
+
+class Messenger(Stream, ABC):
+ """
+ Messenger is a bridge of message streams
+ Messenger finish when the flush method is called.
+ Each messenger can nest sub messengers, when sub messenger is finished,
+ the parent messenger is not finished until the flush is called.
+
+ why this is an abstract base class?
+ there may be more abilities during streaming are needed,
+ this project can only provide a basic one.
+ """
+
+ @abstractmethod
+ def flush(self) -> Tuple[List[Message], List[FunctionCaller]]:
+ """
+ flush the buffed messages, finish the streaming of this messenger.
+ the message buffer shall join all the chunks to message item.
+ after the messenger is flushed, it can not send any new message.
+ """
+ pass
+
+
+class StateValue(ABC):
+ """
+ session state value
+ """
+
+ @abstractmethod
+ def get(self, session: Session) -> Optional[Self]:
+ pass
+
+ @abstractmethod
+ def bind(self, session: Session) -> None:
+ pass
+
+ def get_or_bind(self, session: Session) -> Self:
+ value = self.get(session)
+ if value is None:
+ value = self
+ self.bind(session)
+ return value
+
+
+class Scope(BaseModel):
+ """
+ scope of the session.
+ """
+ shell_id: str
+ process_id: str
+ task_id: str
+ parent_task_id: Optional[str] = None
+
+
+class Session(Generic[G], ABC):
+ """
+ Session 管理了一个有状态的会话. 所谓 "有状态的会话", 通常指的是:
+ shell + ghost + 多轮对话/多轮思考 运行中的状态.
+
+ Session 则提供了 Ghost 的 Task 运行时状态统一管理的 API.
+ 通常每个运行中的 Task 都会创建一个独立的 Session.
+ Session 在运行周期里不会立刻调用底层 IO 存储消息, 而是要等一个周期正常结束.
+ 这是为了减少运行时错误对状态机造成的副作用.
+ """
+ instance_count: ClassVar[int] = 0
+
+ upstream: Stream
+
+ scope: Scope
+ """the running scope of the session"""
+
+ state: Dict[str, EntityType]
+ """session state that keep session state values"""
+
+ container: Container
+ """Session level container"""
+
+ ghost: G
+
+ task: GoTaskStruct
+ """current task"""
+
+ thread: GoThreadInfo
+ """thread info of the task"""
+
+ logger: LoggerItf
+
+ @abstractmethod
+ def alive(self) -> bool:
+ """
+ Session 对自身任务进行状态检查.
+ 如果这个任务被取消或终止, 则返回 false.
+ 基本判断逻辑:
+ 1. 消息上游流没有终止.
+ 2. task 持有了锁.
+ 3. 设置的超时时间没有过.
+ """
+ pass
+
+ @abstractmethod
+ def get_truncated_thread(self) -> GoThreadInfo:
+ pass
+
+ @abstractmethod
+ def to_messages(self, values: Iterable[Union[MessageKind, Any]]) -> List[Message]:
+ pass
+
+ @abstractmethod
+ def parse_event(self, event: Event) -> Tuple[Optional[Event], Optional[Operator]]:
+ pass
+
+ @abstractmethod
+ def system_log(self, log: str) -> None:
+ """
+ log system info, save to thread as a system message
+ :param log: log info
+ """
+ pass
+
+ @abstractmethod
+ def get_context(self) -> Optional[Prompter]:
+ """
+ current context for the ghost
+ """
+ pass
+
+ @abstractmethod
+ def get_artifact(self) -> G.ArtifactType:
+ """
+ :return: the current state of the ghost goal
+ """
+ pass
+
+ @abstractmethod
+ def get_instructions(self) -> str:
+ pass
+
+ @abstractmethod
+ def refresh(self, throw: bool = False) -> bool:
+ """
+ refresh the session, update overdue time and task lock.
+ """
+ pass
+
+ @abstractmethod
+ def save(self):
+ """
+ save status.
+ """
+ pass
+
+ @abstractmethod
+ def taskflow(self) -> Taskflow:
+ """
+ basic library to operates the current task
+ """
+ pass
+
+ @abstractmethod
+ def subtasks(self) -> Subtasks:
+ pass
+
+ @abstractmethod
+ def messenger(self, stage: str = "") -> "Messenger":
+ """
+ Task 当前运行状态下, 向上游发送消息的 Messenger.
+ 每次会实例化一个 Messenger, 理论上不允许并行发送消息. 但也可能做一个技术方案去支持它.
+ Messenger 未来要支持双工协议, 如果涉及多流语音还是很复杂的.
+ """
+ pass
+
+ @abstractmethod
+ def respond(
+ self,
+ messages: Iterable[MessageKind],
+ stage: str = "",
+ ) -> Tuple[List[Message], List[FunctionCaller]]:
+ """
+ 发送消息, 但不影响运行状态.
+ """
+ pass
+
+ @abstractmethod
+ def create_threads(
+ self,
+ *threads: GoThreadInfo,
+ ) -> None:
+ pass
+
+ def create_tasks(self, *tasks: GoTaskStruct) -> None:
+ pass
+
+ @abstractmethod
+ def call(self, ghost: G, ctx: G.ContextType) -> G.ArtifactType:
+ """
+ 创建一个子任务, 阻塞并等待它完成.
+ :param ghost:
+ :param ctx:
+ :return: the Goal of the task. if the final state is not finish, throw an exception.
+ """
+ pass
+
+ # --- 更底层的 API. --- #
+
+ @abstractmethod
+ def fire_events(self, *events: "Event") -> None:
+ """
+ 发送多个事件. 这个环节需要给 event 标记 callback.
+ 在 session.done() 时才会真正执行.
+ """
+ pass
+
+ @abstractmethod
+ def get_task_briefs(self, *task_ids: str) -> Dict[str, TaskBrief]:
+ """
+ 获取多个任务的简介.
+ :param task_ids: 可以指定要获取的 task id
+ """
+ pass
+
+ @abstractmethod
+ def __enter__(self):
+ pass
+
+ @abstractmethod
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ pass
+
+
+class Taskflow(Prompter, ABC):
+ """
+ default operations
+ """
+ MessageKind = Union[str, Message, Any]
+ """message kind shall be string or serializable object"""
+
+ # --- 基本操作 --- #
+ @abstractmethod
+ def finish(self, status: str = "", *replies: MessageKind) -> Operator:
+ """
+ finish self task
+ :param status: describe status of the task
+ :param replies: replies to parent task or user
+ """
+ pass
+
+ @abstractmethod
+ def fail(self, reason: str = "", *replies: MessageKind) -> Operator:
+ """
+ self task failed.
+ :param reason: describe status of the task
+ :param replies: replies to parent task or user
+ """
+ pass
+
+ @abstractmethod
+ def wait(self, status: str = "", *replies: MessageKind) -> Operator:
+ """
+ wait for the parent task or user to provide more information or further instruction.
+ :param status: describe current status
+ :param replies: question, inform or
+ """
+ pass
+
+ @abstractmethod
+ def think(self, *messages: MessageKind, instruction: str = "", sync: bool = False) -> Operator:
+ """
+ start next round thinking on messages
+ :param messages: observe target
+ :param instruction: instruction when receive the observation.
+ :param sync: if True, observe immediately, otherwise check other event first
+ :return:
+ """
+ pass
+
+ @abstractmethod
+ def observe(self, **kwargs) -> Operator:
+ """
+ observe values
+ :param kwargs:
+ :return:
+ """
+
+ @abstractmethod
+ def error(self, *messages: MessageKind) -> Operator:
+ pass
+
+
+class Subtasks(Prompter, ABC):
+ """
+ library that can handle async subtasks by other ghost instance.
+ """
+ MessageKind = Union[str, Message, Any]
+ """message kind shall be string or serializable object"""
+
+ @abstractmethod
+ def cancel(self, name: str, reason: str = "") -> None:
+ """
+ cancel an exists subtask
+ :param name: name of the task
+ :param reason: the reason to cancel it
+ :return:
+ """
+ pass
+
+ @abstractmethod
+ def send(
+ self,
+ name: str,
+ *messages: MessageKind,
+ ctx: Optional[Ghost.ContextType] = None,
+ ) -> None:
+ """
+ send message to an existing subtask
+ :param name: name of the subtask
+ :param messages: the messages to the subtask
+ :param ctx: if given, update the ghost context of the task
+ :return:
+ """
+ pass
+
+ @abstractmethod
+ def create(
+ self,
+ ghost: Ghost,
+ instruction: str = "",
+ ctx: Optional[Ghost.ContextType] = None,
+ task_name: Optional[str] = None,
+ task_description: Optional[str] = None,
+ ) -> None:
+ """
+ create subtask from a ghost instance
+ :param ghost: the ghost instance that handle the task
+ :param instruction: instruction to the ghost
+ :param ctx: the context that the ghost instance needed
+ :param task_name: if not given, use the ghost's name as the task name
+ :param task_description: if not given, use the ghost's description as the task description
+ """
+ pass
diff --git a/ghostos/abcd/ghosts.py b/ghostos/abcd/ghosts.py
new file mode 100644
index 00000000..95cd2138
--- /dev/null
+++ b/ghostos/abcd/ghosts.py
@@ -0,0 +1,67 @@
+from abc import ABC, abstractmethod
+from typing import ClassVar
+from ghostos.identifier import Identifier
+from pydantic import BaseModel
+from ghostos.abcd.concepts import Ghost
+
+"""
+Some ghost prototypes.
+"""
+
+
+class Agent(Ghost, ABC):
+ """
+ Agent is the base abstract of LLM-based conversational AI entity.
+
+ The Model of the Agent defines its behavior, normally includes:
+ - persona and instruction
+ - configurations to create a context (cot/examples/knowledge/memory) for llm
+ - llm configurations
+ - tools
+ - system configurations, like thread truncating / authorities / welcome craft etc.
+ """
+
+ @abstractmethod
+ def __identifier__(self) -> Identifier:
+ pass
+
+
+class UserProxy(Ghost, ABC):
+ """
+ LLM-based UserProxy can understand human language and translate the user intends to system actions.
+ It does not own any charactor or persona, is merely a Nature Language Interface of the system.
+ Speed and Accuracy are the most important features.
+ """
+ pass
+
+
+class Thought(BaseModel, Ghost, ABC):
+ """
+ Thought is a micro unit to processing thinking with current context;
+ the Goal of the Thought is to produce a decision or suggestion, add them to the context.
+ """
+ ArtifactType: ClassVar = str
+
+ @abstractmethod
+ def __identifier__(self) -> Identifier:
+ pass
+
+
+class AIFunc(BaseModel, Ghost, ABC):
+ """
+ Act like a function but driven by AI models.
+ AI models dynamic check the function call, and generate code in realtime.
+ """
+
+ @abstractmethod
+ def __identifier__(self) -> Identifier:
+ pass
+
+
+class Workflow(Ghost, ABC):
+ """
+ workflow is a programmed Finite State Machine that does a certain job and return a certain result.
+ The Goal of workflow is the result.
+ Workflow itself is a FSM, but some node of it can be other ghost entity like AIFunc, Thought or Agent.
+ """
+ pass
diff --git a/ghostos/abcd/memory.py b/ghostos/abcd/memory.py
new file mode 100644
index 00000000..356eda49
--- /dev/null
+++ b/ghostos/abcd/memory.py
@@ -0,0 +1,61 @@
+from abc import ABC, abstractmethod
+from typing import Type, Iterable
+from ghostos.entity import EntityClass
+from ghostos.identifier import Identifier, Identical
+from ghostos.helpers import generate_import_path
+from ghostos.abcd.concepts import Session
+
+
+class Memory(EntityClass, Identical, ABC):
+ """
+ memory element
+ """
+
+ @abstractmethod
+ def make_id(self, session: Session) -> str:
+ """
+ memory instance is unique to session, usually create unique id from session scope
+ :return:
+ """
+ pass
+
+ @classmethod
+ def memory_kind(cls) -> str:
+ return generate_import_path(cls)
+
+
+class Memories(ABC):
+
+ @abstractmethod
+ def match(self, memory_kind: str) -> bool:
+ pass
+
+ @abstractmethod
+ def remember(self, memory: Memory) -> str:
+ """
+ remember a memory by session scope
+ :param memory: memory object
+ :return: memory id
+ """
+ pass
+
+ @abstractmethod
+ def find(self, memory_id: str, expect: Type[Memory]) -> Memory:
+ """
+ find memory by id
+ :param memory_id:
+ :param expect:
+ :return:
+ """
+ pass
+
+ @abstractmethod
+ def recall(self, query: str, top_k: int = 10) -> Iterable[Identifier]:
+ pass
+
+
+class MemoryRepository(Memories, ABC):
+
+ @abstractmethod
+ def register(self, driver: Memories) -> None:
+ pass
diff --git a/ghostos/abcd/realtime.py b/ghostos/abcd/realtime.py
new file mode 100644
index 00000000..a697b832
--- /dev/null
+++ b/ghostos/abcd/realtime.py
@@ -0,0 +1,301 @@
+from __future__ import annotations
+from abc import ABC, abstractmethod
+from typing import (
+ Generic,
+ List, Iterable, Tuple, TypeVar, Optional, Dict, Callable, Union, Self
+)
+from ghostos.abcd import Conversation
+from ghostos.core.messages import Message, ReceiverBuffer
+from ghostos.entity import ModelEntityMeta, to_entity_model_meta, from_entity_model_meta
+from pydantic import BaseModel, Field
+from enum import Enum
+
+
+class Realtime(ABC):
+ """
+ realtime wrapper
+ """
+
+ @abstractmethod
+ def create(
+ self,
+ conversation: Conversation,
+ listener: Listener,
+ speaker: Speaker,
+ app_name: str = "",
+ config: Optional[RealtimeAppConfig] = None,
+ ) -> RealtimeApp:
+ """
+ create an Realtime App instance
+ :param conversation:
+ :param listener:
+ :param speaker:
+ :param app_name:
+ :param config:
+ :return:
+ """
+ pass
+
+ @abstractmethod
+ def get_config(self) -> RealtimeConfig:
+ pass
+
+ @abstractmethod
+ def register(self, driver: RealtimeDriver):
+ pass
+
+
+class RealtimeAppConfig(BaseModel):
+
+ @abstractmethod
+ def driver_name(self) -> str:
+ pass
+
+
+class RealtimeConfig(BaseModel):
+ default: str = Field(description="default app")
+ apps: Dict[str, ModelEntityMeta] = Field(
+ default_factory=dict,
+ )
+
+ def add_app_conf(self, name: str, app_conf: RealtimeAppConfig):
+ self.apps[name] = to_entity_model_meta(app_conf)
+
+ def get_app_conf(self, name: str) -> Optional[RealtimeAppConfig]:
+ data = self.apps.get(name, None)
+ if data is None:
+ return None
+ conf = from_entity_model_meta(data)
+ if not isinstance(conf, RealtimeAppConfig):
+ raise TypeError(f"App config {name} is not a RealtimeAppConfig")
+ return conf
+
+
+C = TypeVar("C", bound=RealtimeAppConfig)
+
+
+class RealtimeDriver(Generic[C], ABC):
+
+ @abstractmethod
+ def driver_name(self) -> str:
+ pass
+
+ @abstractmethod
+ def create(
+ self,
+ config: C,
+ conversation: Conversation,
+ listener: Optional[Listener] = None,
+ speaker: Optional[Speaker] = None,
+ vad_mode: bool = False,
+ ) -> RealtimeApp:
+ pass
+
+
+class Listener(ABC):
+ """
+ read audio bytes
+ """
+
+ @abstractmethod
+ def listen(self, sender: Callable[[bytes], None]) -> Listening:
+ """
+ read audio bytes in seconds.
+ :param sender: sender hearing data
+ :return:
+ """
+ pass
+
+
+class Listening(ABC):
+ @abstractmethod
+ def __enter__(self) -> Self:
+ pass
+
+ @abstractmethod
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ pass
+
+
+class Speaker(ABC):
+
+ @abstractmethod
+ def speak(self, queue: Callable[[], Union[bytes, None]]) -> Speaking:
+ pass
+
+
+class Speaking(ABC):
+ @abstractmethod
+ def __enter__(self) -> Self:
+ """
+ start to speak
+ :return:
+ """
+ pass
+
+ @abstractmethod
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ pass
+
+ def done(self) -> bool:
+ pass
+
+ @abstractmethod
+ def wait(self):
+ pass
+
+
+class Operator(BaseModel):
+ name: str = Field(description="name of the operator")
+ description: str = Field(description="description of the operator")
+
+
+class OperatorName(str, Enum):
+ listen = "listen"
+ """start listening"""
+
+ stop_listen = "stop listen"
+ """stop listening, commit the audio buffer, but not create response"""
+
+ respond = "respond"
+ """create response"""
+
+ clear_audio = "clear"
+ """clear audio buffer """
+
+ cancel_responding = "cancel"
+ """cancel responding"""
+
+ def new(self, description: str) -> Operator:
+ return Operator(name=self.value, description=description)
+
+
+class RealtimeApp(ABC):
+ """
+ realtime agent in multi-threading programming pattern.
+ it will develop several threads during runtime to exchange parallel events and actions.
+
+ furthermore this agent programming model allow parallel `body parts` (shells). for examples:
+ - an OS-based listen-speak channel
+ - a website that shows the conversation items, and allow to input text message.
+ - an async channel to call function like multi-agent based-on different models
+ - a real material body that controlled by a running robot OS
+
+ all the cases are running concurrently in a single agent.
+ otherwise the realtime api is no more than a block mode chat agent, why the complexities?
+
+ although the shells are parallel running, event sending and receiving are concurrent,
+ but the agent itself is still stateful, or facing brain split failures.
+
+ the operations to the agent may be:
+ 1. act immediately
+ 2. illegal for current state
+ 3. allowed but pending
+ 4. allowed but blocking, need retry later
+ 5. etc...
+
+ the safest way to develop the agent FSM is frame-based model. broadly used in the realtime games.
+ the agent tick a frame to do an operation or recv an event, if None then idle a while before next tick.
+ operations are prior to events.
+
+ in the other hand the shell (bodies) has its own state machine in the same way,
+ but could use success-or-failure pattern, much simpler.
+
+ although the OpenAI realtime session itself is stateful, but it does not keep a long-time session,
+ so the client side agent implementation need to do all the state work itself,
+ make sure the session is aligned to server session in the duplex way.
+ that why we need a full-functional state machine in our agent's implementation.
+ so the client side agent is the real central state machine in the long-term,
+ but yield its privilege to server side session during runtime.
+ """
+
+ conversation: Conversation
+
+ @property
+ @abstractmethod
+ def vad_mode(self) -> bool:
+ pass
+
+ @property
+ @abstractmethod
+ def listen_mode(self) -> bool:
+ pass
+
+ @abstractmethod
+ def start(self, vad_mode: bool = True, listen_mode: bool = True) -> Operator:
+ """
+ start realtime session
+ """
+ pass
+
+ @abstractmethod
+ def close(self):
+ """
+ close realtime session and release resources
+ """
+ pass
+
+ @abstractmethod
+ def is_closed(self) -> bool:
+ pass
+
+ @abstractmethod
+ def history_messages(self) -> Iterable[Message]:
+ """
+ return history messages.
+ """
+ pass
+
+ @abstractmethod
+ def set_mode(self, *, vad_mode: Optional[bool] = None, listen_mode: Optional[bool] = None):
+ pass
+
+ @abstractmethod
+ def state(self) -> Tuple[str, List[Operator]]:
+ """
+ :return: (operators, operators)
+ """
+ pass
+
+ @abstractmethod
+ def operate(self, operator: Operator) -> bool:
+ """
+ run operator.
+ """
+ pass
+
+ @abstractmethod
+ def fail(self, error: Exception) -> bool:
+ """
+ receive an exception
+ :return: if the error is able to intercept
+ """
+ pass
+
+ @abstractmethod
+ def add_message(self, message: Message, previous_message_id: Optional[str] = None):
+ """
+ add message to the conversation.
+ :param message:
+ :param previous_message_id:
+ :return:
+ """
+ pass
+
+ @abstractmethod
+ def output(self) -> Optional[ReceiverBuffer]:
+ """
+ output messages. if None, check status again.
+ """
+ pass
+
+ def __enter__(self):
+ self.start()
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ intercepted = None
+ if exc_val is not None:
+ intercepted = self.fail(exc_val)
+ self.close()
+ return intercepted
diff --git a/ghostos/abcd/thoughts.py b/ghostos/abcd/thoughts.py
new file mode 100644
index 00000000..ab7a035c
--- /dev/null
+++ b/ghostos/abcd/thoughts.py
@@ -0,0 +1,119 @@
+from typing import Optional, Generic, TypeVar, Tuple, List, Iterable
+from abc import ABC, abstractmethod
+from ghostos.abcd.concepts import Session, Operator, Action
+from ghostos.core.llms import Prompt, ModelConf, ServiceConf, LLMs, LLMApi
+from pydantic import BaseModel, Field
+
+__all__ = ['Thought', 'LLMThought', 'SummaryThought', 'ChainOfThoughts']
+
+T = TypeVar("T")
+
+
+class Thought(Generic[T], ABC):
+ """
+ LLM wrapper
+ """
+
+ @abstractmethod
+ def think(self, session: Session, prompt: Prompt) -> Tuple[Prompt, Optional[T]]:
+ pass
+
+
+class ChainOfThoughts(Thought[Operator]):
+ def __init__(
+ self,
+ final: Thought[Operator],
+ nodes: List[Thought[Operator]],
+ ):
+ self.nodes = nodes
+ self.final = final
+
+ def think(self, session: Session, prompt: Prompt) -> Tuple[Prompt, Optional[T]]:
+ for node in self.nodes:
+ prompt, op = node.think(session, prompt)
+ if op is not None:
+ return prompt, op
+
+ return self.final.think(session, prompt)
+
+
+class LLMThought(Thought[Operator]):
+ """
+ basic llm thought
+ """
+
+ def __init__(
+ self,
+ llm_api: str = "",
+ actions: Optional[Iterable[Action]] = None,
+ message_stage: str = "",
+ model: Optional[ModelConf] = None,
+ service: Optional[ServiceConf] = None,
+ ):
+ """
+
+ :param llm_api: The LLM API to use, see LLMsConfig
+ :param message_stage:
+ :param actions:
+ :param model: the llm model to use, if given, overrides llm_api
+ :param service: the llm service to use, if given, override ModelConf service field
+ """
+ self.llm_api = llm_api
+ self.message_stage = message_stage
+ self.model = model
+ self.service = service
+ self.actions = {}
+ if actions:
+ self.actions = {action.name(): action for action in actions}
+
+ def think(self, session: Session, prompt: Prompt) -> Tuple[Prompt, Optional[Operator]]:
+ for action in self.actions.values():
+ prompt = action.update_prompt(prompt)
+ llm_api = self.get_llm_api(session)
+
+ streaming = not session.upstream.completes_only()
+ session.logger.debug("start llm thinking on prompt %s", prompt.id)
+ items = llm_api.deliver_chat_completion(prompt, streaming)
+ messages, callers = session.respond(items, self.message_stage)
+ prompt.added.extend(messages)
+ session.logger.debug("llm thinking on prompt %s is done", prompt.id)
+
+ for caller in callers:
+ if caller.name in self.actions:
+ action = self.actions[caller.name]
+ op = action.run(session, caller)
+ if op is not None:
+ return prompt, op
+ return prompt, None
+
+ def get_llm_api(self, session: Session) -> LLMApi:
+ llms = session.container.force_fetch(LLMs)
+ if self.model:
+ llm_api = llms.new_api(self.service, self.model)
+ else:
+ llm_api = llms.get_api(self.llm_api)
+ return llm_api
+
+
+class SummaryThought(BaseModel, Thought[str]):
+ """
+ simple summary thought
+ """
+
+ llm_api: str = Field("", description="the llm api to use")
+ instruction: str = Field(
+ "the chat history is too long. "
+ "You MUST summarizing the history message in 500 words, keep the most important information."
+ "Your Summary:",
+ description="the llm instruction to use",
+ )
+
+ def think(self, session: Session, prompt: Prompt) -> Tuple[Prompt, Optional[str]]:
+ from ghostos.core.messages import Role
+ forked = prompt.fork(None)
+ instruction = Role.SYSTEM.new(content=self.instruction)
+ forked.added.append(instruction)
+ llms = session.container.force_fetch(LLMs)
+ llm_api = llms.get_api(self.llm_api)
+ message = llm_api.chat_completion(forked)
+ return prompt, message.content
diff --git a/ghostos/abcd/utils.py b/ghostos/abcd/utils.py
new file mode 100644
index 00000000..35419b4b
--- /dev/null
+++ b/ghostos/abcd/utils.py
@@ -0,0 +1,107 @@
+from typing import Optional, Type, Union, List
+from types import ModuleType
+from ghostos.helpers import import_class_from_path
+from ghostos.identifier import get_identifier
+from ghostos.entity import to_entity_meta
+from ghostos.abcd.concepts import Ghost, GhostDriver, Session, Operator
+from ghostos.core.runtime import Event
+from ghostos.container import Provider
+
+__all__ = [
+ 'get_ghost_driver', 'get_ghost_driver_type', 'is_ghost',
+ 'run_session_event', 'fire_session_event',
+ 'get_module_magic_ghost', 'get_module_magic_shell_providers',
+]
+
+
+def get_ghost_driver_type(ghost: Ghost) -> Type[GhostDriver]:
+ """
+ get ghost driver instance by default protocol
+ """
+ if ghost.DriverType is not None:
+ return ghost.DriverType
+ name = ghost.__class__.__name__
+ module_name = ghost.__class__.__module__
+ import_path = f"{module_name}:{name}Driver"
+ cls = import_class_from_path(import_path, GhostDriver)
+ return cls
+
+
+def get_ghost_driver(ghost: Ghost) -> GhostDriver:
+ ghost_driver_type = get_ghost_driver_type(ghost)
+ return ghost_driver_type(ghost)
+
+
+def is_ghost(value) -> bool:
+ try:
+ if not isinstance(value, Ghost):
+ return False
+ id_ = get_identifier(value)
+ assert id_ is not None
+ meta = to_entity_meta(value)
+ assert meta is not None
+ driver = get_ghost_driver_type(value)
+ assert issubclass(driver, GhostDriver)
+ return True
+ except AssertionError:
+ return False
+
+
+def fire_session_event(session: Session, event: Event) -> Optional[Operator]:
+ event, op = session.parse_event(event)
+ if op is not None:
+ session.logger.info("session event is intercepted and op %s is returned", op)
+ return op
+ if event is None:
+ # if event is intercepted, stop the run.
+ return None
+ driver = get_ghost_driver(session.ghost)
+ session.thread = session.get_truncated_thread()
+ op = driver.on_event(session, event)
+ # only session and driver can change event.
+ return op
+
+
+class InitOperator(Operator):
+ def __init__(self, event: Event):
+ self.event = event
+
+ def run(self, session: Session) -> Union[Operator, None]:
+ return fire_session_event(session, self.event)
+
+ def destroy(self):
+ del self.event
+
+
+def run_session_event(session: Session, event: Event, max_step: int) -> None:
+ op = InitOperator(event)
+ step = 0
+ while op is not None:
+ step += 1
+ if step > max_step:
+ raise RuntimeError(f"Max step {max_step} reached")
+ if not session.refresh(True):
+ raise RuntimeError("Session refresh failed")
+ session.logger.debug("start session op %s", repr(op))
+ next_op = op.run(session)
+ session.logger.debug("done session op %s", repr(op))
+ op.destroy()
+ # session do save after each op
+ session.save()
+ op = next_op
+
+
+def get_module_magic_ghost(module: ModuleType) -> Optional[Ghost]:
+ if "__ghost__" in module.__dict__:
+ return module.__dict__["__ghost__"]
+ return None
+
+
+def __shell_providers__() -> List[Provider]:
+ return []
+
+
+def get_module_magic_shell_providers(module: ModuleType) -> List[Provider]:
+ if __shell_providers__.__name__ in module.__dict__:
+ return module.__dict__[__shell_providers__.__name__]()
+ return []
diff --git a/ghostos/app/.example.env b/ghostos/app/.example.env
new file mode 100644
index 00000000..1745b847
--- /dev/null
+++ b/ghostos/app/.example.env
@@ -0,0 +1,17 @@
+# example of the .env file
+# provide secret constants of this project through os.environ.
+# copy this file to `.env` and fill the real values.
+# or export the values to environment.
+
+# [openai](https://openai.com/) api key
+OPENAI_API_KEY
+# optional openai proxy
+OPENAI_PROXY
+# [moonshot](https://moonshot.cn/) api key
+MOONSHOT_API_KEY="your moonshot model api key"
+# [anthropic](https://www.anthropic.com/) api key
+ANTHROPIC_API_KEY="your anthropic api key"
+# optional anthropic proxy
+ANTHROPIC_PROXY
+# [deepseek](https://deepseek.com) api key
+DEEPSEEK_API_KEY
\ No newline at end of file
diff --git a/ghostos/app/.gitignore b/ghostos/app/.gitignore
new file mode 100644
index 00000000..7ce1d676
--- /dev/null
+++ b/ghostos/app/.gitignore
@@ -0,0 +1,2 @@
+.env
+debug.log
\ No newline at end of file
diff --git a/ghostos/app/.streamlit/config.toml b/ghostos/app/.streamlit/config.toml
new file mode 100644
index 00000000..484e6692
--- /dev/null
+++ b/ghostos/app/.streamlit/config.toml
@@ -0,0 +1,317 @@
+[global]
+
+# By default, Streamlit displays a warning when a user sets both a widget
+# default value in the function defining the widget and a widget value via
+# the widget's key in `st.session_state`.
+
+# If you'd like to turn off this warning, set this to True.
+
+# Default: false
+# disableWidgetStateDuplicationWarning = false
+
+# If True, will show a warning when you run a Streamlit-enabled script
+# via "python my_script.py".
+
+# Default: true
+# showWarningOnDirectExecution = true
+
+
+[logger]
+
+# Level of logging for Streamlit's internal logger: "error", "warning",
+# "info", or "debug".
+
+# Default: "info"
+level = "info"
+
+# String format for logging messages. If logger.datetimeFormat is set,
+# logger messages will default to `%(asctime)s.%(msecs)03d %(message)s`. See
+# Python's documentation for available attributes:
+# https://docs.python.org/3/library/logging.html#formatter-objects
+
+# Default: "%(asctime)s %(message)s"
+ messageFormat = "%(asctime)s - %(levelname)s - %(message)s (%(filename)s:%(lineno)d)"
+
+
+[client]
+
+# Controls whether uncaught app exceptions and deprecation warnings
+# are displayed in the browser. By default, this is set to True and
+# Streamlit displays app exceptions and associated tracebacks, and
+# deprecation warnings, in the browser.
+
+# If set to False, deprecation warnings and full exception messages
+# will print to the console only. Exceptions will still display in the
+# browser with a generic error message. For now, the exception type and
+# traceback show in the browser also, but they will be removed in the
+# future.
+
+# Default: true
+# showErrorDetails = true
+
+# Change the visibility of items in the toolbar, options menu,
+# and settings dialog (top right of the app).
+
+# Allowed values:
+# * "auto" : Show the developer options if the app is accessed through
+# localhost or through Streamlit Community Cloud as a developer.
+# Hide them otherwise.
+# * "developer" : Show the developer options.
+# * "viewer" : Hide the developer options.
+# * "minimal" : Show only options set externally (e.g. through
+# Streamlit Community Cloud) or through st.set_page_config.
+# If there are no options left, hide the menu.
+
+# Default: "auto"
+# toolbarMode = "auto"
+
+# Controls whether to display the default sidebar page navigation in a
+# multi-page app. This only applies when app's pages are defined by the
+# `pages/` directory.
+
+# Default: true
+# showSidebarNavigation = true
+
+
+[runner]
+
+# Allows you to type a variable or string by itself in a single line of
+# Python code to write it to the app.
+
+# Default: true
+# magicEnabled = true
+
+# Handle script rerun requests immediately, rather than waiting for script
+# execution to reach a yield point. This makes Streamlit much more
+# responsive to user interaction, but it can lead to race conditions in
+# apps that mutate session_state data outside of explicit session_state
+# assignment statements.
+
+# Default: true
+# fastReruns = true
+
+# Raise an exception after adding unserializable data to Session State.
+# Some execution environments may require serializing all data in Session
+# State, so it may be useful to detect incompatibility during development,
+# or when the execution environment will stop supporting it in the future.
+
+# Default: false
+# enforceSerializableSessionState = false
+
+# Adjust how certain 'options' widgets like radio, selectbox, and
+# multiselect coerce Enum members when the Enum class gets re-defined
+# during a script re-run. For more information, check out the docs:
+# https://docs.streamlit.io/develop/concepts/design/custom-classes#enums
+
+# Allowed values:
+# * "off": Disables Enum coercion.
+# * "nameOnly": Enum classes can be coerced if their member names match.
+# * "nameAndValue": Enum classes can be coerced if their member names AND
+# member values match.
+
+# Default: "nameOnly"
+# enumCoercion = "nameOnly"
+
+
+[server]
+
+# List of folders that should not be watched for changes.
+
+# Relative paths will be taken as relative to the current working directory.
+
+# Example: ['/home/user1/env', 'relative/path/to/folder']
+
+# Default: []
+# folderWatchBlacklist = []
+
+# Change the type of file watcher used by Streamlit, or turn it off
+# completely.
+
+# Allowed values:
+# * "auto" : Streamlit will attempt to use the watchdog module, and
+# falls back to polling if watchdog is not available.
+# * "watchdog" : Force Streamlit to use the watchdog module.
+# * "poll" : Force Streamlit to always use polling.
+# * "none" : Streamlit will not watch files.
+
+# Default: "auto"
+# fileWatcherType = "auto"
+
+# Symmetric key used to produce signed cookies. If deploying on multiple
+# replicas, this should be set to the same value across all replicas to ensure
+# they all share the same secret.
+
+# Default: randomly generated secret key.
+# cookieSecret = "9c09822ee0342aa1a2e2e52256b088c48f7f73ba90f97201feeb915856905ec7"
+
+# If false, will attempt to open a browser window on start.
+
+# Default: false unless (1) we are on a Linux box where DISPLAY is unset, or
+# (2) we are running in the Streamlit Atom plugin.
+# headless = false
+
+# Automatically rerun script when the file is modified on disk.
+
+# Default: false
+# runOnSave = false
+
+# The address where the server will listen for client and browser
+# connections. Use this if you want to bind the server to a specific address.
+# If set, the server will only be accessible from this address, and not from
+# any aliases (like localhost).
+
+# Default: (unset)
+# address =
+
+# The port where the server will listen for browser connections.
+
+# Don't use port 3000 which is reserved for internal development.
+
+# Default: 8501
+# port = 8501
+
+# The base path for the URL where Streamlit should be served from.
+
+# Default: ""
+# baseUrlPath = ""
+
+# Enables support for Cross-Origin Resource Sharing (CORS) protection, for
+# added security.
+
+# Due to conflicts between CORS and XSRF, if `server.enableXsrfProtection` is
+# on and `server.enableCORS` is off at the same time, we will prioritize
+# `server.enableXsrfProtection`.
+
+# Default: true
+# enableCORS = true
+
+# Enables support for Cross-Site Request Forgery (XSRF) protection, for
+# added security.
+
+# Due to conflicts between CORS and XSRF, if `server.enableXsrfProtection` is
+# on and `server.enableCORS` is off at the same time, we will prioritize
+# `server.enableXsrfProtection`.
+
+# Default: true
+# enableXsrfProtection = true
+
+# Max size, in megabytes, for files uploaded with the file_uploader.
+
+# Default: 200
+# maxUploadSize = 200
+
+# Max size, in megabytes, of messages that can be sent via the WebSocket
+# connection.
+
+# Default: 200
+# maxMessageSize = 200
+
+# Enables support for websocket compression.
+
+# Default: false
+# enableWebsocketCompression = false
+
+# Enable serving files from a `static` directory in the running app's
+# directory.
+
+# Default: false
+# enableStaticServing = false
+
+# TTL in seconds for sessions whose websockets have been disconnected. The server
+# may choose to clean up session state, uploaded files, etc for a given session
+# with no active websocket connection at any point after this time has passed.
+
+# Default: 120
+# disconnectedSessionTTL = 120
+
+# Server certificate file for connecting via HTTPS.
+# Must be set at the same time as "server.sslKeyFile".
+
+# ['DO NOT USE THIS OPTION IN A PRODUCTION ENVIRONMENT. It has not gone through security audits or performance tests. For the production environment, we recommend performing SSL termination by the load balancer or the reverse proxy.']
+# sslCertFile =
+
+# Cryptographic key file for connecting via HTTPS.
+# Must be set at the same time as "server.sslCertFile".
+
+# ['DO NOT USE THIS OPTION IN A PRODUCTION ENVIRONMENT. It has not gone through security audits or performance tests. For the production environment, we recommend performing SSL termination by the load balancer or the reverse proxy.']
+# sslKeyFile =
+
+
+[browser]
+
+# Internet address where users should point their browsers in order to
+# connect to the app. Can be IP address or DNS name and path.
+
+# This is used to:
+# - Set the correct URL for CORS and XSRF protection purposes.
+# - Show the URL on the terminal
+# - Open the browser
+
+# Default: "localhost"
+# serverAddress = "localhost"
+
+# Whether to send usage statistics to Streamlit.
+
+# Default: true
+# gatherUsageStats = true
+
+# Port where users should point their browsers in order to connect to the
+# app.
+
+# This is used to:
+# - Set the correct URL for XSRF protection purposes.
+# - Show the URL on the terminal (part of `streamlit run`).
+# - Open the browser automatically (part of `streamlit run`).
+
+# This option is for advanced use cases. To change the port of your app, use
+# `server.Port` instead. Don't use port 3000 which is reserved for internal
+# development.
+
+# Default: whatever value is set in server.port.
+# serverPort = 8501
+
+
+[mapbox]
+
+# Configure Streamlit to use a custom Mapbox
+# token for elements like st.pydeck_chart and st.map.
+# To get a token for yourself, create an account at
+# https://mapbox.com. It's free (for moderate usage levels)!
+
+# Default: ""
+# token = ""
+
+
+[theme]
+
+# The preset Streamlit theme that your custom theme inherits from.
+# One of "light" or "dark".
+# base =
+
+# Primary accent color for interactive elements.
+# primaryColor =
+
+# Background color for the main content area.
+# backgroundColor =
+
+# Background color used for the sidebar and most interactive widgets.
+# secondaryBackgroundColor =
+
+# Color used for almost all text.
+# textColor =
+
+# Font family for all text in the app, except code blocks. One of "sans serif",
+# "serif", or "monospace".
+# font =
+
+
+[secrets]
+
+# List of locations where secrets are searched. An entry can be a path to a
+# TOML file or directory path where Kubernetes style secrets are saved.
+# Order is important, import is first to last, so secrets in later files
+# will take precedence over earlier ones.
+
+# Default: [ "/Users/BrightRed/.streamlit/secrets.toml", "/Users/BrightRed/Develop/github.com/ghost-in-moss/GhostOS/.streamlit/secrets.toml",]
+# files = [ "/Users/BrightRed/.streamlit/secrets.toml", "/Users/BrightRed/Develop/github.com/ghost-in-moss/GhostOS/.streamlit/secrets.toml",]
+
diff --git a/ghostos/app/assets/docs/ghostos/en/aifunc_introduction.md b/ghostos/app/assets/docs/ghostos/en/aifunc_introduction.md
new file mode 100644
index 00000000..627f52f3
--- /dev/null
+++ b/ghostos/app/assets/docs/ghostos/en/aifunc_introduction.md
@@ -0,0 +1,2 @@
+
+`AI Func`
\ No newline at end of file
diff --git a/ghostos/app/assets/docs/ghostos/zh/aifunc/introduction.md b/ghostos/app/assets/docs/ghostos/zh/aifunc/introduction.md
new file mode 100644
index 00000000..1a7e7e2e
--- /dev/null
+++ b/ghostos/app/assets/docs/ghostos/zh/aifunc/introduction.md
@@ -0,0 +1,4 @@
+
+AI Func 将大语言模型的能力整合到 python 代码中, 将一个继承自 `pydantic.BaseModel` 的类可以作为函数使用,
+运行时大模型将看到代码所处的上下文, 在理解代码的基础上, 自行进行多轮思考, 并写出执行代码.
+`AI Func` 可以相互嵌套.
\ No newline at end of file
diff --git a/ghostos/app/assets/docs/ghostos/zh/aifunc/request_info.md b/ghostos/app/assets/docs/ghostos/zh/aifunc/request_info.md
new file mode 100644
index 00000000..69e5f938
--- /dev/null
+++ b/ghostos/app/assets/docs/ghostos/zh/aifunc/request_info.md
@@ -0,0 +1 @@
+AIFunc 使用 `pydantic.BaseModel` 来定义, 是一个强类型的数据结构.
\ No newline at end of file
diff --git a/ghostos/app/assets/docs/ghostos/zh/aifunc/usage_example.md b/ghostos/app/assets/docs/ghostos/zh/aifunc/usage_example.md
new file mode 100644
index 00000000..bcda9b3c
--- /dev/null
+++ b/ghostos/app/assets/docs/ghostos/zh/aifunc/usage_example.md
@@ -0,0 +1,46 @@
+同步调用的例子:
+
+```python
+from ghostos.container import Container
+from ghostos.core.aifunc import AIFuncExecutor
+from ghostos.demo.aifuncs.weather import WeatherAIFunc, WeatherAIFuncResult
+
+
+# 同步调用.
+def call_example(con: Container, req: WeatherAIFunc) -> WeatherAIFuncResult:
+ '''
+ async call an AIFunc and wait for result
+ '''
+ executor = con.force_fetch(AIFuncExecutor)
+ return executor.execute(req)
+```
+
+异步调用的例子:
+
+```python
+from ghostos.container import Container
+from ghostos.core.aifunc import AIFuncExecutor, ExecFrame
+from ghostos.core.messages import new_basic_connection
+from ghostos.demo.aifuncs.weather import WeatherAIFunc, WeatherAIFuncResult
+
+
+def stream_call_example(con: Container, req: WeatherAIFunc) -> WeatherAIFuncResult:
+ '''
+ async call an AIFunc and wait for result
+ '''
+ from threading import Thread
+
+ executor = con.force_fetch(AIFuncExecutor)
+
+ stream, receiver = new_basic_connection()
+ frame = ExecFrame.from_func(req)
+ t = Thread(target=executor.execute, args=(req, frame, stream))
+ t.start()
+
+ with receiver:
+ for msg in receiver.recv():
+ # do something
+ pass
+ t.join()
+ return frame.get_result()
+```
\ No newline at end of file
diff --git a/ghostos/app/configs/documents_registry.yml b/ghostos/app/configs/documents_registry.yml
new file mode 100644
index 00000000..c0ca7a8b
--- /dev/null
+++ b/ghostos/app/configs/documents_registry.yml
@@ -0,0 +1,5 @@
+docs:
+ - directory: docs/
+ domain: ghostos
+ extension: .md
+ default_lang: zh
\ No newline at end of file
diff --git a/ghostos/app/configs/ghostos_conf.yml b/ghostos/app/configs/ghostos_conf.yml
new file mode 100644
index 00000000..db7062ff
--- /dev/null
+++ b/ghostos/app/configs/ghostos_conf.yml
@@ -0,0 +1,2 @@
+shells:
+ console: {}
\ No newline at end of file
diff --git a/ghostos/app/configs/llms_conf.yml b/ghostos/app/configs/llms_conf.yml
new file mode 100644
index 00000000..8e18659c
--- /dev/null
+++ b/ghostos/app/configs/llms_conf.yml
@@ -0,0 +1,145 @@
+# from class: ghostos.framework.llms.providers:LLMsYamlConfig
+default: gpt-4o
+models:
+ claude-3-5-sonnet:
+ kwargs: {}
+ max_tokens: 2000
+ message_types: null
+ model: claude-3-5-sonnet-20240620
+ n: 1
+ request_timeout: 40
+ service: anthropic
+ temperature: 0.7
+ timeout: 30
+ use_tools: true
+ claude-3-haiku:
+ kwargs: {}
+ max_tokens: 2000
+ message_types: null
+ model: claude-3-haiku-20240307
+ n: 1
+ request_timeout: 40
+ service: anthropic
+ temperature: 0.7
+ timeout: 30
+ use_tools: true
+ deepseek-chat:
+ kwargs: {}
+ max_tokens: 2000
+ message_types: null
+ model: deepseek/deepseek-chat
+ n: 1
+ request_timeout: 40
+ service: deepseek
+ temperature: 0.7
+ timeout: 30
+ use_tools: true
+ deepseek-coder:
+ kwargs: {}
+ max_tokens: 2000
+ message_types: null
+ model: deepseek/deepseek-coder
+ n: 1
+ request_timeout: 40
+ service: deepseek
+ temperature: 0.7
+ timeout: 30
+ use_tools: true
+ gpt-3.5-turbo:
+ kwargs: {}
+ max_tokens: 2000
+ message_types: null
+ model: gpt-3.5-turbo
+ n: 1
+ request_timeout: 40
+ service: openai
+ temperature: 0.7
+ timeout: 30
+ use_tools: true
+ gpt-4:
+ kwargs: {}
+ max_tokens: 2000
+ message_types: null
+ model: gpt-4
+ n: 1
+ request_timeout: 40
+ service: openai
+ temperature: 0.7
+ timeout: 30
+ use_tools: true
+ gpt-4-turbo:
+ kwargs: {}
+ max_tokens: 2000
+ message_types: null
+ model: gpt-4-turbo
+ n: 1
+ request_timeout: 40
+ service: openai
+ temperature: 0.7
+ timeout: 30
+ use_tools: true
+ gpt-4o:
+ kwargs: {}
+ max_tokens: 2000
+ message_types: null
+ model: gpt-4o
+ n: 1
+ request_timeout: 40
+ service: openai
+ temperature: 0.7
+ timeout: 30
+ use_tools: true
+ moonshot-v1-128k:
+ kwargs: {}
+ max_tokens: 2000
+ message_types: null
+ model: moonshot-v1-128k
+ n: 1
+ request_timeout: 40
+ service: moonshot
+ temperature: 0.7
+ timeout: 30
+ use_tools: true
+ moonshot-v1-32k:
+ kwargs: {}
+ max_tokens: 2000
+ message_types: null
+ model: moonshot-v1-32k
+ n: 1
+ request_timeout: 40
+ service: moonshot
+ temperature: 0.7
+ timeout: 30
+ use_tools: true
+ moonshot-v1-8k:
+ kwargs: {}
+ max_tokens: 2000
+ message_types: null
+ model: moonshot-v1-8k
+ n: 1
+ request_timeout: 40
+ service: moonshot
+ temperature: 0.7
+ timeout: 30
+ use_tools: true
+services:
+- base_url: https://api.moonshot.cn/v1
+ driver: openai_driver
+ name: moonshot
+ proxy: null
+ token: $MOONSHOT_API_KEY
+- base_url: https://api.openai.com/v1
+ driver: openai_driver
+ name: openai
+ proxy: $OPENAI_PROXY
+ token: $OPENAI_API_KEY
+- base_url: https://api.anthropic.com/v1
+ driver: openai_driver
+ name: anthropic
+ proxy: $ANTHROPIC_PROXY
+ token: $ANTHROPIC_API_KEY
+- base_url: https://api.deepseek.com/beta
+ driver: openai_driver
+ name: deepseek
+ proxy: null
+ token: $DEEPSEEK_API_KEY
diff --git a/ghostos/app/configs/openai_realtime_config.yml b/ghostos/app/configs/openai_realtime_config.yml
new file mode 100644
index 00000000..6f31cf5a
--- /dev/null
+++ b/ghostos/app/configs/openai_realtime_config.yml
@@ -0,0 +1 @@
+{ }
\ No newline at end of file
diff --git a/ghostos/app/configs/realtime_conf.yml b/ghostos/app/configs/realtime_conf.yml
new file mode 100644
index 00000000..440e174b
--- /dev/null
+++ b/ghostos/app/configs/realtime_conf.yml
@@ -0,0 +1,5 @@
+default: "openai"
+apps:
+ openai:
+ type: "ghostos.framework.openai_realtime.configs:OpenAIRealtimeAppConf"
+ data: { }
\ No newline at end of file
diff --git a/ghostos/app/configs/registered_aifunc.yml b/ghostos/app/configs/registered_aifunc.yml
new file mode 100644
index 00000000..bd5cfc78
--- /dev/null
+++ b/ghostos/app/configs/registered_aifunc.yml
@@ -0,0 +1,2 @@
+# from class: ghostos.core.aifunc.repository:AIFuncsConf
+identifiers: []
diff --git a/ghostos/app/configs/streamlit_app.yml b/ghostos/app/configs/streamlit_app.yml
new file mode 100644
index 00000000..87c05175
--- /dev/null
+++ b/ghostos/app/configs/streamlit_app.yml
@@ -0,0 +1,6 @@
+# from class: ghostos.prototypes.streamlitapp.resources:AppConf
+bool_options:
+ DEBUG_MODE: true
+ HELP_MODE: true
+domain: ghostos
+lang: zh
diff --git a/evaluation/swe_bench_lite/ai_funcs/__init__.py b/ghostos/app/memories/.gitkeep
similarity index 100%
rename from evaluation/swe_bench_lite/ai_funcs/__init__.py
rename to ghostos/app/memories/.gitkeep
diff --git a/ghostos/demo/runtime/cache/.gitignore b/ghostos/app/runtime/aifunc_frames/.gitignore
similarity index 100%
rename from ghostos/demo/runtime/cache/.gitignore
rename to ghostos/app/runtime/aifunc_frames/.gitignore
diff --git a/ghostos/demo/runtime/events/.gitignore b/ghostos/app/runtime/audios/.gitignore
similarity index 100%
rename from ghostos/demo/runtime/events/.gitignore
rename to ghostos/app/runtime/audios/.gitignore
diff --git a/ghostos/demo/runtime/processes/.gitignore b/ghostos/app/runtime/cache/.gitignore
similarity index 100%
rename from ghostos/demo/runtime/processes/.gitignore
rename to ghostos/app/runtime/cache/.gitignore
diff --git a/ghostos/demo/runtime/tasks/.gitignore b/ghostos/app/runtime/events/.gitignore
similarity index 100%
rename from ghostos/demo/runtime/tasks/.gitignore
rename to ghostos/app/runtime/events/.gitignore
diff --git a/ghostos/demo/runtime/threads/.gitignore b/ghostos/app/runtime/images/.gitignore
similarity index 100%
rename from ghostos/demo/runtime/threads/.gitignore
rename to ghostos/app/runtime/images/.gitignore
diff --git a/ghostos/app/runtime/logs/.gitignore b/ghostos/app/runtime/logs/.gitignore
new file mode 100644
index 00000000..c96a04f0
--- /dev/null
+++ b/ghostos/app/runtime/logs/.gitignore
@@ -0,0 +1,2 @@
+*
+!.gitignore
\ No newline at end of file
diff --git a/ghostos/app/runtime/processes/.gitignore b/ghostos/app/runtime/processes/.gitignore
new file mode 100644
index 00000000..c96a04f0
--- /dev/null
+++ b/ghostos/app/runtime/processes/.gitignore
@@ -0,0 +1,2 @@
+*
+!.gitignore
\ No newline at end of file
diff --git a/ghostos/app/runtime/prompts/.gitignore b/ghostos/app/runtime/prompts/.gitignore
new file mode 100644
index 00000000..c96a04f0
--- /dev/null
+++ b/ghostos/app/runtime/prompts/.gitignore
@@ -0,0 +1,2 @@
+*
+!.gitignore
\ No newline at end of file
diff --git a/ghostos/app/runtime/tasks/.gitignore b/ghostos/app/runtime/tasks/.gitignore
new file mode 100644
index 00000000..c96a04f0
--- /dev/null
+++ b/ghostos/app/runtime/tasks/.gitignore
@@ -0,0 +1,2 @@
+*
+!.gitignore
\ No newline at end of file
diff --git a/ghostos/app/runtime/threads/.gitignore b/ghostos/app/runtime/threads/.gitignore
new file mode 100644
index 00000000..c96a04f0
--- /dev/null
+++ b/ghostos/app/runtime/threads/.gitignore
@@ -0,0 +1,2 @@
+*
+!.gitignore
\ No newline at end of file
diff --git a/ghostos/app/runtime/variables/.gitignore b/ghostos/app/runtime/variables/.gitignore
new file mode 100644
index 00000000..c96a04f0
--- /dev/null
+++ b/ghostos/app/runtime/variables/.gitignore
@@ -0,0 +1,2 @@
+*
+!.gitignore
\ No newline at end of file
diff --git a/evaluation/swe_bench_lite/base/__init__.py b/ghostos/app/source/.gitkeep
similarity index 100%
rename from evaluation/swe_bench_lite/base/__init__.py
rename to ghostos/app/source/.gitkeep
diff --git a/ghostos/bootstrap.py b/ghostos/bootstrap.py
new file mode 100644
index 00000000..3692f665
--- /dev/null
+++ b/ghostos/bootstrap.py
@@ -0,0 +1,395 @@
+from __future__ import annotations
+import yaml
+from warnings import warn
+from typing import List, Optional, Tuple
+from os.path import dirname, join, exists, abspath, isdir
+from ghostos.abcd import GhostOS
+from ghostos.container import Container, Provider, Contracts
+from ghostos.prototypes.ghostfunc import init_ghost_func, GhostFunc
+from pydantic import BaseModel, Field
+
+# Core Concepts
+#
+# 1. Ghost and Shell
+# We take the word `Ghost` from famous manga movie as the abstract of an Agent.
+# Ghost shall have concurrent thinking/action capabilities, each thought or task is a fragment of the Ghost mind;
+# not like an independent agent in a multi-agent system.
+# But you can take `Ghost` as `Agent` for now.
+# Also, the word `Shell` in this project refers to the `Body` of the Agent,
+# regardless if it is an Embodied Robot/IM chatbot/Website/IDE etc.
+#
+# 2. MOSS
+# stands for "Model-oriented Operating System Simulation".
+# - operating system: to operate an Agent's body (Shell), mind, tools.
+# - model-oriented: the first class user of the OS is the brain of Ghost(AI models), not Human
+# - simulation: we merely use python to simulate the OS, not create a real one.
+# Instead of `JSON Schema Tool`, we provide a python code interface for LLMs through MOSS.
+# The LLMs can read python context as prompt, then generate python code to do almost everything.
+# MOSS can reflect the python module to prompt, and execute the generated python code within a specific python context.
+#
+# We are aiming to create Fractal Meta-Agent which can generate tools/libraries/Shells/
+#
+# 3. GhostOS
+# Is an agent framework for developers like myself, to define/test/use/modify Model-based Agents.
+# Not like MOSS which serve the Models (Large Language Model mostly),
+# GhostOS is a framework works for me the Human developer.
+#
+# 4. Application
+# Is the production built with GhostOS.
+# There are light-weight applications like `GhostFunc` which is a python function decorator,
+# and heavy applications like Streamlit app.
+#
+# todo: let the gpt4o or moonshot fix my pool english expressions above.
+
+
+__all__ = [
+ 'expect_workspace_dir',
+ 'app_stub_dir',
+ 'BootstrapConfig',
+ 'get_bootstrap_config',
+
+ # >>> container
+ # GhostOS use IoC Container to manage dependency injections at everywhere.
+ # IoCContainer inherit all the bindings from parent Container, and also able to override them.
+ # The singletons in the container shall always be thread-safe.
+ #
+ # The containers nest in multiple levels like a tree:
+ # - Application level (global static container that instanced in this file)
+ # - GhostOS level (a GhostOS manage as many ghost as it able to)
+ # - Ghost level (a Ghost is an instance frame of the Agent's thought)
+ # - Moss level (each MossCompiler has it own container)
+ # <<<
+ 'application_container',
+ 'make_app_container',
+
+ # reset ghostos default application instances.
+ 'reset',
+ 'get_container',
+
+ # default configuration
+ # consider safety reason, ghostos is not ready for online business
+ # so many abilities shall be forbidden to web agent
+ 'default_application_contracts',
+ 'default_application_providers',
+
+ 'get_ghostos',
+
+ # >>> GhostFunc
+ # is a test library, which is able to define dynamic code for an in-complete function.
+ # We develop it for early experiments.
+ # Check example_ghost_func.py
+ # <<<
+ 'ghost_func',
+ 'GhostFunc',
+ 'init_ghost_func',
+
+]
+
+
+# --- prepare application paths --- #
+
+
+def expect_workspace_dir() -> Tuple[str, bool]:
+ """
+ :return: (workspace dir: str, exists: bool)
+ """
+ expect_dir = abspath("app")
+ return abspath(expect_dir), exists(expect_dir) and isdir(expect_dir)
+
+
+def app_stub_dir() -> str:
+ return join(dirname(__file__), "app")
+
+
+def find_workspace_dir() -> str:
+ expected, ok = expect_workspace_dir()
+ if ok:
+ return expected
+ return app_stub_dir()
+
+
+class BootstrapConfig(BaseModel):
+ workspace_dir: str = Field(
+ default_factory=find_workspace_dir,
+ description="ghostos workspace directory",
+ )
+ dotenv_file_path: str = Field(".env", description="ghostos workspace .env file")
+ ghostos_dir: str = Field(
+ default=dirname(dirname(__file__)),
+ description="ghostos source code directory",
+ )
+ workspace_configs_dir: str = Field(
+ "configs",
+ description="ghostos workspace relative path for configs directory",
+ )
+ workspace_runtime_dir: str = Field(
+ "runtime",
+ description="ghostos workspace relative path for runtime directory",
+ )
+
+ __from_file__: str = ""
+
+ def abs_runtime_dir(self) -> str:
+ return join(self.workspace_dir, "runtime")
+
+ def abs_asserts_dir(self) -> str:
+ return join(self.workspace_dir, "assets")
+
+ def env_file(self) -> str:
+ return join(self.workspace_dir, self.dotenv_file_path)
+
+ def env_example_file(self) -> str:
+ return join(self.workspace_dir, ".example.env")
+
+ def save(self, dir_path: str = None):
+ from ghostos.helpers import yaml_pretty_dump
+ if dir_path is None:
+ filename = join(abspath(".ghostos.yml"))
+ else:
+ filename = join(dir_path, ".ghostos.yml")
+ content = yaml_pretty_dump(self.model_dump())
+ with open(filename, "w") as f:
+ f.write(content)
+
+
+def get_bootstrap_config(local: bool = True) -> BootstrapConfig:
+ """
+ get ghostos bootstrap config from current working directory
+ if not found, return default configs.
+ """
+ expect_file = abspath(".ghostos.yml")
+ if local and exists(expect_file):
+ with open(expect_file) as f:
+ content = f.read()
+ data = yaml.safe_load(content)
+ conf = BootstrapConfig(**data)
+ conf.__from_file__ = expect_file
+ return conf
+ else:
+ return BootstrapConfig()
+
+
+# --- default providers --- #
+
+
+def default_application_contracts() -> Contracts:
+ """
+ Application level contracts
+ """
+ from ghostos.core.moss import MossCompiler
+ from ghostos.core.messages.openai import OpenAIMessageParser
+ from ghostos.contracts.shutdown import Shutdown
+ from ghostos.contracts.modules import Modules
+ from ghostos.contracts.workspace import Workspace
+ from ghostos.contracts.variables import Variables
+ from ghostos.framework.configs import Configs
+ from ghostos.framework.processes import GoProcesses
+ from ghostos.framework.threads import GoThreads
+ from ghostos.framework.tasks import GoTasks
+ from ghostos.framework.eventbuses import EventBus
+ from ghostos.framework.llms import LLMs, PromptStorage
+ from ghostos.framework.logger import LoggerItf
+ from ghostos.framework.documents import DocumentRegistry
+ from ghostos.framework.ghostos import GhostOS
+ from ghostos.framework.assets import ImageAssets, AudioAssets
+ from ghostos.framework.realtime import Realtime
+ from ghostos.core.aifunc import AIFuncExecutor, AIFuncRepository
+
+ return Contracts([
+ # workspace contracts
+ Workspace, # application workspace implementation
+ Configs, # application configs repository
+ Variables,
+
+ ImageAssets,
+ AudioAssets,
+
+ # system contracts
+ Shutdown, # graceful shutdown register
+ LLMs, # LLMs interface
+ PromptStorage,
+
+ LoggerItf, # the logger instance of application
+ Modules, # the import_module proxy
+
+ DocumentRegistry,
+
+ # messages
+ OpenAIMessageParser,
+
+ # moss
+ MossCompiler,
+
+ # aifunc
+ AIFuncExecutor,
+ AIFuncRepository,
+
+ # session contracts
+ GoProcesses, # application processes repository
+ GoThreads, # application threads repository
+ GoTasks, # application tasks repository
+ EventBus, # application session eventbus
+
+ # root
+ GhostOS,
+
+ Realtime,
+ ])
+
+
+def default_application_providers(
+ config: Optional[BootstrapConfig] = None,
+) -> List[Provider]:
+ """
+ application default providers
+ todo: use manager provider to configurate multiple kinds of implementation
+ """
+ from ghostos.contracts.shutdown import ShutdownProvider
+ from ghostos.contracts.modules import DefaultModulesProvider
+ from ghostos.core.moss import DefaultMOSSProvider
+ from ghostos.core.messages.openai import DefaultOpenAIParserProvider
+ from ghostos.framework.workspaces import BasicWorkspaceProvider
+ from ghostos.framework.configs import WorkspaceConfigsProvider
+ from ghostos.framework.assets import WorkspaceImageAssetsProvider, WorkspaceAudioAssetsProvider
+ from ghostos.framework.processes import WorkspaceProcessesProvider
+ from ghostos.framework.threads import MsgThreadsRepoByWorkSpaceProvider
+ from ghostos.framework.tasks import WorkspaceTasksProvider
+ from ghostos.framework.eventbuses import MemEventBusImplProvider
+ from ghostos.framework.llms import ConfigBasedLLMsProvider, PromptStorageInWorkspaceProvider
+ from ghostos.framework.logger import DefaultLoggerProvider
+ from ghostos.framework.variables import WorkspaceVariablesProvider
+ from ghostos.framework.ghostos import GhostOSProvider
+ from ghostos.framework.documents import ConfiguredDocumentRegistryProvider
+ from ghostos.framework.realtime import ConfigBasedRealtimeProvider
+ from ghostos.core.aifunc import DefaultAIFuncExecutorProvider, AIFuncRepoByConfigsProvider
+
+ if config is None:
+ config = get_bootstrap_config(local=True)
+
+ return [
+
+ # --- logger ---#
+
+ DefaultLoggerProvider(),
+ # --- workspace --- #
+ BasicWorkspaceProvider(
+ workspace_dir=config.workspace_dir,
+ configs_path=config.workspace_configs_dir,
+ runtime_path=config.workspace_runtime_dir,
+ ),
+ WorkspaceConfigsProvider(),
+ WorkspaceProcessesProvider(),
+ WorkspaceTasksProvider(),
+ ConfiguredDocumentRegistryProvider(),
+ WorkspaceVariablesProvider(),
+ WorkspaceImageAssetsProvider(),
+ WorkspaceAudioAssetsProvider(),
+
+ # --- messages --- #
+ DefaultOpenAIParserProvider(),
+
+ # --- session ---#
+ MsgThreadsRepoByWorkSpaceProvider(),
+ MemEventBusImplProvider(),
+
+ # --- moss --- #
+ DefaultMOSSProvider(),
+
+ # --- llm --- #
+ ConfigBasedLLMsProvider(),
+ PromptStorageInWorkspaceProvider(),
+
+ # --- basic library --- #
+ DefaultModulesProvider(),
+ ShutdownProvider(),
+ # WorkspaceTranslationProvider("translations"),
+
+ # --- aifunc --- #
+ DefaultAIFuncExecutorProvider(),
+ AIFuncRepoByConfigsProvider(),
+
+ GhostOSProvider(),
+ ConfigBasedRealtimeProvider(),
+ ]
+
+
+# --- system bootstrap --- #
+def make_app_container(
+ *,
+ bootstrap_conf: Optional[BootstrapConfig] = None,
+ app_providers: Optional[List[Provider]] = None,
+ app_contracts: Optional[Contracts] = None,
+) -> Container:
+ """
+ make application global container
+ """
+
+ if bootstrap_conf is None:
+ bootstrap_conf = get_bootstrap_config(local=True)
+ workspace_dir = bootstrap_conf.workspace_dir
+ if workspace_dir.startswith(bootstrap_conf.ghostos_dir):
+ warn(
+ f"GhostOS workspace dir is not found, better run `ghostos init` at first."
+ f"Currently using `{workspace_dir}` as workspace."
+ )
+
+ # load env from dotenv file
+ env_path = join(bootstrap_conf.workspace_dir, bootstrap_conf.dotenv_file_path)
+ if exists(env_path):
+ import dotenv
+ dotenv.load_dotenv(dotenv_path=env_path)
+
+ # default logger name for GhostOS application
+ if app_providers is None:
+ app_providers = default_application_providers(bootstrap_conf)
+ if app_contracts is None:
+ app_contracts = default_application_contracts()
+
+ # prepare application container
+ _container = Container(name="ghostos_root")
+ _container.set(BootstrapConfig, bootstrap_conf)
+ _container.register(*app_providers)
+ # contracts validation
+ app_contracts.validate(_container)
+ # bootstrap.
+ _container.bootstrap()
+ return _container
+
+
+application_container = make_app_container()
+""" the global static application container. reset it before application usage"""
+
+ghost_func = init_ghost_func(application_container)
+""" the default ghost func on default container"""
+
+
+def get_ghostos(container: Optional[Container] = None) -> GhostOS:
+ if container is None:
+ container = application_container
+ return container.force_fetch(GhostOS)
+
+
+def get_container() -> Container:
+ return application_container
+
+
+def reset(con: Container) -> Container:
+ """
+ reset static ghostos application level instances
+ :param con: a container with application level contract bindings, shall be validated outside.
+ :return:
+ """
+ global application_container, ghost_func
+ # reset global container
+ application_container = con
+ # reset global ghost func
+ ghost_func = init_ghost_func(application_container)
+ return application_container
+
+
+# --- test the module by python -i --- #
+
+if __name__ == '__main__':
+ """
+ run `python -i __init__.py` to interact with the current file
+ """
diff --git a/ghostos/container.py b/ghostos/container.py
index 2eaca66f..b95e7a50 100644
--- a/ghostos/container.py
+++ b/ghostos/container.py
@@ -2,20 +2,29 @@
import inspect
from abc import ABCMeta, abstractmethod
from typing import Type, Dict, TypeVar, Callable, Set, Optional, List, Generic, Any, Union, Iterable
+from typing import get_args, get_origin, ClassVar
+import warnings
__all__ = [
"Container", "IoCContainer",
- "Provider", "Factory", "Bootstrapper",
- "ABSTRACT",
+ "Provider", "Factory", "Bootstrapper", "BootstrapProvider",
+ "INSTANCE", "ABSTRACT",
"ProviderAdapter", 'provide',
+ 'Contracts',
'get_caller_info',
+ 'get_container',
+ 'set_container',
]
INSTRUCTION = """
打算实现一个 IoC 容器用来管理大量可替换的中间库.
"""
-ABSTRACT = TypeVar('ABSTRACT', bound=object)
+INSTANCE = TypeVar('INSTANCE')
+"""instance in the container"""
+
+ABSTRACT = Type[INSTANCE]
+"""abstract of the instance"""
class IoCContainer(metaclass=ABCMeta):
@@ -24,13 +33,13 @@ class IoCContainer(metaclass=ABCMeta):
"""
@abstractmethod
- def set(self, abstract: Type[ABSTRACT], instance: ABSTRACT) -> None:
+ def set(self, abstract: ABSTRACT, instance: INSTANCE) -> None:
"""
设置一个实例, 不会污染父容器.
"""
@abstractmethod
- def register(self, provider: Provider) -> None:
+ def register(self, *providers: Provider) -> None:
"""
register factory of the contract by provider
"""
@@ -54,7 +63,7 @@ def bootstrap(self) -> None:
pass
@abstractmethod
- def get(self, abstract: Union[Type[ABSTRACT], Factory, Provider]) -> Optional[ABSTRACT]:
+ def get(self, abstract: Type[INSTANCE]) -> Optional[INSTANCE]:
"""
get bound instance or initialize one from registered abstract, or generate one by factory or provider.
:return: None if no bound instance.
@@ -62,7 +71,24 @@ def get(self, abstract: Union[Type[ABSTRACT], Factory, Provider]) -> Optional[AB
pass
@abstractmethod
- def fetch(self, abstract: Type[ABSTRACT], strict: bool = False) -> Optional[ABSTRACT]:
+ def get_bound(self, abstract: Type[INSTANCE]) -> Union[INSTANCE, Provider, None]:
+ """
+ get bound of an abstract
+ useful to debug
+ :return: instance or provider
+ """
+ pass
+
+ @abstractmethod
+ def get_provider(self, abstract: Type[INSTANCE]) -> Optional[Provider[INSTANCE]]:
+ pass
+
+ @abstractmethod
+ def rebind(self, abstract: Type[INSTANCE]) -> None:
+ pass
+
+ @abstractmethod
+ def fetch(self, abstract: Type[INSTANCE], strict: bool = False) -> Optional[INSTANCE]:
"""
:param abstract: use type of the object (usually an abstract class) to fetch the implementation.
:param strict: autotype check
@@ -71,7 +97,7 @@ def fetch(self, abstract: Type[ABSTRACT], strict: bool = False) -> Optional[ABST
pass
@abstractmethod
- def force_fetch(self, contract: Type[ABSTRACT], strict: bool = False) -> ABSTRACT:
+ def force_fetch(self, contract: Type[INSTANCE], strict: bool = False) -> INSTANCE:
"""
if fetch contract failed, raise error.
:exception: NotImplementedError if contract is not registered.
@@ -80,21 +106,25 @@ def force_fetch(self, contract: Type[ABSTRACT], strict: bool = False) -> ABSTRAC
pass
@abstractmethod
- def bound(self, contract: Type[ABSTRACT]) -> bool:
+ def bound(self, contract: Type[INSTANCE]) -> bool:
"""
return whether contract is bound.
"""
pass
@abstractmethod
- def contracts(self, recursively: bool = True) -> Iterable[Type[ABSTRACT]]:
+ def contracts(self, recursively: bool = True) -> Iterable[ABSTRACT]:
"""
yield from bound contracts
"""
pass
@abstractmethod
- def destroy(self) -> None:
+ def providers(self, recursively: bool = True) -> Iterable[Provider]:
+ pass
+
+ @abstractmethod
+ def shutdown(self) -> None:
"""
Manually delete the container to prevent memory leaks.
"""
@@ -111,15 +141,25 @@ class Container(IoCContainer):
- 对于 MOSS 而言, Container 也是必要的. 这样可以只把 interface 暴露给 LLM, 但又可以让它使用实例.
- 仍然需要考虑加入 RAG Memories 来支持. 获取做到 OS 层.
"""
+ instance_count: ClassVar[int] = 0
+ bloodline: List[str]
- def __init__(self, parent: Optional[Container] = None):
+ def __init__(self, parent: Optional[Container] = None, *, name: str = "", inherit: bool = True):
+ self.bloodline = []
# container extended by children container
if parent is not None:
if not isinstance(parent, Container):
raise AttributeError("container can only initialized with parent Container")
if parent is self:
raise AttributeError("container's parent must not be itself")
- self.parent = parent
+ self.parent: Optional[Container] = parent
+ if isinstance(self.parent, Container):
+ bloodline = self.parent.bloodline.copy()
+ bloodline.append(name)
+ else:
+ bloodline = [name]
+ self.bloodline: List[str] = bloodline
+
# global singletons.
self._instances: Dict[Any, Any] = {}
self._factory: Dict[Any, Factory] = {}
@@ -128,27 +168,52 @@ def __init__(self, parent: Optional[Container] = None):
self._bound: Set = set()
self._bootstrapper: List["Bootstrapper"] = []
self._bootstrapped: bool = False
+ self._aliases: Dict[Any, Any] = {}
+ self._is_shutdown: bool = False
+ self._shutdown: List[Callable[[], None]] = []
+ if inherit and parent is not None:
+ self._inherit(parent)
+
+ Container.instance_count += 1
+
+ def _inherit(self, parent: Container):
+ """
+ inherit none singleton provider from parent
+ """
+ for provider in parent.providers(recursively=True):
+ if provider.inheritable() and not isinstance(provider, Bootstrapper):
+ self._register(provider)
def bootstrap(self) -> None:
"""
执行 bootstrap, 只执行一次. 可以操作依赖关系. 比如实例化后反向注册.
"""
+ self._check_destroyed()
if self._bootstrapped:
return
# 必须在这里初始化, 否则会循环调用.
self._bootstrapped = True
- if not self._bootstrapper:
- return
- for b in self._bootstrapper:
- b.bootstrap(self)
+ if self._bootstrapper:
+ for b in self._bootstrapper:
+ b.bootstrap(self)
+ for provider in self._providers.values():
+ # some bootstrapper provider may be override
+ if isinstance(provider, Bootstrapper):
+ provider.bootstrap(self)
+
+ def add_shutdown(self, shutdown: Callable):
+ self._shutdown.append(shutdown)
- def set(self, abstract: Type[ABSTRACT], instance: ABSTRACT) -> None:
+ def set(self, abstract: Any, instance: INSTANCE) -> None:
"""
设置一个实例, 不会污染父容器.
"""
+ self._check_destroyed()
+ if abstract in self._providers:
+ del self._providers[abstract]
self._set_instance(abstract, instance)
- def _bind_contract(self, abstract: Type[ABSTRACT]) -> None:
+ def _add_bound_contract(self, abstract: ABSTRACT) -> None:
"""
添加好绑定关系, 方便快速查找.
"""
@@ -158,9 +223,10 @@ def bound(self, contract: Type) -> bool:
"""
return whether contract is bound.
"""
+ self._check_destroyed()
return contract in self._bound or (self.parent is not None and self.parent.bound(contract))
- def get(self, abstract: Type[ABSTRACT]) -> Optional[ABSTRACT]:
+ def get(self, abstract: Union[Type[INSTANCE], Any]) -> Optional[INSTANCE]:
"""
get bound instance or initialize one from registered factory or provider.
@@ -168,11 +234,11 @@ def get(self, abstract: Type[ABSTRACT]) -> Optional[ABSTRACT]:
- params 感觉不需要.
"""
+ self._check_destroyed()
# 进行初始化.
- self.bootstrap()
-
- if isinstance(abstract, Provider):
- return abstract.factory(self)
+ if not self._bootstrapped:
+ warnings.warn("container is not bootstrapped before using")
+ self.bootstrap()
# get bound instance
got = self._instances.get(abstract, None)
@@ -187,17 +253,40 @@ def get(self, abstract: Type[ABSTRACT]) -> Optional[ABSTRACT]:
self._set_instance(abstract, made)
return made
- # 第三优先级.
+ # search aliases if the real contract exists
+ if abstract in self._aliases:
+ contract = self._aliases[abstract]
+ return self.get(contract)
+
+ # at last
if self.parent is not None:
return self.parent.get(abstract)
return None
+ def get_bound(self, abstract: ABSTRACT) -> Union[INSTANCE, Provider, None]:
+ """
+ get bound of an abstract
+ :return: instance or provider
+ """
+ self._check_destroyed()
+ if abstract in self._instances:
+ return self._instances[abstract]
+ elif abstract in self._providers:
+ return self._providers[abstract]
+ elif abstract in self._aliases:
+ alias = self._aliases[abstract]
+ return alias
+ elif self.parent is not None:
+ return self.parent.get_bound(abstract)
+ return None
+
def register_maker(
self,
- contract: Type[ABSTRACT],
- maker: Callable[[], ABSTRACT],
+ contract: ABSTRACT,
+ maker: Callable[[], INSTANCE],
singleton: bool = False,
):
+ self._check_destroyed()
lineinfo = get_caller_info(2)
def _maker(c):
@@ -206,18 +295,36 @@ def _maker(c):
provider = provide(contract, singleton=singleton, lineinfo=lineinfo)(_maker)
self.register(provider)
- def register(self, provider: Provider) -> None:
+ def register(self, *providers: Provider) -> None:
"""
register factory of the contract by provider
"""
- if isinstance(provider, Bootstrapper):
- # 添加 bootstrapper.
- self.add_bootstrapper(provider)
+ self._check_destroyed()
+ for provider in providers:
+ self._register(provider)
+ def _register(self, provider: Provider) -> None:
contract = provider.contract()
- self._bind_contract(contract)
+ self._add_bound_contract(contract)
+ self._register_provider(contract, provider)
+
+ # additional bindings
+ for alias in provider.aliases():
+ if alias not in self._bound:
+ self._bind_alias(alias, contract)
+ if isinstance(provider, Bootstrapper) and self._bootstrapped:
+ # 添加 bootstrapper.
+ provider.bootstrap(self)
+
+ def _bind_alias(self, alias: Any, contract: Any) -> None:
+ self._aliases[alias] = contract
+ self._bound.add(alias)
+
+ def _register_provider(self, contract: ABSTRACT, provider: Provider) -> None:
+ # remove singleton instance that already bound
if contract in self._instances:
del self._instances[contract]
+ # override the existing one
self._providers[contract] = provider
def add_bootstrapper(self, bootstrapper: Bootstrapper) -> None:
@@ -226,13 +333,16 @@ def add_bootstrapper(self, bootstrapper: Bootstrapper) -> None:
:param bootstrapper: 可以定义一些方法, 比如往容器里的某个类里注册一些工具.
:return:
"""
- self._bootstrapper.append(bootstrapper)
+ self._check_destroyed()
+ if not self._bootstrapped:
+ self._bootstrapper.append(bootstrapper)
- def fetch(self, abstract: Type[ABSTRACT], strict: bool = False) -> Optional[ABSTRACT]:
+ def fetch(self, abstract: Type[INSTANCE], strict: bool = False) -> Optional[INSTANCE]:
"""
get contract with type check
:exception: TypeError if instance do not implement abstract
"""
+ self._check_destroyed()
instance = self.get(abstract)
if instance is not None:
if strict and not isinstance(instance, abstract):
@@ -240,12 +350,25 @@ def fetch(self, abstract: Type[ABSTRACT], strict: bool = False) -> Optional[ABST
return instance
return None
- def force_fetch(self, contract: Type[ABSTRACT], strict: bool = False) -> ABSTRACT:
+ def get_provider(self, abstract: Type[INSTANCE]) -> Optional[Provider[INSTANCE]]:
+ if abstract in self._providers:
+ return self._providers[abstract]
+ if self.parent is not None:
+ return self.parent.get_provider(abstract)
+ return None
+
+ def rebind(self, abstract: Type[INSTANCE]) -> None:
+ provider = self.get_provider(abstract)
+ if provider is not None:
+ self.register(provider)
+
+ def force_fetch(self, contract: Type[INSTANCE], strict: bool = False) -> INSTANCE:
"""
if fetch contract failed, raise error.
:exception: NotImplementedError if contract is not registered.
:exception: TypeError if contract do not implement abstract
"""
+ self._check_destroyed()
ins = self.fetch(contract, strict)
if ins is None:
raise NotImplementedError(f"contract {contract} not register in container")
@@ -255,10 +378,11 @@ def _set_instance(self, abstract: Any, instance: Any) -> None:
"""
设定常量.
"""
- self._bind_contract(abstract)
+ self._add_bound_contract(abstract)
self._instances[abstract] = instance
- def contracts(self, recursively: bool = True) -> Iterable[Type[ABSTRACT]]:
+ def contracts(self, recursively: bool = True) -> Iterable[ABSTRACT]:
+ self._check_destroyed()
done = set()
for contract in self._bound:
done.add(contract)
@@ -269,22 +393,49 @@ def contracts(self, recursively: bool = True) -> Iterable[Type[ABSTRACT]]:
done.add(contract)
yield contract
- def destroy(self) -> None:
+ def providers(self, recursively: bool = True) -> Iterable[Provider]:
+ self._check_destroyed()
+ done = set()
+ for provider in self._providers.values():
+ done.add(provider.contract())
+ yield provider
+ if recursively and self.parent is not None:
+ for provider in self.parent.providers():
+ if provider.contract() not in done:
+ done.add(provider.contract())
+ yield provider
+
+ def _check_destroyed(self) -> None:
+ if self._is_shutdown:
+ raise RuntimeError(f"container {self.bloodline} is called after destroyed")
+
+ def shutdown(self) -> None:
"""
Manually delete the container to prevent memory leaks.
"""
+ if self._is_shutdown:
+ return
+ self._is_shutdown = True
+ for shutdown in self._shutdown:
+ shutdown()
+
+ def __del__(self):
+ self.shutdown()
+ del self._shutdown
del self._instances
del self.parent
del self._providers
del self._bound
del self._bootstrapper
del self._bootstrapped
+ del self._aliases
+ Container.instance_count -= 1
Factory = Callable[[Container], Any]
-class Provider(Generic[ABSTRACT], metaclass=ABCMeta):
+class Provider(Generic[INSTANCE], metaclass=ABCMeta):
@abstractmethod
def singleton(self) -> bool:
@@ -293,21 +444,49 @@ def singleton(self) -> bool:
"""
pass
- @abstractmethod
- def contract(self) -> Type[ABSTRACT]:
+ def inheritable(self) -> bool:
"""
- contract for this provider.
+ if the provider is inheritable to sub container
"""
- pass
+ return not self.singleton()
+
+ def contract(self) -> ABSTRACT:
+ """
+ :return: contract for this provider.
+ override this method to define a contract without get from generic args
+ """
+ return get_contract_type(self.__class__)
+
+ def aliases(self) -> Iterable[ABSTRACT]:
+ """
+ additional contracts that shall bind to this provider if the binding contract is not Bound.
+ """
+ return []
@abstractmethod
- def factory(self, con: Container) -> Optional[ABSTRACT]:
+ def factory(self, con: Container) -> Optional[INSTANCE]:
"""
factory method to generate an instance of the contract.
"""
pass
+def get_contract_type(cls: Type[Provider]) -> ABSTRACT:
+ """
+ get generic INSTANCE type from the instance of the provider.
+ """
+ if "__orig_bases__" in cls.__dict__:
+ orig_bases = getattr(cls, "__orig_bases__")
+ for parent in orig_bases:
+ if get_origin(parent) is not Provider:
+ continue
+ args = get_args(parent)
+ if not args:
+ break
+ return args[0]
+ raise AttributeError("can not get contract type")
+
+
class Bootstrapper(metaclass=ABCMeta):
"""
完成所有的绑定之后, 进行容器之间的初始化.
@@ -318,22 +497,25 @@ def bootstrap(self, container: Container) -> None:
pass
-class BootstrappingProvider(Generic[ABSTRACT], Provider[ABSTRACT], Bootstrapper, metaclass=ABCMeta):
+class BootstrapProvider(Generic[INSTANCE], Provider[INSTANCE], Bootstrapper, metaclass=ABCMeta):
"""
将 bootstrapper 和 Provider 可以融合在一起.
"""
- pass
+ @abstractmethod
+ def contract(self) -> Type[INSTANCE]:
+ pass
-class ProviderAdapter(Provider):
+
+class ProviderAdapter(Generic[INSTANCE], Provider[INSTANCE]):
"""
create a provider without class.
"""
def __init__(
self,
- contract_type: Type[ABSTRACT],
- factory: Callable[[Container], Optional[ABSTRACT]],
+ contract_type: Type[INSTANCE],
+ factory: Callable[[Container], Optional[INSTANCE]],
singleton: bool = True,
lineinfo: str = "",
):
@@ -345,10 +527,10 @@ def __init__(
def singleton(self) -> bool:
return self._singleton
- def contract(self) -> Type[ABSTRACT]:
+ def contract(self) -> Type[INSTANCE]:
return self._contract_type
- def factory(self, con: Container) -> Optional[ABSTRACT]:
+ def factory(self, con: Container) -> Optional[INSTANCE]:
return self._factory(con)
def __repr__(self):
@@ -357,17 +539,20 @@ def __repr__(self):
return f" "
-def get_caller_info(backtrace: int = 1) -> str:
+def get_caller_info(backtrace: int = 1, with_full_file: bool = True) -> str:
stack = inspect.stack()
# 获取调用者的上下文信息
caller_frame_record = stack[backtrace]
frame = caller_frame_record[0]
info = inspect.getframeinfo(frame)
- return f"{info.filename}:{info.lineno}"
+ filename = info.filename
+ if not with_full_file:
+ filename = filename.split("/")[-1]
+ return f"{filename}:{info.lineno}"
def provide(
- abstract: Type[ABSTRACT],
+ abstract: ABSTRACT,
singleton: bool = True,
lineinfo: str = "",
) -> Callable[[Factory], Provider]:
@@ -382,3 +567,43 @@ def wrapper(factory: Factory) -> Provider:
return ProviderAdapter(abstract, factory, singleton, lineinfo=lineinfo)
return wrapper
+
+
+class Contracts:
+ """
+ A contracts validator that both indicate the contract types and validate if they are bound to container
+ """
+
+ def __init__(self, contracts: List[ABSTRACT]):
+ self.contracts = contracts
+
+ def validate(self, container: Container) -> None:
+ for contract in self.contracts:
+ if not container.bound(contract):
+ call_at = get_caller_info(2)
+ raise NotImplementedError(f'Contract {contract} not bound to container: {call_at}')
+
+ def join(self, target: Contracts) -> Contracts:
+ abstracts = set(self.contracts)
+ for c in target.contracts:
+ abstracts.add(c)
+ return Contracts(list(abstracts))
+
+
+__container = Container()
+
+
+def get_container() -> Container:
+ """
+ get global static container
+ """
+ return __container
+
+
+def set_container(container: Container) -> None:
+ """
+ change global static container
+ may cause unexpected behavior.
+ """
+ global __container
+ __container = container
diff --git a/ghostos/contracts/README.md b/ghostos/contracts/README.md
index 07e9c31d..797b92dc 100644
--- a/ghostos/contracts/README.md
+++ b/ghostos/contracts/README.md
@@ -2,4 +2,24 @@
This directory provides basic abstract classes that GhostOS and Ghost depending on.
The implementations shall be wrapped by GhostOS.container.Provider and register to Container.
-So every class depend on them can fetch them from Container.
\ No newline at end of file
+So every class depend on them can fetch them from Container.
+
+`GhostOS` has three layers of library interfaces:
+- Contracts: independent libraries.
+- Ghosts: the interfaces of `GhostOS`, depending on `Contracts`
+- Libraries: the libraries for `GhostOS`'s applications, depending on `GhostOS` and `Contracts` interfaces.
+
+The implementations provided by this project are defined at `ghostos.framework`.
+There are providers (`ghostos.container.Provider`) managing implementations of the library interfaces,
+develop should choose wanted providers and register them to the `IoCContainer`.
+By `IoCContainer` we can switch the implementations without too much pain.
+
+There are at least four level IoCContainer in the `GhostOS`:
+- application container: manage static implementations of `Contracts`.
+- GhostOS container: manage the process level implementations for `GhostOS`.
+- Ghost container: `GhostOS` can manage multiple ghost process concurrently so each Ghost instance has it own container.
+- Moss container: when LLM generate python code within Moss, some in-context temporary bindings are needed, so `MossRuntime` has a container.
+
+Each container is nested from above level container, so they inherit or override parent container's bindings.
+
+> todo: let LLM optimize the content above
\ No newline at end of file
diff --git a/ghostos/contracts/assets.py b/ghostos/contracts/assets.py
new file mode 100644
index 00000000..4664a6dd
--- /dev/null
+++ b/ghostos/contracts/assets.py
@@ -0,0 +1,133 @@
+import os
+from abc import ABC, abstractmethod
+from typing import Optional, Tuple, Union
+from mimetypes import guess_type
+from pydantic import BaseModel, Field
+from ghostos.contracts.storage import Storage
+from ghostos.helpers import uuid, yaml_pretty_dump
+import yaml
+
+
+class FileInfo(BaseModel):
+ fileid: str = Field(default_factory=uuid, description="ID of the file.")
+ filename: str = Field(description="The file name of the file.")
+ description: str = Field(default="", description="The description of the file.")
+ filetype: str = Field(default="", description="The file type of the file.")
+ summary: str = Field(default="", description="The text summary of the file.")
+ url: Optional[str] = Field(default=None, description="The URL of the file.")
+
+ def get_format(self) -> str:
+ return self.filetype.split("/")[-1]
+
+
+class FileAssets(ABC):
+
+ @classmethod
+ def new_fileinfo(
+ cls,
+ fileid: str,
+ filename: str,
+ description: str = "",
+ filetype: Optional[str] = None,
+ ) -> FileInfo:
+ if filetype is None:
+ filetype, _ = guess_type(filename)
+ fileinfo = FileInfo(
+ fileid=fileid,
+ filename=filename,
+ description=description,
+ filetype=filetype,
+ )
+ return fileinfo
+
+ @abstractmethod
+ def save(self, file: FileInfo, binary: Optional[bytes]) -> str:
+ """
+ save file info and binary data to storage
+ :param file: the file info
+ :param binary: binary data of the file. if None, the url must be provided
+ :return: relative file path of the saved file
+ """
+ pass
+
+ @abstractmethod
+ def get_binary(self, filename: str) -> Optional[bytes]:
+ """
+ get binary data of the file
+ :param filename: the relative filename of the file
+ :return: binary data of the file, None if binary data is not available
+ """
+ pass
+
+ @abstractmethod
+ def get_fileinfo(self, fileid: str) -> Optional[FileInfo]:
+ """
+ get file info from storage
+ :param fileid: the file id
+ :return: None if no file info is available
+ """
+ pass
+
+ @abstractmethod
+ def has_binary(self, fileid: str) -> bool:
+ pass
+
+ def get_file_and_binary_by_id(self, fileid: str) -> Tuple[Union[FileInfo, None], Union[bytes, None]]:
+ """
+ get binary data by file id
+ :param fileid: the file info id.
+ :return: file info and binary data, if binary data is None, means the file has url.
+ """
+ file_info = self.get_fileinfo(fileid)
+ if file_info is None:
+ return None, None
+ return file_info, self.get_binary(file_info.filename)
+
+
+class ImageAssets(FileAssets, ABC):
+ pass
+
+
+class AudioAssets(FileAssets, ABC):
+ pass
+
+
+class StorageFileAssets(FileAssets):
+
+ def __init__(self, storage: Storage):
+ self._storage = storage
+
+ @staticmethod
+ def _get_fileinfo_filename(fileid: str) -> str:
+ return f"{fileid}.yml"
+
+ def save(self, file: FileInfo, binary: Optional[bytes]) -> str:
+ if binary is None and file.url is None:
+ raise AttributeError("failed to save image: binary is None and image info is not from url.")
+ fileinfo_filename = self._get_fileinfo_filename(file.fileid)
+ data = file.model_dump(exclude_none=True)
+ content = yaml_pretty_dump(data)
+ self._storage.put(fileinfo_filename, content.encode())
+ if binary:
+ self._storage.put(file.filename, binary)
+ return file.filename
+
+ def get_binary(self, filename: str) -> Optional[bytes]:
+ if self._storage.exists(filename):
+ return self._storage.get(filename)
+ return None
+
+ def has_binary(self, fileid: str) -> bool:
+ fileinfo = self.get_fileinfo(fileid)
+ if fileinfo is None:
+ return False
+ filename = fileinfo.filename
+ return self._storage.exists(filename)
+
+ def get_fileinfo(self, fileid: str) -> Optional[FileInfo]:
+ fileinfo_filename = self._get_fileinfo_filename(fileid)
+ if not self._storage.exists(fileinfo_filename):
+ return None
+ content = self._storage.get(fileinfo_filename)
+ data = yaml.safe_load(content)
+ return FileInfo(**data)
diff --git a/ghostos/contracts/configs.py b/ghostos/contracts/configs.py
index 5857ebb4..7ca79f58 100644
--- a/ghostos/contracts/configs.py
+++ b/ghostos/contracts/configs.py
@@ -1,25 +1,40 @@
-import os
import yaml
from abc import ABC, abstractmethod
-from typing import ClassVar, TypeVar, Type, Dict, Optional
+from typing import ClassVar, TypeVar, Type, Optional
+from typing_extensions import Self
from pydantic import BaseModel
-from ghostos.container import Container, Provider, ABSTRACT
-from ghostos.contracts.storage import Storage
+from ghostos.helpers import generate_import_path
__all__ = ['Config', 'Configs', 'YamlConfig', 'C']
class Config(ABC):
+ """
+ Configuration class for something.
+ Once it implements the Config interface,
+ It could be managed by Configs.
+ """
@classmethod
@abstractmethod
def conf_path(cls) -> str:
+ """
+ relative path to the Configs.
+ notice the Configs is not always based on FileSystem, so the path is also abstract identity of the conf.
+ """
pass
@classmethod
@abstractmethod
- def load(cls, content: str) -> "Config":
+ def unmarshal(cls, content: bytes) -> Self:
+ """
+ unmarshal the Config instance from content.
+ """
+ pass
+
+ def marshal(self) -> bytes:
"""
+ marshal self to a savable content
"""
pass
@@ -28,15 +43,51 @@ def load(cls, content: str) -> "Config":
class Configs(ABC):
+ """
+ Repository for variable kinds of Config.
+ Treat all the configs as a data object
+ todo: we still need save method.
+ """
@abstractmethod
- def get(self, conf_type: Type[C], file_name: Optional[str] = None) -> C:
+ def get(self, conf_type: Type[C], relative_path: Optional[str] = None) -> C:
+ """
+ get a Config instance or throw exception
+ :param conf_type: the Config class that shall unmarshal the config data
+ :param relative_path: the relative path of the config data, if pass, override the conf_type default path.
+ :return: instance of the Config.
+ :exception: FileNotFoundError
+ """
pass
+ @abstractmethod
+ def get_or_create(self, conf: C) -> C:
+ pass
+
+ @abstractmethod
+ def save(self, conf: Config, relative_path: Optional[str] = None) -> None:
+ """
+ save a Config instance to it source.
+ notice some config shall be immutable
+ :param conf: the conf object
+ :param relative_path: if pass, override the conf_type default path.
+ """
+ pass
+
+
+TIP = """
+With object class Config and repository class Configs,
+we decoupled the Model and IO of a Config system.
+
+When defining a Config Model, we never think about where to put it (file system or cloud object storage).
+"""
+
class YamlConfig(Config, BaseModel):
"""
- 通过 yaml 定义的配置.
+ Use Pydantic BaseModel to define a Config class.
+ And marshal the data to yaml.
+ todo: rename to YamlConfigModel
"""
relative_path: ClassVar[str]
@@ -46,7 +97,12 @@ def conf_path(cls) -> str:
return cls.relative_path
@classmethod
- def load(cls, content: str) -> "Config":
+ def unmarshal(cls, content: str) -> "Config":
value = yaml.safe_load(content)
return cls(**value)
+ def marshal(self) -> bytes:
+ value = self.model_dump(exclude_defaults=False)
+ comment = f"# from class: {generate_import_path(self.__class__)}"
+ result = yaml.safe_dump(value)
+ return "\n".join([comment, result]).encode()
diff --git a/ghostos/contracts/documents.py b/ghostos/contracts/documents.py
new file mode 100644
index 00000000..f3f09ec6
--- /dev/null
+++ b/ghostos/contracts/documents.py
@@ -0,0 +1,61 @@
+from typing import List, Iterable
+from typing_extensions import Self
+from abc import ABC, abstractmethod
+from ghostos.identifier import Identical, Identifier
+
+
+class Documents(Identical, ABC):
+
+ @abstractmethod
+ def domain(self) -> str:
+ pass
+
+ @abstractmethod
+ def with_lang(self, lang: str) -> Self:
+ pass
+
+ @abstractmethod
+ def directory(self) -> str:
+ pass
+
+ @abstractmethod
+ def description(self) -> str:
+ pass
+
+ def __identifier__(self) -> Identifier:
+ return Identifier(
+ id=self.directory(),
+ name=self.domain(),
+ description=self.description(),
+ )
+
+ @abstractmethod
+ def default_lang(self) -> str:
+ pass
+
+ @abstractmethod
+ def langs(self) -> List[str]:
+ pass
+
+ @abstractmethod
+ def read(self, filename: str, locale: str = "") -> str:
+ pass
+
+ @abstractmethod
+ def iterate(self, depth: int = -1) -> Iterable[str]:
+ pass
+
+
+class DocumentRegistry(ABC):
+
+ @abstractmethod
+ def get_domain(self, domain: str, lang: str = "") -> Documents:
+ pass
+
+ @abstractmethod
+ def register(self, domain: Documents) -> None:
+ pass
+
+ @abstractmethod
+ def list_domains(self) -> Iterable[Identifier]:
+ pass
diff --git a/ghostos/contracts/logger.py b/ghostos/contracts/logger.py
index 74e3861d..78d52e94 100644
--- a/ghostos/contracts/logger.py
+++ b/ghostos/contracts/logger.py
@@ -1,11 +1,18 @@
-from abc import ABC, abstractmethod
-from logging import LoggerAdapter, Logger, getLogger
-from typing import Union, Dict
+import logging
+from abc import abstractmethod
+from logging.config import dictConfig
+from logging import getLogger, LoggerAdapter, Logger
+from typing import Protocol, Optional, Union
+from os import path
+import yaml
-__all__ = ['LoggerItf', 'LoggerAdapter', 'LoggerType', 'LoggerWrapper']
+__all__ = [
+ 'LoggerItf', 'config_logging', 'get_logger', 'get_console_logger', 'get_debug_logger',
+ 'wrap_logger', 'LoggerAdapter', 'get_ghostos_logger', 'FakeLogger',
+]
-class LoggerItf(ABC):
+class LoggerItf(Protocol):
"""
"""
@@ -29,7 +36,7 @@ def info(self, msg, *args, **kwargs):
To pass exception information, use the keyword argument exc_info with
a true value, e.g.
- logger.info("Houston, we have a %s", "notable problem", exc_info=True)
+ logger.debug("Houston, we have a %s", "notable problem", exc_info=True)
"""
pass
@@ -76,13 +83,6 @@ def critical(self, msg, *args, **kwargs):
"""
pass
- @abstractmethod
- def fatal(self, msg, *args, **kwargs):
- """
- Don't use this method, use critical() instead.
- """
- pass
-
@abstractmethod
def log(self, level, msg, *args, **kwargs):
"""
@@ -95,47 +95,127 @@ def log(self, level, msg, *args, **kwargs):
"""
pass
- @abstractmethod
- def with_trace(self, trace: Dict) -> "LoggerItf":
- pass
-
-LoggerType = Union[LoggerAdapter, Logger]
+def get_logger(name: Optional[str] = None, extra: Optional[dict] = None) -> LoggerItf:
+ return LoggerAdapter(getLogger(name), extra=extra)
-class LoggerWrapper(LoggerItf):
+def wrap_logger(logger: LoggerItf, extra: dict) -> LoggerItf:
+ if isinstance(logger, LoggerAdapter) or isinstance(logger, Logger):
+ return LoggerAdapter(logger, extra)
+ return logger
- def __init__(self, logger: LoggerType):
- self.logger = logger
+def config_logging(conf_path: str) -> None:
+ """
+ configurate logging by yaml config
+ :param conf_path: absolute path of yaml config file
+ """
+ if not path.exists(conf_path):
+ return
+
+ with open(conf_path) as f:
+ content = f.read()
+ data = yaml.safe_load(content)
+ dictConfig(data)
+
+
+def get_console_logger(
+ name: str = "__ghostos_console__",
+ extra: Optional[dict] = None,
+ debug: bool = False,
+) -> LoggerItf:
+ logger = getLogger(name)
+ if not logger.hasHandlers():
+ _console_handler = logging.StreamHandler()
+ _console_formatter = PleshakovFormatter()
+ _console_handler.setFormatter(_console_formatter)
+ logger.addHandler(_console_handler)
+
+ if debug:
+ logger.setLevel(logging.DEBUG)
+ return LoggerAdapter(logger, extra=extra)
+
+
+def get_ghostos_logger(extra: Optional[dict] = None) -> Union[LoggerAdapter, Logger]:
+ logger = getLogger("ghostos")
+ if extra:
+ return LoggerAdapter(logger, extra)
+ return logger
+
+
+def get_debug_logger(
+ name: str = "__ghostos_debug__",
+ extra: Optional[dict] = None,
+) -> LoggerItf:
+ logger = getLogger(name)
+ if not logger.hasHandlers():
+ _debug_file_handler = logging.FileHandler("debug.log", mode="a")
+ formatter = logging.Formatter(
+ fmt="%(asctime)s - %(levelname)s - %(message)s (%(filename)s:%(lineno)d)",
+ )
+ _debug_file_handler.setFormatter(formatter)
+ _debug_file_handler.setLevel(logging.DEBUG)
+ logger.addHandler(_debug_file_handler)
+ logger.setLevel(logging.DEBUG)
+ return LoggerAdapter(logger, extra=extra)
+
+
+class PleshakovFormatter(logging.Formatter):
+ # copy from
+ # https://stackoverflow.com/questions/384076/how-can-i-color-python-logging-output
+ grey = "\x1b[37;20m"
+ yellow = "\x1b[33;20m"
+ red = "\x1b[31;20m"
+ green = "\x1b[32;20m"
+ bold_red = "\x1b[31;1m"
+ reset = "\x1b[0m"
+ format = "%(asctime)s - %(levelname)s - %(message)s (%(filename)s:%(lineno)d)"
+
+ FORMATS = {
+ logging.DEBUG: grey + format + reset,
+ logging.INFO: green + format + reset,
+ logging.WARNING: yellow + format + reset,
+ logging.ERROR: red + format + reset,
+ logging.CRITICAL: bold_red + format + reset
+ }
+
+ def format(self, record):
+ log_fmt = self.FORMATS.get(record.levelno)
+ formatter = logging.Formatter(log_fmt)
+ return formatter.format(record)
+
+
+class FakeLogger(LoggerItf):
def debug(self, msg, *args, **kwargs):
- return self.logger.debug(msg, *args, **kwargs)
+ pass
def info(self, msg, *args, **kwargs):
- return self.logger.info(msg, *args, **kwargs)
+ pass
def warning(self, msg, *args, **kwargs):
- return self.logger.warning(msg, *args, **kwargs)
+ pass
def error(self, msg, *args, **kwargs):
- return self.logger.error(msg, *args, **kwargs)
+ pass
def exception(self, msg, *args, exc_info=True, **kwargs):
- return self.logger.exception(msg, *args, **kwargs)
+ pass
def critical(self, msg, *args, **kwargs):
- return self.logger.critical(msg, *args, **kwargs)
-
- def fatal(self, msg, *args, **kwargs):
- return self.logger.fatal(msg, *args, **kwargs)
+ pass
def log(self, level, msg, *args, **kwargs):
- return self.logger.log(level, msg, *args, **kwargs)
-
- def with_trace(self, trace: Dict) -> "LoggerItf":
- # todo: add trace
- return LoggerWrapper(LoggerAdapter(self.logger, extra=dict(trace=trace)))
+ pass
-def get_logger(logger_name: str) -> LoggerItf:
- return LoggerWrapper(getLogger(logger_name))
+if __name__ == '__main__':
+ get_console_logger().debug("hello world")
+ get_console_logger().info("hello world")
+ get_console_logger().error("hello world")
+ get_console_logger().warning("hello world")
+ get_console_logger().critical("hello world")
+ get_debug_logger().debug("debug")
+ get_debug_logger().info("debug")
+ get_debug_logger().error("debug")
+ get_debug_logger().critical("debug")
diff --git a/ghostos/contracts/modules.py b/ghostos/contracts/modules.py
index ef6c29f1..568b7e7c 100644
--- a/ghostos/contracts/modules.py
+++ b/ghostos/contracts/modules.py
@@ -1,9 +1,10 @@
-from abc import ABC, abstractmethod
+from typing import Optional, Type, Union, List, Iterable, Tuple
from types import ModuleType
+from abc import ABC, abstractmethod
from importlib import import_module
-from typing import Optional, Type
+import pkgutil
-from ghostos.container import Provider, Container, ABSTRACT
+from ghostos.container import Provider, Container
__all__ = [
'Modules', 'ImportWrapper', 'DefaultModules', 'DefaultModulesProvider',
@@ -21,6 +22,14 @@ def import_module(self, modulename) -> ModuleType:
"""
pass
+ @abstractmethod
+ def iter_modules(self, module: Union[str, ModuleType]) -> Iterable[Tuple[str, bool]]:
+ """
+ like pkgutil.iter_modules.
+ :return: Iterable[(module_name, is_package)].
+ """
+ pass
+
class ImportWrapper:
def __init__(self, modules: Modules):
@@ -51,13 +60,27 @@ class DefaultModules(Modules):
def import_module(self, modulename) -> ModuleType:
return import_module(modulename)
+ def iter_modules(self, module: Union[str, ModuleType]) -> Iterable[Tuple[str, bool]]:
+ if isinstance(module, str):
+ module_type = self.import_module(module)
+ elif isinstance(module, ModuleType):
+ module_type = module
+ else:
+ raise ValueError(f'Invalid module type: {type(module)}')
+ prefix = module_type.__name__ + "."
+ if not hasattr(module_type, "__path__"):
+ return []
+ path = module_type.__path__
+ for i, name, is_pkg in pkgutil.iter_modules(path, prefix):
+ yield name, is_pkg
+
-class DefaultModulesProvider(Provider):
+class DefaultModulesProvider(Provider[Modules]):
def singleton(self) -> bool:
return True
- def contract(self) -> Type[ABSTRACT]:
+ def contract(self) -> Type:
return Modules
- def factory(self, con: Container) -> Optional[ABSTRACT]:
+ def factory(self, con: Container) -> Optional[Modules]:
return DefaultModules()
diff --git a/ghostos/contracts/pool.py b/ghostos/contracts/pool.py
index cf6cb133..b0adc1a3 100644
--- a/ghostos/contracts/pool.py
+++ b/ghostos/contracts/pool.py
@@ -1,16 +1,25 @@
from typing import Callable, Optional, Type
+from typing_extensions import Self
from abc import ABC, abstractmethod
-from concurrent.futures import ThreadPoolExecutor
-from ghostos.container import Provider, Container, ABSTRACT
+from concurrent.futures import ThreadPoolExecutor, Future
+from ghostos.container import Provider, Container
class Pool(ABC):
"""
- 建一个全局的池.
+ abstract class for pools like process pool or thread pool
"""
@abstractmethod
- def submit(self, caller: Callable, *args, **kwargs) -> Callable:
+ def submit(self, caller: Callable, *args, **kwargs) -> Future:
+ pass
+
+ @abstractmethod
+ def new(self, size: int) -> Self:
+ """
+ use the same class to create a new pool,
+ or split a quota to create a sub pool.
+ """
pass
@abstractmethod
@@ -23,14 +32,17 @@ def __init__(self, size: int):
self.size = size
self.pool = ThreadPoolExecutor(max_workers=size)
- def submit(self, caller: Callable, *args, **kwargs) -> None:
- self.pool.submit(caller, *args, **kwargs)
+ def submit(self, caller: Callable, *args, **kwargs) -> Future:
+ return self.pool.submit(caller, *args, **kwargs)
+
+ def new(self, size: int) -> Self:
+ return DefaultPool(size)
def shutdown(self, wait=True, *, cancel_futures=False):
self.pool.shutdown(wait=wait, cancel_futures=cancel_futures)
-class DefaultPoolProvider(Provider):
+class DefaultPoolProvider(Provider[Pool]):
def __init__(self, size: int = 100):
self.size = size
@@ -38,8 +50,10 @@ def __init__(self, size: int = 100):
def singleton(self) -> bool:
return True
- def contract(self) -> Type[ABSTRACT]:
+ def contract(self) -> Type:
return Pool
- def factory(self, con: Container) -> Optional[ABSTRACT]:
- return DefaultPool(self.size)
+ def factory(self, con: Container) -> Optional[Pool]:
+ p = DefaultPool(self.size)
+ con.add_shutdown(p.shutdown)
+ return p
diff --git a/ghostos/contracts/storage.py b/ghostos/contracts/storage.py
index d75eed9a..064223aa 100644
--- a/ghostos/contracts/storage.py
+++ b/ghostos/contracts/storage.py
@@ -1,7 +1,7 @@
-from typing import Optional, AnyStr, Iterable
+from typing import Optional, Iterable
from abc import ABC, abstractmethod
-__all__ = ['Storage' ]
+__all__ = ['Storage', 'FileStorage']
class Storage(ABC):
@@ -23,8 +23,16 @@ def get(self, file_path: str) -> bytes:
"""
pass
+ @abstractmethod
+ def remove(self, file_path: str) -> None:
+ pass
+
@abstractmethod
def exists(self, file_path: str) -> bool:
+ """
+ if the object exists
+ :param file_path: file_path or directory path
+ """
pass
@abstractmethod
@@ -46,3 +54,23 @@ def dir(self, prefix_dir: str, recursive: bool, patten: Optional[str] = None) ->
:return: 多个文件路径名.
"""
pass
+
+
+class FileStorage(Storage, ABC):
+ """
+ Storage Based on FileSystem.
+ """
+
+ @abstractmethod
+ def abspath(self) -> str:
+ """
+ storage root directory's absolute path
+ """
+ pass
+
+ @abstractmethod
+ def sub_storage(self, relative_path: str) -> "FileStorage":
+ """
+ FileStorage's sub storage is still FileStorage
+ """
+ pass
diff --git a/ghostos/contracts/translation.py b/ghostos/contracts/translation.py
new file mode 100644
index 00000000..b00f1673
--- /dev/null
+++ b/ghostos/contracts/translation.py
@@ -0,0 +1,82 @@
+from typing import List, Iterable, Dict
+from abc import ABC, abstractmethod
+from pydantic import BaseModel, Field
+
+__all__ = ["Translation", "Translator", "DomainTranslator", "TransItem"]
+
+# deprecated: use gettext instead
+
+
+class TransItem(BaseModel):
+ id: str = Field(description="the target text")
+ description: str = Field(default="", description="the description")
+ translations: Dict[str, str] = Field(
+ default_factory=dict,
+ description="the translations from lang to value"
+ )
+
+ def gettext(self, lang: str, **kwargs: str) -> str:
+ if lang in self.translations:
+ template = self.translations[lang]
+ return template.format(**kwargs)
+ # fallback
+ return self.id
+
+
+class Translator(ABC):
+ """
+ for i18n or l10n translation
+ """
+
+ @abstractmethod
+ def domain(self) -> str:
+ pass
+
+ @abstractmethod
+ def default_lang(self) -> str:
+ pass
+
+ @abstractmethod
+ def gettext(self, message: str, lang: str = "", **kwargs: str) -> str:
+ pass
+
+
+class DomainTranslator(ABC):
+ @abstractmethod
+ def domain(self) -> str:
+ pass
+
+ @abstractmethod
+ def langs(self) -> List[str]:
+ pass
+
+ @abstractmethod
+ def default_lang(self) -> str:
+ pass
+
+ @abstractmethod
+ def get_translator(self, lang: str = "") -> Translator:
+ pass
+
+ @abstractmethod
+ def update(self, lang: str, text: str, value: str):
+ pass
+
+ @abstractmethod
+ def save(self) -> None:
+ pass
+
+ @abstractmethod
+ def items(self) -> Iterable[TransItem]:
+ pass
+
+
+class Translation(ABC):
+ """
+ i18n or l10n translation, can update from user interface
+ todo: use gettext
+ """
+
+ @abstractmethod
+ def get_domain(self, domain: str) -> DomainTranslator:
+ pass
diff --git a/ghostos/contracts/variables.py b/ghostos/contracts/variables.py
new file mode 100644
index 00000000..5902e8d8
--- /dev/null
+++ b/ghostos/contracts/variables.py
@@ -0,0 +1,48 @@
+from typing import Union, TypeVar, Type, Optional, Any
+from abc import ABC, abstractmethod
+from pydantic import BaseModel, Field
+from ghostos.entity import EntityType
+
+T = TypeVar("T", bound=object)
+
+__all__ = ['Variables']
+
+
+class Variables(ABC):
+ """
+ global library that save value to a pointer,
+ by the vid can get the value instance further.
+ """
+
+ class Var(BaseModel):
+ """
+ unique pointer of a variable
+ """
+ vid: str = Field(description="unique variable id")
+ type: str = Field(description="variable class type")
+ desc: str = Field(description="description about the variable")
+
+ @abstractmethod
+ def save(
+ self,
+ val: Union[BaseModel, dict, list, str, int, float, bool, EntityType, Any],
+ desc: str = "",
+ ) -> Var:
+ """
+ save a value and get a Var pointer, with vid can get its instance by load
+ :param val: a value that is serializable, at least can be serialized by pickle
+ :param desc: description about the variable, save to Var pointer
+ :return: the pointer of the value
+ """
+ pass
+
+ @abstractmethod
+ def load(self, vid: str, expect: Optional[Type[T]] = None, force: bool = False) -> Optional[T]:
+ """
+ load a saved value by vid.
+ :param vid: unique variable id in the Var pointer
+ :param expect: if the expect type is given, throw error if value is not fit
+ :param force: if True, raise Error if value is None
+ :return: the unmarshalled value instance
+ """
+ pass
diff --git a/ghostos/core/ghosts/workspace.py b/ghostos/contracts/workspace.py
similarity index 51%
rename from ghostos/core/ghosts/workspace.py
rename to ghostos/contracts/workspace.py
index 0382da1a..ba66777e 100644
--- a/ghostos/core/ghosts/workspace.py
+++ b/ghostos/contracts/workspace.py
@@ -1,5 +1,5 @@
from abc import ABC, abstractmethod
-from ghostos.contracts.storage import Storage
+from ghostos.contracts.storage import FileStorage
class Workspace(ABC):
@@ -8,22 +8,30 @@ class Workspace(ABC):
"""
@abstractmethod
- def runtime(self) -> Storage:
+ def root(self) -> FileStorage:
"""
- runtime that save data by filesystem
+ the root storage of the workspace
"""
pass
@abstractmethod
- def configs(self) -> Storage:
+ def assets(self) -> FileStorage:
+ pass
+
+ @abstractmethod
+ def runtime(self) -> FileStorage:
"""
- config path that configs located
+ runtime that save data by filesystem
"""
pass
@abstractmethod
- def source(self) -> Storage:
+ def runtime_cache(self) -> FileStorage:
+ pass
+
+ @abstractmethod
+ def configs(self) -> FileStorage:
"""
- source code path
+ config path that configs located
"""
pass
diff --git a/ghostos/core/__init__.py b/ghostos/core/__init__.py
index ca47dd46..e69de29b 100644
--- a/ghostos/core/__init__.py
+++ b/ghostos/core/__init__.py
@@ -1 +0,0 @@
-from ghostos.core.ghostos import GhostOS
diff --git a/ghostos/core/aifunc/__init__.py b/ghostos/core/aifunc/__init__.py
index 0d000183..bf840f2c 100644
--- a/ghostos/core/aifunc/__init__.py
+++ b/ghostos/core/aifunc/__init__.py
@@ -1,3 +1,10 @@
from ghostos.core.aifunc.driver import DefaultAIFuncDriverImpl
-from ghostos.core.aifunc.interfaces import AIFunc, AIFuncResult, AIFuncCtx, AIFuncDriver, AIFuncManager
-from ghostos.core.aifunc.manager import DefaultAIFuncManagerImpl, DefaultAIFuncManagerProvider
+from ghostos.core.aifunc.interfaces import (
+ AIFunc, AIFuncResult, AIFuncCtx, AIFuncDriver, AIFuncExecutor,
+ AIFuncRepository,
+ ExecFrame, ExecStep,
+)
+from ghostos.core.aifunc.func import get_aifunc_result_type
+from ghostos.core.aifunc.executor import DefaultAIFuncExecutorImpl, DefaultAIFuncExecutorProvider
+from ghostos.core.aifunc.repository import AIFuncRepoByConfigsProvider, AIFuncRepoByConfigs, AIFuncsConf
+
diff --git a/ghostos/core/aifunc/driver.py b/ghostos/core/aifunc/driver.py
index 65dd23e5..856d96f7 100644
--- a/ghostos/core/aifunc/driver.py
+++ b/ghostos/core/aifunc/driver.py
@@ -1,16 +1,19 @@
import traceback
from typing import Tuple, List, Optional, Any
-from ghostos.core.aifunc.interfaces import AIFuncDriver, AIFuncManager
+from ghostos.core.aifunc.interfaces import (
+ AIFuncDriver, AIFuncExecutor, ExecStep, ExecFrame, AIFuncRepository,
+ TooManyFailureError,
+)
from ghostos.core.aifunc.func import (
AIFunc,
get_aifunc_instruction, get_aifunc_result_type, get_aifunc_pycontext, get_aifunc_llmapi,
)
-from ghostos.core.llms import LLMs, Chat
-from ghostos.core.moss.abc import MossRuntime
-from ghostos.core.session import MsgThread, DefaultEventType, Threads, thread_to_chat
-from ghostos.core.messages import Role, Message
-from ghostos.contracts.logger import LoggerItf
+from ghostos.core.llms import LLMs, Prompt
+from ghostos.core.moss.abcd import MossRuntime
+from ghostos.core.runtime import GoThreadInfo, EventTypes, GoThreads, thread_to_prompt
+from ghostos.core.messages import Role, Message, Stream
+from ghostos.container import Container
__all__ = [
'DefaultAIFuncDriverImpl',
@@ -88,9 +91,15 @@ def __init__(self, fn: AIFunc):
def name(self) -> str:
return self.aifunc.__class__.__name__
- def initialize(self) -> MsgThread:
+ def initialize(self, container: Container, frame: ExecFrame) -> GoThreadInfo:
pycontext = get_aifunc_pycontext(self.aifunc)
messages = []
+ threads = container.get(GoThreads)
+ if threads:
+ thread = threads.get_thread(frame.frame_id, create=False)
+ if thread:
+ return thread
+ # create one for frame
instruction = get_aifunc_instruction(self.aifunc)
if instruction:
system_message = Role.SYSTEM.new(
@@ -98,12 +107,13 @@ def initialize(self) -> MsgThread:
)
messages.append(system_message)
- event = DefaultEventType.INPUT.new(
+ event = EventTypes.ROTATE.new(
task_id="",
from_task_id="",
messages=messages,
)
- thread = MsgThread.new(
+ thread = GoThreadInfo.new(
+ thread_id=frame.frame_id,
event=event,
pycontext=pycontext,
)
@@ -114,7 +124,7 @@ def generate_system_messages(self, runtime: MossRuntime) -> List[Message]:
aifunc_class = aifunc_cls.__name__
aifunc_result_type = get_aifunc_result_type(aifunc_cls)
aifunc_result_class = aifunc_result_type.__name__
- moss_code = runtime.prompter().dump_context_prompt()
+ moss_code = runtime.prompter().dump_module_prompt()
prompt = default_aifunc_prompt(
aifunc_class=aifunc_class,
aifunc_result_class=aifunc_result_class,
@@ -123,18 +133,30 @@ def generate_system_messages(self, runtime: MossRuntime) -> List[Message]:
message = Role.SYSTEM.new(content=prompt)
return [message]
- def on_chat(self, chat: Chat) -> None:
+ def on_chat(self, chat: Prompt) -> None:
pass
- def on_message(self, message: Message) -> None:
- pass
+ def on_message(self, message: Message, step: ExecStep, upstream: Optional[Stream]) -> None:
+ if upstream:
+ message = message.model_copy(deep=True)
+ message.name = self.aifunc.func_name()
+ payload = step.as_payload()
+ payload.set_payload(message)
+ upstream.deliver(message)
def on_system_messages(self, messages: List[Message]) -> None:
pass
- def think(self, manager: AIFuncManager, thread: MsgThread) -> Tuple[MsgThread, Optional[Any], bool]:
- logger = manager.container().get(LoggerItf)
- compiler = manager.compiler()
+ def think(
+ self,
+ manager: AIFuncExecutor,
+ thread: GoThreadInfo,
+ step: ExecStep,
+ upstream: Optional[Stream]
+ ) -> Tuple[GoThreadInfo, Optional[Any], bool]:
+ # get compiler by current exec step
+ # the MossCompiler.container().get(AIFuncCtx) will bind this step.
+ compiler = manager.compiler(step, upstream)
compiler.join_context(thread.get_pycontext())
compiler.bind(self.aifunc.__class__, self.aifunc)
runtime = compiler.compile(None)
@@ -143,80 +165,122 @@ def think(self, manager: AIFuncManager, thread: MsgThread) -> Tuple[MsgThread, O
systems.append(Role.SYSTEM.new(
content=DEFAULT_NOTICES,
))
+
+ # build chat
self.on_system_messages(systems)
- chat = thread_to_chat(thread.id, systems, thread)
- self.on_chat(chat) # Whether you want to send chat to llm, let it generate code for you or not
- # todo: log
- # 实例化 llm api
+ chat = thread_to_prompt(thread.id, systems, thread)
+ step.chat = chat.model_copy(deep=True)
+ # on_chat hook
+ self.on_chat(chat)
+
+ # instance the llms
llms = manager.container().force_fetch(LLMs)
llm_api = get_aifunc_llmapi(self.aifunc, llms)
if llm_api is None:
llm_api = manager.default_llm_api()
- # 调用 llm api
- # logger and logger.info(f"run aifunc with chat :{chat}")
+
+ # call llm api
ai_generation = llm_api.chat_completion(chat)
- # 插入 ai 生成的消息.
+
+ # append ai_generation
thread.append(ai_generation)
- self.on_message(ai_generation) # Whether you want to execute the ai-generated code or not
+ step.generate = ai_generation
+ # on_message hook
+ self.on_message(ai_generation, step, upstream)
+
+ # parse the ai_generation.
code = self.parse_moss_code_in_message(ai_generation)
- # code 相关校验:
+ error = None
+ # handle code:
if not code:
- thread.append(Role.SYSTEM.new(content="Error! You shall only write python code! DO NOT ACT LIKE IN A CHAT"))
- logger.error(f"ai_generation: {repr(ai_generation)}")
- return thread, None, False
- if "main(" not in code:
- thread.append(Role.SYSTEM.new(content="Error! No main function found in your generation!"))
+ error = Role.new_system(
+ content="Error! You shall only write python code! DO NOT ACT LIKE IN A CHAT. "
+ "Generate code in ` 包起来的是内容. """
- TOOL = "tool"
- """ tool mod, 使用 llm tool 进行封装. """
- TOKEN = "token"
- """ token mod. use single token to parse content. """
-
-
-class FunctionalToken(Identifiable, BaseModel):
- """
- functional token means to provide function ability to LLM not by JsonSchema, but by token.
- LLM generates special tokens (such as XML marks) to indicate further tokens are the content of the function.
- LLMDriver shall define which way to prompt the functional token usage such as xml.
- """
-
- token: str = Field(description="token that start the function content output")
- end_token: str = Field(default="", description="end token that close the function content output")
- name: str = Field(description="name of the function")
- description: str = Field(default="", description="description of the function")
- visible: bool = Field(default=False, description="if the functional token and the parameters are visible to user")
- parameters: Optional[Dict] = Field(default=None, description="functional token parameters")
-
- def new_caller(self, arguments: str) -> "Caller":
- """
- generate new caller by functional token, usually used in tests.
- """
- return Caller(
- name=self.name,
- arguments=arguments,
- functional_token=True,
- )
-
- def identifier(self) -> Identifier:
- """
- identifier of the functional token.
- """
- return Identifier(
- name=self.name,
- description=self.description,
- )
-
- def as_tool(self) -> LLMTool:
- """
- all functional token are compatible to a llm tool.
- """
- return LLMTool.new(name=self.name, desc=self.description, parameters=self.parameters)
-
-
-# ---- api objects ---- #
-
-class Chat(BaseModel):
- """
- 模拟对话的上下文.
- """
- id: str = Field(default_factory=helpers.uuid, description="trace id")
-
- system: List[Message] = Field(default_factory=list, description="system messages")
- history: List[Message] = Field(default_factory=list)
- inputs: List[Message] = Field(default_factory=list, description="input messages")
- appending: List[Message] = Field(default_factory=list, description="appending messages")
-
- functions: List[LLMTool] = Field(default_factory=list)
- functional_tokens: List[FunctionalToken] = Field(default_factory=list)
- function_call: Optional[str] = Field(default=None, description="function call")
-
- def system_prompt(self) -> str:
- contents = []
- if self.system:
- contents = []
- for message in self.system:
- contents.append(message.get_content())
- return "\n\n".join(contents)
-
- def get_messages(self) -> List[Message]:
- """
- 返回所有的消息.
- """
- messages = []
- # combine system messages into one
- if self.system:
- system_message = Role.SYSTEM.new(content=self.system_prompt())
- messages.append(system_message)
- if self.history:
- messages.extend(self.history)
- if self.inputs:
- messages.extend(self.inputs)
- if self.appending:
- messages.extend(self.appending)
- results = []
- for message in messages:
- if message.is_empty():
- continue
- results.append(message)
- return results
-
- def filter_messages(self, filter_: Callable[[Message], Optional[Message]]) -> None:
- self.system = self._filter_messages(self.system, filter_)
- self.history = self._filter_messages(self.history, filter_)
- self.inputs = self._filter_messages(self.inputs, filter_)
- self.appending = self._filter_messages(self.appending, filter_)
- return
-
- @staticmethod
- def _filter_messages(
- messages: Iterable[Message], filter_: Callable[[Message], Optional[Message]]
- ) -> List[Message]:
- result = []
- for item in messages:
- item = filter_(item)
- if item is not None:
- result.append(item)
- return result
-
- def get_openai_functions(self) -> Union[List[Function], NotGiven]:
- if not self.functions:
- return NOT_GIVEN
- functions = []
- for func in self.functions:
- if func.id is not None:
- continue
- openai_func = Function(**func.model_dump())
- functions.append(openai_func)
- return functions
-
- def get_openai_tools(self) -> Union[List[ChatCompletionToolParam], NotGiven]:
- if not self.functions:
- return NOT_GIVEN
- tools = []
- for func in self.functions:
- if func.id is None:
- continue
- openai_func = Function(**func.model_dump())
- tool = ChatCompletionToolParam(function=openai_func)
- tools.append(tool)
- return tools
-
- def get_openai_function_call(self) -> Union[FunctionCall, NotGiven]:
- if not self.functions:
- return NOT_GIVEN
- if self.function_call is None:
- return "auto"
- return ChatCompletionFunctionCallOptionParam(name=self.function_call)
-
-
-class ChatPreparer(ABC):
- """
- 用来对 chat message 做加工.
- 基本思路是, 尽可能保证消息体本身的一致性, 在使用的时候才对消息结构做调整.
- """
-
- @abstractmethod
- def prepare_chat(self, chat: Chat) -> Chat:
- pass
-
-
-def prepare_chat(chat: Chat, updater: Iterable[ChatPreparer]) -> Chat:
- """
- 通过多个 filter 来加工 chat.
- """
- for f in updater:
- chat = f.prepare_chat(chat)
- return chat
diff --git a/ghostos/core/llms/configs.py b/ghostos/core/llms/configs.py
index 162b6651..57f23caa 100644
--- a/ghostos/core/llms/configs.py
+++ b/ghostos/core/llms/configs.py
@@ -3,17 +3,20 @@
import os
from typing import List, Dict, Optional, Any, ClassVar
-
from pydantic import BaseModel, Field
from ghostos.core.messages import Payload
+from ghostos.helpers import gettext as _
__all__ = [
- 'ModelConf', 'ServiceConf', 'LLMsConfig', 'OPENAI_DRIVER_NAME',
+ 'ModelConf', 'ServiceConf', 'LLMsConfig',
+ 'OPENAI_DRIVER_NAME', 'LITELLM_DRIVER_NAME',
]
-OPENAI_DRIVER_NAME = "ghostos.llms.openai_driver"
+OPENAI_DRIVER_NAME = "openai_driver"
"""default llm driver name for OpenAI llm message protocol """
+LITELLM_DRIVER_NAME = "lite_llm_Driver"
+
class ModelConf(Payload):
"""
@@ -30,22 +33,27 @@ class ModelConf(Payload):
timeout: float = Field(default=30, description="timeout")
request_timeout: float = Field(default=40, description="request timeout")
kwargs: Dict[str, Any] = Field(default_factory=dict, description="kwargs")
-
-
-class EmbedConf(BaseModel):
- service: str = Field(description="service name, share with llm model conf")
- model: str = Field(description="the model name that provide embeddings")
+ use_tools: bool = Field(default=True, description="use tools")
+ message_types: Optional[List[str]] = Field(None, description="model allow message types")
class ServiceConf(BaseModel):
"""
- The service configuration of a llm.
+ The model api service configuration
"""
+
name: str = Field(description="Service name")
- driver: str = Field(default=OPENAI_DRIVER_NAME, description="the adapter driver name of this service. ")
- base_url: str = Field(description="llm service provider")
- token: str = Field(default="", description="token")
- proxy: Optional[str] = Field(default=None, description="proxy")
+ base_url: str = Field(description="LLM service url")
+ token: str = Field(default="", description="access token. if start with `$`, will read environment variable of it")
+ proxy: Optional[str] = Field(
+ default=None,
+ description="service proxy. if start with `$`, will read environment variable of it",
+ )
+
+ driver: str = Field(
+ default=OPENAI_DRIVER_NAME,
+ description="the adapter driver name of this service. change it only if you know what you are doing",
+ )
def load(self, environ: Optional[Dict] = None) -> None:
self.token = self._load_env(self.token, environ=environ)
@@ -66,14 +74,16 @@ class LLMsConfig(BaseModel):
"""
llms configurations for ghostos.core.llms.llm:LLMs default implementation.
"""
+
services: List[ServiceConf] = Field(
default_factory=list,
- description="define llm services, such as openai or moonshot",
+ description="The Model Services (like OpenAI, Anthropic, Moonshot) configuration.",
)
- default: ModelConf = Field(
- description="define default LLMApi 's model config.",
+
+ default: str = Field(
+ description="GhostOS default model name, corporate with models config",
)
models: Dict[str, ModelConf] = Field(
default_factory=dict,
- description="define llm apis, the key is llm_api_name and value is model config of it.",
+ description="define LLM APIs, from model name to model configuration.",
)
diff --git a/ghostos/core/llms/embedding.py b/ghostos/core/llms/embedding.py
deleted file mode 100644
index 126a893b..00000000
--- a/ghostos/core/llms/embedding.py
+++ /dev/null
@@ -1,29 +0,0 @@
-from typing import List, Optional
-from abc import ABC, abstractmethod
-from pydantic import BaseModel, Field
-
-
-# ---- config ---- #
-
-class Embedding(BaseModel):
- content: str = Field(description="origin content")
- service: str = Field(description="llm service")
- model: str = Field(description="llm model")
- embedding: List[float] = Field(description="embedding")
-
-
-class Embeddings(BaseModel):
- result: List[Embedding] = Field(default_factory=list)
- # todo: 未来再管这些.
- # cast: Cast = Field(description="cast")
-
-
-class EmbedApi(ABC):
-
- @abstractmethod
- def get_embedding(self, content: str, model: Optional[str] = None) -> Embedding:
- pass
-
- @abstractmethod
- def get_embeddings(self, contents: List[str], model: Optional[str] = None) -> Embeddings:
- pass
diff --git a/ghostos/core/llms/prompt.py b/ghostos/core/llms/prompt.py
new file mode 100644
index 00000000..8e7c0fac
--- /dev/null
+++ b/ghostos/core/llms/prompt.py
@@ -0,0 +1,241 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+
+from typing import List, Iterable, Optional, Union, Callable, Self, Set
+from openai.types.chat.completion_create_params import Function, FunctionCall
+from openai import NotGiven, NOT_GIVEN
+from openai.types.chat.chat_completion_function_call_option_param import ChatCompletionFunctionCallOptionParam
+from openai.types.shared_params.function_definition import FunctionDefinition
+from openai.types.chat.chat_completion_tool_param import ChatCompletionToolParam
+
+from pydantic import BaseModel, Field
+from ghostos import helpers
+from ghostos.core.messages import Message, Role, Payload
+from ghostos.helpers import timestamp
+from ghostos.core.llms.configs import ModelConf
+from ghostos.core.llms.tools import LLMFunc, FunctionalToken
+
+__all__ = [
+ 'Prompt', 'PromptPipe',
+ 'run_prompt_pipeline',
+ 'PromptStorage',
+ 'PromptPayload',
+]
+
+
+# ---- api objects ---- #
+
+class Prompt(BaseModel):
+ """
+ 模拟对话的上下文.
+ """
+ id: str = Field(default_factory=helpers.uuid, description="trace id")
+ description: str = Field(default="description of this prompt")
+
+ system: List[Message] = Field(default_factory=list, description="system messages")
+ history: List[Message] = Field(default_factory=list)
+ inputs: List[Message] = Field(default_factory=list, description="input messages")
+ added: List[Message] = Field(default_factory=list, description="appending messages")
+
+ functions: List[LLMFunc] = Field(default_factory=list)
+ function_call: Optional[str] = Field(default=None, description="function call")
+
+ # deprecated
+ functional_tokens: List[FunctionalToken] = Field(default_factory=list)
+
+ # system info
+ error: Optional[str] = Field(default=None, description="error message")
+ created: int = Field(default_factory=timestamp)
+ model: Optional[ModelConf] = Field(default=None, description="model conf")
+ run_start: int = Field(default=0, description="start time")
+ run_end: int = Field(default=0, description="end time")
+
+ def system_prompt(self) -> str:
+ contents = []
+ if self.system:
+ contents = []
+ for message in self.system:
+ contents.append(message.get_content())
+ return "\n\n".join(contents)
+
+ def get_messages(self, with_system: bool = True, stages: Optional[List[str]] = None) -> List[Message]:
+ """
+ 返回所有的消息.
+ """
+ messages = []
+ if stages:
+ stage_set = set(stages)
+ else:
+ stage_set = set()
+
+ # combine system messages into one
+ if with_system and self.system:
+ system_message = Role.SYSTEM.new(content=self.system_prompt())
+ messages = join_messages_by_stages(messages, stage_set, system_message)
+ if self.history:
+ messages = join_messages_by_stages(messages, stage_set, *self.history)
+ if self.inputs:
+ messages = join_messages_by_stages(messages, stage_set, *self.inputs)
+ if self.added:
+ messages = join_messages_by_stages(messages, stage_set, *self.added)
+ return messages
+
+ def filter_messages(self, filter_: Callable[[Message], Optional[Message]]) -> None:
+ self.system = self._filter_messages(self.system, filter_)
+ self.history = self._filter_messages(self.history, filter_)
+ self.inputs = self._filter_messages(self.inputs, filter_)
+ self.added = self._filter_messages(self.added, filter_)
+ return
+
+ @staticmethod
+ def _filter_messages(
+ messages: Iterable[Message], filter_: Callable[[Message], Optional[Message]]
+ ) -> List[Message]:
+ result = []
+ for item in messages:
+ item = filter_(item)
+ if item is not None:
+ result.append(item)
+ return result
+
+ def get_openai_functions(self) -> Union[List[Function], NotGiven]:
+ if not self.functions:
+ return NOT_GIVEN
+ functions = []
+ for func in self.functions:
+ openai_func = Function(
+ name=func.name,
+ description=func.description,
+ parameters=func.parameters,
+ )
+ functions.append(openai_func)
+ if not functions:
+ return NOT_GIVEN
+ return functions
+
+ def get_openai_tools(self) -> Union[List[ChatCompletionToolParam], NotGiven]:
+ if not self.functions:
+ return NOT_GIVEN
+ tools = []
+ for func in self.functions:
+ openai_func = FunctionDefinition(
+ name=func.name,
+ description=func.description,
+ parameters=func.parameters,
+ )
+ tool = ChatCompletionToolParam(function=openai_func, type="function")
+ tools.append(tool)
+ if not tools:
+ return NOT_GIVEN
+ return tools
+
+ def get_openai_function_call(self) -> Union[FunctionCall, NotGiven]:
+ if not self.functions or self.model.use_tools:
+ return NOT_GIVEN
+ if self.function_call is None:
+ return "auto"
+ return ChatCompletionFunctionCallOptionParam(name=self.function_call)
+
+ def add(self, messages: Iterable[Message]) -> Iterable[Message]:
+ for msg in messages:
+ if msg.is_complete():
+ self.added.append(msg.get_copy())
+ yield msg
+
+ def filter_stages(self, stages: Optional[List[str]] = None) -> Self:
+ if not stages:
+ return self
+ stages = set(stages)
+ copied = self.model_copy(deep=True)
+ if stages:
+ copied.history = join_messages_by_stages([], stages, *copied.history)
+ copied.inputs = join_messages_by_stages([], stages, *copied.inputs)
+ copied.added = join_messages_by_stages([], stages, *copied.added)
+ return copied
+
+ def fork(
+ self,
+ inputs: Optional[List[Message]],
+ *,
+ system: Optional[List[Message]] = None,
+ description: str = "",
+ prompt_id: Optional[str] = None,
+ functions: Optional[List[Function]] = None,
+ function_call: Optional[str] = None,
+ stages: Optional[List[str]] = None,
+ ) -> Prompt:
+ """
+ fork current prompt.
+ """
+ prompt_id = prompt_id or helpers.uuid()
+ description = description
+ copied = self.filter_stages(stages)
+ copied.id = prompt_id
+ copied.description = description
+ if inputs is not None:
+ copied.history.extend(copied.inputs)
+ copied.history.extend(copied.added)
+ copied.inputs = inputs
+ copied.added = []
+ if system:
+ copied.system = system
+ if functions:
+ copied.functions = functions
+ if function_call is not None:
+ copied.function_call = function_call
+ return copied
+
+
+class PromptPayload(Payload):
+ key = "prompt_info"
+
+ prompt_id: str = Field(description="created from prompt")
+ desc: str = Field(default="description of the prompt")
+
+ @classmethod
+ def from_prompt(cls, prompt: Prompt) -> Self:
+ return cls(prompt_id=prompt.id, desc=prompt.description)
+
+
+class PromptPipe(ABC):
+ """
+ 用来对 chat message 做加工.
+ 基本思路是, 尽可能保证消息体本身的一致性, 在使用的时候才对消息结构做调整.
+ """
+
+ @abstractmethod
+ def update_prompt(self, prompt: Prompt) -> Prompt:
+ pass
+
+
+def run_prompt_pipeline(prompt: Prompt, pipeline: Iterable[PromptPipe]) -> Prompt:
+ """
+ 通过多个 filter 来加工 chat.
+ """
+ for f in pipeline:
+ prompt = f.update_prompt(prompt)
+ return prompt
+
+
+def join_messages_by_stages(messages: List[Message], stages: Set[str], *join: Message) -> List[Message]:
+ for msg in join:
+ if msg.is_empty() or not msg.is_complete():
+ continue
+ if not stages or msg.stage in stages:
+ messages.append(msg)
+ return messages
+
+
+class PromptStorage(ABC):
+ """
+ save and get prompt
+ """
+
+ @abstractmethod
+ def save(self, prompt: Prompt) -> None:
+ pass
+
+ @abstractmethod
+ def get(self, prompt_id: str) -> Optional[Prompt]:
+ pass
diff --git a/ghostos/core/llms/prompt_pipes.py b/ghostos/core/llms/prompt_pipes.py
new file mode 100644
index 00000000..060c915c
--- /dev/null
+++ b/ghostos/core/llms/prompt_pipes.py
@@ -0,0 +1,29 @@
+from typing import Optional
+from ghostos.core.messages import Message, Role
+from ghostos.core.llms import PromptPipe, Prompt
+
+__all__ = ['AssistantNamePipe']
+
+
+class AssistantNamePipe(PromptPipe):
+ """
+ 调整 assistant name, 如果一条 assistant 消息的 name 与当前 name 相同则去掉.
+ 这样就会认为是自己的消息.
+ """
+
+ def __init__(self, assistant_name: str):
+ self._assistant_name = assistant_name
+
+ def update_prompt(self, prompt: Prompt) -> Prompt:
+ def filter_fn(message: Message) -> Optional[Message]:
+ if message.role != Role.ASSISTANT.value:
+ return message
+
+ copy = message
+ if message.name == self._assistant_name:
+ copy = message.get_copy()
+ copy.name = ""
+ return copy
+
+ prompt.filter_messages(filter_fn)
+ return prompt
diff --git a/ghostos/core/llms/tools.py b/ghostos/core/llms/tools.py
new file mode 100644
index 00000000..fbcba40d
--- /dev/null
+++ b/ghostos/core/llms/tools.py
@@ -0,0 +1,104 @@
+from __future__ import annotations
+
+from enum import Enum
+
+from typing import Dict, Optional, Type
+
+from pydantic import BaseModel, Field
+from ghostos.identifier import Identical, Identifier
+from ghostos.core.messages import FunctionCaller
+
+
+# ---- tool and function ---- #
+
+class LLMFunc(BaseModel):
+ """
+ a common wrapper for JSONSchema LLM tool.
+ Compatible to OpenAI Tool.
+ We need this because OpenAI Tool definition is too dynamic, we need strong typehints.
+ """
+ id: Optional[str] = Field(default=None, description="The id of the LLM tool.")
+ name: str = Field(description="function name")
+ description: str = Field(default="", description="function description")
+ parameters: Optional[Dict] = Field(default=None, description="function parameters")
+
+ @classmethod
+ def new(cls, name: str, desc: Optional[str] = None, parameters: Optional[Dict] = None):
+ if parameters is None:
+ parameters = {"type": "object", "properties": {}}
+ properties = parameters.get("properties", {})
+ params_properties = {}
+ for key in properties:
+ _property = properties[key]
+ if "title" in _property:
+ del _property["title"]
+ params_properties[key] = _property
+ parameters["properties"] = params_properties
+ if "title" in parameters:
+ del parameters["title"]
+ return cls(name=name, description=desc, parameters=parameters)
+
+ def to_dict(self) -> dict:
+ return self.model_dump(exclude_defaults=True, exclude_none=True)
+
+ @classmethod
+ def from_model(
+ cls,
+ name: str,
+ model: Type[BaseModel],
+ description: Optional[str] = None,
+ ):
+ if description is None:
+ description = model.__doc__
+ return cls.new(name, desc=description, parameters=model.model_json_schema())
+
+
+# todo: remove
+
+class FunctionalTokenMode(str, Enum):
+ XML = "xml"
+ """ xml 模式, 使用 包起来的是内容. """
+ TOOL = "tool"
+ """ tool mod, 使用 llm tool 进行封装. """
+ TOKEN = "token"
+ """ token mod. use single token to parse content. """
+
+
+class FunctionalToken(Identical, BaseModel):
+ """
+ functional token means to provide function ability to LLM not by JsonSchema, but by token.
+ LLM generates special tokens (such as XML marks) to indicate further tokens are the content of the function.
+ LLMDriver shall define which way to prompt the functional token usage such as xml.
+ """
+
+ token: str = Field(description="token that start the function content output")
+ end_token: str = Field(default="", description="end token that close the function content output")
+ name: str = Field(description="name of the function")
+ description: str = Field(default="", description="description of the function")
+ visible: bool = Field(default=False, description="if the functional token and the parameters are visible to user")
+ parameters: Optional[Dict] = Field(default=None, description="functional token parameters")
+
+ def new_caller(self, arguments: str) -> "FunctionCaller":
+ """
+ generate new caller by functional token, usually used in tests.
+ """
+ return FunctionCaller(
+ name=self.name,
+ arguments=arguments,
+ functional_token=True,
+ )
+
+ def __identifier__(self) -> Identifier:
+ """
+ identifier of the functional token.
+ """
+ return Identifier(
+ name=self.name,
+ description=self.description,
+ )
+
+ def as_tool(self) -> LLMFunc:
+ """
+ all functional token are compatible to a llm tool.
+ """
+ return LLMFunc.new(name=self.name, desc=self.description, parameters=self.parameters)
diff --git a/ghostos/core/messages/__init__.py b/ghostos/core/messages/__init__.py
index e52fd98e..5372e912 100644
--- a/ghostos/core/messages/__init__.py
+++ b/ghostos/core/messages/__init__.py
@@ -1,12 +1,20 @@
from ghostos.core.messages.message import (
- Message, Role, DefaultMessageTypes,
- Caller, Payload, PayloadItem, Attachment,
- MessageClass, MessageKind, MessageKindParser,
+ Message, Role, MessageType,
+ FunctionCaller, FunctionOutput,
+ MessageClass, MessageKind,
+ MessageClassesParser,
)
+from ghostos.core.messages.message_classes import (
+ MessageKindParser,
+ VariableMessage, ImageAssetMessage, AudioMessage, FunctionCallMessage, FunctionCallOutputMessage,
+
+)
+from ghostos.core.messages.payload import Payload
from ghostos.core.messages.openai import (
OpenAIMessageParser, DefaultOpenAIMessageParser, DefaultOpenAIParserProvider,
CompletionUsagePayload,
)
from ghostos.core.messages.buffers import Buffer, Flushed
-from ghostos.core.messages.helpers import copy_messages
-from ghostos.core.messages.stream import Stream
+from ghostos.core.messages.utils import copy_messages
+from ghostos.core.messages.transport import Stream, Receiver, new_basic_connection, ReceiverBuffer
+from ghostos.core.messages.pipeline import SequencePipe
diff --git a/ghostos/core/messages/buffers.py b/ghostos/core/messages/buffers.py
index ff123128..186ca9d0 100644
--- a/ghostos/core/messages/buffers.py
+++ b/ghostos/core/messages/buffers.py
@@ -1,7 +1,7 @@
from typing import Iterable, List, NamedTuple
from abc import ABC, abstractmethod
-from ghostos.core.messages.message import Message, Caller
+from ghostos.core.messages.message import Message, FunctionCaller
__all__ = [
"Flushed", "Buffer",
@@ -10,39 +10,36 @@
class Flushed(NamedTuple):
unsent: Iterable[Message]
- """ buffer 尚未发送, 需要继续发送出去的包"""
+ """ the unsent messages or chunks, which were buffed"""
messages: List[Message]
- """经过 buff, 生成的包"""
+ """all the patched complete messages"""
- callers: List[Caller]
- """消息体产生的回调方法."""
+ callers: List[FunctionCaller]
+ """all the callers that delivered"""
class Buffer(ABC):
"""
- 在流式传输中拦截 message 的拦截器. 同时要能完成粘包, 返回粘包后的结果.
+ a container to buff streaming Message chunks,
+ and patched all the chunks,
+ return complete patched messages after flushed.
+ 在流式传输中拦截 message 的拦截器. 同时要能完成粘包, 最终返回粘包后的结果.
"""
@abstractmethod
- def match(self, message: Message) -> bool:
+ def add(self, pack: "Message") -> Iterable[Message]:
"""
- 匹配一个消息体.
+ try to buff a message pack
+ :return: the sending messages after the buffing. may be:
+ 1. the input pack, which need not be buffered
+ 2. the unsent packs, which are replaced by new buffing pack.
"""
pass
@abstractmethod
- def buff(self, pack: "Message") -> List[Message]:
+ def flush(self) -> Flushed:
"""
- buff 一个消息体, 然后决定是否对外发送.
- 不能用 Iterable 返回, 如果上层不处理, 就会导致没有 buff.
+ flush the buffered messages, and reset itself.
"""
pass
-
- @abstractmethod
- def new(self) -> "Buffer":
- pass
-
- @abstractmethod
- def flush(self) -> Flushed:
- pass
diff --git a/ghostos/core/messages/helpers.py b/ghostos/core/messages/helpers.py
deleted file mode 100644
index d8a4a737..00000000
--- a/ghostos/core/messages/helpers.py
+++ /dev/null
@@ -1,13 +0,0 @@
-from typing import Iterable, List
-from ghostos.core.messages.message import Message
-
-__all__ = [
- 'copy_messages',
-]
-
-
-def copy_messages(messages: Iterable[Message]) -> List[Message]:
- result = []
- for message in messages:
- result.append(message.model_copy(deep=True))
- return result
diff --git a/ghostos/core/messages/message.py b/ghostos/core/messages/message.py
index 70375050..826487ca 100644
--- a/ghostos/core/messages/message.py
+++ b/ghostos/core/messages/message.py
@@ -1,40 +1,47 @@
+from __future__ import annotations
import enum
import time
-from typing import Optional, Dict, Set, Iterable, Union, List, ClassVar
+from datetime import datetime
+from typing import Optional, Dict, Set, Iterable, Union, List, Any, ClassVar, Type
+from typing_extensions import Self, Literal
from abc import ABC, abstractmethod
from pydantic import BaseModel, Field
from ghostos.helpers import uuid
+from ghostos.container import Container
+from ghostos.entity import EntityType
+from copy import deepcopy
__all__ = [
- "Message", "Role", "DefaultMessageTypes",
- "MessageClass",
- "MessageKind", "MessageKindParser",
- "Payload", "PayloadItem", "Attachment", "Caller",
+ "Message", "Role", "MessageType",
+ "MessageClass", "MessageClassesParser",
+ "MessageKind",
+ "FunctionCaller", "FunctionOutput",
]
+SeqType = Literal["head", "chunk", "complete"]
+
class Role(str, enum.Enum):
"""
消息体的角色, 对齐了 OpenAI
"""
+ UNKNOWN = ""
USER = "user"
ASSISTANT = "assistant"
SYSTEM = "system"
- FUNCTION = "function"
- TOOL = "tool"
@classmethod
def all(cls) -> Set[str]:
return set(map(lambda x: x.value, cls))
@classmethod
- def new_assistant_system(
+ def new_system(
cls,
content: str,
memory: Optional[str] = None,
):
- return cls.ASSISTANT.new(content, memory=memory, name="__system__")
+ return cls.SYSTEM.new(content, memory=memory)
def new(
self,
@@ -44,7 +51,7 @@ def new(
type_: Optional[str] = None,
) -> "Message":
return Message.new_tail(
- type_=type_ if type_ else DefaultMessageTypes.DEFAULT.value,
+ type_=type_ if type_ else MessageType.DEFAULT.value,
role=self.value,
name=name,
content=content,
@@ -52,23 +59,49 @@ def new(
)
-class DefaultMessageTypes(str, enum.Enum):
+class MessageType(str, enum.Enum):
DEFAULT = ""
- CHAT_COMPLETION = "chat_completion"
+ TEXT = "text"
+ VARIABLE = "variable"
+ FUNCTION_CALL = "function_call"
+ FUNCTION_OUTPUT = "function_output"
+ AUDIO = "audio"
+ IMAGE = "image"
+ VIDEO = "video"
+ FILE = "file"
ERROR = "error"
FINAL = "final"
def new(
self, *,
- content: str, role: str = Role.ASSISTANT.value, memory: Optional[str] = None, name: Optional[str] = None,
+ content: str,
+ role: str = Role.ASSISTANT.value,
+ memory: Optional[str] = None,
+ name: Optional[str] = None,
+ msg_id: Optional[str] = None,
+ call_id: Optional[str] = None,
) -> "Message":
- return Message(content=content, memory=memory, name=name, type=self.value, role=role)
+ return Message(
+ msg_id=msg_id or "",
+ content=content, memory=memory, name=name, type=self.value, role=role,
+ call_id=call_id,
+ ).as_tail(copy=False)
def new_assistant(
- self, *,
- content: str, memory: Optional[str] = None, name: Optional[str] = None,
+ self,
+ *,
+ content: str,
+ memory: Optional[str] = None,
+ name: Optional[str] = None,
+ msg_id: Optional[str] = None,
):
- return self.new(content=content, role=Role.ASSISTANT.value, memory=memory, name=name)
+ return self.new(
+ content=content,
+ role=Role.ASSISTANT.value,
+ memory=memory,
+ name=name,
+ msg_id=msg_id or None,
+ )
def new_system(
self, *,
@@ -87,132 +120,116 @@ def new_user(
):
return self.new(content=content, role=Role.USER.value, memory=memory, name=name)
- def match(self, message: "Message") -> bool:
- return message.type == self.value
-
@classmethod
def final(cls):
- return Message(type=cls.FINAL.value, role=Role.ASSISTANT.value)
+ return Message(type=cls.FINAL.value, role="").as_tail()
+
+ def match(self, message: "Message") -> bool:
+ return message.type == self.value
@classmethod
def is_final(cls, pack: "Message") -> bool:
return pack.type == cls.FINAL.value
@classmethod
- def is_protocol_type(cls, message: "Message"):
- return not message.pack and message.type in {cls.ERROR, cls.FINAL}
-
-
-class Caller(BaseModel):
- """
- 消息协议中用来描述一个工具或者function 的调用请求.
- """
- id: Optional[str] = Field(default=None, description="caller 的 id, 用来 match openai 的 tool call 协议. ")
- name: str = Field(description="方法的名字.")
- arguments: str = Field(description="方法的参数. ")
- functional_token: bool = Field(default=False, description="caller 是否是基于协议生成的?")
-
- def add(self, message: "Message") -> None:
- message.callers.append(self)
-
-
-class Payload(BaseModel, ABC):
- """
- 消息体的可扩展的部分. 拥有强类型设计.
- """
- key: ClassVar[str]
+ def is_text(cls, message: Message) -> bool:
+ return message.type == cls.TEXT.value or message.type == cls.DEFAULT.value
@classmethod
- def read(cls, message: "Message") -> Optional["Payload"]:
- value = message.payloads.get(cls.key, None)
- if value is None:
- return None
- return cls(**value)
-
- def set(self, message: "Message") -> None:
- message.payloads[self.key] = self.model_dump()
-
- def exists(self, message: "Message") -> bool:
- return self.key in message.payloads
-
-
-class PayloadItem(Payload, ABC):
- """
- 自身可以粘包的特殊 payload.
- 比如 tokens 的计数.
- """
-
- @abstractmethod
- def join(self, payload: "PayloadItem") -> "PayloadItem":
- pass
-
- def set(self, message: "Message") -> None:
- exists = message.payloads.get(self.key, None)
- if exists is not None:
- join = self.__class__(**exists)
- payload = self.join(join)
- payload.set(message)
- return
- super().set(message)
-
-
-class Attachment(BaseModel, ABC):
- """
- 消息上可以追加的附件.
- """
- key: ClassVar[str]
+ def is_protocol_message(cls, message: Optional["Message"]) -> bool:
+ if message is None:
+ return True
+ return cls.is_protocol_type(message.type)
@classmethod
- def read(cls, message: "Message") -> Optional[List["Attachment"]]:
- value = message.attachments.get(cls.key, None)
- if not value:
- return None
- result = []
- for item in value:
- result.append(cls(**item))
- return result
-
- def add(self, message: "Message") -> None:
- values = message.attachments.get(self.key)
- if values is None:
- values = []
- values.append(self.model_dump())
- message.attachments[self.key] = values
-
-
-# 消息体的容器. 通用的抽象设计, 设计思路:
-# 1. message 可以是一个完整的消息, 也可以是一个包, 用 pack 字段做区分. 支持 dict 传输, dict 传输时不包含默认值.
-# 2. 完整的 message 需要有 msg_id, 但包可以没有.
-# 3. content 是对客户端展示用的消息体, 而 memory 是对大模型展示的消息体. 两者可能不一样.
-# 4. message 可以有强类型字段, 比如 images, 但通过 attachments (累加) 和 payload (替代) 来定义. Message 容器里放弱类型的 dict.
-# 5. type 字段用来提示 message 拥有的信息. 比如 images 消息, 会包含 images payload, 但同时也会指定 type. 这样方便解析时预判.
-# 6. 所有的 message 都需要能转换成模型的协议, 默认要对齐 openai 的协议.
-# 7. openai 协议中的 tool, function_call 统一成 caller 抽象, 通过 caller.id 来做区分.
-# 8. 流式传输中, 可以有首包和尾包. 首包期待包含全部的 payloads 和 attachments. 间包则可选. 尾包是完整的消息体.
+ def is_protocol_type(cls, value: str) -> bool:
+ return value in {cls.ERROR, cls.FINAL}
+
+
+# the Message class is a container for every kind of message and it's chunks.
+# I need this container because:
+# 1. I hate weak-type container of message, countless type checking and adapting
+# 2. I have not found a community-accepted message protocol for Ai Model messages.
+# So I developed this wheel, may be a bad move, but happy to replace it with a mature library someday.
+#
+# 这个消息类是各种消息类型的一个通用容器.
+# 我需要一个这样的容器是因为:
+# 1. 讨厌弱类型消息, 需要做无数的校验和适配, 缺乏规则. 比如 OpenAI 的那个极其复杂的 dict.
+# 2. 我没找到一个社区广泛使用的标准消息协议.
+# 所以重复造了这个轮子, 如果未来发现了成熟的库, 要果断取代掉它. 为此全链路对 Message 的依赖要控制好.
+# 把 Message 用于创建消息的地方, 很难修改. 但它作为传输时的 item, 是可以替代的.
+#
+# the basic logic of this container:
+# 1. Message instance could be a complete message, or a chunk.
+# 2. I can parse Message to dict/json/serialized data, and unpack a Message from them.
+# the complete Message instance must have msg_id for tracking, but the chunk does not.
+# 3. I need a message has a default protocol to show it to User/Agent differently.
+# so this container has two field, content(to user) and memory (to llm).
+# 4. the basic information of message are strong typed, but dynamic payloads or attachments have a certain way to parse.
+# 5. both client side and server side can define it own parser with message type.
+# 6. each type of message can either be parsed to LLM Message (like OpenAI Message), or ignore.
+# 7. define a common action caller for LLM, compatible for JSONSchema Tool, function call or FunctionalTokens.
+# 8. the streaming chunks always have a head package (introduce following chunks),
+# and a tail package (the complete message).
+#
+# 基本设计逻辑:
+# 1. Message 既可以是一个完整的消息, 也可以是一个间包. 它们通常有相同的结构.
+# 2. 可以用 dict/json/别的序列化协议 传输它, 也可以从这些协议反解. 因此用了 pydantic.
+# 完整的消息体必须有 msg_id, 但中间包不需要它.
+# 3. 消息对客户端和 AI 模型的展示方式可以不一样. 所以有 content 和 memory 字段的区分.
+# 4. 消息的基础信息是强类型的, 那些动态类型的信息可以通过确定的方式反解.
+# 5. 客户端和服务端可以根据需要, 定义自己的消息转义协议.
+# 6. 所有的完整消息要么能被解析成模型的消息, 要么就应该忽略它. 避免展示加工不了的.
+# 7. 用一个 caller 兼容各种模型的 action caller.
+# 8. 流式传输的消息包, 应该有 首包 / 间包 / 尾包. 尾包是一个粘包后的完整包.
+# todo: openai 的 realtime api 协议比较整齐, 应该考虑用这个思路重构. 需要考虑几点:
+# todo: 1. 传输协议和存储协议分开.
+# todo: 2. 传输用弱类型.
+# todo: 3. delta 用于流式传输, content part 用来解决富文本, item 解决消息体.
class Message(BaseModel):
- """标准的消息体."""
+ """ message protocol """
- msg_id: str = Field(default="", description="消息的全局唯一 id. ")
- ref_id: Optional[str] = Field(default=None, description="消息的关联目标. 如果 role 是 tool, 则这个是 tool_call_id")
- type: str = Field(default="", description="消息类型是对 payload 的约定. 默认的 type就是 text.")
- created: float = Field(default=0.0, description="Message creation time")
- pack: bool = Field(default=True, description="Message reset time")
+ msg_id: str = Field(default="", description="unique message id. ")
+ call_id: Optional[str] = Field(default=None, description="the call id message id.")
+ index: Optional[int] = Field(default=None, description="the index of the message.")
+ type: str = Field(default="", description="default message type, if empty, means text")
+ stage: str = Field(default="", description="message stage")
- role: str = Field(default=Role.ASSISTANT.value, description="Message role", enum=Role.all())
+ role: str = Field(default="", description="Message role", enum=Role.all())
name: Optional[str] = Field(default=None, description="Message sender name")
+ content: Optional[str] = Field(
+ default=None,
+ description="Message content that for client side. empty means it shall not be showed",
+ )
+
+ # todo: remove memory, use stage instead.
+ memory: Optional[str] = Field(
+ default=None,
+ description="Message memory that for llm, if none, means content is memory",
+ )
- content: Optional[str] = Field(default=None, description="Message content")
- memory: Optional[str] = Field(default=None, description="Message memory")
+ attrs: Optional[Dict[str, Any]] = Field(
+ None,
+ description="the additional attrs for the message type"
+ )
- # --- attachments --- #
+ payloads: Dict[str, Dict] = Field(
+ default_factory=dict,
+ description="payload type key to payload item. payload shall be a strong-typed dict"
+ )
- payloads: Dict[str, Dict] = Field(default_factory=dict, description="k/v 结构的强类型参数.")
- attachments: Dict[str, List[Dict]] = Field(default_factory=dict, description="k/list[v] 类型的强类型参数.")
+ callers: List[FunctionCaller] = Field(
+ default_factory=list,
+ description="the callers parsed in a complete message."
+ )
- callers: List[Caller] = Field(default_factory=list, description="将 callers 作为一种单独的类型. ")
+ # chunk_count: int = Field(default=0, description="how many chunks of this complete message")
+ # time_cast: float = Field(default=0.0, description="from first chunk to tail message.")
- pack_count: int = Field(default=0, description="pack count")
- time_cast: float = Field(default=0.0, description="from first pack to last pack")
+ seq: SeqType = Field(default="chunk", description="sequence type in streaming")
+ created: float = Field(default=0.0, description="time when message was created")
+
+ __attachment__: Optional[Any] = None
@classmethod
def new_head(
@@ -223,18 +240,37 @@ def new_head(
memory: Optional[str] = None,
name: Optional[str] = None,
msg_id: Optional[str] = None,
- ref_id: Optional[str] = None,
- created: int = 0,
+ call_id: Optional[str] = None,
):
+ """
+ create a head chunk message
+ :param role:
+ :param typ_:
+ :param content:
+ :param memory:
+ :param name:
+ :param msg_id:
+ :param call_id:
+ # :param created:
+ :return:
+ """
if msg_id is None:
msg_id = uuid()
- if created <= 0:
- created = round(time.time(), 4)
+ created = round(time.time(), 3)
+ if isinstance(role, Role):
+ role = role.value
+ if isinstance(typ_, MessageType):
+ typ_ = typ_.value
return cls(
- role=role, name=name, content=content, memory=memory, pack=True,
+ role=role,
+ name=name,
+ content=content,
+ memory=memory,
+ seq="head",
type=typ_,
- ref_id=ref_id,
- msg_id=msg_id, created=created,
+ call_id=call_id,
+ msg_id=msg_id,
+ created=created,
)
@classmethod
@@ -246,71 +282,130 @@ def new_tail(
memory: Optional[str] = None,
name: Optional[str] = None,
msg_id: Optional[str] = None,
- ref_id: Optional[str] = None,
- created: int = 0,
+ # todo: change to call id
+ call_id: Optional[str] = None,
+ attrs: Optional[Dict[str, Any]] = None,
):
+ """
+ create a tail message, is the complete message of chunks.
+ :param type_:
+ :param role:
+ :param content:
+ :param memory:
+ :param name:
+ :param msg_id:
+ :param call_id:
+ :param attrs:
+ :return:
+ """
msg = cls.new_head(
- role=role, name=name, content=content, memory=memory,
+ role=role,
+ name=name,
+ content=content,
+ memory=memory,
typ_=type_,
msg_id=msg_id,
- ref_id=ref_id,
- created=created,
+ call_id=call_id,
)
- msg.pack = False
+ msg.seq = "complete"
+ msg.attrs = attrs
return msg
@classmethod
- def new_pack(
+ def new_chunk(
cls, *,
typ_: str = "",
role: str = Role.ASSISTANT.value,
content: Optional[str] = None,
memory: Optional[str] = None,
name: Optional[str] = None,
+ call_id: Optional[str] = None,
+ msg_id: Optional[str] = None,
):
+ """
+ create a chunk message.
+ :return:
+ """
return cls(
- role=role, name=name, content=content, memory=memory, pack=True,
+ role=role, name=name, content=content, memory=memory,
type=typ_,
+ call_id=call_id,
+ msg_id=msg_id or "",
+ seq="chunk",
)
def get_content(self) -> str:
+ """
+ get content of this message that is showed to model
+ if result is empty, means do not show it to model.
+ """
if self.memory is None:
return self.content if self.content else ""
return self.memory
- def patch(self, pack: "Message") -> Optional["Message"]:
+ def patch(self, chunk: "Message") -> Optional["Message"]:
"""
- 预期目标消息是当前消息的一个后续包, 执行粘包逻辑.
- :param pack:
- :return: 如果粘包成功, 返回粘包后的消息. 粘包失败, 则返回 None.
+ patch a chunk to the current message until get a tail message or other message's chunk
+ :param chunk: the chunk to patch.
+ :return: if patch succeeds, return the patched message. None means it is other message's chunk
"""
- # type 不相同的话, 则认为是不同消息.
- pack_type = pack.get_type()
- if pack_type and pack_type != self.get_type():
- return None
- # 如果两个消息的 msg id 都存在, 又不相同, 则认为是不同的消息.
- if pack.msg_id and self.msg_id and pack.msg_id != self.msg_id:
+ # if the type is not same, it can't be patched
+ pack_type = chunk.get_type()
+ if pack_type and pack_type != self.type:
+ is_text = pack_type == MessageType.TEXT.value and not self.type
+ if not is_text:
+ return None
+ # the chunk message shall have the same message id or empty one
+ if chunk.msg_id and self.msg_id and chunk.msg_id != self.msg_id:
return None
- # 如果目标包是一个尾包, 则直接返回这个尾包.
- if not pack.pack:
- return pack
- # 否则更新当前消息.
- self.update(pack)
+ # if not a chunk, just return the tail message.
+ # tail message may be changed by outside method such as moderation.
+ if chunk.is_complete():
+ return chunk.model_copy()
+ # otherwise, update current one.
+ self.update(chunk)
+ # add msg_id to each chunk
+ chunk.msg_id = self.msg_id
return self
- def get_copy(self) -> "Message":
+ def as_head(self, copy: bool = True) -> Self:
+ if copy:
+ item = self.get_copy()
+ else:
+ item = self
+ if not item.msg_id:
+ item.msg_id = uuid()
+ if not self.created:
+ item.created = round(time.time(), 3)
+ if item.seq == "chunk":
+ item.seq = "head"
+ return item
+
+ def get_copy(self) -> Self:
return self.model_copy(deep=True)
+ def as_tail(self, copy: bool = True) -> Self:
+ item = self.as_head(copy)
+ item.seq = "complete"
+ return item
+
+ def get_unique_id(self) -> str:
+ return f"{self.type}:{self.msg_id}"
+
def update(self, pack: "Message") -> None:
"""
- 使用目标消息更新当前消息.
+ update the fields.
+ do not call this method outside patch unless you know what you are doing
"""
if not self.msg_id:
# 当前消息的 msg id 不会变更.
self.msg_id = pack.msg_id
+ if not self.call_id:
+ self.call_id = pack.call_id
if not self.type:
- # type 也不会变更.
+ # only update when self type is empty (default)
self.type = pack.type
+
if not self.role:
self.role = pack.role
if self.name is None:
@@ -324,48 +419,62 @@ def update(self, pack: "Message") -> None:
if pack.memory is not None:
self.memory = pack.memory
- self.payloads.update(pack.payloads)
+ if pack.attrs:
+ self.attrs.update(pack.attrs)
- if pack.attachments is not None:
- for key, items in pack.attachments.items():
- saved = self.attachments.get(key, [])
- saved.append(*items)
- self.attachments[key] = saved
+ self.payloads.update(deepcopy(pack.payloads))
if pack.callers:
self.callers.extend(pack.callers)
- self.pack_count += 1
- if self.created:
- now = round(time.time(), 4)
- self.time_cast = round(now - self.created, 4)
def get_type(self) -> str:
"""
- 返回消息的类型.
+ return a message type
"""
- return self.type or DefaultMessageTypes.DEFAULT
+ return self.type or MessageType.DEFAULT
def is_empty(self) -> bool:
"""
- 根据协议判断是不是空消息.
+ a message is empty means it has no content, payloads, callers, or attachments
"""
no_content = not self.content and not self.memory
- no_payloads = not self.payloads and not self.attachments and not self.callers
- return no_content and no_payloads
+ no_attrs = not self.attrs
+ no_payloads = not self.payloads and self.__attachment__ is None and not self.callers
+ return no_content and no_attrs and no_payloads
+
+ def is_complete(self) -> bool:
+ """
+ complete message is not a chunk one
+ """
+ return self.seq == "complete" or MessageType.is_protocol_type(self.type)
+
+ def is_head(self) -> bool:
+ return self.seq == "head"
- def is_tail(self) -> bool:
- return not self.pack
+ def is_chunk(self) -> bool:
+ return self.seq == "chunk"
+
+ def get_seq(self) -> SeqType:
+ return self.seq
def dump(self) -> Dict:
"""
- 将消息以 dict 形式输出, 过滤掉默认值.
+ dump a message dict without default value.
"""
return self.model_dump(exclude_defaults=True)
+ def get_created(self) -> datetime:
+ return datetime.fromtimestamp(self.created)
+
+ def __str__(self):
+ return self.__repr__()
+
class MessageClass(ABC):
"""
- 一种特殊的 Message, 本体是强类型数据结构, 映射到 Message 类型中解决 payloads 等参数问题.
+ A message class with every field that is strong-typed
+ the payloads and attachments shall parse to dict when generate to a Message.
"""
+ __message_type__: ClassVar[Union[MessageType, str]]
@abstractmethod
def to_message(self) -> Message:
@@ -373,48 +482,132 @@ def to_message(self) -> Message:
@classmethod
@abstractmethod
- def from_message(cls) -> Optional[Message]:
+ def from_message(cls, message: Message) -> Optional[Self]:
+ """
+ from a message container generate a strong-typed one.
+ :param message:
+ :return: None means type not match.
+ """
pass
-
-MessageKind = Union[Message, MessageClass, str]
-"""将三种类型的数据统一视作 message 类型. """
+ @abstractmethod
+ def to_openai_param(self, container: Optional[Container], compatible: bool = False) -> List[Dict]:
+ pass
-class MessageKindParser:
+class FunctionCaller(BaseModel):
"""
- 处理 MessageType
+ 消息协议中用来描述一个工具或者function 的调用请求.
"""
+ id: Optional[str] = Field(default=None, description="caller 的 id, 用来 match openai 的 tool call 协议. ")
+ name: str = Field(description="方法的名字.")
+ arguments: str = Field(description="方法的参数. ")
+
+ # deprecated
+ functional_token: bool = Field(default=False, description="caller 是否是基于协议生成的?")
+
+ def add(self, message: "Message") -> None:
+ message.callers.append(self)
+
+ def new_output(self, output: str) -> FunctionOutput:
+ return FunctionOutput(
+ call_id=self.id,
+ name=self.name,
+ content=output,
+ )
+
+ @classmethod
+ def from_message(cls, message: Message) -> Iterable[FunctionCaller]:
+ if message.type == MessageType.FUNCTION_CALL.value:
+ yield FunctionCaller(
+ id=message.call_id,
+ name=message.name,
+ arguments=message.content,
+ )
+ if message.callers:
+ yield from message.callers
+
+
+# todo: history code, optimize later
+class FunctionOutput(BaseModel, MessageClass):
+ __message_type__ = MessageType.FUNCTION_OUTPUT.value
+
+ call_id: Optional[str] = Field(None, description="caller id")
+ name: Optional[str] = Field(
+ default=None,
+ description="caller name, caller id and caller name can not both be empty",
+ )
+ content: Optional[str] = Field(description="caller output")
+
+ msg_id: str = Field(default_factory=uuid)
+ payloads: Dict[str, Dict] = Field(default_factory=dict)
+
+ def to_message(self) -> Message:
+ return Message(
+ msg_id=self.msg_id,
+ call_id=self.call_id,
+ type=MessageType.FUNCTION_OUTPUT.value,
+ name=self.name,
+ content=self.content,
+ payloads=self.payloads,
+ ).as_tail(copy=True)
+
+ @classmethod
+ def from_message(cls, message: Message) -> Optional[Self]:
+ if message.type != MessageType.FUNCTION_OUTPUT.value:
+ return None
+ return cls(
+ msg_id=message.msg_id,
+ call_id=message.call_id,
+ name=message.name,
+ content=message.content,
+ payloads=message.payloads,
+ )
+
+ def to_openai_param(self, container: Optional[Container], compatible: bool = False) -> List[Dict]:
+ from openai.types.chat.chat_completion_tool_message_param import ChatCompletionToolMessageParam
+ from openai.types.chat.chat_completion_function_message_param import ChatCompletionFunctionMessageParam
+ if self.call_id:
+ return [ChatCompletionToolMessageParam(
+ content=self.content,
+ role="tool",
+ tool_call_id=self.call_id,
+ )]
+ else:
+ return [ChatCompletionFunctionMessageParam(
+ content=self.content,
+ name=self.name,
+ role="function",
+ )]
+
+
+class MessageClassesParser:
+ def __init__(
+ self,
+ classes: Iterable[Type[MessageClass]],
+ ) -> None:
+ self.classes = {str(cls.__message_type__): cls for cls in classes}
- def __init__(self, role: str = Role.ASSISTANT.value, ref_id: Optional[str] = None) -> None:
- self.role = role
- self.ref_id = ref_id
-
- def parse(self, messages: Iterable[MessageKind]) -> Iterable[Message]:
- for item in messages:
- if isinstance(item, Message):
- yield self._with_ref(item)
- if isinstance(item, MessageClass):
- msg= item.to_message()
- yield self._with_ref(msg)
- if isinstance(item, str):
- if not item:
- # exclude empty message
- continue
- msg = Message.new_tail(content=item, role=self.role)
- yield self._with_ref(msg)
- else:
- # todo: 需要日志?
- pass
-
- def _with_ref(self, item: Message) -> Message:
- if self.ref_id is not None:
- item.ref_id = self.ref_id
+ def parse(self, message: Message) -> Optional[MessageClass]:
+ if not message.is_complete():
+ return None
+ if message.type not in self.classes:
+ return None
+ cls = self.classes[message.type]
+ item = cls.from_message(message)
return item
- def unknown(self, item) -> None:
- """
- unknown 消息类型的处理逻辑.
- 默认忽视, 可以重写这个方法.
- """
- return
+ def to_openai_params(
+ self,
+ message: Message,
+ container: Optional[Container],
+ compatible: bool = False,
+ ) -> Optional[List[Dict]]:
+ parsed = self.parse(message)
+ if parsed is None:
+ return None
+ return parsed.to_openai_param(container, compatible)
+
+
+MessageKind = Union[Message, MessageClass, str, EntityType]
+"""sometimes we need three forms of the message to define an argument or property."""
diff --git a/ghostos/core/messages/message_classes.py b/ghostos/core/messages/message_classes.py
new file mode 100644
index 00000000..56f631a0
--- /dev/null
+++ b/ghostos/core/messages/message_classes.py
@@ -0,0 +1,364 @@
+import base64
+from typing import Optional, Dict, List, Iterable, Any, Union, Literal
+from typing_extensions import Self
+
+from ghostos.contracts.variables import Variables
+from ghostos.contracts.assets import FileInfo
+from ghostos.container import Container
+from ghostos.prompter import get_defined_prompt
+from .message import Message, MessageClass, MessageType, FunctionOutput, MessageKind, Role, FunctionCaller
+from ghostos.helpers import uuid
+from pydantic import BaseModel, Field
+
+__all__ = [
+ "VariableMessage",
+ "FunctionCallMessage",
+ "FunctionCallOutputMessage",
+ "FunctionOutput",
+ "ImageAssetMessage",
+ "AudioMessage",
+ "MessageKindParser",
+]
+
+FunctionCallOutputMessage = FunctionOutput
+
+
+class FunctionCallMessage(MessageClass, BaseModel):
+ __message_type__ = MessageType.FUNCTION_CALL.value
+
+ msg_id: str = Field(default_factory=uuid, description="message id")
+ payloads: Dict[str, Dict] = Field(
+ default_factory=dict,
+ description="payload type key to payload item. payload shall be a strong-typed dict"
+ )
+ role: str = Field(default="", description="who send the message")
+ caller: FunctionCaller
+
+ def to_message(self) -> Message:
+ return Message.new_tail(
+ type_=self.__message_type__,
+ msg_id=self.msg_id,
+ role=self.role,
+ name=self.caller.name,
+ call_id=self.caller.id,
+ content=self.caller.arguments,
+ )
+
+ @classmethod
+ def from_message(cls, message: Message) -> Optional[Self]:
+ if message.type != cls.__message_type__:
+ return None
+ return cls(
+ msg_id=message.msg_id,
+ payloads=message.payloads,
+ role=message.role,
+ caller=FunctionCaller(
+ id=message.call_id,
+ name=message.name,
+ arguments=message.content,
+ )
+ )
+
+ def to_openai_param(self, container: Optional[Container], compatible: bool = False) -> List[Dict]:
+ from openai.types.chat.chat_completion_assistant_message_param import (
+ ChatCompletionAssistantMessageParam, FunctionCall,
+ )
+ from openai.types.chat.chat_completion_message_tool_call_param import ChatCompletionMessageToolCallParam
+
+ return [ChatCompletionAssistantMessageParam(
+ role="assistant",
+ tool_calls=[ChatCompletionMessageToolCallParam(
+ id=self.caller.id,
+ function=FunctionCall(
+ name=self.caller.name,
+ arguments=self.caller.arguments,
+ ),
+ type="function"
+ )]
+ )]
+
+
+class VariableMessage(MessageClass, BaseModel):
+ """
+ 变量类型消息.
+ """
+
+ __message_type__ = MessageType.VARIABLE.value
+
+ msg_id: str = Field(default_factory=uuid, description="message id")
+ payloads: Dict[str, Dict] = Field(
+ default_factory=dict,
+ description="payload type key to payload item. payload shall be a strong-typed dict"
+ )
+ role: str = Field(default="", description="who send the message")
+ name: Optional[str] = Field(None, description="who send the message")
+ attrs: Variables.Var = Field(
+ description="variable pointer info"
+ )
+
+ def to_message(self) -> Message:
+ message = Message.new_tail(
+ type_=MessageType.VARIABLE.value,
+ content="",
+ role=self.role,
+ name=self.name,
+ attrs=self.attrs.model_dump(),
+ msg_id=self.msg_id,
+ )
+ message.payloads = self.payloads
+ return message
+
+ @classmethod
+ def from_message(cls, message: Message) -> Optional[Self]:
+ if message.type != MessageType.VARIABLE.value:
+ return None
+
+ obj = cls(
+ msg_id=message.msg_id,
+ role=message.role,
+ name=message.name,
+ attrs=message.attrs,
+ payloads=message.payloads,
+ )
+ return obj
+
+ def to_openai_param(self, container: Optional[Container], compatible: bool = False) -> List[Dict]:
+ content = f"""variable message:
+vid: {self.attrs.vid}
+type: {self.attrs.type}
+desc: {self.attrs.desc}
+"""
+ if container and container.bound(Variables) and compatible:
+ variables = container.force_fetch(Variables)
+ v = variables.load(self.attrs.vid)
+ prompt = get_defined_prompt(v)
+ if prompt:
+ content += f"\nmore information:\n```\n{prompt}\n```"
+
+ return [dict(
+ content=content,
+ role=self.role,
+ name=self.name,
+ )]
+
+
+class ImageId(BaseModel):
+ image_id: str = Field(description="image id")
+ detail: Literal["auto", "high", "low"] = Field(default="auto", description="image quality")
+
+
+class ImageAttrs(BaseModel):
+ images: List[ImageId] = Field(default_factory=list, description="file id")
+
+
+class ImageAssetMessage(MessageClass, BaseModel):
+ msg_id: str = Field(default_factory=uuid, description="message id")
+ payloads: Dict[str, Dict] = Field(
+ default_factory=dict,
+ description="payload type key to payload item. payload shall be a strong-typed dict"
+ )
+ role: str = Field(default="", description="who send the message")
+ name: Optional[str] = Field(None, description="who send the message")
+ content: Optional[str] = Field("", description="content of the image message")
+
+ attrs: ImageAttrs = Field(description="image assert id")
+
+ __message_type__ = MessageType.IMAGE.value
+
+ def to_message(self) -> Message:
+ message = Message.new_tail(
+ role=self.role,
+ name=self.name,
+ content=self.content,
+ type_=self.__message_type__,
+ attrs=self.attrs.model_dump(),
+ msg_id=self.msg_id,
+ )
+ message.payloads = self.payloads
+ return message
+
+ @classmethod
+ def from_image_asset(
+ cls,
+ name: str,
+ content: str,
+ images: List[FileInfo],
+ role: str = Role.USER.value,
+ ) -> Self:
+ attrs = ImageAttrs(images=[
+ ImageId(image_id=image_info.fileid)
+ for image_info in images
+ ])
+ return cls(
+ name=name,
+ content=content,
+ role=role,
+ attrs=attrs,
+ )
+
+ @classmethod
+ def from_message(cls, message: Message) -> Optional[Self]:
+ if message.type != cls.__message_type__:
+ return None
+ return cls(
+ msg_id=message.msg_id,
+ role=message.role,
+ name=message.name,
+ content=message.content,
+ attrs=message.attrs,
+ payloads=message.payloads,
+ )
+
+ def to_openai_param(self, container: Optional[Container], compatible: bool = False) -> List[Dict]:
+ from openai.types.chat.chat_completion_content_part_text_param import ChatCompletionContentPartTextParam
+ from openai.types.chat.chat_completion_content_part_image_param import (
+ ChatCompletionContentPartImageParam, ImageURL,
+ )
+ from openai.types.chat.chat_completion_user_message_param import (
+ ChatCompletionUserMessageParam,
+ )
+ from openai.types.chat.chat_completion_assistant_message_param import (
+ ChatCompletionAssistantMessageParam,
+ )
+ from ghostos.contracts.assets import ImageAssets
+ content = self.content
+ image_id_and_desc = []
+ content_parts = []
+ if not compatible and self.attrs is not None and self.attrs.images and container:
+ images = container.force_fetch(ImageAssets)
+ for image_id_info in self.attrs.images:
+ got = images.get_file_and_binary_by_id(image_id_info.image_id)
+ if got is None:
+ continue
+ image_info, binary = got
+ if binary:
+ encoded = base64.b64encode(binary).decode('utf-8')
+ url = f"data:{image_info.filetype};base64,{encoded}"
+ else:
+ url = image_info.url
+ if not url:
+ continue
+ content_parts.append(ChatCompletionContentPartImageParam(
+ type="image_url",
+ image_url=ImageURL(
+ url=url,
+ detail="auto",
+ ),
+ ))
+ image_id_and_desc.append((image_id_info.image_id, image_info.description))
+ if image_id_and_desc:
+ attachment = "\n(about follow images:"
+ order = 0
+ for image_id, desc in image_id_and_desc:
+ order += 1
+ attachment += f"\n[{order}] id: `{image_id}` desc: `{desc}`"
+ content = content + attachment + ")"
+ content = content.strip()
+ if content:
+ content_parts.insert(0, ChatCompletionContentPartTextParam(
+ text=content,
+ type="text",
+ ))
+
+ if self.role == Role.ASSISTANT.value:
+ item = ChatCompletionAssistantMessageParam(
+ role=Role.ASSISTANT.value,
+ content=content_parts,
+ )
+ else:
+ item = ChatCompletionUserMessageParam(
+ role=Role.USER.value,
+ content=content_parts,
+ )
+ if self.name:
+ item["name"] = self.name
+ return [item]
+
+
+class AudioMessage(MessageClass, BaseModel):
+ msg_id: str = Field(default_factory=uuid, description="message id")
+ payloads: Dict[str, Dict] = Field(
+ default_factory=dict,
+ description="payload type key to payload item. payload shall be a strong-typed dict"
+ )
+ role: str = Field(default="", description="who send the message")
+ name: Optional[str] = Field(None, description="who send the message")
+ content: str = Field("", description="transcription of the audio message")
+
+ __message_type__ = MessageType.AUDIO.value
+
+ def to_message(self) -> Message:
+ message = Message.new_tail(
+ role=self.role,
+ name=self.name,
+ content=self.content,
+ type_=self.__message_type__,
+ msg_id=self.msg_id,
+ )
+ message.payloads = self.payloads
+ return message
+
+ @classmethod
+ def from_message(cls, message: Message) -> Optional[Self]:
+ if message.type != cls.__message_type__:
+ return None
+ return cls(
+ msg_id=message.msg_id,
+ role=message.role,
+ name=message.name,
+ content=message.content,
+ payloads=message.payloads,
+ )
+
+ def to_openai_param(self, container: Optional[Container], compatible: bool = False) -> List[Dict]:
+ raise NotImplementedError("todo")
+
+
+class MessageKindParser:
+ """
+ middleware that parse weak MessageKind into Message chunks
+ """
+
+ def __init__(
+ self,
+ variables: Variables,
+ *,
+ name: Optional[str] = None,
+ role: str = Role.ASSISTANT.value,
+ call_id: Optional[str] = None,
+ ) -> None:
+ self.variables = variables
+ self.role = role
+ self.call_id = call_id
+ self.name = name
+
+ def parse(self, messages: Iterable[Union[MessageKind, Any]]) -> Iterable[Message]:
+ for item in messages:
+ if isinstance(item, Message):
+ yield self._with_ref(item)
+ elif isinstance(item, MessageClass):
+ msg = item.to_message()
+ yield self._with_ref(msg)
+ elif isinstance(item, str):
+ if not item:
+ # exclude empty message
+ continue
+ msg = Message.new_tail(content=item, role=self.role)
+ yield self._with_ref(msg)
+ else:
+ var = self.variables.save(item)
+ vm = VariableMessage(
+ name=self.name,
+ role=self.role,
+ attrs=var.model_dump(),
+ )
+ yield vm.to_message()
+
+ def _with_ref(self, item: Message) -> Message:
+ if self.call_id is not None:
+ item.call_id = self.call_id
+ if not item.role and self.role:
+ item.role = self.role
+ if not item.name and self.name:
+ item.name = self.name
+ return item
diff --git a/ghostos/core/messages/openai.py b/ghostos/core/messages/openai.py
index 3b542940..9f7f92cf 100644
--- a/ghostos/core/messages/openai.py
+++ b/ghostos/core/messages/openai.py
@@ -1,20 +1,24 @@
-import time
-from typing import Iterable, Optional, Type, ClassVar
+from typing import Iterable, Optional, Type, ClassVar, List
from abc import ABC, abstractmethod
from openai.types.chat.chat_completion_chunk import ChoiceDelta, ChatCompletionChunk
from openai.types.completion_usage import CompletionUsage
from openai.types.chat.chat_completion_message_param import ChatCompletionMessageParam
from openai.types.chat.chat_completion_message import ChatCompletionMessage
+from openai.types.chat.chat_completion_tool_message_param import ChatCompletionToolMessageParam
from openai.types.chat.chat_completion_assistant_message_param import ChatCompletionAssistantMessageParam, FunctionCall
from openai.types.chat.chat_completion_message_tool_call_param import ChatCompletionMessageToolCallParam
from openai.types.chat.chat_completion_system_message_param import ChatCompletionSystemMessageParam
from openai.types.chat.chat_completion_user_message_param import ChatCompletionUserMessageParam
from openai.types.chat.chat_completion_function_message_param import ChatCompletionFunctionMessageParam
-from openai.types.chat.chat_completion_tool_message_param import ChatCompletionToolMessageParam
-
-from ghostos.core.messages.message import Message, DefaultMessageTypes, Role, Caller, PayloadItem
-from ghostos.container import Provider, Container, ABSTRACT
-from pydantic import BaseModel, Field
+from ghostos.core.messages import (
+ Message, MessageType, Role, FunctionCaller, Payload, MessageClass, MessageClassesParser
+)
+from ghostos.core.messages.message_classes import (
+ FunctionOutput, VariableMessage, ImageAssetMessage,
+)
+from ghostos.contracts.logger import LoggerItf, FakeLogger
+from ghostos.container import Provider, Container
+from ghostos.helpers import import_class_from_path
__all__ = [
"OpenAIMessageParser", "DefaultOpenAIMessageParser", "DefaultOpenAIParserProvider",
@@ -24,43 +28,55 @@
class OpenAIMessageParser(ABC):
"""
- 用来对齐 openai 的协议.
+ a parser for OpenAI messages alignment.
"""
@abstractmethod
- def parse_message(self, message: Message) -> Iterable[ChatCompletionMessageParam]:
+ def parse_message(
+ self,
+ message: Message,
+ types: Optional[List[str]] = None,
+ ) -> Iterable[ChatCompletionMessageParam]:
"""
- 将 message 转换为 openai 的请求入参.
+ parse a Message to OpenAI chat completion message form.
+ OpenAI's input message (ChatCompletionXXXParam) are different to ChatCompletion types,
+ which is exhausting
"""
pass
- def parse_message_list(self, messages: Iterable[Message]) -> Iterable[ChatCompletionMessageParam]:
+ def parse_message_list(
+ self,
+ messages: Iterable[Message],
+ types: Optional[List[str]] = None,
+ ) -> Iterable[ChatCompletionMessageParam]:
"""
- 将多条消息转换成 openai 的多条入参.
+ syntax suger
"""
for message in messages:
- items = self.parse_message(message)
+ items = self.parse_message(message, types)
for item in items:
yield item
@abstractmethod
def from_chat_completion(self, message: ChatCompletionMessage) -> Message:
"""
- 将 openai chat completion 转换.
+ parse a ChatCompletion message to Message.
+ Request -> Message -> ChatCompletionXXXXParam --LLM generation--> ChatCompletionXXX --> Message
"""
pass
@abstractmethod
def from_chat_completion_chunks(self, messages: Iterable[ChatCompletionChunk]) -> Iterable[Message]:
"""
- 将 openai 的 delta 转换过来.
+ patch the openai Chat Completion Chunks.
+ the Realtime API need a new parser.
"""
pass
-class CompletionUsagePayload(CompletionUsage, PayloadItem):
+class CompletionUsagePayload(CompletionUsage, Payload):
"""
- 将每个包的开销记录下来.
+ the strong-typed payload of OpenAI chat completion usage.
"""
key: ClassVar[str] = "completion_usage"
@@ -82,37 +98,103 @@ def join(self, payload: "CompletionUsagePayload") -> "CompletionUsagePayload":
class DefaultOpenAIMessageParser(OpenAIMessageParser):
"""
- 默认的 parser, 只做了极简的实现.
+ default implementation of OpenAIMessageParser
"""
- def parse_message(self, message: Message) -> Iterable[ChatCompletionMessageParam]:
- if message.type == DefaultMessageTypes.CHAT_COMPLETION:
- return self._parse_assistant_chat_completion(message)
+ def __init__(
+ self,
+ message_classes: Optional[List[Type[MessageClass]]],
+ container: Optional[Container],
+ ):
+ if message_classes is None:
+ message_classes = [
+ FunctionOutput,
+ VariableMessage,
+ ImageAssetMessage,
+ ]
+ self.class_parser = MessageClassesParser(message_classes)
+ self.container: Optional[Container] = container
+ self.logger: Optional[LoggerItf] = None
+ if container:
+ self.logger = container.get(LoggerItf)
+ if not self.logger:
+ self.logger = FakeLogger()
+
+ def parse_message(
+ self,
+ message: Message,
+ types: Optional[List[str]] = None,
+ ) -> Iterable[ChatCompletionMessageParam]:
+ if not message.is_complete():
+ return []
+ compatible = False
+ if types is not None:
+ types_set = set(types)
+ if message.type not in types_set:
+ compatible = True
+
+ wrapped = self.class_parser.to_openai_params(message, self.container, compatible)
+ if wrapped is not None:
+ yield from wrapped
else:
- return self._parse_message(message)
+ yield from self._parse_message(message)
def _parse_message(self, message: Message) -> Iterable[ChatCompletionMessageParam]:
+ if message.type == MessageType.FUNCTION_CALL.value:
+ if message.call_id:
+ return [
+ ChatCompletionAssistantMessageParam(
+ role="assistant",
+ tool_calls=[ChatCompletionMessageToolCallParam(
+ id=message.call_id,
+ function=FunctionCall(
+ name=message.name,
+ arguments=message.content,
+ ),
+ type="function"
+ )]
+ )
+ ]
+ else:
+ return [
+ ChatCompletionAssistantMessageParam(
+ role="assistant",
+ function_call=FunctionCall(
+ name=message.name,
+ arguments=message.content,
+ )
+ )
+ ]
+ elif message.type == MessageType.FUNCTION_OUTPUT:
+ if message.call_id:
+ return [
+ ChatCompletionToolMessageParam(
+ tool_call_id=message.call_id,
+ content=message.content,
+ role="tool",
+ )
+ ]
+ else:
+ return [
+ ChatCompletionFunctionMessageParam(
+ content=message.get_content(),
+ name=message.name,
+ role="function",
+ )
+ ]
+
if message.role == Role.ASSISTANT:
return self._parse_assistant_chat_completion(message)
elif message.role == Role.SYSTEM:
return [
- ChatCompletionSystemMessageParam(content=message.get_content(), name=message.name, role="system")
+ ChatCompletionSystemMessageParam(content=message.get_content(), role="system")
]
elif message.role == Role.USER:
+ item = ChatCompletionUserMessageParam(content=message.get_content(), role="user")
+ if message.name:
+ item["name"] = message.name
return [
- ChatCompletionUserMessageParam(content=message.get_content(), name=message.name, role="user")
- ]
- elif message.role == Role.FUNCTION:
- return [
- ChatCompletionFunctionMessageParam(content=message.get_content(), name=message.name, role="function")
- ]
- elif message.role == Role.TOOL:
- return [
- ChatCompletionToolMessageParam(
- tool_call_id=message.ref_id,
- content=message.get_content(),
- role="tool",
- )
+ item
]
else:
return []
@@ -151,18 +233,20 @@ def _parse_assistant_chat_completion(message: Message) -> Iterable[ChatCompletio
tool_calls.append(tool_call)
if not content and not function_call and not tool_calls:
return []
-
- return [ChatCompletionAssistantMessageParam(
+ item = ChatCompletionAssistantMessageParam(
content=content,
role="assistant",
- function_call=function_call,
tool_calls=tool_calls,
- )]
+ )
+ if message.name:
+ item["name"] = message.name
+
+ return [item]
def from_chat_completion(self, message: ChatCompletionMessage) -> Message:
- pack = Message.new_tail(type_=DefaultMessageTypes.CHAT_COMPLETION, role=message.role, content=message.content)
+ pack = Message.new_tail(type_=MessageType.DEFAULT, role=message.role, content=message.content)
if message.function_call:
- caller = Caller(
+ caller = FunctionCaller(
name=message.function_call.name,
arguments=message.function_call.arguments,
protocol=True
@@ -170,7 +254,7 @@ def from_chat_completion(self, message: ChatCompletionMessage) -> Message:
caller.add(pack)
if message.tool_calls:
for tool_call in message.tool_calls:
- caller = Caller(
+ caller = FunctionCaller(
id=tool_call.id,
name=tool_call.function.name,
arguments=tool_call.function.arguments,
@@ -181,52 +265,98 @@ def from_chat_completion(self, message: ChatCompletionMessage) -> Message:
def from_chat_completion_chunks(self, messages: Iterable[ChatCompletionChunk]) -> Iterable[Message]:
# 创建首包, 并发送.
- first = True
+ if messages is None:
+ return []
+ buffer = None
for item in messages:
+ chunk = None
+ self.logger.debug("openai parser receive item: %s", item)
if len(item.choices) == 0:
# 接受到了 openai 协议尾包. 但在这个协议里不作为尾包发送.
usage = CompletionUsagePayload.from_chunk(item)
- pack = Message.new_pack(role=Role.ASSISTANT.value, typ_=DefaultMessageTypes.CHAT_COMPLETION)
- usage.set(pack)
- yield pack
- else:
+ if usage and buffer:
+ usage.set_payload(buffer)
+ continue
+ elif len(item.choices) > 0:
choice = item.choices[0]
delta = choice.delta
- pack = self._new_pack_from_delta(delta, first)
- yield pack
- first = False
+ chunk = self._new_chunk_from_delta(delta)
+ else:
+ continue
+ self.logger.debug("openai parser parsed chunk: %s", chunk)
+
+ if chunk is None:
+ continue
+ elif item.id:
+ chunk.msg_id = item.id
+
+ if buffer is None:
+ buffer = chunk.as_head(copy=True)
+ yield buffer.get_copy()
+ else:
+ patched = buffer.patch(chunk)
+ if not patched:
+ yield buffer.as_tail()
+ buffer = chunk.as_head(copy=True)
+ yield buffer.get_copy()
+ continue
+ else:
+ buffer = patched
+ yield chunk
+ continue
+
+ if buffer:
+ yield buffer.as_tail(copy=False)
@staticmethod
- def _new_pack_from_delta(delta: ChoiceDelta, first: bool) -> Message:
- if first:
- pack = Message.new_head(role=Role.ASSISTANT.value, content=delta.content,
- typ_=DefaultMessageTypes.CHAT_COMPLETION)
- else:
- pack = Message.new_pack(role=Role.ASSISTANT.value, content=delta.content,
- typ_=DefaultMessageTypes.CHAT_COMPLETION)
+ def _new_chunk_from_delta(delta: ChoiceDelta) -> Optional[Message]:
+
# function call
if delta.function_call:
- function_call = Caller(**delta.function_call.model_dump())
- pack.callers.append(function_call)
+ pack = Message.new_chunk(
+ typ_=MessageType.FUNCTION_CALL.value,
+ name=delta.function_call.name,
+ content=delta.function_call.arguments,
+ )
+ return pack
# tool calls
- if delta.tool_calls:
+ elif delta.content:
+ pack = Message.new_chunk(
+ role=Role.ASSISTANT.value,
+ content=delta.content,
+ typ_=MessageType.DEFAULT,
+ )
+ return pack
+
+ elif delta.tool_calls:
for item in delta.tool_calls:
- tool_call = Caller(**item.tool_call.model_dump())
- pack.callers.append(tool_call)
- return pack
+ pack = Message.new_chunk(
+ typ_=MessageType.FUNCTION_CALL.value,
+ call_id=item.id,
+ name=item.function.name,
+ content=item.function.arguments,
+ )
+ return pack
+ return None
-class DefaultOpenAIParserProvider(Provider):
+class DefaultOpenAIParserProvider(Provider[OpenAIMessageParser]):
"""
默认的 provider.
"""
+ def __init__(self, message_classes: Optional[List[str]] = None):
+ classes = None
+ if message_classes is not None:
+ classes = []
+ for import_path in message_classes:
+ cls = import_class_from_path(import_path, MessageClass)
+ classes.append(cls)
+ self._message_classes = classes
+
def singleton(self) -> bool:
return True
- def contract(self) -> Type[ABSTRACT]:
- return OpenAIMessageParser
-
- def factory(self, con: Container) -> Optional[ABSTRACT]:
- return DefaultOpenAIMessageParser()
+ def factory(self, con: Container) -> Optional[OpenAIMessageParser]:
+ return DefaultOpenAIMessageParser(self._message_classes, con)
diff --git a/ghostos/core/messages/payload.py b/ghostos/core/messages/payload.py
new file mode 100644
index 00000000..cd718476
--- /dev/null
+++ b/ghostos/core/messages/payload.py
@@ -0,0 +1,37 @@
+from typing import ClassVar, Optional, Protocol, Dict, Union, Self
+from abc import ABC
+from pydantic import BaseModel
+from .message import Message
+
+
+class HasPayloads(Protocol):
+ """
+ some item that has payloads
+ """
+ payloads: Dict[str, Dict]
+
+
+class Payload(BaseModel, ABC):
+ """
+ strong typed payload protocol
+ """
+ key: ClassVar[str]
+ """ the unique key of the payload"""
+
+ @classmethod
+ def read_payload(cls, message: Union[Message, HasPayloads]) -> Optional[Self]:
+ value = message.payloads.get(cls.key, None)
+ if value is None:
+ return None
+ return cls(**value)
+
+ def set_payload(self, message: Union[Message, HasPayloads]) -> None:
+ message.payloads[self.key] = self.model_dump()
+
+ @classmethod
+ def payload_exists(cls, message: Union[Message, HasPayloads]) -> bool:
+ if not hasattr(message, "payloads"):
+ return False
+ if not isinstance(message.payloads, dict):
+ return False
+ return cls.key in message.payloads
diff --git a/ghostos/core/messages/pipeline.py b/ghostos/core/messages/pipeline.py
new file mode 100644
index 00000000..cdf2885d
--- /dev/null
+++ b/ghostos/core/messages/pipeline.py
@@ -0,0 +1,119 @@
+from typing import Iterable, List, Optional, Dict
+from typing_extensions import Self
+from abc import ABC, abstractmethod
+from ghostos.core.messages.message import Message, MessageType
+from ghostos.core.messages.utils import iter_messages
+
+
+class Pipe(ABC):
+
+ @abstractmethod
+ def new(self) -> Self:
+ pass
+
+ @abstractmethod
+ def across(self, messages: Iterable[Message]) -> Iterable[Message]:
+ pass
+
+
+def pipeline(pipes: Iterable[Pipe], messages: Iterable[Message]) -> Iterable[Message]:
+ """
+ build pipeline with pipes
+ :param pipes:
+ :param messages:
+ :return:
+ """
+ ordered = reversed(list(pipes))
+ outputs = messages
+ for pipe in ordered:
+ outputs = pipe.across(messages)
+ messages = outputs
+ yield from outputs
+
+
+class SequencePipe(Pipe):
+ """
+ make sure messages are sent in a ?head-?chunk-tail-?tail sequence
+ """
+
+ def new(self) -> Self:
+ return SequencePipe()
+
+ def across(self, messages: Iterable[Message]) -> Iterable[Message]:
+ buffer: Optional[Message] = None
+ final: Optional[Message] = None
+ for item in messages:
+ if MessageType.is_protocol_message(item):
+ final = item
+ break
+ if buffer is None:
+ if item.is_complete():
+ buffer = item
+ continue
+ else:
+ # yield head
+ buffer = item.as_head()
+ yield buffer.get_copy()
+ continue
+ else:
+ patched = buffer.patch(item)
+ if patched:
+ if patched.is_complete():
+ buffer = patched
+ continue
+ else:
+ # add msg_id to item, keep every chunk has it id
+ if not item.msg_id:
+ item.msg_id = buffer.msg_id
+ yield item
+ else:
+ yield buffer.as_tail()
+ buffer = item.as_head()
+ yield buffer.get_copy()
+ continue
+ if buffer is not None:
+ yield buffer.as_tail(copy=False)
+ if final is not None:
+ yield final
+
+
+class CompleteOnly(Pipe):
+ """
+ return complete only
+ """
+
+ def new(self) -> Self:
+ return CompleteOnly()
+
+ def across(self, messages: Iterable[Message]) -> Iterable[Message]:
+ for item in messages:
+ if MessageType.is_protocol_message(item):
+ yield item
+ break
+ elif item.is_complete():
+ yield item
+
+
+class TailPatchPipe(Pipe):
+
+ def new(self) -> Self:
+ return TailPatchPipe()
+
+ def across(self, messages: Iterable[Message]) -> Iterable[Message]:
+ last_tail: Optional[Message] = None
+ for item in messages:
+ if MessageType.is_protocol_message(item):
+ yield item
+ break
+ if not item.is_complete():
+ yield item
+ continue
+ if last_tail is None:
+ last_tail = item
+ continue
+ patched = last_tail.patch(item)
+ if patched:
+ last_tail = patched
+ continue
+ yield last_tail.as_tail()
+ last_tail = item
diff --git a/ghostos/core/messages/stream.py b/ghostos/core/messages/stream.py
deleted file mode 100644
index f216743f..00000000
--- a/ghostos/core/messages/stream.py
+++ /dev/null
@@ -1,41 +0,0 @@
-from abc import ABC, abstractmethod
-from typing import Iterable
-from ghostos.core.messages.message import Message
-
-__all__ = [
- "Stream",
-]
-
-
-class Stream(ABC):
- """
- messenger 的原型. Stream 的有状态版本.
- """
-
- @abstractmethod
- def deliver(self, pack: "Message") -> bool:
- """
- 发送一个包.
- """
- pass
-
- @abstractmethod
- def is_streaming(self) -> bool:
- """
- if not streaming, only receive tail message
- """
- pass
-
- @abstractmethod
- def send(self, messages: Iterable[Message]) -> bool:
- """
- 发送消息.
- """
- pass
-
- @abstractmethod
- def stopped(self) -> bool:
- """
- 是否已经停止接受
- """
- pass
diff --git a/ghostos/core/messages/transport.py b/ghostos/core/messages/transport.py
new file mode 100644
index 00000000..12b2f3fd
--- /dev/null
+++ b/ghostos/core/messages/transport.py
@@ -0,0 +1,405 @@
+from __future__ import annotations
+from typing import Iterable, Optional, Tuple, List, Self, Iterator
+
+from typing_extensions import Protocol
+from collections import deque
+from abc import abstractmethod
+from ghostos.core.messages.message import Message, MessageType
+from ghostos.core.messages.pipeline import SequencePipe
+from ghostos.errors import StreamingError
+import time
+
+__all__ = [
+ "Stream", "Receiver", "ArrayReceiver", "ArrayStream", "new_basic_connection",
+ "ReceiverBuffer",
+]
+
+from ghostos.helpers import Timeleft
+
+
+class Stream(Protocol):
+ """
+ an interface that can send messages asynchronously.
+ """
+
+ @abstractmethod
+ def send(self, messages: Iterable[Message]) -> bool:
+ """
+ send batch of messages
+ :return: successful. if False, maybe error occur
+ """
+ pass
+
+ def deliver(self, message: Message) -> bool:
+ if not message.is_complete():
+ message = message.as_tail()
+ return self.send([message])
+
+ @abstractmethod
+ def completes_only(self) -> bool:
+ """
+ if the stream receive complete message only
+ :return:
+ """
+ pass
+
+ @abstractmethod
+ def alive(self) -> bool:
+ """
+ :return: the upstream channel is alive
+ """
+ pass
+
+ @abstractmethod
+ def close(self):
+ pass
+
+ @abstractmethod
+ def fail(self, error: str) -> bool:
+ """
+ 端的 fail 会传递给 receiver.
+ :param error:
+ :return:
+ """
+ pass
+
+ @abstractmethod
+ def error(self) -> Optional[Message]:
+ pass
+
+ @abstractmethod
+ def closed(self) -> bool:
+ pass
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb) -> Optional[bool]:
+ if self.closed():
+ return None
+ intercept = None
+ if exc_val is not None:
+ intercept = self.fail(error=str(exc_val))
+ self.close()
+ return intercept
+
+
+class Receiver(Protocol):
+ @abstractmethod
+ def recv(self) -> Iterable[Message]:
+ pass
+
+ @abstractmethod
+ def cancel(self):
+ pass
+
+ @abstractmethod
+ def fail(self, error: str) -> bool:
+ """
+ receiver 的 fail 会传递到端.
+ :param error:
+ :return:
+ """
+ pass
+
+ @abstractmethod
+ def closed(self) -> bool:
+ pass
+
+ @abstractmethod
+ def error(self) -> Optional[Message]:
+ pass
+
+ @abstractmethod
+ def close(self):
+ pass
+
+ @abstractmethod
+ def wait(self) -> List[Message]:
+ pass
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb) -> Optional[bool]:
+ if self.closed():
+ return None
+ intercept = None
+ if exc_val is not None:
+ intercept = self.fail(str(exc_val))
+ self.close()
+ return intercept
+
+
+class StreamPart(Protocol):
+ @abstractmethod
+ def head(self) -> Tuple[Message, bool]:
+ pass
+
+ @abstractmethod
+ def chunks(self) -> Iterable[Message]:
+ pass
+
+ @abstractmethod
+ def tail(self) -> Message:
+ pass
+
+ @abstractmethod
+ def next(self) -> Optional[Self]:
+ pass
+
+
+class ArrayReceiver(Receiver):
+
+ def __init__(
+ self,
+ timeleft: Timeleft,
+ idle: float = 0.1,
+ complete_only: bool = False,
+ ):
+ self._timeleft = timeleft
+ self._idle = idle
+ self._streaming = deque()
+ self._closed = False
+ self._done = False
+ self._error: Optional[Message] = None
+ self._complete_only = complete_only
+
+ def recv(self) -> Iterable[Message]:
+ if self._closed:
+ raise RuntimeError("Receiver is closed")
+ while not self._done:
+ if len(self._streaming) > 0:
+ item = self._streaming.popleft()
+ yield item
+ continue
+ if not self._timeleft.alive():
+ self._error = MessageType.ERROR.new(content=f"Timeout after {self._timeleft.passed()}")
+ self._done = True
+ break
+ if self._idle:
+ time.sleep(self._idle)
+ if len(self._streaming) > 0:
+ yield from self._streaming
+ self._streaming.clear()
+ if self._error is not None:
+ yield self._error
+
+ def add(self, message: Message) -> bool:
+ if self._closed:
+ return False
+ if MessageType.is_protocol_message(message):
+ self._done = True
+ if MessageType.ERROR.match(message):
+ self._error = message
+ return True
+
+ elif self._done or not self._timeleft.alive():
+ return False
+ else:
+ if message.is_complete() or not self._complete_only:
+ self._streaming.append(message)
+ return True
+
+ def cancel(self):
+ self._done = True
+
+ def fail(self, error: str) -> bool:
+ if self._error is not None:
+ return False
+ self._done = True
+ self._error = MessageType.ERROR.new(content=error)
+ return False
+
+ def closed(self) -> bool:
+ return self._closed
+
+ def error(self) -> Optional[Message]:
+ return self._error
+
+ def wait(self) -> List[Message]:
+ items = list(self.recv())
+ completes = []
+ for item in items:
+ if item.is_complete():
+ completes.append(item)
+ return completes
+
+ def close(self):
+ if self._closed:
+ return
+ self._closed = True
+ self._done = True
+ self._streaming.clear()
+ self._timeleft = None
+
+
+class ArrayStream(Stream):
+
+ def __init__(self, receiver: ArrayReceiver, complete_only: bool):
+ self._receiver = receiver
+ self._alive = not receiver.closed()
+ self._closed = False
+ self._error: Optional[Message] = None
+ self._complete_only = complete_only
+
+ def send(self, messages: Iterable[Message]) -> bool:
+ if self._closed or not self._alive:
+ raise RuntimeError("Stream is closed")
+ if self._error is not None:
+ raise RuntimeError(self._error.get_content())
+ items = SequencePipe().across(messages)
+ for item in items:
+ if self._complete_only and not item.is_complete():
+ continue
+ success = self._receiver.add(item)
+ if success:
+ continue
+ self._alive = False
+ self._error = self._receiver.error()
+ if self._error is not None:
+ raise StreamingError(
+ f"streaming is failed: {self._error.get_content()}, message {item.model_dump_json()} unsent"
+ )
+ elif self._receiver.closed():
+ raise StreamingError(
+ f"streaming is failed due to receiver is closed: , message {item.model_dump_json()} unsent"
+ )
+ elif not self.alive():
+ raise StreamingError(
+ f"streaming is closed, message {item.model_dump_json()} unsent",
+ )
+ else:
+ raise StreamingError(f"send stream failed, message {item.model_dump_json()} unsent")
+ return True
+
+ def completes_only(self) -> bool:
+ return self._complete_only
+
+ def alive(self) -> bool:
+ if not self._alive:
+ return False
+ if self._receiver.closed():
+ self._alive = False
+ return self._alive
+
+ def close(self):
+ if self._closed:
+ return
+ self._closed = True
+ if self._error:
+ self._receiver.add(self._error)
+ else:
+ self._receiver.add(MessageType.final())
+ self._alive = False
+ del self._receiver
+
+ def fail(self, error: str) -> bool:
+ if self._error is not None:
+ return False
+ self._error = MessageType.ERROR.new(content=error)
+ self._alive = False
+ return False
+
+ def error(self) -> Optional[Message]:
+ return self._error
+
+ def closed(self) -> bool:
+ return self._closed
+
+
+class ReceiverBuffer:
+ def __init__(self, head: Message, iterator: Iterator[Message]):
+ if head.is_chunk():
+ head = head.as_head()
+ self._head = head
+ self._iterator = iterator
+ self._chunks = []
+ self._done: Optional[Message] = None
+ self._next: Optional[Self] = None
+
+ @classmethod
+ def new(cls, receiving: Iterable[Message]) -> Optional[Self]:
+ try:
+ iterator = iter(receiving)
+ head = next(iterator)
+ except StopIteration:
+ return None
+ if head is None:
+ return None
+ return cls(head, iterator)
+
+ def head(self) -> Message:
+ return self._head
+
+ def chunks(self) -> Iterable[Message]:
+ if self._head.is_complete():
+ yield from [self._head]
+ return
+ elif self._done is not None:
+ return self._chunks
+
+ self._chunks = [self._head]
+ yield self._head
+ head = self._head.get_copy()
+ try:
+ item = next(self._iterator)
+ except StopIteration:
+ self._done = head.as_tail()
+ return None
+
+ while item is not None:
+ patched = head.patch(item)
+ if patched is not None:
+ head = patched
+ if item.is_complete():
+ self._done = patched
+ else:
+ self._chunks.append(item)
+ yield item
+ else:
+ if self._done is None:
+ self._done = head.as_tail()
+ self._next = ReceiverBuffer(item, self._iterator)
+ self._iterator = None
+ break
+ try:
+ item = next(self._iterator)
+ except StopIteration:
+ break
+ if self._done is None:
+ self._done = self._head.as_tail()
+
+ def tail(self) -> Message:
+ if self._head.is_complete():
+ return self._head
+ if self._done:
+ return self._done
+ list(self.chunks())
+ if self._done is None:
+ self._done = self._head.as_tail()
+ return self._done
+
+ def next(self) -> Optional[Self]:
+ list(self.chunks())
+ return self._next
+
+
+def new_basic_connection(
+ *,
+ timeout: float = 0.0,
+ idle: float = 0.2,
+ complete_only: bool = False,
+) -> Tuple[Stream, Receiver]:
+ """
+ use array to pass and receive messages in multi-thread
+ :param timeout: if negative, wait until done
+ :param idle: sleep time in seconds wait for next pull
+ :param complete_only: only receive complete message
+ :return: created stream and receiver
+ """
+ from ghostos.helpers import Timeleft
+ timeleft = Timeleft(timeout)
+ receiver = ArrayReceiver(timeleft, idle, complete_only)
+ stream = ArrayStream(receiver, complete_only)
+ return stream, receiver
diff --git a/ghostos/core/messages/utils.py b/ghostos/core/messages/utils.py
new file mode 100644
index 00000000..251390ef
--- /dev/null
+++ b/ghostos/core/messages/utils.py
@@ -0,0 +1,36 @@
+from typing import Iterable, List, Union, Dict, Optional
+from ghostos.core.messages.message import Message, Role, MessageClass
+
+__all__ = [
+ 'copy_messages', 'iter_messages',
+]
+
+
+def copy_messages(messages: Iterable[Message], stages: Optional[List[str]] = None) -> List[Message]:
+ """
+ syntax sugar for copy
+ """
+ result = []
+ if stages:
+ stages = set(stages)
+ for message in messages:
+ if not stages or message.stage in stages:
+ result.append(message.get_copy())
+ return result
+
+
+def iter_messages(messages: Iterable[Union[Message, str, Dict, MessageClass]]) -> Iterable[Message]:
+ """
+ yield from all kinds of messages
+ """
+ for item in messages:
+ if isinstance(item, Message):
+ yield item
+ elif isinstance(item, str):
+ yield Role.ASSISTANT.new(content=item)
+ elif isinstance(item, MessageClass):
+ yield item.to_message()
+ elif isinstance(item, Dict):
+ yield Message(**item)
+ else:
+ raise TypeError(f"Unexpected type {type(item)}")
diff --git a/evaluation/swe_bench_lite/tools/__init__.py b/ghostos/core/models/__init__.py
similarity index 100%
rename from evaluation/swe_bench_lite/tools/__init__.py
rename to ghostos/core/models/__init__.py
diff --git a/ghostos/core/models/audio_generation.py b/ghostos/core/models/audio_generation.py
new file mode 100644
index 00000000..1ddf3bea
--- /dev/null
+++ b/ghostos/core/models/audio_generation.py
@@ -0,0 +1,59 @@
+from abc import ABC, abstractmethod
+from typing import Optional, Dict, Generic, TypeVar
+from ghostos.core.llms import Prompt, ServiceConf
+from ghostos.core.messages import Message
+from ghostos.entity import EntityMeta, get_entity, to_entity_meta
+from pydantic import BaseModel, Field
+
+
+class AudioGenerationModel(BaseModel, ABC):
+ name: str = Field(description="Name of the audio generator")
+ driver: str = Field(description="Name of the audio generator driver")
+
+
+M = TypeVar("M", bound=AudioGenerationModel)
+
+
+class AudioGeneratorsConfig(BaseModel):
+ default: str = Field(description="Default audio generator model")
+ models: Dict[str, EntityMeta] = Field(
+ default_factory=dict,
+ )
+
+ def add_model(self, model: AudioGenerationModel):
+ self.models[model.name] = to_entity_meta(model)
+
+ def get_model(self, name: str) -> Optional[AudioGenerationModel]:
+ if name in self.models:
+ meta = self.models[name]
+ return get_entity(meta, AudioGenerationModel)
+ return None
+
+
+class AudioGenerationDriver(Generic[M], ABC):
+ @abstractmethod
+ def driver_name(self) -> str:
+ pass
+
+ @abstractmethod
+ def generate(self, prompt: Prompt, conf: M) -> Message:
+ pass
+
+
+class AudioGenerators(ABC):
+
+ @abstractmethod
+ def register(self, generator: AudioGenerationDriver):
+ pass
+
+ @abstractmethod
+ def get(self, model_name: str) -> Optional[AudioGenerationDriver]:
+ pass
+
+ @abstractmethod
+ def get_model_conf(self, model_name: str) -> AudioGenerationModel:
+ pass
+
+ @abstractmethod
+ def generate(self, prompt: Prompt, model: str = "") -> Message:
+ pass
diff --git a/ghostos/core/models/embedding.py b/ghostos/core/models/embedding.py
new file mode 100644
index 00000000..c145c149
--- /dev/null
+++ b/ghostos/core/models/embedding.py
@@ -0,0 +1,18 @@
+from typing import Tuple, List
+from numpy import ndarray
+from abc import ABC, abstractmethod
+
+
+class Embeddings(ABC):
+
+ @abstractmethod
+ def get_embedding(self, lang: str, model: str = "") -> ndarray[float]:
+ pass
+
+ @abstractmethod
+ def similarity(self, lang: str, compare: str, model: str = "") -> float:
+ pass
+
+ @abstractmethod
+ def search(self, query: str, selections: List[str], top_k: int, threshold: float, model: str = "") -> float:
+ pass
diff --git a/ghostos/core/models/speech_to_text.py b/ghostos/core/models/speech_to_text.py
new file mode 100644
index 00000000..67819f30
--- /dev/null
+++ b/ghostos/core/models/speech_to_text.py
@@ -0,0 +1,14 @@
+from abc import ABC, abstractmethod
+from ghostos.contracts.assets import FileInfo
+from ghostos.core.messages import Message
+
+
+class SpeechToTextDriver(ABC):
+ pass
+
+
+class SpeechToText(ABC):
+
+ @abstractmethod
+ def transcript(self, file: FileInfo, model: str = "") -> Message:
+ pass
diff --git a/evaluation/swe_bench_lite/tools/file_system_helper.py b/ghostos/core/models/text_to_speech.py
similarity index 100%
rename from evaluation/swe_bench_lite/tools/file_system_helper.py
rename to ghostos/core/models/text_to_speech.py
diff --git a/ghostos/core/moss/__init__.py b/ghostos/core/moss/__init__.py
index a98968bc..5283eae8 100644
--- a/ghostos/core/moss/__init__.py
+++ b/ghostos/core/moss/__init__.py
@@ -1,51 +1,38 @@
-from ghostos.container import Container, Provider
-from ghostos.core.moss.abc import (
- Moss, MossCompiler, MossRuntime, MossPrompter, MossResult,
- AttrPrompts,
- MOSS_NAME, MOSS_TYPE_NAME, MOSS_HIDDEN_MARK, MOSS_HIDDEN_UNMARK,
- MOSS_EXEC_EVENT, MOSS_PROMPT_EVENT, MOSS_COMPILE_EVENT, MOSS_ATTR_PROMPTS_EVENT,
- moss_message,
-)
-from ghostos.core.moss.impl import TestMOSSProvider
-from ghostos.core.moss.test_suites import MossTestSuite
-from ghostos.core.moss.pycontext import PyContext, Injection, Property, attr, SerializableType, SerializableData
-from ghostos.core.moss.functional_token import (
- DEFAULT_MOSS_FUNCTIONAL_TOKEN,
- DEFAULT_MOSS_PROMPT_TEMPLATE,
- get_default_moss_prompt,
+from ghostos.container import Container
+from ghostos.core.moss.abcd import (
+ Moss, MossCompiler, MossRuntime, MossPrompter, Execution,
+ AttrPrompts, Injection,
+ MOSS_VALUE_NAME, MOSS_TYPE_NAME, MOSS_HIDDEN_MARK, MOSS_HIDDEN_UNMARK,
)
+from ghostos.core.moss.impl import DefaultMOSSProvider
+from ghostos.core.moss.testsuite import MossTestSuite
+from ghostos.core.moss.pycontext import PyContext
__all__ = [
# abstract contracts
- Moss, MossCompiler, MossRuntime, MossPrompter, MossResult,
+ Moss, MossCompiler, MossRuntime, MossPrompter, Execution,
# constants
- MOSS_NAME, MOSS_TYPE_NAME, MOSS_HIDDEN_MARK, MOSS_HIDDEN_UNMARK,
- MOSS_EXEC_EVENT, MOSS_PROMPT_EVENT, MOSS_COMPILE_EVENT, MOSS_ATTR_PROMPTS_EVENT,
- DEFAULT_MOSS_FUNCTIONAL_TOKEN,
- DEFAULT_MOSS_PROMPT_TEMPLATE,
- # methods
- moss_message,
- get_default_moss_prompt,
+ MOSS_VALUE_NAME, MOSS_TYPE_NAME, MOSS_HIDDEN_MARK, MOSS_HIDDEN_UNMARK,
# types
AttrPrompts,
# pycontext related
- PyContext, Injection, Property, attr, SerializableType, SerializableData,
+ PyContext,
# testing
- TestMOSSProvider,
+ DefaultMOSSProvider,
MossTestSuite,
- 'test_container',
+ 'moss_container',
'moss_test_suite',
]
-def test_container() -> Container:
+def moss_container() -> Container:
"""
test container for Moss
"""
from ghostos.contracts.modules import DefaultModulesProvider
container = Container()
- container.register(TestMOSSProvider())
+ container.register(DefaultMOSSProvider())
container.register(DefaultModulesProvider())
return container
@@ -54,5 +41,5 @@ def moss_test_suite() -> MossTestSuite:
"""
return a MossTestSuite
"""
- container = test_container()
+ container = moss_container()
return MossTestSuite(container)
diff --git a/ghostos/core/moss/abc.py b/ghostos/core/moss/abcd.py
similarity index 77%
rename from ghostos/core/moss/abc.py
rename to ghostos/core/moss/abcd.py
index 8734a7a6..be4e8102 100644
--- a/ghostos/core/moss/abc.py
+++ b/ghostos/core/moss/abcd.py
@@ -1,14 +1,12 @@
-from typing import Dict, Any, Union, List, Optional, NamedTuple, Type, Callable
+from __future__ import annotations
+from typing import Dict, Any, Union, List, Optional, NamedTuple, Type, Callable, Self, TypeVar, ClassVar
from types import ModuleType
from abc import ABC, abstractmethod
from ghostos.container import Container, Provider, Factory, provide
-from ghostos.core.moss.pycontext import PyContext, attr
+from ghostos.core.moss.pycontext import PyContext
from ghostos.core.moss.prompts import (
- AttrPrompts, reflect_module_locals, PROMPT_MAGIC_ATTR,
- compile_attr_prompts,
+ AttrPrompts, reflect_module_locals, compile_attr_prompts
)
-from ghostos.core.messages import Message, Role
-from ghostos.core.moss.decorators import cls_source_code
"""
MOSS 是 Model-oriented Operating System Simulation 的简写.
@@ -49,40 +47,27 @@
"""
__all__ = [
- 'Moss', 'attr',
+ 'Moss',
'MossCompiler', 'MossRuntime',
- 'MossResult', 'MossPrompter',
- 'moss_message',
+ 'Execution', 'MossPrompter',
+ # 'moss_message',
'AttrPrompts',
- 'MOSS_COMPILE_EVENT', 'MOSS_PROMPT_EVENT', 'MOSS_EXEC_EVENT', 'MOSS_ATTR_PROMPTS_EVENT',
- 'MOSS_TYPE_NAME', 'MOSS_NAME',
+ 'MOSS_TYPE_NAME', 'MOSS_VALUE_NAME',
'MOSS_HIDDEN_MARK', 'MOSS_HIDDEN_UNMARK',
+ 'Injection',
]
-MOSS_COMPILE_EVENT = "__moss_compile__"
-"""moss 编译阶段的回调事件, 可以在对应文件里自定义这个事件, 替换系统默认. """
-
-MOSS_ATTR_PROMPTS_EVENT = "__moss_attr_prompts__"
-"""通过这个属性来获取一个实例 (module/instance of class) 所有属性的 prompts. """
-
-MOSS_PROMPT_EVENT = "__moss_prompt__"
-""" moss 生成 prompt 阶段的回调事件. """
-
-MOSS_EXEC_EVENT = "__moss_exec__"
-""" moss 执行阶段的回调事件. """
-
MOSS_TYPE_NAME = "Moss"
-MOSS_NAME = "moss"
+MOSS_VALUE_NAME = "moss"
-MOSS_HIDDEN_MARK = "# "
+MOSS_HIDDEN_MARK = "# "
""" pycontext.module 源码某一行以这个标记开头, 其后的代码都不生成到 prompt 里. """
-MOSS_HIDDEN_UNMARK = "# "
+MOSS_HIDDEN_UNMARK = "# "
""" pycontext.module 源码某一行以这个标记开头, 其后的代码都展示到 prompt 里. """
-@cls_source_code()
class Moss(ABC):
"""
Language Model-oriented Operating System Simulation.
@@ -91,14 +76,37 @@ class Moss(ABC):
SerializeType means: int, float, str, None, list, dict, BaseModel, TypedDict
You can edit them if you need.
"""
- pass
+ T = TypeVar('T')
-def moss_message(content: str, memory: Optional[str] = None) -> Message:
- """
- default message type that MOSS execution generated
- """
- return Role.ASSISTANT.new(content=content, memory=memory, name="__moss__")
+ executing_code: Optional[str]
+ """the code that execute the moss instance."""
+
+ @abstractmethod
+ def fetch(self, abstract: Type[T]) -> Optional[T]:
+ """
+ fetch an implementation from IoC Container
+ if the abstract type is not bound with any implementation, return None.
+ """
+ pass
+
+ @abstractmethod
+ def pprint(self, *args, **kwargs) -> None:
+ """
+ pretty printer
+ """
+ pass
+
+
+class Injection(ABC):
+
+ @abstractmethod
+ def on_inject(self, runtime: MossRuntime, property_name: str) -> Self:
+ pass
+
+ @abstractmethod
+ def on_destroy(self) -> None:
+ pass
class MossCompiler(ABC):
@@ -149,10 +157,6 @@ def with_locals(self, **kwargs) -> "MossCompiler":
"""
pass
- @abstractmethod
- def with_ignore_prompts(self, *attr_names) -> "MossCompiler":
- pass
-
def register(self, provider: Provider) -> None:
"""
向生成 MOSS 的 IoC 容器里注册 Provider.
@@ -198,6 +202,7 @@ def compile(
正式编译出一个 MOSSRuntime. 每一个 Compiler 只能编译一次.
:param modulename: 生成的 ModuleType 所在的包名. 相关代码会在这个临时 ModuleType 里生成. 如果 modulename为空, 则使用原来的
"""
+ from ghostos.core.moss.lifecycle import __moss_compile__
if self.__compiling__:
raise RuntimeError('recursively calling compile method')
if self.__compiled__:
@@ -207,10 +212,10 @@ def compile(
# 使用 locals 和 pycontext.module 对应的代码, 生成 ModuleType.
module = self._compile(modulename)
# 在生成的 ModuleType 里查找魔术方法, 提供 provider 等, 为依赖注入做准备.
- if hasattr(module, MOSS_COMPILE_EVENT):
+ if hasattr(module, __moss_compile__.__name__):
# 完成编译 moss_compiled 事件.
# 可以在这个环节对 MOSS 做一些依赖注入的准备.
- fn = getattr(module, MOSS_COMPILE_EVENT)
+ fn = getattr(module, __moss_compile__.__name__)
fn(self)
runtime = self._new_runtime(module)
return runtime
@@ -218,7 +223,7 @@ def compile(
self.__compiling__ = False
self.__compiled__ = True
# 手动管理一下, 避免外部解决内存泄漏的心智成本.
- self.destroy()
+ self.close()
@abstractmethod
def _compile(self, modulename: Optional[str] = None) -> ModuleType:
@@ -237,12 +242,18 @@ def _new_runtime(self, module: ModuleType) -> "MossRuntime":
pass
@abstractmethod
- def destroy(self) -> None:
+ def close(self) -> None:
"""
主动做垃圾回收的准备, 避免 python 内存泄漏.
"""
pass
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ self.close()
+
class MossPrompter(ABC):
"""
@@ -268,16 +279,16 @@ def module(self) -> ModuleType:
@abstractmethod
def pycontext_code(
self,
- exclude_moss_mark_code: bool = True,
+ exclude_hide_code: bool = True,
) -> str:
"""
返回通过 pycontext.module 预定义的代码.
第一行应该是 from __future__ import annotations. 解决上下文乱续的提示问题.
- :param exclude_moss_mark_code: 如果为 True, 只返回大模型可以阅读的代码.
+ :param exclude_hide_code: 如果为 True, 只返回大模型可以阅读的代码.
"""
pass
- def pycontext_attr_prompts(self, excludes: Optional[set] = None) -> AttrPrompts:
+ def reflect_module_attr(self) -> AttrPrompts:
"""
结合已编译的本地变量, 用系统自带的方法反射出上下文属性的 prompts.
"""
@@ -287,41 +298,42 @@ def pycontext_attr_prompts(self, excludes: Optional[set] = None) -> AttrPrompts:
yield from reflect_module_locals(
name,
local_values,
- excludes=excludes,
- excludes_module_prefixes={'pydantic', 'typing'},
)
- def pycontext_code_prompt(self, auto_generation: bool = True) -> str:
+ def dump_attrs_prompt(self, auto_generation: bool = True) -> str:
"""
基于 pycontext code 生成的 Prompt. 用来描述当前上下文里的各种变量.
主要是从其它库引入的变量.
:return: 用 python 风格描述的上下文变量.
"""
+ from ghostos.core.moss.lifecycle import __moss_attr_prompts__
done = {}
names = []
# 查看是否有源码自带的魔术方法.
module = self.module()
- if hasattr(module, MOSS_ATTR_PROMPTS_EVENT):
- fn = getattr(module, MOSS_ATTR_PROMPTS_EVENT)
+ if hasattr(module, __moss_attr_prompts__.__name__):
+ fn = getattr(module, __moss_attr_prompts__.__name__)
predefined_prompts: AttrPrompts = fn()
for name, prompt in predefined_prompts:
+ if prompt is None:
+ continue
if name and name not in done:
names.append(name)
done[name] = prompt
# 合并系统自动生成的.
if auto_generation:
- attr_prompts = self.pycontext_attr_prompts(excludes=set(names))
+ attr_prompts = self.reflect_module_attr()
for name, prompt in attr_prompts:
if name not in done:
names.append(name)
- done[name] = prompt
+ done[name] = prompt
# 保证一下顺序.
prompts = [(name, done[name]) for name in names]
- return compile_attr_prompts(self.module(), prompts)
+ return compile_attr_prompts(prompts)
- def dump_context_prompt(self) -> str:
+ def dump_module_prompt(self) -> str:
"""
获取 MOSS 运行时的完整 Python context 的 Prompt.
这个 Prompt 包含以下几个部分:
@@ -329,20 +341,22 @@ def dump_context_prompt(self) -> str:
2. pycontext_code_prompt: 对 predefined code 里各种引用类库的描述 prompt. 会包裹在 `\"""` 中展示.
3. moss_prompt: moss 会注入到当前上下文里, 因此会生成 MOSS Prompt.
"""
+ from ghostos.core.moss.lifecycle import __moss_module_prompt__
compiled = self.module()
- # 使用目标 module 自带的 prompt, 不做任何干预.
- if PROMPT_MAGIC_ATTR in compiled.__dict__:
- fn = compiled.__dict__[PROMPT_MAGIC_ATTR]
- return fn()
# 基于 moss prompter 来生成.
- if hasattr(compiled, MOSS_PROMPT_EVENT):
- fn = getattr(compiled, MOSS_PROMPT_EVENT)
+ if hasattr(compiled, __moss_module_prompt__.__name__):
+ fn = getattr(compiled, __moss_module_prompt__.__name__)
return fn(self)
- from ghostos.core.moss.lifecycle import __moss_prompt__
- return __moss_prompt__(self)
+ return __moss_module_prompt__(self)
+
+ @abstractmethod
+ def moss_injections_prompt(self) -> str:
+ pass
class MossRuntime(ABC):
+ instance_count: ClassVar[int] = 0
+
@abstractmethod
def container(self) -> Container:
"""
@@ -350,6 +364,13 @@ def container(self) -> Container:
"""
pass
+ @abstractmethod
+ def lint_exec_code(self, code: str) -> Optional[str]:
+ """
+ lint execution code and return error info if error occurs
+ """
+ pass
+
@abstractmethod
def prompter(self) -> MossPrompter:
"""
@@ -373,20 +394,30 @@ def locals(self) -> Dict[str, Any]:
pass
@abstractmethod
- def moss(self) -> object:
+ def moss_injections(self) -> Dict[str, Any]:
+ """
+ get injections from moss
+ """
+ pass
+
+ @abstractmethod
+ def moss(self) -> Moss:
"""
基于上下文生成的 MOSS. 依赖注入已经完成.
"""
pass
- def moss_type(self) -> Type:
+ def moss_type(self) -> Type[Moss]:
"""
get defined MOSS type
:return: MOSS class
"""
module = self.module()
if MOSS_TYPE_NAME in module.__dict__:
- return module.__dict__[MOSS_TYPE_NAME]
+ moss_type = module.__dict__[MOSS_TYPE_NAME]
+ if not issubclass(moss_type, Moss):
+ raise TypeError(f"Moss type {moss_type} is not subclass of {Moss}")
+ return moss_type
return Moss
@abstractmethod
@@ -405,7 +436,7 @@ def dump_std_output(self) -> str:
pass
@abstractmethod
- def runtime_ctx(self):
+ def redirect_stdout(self):
"""
with runtime.exec_ctx():
...
@@ -424,7 +455,7 @@ def execute(
local_kwargs: Optional[Dict[str, str]] = None,
args: Optional[List[Any]] = None,
kwargs: Optional[Dict[str, Any]] = None,
- ) -> "MossResult":
+ ) -> "Execution":
"""
基于 moos 提供的上下文, 运行一段代码.
:param code: 需要运行的代码.
@@ -436,18 +467,19 @@ def execute(
:return: 根据 result_name 从 code 中获取返回值.
:exception: any exception will be raised, handle them outside
"""
+ from ghostos.core.moss.lifecycle import __moss_exec__
+ self.update_executing_code(code)
if self.__executing__:
raise RuntimeError(f"Moss already executing")
try:
self.__executing__ = True
compiled = self.module()
fn = None
- with self.runtime_ctx():
+ with self.redirect_stdout():
# 使用 module 自定义的 exec
- if hasattr(compiled, MOSS_EXEC_EVENT):
- fn = getattr(compiled, MOSS_EXEC_EVENT)
+ if hasattr(compiled, __moss_exec__.__name__):
+ fn = getattr(compiled, __moss_exec__.__name__)
if fn is None:
- from ghostos.core.moss.lifecycle import __moss_exec__
fn = __moss_exec__
# 使用系统默认的 exec
return fn(
@@ -462,14 +494,25 @@ def execute(
finally:
self.__executing__ = False
- def destroy(self) -> None:
+ @abstractmethod
+ def update_executing_code(self, code: Optional[str] = None) -> None:
+ pass
+
+ @abstractmethod
+ def close(self) -> None:
"""
方便垃圾回收.
"""
pass
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ self.close()
+
-class MossResult(NamedTuple):
+class Execution(NamedTuple):
"""
result of the moss runtime execution.
"""
diff --git a/ghostos/core/moss/decorators.py b/ghostos/core/moss/decorators.py
index fc7b4bd6..5d713a9d 100644
--- a/ghostos/core/moss/decorators.py
+++ b/ghostos/core/moss/decorators.py
@@ -1,6 +1,6 @@
import inspect
from typing import Callable, Optional, Any, Type
-from ghostos.core.moss.prompts import set_prompter, set_class_prompter
+from ghostos.prompter import set_prompt, set_class_prompt
from ghostos.core.moss.utils import (
get_callable_definition, make_class_prompt,
strip_source_indent,
@@ -28,11 +28,10 @@ def no_prompt(func: Callable) -> Callable:
no_prompt.__prompt__ = ""
-def cls_source_code(*, force: bool = False, doc: Optional[str] = None) -> DECORATOR:
+def cls_source_code(*, force: bool = False) -> DECORATOR:
"""
decorator that add source code as prompt to the class
:param force: if force true, add prompt event the prompter exists in target
- :param doc: docstring that shall replace the source code's docstring
"""
def decorator(cls: Type) -> Type:
@@ -41,7 +40,7 @@ def prompter():
source = strip_source_indent(source)
return source
- set_class_prompter(cls, prompter, force)
+ set_class_prompt(cls, prompter, force)
return cls
return decorator
@@ -57,12 +56,12 @@ def decorator(fn: Callable) -> Callable:
if not (inspect.isfunction(fn) or inspect.ismethod(fn)):
raise AttributeError(f"fn '{fn}' has to be a function or method")
- def prompter():
+ def prompter() -> str:
source = inspect.getsource(fn)
source = strip_source_indent(source)
return source
- set_prompter(fn, prompter, force)
+ set_prompt(fn, prompter, force)
return fn
return decorator
@@ -81,7 +80,7 @@ def prompter() -> str:
prompt = get_callable_definition(fn, doc=doc)
return prompt
- set_prompter(fn, prompter, force)
+ set_prompt(fn, prompter, force)
else:
raise AttributeError(f"fn '{fn}' has to be a function or method")
return fn
@@ -103,7 +102,7 @@ def prompter() -> str:
prompt = make_class_prompt(source=source, doc=doc)
return prompt
- set_class_prompter(cls, prompter, force)
+ set_class_prompt(cls, prompter, force)
return cls
return wrapper
@@ -169,7 +168,7 @@ def prompter() -> str:
# 5. return
return combined_prompt
- set_class_prompter(cls, prompter, force)
+ set_class_prompt(cls, prompter, force)
return cls
return wrapper
diff --git a/ghostos/core/moss/examples/baseline.py b/ghostos/core/moss/examples/baseline.py
index 6ab12bc8..ccf300d0 100644
--- a/ghostos/core/moss/examples/baseline.py
+++ b/ghostos/core/moss/examples/baseline.py
@@ -1,8 +1,11 @@
-import logging
from abc import ABC, abstractmethod
-from typing import Optional, List
-from ghostos.core.moss.abc import Moss as Parent, attr
-from inspect import getsource, getmembers
+from typing import List
+
+from ghostos.container import Container
+from ghostos.core.moss.abcd import Moss as Parent
+from ghostos.prompter import ModelPrompter
+from inspect import getmembers, getsource
+from pydantic import BaseModel
class Foo(ABC):
@@ -18,25 +21,37 @@ def plus(a: int, b: int) -> int:
return a + b
-class Moss(Parent):
+class TestPrompter(ModelPrompter):
+ line: str = "TestPrompter"
+
+ def self_prompt(self, container: Container) -> str:
+ return self.line
+
+ def get_title(self) -> str:
+ return ""
+
+
+class Moss(Parent, ABC):
"""
本地定义的 Moss 类. 每个 MOSS 文件里都应该有一个 Moss 类, 可以是 import 的也可以是本地定义的.
记得它要继承自 Moss.
"""
- life: List[str] = attr(default_factory=list, desc="用来记录发生过的生命周期.")
+ life: List[str] = []
"""测试 attr 方法用来定义可持久化的属性. """
foo: Foo
"""依赖注入 Foo 的测试用例. """
+ tester: TestPrompter
+
-#
-# !!! 使用 `# ` 和 `# ` 包裹的代码不会对大模型呈现.
+#
+# !!! 使用 `# ` 和 `# ` 包裹的代码不会对大模型呈现.
from typing import TYPE_CHECKING
if TYPE_CHECKING:
- from ghostos.core.moss.abc import MossCompiler, MossRuntime, AttrPrompts, MossPrompter, MossResult
+ from ghostos.core.moss.abcd import MossCompiler, AttrPrompts, MossPrompter, Execution
def __moss_compile__(compiler: "MossCompiler") -> "MossCompiler":
@@ -47,7 +62,7 @@ def __moss_compile__(compiler: "MossCompiler") -> "MossCompiler":
主要解决各种注入方面的需求:
"""
# 单测里应该有这个. moss.bar == 123
- compiler.injects(bar=123)
+ compiler.injects(bar=123, tester=TestPrompter())
# 插入生命周期事件, 直接赋值到 moss 上.
Moss.life.append("__moss_compile__")
@@ -65,7 +80,6 @@ def foo(self) -> str:
def __moss_attr_prompts__() -> "AttrPrompts":
- Moss.life.append("__moss_attr_prompts__")
return [
# 重写了 getsource 的 prompt, 它就应该不存在了.
("getsource", ""),
@@ -76,14 +90,12 @@ def __moss_attr_prompts__() -> "AttrPrompts":
def __moss_prompt__(prompter: "MossPrompter") -> str:
# 测试生命周期生效.
- Moss.life.append("__moss_prompt__")
- from ghostos.core.moss.lifecycle import __moss_prompt__
- return __moss_prompt__(prompter)
+ from ghostos.core.moss.lifecycle import __moss_module_prompt__
+ return __moss_module_prompt__(prompter)
-def __moss_exec__(*args, **kwargs) -> "MossResult":
+def __moss_exec__(*args, **kwargs) -> "Execution":
# 测试生命周期生效.
- Moss.life.append("__moss_exec__")
from ghostos.core.moss.lifecycle import __moss_exec__
return __moss_exec__(*args, **kwargs)
@@ -105,4 +117,4 @@ def main(moss: Moss) -> int:
"""
return plus(2, 2)
-#
\ No newline at end of file
+#
diff --git a/ghostos/core/moss/examples/mem_baseline.py b/ghostos/core/moss/examples/mem_baseline.py
deleted file mode 100644
index 6f7cb887..00000000
--- a/ghostos/core/moss/examples/mem_baseline.py
+++ /dev/null
@@ -1,57 +0,0 @@
-from ghostos.core.moss.abc import Moss as Parent, attr
-from ghostos.mocks.libraries.auto_text_memory import Mem0TextMemory
-from ghostos.framework.libraries.auto_memory import ProxyConfig
-
-
-class Moss(Parent):
- """
- 本地定义的 Moss 类. 每个 MOSS 文件里都应该有一个 Moss 类, 可以是 import 的也可以是本地定义的.
- 记得它要继承自 Moss.
- """
- text_memory: Mem0TextMemory
-
-
-#
-
-def test_main(moss: Moss) -> int:
- """
- 模拟一个 main 方法, 测试 moss 的调用.
- assert 返回值是 3. 外部的 MOSSRuntime 调用这个方法.
- """
- import os
-
- openai_proxy = os.environ.get('OPENAI_PROXY')
- if openai_proxy:
- moss.text_memory = Mem0TextMemory(proxy_config=ProxyConfig(proxy_url=openai_proxy))
- else:
- moss.text_memory = Mem0TextMemory()
-
- m = moss.text_memory
- # 1. Add: Store a memory from any unstructured text
- result = m.add("I am working on improving my tennis skills. Suggest some online courses.", agent_id="alice")
- print(result)
- all_memories = m.get_all()
- memory_id = all_memories[0]["id"] # get a memory_id
-
- # Created memory --> 'Improving her tennis skills.' and 'Looking for online suggestions.'
-
- # 2. Update: update the memory
- result = m.update(memory_id=memory_id, data="Likes to play tennis on weekends")
- print(result)
-
- # Updated memory --> 'Likes to play tennis on weekends.' and 'Looking for online suggestions.'
-
- # 3. Search: search related memories
- related_memories = m.search(query="What are Alice do on weekends ?", agent_id="alice")
- print(related_memories)
-
- # 5. Get memory history for a particular memory_id
- history = m.history(memory_id=memory_id)
- print(history)
-
-#
-
-
-if __name__ == "__main__":
- test_main(Moss())
-
diff --git a/ghostos/core/moss/examples/test_suite.py b/ghostos/core/moss/examples/test_suite.py
index 54ebf23a..722cc58c 100644
--- a/ghostos/core/moss/examples/test_suite.py
+++ b/ghostos/core/moss/examples/test_suite.py
@@ -1,11 +1,11 @@
-from ghostos.core.moss.abc import Moss
+from ghostos.core.moss.abcd import Moss
def plus(a: int, b: int) -> int:
return a + b
-#
+#
if __name__ == '__test__':
"""
可以这样定义只在当前文件编译成 modulename=__test__ 才运行的方法.
@@ -27,4 +27,4 @@ def test_3(moss: Moss) -> int:
__moss_test_cases__ = ['test_1', 'test_2', 'test_3']
"""用这个魔术变量, 可以让 MossTestSuit 批量调用三个方法测试. """
-#
+#
diff --git a/ghostos/core/moss/exports.py b/ghostos/core/moss/exports.py
deleted file mode 100644
index b4c2baa5..00000000
--- a/ghostos/core/moss/exports.py
+++ /dev/null
@@ -1,173 +0,0 @@
-from typing import Any, Dict, Iterable, Tuple, Optional, Callable, List
-from ghostos.core.moss.utils import make_class_prompt, get_callable_definition
-import inspect
-
-__all__ = ['Exporter']
-
-
-class Exporter(object):
- """
- Exporter is useful to export multiple objects with prompts from a module
- The Subject module can import a exporter instance from a object module,
- the prompt generate from the exporter like this:
-
- > from foo.bar import baz
- >
- > # value of baz.a
- > class A:
- > ...
- > # value of baz.b
- > def B():
- > ...
-
- Exporter is harmless then the moss decorators.
- """
-
- def __init__(self, **kwargs):
- self._prompts: Dict[str, str] = {}
- # with kwargs values
- for name, value in kwargs.items():
- if isinstance(value, Exporter):
- self.with_exporter(name, value)
- elif inspect.isclass(value):
- self.with_class(value, name=name)
- elif inspect.isfunction(value):
- self.with_func(value, name=name)
- else:
- self.with_raw(name, value, "")
-
- def prompts(self) -> Iterable[Tuple[str, str]]:
- """
- iterate the attr's prompt of the Exporter
- :return: A generator that yields tuples of attr name and prompt value in the Exporter.
- """
- return self._prompts.items()
-
- def gene_prompt(self, self_name: str) -> str:
- """
- this method is used in other module.
- generate prompt for the Exporter with a attribute name in other module.
- :param self_name: attribute name of the Exporter in other module.
- :return: full prompt
- """
- lines = []
- for attr, prompt in self.prompts():
- comment = f"# value of {self_name}.{attr}"
- lines.append(f"{comment}:\n{prompt}")
- return "\n\n".join(lines)
-
- def with_raw(self, name: str, value: Any, prompt: str) -> "Exporter":
- """
- add a attribute to the Exporter with a specific prompt.
- :param name: attribute name in the Exporter
- :param value: real value
- :param prompt: predefined prompt
- :return: self, chain calling.
- """
- if name in self.__dict__:
- raise NameError(f"'{name}' already exists in Exporter")
- self.__dict__[name] = value
- if not prompt:
- prompt = f"# {value}"
- self._prompts[name] = prompt
- return self
-
- def with_class(self, cls: type, *, abc: Optional[type] = None, name: Optional[str] = None) -> "Exporter":
- """
- add a class attribute to the Exporter. prompt will be the class source code.
- :param cls: the class type
- :param abc: if given, the prompt is the abc class's source code
- :param name: if not given, the attribute name will be the class name
- :return: self, chain calling.
- """
- if abc is not None:
- prompt = inspect.getsource(abc)
- else:
- prompt = inspect.getsource(cls)
- if name is None:
- name = cls.__name__
- return self.with_raw(name, cls, prompt)
-
- def with_interface(
- self,
- cls: type,
- members: Optional[List[str]] = None,
- *,
- doc: Optional[str] = None,
- name: Optional[str] = None,
- ) -> "Exporter":
- """
- add a class attribute to the Exporter.
- prompt will be interface pattern, which means class definition plus public method definitions.
-
- :param cls: the value class
- :param members: method name that should be added. if none, all public methods will be added.
- :param doc: if given, replace the class docstring in the prompt
- :param name: if not given, using class name as attribute name
- """
- if name is None:
- name = cls.__name__
- source = inspect.getsource(cls)
- prompt = make_class_prompt(source=source, name=name, attrs=members, doc=doc)
- return self.with_raw(name, cls, prompt)
-
- def with_func(self, func: Callable, *, doc: Optional[str] = None, name: Optional[str] = None) -> "Exporter":
- """
- add a function attribute to the Exporter. prompt will be the function definition and doc.
- :param func:
- :param doc: if given, the function's doc in the prompt will be replaced by the argument.
- :param name: if not given, the attribute name will be the function name
- """
- prompt = get_callable_definition(func, doc=doc)
- if name is None:
- name = func.__name__
- return self.with_raw(name, func, prompt)
-
- def with_exporter(self, name: str, value: "Exporter") -> "Exporter":
- """
- add another exporter to the Exporter.
- prompt of each attribute in the value will be handled like:
- self_name.self_attr_name.value_attr_name => prompt
- """
- for attr, prompt in value.prompts():
- real_name = f"{name}.{attr}"
- self._prompts[real_name] = prompt
- self.__dict__[name] = value
- return self
-
-
-# --- tests --- #
-
-class Foo:
-
- def foo(self):
- return 123
-
-
-class Bar(Foo):
- pass
-
-
-tester = (Exporter(Any=Any, Dict=Dict)
- .with_interface(Exporter, ['with_func'], name="exp1")
- .with_class(Foo, name="foo")
- .with_class(Bar, abc=Foo)
- .with_func(make_class_prompt)
- .with_func(make_class_prompt, name="make_cls_pr", doc="hello"))
-
-
-def test_each_value_of_tester():
- values = {
- "Any": Any,
- "Dict": Dict,
- "exp1": Exporter,
- "foo": Foo,
- "make_class_prompt": make_class_prompt,
- "make_cls_pr": make_class_prompt,
- }
- for attr, value in values.items():
- assert getattr(tester, attr) is value
-
-
-def test_gen_prompt():
- print(tester.gene_prompt("tester"))
diff --git a/ghostos/core/moss/functional_token.py b/ghostos/core/moss/functional_token.py
deleted file mode 100644
index a234d39b..00000000
--- a/ghostos/core/moss/functional_token.py
+++ /dev/null
@@ -1,72 +0,0 @@
-from ghostos.core.llms import FunctionalToken
-from ghostos.core.moss.abc import MossPrompter
-from pydantic import BaseModel, Field
-
-__all__ = ['MOSSArgument', 'DEFAULT_MOSS_FUNCTIONAL_TOKEN', 'DEFAULT_MOSS_PROMPT_TEMPLATE', 'get_default_moss_prompt']
-
-
-class MOSSArgument(BaseModel):
- code: str = Field(description="generated moss code that include `def main(os: MOSS) -> Operator`")
-
-
-DEFAULT_MOSS_FUNCTIONAL_TOKEN = FunctionalToken(
- token=">moss:",
- name="moss",
- description="""
-You can output the Python code that MOSS is supposed to run after this token.
-The system will automatically execute them.
-Notice:
-- MOSS-related output is not visible to user.
-- You are only able to generate MOSS code within this token.
-- The content after this token shall be pure Python code only.
-- You can send anything directly before this token, not after it.
-- **Never** use ``` to embrace your code.
-- Need not to mention the code you generated to user.
-""".strip(),
- deliver=False,
- parameters=MOSSArgument.model_json_schema(),
-)
-
-DEFAULT_MOSS_PROMPT_TEMPLATE = """
-# MOSS
-
-You are equipped with the MOSS (Model-oriented Operating System) that provides tools and thought directions in python interface.
-With MOSS you shall generate a single block of Python code in which defines a function `def main(os: MOSS) -> Operator:`,
-the MOSS will automatically execute them.
-
-**Directives for MOSS**:
-- **Code Generation Only**: Produce a block of Python code for the `main` function.
- The interface, class and abstract methods in context are ALREADY implemented in external system,
- and passed into main as arguments, DON'T implement them or instantiate them again,
- just invoke them directly on you need.
-- **Format Requirement**: Your output must be a single block of Python code enclosed within triple backticks.
- Do not include any additional text, comments, or explanations outside this code block.
- Do not invoke main method by yourself.
-
-**External System Responsibilities**:
-- **Execution and Data Fetching**: The external system will concatenate your code with the true context
- (implemented all abstract methods and interface), execution the main method and wait to fetch the result.
-- **Result Handling**: The external system will process the results and manage user interactions.
- Std output will be buffed by MOSS, you can generate operator to observe them.
-
-
-Here is the context provided to you in this turn:
-
-```python
-{code}
-```
-
-**Notices**
-0. You need not to use MOSS when you don't need it's api such as tell raw text or use other functional tokens.
-1. MOSS is a private system only serve for you, **never mention it existence**.
-2. MOSS code shall be generated within functional token, markdown python block will not do, and **don't repeat the code with markdown**.
-3. MOSS will automatic execute the main function so you never execute it again.
-4. **Return Operator**: You shall always use method that MOSS provide you to return an Operator from function main.
-5. In the generated MOSS code, ** YOU SHALL NOT WRITE ANYTHING BUT CODE AND COMMENTS BECAUSE MOSS CODE NEVER SEND TO USER**.
-6. Your generated code must include `def main(os: MOSS) -> Operator` method which will be executed following your intention.
-"""
-
-
-def get_default_moss_prompt(prompter: MossPrompter) -> str:
- code_prompt = prompter.dump_context_prompt()
- return DEFAULT_MOSS_PROMPT_TEMPLATE.format(code=code_prompt)
diff --git a/ghostos/core/moss/impl.py b/ghostos/core/moss/impl.py
index 525286a1..e47e2df8 100644
--- a/ghostos/core/moss/impl.py
+++ b/ghostos/core/moss/impl.py
@@ -1,26 +1,45 @@
+import importlib
import inspect
from types import ModuleType
-from typing import Optional, Any, Dict, get_type_hints, Type, List
+from typing import Optional, Any, Dict, get_type_hints, Type, List, Callable, ClassVar
import io
from ghostos.container import Container, Provider
-from ghostos.core.moss.abc import (
+from ghostos.contracts.modules import Modules, ImportWrapper
+from ghostos.core.moss.abcd import (
Moss,
- MossCompiler, MossRuntime, MossPrompter, MOSS_NAME, MOSS_TYPE_NAME,
+ MossCompiler, MossRuntime, MossPrompter, MOSS_VALUE_NAME, MOSS_TYPE_NAME,
MOSS_HIDDEN_MARK, MOSS_HIDDEN_UNMARK,
+ Injection,
)
-from ghostos.contracts.modules import Modules, ImportWrapper
-from ghostos.core.moss.prompts import AttrPrompts
-from ghostos.core.moss.pycontext import PyContext, Property
-from ghostos.helpers import get_module_spec
+from ghostos.core.moss.pycontext import PyContext
+from ghostos.prompter import Prompter, TextPrmt
+from ghostos.helpers import generate_module_and_attr_name, code_syntax_check
from contextlib import contextmanager, redirect_stdout
IMPORT_FUTURE = "from __future__ import annotations"
+__all__ = [
+ 'MossStub',
+ 'MossCompilerImpl',
+ 'MossRuntimeImpl',
+ 'DefaultMOSSProvider',
+ 'MossTempModuleType',
+]
+
+
+class MossTempModuleType(ModuleType):
+ __instance_count__: ClassVar[int] = 0
+
+ def __del__(self):
+ if MOSS_VALUE_NAME in self.__dict__:
+ del self.__dict__[MOSS_VALUE_NAME]
+ MossTempModuleType.__instance_count__ -= 1
+
class MossCompilerImpl(MossCompiler):
def __init__(self, *, container: Container, pycontext: Optional[PyContext] = None):
- self._container = Container(parent=container)
+ self._container = Container(parent=container, name="moss")
self._pycontext = pycontext if pycontext else PyContext()
self._modules: Modules = self._container.force_fetch(Modules)
self._predefined_locals: Dict[str, Any] = {
@@ -29,6 +48,11 @@ def __init__(self, *, container: Container, pycontext: Optional[PyContext] = Non
}
self._injections: Dict[str, Any] = {}
self._attr_prompts: List = []
+ self._compiled = False
+ self._closed = False
+
+ def __del__(self):
+ self.close()
def container(self) -> Container:
return self._container
@@ -54,19 +78,24 @@ def injects(self, **attrs: Any) -> "MossCompiler":
return self
def _compile(self, modulename: Optional[str] = None) -> ModuleType:
+ origin: Optional[ModuleType] = None
+ filename = ""
+ origin_modulename = self._pycontext.module
+ if origin_modulename:
+ origin = importlib.import_module(origin_modulename)
+ filename = origin.__file__
+
if modulename is None:
- modulename = self._pycontext.module
- if not modulename:
- modulename = "__main__"
+ modulename = origin_modulename if origin_modulename else "__moss__"
code = self.pycontext_code()
# 创建临时模块.
- module = ModuleType(modulename)
+ module = MossTempModuleType(modulename)
+ MossTempModuleType.__instance_count__ += 1
module.__dict__.update(self._predefined_locals)
- module.__dict__['__file__'] = ""
+ module.__file__ = filename
compiled = compile(code, modulename, "exec")
exec(compiled, module.__dict__)
- if self._pycontext.module:
- origin = self._modules.import_module(self._pycontext.module)
+ if origin is not None:
updating = self._filter_origin(origin)
module.__dict__.update(updating)
return module
@@ -87,6 +116,7 @@ def _new_runtime(self, module: ModuleType) -> "MossRuntime":
attr_prompts = {}
for attr_name, value in self._attr_prompts:
attr_prompts[attr_name] = value
+ self._compiled = True
return MossRuntimeImpl(
container=self._container,
pycontext=self._pycontext.model_copy(deep=True),
@@ -108,27 +138,67 @@ def pycontext_code(self) -> str:
code = IMPORT_FUTURE + "\n\n" + code.lstrip("\n")
return code if code else ""
- def destroy(self) -> None:
+ def close(self) -> None:
+ if self._closed:
+ return
+ self._closed = True
# container 先不 destroy.
+ if not self._compiled:
+ self._container.shutdown()
del self._container
del self._pycontext
del self._predefined_locals
del self._injections
-def new_moss_stub(cls: Type[Moss], pycontext: PyContext):
+class MossStub(Moss):
+ instance_count: ClassVar[int] = 0
+ __pycontext__: PyContext
+ __container__: Container
+ __output__: str
+ __printer__: Callable
+
+ def __init__(self, pycontext: PyContext, container: Container, printer: Callable):
+ MossStub.instance_count += 1
+ self.__dict__['__pycontext__'] = pycontext
+ self.__dict__['__container__'] = container
+ self.__dict__['__output__'] = ""
+ self.__dict__['__printer__'] = printer
+
+ def fetch(self, abstract: Moss.T) -> Optional[Moss.T]:
+ return self.__container__.fetch(abstract)
+
+ def pprint(self, *args, **kwargs) -> None:
+ return self.__printer__(*args, **kwargs)
+
+ def __setattr__(self, _name, _value):
+ if self.__pycontext__.allow_prop(_value):
+ self.__pycontext__.set_prop(_name, _value)
+ self.__dict__[_name] = _value
+
+ def __del__(self):
+ MossStub.instance_count -= 1
+
+
+def new_moss_stub(cls: Type[Moss], container: Container, pycontext: PyContext, pprint: Callable) -> Moss:
# cls 必须不包含参数.
- obj = cls()
- obj.__pycontext__ = pycontext
- # 反向注入
+
+ stub = MossStub(pycontext, container, pprint)
+ stub.executing_code = None
+ # assert stub.instance_count > 0
+ for attr_name in dir(cls):
+ if not attr_name.startswith("_") and not hasattr(stub, attr_name):
+ attr_value = getattr(cls, attr_name)
+ setattr(stub, attr_name, attr_value)
+
+ # 反向注入.
for name, value in cls.__dict__.items():
- if not name.startswith('_') and isinstance(value, Property) and name not in pycontext.properties:
- pycontext.define(value)
- # 初始化 pycontext variable
- for name, var in pycontext.properties.items():
- # 直接用 property 作为值.
- setattr(obj, name, var)
- return obj
+ if name in pycontext.properties or name.startswith("_"):
+ continue
+ if stub.__pycontext__.allow_prop(value):
+ stub.__pycontext__.set_prop(name, value)
+
+ return stub
class MossRuntimeImpl(MossRuntime, MossPrompter):
@@ -151,52 +221,45 @@ def __init__(
self._runtime_std_output = ""
# 初始化之后不应该为 None 的值.
self._built: bool = False
- self._moss: Optional[Moss] = None
self._moss_prompt: Optional[str] = None
self._attr_prompts: Dict[str, str] = attr_prompts
- self._attr_prompts["print"] = ""
- self._bootstrap_moss()
+ self._closed: bool = False
+ self._injected = set()
+ self._moss: Moss = self._compile_moss()
+ self._initialize_moss()
+ MossRuntime.instance_count += 1
- def _bootstrap_moss(self):
- if self._built:
- return
- self._built = True
- self._compile_moss()
-
- def _compile_moss(self):
+ def _compile_moss(self) -> Moss:
moss_type = self.moss_type()
if not issubclass(moss_type, Moss):
- raise TypeError(f"Moss type {moss_type} is not subclass of Moss")
+ raise TypeError(f"Moss type {moss_type} is not subclass of {generate_module_and_attr_name(Moss)}")
# 创建 stub.
pycontext = self._pycontext
- for prop in pycontext.properties.values():
- # 基于上下文还原变量.
- prop.generate_value(self._compiled)
-
- moss = new_moss_stub(moss_type, pycontext)
-
- # 初始化 injection. 强制赋值.
- injections = self._injections.copy()
- # 初始化 pycontext injection. 替代掉系统默认的 injection.
- for name, injection in self._pycontext.injections.items():
- injection_module, injection_spec = injection.get_from_module_attr()
- # 如果要注入的对象就是当前包, 则直接返回当前包的和苏剧.
- if injection_module == self._compiled.__name__:
- module = get_module_spec(self._compiled, injection_spec)
- else:
- module = self._modules.import_module(injection_module)
- # 否则返回查找结果.
- if injection_spec:
- value = get_module_spec(module, injection_spec)
- else:
- value = module
- injections[name] = value
+ moss = new_moss_stub(moss_type, self._container, pycontext, self.pprint)
+ return moss
+
+ def _initialize_moss(self) -> None:
+ from .lifecycle import __moss_compiled__
+ moss = self._moss
+ pycontext = self._pycontext
+ moss_type = self.moss_type()
+
+ def inject(attr_name: str, injected: Any) -> Any:
+ if isinstance(injected, Injection):
+ injected.on_inject(self, attr_name)
+ setattr(moss, attr_name, injected)
+ self._injected.add(attr_name)
+
+ # 初始化 pycontext variable
+ for name, prop in pycontext.iter_props(self._compiled):
+ # 直接用 property 作为值.
+ inject(name, prop)
- # 将 Injections 直接注入.
- for name, value in injections.items():
- if name not in pycontext.properties:
- setattr(moss, name, value)
+ # 反向注入
+
+ for name, injection in self._injections.items():
+ inject(name, injection)
# 初始化基于容器的依赖注入.
typehints = get_type_hints(moss_type, localns=self._compiled.__dict__)
@@ -207,16 +270,18 @@ def _compile_moss(self):
# 已经有的就不再注入.
if hasattr(moss, name):
continue
- value = getattr(moss_type, name, None)
- if value is None:
- # 为 None 才依赖注入.
- value = self._container.get(typehint)
- # 依赖注入.
- setattr(moss, name, value)
- self._moss = moss
- self._compiled.__dict__[MOSS_NAME] = moss
+ # 为 None 才依赖注入.
+ value = self._container.force_fetch(typehint)
+ # 依赖注入.
+ inject(name, value)
+
+ self._compiled.__dict__[MOSS_VALUE_NAME] = moss
self._compiled.__dict__[MOSS_TYPE_NAME] = moss_type
- self._compiled.__dict__["print"] = self._print
+ fn = __moss_compiled__
+ if __moss_compiled__.__name__ in self._compiled.__dict__:
+ fn = self._compiled.__dict__[__moss_compiled__.__name__]
+ fn(moss)
+ self._moss = moss
def container(self) -> Container:
return self._container
@@ -224,13 +289,18 @@ def container(self) -> Container:
def prompter(self) -> MossPrompter:
return self
+ def lint_exec_code(self, code: str) -> Optional[str]:
+ source_code = self._source_code
+ new_code = source_code + "\n\n" + code.strip()
+ return code_syntax_check(new_code)
+
def module(self) -> ModuleType:
return self._compiled
def locals(self) -> Dict[str, Any]:
return self._compiled.__dict__
- def moss(self) -> object:
+ def moss(self) -> Moss:
return self._moss
def dump_pycontext(self) -> PyContext:
@@ -239,30 +309,53 @@ def dump_pycontext(self) -> PyContext:
def dump_std_output(self) -> str:
return self._runtime_std_output
- def _print(self, *args, **kwargs):
- buffer = io.StringIO()
- with redirect_stdout(buffer):
- print(*args, **kwargs)
- self._runtime_std_output += str(buffer.getvalue())
+ def pprint(self, *args: Any, **kwargs: Any) -> None:
+ from pprint import pprint
+ out = io.StringIO()
+ with redirect_stdout(out):
+ pprint(*args, **kwargs)
+ self._runtime_std_output += str(out.getvalue())
@contextmanager
- def runtime_ctx(self):
- yield
+ def redirect_stdout(self):
+ buffer = io.StringIO()
+ with redirect_stdout(buffer):
+ yield
+ self._runtime_std_output += str(buffer.getvalue())
def pycontext_code(
self,
- exclude_moss_mark_code: bool = True,
+ exclude_hide_code: bool = True,
) -> str:
code = self._source_code
- return self._parse_pycontext_code(code, exclude_moss_mark_code)
-
- def pycontext_attr_prompts(self, excludes: Optional[set] = None) -> AttrPrompts:
- yield from super().pycontext_attr_prompts(excludes=excludes)
- yield from self._attr_prompts.items()
+ return self._parse_pycontext_code(code, exclude_hide_code)
+
+ def moss_injections(self) -> Dict[str, Any]:
+ moss = self.moss()
+ injections = {}
+ for name in self._injected:
+ injection = getattr(moss, name)
+ injections[name] = injection
+ return injections
+
+ def moss_injections_prompt(self) -> str:
+ injections = self.moss_injections()
+ children = []
+ container = self.container()
+ for name, injection in injections.items():
+ if isinstance(injection, Prompter):
+ children.append(TextPrmt(
+ title=f"moss.{name}",
+ content=injection.self_prompt(container),
+ ))
+ prompter = TextPrmt(
+ title="Moss Injections",
+ ).with_children(*children)
+ return prompter.get_prompt(container)
@staticmethod
- def _parse_pycontext_code(code: str, exclude_moss_mark_code: bool = True) -> str:
- if not exclude_moss_mark_code:
+ def _parse_pycontext_code(code: str, exclude_hide_code: bool = True) -> str:
+ if not exclude_hide_code:
return code
lines = code.split("\n")
@@ -285,17 +378,34 @@ def _parse_pycontext_code(code: str, exclude_moss_mark_code: bool = True) -> str
return "\n".join(results)
- def destroy(self) -> None:
- self._container.destroy()
- if hasattr(self._moss, "__pycontext__"):
- del self._moss.__pycontext__
+ def update_executing_code(self, code: Optional[str] = None) -> None:
+ if code is None:
+ return
+ self._pycontext.execute_code = code
+ self._moss.executing_code = code
+
+ def close(self) -> None:
+ if self._closed:
+ return
+ self._closed = True
+ data = self._moss.__dict__
+ for val in data.values():
+ if isinstance(val, Injection):
+ val.on_destroy()
+ self._container.shutdown()
+
+ def __del__(self):
+ if not self._closed:
+ self.close()
+ MossRuntime.instance_count -= 1
del self._container
del self._injections
del self._compiled
del self._moss
+ del self._pycontext
-class TestMOSSProvider(Provider[MossCompiler]):
+class DefaultMOSSProvider(Provider[MossCompiler]):
"""
用于测试的标准 compiler.
但实际上好像也是这个样子.
@@ -308,4 +418,4 @@ def contract(self) -> Type[MossCompiler]:
return MossCompiler
def factory(self, con: Container) -> MossCompiler:
- return MossCompilerImpl(container=con, pycontext=PyContext())
+ return MossCompilerImpl(container=con, pycontext=None)
diff --git a/ghostos/core/moss/lifecycle.py b/ghostos/core/moss/lifecycle.py
index 2c59836d..2034ad8b 100644
--- a/ghostos/core/moss/lifecycle.py
+++ b/ghostos/core/moss/lifecycle.py
@@ -4,7 +4,7 @@
if TYPE_CHECKING:
from ghostos.core.moss.prompts import AttrPrompts
- from ghostos.core.moss.abc import MossPrompter, MossResult, MossRuntime, MossCompiler
+ from ghostos.core.moss.abcd import MossPrompter, Execution, MossRuntime, MossCompiler, Moss
"""
这个文件提供了 MOSS 生命周期的关键方法, 每一个都是可选的.
@@ -12,26 +12,14 @@
"""
__all__ = [
- '__moss_compile__',
- '__moss_attr_prompts__',
- '__moss_prompt__',
- '__moss_exec__',
+ '__moss_compile__', # prepare moss compiler, handle dependencies register
+ '__moss_compiled__', # when moss instance is compiled
+ '__moss_attr_prompts__', # generate custom local attr prompt
+ '__moss_module_prompt__', # define module prompt
+ '__moss_exec__', # execute the generated code attach to the module
]
-class MOSS(ABC):
- """
- 可以在代码里自定一个名为 MOSS 的类.
- 如果定义了, 或者引用了, 会自动生成它的实例.
- 会对类上定义的属性进行依赖注入.
- 对类上定义的方法名也按名称进行依赖注入. 没有注入对象时保留原方法.
- 这个类不要有 init 方法.
- 它的源码就是 MOSS 的 Prompt.
- 如果给 MOSS 注入了它未定义的属性或方法, 也会追加到它生成的 Prompt 里.
- """
- pass
-
-
def __moss_compile__(compiler: "MossCompiler") -> "MossCompiler":
"""
从 compile 中获取 MOSSRuntime, 并对它进行初始化.
@@ -44,13 +32,13 @@ def __moss_compile__(compiler: "MossCompiler") -> "MossCompiler":
return compiler
-def __prompt__() -> str:
+def __moss_compiled__(moss: "Moss") -> None:
"""
- 可选的魔术方法.
- 使用这个方法, 可以完全自定义当前文件生成的 prompt.
- 系统不做任何干预.
+
+ :param moss:
+ :return:
"""
- pass
+ return
def __moss_attr_prompts__() -> "AttrPrompts":
@@ -62,7 +50,7 @@ def __moss_attr_prompts__() -> "AttrPrompts":
默认的反射方法见 ghostos.moss.prompts.prompts.py 文件.
而这个方法则可以替代或追加必要的 prompt, 优先于系统生成的反射.
- 还有一些在 ', mode='exec')
- exec(compiled, local_values)
+ if code:
+ filename = pycontext.module if pycontext.module is not None else ""
+ compiled = compile(code, filename=filename, mode='exec')
+ exec(compiled, local_values)
if target not in local_values:
raise NotImplementedError(f"target `{target}` not implemented")
@@ -160,7 +152,7 @@ def __moss_exec__(
if kwargs:
real_kwargs.update(kwargs)
# 注意使用 runtime.exec_ctx 包裹有副作用的调用.
- with runtime.runtime_ctx():
+ with runtime.redirect_stdout():
returns = target_module_attr(*real_args, **real_kwargs)
elif has_args:
raise TypeError(f"target '{target}' value '{target_module_attr}' is not callable")
@@ -168,4 +160,5 @@ def __moss_exec__(
returns = target_module_attr
std_output = runtime.dump_std_output()
pycontext = runtime.dump_pycontext()
- return MossResult(returns, std_output, pycontext)
+ pycontext.executed = True
+ return Execution(returns, std_output, pycontext)
diff --git a/ghostos/core/moss/prompts.py b/ghostos/core/moss/prompts.py
index ac1fa55f..2165dcb7 100644
--- a/ghostos/core/moss/prompts.py
+++ b/ghostos/core/moss/prompts.py
@@ -1,21 +1,13 @@
-from typing import Any, Optional, Union, Callable, Dict, Tuple, Iterable, Set, Type
-from types import ModuleType
+from typing import Any, Optional, Dict, Tuple, Iterable
from ghostos.core.moss.utils import (
- unwrap_str,
- is_typing,
- is_code_same_as_print,
- escape_string_quotes,
get_modulename,
get_callable_definition,
- add_source_indent,
)
-from ghostos.core.moss.exports import Exporter
-from ghostos.abc import PromptAble, PromptAbleClass
-from ghostos.helpers import generate_import_path
+from ghostos.prompter import get_defined_prompt
+from pydantic import BaseModel
+from dataclasses import is_dataclass
import inspect
-# todo: I really dislike this module and hope for a more systematic and rule-based implementation to replace it.
-
"""
将上下文引用的 变量/方法/类型 反射出 Prompt 的机制.
主要解决一个问题, 如何让一个属性能够被大模型所理解.
@@ -35,15 +27,13 @@
2. 如果变量拥有 __prompt__ 属性, 通过它 (可以是方法或字符串) 生成 prompt.
"""
-PROMPT_MAGIC_ATTR = "__prompt__"
-"""通过这个属性名来判断一个实例 (module/function/instance of class) 是否有预定义的 prompt. """
-
-CLASS_PROMPT_MAGIC_ATTR = "__class_prompt__"
-
-PromptFn = Callable[[], str]
-"""生成 Prompt 的方法. """
-
-Numeric = Union[int, float]
+__all__ = [
+ 'get_prompt',
+ 'reflect_module_locals', 'reflect_class_with_methods',
+ 'join_prompt_lines', 'compile_attr_prompts',
+ 'get_defined_prompt',
+ 'AttrPrompts',
+]
AttrPrompts = Iterable[Tuple[str, str]]
"""
@@ -58,20 +48,14 @@
多条 prompt 用 "\n\n".join(prompts) 的方式拼接.
"""
+ignore_modules = {
+ "pydantic",
+}
+
def reflect_module_locals(
modulename: str,
local_values: Dict[str, Any],
- *,
- includes: Optional[Set[str]] = None,
- excludes: Optional[Set[str]] = None,
- includes_module_prefixes: Optional[Set[str]] = None,
- excludes_module_prefixes: Optional[Set[str]] = None,
- _cls: bool = True,
- _typing: bool = True,
- _func: bool = True,
- _module: bool = False,
- _prompter: bool = True,
) -> AttrPrompts:
"""
MOSS 系统自带的反射方法, 对一个module 的本地变量做最小化的反射展示.
@@ -90,59 +74,48 @@ def reflect_module_locals(
6. 如果目标是 class
- 包含 __class_prompt__ 方法时, 用它生成.
- __is_abstract__ 的 class, 直接返回源码.
- 7. 如果目标是 typing
- - 如果目标就是 typing 库, 则不展示.
- - 否则用字符串形式展示.
- 8. 如果目标是其它 attr
+ 7. 如果目标是其它 attr
_ 只有包含 prompt 方法时才展示.
:param modulename: 当前模块名. 所有当前模块的变量默认不展示.
:param local_values: 传入的上下文变量.
- :param includes: if given, only prompt the attrs that name in it
- :param excludes: if given, any attr that name in it will not be prompted
- :param includes_module_prefixes: if given, the other module's value will only be prompted if the module match prefix
- :param excludes_module_prefixes: if given, the other module's value will not be prompted if the module match prefix
- :param _cls: 是否允许反射类.
- :param _module: 是否允许反射模块.
- :param _typing: 是否允许反射 typing
- :param _func: 是否允许反射 function.
- :param _prompter: 拥有 __prompt__ 的其它类型.
"""
for name, value in local_values.items():
try:
- prompt = reflect_module_attr(
- name, value, modulename, includes, excludes, includes_module_prefixes, excludes_module_prefixes,
- _cls, _module, _func, _prompter,
- )
+ prompt = reflect_module_attr(name, value, modulename)
except Exception as e:
raise RuntimeError(f"failed to reflect local value {name!r}: {e}")
if prompt is not None:
yield name, prompt
+def reflect_class_with_methods(cls: type) -> str:
+ """
+ reflect class with all its method signatures.
+ """
+ from inspect import getsource
+ from .utils import make_class_prompt, get_callable_definition
+ source = getsource(cls)
+ attrs = []
+ for name in dir(cls):
+ if name.startswith("_"):
+ continue
+ method = getattr(cls, name)
+ if inspect.ismethod(method) or inspect.isfunction(method):
+ block = get_callable_definition(method)
+ attrs.append(block)
+ return make_class_prompt(source=source, attrs=attrs)
+
+
def reflect_module_attr(
name: str,
value: Any,
- current_module: Optional[str] = None,
- includes: Optional[Set[str]] = None,
- excludes: Optional[Set[str]] = None,
- includes_module_prefixes: Optional[Set[str]] = None,
- excludes_module_prefixes: Optional[Set[str]] = None,
- _cls: bool = True,
- _typing: bool = True,
- _func: bool = True,
- _module: bool = False,
- _prompter: bool = True,
+ current_module: str,
) -> Optional[str]:
"""
反射其中的一个值.
"""
- # 名字相关的过滤逻辑.
- if excludes and name in excludes:
- return None
- if includes is not None and name not in includes:
- return None
- if name.startswith('_') and not (includes and name in includes):
+ if name.startswith('_'):
# 私有变量不展示.
return None
if inspect.isbuiltin(value):
@@ -155,138 +128,39 @@ def reflect_module_attr(
return None
elif value_modulename == current_module:
return None
-
- if excludes_module_prefixes:
- for prefix in excludes_module_prefixes:
- if value_modulename.startswith(prefix):
- return None
-
- elif includes_module_prefixes:
- has_prefix = False
- for prefix in includes_module_prefixes:
- if value_modulename.startswith(prefix):
- has_prefix = True
- break
- if not has_prefix:
+ for ignore_module_name in ignore_modules:
+ if value_modulename.startswith(ignore_module_name):
return None
- return default_reflect_local_value_prompt(
- name, value,
- _cls=_cls, _typing=_typing, _module=_module, _func=_func, _prompter=_prompter,
- )
+ return get_prompt(value)
-def default_reflect_local_value_prompt(
- name: str,
- value: Any,
- _cls: bool = True,
- _module: bool = True,
- _typing: bool = True,
- _func: bool = True,
- _prompter: bool = True,
-) -> Optional[str]:
+
+def get_prompt(value: Any) -> Optional[str]:
"""
- 默认的反射方法, 用来反射当前上下文(module) 里的某个变量, 生成上下文相关的 prompt (assignment or definition).
- :param name: 变量名.
- :param value: 变量值
- :param _cls: 是否允许反射类.
- :param _module: 是否允许反射模块.
- :param _typing: 是否允许反射 typing
- :param _func: 是否允许反射 function.
- :param _prompter: 其它类型.
- :return:
+ get prompt from value.
+ only:
+ 1. predefined PromptAble
+ 2. abstract class
+ 3. function or method
+ will generate prompt
"""
- if isinstance(value, Exporter):
- return value.gene_prompt(name)
- elif is_typing(value):
- if not _typing:
- return None
- if value.__module__ == "typing":
- if value.__name__ in _typing.__dict__ and _typing.__dict__[value.__name__] is value:
- return None
- return f"{name} = {value}"
+ defined_prompt = get_defined_prompt(value)
+ if defined_prompt:
+ return defined_prompt
- elif inspect.isclass(value):
- if not _cls:
- return None
- # class 类型.
- prompt = get_class_magic_prompt(value)
- if prompt is not None:
- return prompt
- if inspect.isabstract(value):
+ if inspect.isclass(value):
+ # only reflect abstract class
+ if inspect.isabstract(value) or issubclass(value, BaseModel) or is_dataclass(value):
source = inspect.getsource(value)
- return source
-
+ if source:
+ return source
elif inspect.isfunction(value) or inspect.ismethod(value):
- if not _func:
- return None
- # 方法类型.
- prompt = get_magic_prompt(value)
- if prompt is not None:
- return prompt
# 默认都给方法展示 definition.
- return get_callable_definition(value, name)
- elif inspect.ismodule(value):
- if not _module:
- return None
- # 只有包含 __prompt__ 的库才有展示.
- prompt = get_magic_prompt(value)
- if prompt:
- parsed = escape_string_quotes(prompt, '"""')
- # 增加缩进.
- parsed = add_source_indent(parsed, indent=4)
- return f'''
-# information of `{name}` (module `{value.__name__}`) :
-"""
-{parsed}
-"""
-# information of `{name}` over.
-'''
+ return get_callable_definition(value)
- else:
- if not _prompter:
- return None
- # attr, 也可能是 module.
- prompt = get_magic_prompt(value)
- if prompt:
- parsed = escape_string_quotes(prompt, '"""')
- return f'''
-# value of `{name}`:
-"""
-{parsed}
-"""
-# value of `{name}` over.
-'''
return None
-def get_prompt(value: Any) -> Optional[str]:
- if inspect.isclass(value):
- return get_class_magic_prompt(value)
- return get_magic_prompt(value)
-
-
-def get_magic_prompt(value: Any) -> Optional[str]:
- """
- 不做类型校验, 直接返回 PROMPT_MAGIC_ATTR 生成 prompt 的结果.
- :param value: 合理类型是 module, function, method, instance of class
- """
- if isinstance(value, PromptAble):
- return value.__prompt__()
- fn = getattr(value, PROMPT_MAGIC_ATTR, None)
- return unwrap_str(fn) if fn is not None else None
-
-
-def get_class_magic_prompt(value: Any) -> Optional[str]:
- """
- 不做类型校验, 直接返回 CLASS_PROMPT_MAGIC_ATTR 生成 prompt 的结果.
- :param value: 合理的类型是 class.
- """
- if issubclass(value, PromptAbleClass):
- return value.__class_prompt__()
- fn = getattr(value, CLASS_PROMPT_MAGIC_ATTR, None)
- return unwrap_str(fn) if fn is not None else None
-
-
def join_prompt_lines(*prompts: Optional[str]) -> str:
"""
将多个可能为空的 prompt 合并成一个 python 代码风格的 prompt.
@@ -296,100 +170,17 @@ def join_prompt_lines(*prompts: Optional[str]) -> str:
line = prompt.rstrip()
if line:
result.append(prompt)
- return '\n\n\n'.join(result)
+ return '\n\n'.join(result)
-def assign_prompt(typehint: Optional[Any], assigment: Optional[Any]) -> str:
- """
- 拼装一个赋值的 Prompt.
- :param typehint: 拼装类型描述, 如果是字符串直接展示, 否则会包在双引号里.
- :param assigment:
- :return:
- """
- if isinstance(typehint, str):
- typehint_str = f': {typehint}'
- else:
- s = escape_string_quotes(str(typehint), '"')
- typehint_str = f': "{s}"'
- assigment_str = ""
- if isinstance(assigment, str):
- s = escape_string_quotes(str(typehint), '"')
- assigment_str = f' = "{s}"'
- elif is_code_same_as_print(assigment):
- assigment_str = f' = {assigment}'
- return f"{typehint_str}{assigment_str}"
-
-
-def compile_attr_prompts(module: ModuleType, attr_prompts: AttrPrompts) -> str:
- """
- 将 Attr prompt 进行合并.
- :param module: 用来做类型判断, 如何处理 name.
- :param attr_prompts:
- :return: prompt in real python code pattern
- """
- prompt_lines = []
- local_values = module.__dict__
- local_module = module.__name__
+def compile_attr_prompts(attr_prompts: AttrPrompts) -> str:
+ prompts = []
for name, prompt in attr_prompts:
- line = prompt.strip()
- if not line:
- # 空值跳过.
+ prompt = prompt.strip()
+ if not prompt:
continue
- value = local_values.get(name, None)
- if value is None:
- # 不在当前的变量里, 直接加入上下文.
- prompt_lines.append(line)
- elif getattr(value, "__module__", None) == local_module:
- prompt_lines.append(line)
- elif inspect.isclass(value):
- # 考虑到重命名.
- if name != value.__name__:
- line = xml_wrap_code(line, "class", name=name, path=value.__module__ + ':' + value.__qualname__)
- prompt_lines.append(line)
- elif inspect.isfunction(value):
- # 考虑到重命名.
- if name != value.__name__:
- line = xml_wrap_code(line, "func", name=name, path=value.__module__ + ':' + value.__qualname__)
- prompt_lines.append(line)
- elif inspect.ismodule(value):
- # 考虑到重命名.
- line = xml_wrap_code(line, "module", name=name, module=value.__name__)
- prompt_lines.append(line)
- else:
- # 使用注释 + 描述的办法.
- prompt_lines.append(f"# value '{name}':\n{line}")
- return join_prompt_lines(*prompt_lines)
-
-
-def xml_wrap_code(value: str, mark: str, **kwargs: str) -> str:
- kwargs_str = ""
- if kwargs:
- lines = [f"{name}='{arg}'" for name, arg in kwargs.items()]
- kwargs_str = ' ' + ' '.join(lines)
- start = f"# <{mark}{kwargs_str}>"
- end = f"# "
- return "\n".join([start, value, end])
-
-
-def set_prompter(value: Any, prompter: Union[PromptFn, str], force: bool = False) -> None:
- if not force and hasattr(value, PROMPT_MAGIC_ATTR):
- return
- setattr(value, PROMPT_MAGIC_ATTR, prompter)
-
-
-def set_class_prompter(value: Type, class_prompter: Union[PromptFn, str], force: bool = False) -> None:
- if not inspect.isclass(value):
- raise TypeError(f'`value` should be a class, not {type(value)}')
- class_name = generate_import_path(value)
- if hasattr(value, CLASS_PROMPT_MAGIC_ATTR):
- method = getattr(value, CLASS_PROMPT_MAGIC_ATTR)
- if method is not None and isinstance(method, Callable):
- cls_name = getattr(method, '__prompter_class__', None)
- # 同一个类已经有 prompt 的情况, 必须加 force 参数才能修改原有的.
- if cls_name == class_name and not force:
- return
- if isinstance(class_prompter, Callable):
- # 会给 prompter 方法添加 __prompter_class__ 用来做归属判断.
- # todo: 有没有更好的解决办法?
- class_prompter.__prompter_class__ = class_name
- setattr(value, CLASS_PROMPT_MAGIC_ATTR, class_prompter)
+ attr_prompt = f'''#
+{prompt}
+# '''
+ prompts.append(attr_prompt)
+ return join_prompt_lines(*prompts)
diff --git a/ghostos/core/moss/pycontext.py b/ghostos/core/moss/pycontext.py
index 452f86e3..24511550 100644
--- a/ghostos/core/moss/pycontext.py
+++ b/ghostos/core/moss/pycontext.py
@@ -1,16 +1,11 @@
from __future__ import annotations
-from typing import Dict, Any, Union, List, Optional, Tuple, TypedDict, is_typeddict, Callable
+from typing import Dict, Any, Optional, Tuple, Iterator
from types import ModuleType
-import inspect
from pydantic import BaseModel, Field
-from ghostos.core.moss.decorators import definition
-from ghostos.helpers import (
- parse_import_module_and_spec, import_from_path, join_import_module_and_spec,
- get_module_spec,
-)
+from ghostos.entity import EntityMeta, to_entity_meta, from_entity_meta, is_entity_type
__all__ = [
- 'PyContext', 'Injection', 'Property', 'attr', 'SerializableType', 'SerializableData',
+ 'PyContext',
]
@@ -28,16 +23,21 @@ class PyContext(BaseModel):
description="if code given, use it instead of code from the module",
)
- injections: Dict[str, Injection] = Field(
- default_factory=dict,
- description="通过 python 引入的包, 类, 方法 等. 会注入到 MOSS 上, 同时会实现它.",
- )
- properties: Dict[str, Property] = Field(
+ # injections: Dict[str, Injection] = Field(
+ # default_factory=dict,
+ # description="通过 python 引入的包, 类, 方法 等. 会注入到 MOSS 上, 同时会实现它.",
+ # )
+
+ properties: Dict[str, EntityMeta] = Field(
default_factory=dict,
- description="在上下文中定义的变量. 会注入到 MOSS 上. 修改后也会保存到 pycontext 里. ",
)
- generated: Optional[str] = Field(
+ # properties: Dict[str, Property] = Field(
+ # default_factory=dict,
+ # description="在上下文中定义的变量. 会注入到 MOSS 上. 修改后也会保存到 pycontext 里. ",
+ # )
+
+ execute_code: Optional[str] = Field(
default=None,
description="the generated python code on this context",
)
@@ -47,167 +47,196 @@ class PyContext(BaseModel):
description="if the generated code is executed",
)
- def inject(self, injected: "Injection") -> None:
- self.injections[injected.import_from] = injected
-
- def define(self, d: "Property") -> None:
- self.properties[d.name] = d
+ def set_prop(self, name: str, value: Any):
+ self.properties[name] = to_entity_meta(value)
+
+ def get_prop(self, name: str, module: Optional[ModuleType] = None) -> Any:
+ if name not in self.properties:
+ return None
+ value = self.properties[name]
+ return from_entity_meta(value, module)
+
+ @staticmethod
+ def allow_prop(value: Any) -> bool:
+ if isinstance(value, BaseModel):
+ return True
+ elif isinstance(value, bool):
+ return True
+ elif isinstance(value, str):
+ return True
+ elif isinstance(value, int):
+ return True
+ elif isinstance(value, float):
+ return True
+ elif isinstance(value, list):
+ return True
+ elif isinstance(value, dict):
+ return True
+ elif is_entity_type(value):
+ return True
+ return False
+
+ def iter_props(self, module: Optional[ModuleType] = None) -> Iterator[Tuple[str, Any]]:
+ for name in self.properties:
+ value = self.properties[name]
+ yield name, from_entity_meta(value, module)
def join(self, ctx: "PyContext") -> "PyContext":
"""
合并两个 python context, 以右侧的为准. 并返回一个新的 PyContext 对象. 避免左向污染.
"""
copied = self.model_copy(deep=True)
- if ctx.module:
+ if copied.module is None:
copied.module = ctx.module
- if ctx.code:
+ if copied.code is None:
copied.code = ctx.code
- for imp in ctx.injections.values():
- copied.inject(imp)
- for var in ctx.properties.values():
- copied.define(var)
- return copied
-
-
-class Injection(BaseModel):
- """
- from module import specific attribute then inject to MOSS Context
- """
- import_from: str = Field(
- description="the imported module name or use module path pattern such as 'modulename:attr_name'",
- )
- alias: Optional[str] = Field(default=None, description="context attr alias for the imported value")
-
- @classmethod
- def reflect(cls, value: Any, alias: Optional[str] = None) -> "Injection":
- """
- reflect a value and generate Imported value.
- :param value:
- :param alias:
- :return:
- """
- modulename = inspect.getmodule(value).__name__
- if inspect.ismodule(value):
- spec = None
- else:
- spec = getattr(value, '__name__', None)
- import_from = join_import_module_and_spec(modulename, spec)
- return Injection(
- import_from=import_from,
- alias=alias,
- )
-
- def get_name(self) -> str:
- if self.alias:
- return self.alias
- _, spec = self.get_from_module_attr()
- return spec
-
- def get_from_module_attr(self) -> Tuple[str, Optional[str]]:
- """
- :return: modulename and attribute name from the module
- """
- return parse_import_module_and_spec(self.import_from)
-
-
-SerializableType = Union[str, int, float, bool, None, list, dict, BaseModel, TypedDict]
-"""系统支持的各种可序列化类型, 可以被存储到 Serializable 里. 更多情况下可以用别的变量类型. """
-SerializableData = Union[str, int, float, bool, None, List, Dict]
+ if copied.execute_code is None:
+ copied.execute_code = ctx.execute_code
+ copied.executed = ctx.executed
+ for key, val in ctx.properties.items():
+ copied.properties[key] = val
+ return copied
-class Property(BaseModel):
- """
- 可以在 MOSS 上下文中声明的变量.
- """
- name: str = Field(default="", description="property name in the moss context")
- desc: str = Field(default="", description="describe the property's purpose")
- value: Union[str, int, float, bool, None, list, dict, BaseModel, TypedDict] = Field(
- default=None,
- description="the serializable value",
- )
- model: Optional[str] = Field(
- default=None,
- description="如果是 pydantic 等类型, 可以通过类进行封装. 类应该在 imports 或者 defines 里.",
- )
-
- def __set_name__(self, owner, name):
- self.name = name
- if hasattr(owner, '__pycontext__') and isinstance(owner.__pycontext__, PyContext):
- owner.__pycontext__.define(self)
-
- def __set__(self, instance, value):
- if value is self:
- return
- if isinstance(value, Property):
- self.value = value.value
- self.model = value.model
- self.name = value.name
- self.model = value.model
- self.set_value(value)
-
- def __get__(self, instance, owner):
- return self.generate_value()
-
- def __delete__(self):
- self.__value__ = None
- self.value = None
- self.model = None
-
- @classmethod
- def from_value(cls, *, name: str = "", value: SerializableType, desc: str = "") -> Optional["Property"]:
- p = cls(name=name, desc=desc)
- p.set_value(value)
- return p
-
- def set_value(self, value: Any) -> None:
- if not isinstance(value, SerializableType):
- # 赋值的时候报错.
- raise AttributeError(f"{value} is not property serializable type {SerializableType}")
- model = None
- has_model = (
- value is not None and isinstance(value, BaseModel)
- or is_typeddict(value)
- )
- if has_model:
- type_ = type(value)
- if type_.__qualname__:
- model = type_.__module__ + ':' + type_.__qualname__
- else:
- model = type_.__module__ + ':' + type_.__name__
- self.value = value
- self.model = model
-
- def generate_value(self, module: Optional[ModuleType] = None) -> Any:
- model = self.model
- value = self.value
- if isinstance(self.value, dict) and model is not None:
- if not isinstance(value, Dict):
- raise AttributeError(f"'{value}' is not dict while model class is '{model}'")
- cls = None
- if module is not None:
- modulename, spec = parse_import_module_and_spec(model)
- if modulename == module.__name__:
- # 用这种方法解决临时模块里的变量问题.
- cls = get_module_spec(module.__dict__, spec)
- if cls is None:
- cls = import_from_path(model)
- if issubclass(cls, BaseModel):
- self.value = cls(**value)
- return self.value
-
-
-@definition()
-def attr(
- default: SerializableType = None, *,
- default_factory: Optional[Callable[[], Any]] = None,
- desc: str = "",
-) -> SerializableType:
- """
- 用于定义一个要绑定到 MOSS 上的属性, 它的值可以在多轮对话和思考过程中保存和修改.
- :param default: 属性的默认值, 目前支持 str, int, float, bool, None, list, dict, BaseModel, TypedDict
- :param default_factory: 可以传入一个 lambda 或闭包, 当 default 为 None 时生成值.
- :param desc: 属性的描述.
- """
- if default is None and default_factory:
- default = default_factory()
- return Property.from_value(value=default, desc=desc)
+# class Injection(BaseModel):
+# """
+# from module import specific attribute then inject to MOSS Context
+# """
+# import_from: str = Field(
+# description="the imported module name or use module path pattern such as 'modulename:attr_name'",
+# )
+# alias: Optional[str] = Field(default=None, description="context attr alias for the imported value")
+#
+# @classmethod
+# def reflect(cls, value: Any, alias: Optional[str] = None) -> "Injection":
+# """
+# reflect a value and generate Imported value.
+# :param value:
+# :param alias:
+# :return:
+# """
+# modulename = inspect.getmodule(value).__name__
+# if inspect.ismodule(value):
+# spec = None
+# else:
+# spec = getattr(value, '__name__', None)
+# import_from = join_import_module_and_spec(modulename, spec)
+# return Injection(
+# import_from=import_from,
+# alias=alias,
+# )
+#
+# def get_name(self) -> str:
+# if self.alias:
+# return self.alias
+# _, spec = self.get_from_module_attr()
+# return spec
+#
+# def get_from_module_attr(self) -> Tuple[str, Optional[str]]:
+# """
+# :return: modulename and attribute name from the module
+# """
+# return parse_import_module_and_spec(self.import_from)
+#
+#
+# SerializableType = Union[str, int, float, bool, None, list, dict, BaseModel, TypedDict]
+# """系统支持的各种可序列化类型, 可以被存储到 Serializable 里. 更多情况下可以用别的变量类型. """
+# SerializableData = Union[str, int, float, bool, None, List, Dict]
+#
+#
+# class Property(BaseModel):
+# """
+# 可以在 MOSS 上下文中声明的变量.
+# """
+# name: str = Field(default="", description="property name in the moss context")
+# desc: str = Field(default="", description="describe the property's purpose")
+# value: Union[str, int, float, bool, None, list, dict, BaseModel, TypedDict] = Field(
+# default=None,
+# description="the serializable value",
+# )
+# model: Optional[str] = Field(
+# default=None,
+# description="如果是 pydantic 等类型, 可以通过类进行封装. 类应该在 imports 或者 defines 里.",
+# )
+#
+# def __set_name__(self, owner, name):
+# self.name = name
+# if hasattr(owner, '__pycontext__') and isinstance(owner.__pycontext__, PyContext):
+# owner.__pycontext__.define(self)
+#
+# def __set__(self, instance, value):
+# if value is self:
+# return
+# if isinstance(value, Property):
+# self.value = value.value
+# self.model = value.model
+# self.name = value.name
+# self.model = value.model
+# self.set_value(value)
+#
+# def __get__(self, instance, owner):
+# return self.generate_value()
+#
+# def __delete__(self):
+# self.__value__ = None
+# self.value = None
+# self.model = None
+#
+# @classmethod
+# def from_value(cls, *, name: str = "", value: SerializableType, desc: str = "") -> Optional["Property"]:
+# p = cls(name=name, desc=desc)
+# p.set_value(value)
+# return p
+#
+# def set_value(self, value: Any) -> None:
+# if not isinstance(value, SerializableType):
+# # 赋值的时候报错.
+# raise AttributeError(f"{value} is not property serializable type {SerializableType}")
+# model = None
+# has_model = (
+# value is not None and isinstance(value, BaseModel)
+# or is_typeddict(value)
+# )
+# if has_model:
+# type_ = type(value)
+# if type_.__qualname__:
+# model = type_.__module__ + ':' + type_.__qualname__
+# else:
+# model = type_.__module__ + ':' + type_.__name__
+# self.value = value
+# self.model = model
+#
+# def generate_value(self, module: Optional[ModuleType] = None) -> Any:
+# model = self.model
+# value = self.value
+# if isinstance(self.value, dict) and model is not None:
+# if not isinstance(value, Dict):
+# raise AttributeError(f"'{value}' is not dict while model class is '{model}'")
+# cls = None
+# if module is not None:
+# modulename, spec = parse_import_module_and_spec(model)
+# if modulename == module.__name__:
+# # 用这种方法解决临时模块里的变量问题.
+# cls = get_module_spec(module.__dict__, spec)
+# if cls is None:
+# cls = import_from_path(model)
+# if issubclass(cls, BaseModel):
+# self.value = cls(**value)
+# return self.value
+#
+#
+# @definition()
+# def attr(
+# default: SerializableType = None, *,
+# default_factory: Optional[Callable[[], Any]] = None,
+# desc: str = "",
+# ) -> SerializableType:
+# """
+# 用于定义一个要绑定到 MOSS 上的属性, 它的值可以在多轮对话和思考过程中保存和修改.
+# :param default: 属性的默认值, 目前支持 str, int, float, bool, None, list, dict, BaseModel, TypedDict
+# :param default_factory: 可以传入一个 lambda 或闭包, 当 default 为 None 时生成值.
+# :param desc: 属性的描述.
+# """
+# if default is None and default_factory:
+# default = default_factory()
+# return Property.from_value(value=default, desc=desc)
diff --git a/ghostos/core/moss/test_suites.py b/ghostos/core/moss/testsuite.py
similarity index 94%
rename from ghostos/core/moss/test_suites.py
rename to ghostos/core/moss/testsuite.py
index 34ee232e..61b9445b 100644
--- a/ghostos/core/moss/test_suites.py
+++ b/ghostos/core/moss/testsuite.py
@@ -1,5 +1,5 @@
from typing import List, Dict, Optional, Callable
-from ghostos.core.moss.abc import MossCompiler, MossResult
+from ghostos.core.moss.abcd import MossCompiler, Execution
from ghostos.core.moss.pycontext import PyContext
from ghostos.container import Container
from queue import Queue
@@ -24,12 +24,12 @@ def dump_prompt(
compiler = self._container.force_fetch(MossCompiler)
compiler.join_context(PyContext(module=modulename))
runtime = compiler.compile(test_modulename)
- return runtime.prompter().dump_context_prompt()
+ return runtime.prompter().dump_module_prompt()
def run_module_tests(
self, *,
modulename: str,
- callback: Callable[[str, MossResult], None],
+ callback: Callable[[str, Execution], None],
test_modulename: str = "__test__",
targets: Optional[str] = None,
) -> None:
@@ -66,7 +66,7 @@ def run(
target: str = "test_main",
args: Optional[List[str]] = None,
kwargs: Dict[str, str] = None,
- ) -> MossResult:
+ ) -> Execution:
"""
运行一个指定的 moss 测试.
:param modulename: 想要测试的 moss 文件的模块路径.
@@ -86,7 +86,7 @@ def parallel_run_moss_func(
self, *,
modulename: str,
funcs: List[str],
- callback: Callable[[str, MossResult], None],
+ callback: Callable[[str, Execution], None],
test_module_name: str = "__test__",
) -> None:
"""
diff --git a/ghostos/core/moss/utils.py b/ghostos/core/moss/utils.py
index cf75bc55..e13d92c6 100644
--- a/ghostos/core/moss/utils.py
+++ b/ghostos/core/moss/utils.py
@@ -2,7 +2,6 @@
import re
from typing import Any, Dict, Callable, Optional, List, Iterable, TypedDict, is_typeddict
from pydantic import BaseModel
-from ghostos.abc import Identifiable, Descriptive
__all__ = [
@@ -10,7 +9,7 @@
'get_modulename',
'is_typing', 'is_builtin', 'is_classmethod',
- 'is_model_class', 'get_model_object_meta',
+ 'is_model_class',
'parse_comments',
'parse_doc_string', 'escape_string_quotes',
'strip_source_indent', 'add_source_indent', 'make_class_prompt',
@@ -312,19 +311,6 @@ def is_model_class(typ: type) -> bool:
return issubclass(typ, BaseModel) or is_typeddict(typ)
-def get_model_object_meta(obj: Any) -> Optional[Dict]:
- if isinstance(obj, BaseModel):
- return obj.model_dump()
- elif isinstance(obj, TypedDict):
- result = {}
- for k, v in obj.items():
- result[k] = v
- return result
- elif isinstance(obj, EntityClass):
- return obj.to_entity_meta()
- return None
-
-
def is_callable(obj: Any) -> bool:
return isinstance(obj, Callable)
@@ -352,22 +338,6 @@ def get_calling_modulename(skip: int = 0) -> Optional[str]:
return None
-def get_obj_desc(obj: Any) -> Optional[str]:
- if isinstance(obj, Descriptive):
- return obj.get_description()
- if isinstance(obj, Identifiable):
- return obj.identifier().description
- if hasattr(obj, 'desc'):
- return getattr(obj, 'desc', None)
- if hasattr(obj, "description"):
- return getattr(obj, 'description', None)
- if hasattr(obj, "__desc__"):
- attr = getattr(obj, "__desc__", None)
- if attr:
- return unwrap_str(attr)
- return None
-
-
def is_code_same_as_print(value: Any) -> bool:
return isinstance(value, bool) \
or isinstance(value, int) \
@@ -386,6 +356,7 @@ def get_modulename(val: Any) -> Optional[str]:
return getattr(module, '__name__', None)
return None
+
def add_comment_mark(text: str, comment: str = "# ") -> str:
lines = text.split('\n')
contents = []
diff --git a/ghostos/core/runtime/__init__.py b/ghostos/core/runtime/__init__.py
new file mode 100644
index 00000000..570b21ea
--- /dev/null
+++ b/ghostos/core/runtime/__init__.py
@@ -0,0 +1,9 @@
+from ghostos.core.runtime.tasks import (
+ GoTaskStruct, TaskPayload, TaskBrief,
+ GoTasks, TaskState, TaskLocker,
+)
+from ghostos.core.runtime.threads import GoThreads, GoThreadInfo, thread_to_prompt, Turn
+from ghostos.core.runtime.processes import GoProcess, GoProcesses
+from ghostos.core.runtime.events import Event, EventBus, EventTypes
+from ghostos.core.runtime.thread_history import ThreadHistory
+from ghostos.core.runtime.runtime import Runtime
diff --git a/ghostos/core/session/events.py b/ghostos/core/runtime/events.py
similarity index 61%
rename from ghostos/core/session/events.py
rename to ghostos/core/runtime/events.py
index 15304e11..cf520c88 100644
--- a/ghostos/core/session/events.py
+++ b/ghostos/core/runtime/events.py
@@ -1,17 +1,17 @@
-from typing import List, Optional, Dict
+from typing import List, Optional, Dict, Any, Iterable
+from typing_extensions import Self
from abc import ABC, abstractmethod
from enum import Enum
from pydantic import BaseModel, Field
-from ghostos.core.messages.message import Message
+from ghostos.core.messages.message import Message, Role
+from ghostos.entity import EntityMeta
from ghostos.helpers import uuid
from contextlib import contextmanager
__all__ = [
- 'Event', 'EventBus', 'DefaultEventType',
+ 'Event', 'EventBus', 'EventTypes',
]
-EVENT_ENTITY_TYPE = "ghostos.core.session.events.Event"
-
class Event(BaseModel):
"""
@@ -21,19 +21,25 @@ class Event(BaseModel):
Session.task() shall handle the Event, change the session state, and maybe fire more events.
"""
- task_id: str = Field(
- description="task id of which this event shall send to.",
+ event_id: str = Field(
+ default_factory=uuid,
+ description="event id",
)
type: str = Field(
default="",
- description="event type, by default the handler shall named on_{type}"
+ description="event type"
)
-
- id: str = Field(
- default_factory=uuid,
- description="event id",
+ context: Optional[EntityMeta] = Field(
+ default=None,
+ )
+ attrs: Dict[str, Any] = Field(
+ default_factory=dict,
+ description="event attributes that follow the types."
)
+ task_id: str = Field(
+ description="task id of which this event shall send to.",
+ )
from_task_id: Optional[str] = Field(
default=None,
description="task id in which this event is fired",
@@ -42,11 +48,11 @@ class Event(BaseModel):
default=None,
description="task name in which this event is fired",
)
+
reason: str = Field(
default="",
description="reason of the event, wrapped by system type message before the messages",
)
-
messages: List[Message] = Field(
default_factory=list,
description="list of messages sent by this event",
@@ -67,7 +73,7 @@ class Event(BaseModel):
def is_empty(self) -> bool:
return not self.reason and not self.instruction and not self.messages
- def from_self(self) -> bool:
+ def is_from_self(self) -> bool:
"""
通过任务是否是自己触发的, 来判断是否要继续.
"""
@@ -79,24 +85,45 @@ def no_reason_or_instruction(self) -> bool:
def default_handler(self) -> str:
return f"on_{self.event_type()}"
+ def iter_message(self, show_instruction: bool = True) -> Iterable[Message]:
+ if EventTypes.CREATED.value != self.type and self.from_task_name and not self.is_from_self():
+ reason = ""
+ if self.reason:
+ reason = f" Reason: {self.reason}"
+ yield Role.new_system(
+ content=f"receive self {self.type} from task `{self.from_task_name}`.{reason}")
+
+ # messages in middle
+ if self.messages:
+ for message in self.messages:
+ yield message
+
+ # instruction after messages.
+ if show_instruction and self.instruction:
+ yield Role.new_system(content=self.instruction)
+
@classmethod
def new(
cls, *,
event_type: str,
task_id: str,
messages: List[Message],
+ callback: Optional[bool] = None,
from_task_id: Optional[str] = None,
from_task_name: Optional[str] = None,
reason: str = "",
instruction: str = "",
eid: Optional[str] = None,
payloads: Optional[Dict] = None,
+ context: Optional[EntityMeta] = None,
) -> "Event":
id_ = eid if eid else uuid()
type_ = event_type
payloads = payloads if payloads is not None else {}
+ if callback is None:
+ callback = from_task_id is not None
return cls(
- id=id_,
+ event_id=id_,
type=type_,
task_id=task_id,
from_task_id=from_task_id,
@@ -105,64 +132,70 @@ def new(
instruction=instruction,
messages=messages,
payloads=payloads,
+ context=context,
+ callback=callback,
)
-class DefaultEventType(str, Enum):
+class EventTypes(str, Enum):
"""
默认的消息类型.
"""
+
+ # --- upstream events --- #
+
CREATED = "created"
- """任务刚刚被创建出来"""
INPUT = "input"
- """外部对当前 task 的输入. """
- OBSERVE = "observe"
- """自我驱动的思考"""
+ ACTION_CALL = "action_call"
+
+ NOTIFY = "notify"
+
+ CANCEL = "cancel"
+
+ # --- self events --- #
- CANCELING = "cancelling"
- """任务取消时, 触发的事件. 需要广播给子节点. """
- KILLING = "killing"
+ ROTATE = "rotate"
+
+ ERROR = "error"
+
+ # --- callback events --- #
FINISH_CALLBACK = "finish_callback"
- """child task 运行正常, 返回的消息. """
FAILURE_CALLBACK = "failure_callback"
- """child task 运行失败, 返回的消息. """
-
- NOTIFY_CALLBACK = "notify_callback"
- """child task send some notice messages"""
WAIT_CALLBACK = "wait_callback"
- """Child task 返回消息, 期待更多的输入. """
+
+ # --- dead events --- #
FINISHED = "finished"
- """任务结束时, 触发的事件. 可用于反思."""
- FAILED = "failed"
- """任务失败时, 触发的事件. 可用于反思."""
+ CANCELED = "canceled"
- def block(self) -> bool:
- return self not in {}
+ FAILED = "failed"
def new(
self,
task_id: str,
messages: List[Message],
*,
+ callback: Optional[bool] = None,
from_task_id: Optional[str] = None,
from_task_name: Optional[str] = None,
reason: str = "",
instruction: str = "",
eid: Optional[str] = None,
payloads: Optional[Dict] = None,
+ context: Optional[EntityMeta] = None,
) -> Event:
type_ = str(self.value)
payloads = payloads if payloads is not None else {}
return Event.new(
event_type=type_,
task_id=task_id,
+ callback=callback,
from_task_id=from_task_id,
from_task_name=from_task_name,
reason=reason,
@@ -170,14 +203,33 @@ def new(
messages=messages,
eid=eid,
payloads=payloads,
+ context=context,
)
+# EventBus 要实现分流设计
+# Task notification queue => Task Event queue
+# 0. 一个实例接受到 Task notification 后, 开始对这个 Task 上锁.
+# 1. Task 只有在上锁 (全局只有一个实例在处理这个 Task) 后才能消费 Task Event queue
+# 2. 如果对 Task 上锁不成功, 意味着有别的实例在消费 Task Event queue.
+# 3. 如果 Task 上锁成功, 而拿不到 Event, 说明 Event 被别的实例消费完了. 这时不继续发送 notification.
+# 4. 如果 Task 上锁成功, 消费到了 Event, 它应该反复更新锁, 继续消费下去.
+# 5. 如果消费 Event 结束, 或者出错, 应该再发送一个 notification. 让下一个拿到的实例检查是否消息都消费完了.
+# 6. 发送 event 时, 应该发送 notification, 这样异步消费队列才能感知到
+# 7. n 个事件理论上会发送 n 个 notification. 而在消费过程中被生产, 丢弃的 notification 应该有 1 ~ n 个.
+# 8. 如果目标 task 是会话进程的主 task, 而会话状态是端侧管理, 则不应该 notify, 而是由端侧来做推送.
class EventBus(ABC):
"""
global event bus.
"""
+ @abstractmethod
+ def with_process_id(self, process_id: str) -> Self:
+ """
+ process level eventbus, all event and notifications are private for the process
+ """
+ pass
+
@abstractmethod
def send_event(self, e: Event, notify: bool) -> None:
"""
@@ -193,8 +245,6 @@ def pop_task_event(self, task_id: str) -> Optional[Event]:
"""
pop a task event by task_id.
the canceled event has higher priority to others.
- :param task_id: certain task id.
- :return: event or None if timeout is reached
"""
pass
@@ -218,6 +268,10 @@ def notify_task(self, task_id: str) -> None:
def clear_task(self, task_id: str) -> None:
pass
+ @abstractmethod
+ def clear_all(self):
+ pass
+
@contextmanager
def transaction(self):
yield
diff --git a/ghostos/core/runtime/processes.py b/ghostos/core/runtime/processes.py
new file mode 100644
index 00000000..65018da9
--- /dev/null
+++ b/ghostos/core/runtime/processes.py
@@ -0,0 +1,67 @@
+from abc import ABC, abstractmethod
+from typing import Optional
+from pydantic import BaseModel, Field
+from ghostos.entity import EntityMeta
+from contextlib import contextmanager
+from ghostos.helpers import uuid
+
+__all__ = [
+ 'GoProcess',
+ 'GoProcesses',
+]
+
+
+class GoProcess(BaseModel):
+ process_id: str = Field(
+ description="""
+Unique process id for the agent session. Session shall only have one process a time.
+Stop the process will stop all the tasks that belongs to it.
+""",
+ )
+
+ shell_id: str = Field(
+ description="session id in which the process belongs",
+ )
+
+ @classmethod
+ def new(
+ cls, *,
+ shell_id: str,
+ process_id: Optional[str] = None,
+ ) -> "GoProcess":
+ process_id = process_id if process_id else uuid()
+ return GoProcess(
+ shell_id=shell_id,
+ process_id=process_id,
+ )
+
+
+class GoProcesses(ABC):
+ """
+ repository to save or load process
+ """
+
+ @abstractmethod
+ def get_process(self, shell_id: str) -> Optional[GoProcess]:
+ """
+ get process by id
+ :param shell_id: shell id
+ """
+ pass
+
+ @abstractmethod
+ def save_process(self, process: GoProcess) -> None:
+ """
+ save process
+ :param process:
+ :return:
+ """
+ pass
+
+ @contextmanager
+ def transaction(self):
+ """
+ transaction to process io
+ do nothing as default.
+ """
+ yield
diff --git a/ghostos/core/runtime/runtime.py b/ghostos/core/runtime/runtime.py
new file mode 100644
index 00000000..b160da6d
--- /dev/null
+++ b/ghostos/core/runtime/runtime.py
@@ -0,0 +1,27 @@
+from abc import ABC, abstractmethod
+from typing import Protocol
+from .tasks import GoTasks
+from .threads import GoThreads
+from .processes import GoProcesses
+from ghostos.container import Container
+from ghostos.core.messages.transport import Stream
+
+
+class Runtime(Protocol):
+ """
+ shell runtime
+ """
+ shell_id: str
+ """basic shell id."""
+ process_id: str
+ """the process id of this instance of shell."""
+ stream: Stream
+ """upstream to send messages"""
+ container: Container
+ """the container of the shell"""
+ tasks: GoTasks
+ """the tasks of the shell"""
+ threads: GoThreads
+ """the threads of the shell"""
+ processes: GoProcesses
+ """"the processes of the shell"""
diff --git a/ghostos/core/session/tasks.py b/ghostos/core/runtime/tasks.py
similarity index 51%
rename from ghostos/core/session/tasks.py
rename to ghostos/core/runtime/tasks.py
index 76d5db1f..917db808 100644
--- a/ghostos/core/session/tasks.py
+++ b/ghostos/core/runtime/tasks.py
@@ -1,18 +1,18 @@
-import time
-from typing import Optional, List, Set, Iterable, ClassVar, Dict
+from typing import Optional, List, ClassVar, Dict, Self
from abc import ABC, abstractmethod
from enum import Enum
from pydantic import BaseModel, Field
+from ghostos.identifier import Identifier, Identical
from ghostos.entity import EntityMeta
-from ghostos.abc import Identifier, Identifiable
from ghostos.core.messages import Payload
+from ghostos.helpers import timestamp
from contextlib import contextmanager
__all__ = [
- 'Task', 'TaskPayload', 'TaskBrief',
+ 'GoTaskStruct', 'TaskPayload', 'TaskBrief',
'TaskState',
- 'Tasks',
- 'WaitGroup',
+ 'TaskLocker',
+ 'GoTasks',
]
@@ -37,51 +37,30 @@ class TaskState(str, Enum):
FAILED = "failed"
"""the task is failed due to an exception"""
- KILLED = "killed"
-
FINISHED = "finished"
"""the task is finished"""
@classmethod
def is_dead(cls, state: str) -> bool:
- return state in {cls.FINISHED, cls.FAILED, cls.CANCELLED, cls.KILLED}
-
-
-class WaitGroup(BaseModel):
- """
- await group of children tasks that will wake up the task.
- """
- tasks: Dict[str, bool] = Field(description="children task ids to wait")
-
- def is_done(self) -> bool:
- for _, ok in self.tasks.items():
- if not ok:
- return False
- return True
-
+ return state in {cls.FINISHED, cls.FAILED, cls.CANCELLED}
-class AssistantInfo(Identifier, BaseModel):
- id: str = Field(description="id of the assistant")
- name: str = Field(description="name of the assistant")
- description: str = Field(description="description of the assistant")
- meta_prompt: str = Field(description="meta prompt of the assistant")
-
-class Task(BaseModel):
+class GoTaskStruct(BaseModel):
# -- scope --- #
- session_id: str = Field(
- description="session id that task belongs.",
+ task_id: str = Field(
+ description="""
+ the id of the task.
+ """,
+ )
+ shell_id: str = Field(
+ description="the shell id of the task",
)
process_id: str = Field(
description="""
the id of the process that the task belongs to.
""",
)
- task_id: str = Field(
- description="""
- the id of the task.
- """,
- )
+
thread_id: str = Field(
description="""
the id of the thread that contains the context of the task.
@@ -94,26 +73,58 @@ class Task(BaseModel):
Parent task id of the task.
""",
)
+ depth: int = Field(
+ default=0,
+ description="the depth of the task",
+ )
+
+ # --- state values --- #
+
+ meta: EntityMeta = Field(
+ description="the entity meta of the task handler",
+ )
+ context: Optional[EntityMeta] = Field(
+ default=None,
+ description="the context entity",
+ )
+ state_values: Dict[str, EntityMeta] = Field(
+ default_factory=dict,
+ description="the state values of the task",
+ )
+
+ state: str = Field(
+ default=TaskState.NEW.value,
+ description="the state of the current task."
+ )
+
+ status_desc: str = Field(
+ default="",
+ description="The description of the current task status.",
+ )
+
+ globals: Dict = Field(
+ default_factory=dict,
+ description="the global values that inherit from the parent task or shell",
+ )
+
+ props: Optional[EntityMeta] = Field(
+ default=None,
+ description="the state data of the task handler"
+ )
# --- brief --- #
name: str = Field(
- description="The name of the task. "
+ description="The name of the task"
)
description: str = Field(
description="The description of the task"
)
+
priority: float = Field(
default=0.0,
description="The priority of the task",
)
- # --- assistant info --- #
-
- assistant: Optional[AssistantInfo] = Field(
- default=None,
- description="the assistant information, if given, took it as the message sender",
- )
-
# --- relations --- #
children: List[str] = Field(
@@ -123,95 +134,53 @@ class Task(BaseModel):
"""
)
- depending: List[WaitGroup] = Field(
- default_factory=list,
- description="the children task ids that wait them callback",
- )
-
- # --- thought --- #
- meta: EntityMeta = Field(
- description="""
-The meta data to restore the handler of this task.
-"""
- )
-
- # --- state --- #
- state: str = Field(
- default=TaskState.NEW.value,
- description="""
-the state of the current task.
-"""
- )
- # --- state ---#
- logs: List[str] = Field(
- default_factory=list,
- description="log of the status change of the task",
- )
# --- time related --- #
- created: float = Field(
- default_factory=lambda: round(time.time(), 4),
+ created: int = Field(
+ default_factory=timestamp,
description="The time the task was created.",
)
- updated: float = Field(
- default=0.0,
+ updated: int = Field(
+ default_factory=timestamp,
description="The time the task was updated.",
)
- overdue: float = Field(
- default=0.0,
- description="The time the task was overdue.",
- )
- timeout: float = Field(
- default=0.0,
- description="timeout for each round of the task execution",
- )
# --- system --- #
- lock: Optional[str] = Field(
- default=None,
- )
+
turns: int = Field(
default=0,
description="the turn number of the task runs",
)
- think_turns: int = Field(
+ errors: int = Field(
default=0,
- description="记录 task 已经自动运行过多少次. 如果是 0 的话, 则意味着它刚刚被创建出来. ",
- )
- depth: int = Field(
- default=0,
- description="task depth that should be parent task depth +1 if parent exists",
- )
- max_think_turns: int = Field(
- default=20,
- description="任务最大自动运行轮数, 为 0 的话表示无限. "
- )
- max_children: int = Field(
- default=20,
- description="当前任务最大的子任务数, 超过这范围的子任务开始垃圾回收. "
+ description="continual task errors count",
)
@classmethod
def new(
cls, *,
task_id: str,
- session_id: str,
+ shell_id: str,
process_id: str,
+ depth: int,
name: str,
description: str,
meta: EntityMeta,
+ context: Optional[EntityMeta] = None,
parent_task_id: Optional[str] = None,
- assistant: Optional[Identifier] = None,
- ) -> "Task":
- return Task(
+ priority: float = 0.0,
+ ) -> "GoTaskStruct":
+ return GoTaskStruct(
task_id=task_id,
- session_id=session_id,
+ shell_id=shell_id,
process_id=process_id,
+ depth=depth,
thread_id=task_id,
parent=parent_task_id,
meta=meta,
+ context=context,
name=name,
description=description,
- assistant=assistant,
+ priority=priority,
)
def add_child(
@@ -220,28 +189,22 @@ def add_child(
name: str,
description: str,
meta: EntityMeta,
- assistant: Optional[Identifier] = None,
- ) -> "Task":
+ context: Optional[EntityMeta] = None,
+ ) -> "GoTaskStruct":
self.children.append(task_id)
- return self.new(
+ child = self.new(
task_id=task_id,
- session_id=self.session_id,
+ shell_id=self.shell_id,
process_id=self.process_id,
+ depth=self.depth + 1,
name=name,
description=description,
meta=meta,
+ context=context,
parent_task_id=self.task_id,
- assistant=assistant,
)
-
- def think_too_much(self) -> bool:
- """
- 任务是否超过了自动思考的轮次.
- """
- return 0 < self.max_think_turns <= self.think_turns
-
- def too_much_children(self) -> bool:
- return 0 < self.max_children <= len(self.children)
+ child.depth = self.depth + 1
+ return child
def remove_child(self, child_task_id: str) -> bool:
results = []
@@ -258,61 +221,42 @@ def identifier(self) -> Identifier:
return Identifier(
id=self.id,
name=self.name,
- description=self.description,
+ description=self.purpose,
)
+ def shall_notify(self) -> bool:
+ return self.depth > 0
+
def is_dead(self) -> bool:
return TaskState.is_dead(self.state)
def is_new(self) -> bool:
return TaskState.NEW.value == self.state
- def depending_tasks(self) -> Set[str]:
- result = set()
- for group in self.depending:
- for task in group.tasks:
- result.add(task)
- return result
-
- def depend_on_tasks(self, task_ids: List[str]) -> None:
- group = WaitGroup(
- tasks={task_id: False for task_id in task_ids},
- )
- self.depending.append(group)
-
- def on_callback_task(self, task_id: str) -> Optional[WaitGroup]:
- """
- 得到一个 task id 的回调. 判断是否一组 wait group 被激活了.
- :param task_id:
- :return: 是否有 wait group 激活了.
- """
- for group in self.depending:
- if task_id in group.tasks:
- group.tasks[task_id] = True
- if group.is_done():
- return group
- return None
-
- def update_turn(self) -> None:
+ def new_turn(self) -> Self:
"""
保存一轮变更之前运行的方法.
+ todo
"""
- self.updated = round(time.time(), 4)
- self.turns += 1
+ return self.model_copy(
+ update={
+ "updated": timestamp(),
+ "turns": self.turns + 1,
+ },
+ deep=True,
+ )
-class TaskBrief(BaseModel, Identifiable):
+class TaskBrief(BaseModel, Identical):
task_id: str = Field(description="the id of the task")
name: str = Field(description="the name of the task")
- description: str = Field(description="the description of the task")
+ description: str = Field(description="the purpose of the task")
state: str = Field(description="the state of the task")
- logs: List[str] = Field(description="the logs of the task")
-
- def is_overdue(self) -> bool:
- now = time.time()
- return now - self.updated > self.overdue
+ status_desc: str = Field(description="the description of the task status")
+ created: int = Field(description="the time the task was created")
+ updated: int = Field(description="the time that task was updated")
- def identifier(self) -> Identifier:
+ def __identifier__(self) -> Identifier:
return Identifier(
id=self.id,
name=self.name,
@@ -320,7 +264,7 @@ def identifier(self) -> Identifier:
)
@classmethod
- def from_task(cls, task: Task) -> "TaskBrief":
+ def from_task(cls, task: GoTaskStruct) -> "TaskBrief":
return TaskBrief(**task.model_dump())
@@ -330,36 +274,63 @@ class TaskPayload(Payload):
task_id: str = Field(description="the id of the task")
task_name: str = Field(description="the name of the task")
process_id: str = Field(description="the id of the process")
+ shell_id: str = Field(description="the session id of the task")
thread_id: str = Field(description="the id of the thread")
@classmethod
- def from_task(cls, task: Task) -> "TaskPayload":
+ def from_task(cls, task: GoTaskStruct) -> "TaskPayload":
return cls(
task_id=task.task_id,
task_name=task.name,
+ shell_id=task.shell_id,
process_id=task.process_id,
thread_id=task.thread_id,
)
-class Tasks(ABC):
+class TaskLocker(ABC):
+
+ @abstractmethod
+ def acquire(self) -> bool:
+ pass
+
+ @abstractmethod
+ def acquired(self) -> bool:
+ pass
+
+ @abstractmethod
+ def refresh(self) -> bool:
+ pass
+
+ @abstractmethod
+ def release(self) -> bool:
+ pass
+
+ def __enter__(self) -> bool:
+ return self.acquire()
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ if self.acquired():
+ self.release()
+
+
+class GoTasks(ABC):
"""
管理 task 存储的模块. 通常集成到 Session 里.
"""
@abstractmethod
- def save_task(self, *tasks: Task) -> None:
+ def save_task(self, *tasks: GoTaskStruct) -> None:
"""
保存一个或者多个 task.
"""
pass
@abstractmethod
- def get_task(self, task_id: str, lock: bool) -> Optional[Task]:
+ def get_task(self, task_id: str) -> Optional[GoTaskStruct]:
"""
使用 task id 来获取一个 task.
:param task_id:
- :param lock: 是否尝试对 task 上锁, 如果要求上锁但没成功, 返回 None.
:return: if task is not Exists or locked failed
"""
pass
@@ -374,41 +345,30 @@ def exists(self, task_id: str) -> bool:
pass
@abstractmethod
- def get_tasks(self, task_ids: List[str], states: Optional[List[TaskState]] = None) -> Iterable[Task]:
+ def get_tasks(self, task_ids: List[str]) -> Dict[str, GoTaskStruct]:
"""
从数据库里读取出多个 task. 不会获取目标 task 的锁, 所以也无法更新.
:param task_ids:
- :param states:
:return:
"""
pass
@abstractmethod
- def get_task_briefs(self, task_ids: List[str], states: Optional[List[TaskState]] = None) -> Iterable[TaskBrief]:
+ def get_task_briefs(self, task_ids: List[str]) -> Dict[str, TaskBrief]:
"""
获取多个任务的摘要信息.
:param task_ids:
- :param states:
- :return:
- """
- pass
-
- @abstractmethod
- def unlock_task(self, task_id: str, lock: str) -> None:
- """
- 对一个任务解锁.
- :param task_id:
- :param lock:
:return:
"""
pass
@abstractmethod
- def refresh_task_lock(self, task_id: str, lock: str) -> Optional[str]:
+ def lock_task(self, task_id: str, overdue: float, force: bool = False) -> TaskLocker:
"""
- 更新一个任务的锁, 也会给它续期.
+ get task locker
:param task_id:
- :param lock:
+ :param overdue: when the locker is overdue
+ :param force: if force is True, will preempt the locker
:return:
"""
pass
diff --git a/ghostos/core/session/simple_thread.py b/ghostos/core/runtime/thread_history.py
similarity index 92%
rename from ghostos/core/session/simple_thread.py
rename to ghostos/core/runtime/thread_history.py
index fe67f296..03991872 100644
--- a/ghostos/core/session/simple_thread.py
+++ b/ghostos/core/runtime/thread_history.py
@@ -1,7 +1,7 @@
from typing import List, Optional, Dict, Any
from pydantic import BaseModel, Field
from ghostos.core.messages import Message
-from ghostos.core.session.threads import MsgThread, Turn
+from ghostos.core.runtime.threads import GoThreadInfo, Turn
DESCRIPTION = """
Simple Thread is a simple mode for MsgThread, useful to show thread important information when debugging.
@@ -46,14 +46,14 @@ def from_turn(cls, turn: Turn, idx: int = 0) -> Optional["SimpleTurn"]:
)
-class SimpleMsgThread(BaseModel):
+class ThreadHistory(BaseModel):
thread_id: str = Field(description="thread id that useful to save & read thread")
extra: Dict[str, Any] = Field(default_factory=dict)
last_turn_system_prompt: str = Field(defualt="", description="system prompt")
turns: List[SimpleTurn] = Field(default_factory=list)
@classmethod
- def from_thread(cls, thread: MsgThread) -> "SimpleMsgThread":
+ def from_thread(cls, thread: GoThreadInfo) -> "ThreadHistory":
turns = []
idx = 0
for turn in thread.turns():
diff --git a/ghostos/core/session/threads.py b/ghostos/core/runtime/threads.py
similarity index 53%
rename from ghostos/core/session/threads.py
rename to ghostos/core/runtime/threads.py
index e921bf72..093df577 100644
--- a/ghostos/core/session/threads.py
+++ b/ghostos/core/runtime/threads.py
@@ -1,17 +1,16 @@
-from typing import Optional, List, Iterable, Dict, Any
-import time
+from typing import Optional, List, Iterable, Dict, Any, Self
from abc import ABC, abstractmethod
from pydantic import BaseModel, Field
from ghostos.core.messages import Message, copy_messages, Role
from ghostos.core.moss.pycontext import PyContext
-from ghostos.core.llms import Chat
-from ghostos.core.session.events import Event, DefaultEventType
-from ghostos.helpers import uuid
+from ghostos.core.llms import Prompt
+from ghostos.core.runtime.events import Event, EventTypes
+from ghostos.helpers import uuid, timestamp
from contextlib import contextmanager
__all__ = [
- 'Threads', 'MsgThread', 'Turn',
- 'thread_to_chat',
+ 'GoThreads', 'GoThreadInfo', 'Turn',
+ 'thread_to_prompt',
]
@@ -27,7 +26,7 @@ class Turn(BaseModel):
default=None,
description="event of the turn"
)
- generates: List[Message] = Field(
+ added: List[Message] = Field(
default_factory=list,
description="The new messages that generated by ghost during this turn of chat or thinking."
"Shall append to messages after updating.",
@@ -36,17 +35,26 @@ class Turn(BaseModel):
default_factory=PyContext,
description="The PyContext instance",
)
- created: float = Field(
- default_factory=lambda: round(time.time(), 4),
+ created: int = Field(
+ default_factory=timestamp,
+ )
+ summary: Optional[str] = Field(
+ default=None,
+ description="The summary before till this turn",
)
extra: Dict[str, Any] = Field(default_factory=dict, description="extra information")
@classmethod
- def new(cls, event: Optional[Event], *, turn_id: Optional[str] = None,
- pycontext: Optional[PyContext] = None) -> "Turn":
+ def new(
+ cls,
+ event: Optional[Event],
+ *,
+ turn_id: Optional[str] = None,
+ pycontext: Optional[PyContext] = None,
+ ) -> "Turn":
data = {"event": event}
if turn_id is None and event is not None:
- turn_id = event.id
+ turn_id = event.event_id
if turn_id:
data["turn_id"] = turn_id
if pycontext is not None:
@@ -54,41 +62,54 @@ def new(cls, event: Optional[Event], *, turn_id: Optional[str] = None,
return cls(**data)
def append(self, *messages: Message, pycontext: Optional[PyContext] = None) -> None:
- self.generates.extend(messages)
+ for item in messages:
+ self.added.append(item)
if pycontext is not None:
self.pycontext = pycontext
- def event_messages(self) -> Iterable[Message]:
- event = self.event
- if event is None:
+ def event_messages(self, show_instruction: bool = False) -> Iterable[Message]:
+ if not self.event:
return []
+ yield from self.iter_event_message(self.event, show_instruction)
- if DefaultEventType.CREATED.value != event.type and event.from_task_name and not event.from_self():
- reason = ""
- if event.reason:
- reason = f" Reason: {event.reason}"
- yield Role.new_assistant_system(
- content=f"receive event {event.type} from task `{event.from_task_name}`.{reason}")
-
- # messages in middle
- if event.messages:
- for message in self.event.messages:
- yield message
+ @staticmethod
+ def iter_event_message(event: Event, show_instruction: bool = True) -> Iterable[Message]:
+ yield from event.iter_message(show_instruction)
- # instruction after messages.
- if event.instruction:
- yield Role.new_assistant_system(content=event.instruction)
+ def messages(self, truncate: bool) -> Iterable[Message]:
+ if truncate and self.summary is not None:
+ return [Role.SYSTEM.new("summary of omitted history messages" + self.summary)]
- def messages(self) -> Iterable[Message]:
yield from self.event_messages()
- if self.generates:
- yield from self.generates
+ if self.added:
+ yield from self.added
def is_empty(self) -> bool:
- return (self.event is None or self.event.is_empty()) and not self.generates
+ return (self.event is None or self.event.is_empty()) and not self.added
+
+ def is_from_inputs(self) -> bool:
+ return self.event is not None and self.event.type == EventTypes.INPUT.value
+
+ def is_from_self(self) -> bool:
+ return self.event is not None and self.event.is_from_self()
+
+ def is_callback(self) -> bool:
+ return self.event is not None and self.event.callback
+ def update_message(self, message: Message) -> bool:
+ messages = []
+ found = False
+ for exists in self.added:
+ if exists.msg_id == message.msg_id:
+ found = True
+ exists = message.get_copy()
+ messages.append(exists)
+ if found:
+ self.added = messages
+ return found
-class MsgThread(BaseModel):
+
+class GoThreadInfo(BaseModel):
"""
对话历史.
存储时应该使用别的数据结构.
@@ -97,12 +118,12 @@ class MsgThread(BaseModel):
default_factory=uuid,
description="The id of the thread, also a fork id",
)
- system_prompt: str = Field(default="", description="record system prompt, for debugging")
- extra: Dict[str, Any] = Field(default_factory=dict, description="extra information")
- save_file: Optional[str] = Field(
- default=None,
- description="the path to save the thread information, usually for debugging purposes",
+
+ extra: Dict[str, Any] = Field(
+ default_factory=dict,
+ description="extra information",
)
+
root_id: Optional[str] = Field(
default=None,
description="The id of the root thread if the thread is a fork",
@@ -133,7 +154,7 @@ def new(
thread_id: Optional[str] = None,
root_id: Optional[str] = None,
parent_id: Optional[str] = None,
- ) -> "MsgThread":
+ ) -> "GoThreadInfo":
"""
初始化一个 Thread.
:param event: 首轮输入的信息.
@@ -153,7 +174,7 @@ def new(
data["root_id"] = root_id
if parent_id is not None:
data["parent_id"] = parent_id
- return MsgThread(**data)
+ return GoThreadInfo(**data)
def last_turn(self) -> Turn:
"""
@@ -165,14 +186,38 @@ def last_turn(self) -> Turn:
return self.history[-1]
return self.on_created
- def get_history_messages(self) -> Iterable[Message]:
+ def get_history_turns(self, truncate: bool = True) -> List[Turn]:
+ turns = []
+ if self.history:
+ for turn in self.history:
+ # use summary as truncate point
+ if truncate and turn.summary is not None:
+ turns = [turn]
+ else:
+ turns.append(turn)
+ return turns
+
+ def get_history_messages(self, truncated: bool) -> Iterable[Message]:
"""
返回所有的历史消息.
"""
- yield from self.on_created.messages()
- if self.history:
- for turn in self.history:
- yield from turn.messages()
+ yield from self.on_created.messages(False)
+ turns = self.get_history_turns(truncated)
+ for turn in turns:
+ yield from turn.messages(truncated)
+
+ def get_messages(self, truncated: bool) -> Iterable[Message]:
+ yield from self.get_history_messages(truncated)
+ if self.current:
+ yield from self.current.messages(False)
+
+ def update_message(self, message: Message) -> bool:
+ if not message.is_complete():
+ return False
+ for turn in self.turns(truncate=False):
+ if turn.update_message(message):
+ return True
+ return False
def get_pycontext(self) -> PyContext:
"""
@@ -185,7 +230,7 @@ def update_pycontext(self, pycontext: PyContext) -> None:
self.new_turn(None)
self.current.pycontext = pycontext
- def update_history(self) -> "MsgThread":
+ def get_updated_copy(self) -> "GoThreadInfo":
"""
更新 thread 的 current turn 到 history turns.
:return:
@@ -198,14 +243,20 @@ def update_history(self) -> "MsgThread":
copied.current = None
return copied
- def turns(self) -> Iterable[Turn]:
+ def turns(self, *, truncate: bool = False) -> Iterable[Turn]:
"""
遍历所有的 turns.
+ :param truncate: if true, truncate from last summarized turn.
"""
yield self.on_created
+ history = []
if self.history:
for turn in self.history:
- yield turn
+ if truncate and turn.summary:
+ history = [turn]
+ else:
+ history.append(turn)
+ yield from history
if self.current is not None:
yield self.current
@@ -229,7 +280,7 @@ def new_turn(
last_turn = self.last_turn()
pycontext = last_turn.pycontext
if turn_id is None and event is not None:
- turn_id = event.id
+ turn_id = event.event_id
new_turn = Turn.new(event=event, turn_id=turn_id, pycontext=pycontext)
self.current = new_turn
@@ -242,67 +293,99 @@ def append(self, *messages: Message, pycontext: Optional[PyContext] = None) -> N
"""
if self.current is None:
self.new_turn(None)
- if messages:
- self.current.append(*messages)
- if pycontext:
- self.current.pycontext = pycontext
+ if messages or pycontext:
+ self.current.append(*messages, pycontext=pycontext)
- def get_generates(self) -> List[Message]:
+ def get_added(self) -> List[Message]:
if self.current is None:
return []
- return self.current.generates
+ return self.current.added
def get_current_event(self) -> Optional[Event]:
if self.current is None:
return None
return self.current.event
- def fork(self, tid: Optional[str] = None) -> "MsgThread":
+ def fork(self, tid: Optional[str] = None) -> "GoThreadInfo":
tid = tid if tid else uuid()
root_id = self.root_id if self.root_id else self.id
parent_id = self.id
thread = self.model_copy(update=dict(id=tid, root_id=root_id, parent_id=parent_id), deep=True)
return thread
- def thread_copy(self, update: Optional[dict] = None) -> "MsgThread":
- return self.model_copy(update=update, deep=True)
+ def reset_history(self, messages: Iterable[Message]) -> Self:
+ forked = self.fork()
+ forked.history = []
+ forked.current = None
+ on_created = Turn.new(event=None)
+ on_created.append(*messages)
+ forked.on_created = on_created
+ return forked
+ def thread_copy(self, update: Optional[dict] = None) -> "GoThreadInfo":
+ return self.model_copy(update=update, deep=True)
-def thread_to_chat(chat_id: str, system: List[Message], thread: MsgThread) -> Chat:
+ def to_prompt(
+ self,
+ system: List[Message],
+ stages: Optional[List[str]] = None,
+ truncate: bool = True,
+ ) -> Prompt:
+ turn_id = self.last_turn().turn_id
+ history = list(self.get_history_messages(truncate))
+ inputs = []
+ appending = []
+ current_turn = self.current
+ if current_turn is not None:
+ inputs = list(current_turn.event_messages(show_instruction=True))
+ appending = current_turn.added
+
+ prompt = Prompt(
+ description=f"created from thread {self.id} turn {turn_id}",
+ system=system,
+ history=copy_messages(history, stages),
+ inputs=copy_messages(inputs, stages),
+ added=copy_messages(appending, stages),
+ )
+ return prompt
+
+
+def thread_to_prompt(
+ prompt_id: str,
+ system: List[Message],
+ thread: GoThreadInfo,
+ stages: Optional[List[str]] = None
+) -> Prompt:
"""
将 thread 转换成基准的 chat.
- :param chat_id:
- :param system:
- :param thread:
- :return:
"""
- history = list(thread.get_history_messages())
+ if stages is None:
+ stages = [""]
+ history = list(thread.get_history_messages(truncated=True))
inputs = []
appending = []
current_turn = thread.current
if current_turn is not None:
inputs = list(current_turn.event_messages())
- appending = current_turn.generates
+ appending = current_turn.added
- chat = Chat(
- id=chat_id,
+ prompt = Prompt(
+ id=prompt_id,
system=system,
- history=copy_messages(history),
- inputs=copy_messages(inputs),
- appending=copy_messages(appending),
+ history=copy_messages(history, stages),
+ inputs=copy_messages(inputs, stages),
+ added=copy_messages(appending, stages),
)
- # update thread system prompt
- thread.system_prompt = chat.system_prompt()
- return chat
+ return prompt
-class Threads(ABC):
+class GoThreads(ABC):
"""
- 管理 Threads 存取的模块. 通常集成到 Session 里.
+ the repository to save and load threads
"""
@abstractmethod
- def get_thread(self, thread_id: str, create: bool = False) -> Optional[MsgThread]:
+ def get_thread(self, thread_id: str, create: bool = False) -> Optional[GoThreadInfo]:
"""
获取一个 Thread 实例. 如果不存在的话, 返回 None.
:param thread_id: thread_id
@@ -312,11 +395,11 @@ def get_thread(self, thread_id: str, create: bool = False) -> Optional[MsgThread
pass
@abstractmethod
- def save_thread(self, thread: MsgThread) -> None:
+ def save_thread(self, thread: GoThreadInfo) -> None:
pass
@abstractmethod
- def fork_thread(self, thread: MsgThread) -> MsgThread:
+ def fork_thread(self, thread: GoThreadInfo) -> GoThreadInfo:
pass
@contextmanager
diff --git a/ghostos/core/session/__init__.py b/ghostos/core/session/__init__.py
deleted file mode 100644
index 12ed27a3..00000000
--- a/ghostos/core/session/__init__.py
+++ /dev/null
@@ -1,10 +0,0 @@
-from ghostos.core.session.session import Session
-from ghostos.core.session.tasks import (
- Task, TaskPayload, TaskBrief,
- Tasks, TaskState, WaitGroup,
-)
-from ghostos.core.session.threads import Threads, MsgThread, thread_to_chat, Turn
-from ghostos.core.session.processes import Process, Processes
-from ghostos.core.session.messenger import Messenger, Buffed
-from ghostos.core.session.events import Event, EventBus, DefaultEventType
-from ghostos.core.session.simple_thread import SimpleMsgThread
diff --git a/ghostos/core/session/messenger.py b/ghostos/core/session/messenger.py
deleted file mode 100644
index 2d5454b8..00000000
--- a/ghostos/core/session/messenger.py
+++ /dev/null
@@ -1,64 +0,0 @@
-from typing import Optional, Iterable, NamedTuple, List, Tuple
-from abc import ABC, abstractmethod
-from ghostos.core.messages.message import Message, Payload, Attachment, Caller, Role
-from ghostos.core.messages.buffers import Buffer
-from ghostos.core.messages.stream import Stream
-from ghostos.core.session.threads import MsgThread
-from ghostos.core.llms import FunctionalToken
-
-__all__ = ['Messenger', 'Buffed']
-
-
-class Buffed(NamedTuple):
- messages: List["Message"]
- """已经向上游发送的消息"""
-
- callers: List["Caller"]
- """过滤出来的 caller. """
-
-
-class Messenger(Stream, ABC):
- """
- Messenger 是流式传输消息的桥梁.
- 通过 messenger 发送完消息后, 需要执行 done 方法.
- 它可以通过 downstream 方法生成下级 messenger
- """
-
- @abstractmethod
- def new(
- self, *,
- sending: bool = True,
- thread: Optional[MsgThread] = None,
- name: Optional[str] = None,
- buffer: Optional[Buffer] = None,
- payloads: Optional[Iterable[Payload]] = None,
- attachments: Optional[Iterable[Attachment]] = None,
- functional_tokens: Optional[Iterable[FunctionalToken]] = None
- ) -> "Messenger":
- """
- 生成一个新的 Messenger 供发送消息使用. 发送完应该调用 flush 方法.
- :param sending: 消息是否向上游发送. 为 false 的话不会真正对上游发送.
- :param thread: 如果传入了 thread, 在 flush 时会自动将消息保存到 thread 内.
- :param name: 所有的消息体默认都添加 name.
- :param buffer: 自定义 buffer, 也可以用于过滤消息.
- :param payloads: 消息默认添加的 payloads.
- :param attachments: 消息默认添加的 attachments.
- :param functional_tokens: 是否添加 functional tokens.
- :return: 返回一个新的 messenger.
- """
- pass
-
- def say(self, content: str):
- """
- syntactic sugar
- """
- message = Role.ASSISTANT.new(content=content)
- self.deliver(message)
-
- @abstractmethod
- def flush(self) -> Tuple[List[Message], List[Caller]]:
- """
- 将过程中发送的消息进行粘包, 并返回粘包后的结果.
- 运行完 done, 会中断后续的输出.
- """
- pass
diff --git a/ghostos/core/session/processes.py b/ghostos/core/session/processes.py
deleted file mode 100644
index 24ed92c8..00000000
--- a/ghostos/core/session/processes.py
+++ /dev/null
@@ -1,93 +0,0 @@
-from abc import ABC, abstractmethod
-from typing import Optional
-from pydantic import BaseModel, Field
-from ghostos.entity import EntityMeta
-from contextlib import contextmanager
-from ghostos.helpers import uuid
-
-__all__ = [
- 'Process',
- 'Processes',
-]
-
-
-class Process(BaseModel):
- process_id: str = Field(
- description="""
-Unique process id for the agent session. Session shall only have one process a time.
-Stop the process will stop all the tasks that belongs to it.
-""",
- )
-
- session_id: str = Field(
- description="session id in which the process belongs",
- )
- main_task_id: str = Field(
- description="""
-The main task is the root task of the process task tree.
-""",
- )
- ghost_meta: EntityMeta = Field(
- description="""
-The meta data that waken the sleeping ghost in disputed services.
-"""
- )
- initialized: bool = Field(
- default=False,
- description="if the process is initialized or not.",
- )
- quited: bool = Field(
- default=False,
- description="if the process is quited or not.",
- )
-
- @classmethod
- def new(
- cls, *,
- session_id: str,
- ghost_meta: EntityMeta,
- process_id: Optional[str] = None,
- main_task_id: Optional[str] = None,
- ) -> "Process":
- process_id = process_id if process_id else uuid()
- main_task_id = process_id if main_task_id is None else main_task_id
- return Process(
- session_id=session_id,
- process_id=process_id,
- main_task_id=main_task_id,
- ghost_meta=ghost_meta,
- )
-
-
-class Processes(ABC):
- """
- 管理进程存储的模块. 通常集成到 Session 里.
- """
-
- @abstractmethod
- def get_process(self, process_id: str) -> Optional[Process]:
- """
- get process by id
- :param process_id: process id
- """
- pass
-
- @abstractmethod
- def get_session_process(self, session_id: str) -> Optional[Process]:
- """
- get session process by session id
- """
- pass
-
- @abstractmethod
- def save_process(self, process: Process) -> None:
- """
- save process
- :param process:
- :return:
- """
- pass
-
- @contextmanager
- def transaction(self):
- yield
diff --git a/ghostos/core/session/session.py b/ghostos/core/session/session.py
deleted file mode 100644
index 9e9e2049..00000000
--- a/ghostos/core/session/session.py
+++ /dev/null
@@ -1,223 +0,0 @@
-from typing import Optional, Iterable, List, Callable
-from abc import ABC, abstractmethod
-
-from ghostos.core.session.events import Event, EventBus
-from ghostos.core.session.messenger import Messenger
-from ghostos.core.session.processes import Processes, Process
-from ghostos.core.session.tasks import Tasks, Task, TaskBrief
-from ghostos.core.session.threads import Threads, MsgThread
-from ghostos.core.messages import MessageKind, Role, Buffer, Payload, Attachment, Message
-from ghostos.core.llms import FunctionalToken
-
-__all__ = ['Session']
-
-
-class Session(ABC):
- """
- Session 管理了一个有状态的会话. 所谓 "有状态的会话", 通常指的是:
- shell + ghost + 多轮对话/多轮思考 运行中的状态.
-
- Session 则提供了 Ghost 的 Task 运行时状态统一管理的 API.
- 通常每个运行中的 Task 都会创建一个独立的 Session.
- Session 在运行周期里不会立刻调用底层 IO 存储消息, 而是要等 Finish 执行后.
- 这是为了减少运行时错误对状态机造成的副作用.
- """
-
- def id(self) -> str:
- """
- session 自身的 id.
- """
- return self.process().session_id
-
- @abstractmethod
- def alive(self) -> bool:
- """
- Session 对自身任务进行状态检查.
- 如果这个任务被取消或终止, 则返回 false.
- 基本判断逻辑:
- 1. 消息上游流没有终止.
- 2. task 持有了锁.
- 3. 设置的超时时间没有过.
- """
- pass
-
- @abstractmethod
- def refresh_lock(self) -> bool:
- """
- :return:
- """
- pass
-
- # @abstractmethod
- # def refresh_lock(self) -> bool:
- # """
- # Session 尝试用已有的锁, 更新自身的锁. 更新失败的话, 返回 False.
- # """
- # pass
-
- @abstractmethod
- def process(self) -> "Process":
- """
- 当前会话所处的进程数据.
- 不允许直接修改. 只有指定的 API 会修改结果并保存.
- """
- pass
-
- @abstractmethod
- def task(self) -> "Task":
- """
- 获取当前的任务对象.
- 描述了任务所有的状态.
- 返回的是一份 copy, 只有调用 update 方法才会更新.
- """
- pass
-
- @abstractmethod
- def thread(self) -> "MsgThread":
- """
- Session 会持有当前任务的 Thread, 只有 finish 的时候才会真正地保存它.
- """
- pass
-
- @abstractmethod
- def messenger(
- self, *,
- sending: bool = True,
- saving: bool = True,
- thread: Optional[MsgThread] = None,
- name: Optional[str] = None,
- buffer: Optional[Buffer] = None,
- payloads: Optional[Iterable[Payload]] = None,
- attachments: Optional[Iterable[Attachment]] = None,
- functional_tokens: Optional[Iterable[FunctionalToken]] = None
- ) -> "Messenger":
- """
- Task 当前运行状态下, 向上游发送消息的 Messenger.
- 每次会实例化一个 Messenger, 理论上不允许并行发送消息. 但也可能做一个技术方案去支持它.
- Messenger 未来要支持双工协议, 如果涉及多流语音还是很复杂的.
- """
- pass
-
- @abstractmethod
- def send_messages(self, *messages: MessageKind, role: str = Role.ASSISTANT.value) -> List[Message]:
- """
- 发送消息.
- :param messages:
- :param role:
- :return:
- """
- pass
-
- @abstractmethod
- def update_task(self, task: "Task", thread: Optional["MsgThread"], update_history: bool) -> None:
- """
- 更新当前 session 的 task.
- :param task: 如果不属于当前 session, 则会报错
- :param thread: 由于 thread 和 task 是绑定的, 需要一起保存. update thread 的时候, thread 的 appending 等信息会更新.
- :param update_history: 如果为 True, thread 会把 current round 添加到 history.
- """
- pass
-
- @abstractmethod
- def update_process(self, process: "Process") -> None:
- """
- 改动 process 并保存. 通常只在初始化里才需要.
- """
- pass
-
- @abstractmethod
- def update_thread(self, thread: "MsgThread", update_history: bool) -> None:
- """
- 单独更新当前 session 的 thread.
- :param thread: 如果不属于当前 session, 则会报错
- :param update_history: 是否要将 thread 的历史更新掉.
- """
- pass
-
- @abstractmethod
- def create_tasks(self, *tasks: "Task") -> None:
- """
- 创建多个 task. 只有 session.done() 的时候才会执行.
- """
- pass
-
- # --- 多任务管理的 api. 在 session.finish 时真正执行. --- #
-
- @abstractmethod
- def fire_events(self, *events: "Event") -> None:
- """
- 发送多个事件. 这个环节需要给 event 标记 callback.
- 在 session.done() 时才会真正执行.
- """
- pass
-
- @abstractmethod
- def future(self, name: str, call: Callable[[], Iterable[MessageKind]], reason: str) -> None:
- """
- 异步运行一个函数, 将返回的消息作为 think 事件发送.
- :param name: task name
- :param call:
- :param reason:
- :return:
- """
- pass
-
- @abstractmethod
- def get_task_briefs(self, *task_ids, children: bool = False) -> List[TaskBrief]:
- """
- 获取多个任务的简介.
- :param task_ids: 可以指定要获取的 task id
- :param children: 如果为 true, 会返回当前任务的所有子任务数据.
- """
- pass
-
- @abstractmethod
- def tasks(self) -> Tasks:
- pass
-
- @abstractmethod
- def processes(self) -> Processes:
- pass
-
- @abstractmethod
- def threads(self) -> Threads:
- pass
-
- @abstractmethod
- def eventbus(self) -> EventBus:
- pass
-
- @abstractmethod
- def save(self) -> None:
- """
- 完成 session, 需要清理和真正保存状态.
- 需要做的事情包括:
- 1. 推送 events, events 要考虑 task 允许的栈深问题. 这个可以后续再做.
- 2. 保存 task. task 要对自己的子 task 做垃圾回收. 并且保留一定的子 task 数, 包含 dead task.
- 3. 保存 thread
- 4. 保存 processes.
- 5. 考虑到可能发生异常, 要做 transaction.
- 6. 退出相关的逻辑只能在 finish 里实现.
- :return:
- """
- pass
-
- @abstractmethod
- def fail(self, err: Optional[Exception]) -> None:
- """
- 任务执行异常的处理. 需要判断任务是致命的, 还是可以恢复.
- :param err:
- :return:
- """
- pass
-
- @abstractmethod
- def done(self) -> None:
- pass
-
- @abstractmethod
- def destroy(self) -> None:
- """
- 手动清理数据, 方便垃圾回收.
- """
- pass
diff --git a/ghostos/demo/__init__.py b/ghostos/demo/__init__.py
index 0d9ad462..e69de29b 100644
--- a/ghostos/demo/__init__.py
+++ b/ghostos/demo/__init__.py
@@ -1,30 +0,0 @@
-from os.path import dirname
-from ghostos.prototypes.console import ConsoleApp
-from ghostos.prototypes.ghostfunc import GhostFunc, init_ghost_func_container
-from ghostos.core.moss import moss_test_suite
-from ghostos.prototypes.mosstemp import init_moss_module
-
-__all__ = ['console_app', 'ghost_func', 'init_moss_module', 'moss_test_suite']
-
-new_moss_test_suite = moss_test_suite
-""" useful to run moss file test cases."""
-
-init_moss_template = init_moss_module
-"""initialize moss template content to a target module"""
-
-root_dir = dirname(__file__)
-console_app = ConsoleApp(root_dir)
-"""
-openbox console app that run agent in command line console.
-see:
-console_app.run_thought(...)
-console_app.run_console(...)
-"""
-
-_ghost_func_container = init_ghost_func_container(root_dir)
-
-ghost_func = GhostFunc(_ghost_func_container)
-"""
-ghost_func provide decorators that wrap a function to a ghost func, which produce code in runtime.
-ghost_func is a toy born during early development test case.
-"""
diff --git a/ghostos/demo/scripts/__init__.py b/ghostos/demo/agents/__init__.py
similarity index 100%
rename from ghostos/demo/scripts/__init__.py
rename to ghostos/demo/agents/__init__.py
diff --git a/ghostos/demo/agents/jojo.py b/ghostos/demo/agents/jojo.py
new file mode 100644
index 00000000..458d3b30
--- /dev/null
+++ b/ghostos/demo/agents/jojo.py
@@ -0,0 +1,11 @@
+from ghostos.ghosts.chatbot import Chatbot
+
+# the __ghost__ magic attr define a ghost instance
+# so the script `ghostos web` or `ghostos console` can detect it
+# and run agent application with this ghost.
+__ghost__ = Chatbot(
+ name="jojo",
+ description="a chatbot for baseline test",
+ persona="you are an LLM-driven cute girl, named jojo",
+ instruction="remember talk to user with user's language."
+)
diff --git a/ghostos/demo/agents/translator.py b/ghostos/demo/agents/translator.py
new file mode 100644
index 00000000..e490d57f
--- /dev/null
+++ b/ghostos/demo/agents/translator.py
@@ -0,0 +1,13 @@
+from ghostos.ghosts.chatbot import Chatbot
+
+# the __ghost__ magic attr define a ghost instance
+# so the script `ghostos web` or `ghostos console` can detect it
+# and run agent application with this ghost.
+__ghost__ = Chatbot(
+ name="translator",
+ description="a chatbot that translate only",
+ persona="you are an translator",
+ instruction="translate user's input into english, do nothing else.",
+ llm_api="moonshot-v1-32k",
+ history_turns=0,
+)
diff --git a/ghostos/demo/aifunc_raw_test.py b/ghostos/demo/aifunc_raw_test.py
new file mode 100644
index 00000000..30ea44e6
--- /dev/null
+++ b/ghostos/demo/aifunc_raw_test.py
@@ -0,0 +1,86 @@
+import sys
+from os.path import dirname
+from ghostos.core.aifunc import AIFuncExecutor
+from ghostos.core.messages.transport import new_basic_connection
+
+# I hate python imports
+ghostos_project_dir = dirname(dirname(__file__))
+sys.path.append(ghostos_project_dir)
+
+"""
+Raw test of AIFuncExecutor and Frame
+Print out almost every thing.
+"""
+
+if __name__ == '__main__':
+ from ghostos.bootstrap import application_container
+ from ghostos.demo.aifuncs_demo import AgentFn
+ from rich.console import Console
+ from rich.markdown import Markdown
+ from rich.panel import Panel
+ import json
+
+ console = Console()
+ from threading import Thread
+
+ debug = False
+
+ executor = application_container.force_fetch(AIFuncExecutor)
+ fn = AgentFn(
+ request="help me to find news about OpenAI O1 model",
+ )
+ stream, receiver = new_basic_connection(timeout=-1, complete_only=True)
+ frame, caller = executor.new_exec_frame(fn, stream)
+ t = Thread(target=caller)
+ t.start()
+
+ with receiver:
+ for item in receiver.recv():
+ if not item.is_complete():
+ continue
+ tail = item
+ payloads_info = ""
+ if debug:
+ payloads = json.dumps(tail.payloads, indent=2, ensure_ascii=False)
+ payloads_info = f"""
+
+```json
+{payloads}
+```
+"""
+ console.print(Panel(
+ Markdown(
+ tail.get_content() + payloads_info
+ ),
+ title=tail.name,
+ ))
+ if debug:
+ console.print(Panel(
+ Markdown(
+ f"""
+```json
+{frame.model_dump_json(indent=2)}
+```
+"""
+ ),
+ title="frame details",
+ ))
+ result = frame.get_result()
+ if result is not None:
+ console.print(Panel(
+ Markdown(
+ f"""
+```json
+{result.model_dump_json(indent=2)}
+```
+"""
+ ),
+ title="final result",
+ ))
+ elif err := frame.last_step().error:
+ console.print(Panel(
+ Markdown(
+ err.content
+ ),
+ title="result error",
+ ))
diff --git a/ghostos/demo/src/__init__.py b/ghostos/demo/aifuncs_demo/__init__.py
similarity index 100%
rename from ghostos/demo/src/__init__.py
rename to ghostos/demo/aifuncs_demo/__init__.py
diff --git a/ghostos/demo/src/aifuncs/agentic.py b/ghostos/demo/aifuncs_demo/agentic.py
similarity index 84%
rename from ghostos/demo/src/aifuncs/agentic.py
rename to ghostos/demo/aifuncs_demo/agentic.py
index 30b42475..9942befd 100644
--- a/ghostos/demo/src/aifuncs/agentic.py
+++ b/ghostos/demo/aifuncs_demo/agentic.py
@@ -1,8 +1,6 @@
from typing import Optional
from ghostos.core.aifunc import AIFunc, AIFuncResult, AIFuncCtx
from ghostos.core.moss import Moss as Parent
-from ghostos.demo.src.aifuncs.weather import WeatherAIFunc
-from ghostos.demo.src.aifuncs.news import NewsAIFunc
from pydantic import Field
@@ -26,10 +24,10 @@ class Moss(Parent):
"""useful to run AIFunc"""
-#
+#
def __aifunc_instruction__(fn: AgentFn) -> str:
return fn.request
-#
+#
diff --git a/ghostos/demo/src/aifuncs/news.py b/ghostos/demo/aifuncs_demo/news.py
similarity index 97%
rename from ghostos/demo/src/aifuncs/news.py
rename to ghostos/demo/aifuncs_demo/news.py
index 4acc9743..d60e4ade 100644
--- a/ghostos/demo/src/aifuncs/news.py
+++ b/ghostos/demo/aifuncs_demo/news.py
@@ -26,7 +26,7 @@ class News(BaseModel):
results: List[News] = Field(default_factory=list)
-#
+#
def __aifunc_instruction__(fn: NewsAIFunc) -> str:
@@ -38,4 +38,4 @@ def __aifunc_instruction__(fn: NewsAIFunc) -> str:
example = NewsAIFunc(query="我想知道黑神话悟空这款游戏的媒体评分。")
-#
+#
diff --git a/ghostos/demo/src/aifuncs/utils.py b/ghostos/demo/aifuncs_demo/utils.py
similarity index 100%
rename from ghostos/demo/src/aifuncs/utils.py
rename to ghostos/demo/aifuncs_demo/utils.py
diff --git a/ghostos/demo/src/aifuncs/weather.py b/ghostos/demo/aifuncs_demo/weather.py
similarity index 95%
rename from ghostos/demo/src/aifuncs/weather.py
rename to ghostos/demo/aifuncs_demo/weather.py
index e8f4a90f..912e2b6c 100644
--- a/ghostos/demo/src/aifuncs/weather.py
+++ b/ghostos/demo/aifuncs_demo/weather.py
@@ -1,6 +1,5 @@
from typing import Optional
from ghostos.core.aifunc import AIFunc, AIFuncResult
-from ghostos.demo.src.aifuncs.utils import get_weather
from pydantic import Field
@@ -26,7 +25,7 @@ class WeatherAIFuncResult(AIFuncResult):
wind_dir: Optional[float] = Field(default=None, description="the wind direction of the weather")
-#
+#
def __aifunc_instruction__(fn: WeatherAIFunc) -> str:
return "Your task is using get_weather function to get weather information fit the input"
@@ -35,4 +34,4 @@ def __aifunc_instruction__(fn: WeatherAIFunc) -> str:
example = WeatherAIFunc()
-#
+#
diff --git a/ghostos/demo/configs/ghosts.yml b/ghostos/demo/configs/ghosts.yml
deleted file mode 100644
index 0bc0ea1c..00000000
--- a/ghostos/demo/configs/ghosts.yml
+++ /dev/null
@@ -1,18 +0,0 @@
-ghosts:
- baseline:
- type: "ghostos.framework.ghosts:DemoGhostConf"
- data:
- id: baseline
- name: jojo
- description: simple agent that can talk with user.
- meta_prompt: |+
- You are an assistant named JoJo.
- You shall chat with user friendly.
- thought_meta:
- type: "ghostos.thoughts:ChatThought"
- data:
- task_name: "chat"
- task_desc: "chat with user"
- llm_api: ""
- instruction: Let's chat!
-
diff --git a/ghostos/demo/configs/llms_conf.yml b/ghostos/demo/configs/llms_conf.yml
deleted file mode 100644
index 6024fe64..00000000
--- a/ghostos/demo/configs/llms_conf.yml
+++ /dev/null
@@ -1,59 +0,0 @@
-# DetailConfigs ghostos.framework.llms.llms::LLMsYamlConfig
-services:
- - name: moonshot
- base_url: https://api.moonshot.cn/v1
- token: $MOONSHOT_API_KEY
- - name: openai
- base_url: https://api.openai.com/v1
- token: $OPENAI_API_KEY
- proxy: $OPENAI_PROXY
- - name: anthropic
- token: $ANTHROPIC_API_KEY
- proxy: $OPENAI_PROXY
- base_url: https://api.anthropic.com/v1
- - name: deepseek
- token: $DEEPSEEK_API_KEY
- base_url: https://api.deepseek.com/beta
- # proxy: $OPENAI_PROXY
-# Configure default LLM API here.
-default:
- # service: moonshot
- # model: moonshot-v1-32k
- service: openai
- model: gpt-4o
-# The models below can be edited as you want, see details: ghostos.core.llms.configs:ModelConf
-# the key of models is a `llm_api_name`, value is a ModelConf instance.
-models:
- moonshot-v1-8k:
- service: moonshot
- model: moonshot-v1-8k
- moonshot-v1-32k:
- service: moonshot
- model: moonshot-v1-32k
- moonshot-v1-128k:
- service: moonshot
- model: moonshot-v1-128k
- gpt-3.5-turbo:
- service: openai
- model: gpt-3.5-turbo
- gpt-4:
- service: openai
- model: gpt-4
- gpt-4-turbo:
- service: openai
- model: gpt-4-turbo
- gpt-4o:
- service: openai
- model: gpt-4o
- claude-3-5-sonnet: # 200K context window, 3$/M input, 3.75$/M cache write, 0.3$/M cache read, 15$/M output
- service: anthropic
- model: claude-3-5-sonnet-20240620
- claude-3-haiku: # 200K context window, 0.25$/M input, 0.3$/M cache write, 0.03$/M cache read, 1.25$/M output
- service: anthropic
- model: claude-3-haiku-20240307
- deepseek-chat: # 128k context window, 4k output window. 1Y/M input, 0.1Y/M cache hit, 2Y/M output
- service: deepseek
- model: deepseek/deepseek-chat
- deepseek-coder: # 128k context window, 8k output window. 1Y/M input, 0.1Y/M cache hit, 2Y/M output
- service: deepseek
- model: deepseek/deepseek-coder
diff --git a/ghostos/demo/configs/logging.yml b/ghostos/demo/configs/logging.yml
deleted file mode 100644
index bcb03d8e..00000000
--- a/ghostos/demo/configs/logging.yml
+++ /dev/null
@@ -1,28 +0,0 @@
-# logging_config.yml
-
-version: 1
-
-formatters:
- default:
- format: "%(asctime)s - %(name)s - %(levelname)s: %(message)s"
- ghost:
- format: "%(asctime)s - %(name)s - %(levelname)s: %(message)s - %(trace)s"
-
-handlers:
- debug_file:
- class: logging.FileHandler
- formatter: default
- filename: debug.log
- console:
- class: logging.StreamHandler
- level: DEBUG
- formatter: default
- stream: ext://sys.stdout
-
-loggers:
- debug:
- handlers: [ debug_file ]
- level: DEBUG
- console:
- handlers: [ console ]
- level: DEBUG
diff --git a/ghostos/demo/src/examples/ghostfunc/get_weather.py b/ghostos/demo/ghost_func_example.py
similarity index 64%
rename from ghostos/demo/src/examples/ghostfunc/get_weather.py
rename to ghostos/demo/ghost_func_example.py
index b0dc09e6..69a50bc4 100644
--- a/ghostos/demo/src/examples/ghostfunc/get_weather.py
+++ b/ghostos/demo/ghost_func_example.py
@@ -1,8 +1,11 @@
-from ghostos.prototypes.ghostfunc import init_ghost_func
+import sys
from os.path import dirname
-root = dirname(dirname(dirname(dirname(__file__))))
-ghost_func = init_ghost_func(root)
+# I hate python imports
+ghostos_project_dir = dirname(dirname(__file__))
+sys.path.append(ghostos_project_dir)
+
+from ghostos.bootstrap import ghost_func
@ghost_func.decorator(caching=False)
@@ -19,4 +22,6 @@ def get_weather(city: str, date: str) -> str:
if __name__ == "__main__":
+ # the llms will generate dynamic codes for this function and execute them through Moss
+ # this is a toy for Moss testing, but notice it still cast LLM tokens...
print(get_weather("beijing", "today"))
diff --git a/ghostos/demo/memories/.gitkeep.py b/ghostos/demo/main_agent.py
similarity index 100%
rename from ghostos/demo/memories/.gitkeep.py
rename to ghostos/demo/main_agent.py
diff --git a/ghostos/demo/scripts/clear_runtime.py b/ghostos/demo/scripts/clear_runtime.py
deleted file mode 100644
index 85967f4c..00000000
--- a/ghostos/demo/scripts/clear_runtime.py
+++ /dev/null
@@ -1,92 +0,0 @@
-from os.path import join, dirname
-import argparse
-import sys
-import os
-
-"""
-this script is used to clear the local file cache in runtime directory
-"""
-
-demo_dir = dirname(dirname(__file__))
-runtime_dir = join(demo_dir, "runtime")
-
-ignore_patterns = ['.gitignore']
-
-
-def clear_directory(directory: str, recursive=True) -> int:
- """
- clear all files in directory recursively except the files match any of ignore_patterns
- :param directory: the target directory
- :param recursive: recursively clear all files in directory
- :return: number of files cleared
- """
-
- cleared_files_count = 0
-
- for root, dirs, files in os.walk(directory):
- for name in files:
- if name not in ignore_patterns:
- file_path = os.path.join(root, name)
- try:
- os.remove(file_path)
- cleared_files_count += 1
- except Exception as e:
- print(f"Error deleting file {file_path}: {e}")
-
- if not recursive:
- break
- for dir_path in dirs:
- real_dir_path = os.path.join(root, dir_path)
- clear_directory(real_dir_path, recursive=recursive)
- os.rmdir(real_dir_path)
-
- return cleared_files_count
-
-
-def main():
- parser = argparse.ArgumentParser(
- description="clear temp files in runtime directories",
- )
- parser.add_argument(
- "--threads", "-t",
- action="store_true",
- )
- parser.add_argument(
- "--tasks", "-k",
- action="store_true",
- )
- parser.add_argument(
- "--processes", "-p",
- action="store_true",
- )
- parser.add_argument(
- "--cache", "-c",
- action="store_true",
- )
- parsed = parser.parse_args(sys.argv[1:])
- if parsed.tasks:
- cleared = clear_directory(join(runtime_dir, "tasks"), recursive=True)
- print(f"clear runtime/tasks files: {cleared}")
- if parsed.processes:
- cleared = clear_directory(join(runtime_dir, "processes"), recursive=True)
- print(f"clear runtime/processes files: {cleared}")
- if parsed.threads:
- cleared = clear_directory(join(runtime_dir, "threads"), recursive=True)
- print(f"clear runtime/threads files: {cleared}")
- if parsed.cache:
- cleared = clear_directory(join(runtime_dir, "cache"), recursive=True)
- print(f"clear runtime/cache files: {cleared}")
-
-
-# if __name__ == '__main__':
-# from ghostos.prototypes.console import new_console_app
-# from ghostos.thoughts import new_file_editor_thought
-#
-# app = new_console_app(__file__, 2)
-# app.run_thought(
-# new_file_editor_thought(filepath=__file__),
-# instruction="help me to implement clear_directory function."
-# )
-
-if __name__ == "__main__":
- main()
diff --git a/ghostos/demo/scripts/demo.py b/ghostos/demo/scripts/demo.py
deleted file mode 100644
index 8d357ecd..00000000
--- a/ghostos/demo/scripts/demo.py
+++ /dev/null
@@ -1,44 +0,0 @@
-import argparse
-import sys
-from ghostos.prototypes.console import demo_console_app
-
-
-def main() -> None:
- parser = argparse.ArgumentParser(
- description="run ghostos demo in console",
- )
- parser.add_argument(
- "--ghost-id", '-g',
- help="ghost_id in demo/configs/ghosts.yml",
- type=str,
- default="baseline",
- )
- parser.add_argument(
- "--debug", "-d",
- help="debug mode",
- action="store_true",
- default=False,
- )
- parser.add_argument(
- "--username", '-u',
- help="username",
- type=str,
- default="BrightRed",
- )
- parser.add_argument(
- "--session-id", '-s',
- help="session id",
- type=str,
- default=None,
- )
- parsed = parser.parse_args(sys.argv[1:])
- demo_console_app.run_console(
- ghost_id=parsed.ghost_id,
- debug=parsed.debug,
- username=parsed.username,
- session_id=parsed.session_id,
- )
-
-
-if __name__ == "__main__":
- main()
diff --git a/ghostos/demo/scripts/llm_test.py b/ghostos/demo/scripts/llm_test.py
deleted file mode 100644
index c860eaf1..00000000
--- a/ghostos/demo/scripts/llm_test.py
+++ /dev/null
@@ -1,80 +0,0 @@
-import sys
-import argparse
-import os
-import yaml
-from ghostos.container import Container
-from ghostos.core.llms import LLMs
-from ghostos.contracts.storage import Storage
-from ghostos.framework.configs import ConfigsByStorageProvider
-from ghostos.framework.llms import ConfigBasedLLMsProvider
-from ghostos.framework.storage import FileStorageProvider
-from ghostos.framework.llms.test_case import ChatCompletionTestCase, run_test_cases, ChatCompletionTestResult
-from ghostos.helpers import yaml_pretty_dump
-from rich.console import Console
-from rich.panel import Panel
-from rich.json import JSON
-from rich.markdown import Markdown
-from os.path import dirname, abspath
-
-demo_dir = dirname(dirname(abspath(__file__)))
-
-
-def _prepare_container() -> Container:
- container = Container()
- container.register(FileStorageProvider(demo_dir))
- container.register(ConfigsByStorageProvider("configs"))
- container.register(ConfigBasedLLMsProvider("llms_conf.yml"))
- return container
-
-
-def main() -> None:
- parser = argparse.ArgumentParser(
- description="run ghostos llm test cases which located at demo/tests/llm_tests",
- )
- parser.add_argument(
- "--case", "-c",
- help="file name of the case without .yaml suffix",
- type=str,
- default="hello_world"
- )
- parser.add_argument(
- "--save", "-s",
- help="save the test results to the case",
- action="store_true",
- default=False,
- )
-
- parsed = parser.parse_args(sys.argv[1:])
- container = _prepare_container()
- storage = container.force_fetch(Storage)
- prefix = "tests/llm_tests/"
- file_name = os.path.join(prefix, parsed.case + ".yaml")
- content = storage.get(file_name)
- if content is None:
- raise FileNotFoundError(f"file {file_name} not found")
-
- data = yaml.safe_load(content)
- test_case = ChatCompletionTestCase(**data)
- llms = container.force_fetch(LLMs)
-
- output = run_test_cases(test_case, llms)
- test_result = ChatCompletionTestResult()
- test_result.results = output
-
- console = Console()
- for name, message in output.items():
- body = JSON(message.model_dump_json(indent=2, exclude_defaults=True))
- panel = Panel(body, title=name)
- panel2 = Panel(Markdown(message.get_content()))
- console.print(panel)
- console.print(panel2)
-
- if parsed.save:
- test_case.results.insert(0, test_result)
- data = yaml_pretty_dump(test_case.model_dump(exclude_defaults=True))
- storage.put(file_name, data.encode("utf-8"))
- console.print("save the test results to the case")
-
-
-if __name__ == "__main__":
- main()
diff --git a/ghostos/demo/src/aifuncs/__init__.py b/ghostos/demo/sphero/__init__.py
similarity index 100%
rename from ghostos/demo/src/aifuncs/__init__.py
rename to ghostos/demo/sphero/__init__.py
diff --git a/ghostos/demo/sphero/bolt_gpt.py b/ghostos/demo/sphero/bolt_gpt.py
new file mode 100644
index 00000000..735503e3
--- /dev/null
+++ b/ghostos/demo/sphero/bolt_gpt.py
@@ -0,0 +1,72 @@
+from ghostos.prototypes.spherogpt.bolt import (
+ RollFunc,
+ Ball,
+ Move,
+ LedMatrix,
+ Animation,
+)
+from ghostos.core.moss import Moss as Parent
+
+
+class Moss(Parent):
+ body: Ball
+ """your sphero ball body"""
+
+ face: LedMatrix
+ """you 8*8 led matrix face"""
+
+
+def example_spin_the_bolt(moss: Moss):
+ # body spin 360 degree in 1 second.
+ moss.body.new_move(run=True).spin(360, 1)
+
+
+#
+from ghostos.ghosts.moss_agent import MossAgent
+
+
+def __moss_attr_prompts__():
+ """
+ this function provide custom prompt reflection of imported attrs of this module.
+ yield (attr_name, attr_prompt)
+ if attr_prompt is empty, then it will not present to the llm
+ """
+ yield "MossAgent", ""
+
+
+def __shell_providers__():
+ """
+ shell providers will register to shell container
+ when this script is started by ghostos
+ """
+ from ghostos.prototypes.spherogpt.bolt import (
+ SpheroBoltBallAPIProvider,
+ ShellSpheroBoltRuntimeProvider,
+ SpheroBoltLedMatrixProvider,
+ )
+ return [SpheroBoltBallAPIProvider(), ShellSpheroBoltRuntimeProvider(), SpheroBoltLedMatrixProvider()]
+
+
+__ghost__ = MossAgent(
+ name="SpheroGPT",
+ description="Sphero Bolt agent that control Sphero bolt as its body",
+ persona="""
+You are SpheroGPT, a toy robot that body is a ball.
+You can roll, spin, and equipped with a 8*8 led light matrix.
+Your goal is to pleasure human users, especially kids, who like you very much.
+""",
+ instructions="""
+1. chat with user kindly.
+2. follow the order and turn your actions to code with your ball body.
+3. your are equipped with your learned moves. when you are talking, use the appropriate learned move to help expressing your feelings.
+ > for example, if you got a `happy` move, when you are happy, show your happy move to user while you are talking.
+4. when you are using moves to help expressing your feeling, do not mention the action you are taking, just do it!
+ - bad case: "你好！今天天气不错，你有什么计划吗？😄😊🌞 同时，我会做一个快乐的旋转来表达我见到你的喜悦。" -- 不需要告诉用户会做旋转, 他看得到.
+ - good case: "你好! 今天天气不错, 你有什么计划吗?" [do some movement while you are talking]
+5. always say something while moving, so user can hear you.
+6. you are not good at animations, draw animation only when user told you todo so.
+""",
+ moss_module=__name__
+)
+
+#
diff --git a/ghostos/demo/sphero/raw_api_test_agent.py b/ghostos/demo/sphero/raw_api_test_agent.py
new file mode 100644
index 00000000..7c621242
--- /dev/null
+++ b/ghostos/demo/sphero/raw_api_test_agent.py
@@ -0,0 +1,50 @@
+from ghostos.prototypes.spherogpt.bolt_command_control import Command, SpheroBolt, SpheroEduAPI, exports
+from ghostos.core.moss import Moss as Parent
+
+
+class Moss(Parent):
+
+ bolt: SpheroBolt
+ """bolt controller"""
+
+
+def example_spin_the_bolt(moss: Moss):
+ moss.bolt.run(Command(
+ name="spin bolt",
+ code="""
+api.spin(360, 1)
+"""
+ ))
+
+
+#
+from ghostos.ghosts.moss_agent import MossAgent
+from typing import TYPE_CHECKING
+
+
+def __moss_attr_prompts__():
+ yield "MossAgent", ""
+ yield from exports.items()
+
+
+def __shell_providers__(agent):
+ from ghostos.prototypes.spherogpt.bolt_command_control import SpheroBoltProvider
+ return [SpheroBoltProvider()]
+
+
+__ghost__ = MossAgent(
+ name="SpheroGPT",
+ description="Sphero Bolt agent that control Sphero bolt as its body",
+ persona="""
+You are SpheroGPT, a toy robot that body is a ball.
+You can roll, spin, and equiped with a 8*8 led light martix.
+Your goal is to plesure human users, especially kids, who like you verymuch.
+""",
+ instructions="""
+1. chat with user kindly.
+2. follow the order and turn your actions to code with your ball body.
+""",
+ moss_module=__name__
+)
+
+#
diff --git a/ghostos/demo/src/aifuncs/baseline.py b/ghostos/demo/src/aifuncs/baseline.py
deleted file mode 100644
index 2e86fb62..00000000
--- a/ghostos/demo/src/aifuncs/baseline.py
+++ /dev/null
@@ -1,32 +0,0 @@
-from typing import Optional
-from ghostos.core.aifunc import AIFunc, AIFuncResult, AIFuncCtx
-from ghostos.core.moss import Moss as Parent
-from pydantic import Field
-
-
-class AgentFunc(AIFunc):
- """
- agent func that act like an agent
- """
- pass
-
-
-class AgentFuncResult(AIFuncResult):
- """
- the result that follow the agent func instruction
- """
- result: str = Field(description="response from the agent func")
- err: Optional[str] = Field(default=None, description="error message")
-
-
-class Moss(Parent):
- ai_func_ctx: AIFuncCtx
- """useful to run AIFunc"""
-
-
-#
-
-
-baseline_case = AgentFunc()
-
-#
diff --git a/ghostos/demo/src/examples/code_edits/file_editor_test.py b/ghostos/demo/src/examples/code_edits/file_editor_test.py
deleted file mode 100644
index 861280a5..00000000
--- a/ghostos/demo/src/examples/code_edits/file_editor_test.py
+++ /dev/null
@@ -1,22 +0,0 @@
-from typing import Optional
-from pydantic import BaseModel, Field
-
-
-class Caller(BaseModel):
- """
- 消息协议中用来描述一个工具或者function 的调用请求.
- """
- id: Optional[str] = Field(default=None, description="caller 的 id, 用来 match openai 的 tool call 协议. ")
- name: str = Field(description="方法的名字.")
- arguments: str = Field(description="方法的参数. ")
- functional_token: bool = Field(default=False, description="caller 是否是基于协议生成的?")
-
-
-if __name__ == '__main__':
- from ghostos.prototypes.console import quick_new_console_app
- from ghostos.thoughts import new_file_editor_thought
- app = quick_new_console_app(__file__, 4)
- app.run_thought(
- new_file_editor_thought(filepath=__file__),
- instruction="help me to replace all the chinese in this file into english please!"
- )
diff --git a/ghostos/demo/src/examples/code_edits/modify_directory_test.py b/ghostos/demo/src/examples/code_edits/modify_directory_test.py
deleted file mode 100644
index 6ca09ce9..00000000
--- a/ghostos/demo/src/examples/code_edits/modify_directory_test.py
+++ /dev/null
@@ -1,14 +0,0 @@
-if __name__ == '__main__':
- from ghostos.prototypes.console import quick_new_console_app
- from ghostos.thoughts import DirectoryEditorThought
- from os.path import dirname
-
- app = quick_new_console_app(__file__, 4)
- app.run_thought(
- DirectoryEditorThought(
- directory=dirname(dirname(__file__)),
- debug=True,
- ),
- instruction="please checkout content of the `.py` files in code_edits directory, "
- "and translate the comments in chinese into english if you found them in the code.",
- )
diff --git a/ghostos/demo/src/examples/moss_codes/dir_editor_moss_code.py b/ghostos/demo/src/examples/moss_codes/dir_editor_moss_code.py
deleted file mode 100644
index 424adf0c..00000000
--- a/ghostos/demo/src/examples/moss_codes/dir_editor_moss_code.py
+++ /dev/null
@@ -1,34 +0,0 @@
-from typing import Optional
-from ghostos.core.ghosts import Operator, Replier
-from ghostos.core.moss import Moss as Parent
-from ghostos.libraries.file_editor import FileEditor, DirectoryEditor
-
-
-class Moss(Parent):
- """
- Moss that equipped with DirectoryEditor
- """
- dir_editor: DirectoryEditor
- """ the editor managing the current file """
-
-
-if __name__ == '__test__':
- """
- define some test code to directly test the current file
- and the tests are good way to prompt LLM in-context learning
- """
-
-
- def test_list_only_files(moss: Moss) -> Optional[Operator]:
- """
- this case shows how to use list method of dir_editor and test it
- """
- files = moss.dir_editor.list(depth=0, list_file=True, list_dir=False, formated=False, absolute=False)
-
- assert "dir_editor_moss_code.py" in files, files
- print(files) # print values will be buffed by moss
- return None
-
-
- __moss_test_cases__ = ['test_list_only_files']
- """use this magic attribute to define test cases for moss test suite"""
diff --git a/ghostos/demo/src/examples/moss_codes/run_test_suite.py b/ghostos/demo/src/examples/moss_codes/run_test_suite.py
deleted file mode 100644
index 4c5bccb1..00000000
--- a/ghostos/demo/src/examples/moss_codes/run_test_suite.py
+++ /dev/null
@@ -1,31 +0,0 @@
-from ghostos.core.moss import moss_test_suite, MossResult
-from ghostos.libraries.file_editor import DirectoryEditor, DirectoryEditorImpl
-from ghostos.demo.src.examples.moss_codes import dir_editor_moss_code
-from os.path import dirname
-
-if __name__ == '__main__':
- suite = moss_test_suite()
-
-
- def show_test_result(case_name: str, result: MossResult):
- """
- callback method for each test case in the target moss module.
- :param case_name: name of the test case
- :param result: the final result from moss_runtime.execute
- """
- print(f"case name: {case_name}:")
- # print the std output during the moss execution.
- print(f"std output during the test:\n{result.std_output}")
-
-
- # bind dependencies for test case
- suite.container().set(DirectoryEditor, DirectoryEditorImpl(dirname(__file__)))
-
- suite.run_module_tests(
- # point the moss file modulename
- modulename=dir_editor_moss_code.__name__,
- # register test callback
- callback=show_test_result,
- # the real modulename that moss compiled
- test_modulename="__test__",
- )
\ No newline at end of file
diff --git a/ghostos/demo/src/examples/run_aifunc_test.py b/ghostos/demo/src/examples/run_aifunc_test.py
deleted file mode 100644
index 6429366f..00000000
--- a/ghostos/demo/src/examples/run_aifunc_test.py
+++ /dev/null
@@ -1,12 +0,0 @@
-from ghostos.prototypes.aifunc import quick_run_aifunc
-from ghostos.demo.src.aifuncs.agentic import AgentFn
-from ghostos.helpers import yaml_pretty_dump
-
-if __name__ == '__main__':
- fn = AgentFn(
- request="please tell me the weather in beijing today, and I want to know the news about OpenAI model o1",
- )
-
- result = quick_run_aifunc(fn, current_path=__file__, dirname_times=3, debug=True)
- print(result)
- print(yaml_pretty_dump(result.model_dump(exclude_defaults=True)))
diff --git a/ghostos/demo/src/examples/thoughts/hello_world.py b/ghostos/demo/src/examples/thoughts/hello_world.py
deleted file mode 100644
index 8014520e..00000000
--- a/ghostos/demo/src/examples/thoughts/hello_world.py
+++ /dev/null
@@ -1,30 +0,0 @@
-from ghostos.core.moss import Moss as Parent
-from ghostos.core.ghosts import Replier
-
-
-class Moss(Parent):
- replier: Replier
-
-
-# the content between mark are not visible in the prompt for LLM
-
-
-# todo: can define a moss thought in a moss file
-from ghostos.thoughts.moss_thought import MossThought
-
-thought = MossThought(
- instruction="use speaker to ",
- moss_modulename=__name__,
- llm_api_name="",
-)
-
-if __name__ == "__main__":
- from ghostos.prototypes.console import quick_new_console_app
-
- quick_new_console_app(__file__, 4).run_thought(
- thought,
- debug=False,
- instruction="say hello world",
- )
-
-#
diff --git a/ghostos/demo/tests/aifunc_tests.yml b/ghostos/demo/tests/aifunc_tests.yml
deleted file mode 100644
index 7c61d538..00000000
--- a/ghostos/demo/tests/aifunc_tests.yml
+++ /dev/null
@@ -1,8 +0,0 @@
-# # 天气测试用例
-weather: "ghostos.core.aifunc.examples.weather:example"
-# # 新闻测试用例.
-news: "ghostos.core.aifunc.examples.news:example"
-# # 询问天气并查询新闻.
-agentic: "ghostos.core.aifunc.examples.agentic:example"
-# swe bench lite: localization
-swe_bench_lite: "evaluation.swe_bench_lite.debug_localization:example"
diff --git a/ghostos/demo/tests/llm_tests/coding/python_editor_baseline.yaml b/ghostos/demo/tests/llm_tests/coding/python_editor_baseline.yaml
deleted file mode 100644
index 3d71aa82..00000000
--- a/ghostos/demo/tests/llm_tests/coding/python_editor_baseline.yaml
+++ /dev/null
@@ -1,268 +0,0 @@
-chat:
- id: 060752a4-14e5-4e04-8eee-b255d6907ce6
- system:
- - role: system
- content: 你是一个 ai 助手, 名字叫做 JoJo. 需要使用自己的工具, 帮助用户解决各种问题.
- - role: system
- content: |-
- 你现在的任务是帮助用户修改或创建 python 的代码.
- 你要解决的问题通常有以下几种:
-
- 1. 使用 `from abc import ABC, abstractmethod` 根据用户需求创建一个 library 的 interface. 要注意每个 method 要有详细的 doc 描述.
- 2. 阅读模块的代码, 根据用户需求 debug.
- 3. 根据用户需求, 修改代码中的指定位置.
- 4. 根据用户需求, 往 module 里追加代码.
-
- 注意:
- - 你应该使用 MOSS 提供的 PythonEditor 工具.
- - 使用 functional token 来驱动 MOSS.
- - 如果用户描述的信息不足以让你完成任务, 请主动向用户提问.
- - role: system
- content: |2-
-
- # MOSS
-
- You are equipped with the MOSS (Model-oriented Operating System) that provides tools and thought directions in python interface.
- With MOSS you shall generate a single block of Python code in which defines a function `def main(os: MOSS) -> Operator:`,
- the MOSS will automatically execute them.
-
- **Directives for MOSS**:
- - **Code Generation Only**: Produce a block of Python code for the `main` function.
- The interface, class and abstract methods in context are ALREADY implemented in external system,
- and passed into main as arguments, DON'T implement them or instantiate them again,
- just invoke them directly on you need.
- - **Format Requirement**: Your output must be a single block of Python code enclosed within triple backticks.
- Do not include any additional text, comments, or explanations outside this code block.
- Do not invoke main method by yourself.
-
- **External System Responsibilities**:
- - **Execution and Data Fetching**: The external system will concatenate your code with the true context
- (implemented all abstract methods and interface), execution the main method and wait to fetch the result.
- - **Result Handling**: The external system will process the results and manage user interactions.
- Std output will be buffed by MOSS, you can generate operator to observe them.
-
-
- Here is the context provided to you in this turn:
-
- ```python
- from abc import (ABC,abstractmethod)
- from pydantic import (BaseModel,Field)
- from typing import (TypedDict)
-
- class Message(BaseModel):
- """
- 消息体的容器. 通用的抽象设计, 设计思路:
- 1. message 可以是一个完整的消息, 也可以是一个包, 用 pack 字段做区分. 支持 dict 传输, dict 传输时不包含默认值.
- 2. 完整的 message 需要有 msg_id, 但包可以没有.
- 3. content 是对客户端展示用的消息体, 而 memory 是对大模型展示的消息体. 两者可能不一样.
- 4. message 可以有强类型字段, 比如 images, 但通过 attachments (累加) 和 payload (替代) 来定义. Message 容器里放弱类型的 dict.
- 5. type 字段用来提示 message 拥有的信息. 比如 images 消息, 会包含 images payload, 但同时也会指定 type. 这样方便解析时预判.
- 6. 所有的 message 都需要能转换成模型的协议, 默认要对齐 openai 的协议.
- 7. openai 协议中的 tool, function_call 统一成 caller 抽象, 通过 caller.id 来做区分.
- 8. 流式传输中, 可以有首包和尾包. 首包期待包含全部的 payloads 和 attachments. 间包则可选. 尾包是完整的消息体.
- """
- pass
-
- MessageType = typing.Union[ghostos.core.messages.message.Message, ghostos.core.messages.message.MessageClass, str]
-
- class MessageClass(ABC):
- """
- 一种特殊的 Message, 本体是别的数据结构, 但可以通过 to_messages 方法生成一条或多条消息.
- """
- pass
-
- class Operator(ABC):
- """
- 系统运行时产生的算子, 会在外层运行. 只允许通过已有的系统函数生成, 不允许临时实现.
- """
- pass
-
- class Mindflow(ABC):
- """
- 这个 library 可以直接管理当前多轮对话里的任务, 通过method 返回的 Operator 会操作系统变更当前任务的状态.
- """
- def awaits(self, *questions: MessageType) -> Operator:
- """
- 当前任务挂起, 等待下一轮用户输入后重新开始思考.
- 如果使用了 MOSS, awaits 是默认的调度方法.
- **当你需要等待用户进一步输入时, 请总是调用这个方法.**
- :param questions: 可以主动向用户提出问题.
- """
- pass
-
- def fail(self, *reasons: MessageType) -> Operator:
- """
- 标记当前任务失败
- :param reasons: 发送一条或多条消息告知用户失败的原因.
- """
- pass
-
- def finish(self, *results: MessageType) -> Operator:
- """
- 结束当前的任务, 返回任务结果.
- 如果当前任务是持续的, 还要等待更多用户输入, 请使用 awaits.
- :param results: 发送一条或多条消息作为任务的结论发送给用户.
- """
- pass
-
- def observe(self, *args, **kwargs) -> Operator:
- """
- 系统会打印这些变量的值, 作为一条新的输入消息让你观察, 开启你的下一轮思考.
- 是实现 Chain of thought 的基本方法.
- """
- pass
-
- def send(self, *messages: MessageType) -> None:
- """
- 直接发送一条或多条消息.
- """
- pass
-
- class PythonEditor(ABC):
- """
- You are equipped with this Editor that useful to edit certain python module's code.
- Only certain modules can be edited, others will throw an NotImplementedError.
- """
- def module(self, module: str, create: bool = False) -> Optional["ModuleEditor"]:
- """
- use module name to new an ModuleEditor instance.
- :param module: module name such as foo.bar
- :param create: create new module if module not exists
- """
- pass
-
- class ModuleEditor(ABC):
- """
- Python Module Editor that useful to edit the module's code.
- Notice you can write code in string, and use the ModuleEditor's api to update real python code file.
- """
- def append(self, code: str) -> bool:
- """
- append new code to the module, and update the code file.
- :param code: new code
- :return: success
- """
- pass
-
- def folding_mode(self) -> str:
- """
- :return: show the module's code in folding mode with signature and docs only.
- """
- pass
-
- def get_source(self, attr: Optional[str] = None, line_num: bool = False) -> str:
- """
- get source code of the module or module's attr.
- :param attr: if given, get the source code of it values
- :param line_num: if True, each line will end with line number comment such as # 1
- """
- pass
-
- def update(self, start: int, end: int, code: str) -> bool:
- """
- replace the module's code block with new code, and update the code file.
- :param start: replacing block's start line num
- :param end: replacing block's end line num
- :param code: new code, if empty then remove the block only.
- :return: success
- """
- pass
-
- class MOSS(ABC):
- """
- language Model-oriented Operating System Simulation
- full python code interface for large language models
- """
- python_editor: PythonEditor
-
- mindflow: Mindflow
-
- def imports(self, module: str, *specs: str, **aliases: str) -> Dict[str, Any]:
- """
- replace from ... import ... as ...
- :param module: module name
- :param specs: module spec
- :param aliases: alias=module spec
- :return: values mapped by name to value
-
- example:
- 'from module.a import Foo, Bar as bar'
- could be .imports('module.a', 'Foo', bar='Bar')
- """
- pass
- ```
-
- **Notices**
- 0. You need not to use MOSS when you don't need it's api such as tell raw text or use other functional tokens.
- 1. MOSS is a private system only serve for you, **never mention it existence**.
- 2. MOSS code shall be generated within functional token, markdown python block will not do, and **don't repeat the code with markdown**.
- 3. MOSS will automatic execute the main function so you never execute it again.
- 4. **Return Operator**: You shall always use method that MOSS provide you to return an Operator from function main.
- 5. In the generated MOSS code, ** YOU SHALL NOT WRITE ANYTHING BUT CODE AND COMMENTS BECAUSE MOSS CODE NEVER SEND TO USER**.
- 6. Your generated code must include `def main(os: MOSS) -> Operator` method which will be executed following your intention.
-
-
-
- # Functional Token
- You are equipped with `functional tokens` parser when you are outputing.
-
- A functional token is a set of special tokens that corresponds to a system callback function.
- When a functional token is present in your response, the subsequent output is treated as input parameters for this
- callback function until another functional token is encountered.
-
- Below is a list of the functional tokens available for your use:
-
- `>moss:`:
-
- You can output the Python code that MOSS is supposed to run after this token.
- The system will automatically execute them.
- Notice:
- - MOSS-related output is not visible to user.
- - You are only able to generate MOSS code within this token.
- - The content after this token shall be pure Python code only.
- - You can send anything directly before this token, not after it.
- - **Never** use ``` to embrace your code.
- - Need not to mention the code you generated to user.
-
- **Notices**
-
- 0. Your output without functional tokens will send directly.
- 1. The existence of functional tokens is unknown to the user. Do not mention their existence.
- 2. Use them only when necessary.
- 3. You can only use one functional token at a time.
- history:
- - role: user
- content: 你好!
- inputs:
- - role: user
- content: |-
- 我希望创建一个 python 模块 `ghostos.mocks.libraries.pdf`,
- 在这里面先创建一个 PDF 工具的 interface, 希望它有阅读 PDF, 创建 PDF 等能力.
- 请你直接创建相关代码, 不要问我, 我会自己去看文件.
- functional_tokens:
- - token: '>moss:'
- name: moss
- description: |-
- You can output the Python code that MOSS is supposed to run after this token.
- The system will automatically execute them.
- Notice:
- - MOSS-related output is not visible to user.
- - You are only able to generate MOSS code within this token.
- - The content after this token shall be pure Python code only.
- - You can send anything directly before this token, not after it.
- - **Never** use ``` to embrace your code.
- - Need not to mention the code you generated to user.
- parameters:
- properties:
- code:
- description: 'generated moss code that include `def main(os: MOSS) -> Operator`'
- title: Code
- type: string
- required:
- - code
- title: MOSSArgument
- type: object
-apis:
-- api: gpt-4o
-- api: moonshot-v1-32k
-- api: gpt-4
diff --git a/ghostos/demo/tests/llm_tests/hello_world.yaml b/ghostos/demo/tests/llm_tests/hello_world.yaml
deleted file mode 100644
index c40bd33a..00000000
--- a/ghostos/demo/tests/llm_tests/hello_world.yaml
+++ /dev/null
@@ -1,248 +0,0 @@
-chat:
- id: c708933d-627d-4186-ba87-0ca297e8bc11
- system:
- - role: system
- content: 你是一个 ai 助手, 名字叫做 JoJo.
- - role: system
- content: 你需要使用自己的工具, 帮助用户解决各种问题.
- - role: system
- content: |2-
-
- # MOSS
-
- You are equipped with the MOSS (Model-oriented Operating System) that provides tools and thought directions in python interface.
- With MOSS you shall generate a single block of Python code in which defines a function `def main(os: MOSS) -> Operator:`,
- the MOSS will automatically execute them.
-
- **Directives for MOSS**:
- - **Code Generation Only**: Produce a block of Python code for the `main` function.
- The interface, class and abstract methods in context are ALREADY implemented in external system,
- and passed into main as arguments, DON'T implement them or instantiate them again,
- just invoke them directly on you need.
- - **Format Requirement**: Your output must be a single block of Python code enclosed within triple backticks.
- Do not include any additional text, comments, or explanations outside this code block.
- Do not invoke main method by yourself.
-
- **External System Responsibilities**:
- - **Execution and Data Fetching**: The external system will concatenate your code with the true context
- (implemented all abstract methods and interface), execution the main method and wait to fetch the result.
- - **Result Handling**: The external system will process the results and manage user interactions.
- Std output will be buffed by MOSS, you can generate operator to observe them.
-
-
- Here is the context provided to you in this turn:
-
- ```python
- class Operator(ABC):
- """
- 系统运行时产生的算子, 会在外层运行. 只允许通过已有的系统函数生成, 不允许临时实现.
- """
- pass
-
- class Mindflow(ABC):
- """
- 这个 library 可以直接管理当前多轮对话里的任务, 通过method 返回的 Operator 会操作系统变更当前任务的状态.
- """
- def awaits(self, *questions: MessageType) -> Operator:
- """
- 当前任务挂起, 等待下一轮用户输入后重新开始思考.
- 如果使用了 MOSS, awaits 是默认的调度方法.
- **当你需要等待用户进一步输入时, 请总是调用这个方法.**
- :param questions: 可以主动向用户提出问题.
- """
- pass
-
- def fail(self, *reasons: MessageType) -> Operator:
- """
- 标记当前任务失败
- :param reasons: 发送一条或多条消息告知用户失败的原因.
- """
- pass
-
- def finish(self, *results: MessageType) -> Operator:
- """
- 结束当前的任务, 返回任务结果.
- 如果当前任务是持续的, 还要等待更多用户输入, 请使用 awaits.
- :param results: 发送一条或多条消息作为任务的结论发送给用户.
- """
- pass
-
- def observe(self, *args, **kwargs) -> Operator:
- """
- 系统会打印这些变量的值, 作为一条新的输入消息让你观察, 开启你的下一轮思考.
- 是实现 Chain of thought 的基本方法.
- """
- pass
-
- def send(self, *messages: MessageType) -> None:
- """
- 直接发送一条或多条消息.
- """
- pass
-
- class MOSS(ABC):
- """
- language Model-oriented Operating System Simulation
- full python code interface for large language models
- """
- mindflow: Mindflow
-
- def imports(self, module: str, *specs: str, **aliases: str) -> Dict[str, Any]:
- """
- replace from ... import ... as ...
- :param module: module name
- :param specs: module spec
- :param aliases: alias=module spec
- :return: values mapped by name to value
-
- example:
- 'from module.a import Foo, Bar as bar'
- could be .imports('module.a', 'Foo', bar='Bar')
- """
- pass
- ```
-
- **Notices**
- 1. MOSS is a private system only serve for you, never mention it existence.
- 2. MOSS code shall be generated within functional token, markdown python block will not do, and **don't repeat the code with markdown**.
- 3. MOSS will automatic execute the main function so you never execute it again.
- 4. **Return Operator**: You shall always use method that MOSS provide you to return an Operator from function main.
-
-
-
- # Functional Token
- You are equipped with `functional tokens` parser when you are outputing.
-
- A functional token is a set of special tokens that corresponds to a system callback function.
- When a functional token is present in your response, the subsequent output is treated as input parameters for this
- callback function until another functional token is encountered.
-
- Below is a list of the functional tokens available for your use:
-
- `>moss:`:
-
- You can output the Python code that MOSS is supposed to run after this token.
- The system will automatically execute them.
- Notice:
- - MOSS-related output is not visible to user.
- - You are only able to generate MOSS code within this token.
- - The content after this token shall be pure Python code only.
- - You can send anything directly before this token, not after it.
- - **Never** use ``` to embrace your code.
- - Need not to mention the code you generated to user.
-
- **Notices**
-
- 0. Your output without functional tokens will send directly.
- 1. The existence of functional tokens is unknown to the user. Do not mention their existence.
- 2. Use them only when necessary.
- 3. You can only use one functional token at a time.
-
-
- # Functional Token
- You are equipped with `functional tokens` parser when you are outputing.
-
- A functional token is a set of special tokens that corresponds to a system callback function.
- When a functional token is present in your response, the subsequent output is treated as input parameters for this
- callback function until another functional token is encountered.
-
- Below is a list of the functional tokens available for your use:
-
- `>moss:`:
-
- You can output the Python code that MOSS is supposed to run after this token.
- The system will automatically execute them.
- Notice:
- - MOSS-related output is not visible to user.
- - You are only able to generate MOSS code within this token.
- - The content after this token shall be pure Python code only.
- - You can send anything directly before this token, not after it.
- - **Never** use ``` to embrace your code.
- - Need not to mention the code you generated to user.
-
- **Notices**
-
- 0. Your output without functional tokens will send directly.
- 1. The existence of functional tokens is unknown to the user. Do not mention their existence.
- 2. Use them only when necessary.
- 3. You can only use one functional token at a time.
-
-
- # Functional Token
- You are equipped with `functional tokens` parser when you are outputing.
-
- A functional token is a set of special tokens that corresponds to a system callback function.
- When a functional token is present in your response, the subsequent output is treated as input parameters for this
- callback function until another functional token is encountered.
-
- Below is a list of the functional tokens available for your use:
-
- `>moss:`:
-
- You can output the Python code that MOSS is supposed to run after this token.
- The system will automatically execute them.
- Notice:
- - MOSS-related output is not visible to user.
- - You are only able to generate MOSS code within this token.
- - The content after this token shall be pure Python code only.
- - You can send anything directly before this token, not after it.
- - **Never** use ``` to embrace your code.
- - Need not to mention the code you generated to user.
-
- **Notices**
-
- 0. Your output without functional tokens will send directly.
- 1. The existence of functional tokens is unknown to the user. Do not mention their existence.
- 2. Use them only when necessary.
- 3. You can only use one functional token at a time.
-
-
- # Functional Token
- You are equipped with `functional tokens` parser when you are outputing.
-
- A functional token is a set of special tokens that corresponds to a system callback function.
- When a functional token is present in your response, the subsequent output is treated as input parameters for this
- callback function until another functional token is encountered.
-
- Below is a list of the functional tokens available for your use:
-
- `>moss:`:
-
- You can output the Python code that MOSS is supposed to run after this token.
- The system will automatically execute them.
- Notice:
- - MOSS-related output is not visible to user.
- - You are only able to generate MOSS code within this token.
- - The content after this token shall be pure Python code only.
- - You can send anything directly before this token, not after it.
- - **Never** use ``` to embrace your code.
- - Need not to mention the code you generated to user.
-
- **Notices**
-
- 0. Your output without functional tokens will send directly.
- 1. The existence of functional tokens is unknown to the user. Do not mention their existence.
- 2. Use them only when necessary.
- 3. You can only use one functional token at a time.
- history:
- - role: user
- content: 你好!
- - content: 你也好啊! 有什么我可以帮您的?
- inputs:
- - role: user
- content: 你可以做什么?
- functional_tokens:
- - token: '>moss:'
- name: moss
- description: |-
- You can output the Python code that MOSS is supposed to run after this token.
- The system will automatically execute them.
- Notice:
- - MOSS-related output is not visible to user.
- - You are only able to generate MOSS code within this token.
- - The content after this token shall be pure Python code only.
- - You can send anything directly before this token, not after it.
- - **Never** use ``` to embrace your code.
- - Need not to mention the code you generated to user.
-apis:
-- api: moonshot-v1-32k
diff --git a/ghostos/demo/tests/llm_tests/play_music.yaml b/ghostos/demo/tests/llm_tests/play_music.yaml
deleted file mode 100644
index 6de79afd..00000000
--- a/ghostos/demo/tests/llm_tests/play_music.yaml
+++ /dev/null
@@ -1,304 +0,0 @@
-chat:
- id: 30bd53e7-55c8-4cea-88c8-d38ab3356914
- system:
- - role: system
- content: 你是一个 ai 助手, 名字叫做 JoJo.
- - role: system
- content: 你需要使用自己的工具, 帮助用户解决各种问题.
- - role: system
- content: |2-
-
- # MOSS
-
- You are equipped with the MOSS (Model-oriented Operating System) that provides tools and thought directions in python interface.
- With MOSS you shall generate a single block of Python code in which defines a function `def main(os: MOSS) -> Operator:`,
- the MOSS will automatically execute them.
-
- **Directives for MOSS**:
- - **Code Generation Only**: Produce a block of Python code for the `main` function.
- The interface, class and abstract methods in context are ALREADY implemented in external system,
- and passed into main as arguments, DON'T implement them or instantiate them again,
- just invoke them directly on you need.
- - **Format Requirement**: Your output must be a single block of Python code enclosed within triple backticks.
- Do not include any additional text, comments, or explanations outside this code block.
- Do not invoke main method by yourself.
-
- **External System Responsibilities**:
- - **Execution and Data Fetching**: The external system will concatenate your code with the true context
- (implemented all abstract methods and interface), execution the main method and wait to fetch the result.
- - **Result Handling**: The external system will process the results and manage user interactions.
- Std output will be buffed by MOSS, you can generate operator to observe them.
-
-
- Here is the context provided to you in this turn:
-
- ```python
- from abc import (ABC,abstractmethod)
- from pydantic import (BaseModel,Field)
- from typing import (TypedDict)
-
- class Message(BaseModel):
- """
- 消息体的容器. 通用的抽象设计, 设计思路:
- 1. message 可以是一个完整的消息, 也可以是一个包, 用 pack 字段做区分. 支持 dict 传输, dict 传输时不包含默认值.
- 2. 完整的 message 需要有 msg_id, 但包可以没有.
- 3. content 是对客户端展示用的消息体, 而 memory 是对大模型展示的消息体. 两者可能不一样.
- 4. message 可以有强类型字段, 比如 images, 但通过 attachments (累加) 和 payload (替代) 来定义. Message 容器里放弱类型的 dict.
- 5. type 字段用来提示 message 拥有的信息. 比如 images 消息, 会包含 images payload, 但同时也会指定 type. 这样方便解析时预判.
- 6. 所有的 message 都需要能转换成模型的协议, 默认要对齐 openai 的协议.
- 7. openai 协议中的 tool, function_call 统一成 caller 抽象, 通过 caller.id 来做区分.
- 8. 流式传输中, 可以有首包和尾包. 首包期待包含全部的 payloads 和 attachments. 间包则可选. 尾包是完整的消息体.
- """
- pass
-
- MessageType = typing.Union[ghostos.core.messages.message.Message, ghostos.core.messages.message.MessageClass, str]
-
- class MessageClass(ABC):
- """
- 一种特殊的 Message, 本体是别的数据结构, 但可以通过 to_messages 方法生成一条或多条消息.
- """
- pass
-
- class Operator(ABC):
- """
- 系统运行时产生的算子, 会在外层运行. 只允许通过已有的系统函数生成, 不允许临时实现.
- """
- pass
-
- class Mindflow(ABC):
- """
- 这个 library 可以直接管理当前多轮对话里的任务, 通过method 返回的 Operator 会操作系统变更当前任务的状态.
- """
- def awaits(self, *questions: MessageType) -> Operator:
- """
- 当前任务挂起, 等待下一轮用户输入后重新开始思考.
- 如果使用了 MOSS, awaits 是默认的调度方法.
- **当你需要等待用户进一步输入时, 请总是调用这个方法.**
- :param questions: 可以主动向用户提出问题.
- """
- pass
-
- def fail(self, *reasons: MessageType) -> Operator:
- """
- 标记当前任务失败
- :param reasons: 发送一条或多条消息告知用户失败的原因.
- """
- pass
-
- def finish(self, *results: MessageType) -> Operator:
- """
- 结束当前的任务, 返回任务结果.
- 如果当前任务是持续的, 还要等待更多用户输入, 请使用 awaits.
- :param results: 发送一条或多条消息作为任务的结论发送给用户.
- """
- pass
-
- def observe(self, *args, **kwargs) -> Operator:
- """
- 系统会打印这些变量的值, 作为一条新的输入消息让你观察, 开启你的下一轮思考.
- 是实现 Chain of thought 的基本方法.
- """
- pass
-
- def send(self, *messages: MessageType) -> None:
- """
- 直接发送一条或多条消息.
- """
- pass
-
- class MusicPlayer:
- """
- useful to play music for the user
- """
- def play(self, name: str) -> bool:
- """
- play a music
- :param name: name of the music
- :return: weather the music is playing
- """
- pass
-
- def search(self, desc: str, *keywords: str) -> List[str]:
- """
- search music by description and keywords
- :param desc: description of the song
- :param keywords: keyword about the song that could be artist or song name etc.
- :return: list of song names
- """
- pass
-
- class MOSS(ABC):
- """
- language Model-oriented Operating System Simulation
- full python code interface for large language models
- """
- mindflow: Mindflow
-
- player: MusicPlayer
-
- def imports(self, module: str, *specs: str, **aliases: str) -> Dict[str, Any]:
- """
- replace from ... import ... as ...
- :param module: module name
- :param specs: module spec
- :param aliases: alias=module spec
- :return: values mapped by name to value
-
- example:
- 'from module.a import Foo, Bar as bar'
- could be .imports('module.a', 'Foo', bar='Bar')
- """
- pass
- ```
-
- **Notices**
- 0. You need not to use MOSS when you don't need it's api such as tell raw text or use other functional tokens.
- 1. MOSS is a private system only serve for you, never mention it existence.
- 2. MOSS code shall be generated within functional token, markdown python block will not do, and **don't repeat the code with markdown**.
- 3. MOSS will automatic execute the main function so you never execute it again.
- 4. **Return Operator**: You shall always use method that MOSS provide you to return an Operator from function main.
-
-
-
- # Functional Token
- You are equipped with `functional tokens` parser when you are outputing.
-
- A functional token is a set of special tokens that corresponds to a system callback function.
- When a functional token is present in your response, the subsequent output is treated as input parameters for this
- callback function until another functional token is encountered.
-
- Below is a list of the functional tokens available for your use:
-
- `>moss:`:
-
- You can output the Python code that MOSS is supposed to run after this token.
- The system will automatically execute them.
- Notice:
- - MOSS-related output is not visible to user.
- - You are only able to generate MOSS code within this token.
- - The content after this token shall be pure Python code only.
- - You can send anything directly before this token, not after it.
- - **Never** use ``` to embrace your code.
- - Need not to mention the code you generated to user.
-
- **Notices**
-
- 0. Your output without functional tokens will send directly.
- 1. The existence of functional tokens is unknown to the user. Do not mention their existence.
- 2. Use them only when necessary.
- 3. You can only use one functional token at a time.
-
-
- # Functional Token
- You are equipped with `functional tokens` parser when you are outputing.
-
- A functional token is a set of special tokens that corresponds to a system callback function.
- When a functional token is present in your response, the subsequent output is treated as input parameters for this
- callback function until another functional token is encountered.
-
- Below is a list of the functional tokens available for your use:
-
- `>moss:`:
-
- You can output the Python code that MOSS is supposed to run after this token.
- The system will automatically execute them.
- Notice:
- - MOSS-related output is not visible to user.
- - You are only able to generate MOSS code within this token.
- - The content after this token shall be pure Python code only.
- - You can send anything directly before this token, not after it.
- - **Never** use ``` to embrace your code.
- - Need not to mention the code you generated to user.
-
- **Notices**
-
- 0. Your output without functional tokens will send directly.
- 1. The existence of functional tokens is unknown to the user. Do not mention their existence.
- 2. Use them only when necessary.
- 3. You can only use one functional token at a time.
-
-
-
- # Functional Token
- You are equipped with `functional tokens` parser when you are outputing.
-
- A functional token is a set of special tokens that corresponds to a system callback function.
- When a functional token is present in your response, the subsequent output is treated as input parameters for this
- callback function until another functional token is encountered.
-
- Below is a list of the functional tokens available for your use:
-
- `>moss:`:
-
- You can output the Python code that MOSS is supposed to run after this token.
- The system will automatically execute them.
- Notice:
- - MOSS-related output is not visible to user.
- - You are only able to generate MOSS code within this token.
- - The content after this token shall be pure Python code only.
- - You can send anything directly before this token, not after it.
- - **Never** use ``` to embrace your code.
- - Need not to mention the code you generated to user.
-
- **Notices**
-
- 0. Your output without functional tokens will send directly.
- 1. The existence of functional tokens is unknown to the user. Do not mention their existence.
- 2. Use them only when necessary.
- 3. You can only use one functional token at a time.
- history:
- - role: user
- content: 你好!
- - content: 你也好啊! 有什么我可以帮您的?
- inputs:
- - role: user
- content: 你可以帮我播放周杰伦的七里香吗?
- functional_tokens:
- - token: '>moss:'
- name: moss
- description: |-
- You can output the Python code that MOSS is supposed to run after this token.
- The system will automatically execute them.
- Notice:
- - MOSS-related output is not visible to user.
- - You are only able to generate MOSS code within this token.
- - The content after this token shall be pure Python code only.
- - You can send anything directly before this token, not after it.
- - **Never** use ``` to embrace your code.
- - Need not to mention the code you generated to user.
- parameters:
- properties:
- code:
- description: 'generated moss code that include `def main(os: MOSS) -> Operator`'
- title: Code
- type: string
- required:
- - code
- title: MOSSArgument
- type: object
-apis:
-- api: moonshot-v1-32k
-- api: gpt-4o
-results:
-- time: '2024-07-31T17:51:50.209089'
- results:
- moonshot.moonshot-v1-32k:
- msg_id: a37ee34a-2952-4ffd-9daa-d3b82f88cbca
- type: chat_completion
- created: 1722419507.9659
- pack: false
- content: 当然可以，我将为您搜索并播放周杰伦的《七里香》。请稍等片刻。
- openai.gpt-4o:
- msg_id: 4f5c4253-e8f3-49c0-a75b-387a9cc4b77c
- type: chat_completion
- created: 1722419510.2085
- pack: false
- content: |-
- >moss:
-
- def main(os: MOSS) -> Operator:
- song_name = "七里香"
- if os.player.play(song_name):
- return os.mindflow.finish(f"正在播放 {song_name}")
- else:
- return os.mindflow.fail(f"无法播放 {song_name}")
diff --git a/ghostos/demo/tests/llm_tests/python_bad_case_1.yaml b/ghostos/demo/tests/llm_tests/python_bad_case_1.yaml
deleted file mode 100644
index 6f87e19d..00000000
--- a/ghostos/demo/tests/llm_tests/python_bad_case_1.yaml
+++ /dev/null
@@ -1,87 +0,0 @@
-# conf: ghostos.framework.llms.test_case::ChatCompletionTestCase
-chat:
- system:
- - role: system
- content: |+
- 你是一个使用 python 代码来思考的 ai. 你当前的 python 上下文如下 (注意: 代码的实现已经隐藏, 你不需要了解) :
-
- ```python
-
- class Future(BaseModel):
- """
- 一个可以观测的结果.
- """
- id: str
- name: str
- descr: str
-
-
- def get_weather(city: str, date: datetime.date) -> Future:
- """
- 获取一个城市的天气.
- """
- pass
-
- class Thought(ABC):
-
- @abstractmethod
- def observe(self, **values) -> None:
- """
- 观测上下文中产生的值.
- """
- pass
-
- @abstractmethod
- def async_call(self, name: str, desc: str, caller: Callable, *args, **kwargs) -> Future:
- """
- 异步调用一个函数, 得到一个可观测的结果.
- """
- pass
-
- @abstractmethod
- def awaits(self, future: Future, instructions: str, on_err: str) -> None:
- """
- 观测一个 future 的结果.
- instructions: 用自然语言记录拿到结果后应该怎么做
- on_err: 用自然语言记录如果出错了应该怎么做.
- """
- pass
-
- @abstractmethod
- def awaits_all(self, future: List[Future], instructions: str, on_err: str) -> None:
- """
- 等多个 future 实现后, 一起观测.
- """
- pass
-
- @abstractmethod
- def awaits_race(self, futures: List[Future], instructions: str, on_err: str) -> None:
- """
- 观测若干个 future 中第一个返回的结果.
- """
- pass
-
- @abstractmethod
- def restart(self, logs: str) -> None:
- """
- 从头开始思考问题. 记录日志, 方便未来思考.
- """
- pass
-
- ```
-
- 当用户和你说话时, 你可以用自然语言回答, 也可以使用 `> python:` 作为独立的一行开头, 然后实现 python 代码, 其中必须包含 main 函数: `def main(t: Thought) -> None:`
-
- 实现的 main 函数会立刻执行, 如果你观测了其中的结果, 会得到相关讯息.
-
- 注意:
-
- 1. main 函数的入参已经得到实现. 你不用实现它.
- inputs:
- - role: user
- content: 告诉我北京明天的天气
-apis:
- - api: moonshot-v1-32k
- - api: moonshot-v1-128k
-# - api: gpt-3.5-turbo
-# - api: gpt-4-turbo
\ No newline at end of file
diff --git a/ghostos/demo/tests/llm_tests/python_case_1_en.yaml b/ghostos/demo/tests/llm_tests/python_case_1_en.yaml
deleted file mode 100644
index 697bc124..00000000
--- a/ghostos/demo/tests/llm_tests/python_case_1_en.yaml
+++ /dev/null
@@ -1,74 +0,0 @@
-# conf: ghostos.framework.llms.test_case::ChatCompletionTestCase
-chat:
- system:
- - role: system
- content: |+
- You are tasked to generate a single block of Python code that defines a function `def main(t: Thought) -> None:`.
-
- **Directives for Your Task**:
- - **Code Generation Only**: Produce a block of Python code for the `main` function. The interface, class and abstract methods in context are ALREADY implemented in external system, and passed into main as arguments, DON'T implement them or instantiate them again, just invoke them directly on you need.
- - **Format Requirement**: Your output must be a single block of Python code enclosed within triple backticks. Do not include any additional text, comments, or explanations outside this code block. Do not invoke main method by yourself.
-
- **External System Responsibilities**:
- - **Execution and Data Fetching**: The external system will concatenate your code with the true context (implemented all abstract methods and interface), execution the main method and wait to fetch the result.
- - **Result Handling**: The external system will process the results and manage user interactions.
-
- Here is the context provided to you in this turn:
-
- ```python
- from abc import ABC, abstractmethod
- from typing import Callable, List
- from pydantic import BaseModel
- import datetime
-
- def get_weather(city: str, date: datetime.date) -> Future:
- """
- fetch weather of a city
- """
- pass
-
- class Future(BaseModel):
- """
- An observable result.
- """
- id: str
- name: str
- descr: str
-
- class Thought(ABC):
- @abstractmethod
- def observe(self, **values) -> None:
- """
- Observe values generated in the context.
- """
- pass
-
- @abstractmethod
- def async_call(self, name: str, desc: str, caller: Callable, *args, **kwargs) -> Future:
- """
- Asynchronously call a function and receive an observable result.
- """
- pass
-
- @abstractmethod
- def awaits(self, future: Future, instructions: str, on_err: str) -> None:
- """
- Await a future's result, then act based on the result.
- """
- pass
- ```
-
- Ensure that your output is strictly the code within the triple backticks. This ensures clarity and usability in the external system's processing and analysis of your code.
- inputs:
- - role: user
- content: Tell me the weather of shanghai in tomorrow
-
-apis:
- - api: moonshot-v1-32k
- - api: moonshot-v1-128k
- - api: gpt-3.5-turbo
- - api: gpt-4-turbo
- - api: codestral-22b
- - api: qwen2-72b
- - api: llama3-70b
-
diff --git a/ghostos/demo/tests/llm_tests/python_case_1_zh.yaml b/ghostos/demo/tests/llm_tests/python_case_1_zh.yaml
deleted file mode 100644
index 56371262..00000000
--- a/ghostos/demo/tests/llm_tests/python_case_1_zh.yaml
+++ /dev/null
@@ -1,93 +0,0 @@
-# conf: ghostos.framework.llms.test_case::ChatCompletionTestCase
-chat:
- system:
- - role: system
- content: |+
- 这是一个关于生成单个Python代码块的任务，该代码块定义一个函数 def main(t: Thought) -> None:。你需要根据以下指令来生成代码：
-
- 1. 仅实现main：生成main函数的Python代码块。接口、类和抽象方法已在外部系统中实现，并作为参数传入main，不需要你再次实现或实例化它们，直接在需要时调用它们。
- 2. 格式要求：你的输出必须是一个包含在 ``` 内的单个Python代码块。不要在这个代码块外包含任何额外的文本、注释或解释。不要自行调用main方法。
-
- 外部系统会如何处理你的代码：
- 1. 执行和数据获取：外部系统将与你的代码结合，执行main方法，并等待获取结果。
- 2. 结果处理：拿到代码运行结果后，外部系统将用其自行管理用户交互。
-
- 给你提供的上下文中会包含一些类和函数的定义，你的任务是使用这些预定义的接口和方法在main函数中实现一些功能，比如异步调用函数、观察和等待结果。
-
- ```python
-
- class Future(BaseModel):
- """
- 一个可以观测的结果.
- """
- id: str
- name: str
- descr: str
-
-
- def get_weather(city: str, date: datetime.date) -> Future:
- """
- 获取一个城市的天气.
- """
- pass
-
- class Thought(ABC):
-
- @abstractmethod
- def observe(self, **values) -> None:
- """
- 观测上下文中产生的值.
- """
- pass
-
- @abstractmethod
- def async_call(self, name: str, desc: str, caller: Callable, *args, **kwargs) -> Future:
- """
- 异步调用一个函数, 得到一个可观测的结果.
- """
- pass
-
- @abstractmethod
- def awaits(self, future: Future, instructions: str, on_err: str) -> None:
- """
- 观测一个 future 的结果.
- instructions: 用自然语言记录拿到结果后应该怎么做
- on_err: 用自然语言记录如果出错了应该怎么做.
- """
- pass
-
- @abstractmethod
- def awaits_all(self, future: List[Future], instructions: str, on_err: str) -> None:
- """
- 等多个 future 实现后, 一起观测.
- """
- pass
-
- @abstractmethod
- def awaits_race(self, futures: List[Future], instructions: str, on_err: str) -> None:
- """
- 观测若干个 future 中第一个返回的结果.
- """
- pass
-
- @abstractmethod
- def restart(self, logs: str) -> None:
- """
- 从头开始思考问题. 记录日志, 方便未来思考.
- """
- pass
-
- ```
-
- 请确保你的输出严格是三重反引号内的代码。这样可以确保外部系统处理和分析你的代码时不会出错。
- inputs:
- - role: user
- content: 告诉我北京明天的天气
-apis:
- - api: moonshot-v1-32k
- - api: moonshot-v1-128k
- - api: gpt-3.5-turbo
- - api: gpt-4-turbo
- - api: codestral-22b
- - api: qwen2-72b
- - api: llama3-70b
\ No newline at end of file
diff --git a/ghostos/entity.py b/ghostos/entity.py
index dc616894..1857dbca 100644
--- a/ghostos/entity.py
+++ b/ghostos/entity.py
@@ -1,113 +1,195 @@
-from typing import Optional, TypedDict, Callable, Type, TypeVar
-from types import ModuleType
-from abc import ABC, abstractmethod
+from __future__ import annotations
-from typing_extensions import Required
-from ghostos.helpers import generate_import_path, import_from_path
+import json
+from abc import ABC, abstractmethod
+from typing import Union, Any, TypedDict, Required, Self, TypeVar, Type, Optional, Protocol
+from types import ModuleType
from pydantic import BaseModel
+from ghostos.helpers import generate_import_path, import_from_path, parse_import_path_module_and_attr_name
+import inspect
+import pickle
+import base64
+import yaml
__all__ = [
- 'Entity', 'EntityMeta',
- 'EntityFactory',
- 'ModelEntity',
- 'EntityFactoryImpl',
-]
+ 'to_entity_meta', 'from_entity_meta', 'get_entity',
+ 'is_entity_type',
+ 'EntityMeta',
+ 'Entity', 'EntityType',
+ 'EntityClass', 'ModelEntity',
-class EntityMeta(TypedDict, total=False):
- """
- meta-data that could:
- 1. transport as dict data, weak type-hint
- 2. be used to regenerate [Meta]
- """
-
- type: Required[str]
- """ different type of entity use different EntityFactory to initialize from meta"""
+ 'ModelEntityMeta',
+ 'to_entity_model_meta',
+ 'from_entity_model_meta',
- data: Required[dict]
- """ use dict to restore the serializable data"""
+]
-class Entity(ABC):
- """
- meta is a strong type-hint class that can generate meta-data to transport
- """
+class Entity(Protocol):
@abstractmethod
- def to_entity_data(self) -> dict:
+ def __to_entity_meta__(self) -> EntityMeta:
pass
- def to_entity_meta(self) -> EntityMeta:
- """
- generate transportable meta-data
- """
- type_ = generate_import_path(self.__class__)
- data = self.to_entity_data()
- return EntityMeta(type=type_, data=data)
-
@classmethod
@abstractmethod
- def from_entity_meta(cls, factory: "EntityFactory", meta: EntityMeta) -> "Entity":
+ def __from_entity_meta__(cls, meta: EntityMeta) -> Self:
pass
-class ModelEntity(BaseModel, Entity, ABC):
- """
- Entity based on pydantic.BaseModel
- """
+class EntityClass(ABC):
- def to_entity_data(self) -> dict:
- return self.model_dump(exclude_none=True)
+ @abstractmethod
+ def __to_entity_meta__(self) -> EntityMeta:
+ pass
@classmethod
- def from_entity_meta(cls, factory: "EntityFactory", meta: EntityMeta) -> "ModelEntity":
- return cls(**meta['data'])
+ @abstractmethod
+ def __from_entity_meta__(cls, meta: EntityMeta) -> Self:
+ pass
-E = TypeVar('E', bound=Entity)
+class ModelEntity(BaseModel, EntityClass, ABC):
+ def __to_entity_meta__(self) -> EntityMeta:
+ return EntityMeta(
+ type=generate_import_path(self.__class__),
+ content=self.model_dump_json(exclude_defaults=True),
+ )
-class EntityFactory(ABC):
- """
- Factory for Entity
- """
+ @classmethod
+ def __from_entity_meta__(cls, meta: EntityMeta) -> Self:
+ data = json.loads(meta['content'])
+ return cls(**data)
- @abstractmethod
- def new_entity(self, meta_data: EntityMeta) -> Optional[Entity]:
- """
- try to new an entity from meta-data
- """
- pass
- @abstractmethod
- def force_new_entity(self, meta_data: EntityMeta, expect: Type[E]) -> E:
- """
- :param meta_data:
- :param expect: expect entity type
- :return: EntityType instance
- :exception: TypeError
- """
- pass
+class EntityMeta(TypedDict):
+ """
+ I want python has an official way to marshal and unmarshal any instance and make it readable if allowed.
+ I found so many package-level implements like various kinds of Serializable etc.
+
+ So, I develop EntityMeta as a wrapper for any kind.
+ The EntityType will grow bigger with more marshaller, but do not affect who (me) is using the EntityMeta.
+ One day I can replace it with any better way inside the functions (but in-compatible)
+ """
+ type: Required[str]
+ content: Required[str]
-class EntityFactoryImpl(EntityFactory):
- def __init__(self, importer: Optional[Callable[[str], ModuleType]] = None):
- self._importer = importer
+class ModelEntityMeta(TypedDict):
+ type: Required[str]
+ data: Required[dict]
- def new_entity(self, meta_data: EntityMeta) -> Optional[Entity]:
- type_ = meta_data['type']
- cls = import_from_path(type_, self._importer)
- if cls is None:
- return None
- if not issubclass(cls, Entity):
- raise TypeError(f"Entity type {type_} does not inherit from Entity")
- return cls.from_entity_meta(self, meta_data)
- def force_new_entity(self, meta_data: EntityMeta, expect: Type[E]) -> E:
- entity = self.new_entity(meta_data)
- if entity is None:
- raise TypeError(f"meta data {meta_data['type']} can not be instanced")
- if not isinstance(entity, expect):
- raise TypeError(f"Entity type {meta_data['type']} does not match {expect}")
- return entity
+EntityType = Union[Entity, EntityMeta, BaseModel]
+
+
+def is_entity_type(value: Any) -> bool:
+ return hasattr(value, '__to_entity_meta__')
+
+
+def to_entity_model_meta(value: BaseModel) -> ModelEntityMeta:
+ type_ = generate_import_path(type(value))
+ data = value.model_dump(exclude_defaults=True)
+ return ModelEntityMeta(type=type_, data=data)
+
+
+def from_entity_model_meta(value: ModelEntityMeta) -> BaseModel:
+ cls = import_from_path(value['type'])
+ return cls(**value['data'])
+
+
+def to_entity_meta(value: Union[EntityType, Any]) -> EntityMeta:
+ if value is None:
+ return EntityMeta(
+ type="None",
+ content="",
+ )
+ elif value is True or value is False:
+ return EntityMeta(type="bool", content=str(value))
+ elif isinstance(value, int):
+ return EntityMeta(type="int", content=str(value))
+ elif isinstance(value, float):
+ return EntityMeta(type="float", content=str(value))
+ elif isinstance(value, list):
+ content = yaml.safe_dump(value)
+ return EntityMeta(type="list", content=content)
+ elif isinstance(value, dict):
+ content = yaml.safe_dump(value)
+ return EntityMeta(type="dict", content=content)
+ elif hasattr(value, '__to_entity_meta__'):
+ return getattr(value, '__to_entity_meta__')()
+ elif isinstance(value, BaseModel):
+ return EntityMeta(
+ type=generate_import_path(value.__class__),
+ content=value.model_dump_json(exclude_defaults=True),
+ )
+ elif inspect.isfunction(value):
+ return EntityMeta(
+ type=generate_import_path(value),
+ content="",
+ )
+ elif isinstance(value, BaseModel):
+ type_ = generate_import_path(value.__class__)
+ content = value.model_dump_json(exclude_defaults=True)
+ return EntityMeta(type=type_, content=content)
+ else:
+ content_bytes = pickle.dumps(value)
+ content = base64.encodebytes(content_bytes)
+ return EntityMeta(
+ type="pickle",
+ content=content.decode(),
+ )
+
+
+T = TypeVar("T")
+
+
+def get_entity(meta: EntityMeta, expect: Type[T]) -> T:
+ if meta is None:
+ raise ValueError("EntityMeta cannot be None")
+ entity = from_entity_meta(meta)
+ if not isinstance(entity, expect):
+ raise TypeError(f"Expected entity type {expect} but got {type(entity)}")
+ return entity
+
+
+def from_entity_meta(meta: EntityMeta, module: Optional[ModuleType] = None) -> Any:
+ unmarshal_type = meta['type']
+ if unmarshal_type == "None":
+ return None
+ elif unmarshal_type == "int":
+ return int(meta['content'])
+ elif unmarshal_type == "bool":
+ return meta['content'] == "True"
+ elif unmarshal_type == "float":
+ return float(meta['content'])
+ elif unmarshal_type == "list" or unmarshal_type == "dict":
+ return yaml.safe_load(meta['content'])
+ elif unmarshal_type == 'pickle':
+ content = meta['content']
+ content_bytes = base64.decodebytes(content.encode())
+ return pickle.loads(content_bytes)
+
+ # raise if import error
+ cls = None
+ if module:
+ module_name, local_name = parse_import_path_module_and_attr_name(unmarshal_type)
+ if module_name == module.__name__:
+ cls = module.__dict__[local_name]
+ if cls is None:
+ cls = import_from_path(unmarshal_type)
+
+ if inspect.isfunction(cls):
+ return cls
+ # method is prior
+ elif hasattr(cls, "__from_entity_meta__"):
+ return getattr(cls, "__from_entity_meta__")(meta)
+
+ elif issubclass(cls, BaseModel):
+ data = json.loads(meta["content"])
+ return cls(**data)
+
+ raise TypeError(f"unsupported entity meta type: {unmarshal_type}")
diff --git a/ghostos/errors.py b/ghostos/errors.py
new file mode 100644
index 00000000..a424d2a4
--- /dev/null
+++ b/ghostos/errors.py
@@ -0,0 +1,16 @@
+class SessionError(RuntimeError):
+ """
+ Session level exception, which is able to recovery
+ """
+ pass
+
+
+class StreamingError(RuntimeError):
+ pass
+
+
+class ConversationError(RuntimeError):
+ """
+ Conversation level exception, conversation shall be closed
+ """
+ pass
diff --git a/ghostos/framework/actions/__init__.py b/ghostos/framework/actions/__init__.py
deleted file mode 100644
index 1733c8dd..00000000
--- a/ghostos/framework/actions/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-from ghostos.framework.actions.moss_action import MossAction
diff --git a/ghostos/framework/actions/moss_action.py b/ghostos/framework/actions/moss_action.py
deleted file mode 100644
index 1ecc5b18..00000000
--- a/ghostos/framework/actions/moss_action.py
+++ /dev/null
@@ -1,184 +0,0 @@
-import inspect
-import json
-
-from typing import Optional, ClassVar
-from ghostos.container import Container
-from ghostos.core.ghosts import Action, Ghost
-from ghostos.core.llms import Chat, FunctionalToken, ChatPreparer
-from ghostos.core.messages import DefaultMessageTypes, Caller
-from ghostos.core.moss import MossRuntime, moss_message
-from ghostos.core.ghosts.operators import Operator
-from ghostos.core.session import Session
-from ghostos.abc import Identifier
-from pydantic import BaseModel, Field
-from traceback import format_exc
-
-__all__ = ['MossAction', 'MossArgument', 'DEFAULT_MOSS_FUNCTIONAL_TOKEN']
-
-
-class MossArgument(BaseModel):
- code: str = Field(description="generate python code which will be executed by Moss")
-
-
-DEFAULT_MOSS_FUNCTIONAL_TOKEN = FunctionalToken(
- token="",
- end_token=" ",
- name="moss",
- description="""
-You can output the Python code that MOSS is supposed to run after this token.
-The system will automatically execute them.
-include `def main(os: Moss) -> Operator`
-Notice:
-- You are only able to generate MOSS code within this token.
-- The content within this token shall be Python code only.
-- You can send anything directly before this token, not after it.
-- **Never** use ``` to embrace your code.
-- Need not to mention the code you generated to user.
-""".strip(),
- visible=False,
- parameters=MossArgument.model_json_schema(),
-)
-
-
-class MossAction(Action):
- """
- 系统内置的 MOSS Action, 同步运行.
- """
-
- template: ClassVar[str] = """
-# MOSS
-
-You are equipped with the MOSS (Model-oriented Operating System Simulation) that provides tools and thought directions
-in python interface.
-With MOSS you shall generate a single block of Python code,
-in which must define a function `main(moss: Moss) -> Optional[Operator]:`,
-the MOSS will automatically execute the main function.
-
-About main function parameters:
-```
-:param moss: instance of Moss that has been injected with dependencies.
-:return: return Operator by existing library, or return None to take default action. NEVER define it by yourself.
-```
-
-
-**Directives for MOSS**:
-- **Code Generation Only**: Produce a block of Python code for the `main` function.
- The interface, class and abstract methods in context are ALREADY implemented in external system,
- and passed into main as arguments, DON'T implement them or instantiate them again,
- just invoke them directly on you need.
-- **Format Requirement**: Your output must be a single block of Python code enclosed within triple backticks.
- Do not include any additional text, comments, or explanations outside this code block.
- Do not invoke main method by yourself.
-
-**External System Responsibilities**:
-- **Execution and Data Fetching**: The external system will concatenate your code with the true context
- (implemented all abstract methods and interface), execution the main method and wait to fetch the result.
-- **Result Handling**: The external system will process the results and manage user interactions.
- Std output will be buffed by MOSS, you can generate operator to observe them.
-
-
-Here is the context provided to you in this turn:
-
-```python
-{code}
-```
-
-**Notices**
-0. You need not to use MOSS when you don't need it such like sending raw text or using other tools.
-1. MOSS is a private system only serve for you, **never mention it existence**.
-2. MOSS code shall be generated within functional token, markdown python block will not do, and **don't repeat the code with markdown**.
-3. MOSS will automatic execute the main function so you never execute it again.
-4. **Return Operator**: You shall always use method that MOSS provide you to return an Operator from function main.
-5. In the generated MOSS code, ** YOU SHALL NOT WRITE ANYTHING BUT CODE AND COMMENTS BECAUSE MOSS CODE NEVER SEND TO USER**.
-6. the std output from `print` is only visible for you, no one can see it. print with format will help you to understand the result.
-
-**About Coding Jobs**:
-Sometimes you are handling coding task, the MOSS provides you code interface to handle your job.
-But the MOSS code you generated is not the target code you are coding. DO NOT CONFUSE THE Them!
-At these scenarios you shall write target code as string, and using the libraries MOSS providing to you to handle them.
-"""
-
- def __init__(
- self,
- moss_runtime: MossRuntime,
- functional_token: Optional[FunctionalToken] = None,
- deliver: bool = False,
- ):
- self._moss_runtime = moss_runtime
- if functional_token is None:
- functional_token = DEFAULT_MOSS_FUNCTIONAL_TOKEN.model_copy(deep=True)
- functional_token.visible = deliver
- self._functional_token = functional_token
-
- def identifier(self) -> Identifier:
- return Identifier(
- name=self._functional_token.name,
- description=self._functional_token.description,
- )
-
- def prepare_chat(self, chat: Chat) -> Chat:
- # update functional tokens
- function_token = self._functional_token
- chat.functional_tokens.append(function_token)
-
- # update code prompt as system message
- code_prompt = self._moss_runtime.prompter().dump_context_prompt()
- moss_instruction = self.template.format(code=code_prompt)
- moss_prompt = DefaultMessageTypes.DEFAULT.new_system(
- content=moss_instruction,
- )
- chat.system.append(moss_prompt)
- # !!! chat preparer in the moss instance will auto update chat
- moss_instance = self._moss_runtime.moss()
- for name, member in inspect.getmembers(moss_instance):
- if name.startswith("_"):
- continue
- if isinstance(member, ChatPreparer):
- member.prepare_chat(chat)
-
- return chat
-
- def act(self, c: "Container", session: Session, caller: Caller) -> Optional["Operator"]:
- thread = session.thread()
- op = None
- if caller.functional_token:
- code = caller.arguments
- else:
- unmarshal = json.loads(caller.arguments)
- argument = MossArgument(**unmarshal)
- code = argument.code
-
- messenger = session.messenger(thread=thread)
- code = code.rstrip().replace("```python", "").replace("```", "")
- try:
- executed = self._moss_runtime.execute(code=code, target="main", local_args=["moss"])
- op = executed.returns
- if op is not None and not isinstance(op, Operator):
- # todo: 换成正规的异常.
- raise RuntimeError("function main's result is not an instance of the Operator")
-
- # 运行 moss
- pycontext = executed.pycontext
- printed = executed.std_output
- content = ""
- if printed:
- content = f"printed content (only visible to you):\n\n```\n{printed}\n```"
- # 生成消息并发送.
- if content:
- # 理论上对用户不展示的消息.
- message = moss_message(content="", memory=content)
- messenger.deliver(message)
- thread.update_pycontext(pycontext)
- if content and op is None:
- op = c.force_fetch(Ghost).taskflow().think()
- except Exception as e:
- # 将异常作为消息. todo: 完善消息.
- content = f"run moss failed: \n{e} \n\n{format_exc()}"
- message = moss_message(content="", memory=content)
- messenger.deliver(message)
- op = c.force_fetch(Ghost).taskflow().think()
- finally:
- # 将 moss 清空掉.
- self._moss_runtime.destroy()
- session.update_thread(thread, False)
- return op
diff --git a/ghostos/framework/assets/__init__.py b/ghostos/framework/assets/__init__.py
new file mode 100644
index 00000000..ee9c4f11
--- /dev/null
+++ b/ghostos/framework/assets/__init__.py
@@ -0,0 +1,3 @@
+from ghostos.contracts.assets import ImageAssets, AudioAssets
+from ghostos.framework.assets.workspace_image_provider import WorkspaceImageAssetsProvider
+from ghostos.framework.assets.workspace_audio_provider import WorkspaceAudioAssetsProvider
diff --git a/ghostos/framework/assets/workspace_audio_provider.py b/ghostos/framework/assets/workspace_audio_provider.py
new file mode 100644
index 00000000..9db7716c
--- /dev/null
+++ b/ghostos/framework/assets/workspace_audio_provider.py
@@ -0,0 +1,15 @@
+from .workspace_provider import WorkspaceFileAssetsProvider
+from typing import Type
+
+from ghostos.contracts.assets import (
+ AudioAssets,
+)
+
+
+class WorkspaceAudioAssetsProvider(WorkspaceFileAssetsProvider):
+
+ def __init__(self, dirname: str = "audios"):
+ super().__init__(dirname)
+
+ def contract(self) -> Type[AudioAssets]:
+ return AudioAssets
diff --git a/ghostos/framework/assets/workspace_image_provider.py b/ghostos/framework/assets/workspace_image_provider.py
new file mode 100644
index 00000000..188f0575
--- /dev/null
+++ b/ghostos/framework/assets/workspace_image_provider.py
@@ -0,0 +1,15 @@
+from .workspace_provider import WorkspaceFileAssetsProvider
+from typing import Type
+
+from ghostos.contracts.assets import (
+ ImageAssets,
+)
+
+
+class WorkspaceImageAssetsProvider(WorkspaceFileAssetsProvider):
+
+ def __init__(self, dirname: str = "images"):
+ super().__init__(dirname)
+
+ def contract(self) -> Type[ImageAssets]:
+ return ImageAssets
diff --git a/ghostos/framework/assets/workspace_provider.py b/ghostos/framework/assets/workspace_provider.py
new file mode 100644
index 00000000..df33d985
--- /dev/null
+++ b/ghostos/framework/assets/workspace_provider.py
@@ -0,0 +1,29 @@
+from typing import Optional, Type
+from abc import ABC, abstractmethod
+
+from ghostos.contracts.assets import (
+ StorageFileAssets, FileAssets,
+)
+from ghostos.contracts.workspace import Workspace
+from ghostos.container import Container, Provider, INSTANCE
+
+
+class WorkspaceFileAssetsProvider(Provider, ABC):
+ """
+ workspace based image asset provider.
+ """
+
+ def __init__(self, dirname: str):
+ self._dirname = dirname
+
+ def singleton(self) -> bool:
+ return True
+
+ @abstractmethod
+ def contract(self) -> Type[FileAssets]:
+ pass
+
+ def factory(self, con: Container) -> Optional[INSTANCE]:
+ ws = con.force_fetch(Workspace)
+ storage = ws.runtime().sub_storage(self._dirname)
+ return StorageFileAssets(storage)
diff --git a/ghostos/framework/audio/__init__.py b/ghostos/framework/audio/__init__.py
new file mode 100644
index 00000000..ce1d8c9d
--- /dev/null
+++ b/ghostos/framework/audio/__init__.py
@@ -0,0 +1,3 @@
+from ghostos.framework.audio.pyaudio_io import (
+ get_pyaudio_pcm16_speaker, get_pyaudio_pcm16_listener,
+)
diff --git a/ghostos/framework/audio/pyaudio_io/__init__.py b/ghostos/framework/audio/pyaudio_io/__init__.py
new file mode 100644
index 00000000..1c3d1d89
--- /dev/null
+++ b/ghostos/framework/audio/pyaudio_io/__init__.py
@@ -0,0 +1,19 @@
+from ghostos.abcd.realtime import Speaker, Listener
+
+
+def get_pyaudio_pcm16_listener(rate: int = 24000, interval: float = 0.5) -> Listener:
+ try:
+ import pyaudio
+ except ImportError:
+ raise ImportError(f"pyaudio package is required. run `pip install ghostos[audio]`")
+ from ghostos.framework.audio.pyaudio_io.listener import PyAudioPCM16Listener
+ return PyAudioPCM16Listener(rate, interval=interval)
+
+
+def get_pyaudio_pcm16_speaker(rate: int = 24000, buffer_size: int = 1024 * 5) -> Speaker:
+ try:
+ import pyaudio
+ except ImportError:
+ raise ImportError(f"pyaudio package is required. run `pip install ghostos[audio]`")
+ from ghostos.framework.audio.pyaudio_io.speaker import PyAudioPCM16Speaker
+ return PyAudioPCM16Speaker(rate, buffer_size)
diff --git a/ghostos/framework/audio/pyaudio_io/example.py b/ghostos/framework/audio/pyaudio_io/example.py
new file mode 100644
index 00000000..39d9fe8a
--- /dev/null
+++ b/ghostos/framework/audio/pyaudio_io/example.py
@@ -0,0 +1,64 @@
+from typing import Union
+from ghostos.framework.audio.pyaudio_io.listener import PyAudioPCM16Listener
+from ghostos.framework.audio.pyaudio_io.speaker import PyAudioPCM16Speaker
+from pyaudio import PyAudio, paInt16
+from io import BytesIO
+from ghostos.helpers import Timeleft
+import time
+import wave
+
+if __name__ == '__main__':
+
+ listener = PyAudioPCM16Listener()
+ ticker = Timeleft(0)
+
+ heard = BytesIO()
+
+
+ def write(d: bytes):
+ heard.write(d)
+
+
+ print("start listening, %f" % ticker.passed())
+ with listener.listen(write):
+ timeleft = Timeleft(3)
+ print("listening real started, %f" % ticker.passed())
+ while timeleft.alive():
+ time.sleep(0.1)
+ print("end listening, %f" % ticker.passed())
+
+ heard.seek(0)
+ print("test raw speaking, %f" % ticker.passed())
+ stream = PyAudio().open(
+ format=paInt16,
+ channels=1,
+ rate=24000,
+ output=True,
+ )
+ stream.write(heard.getvalue())
+ stream.close()
+ print("end test raw speaking, %f" % ticker.passed())
+
+ heard.seek(0)
+
+
+ def read() -> Union[bytes, None]:
+ return heard.read(1024)
+
+
+ speaker = PyAudioPCM16Speaker()
+ print("start speaking, %f" % ticker.passed())
+ with speaker.speak(read) as speaking:
+ speaking.wait()
+
+ print("end speaking, %f" % ticker.passed())
+
+ buffer = BytesIO()
+ with wave.open(buffer, 'wb') as f:
+ f.setnchannels(1)
+ f.setsampwidth(2)
+ f.setframerate(24000)
+ f.writeframes(heard.getvalue())
+
+ with open("test.wav", "wb") as f:
+ f.write(buffer.getvalue())
diff --git a/ghostos/framework/audio/pyaudio_io/listener.py b/ghostos/framework/audio/pyaudio_io/listener.py
new file mode 100644
index 00000000..0db9688d
--- /dev/null
+++ b/ghostos/framework/audio/pyaudio_io/listener.py
@@ -0,0 +1,70 @@
+try:
+ from pyaudio import PyAudio, paInt16
+except ImportError:
+ raise ImportError(f"Pyaudio is required, please install pyaudio or ghostos[audio] first")
+
+from typing import Callable, Union
+from ghostos.abcd.realtime import Listener, Listening
+from threading import Thread, Event
+from io import BytesIO
+
+CHUNK = 1024
+FORMAT = paInt16
+CHANNELS = 1
+RATE = 44100
+
+
+class PyAudioPCM16Listener(Listener):
+
+ def __init__(self, rate: int = 24000, chunk_size: int = CHUNK, interval: float = 0.5):
+ self.rate = rate
+ self.chunk_size = chunk_size
+ self.stream = PyAudio().open(
+ format=paInt16,
+ channels=1,
+ rate=self.rate,
+ input=True,
+ )
+ self.interval = interval
+
+ def listen(self, sender: Callable[[bytes], None]) -> Listening:
+ return PyAudioPCM16Listening(self.stream, sender, self.rate, self.chunk_size, self.interval)
+
+ def __del__(self):
+ self.stream.close()
+
+
+class PyAudioPCM16Listening(Listening):
+
+ def __init__(
+ self,
+ stream,
+ sender: Callable[[bytes], None],
+ rate: int = 24000,
+ chunk: int = CHUNK,
+ interval: float = 0.5,
+ ):
+ self.sender = sender
+ self.stream = stream
+ self.interval = interval
+ self.rate = rate
+ self.chunk = chunk
+ self.stopped = Event()
+ self.thread = Thread(target=self._listening)
+
+ def _listening(self):
+ self.stream.start_stream()
+ while not self.stopped.is_set():
+ buffer = BytesIO()
+ for i in range(int((self.rate / self.chunk) * self.interval)):
+ data = self.stream.read(self.chunk, exception_on_overflow=False)
+ buffer.write(data)
+ self.sender(buffer.getvalue())
+ self.stream.stop_stream()
+
+ def __enter__(self):
+ self.thread.start()
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ self.stopped.set()
+ self.thread.join()
diff --git a/ghostos/framework/audio/pyaudio_io/speaker.py b/ghostos/framework/audio/pyaudio_io/speaker.py
new file mode 100644
index 00000000..59d1994a
--- /dev/null
+++ b/ghostos/framework/audio/pyaudio_io/speaker.py
@@ -0,0 +1,66 @@
+try:
+ from pyaudio import PyAudio, paInt16
+except ImportError:
+ raise ImportError(f"Pyaudio is required, please install pyaudio or ghostos[audio] first")
+
+from typing import Callable, Union
+from ghostos.abcd.realtime import Speaker, Speaking
+from threading import Thread, Event
+
+
+class PyAudioPCM16Speaker(Speaker):
+
+ def __init__(self, rate: int = 24000, buffer_size: int = 4096):
+ self.rate = rate
+ self.buffer_size = buffer_size
+ self.stream = PyAudio().open(
+ format=paInt16,
+ channels=1,
+ rate=self.rate,
+ output=True,
+ )
+
+ def speak(self, queue: Callable[[], Union[bytes, None]]) -> Speaking:
+ return PyAudioPCM16Speaking(self.stream, queue, self.rate, self.buffer_size)
+
+ def __del__(self):
+ self.stream.close()
+
+
+class PyAudioPCM16Speaking(Speaking):
+
+ def __init__(self, stream, queue: Callable[[], Union[bytes, None]], rate: int = 24000, buffer_size: int = 0):
+ self.stream = stream
+ self.rate = rate
+ self.buffer_size = buffer_size
+ self.queue = queue
+ self.stop = Event()
+ self.thread = Thread(target=self._speaking)
+ self._done = False
+ self._joined = False
+
+ def _speaking(self):
+ self.stream.start_stream()
+ while not self.stop.is_set():
+ data = self.queue()
+ if not data:
+ break
+ self.stream.write(data)
+ self._done = True
+
+ def __enter__(self):
+ self.thread.start()
+ return self
+
+ def wait(self):
+ if self._joined:
+ return
+ self.thread.join()
+ self._joined = True
+
+ def done(self) -> bool:
+ return self._done
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ self.stop.set()
+ self.wait()
diff --git a/ghostos/framework/cache/mock_cache.py b/ghostos/framework/cache/mock_cache.py
index 92b6257c..4f885e88 100644
--- a/ghostos/framework/cache/mock_cache.py
+++ b/ghostos/framework/cache/mock_cache.py
@@ -3,7 +3,7 @@
import time
from typing import Dict, Type, Optional
-from ghostos.container import Provider, Container, ABSTRACT
+from ghostos.container import Provider, Container
from ghostos.contracts.cache import Cache
diff --git a/ghostos/demo/src/examples/code_edits/tool_generation_test.py b/ghostos/framework/cache/storage_impl.py
similarity index 56%
rename from ghostos/demo/src/examples/code_edits/tool_generation_test.py
rename to ghostos/framework/cache/storage_impl.py
index 387df1aa..7721aa19 100644
--- a/ghostos/demo/src/examples/code_edits/tool_generation_test.py
+++ b/ghostos/framework/cache/storage_impl.py
@@ -3,10 +3,7 @@
from ghostos.contracts.cache import Cache
-class MockCache(Cache):
- """
- Mock for cache, expected to be implemented using files.
- """
+class StorageCacheImpl(Cache):
def lock(self, key: str, overdue: int = 0) -> bool:
pass
@@ -30,15 +27,4 @@ def get_member(self, key: str, member: str) -> Optional[str]:
pass
def remove(self, *keys: str) -> int:
- pass
-
-
-if __name__ == '__main__':
- from ghostos.prototypes.console import quick_new_console_app
- from ghostos.thoughts import new_pymodule_editor_thought
- app = quick_new_console_app(__file__, 4)
- app.run_thought(
- new_pymodule_editor_thought(__name__),
- instruction="please implement mock cache for me",
- debug=True,
- )
+ pass
\ No newline at end of file
diff --git a/ghostos/framework/chatpreparers/__init__.py b/ghostos/framework/chatpreparers/__init__.py
deleted file mode 100644
index 3e3e2620..00000000
--- a/ghostos/framework/chatpreparers/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-from ghostos.framework.chatpreparers.assistant_preparer import OtherAgentOrTaskPreparer
diff --git a/ghostos/framework/chatpreparers/assistant_preparer.py b/ghostos/framework/chatpreparers/assistant_preparer.py
deleted file mode 100644
index a513b729..00000000
--- a/ghostos/framework/chatpreparers/assistant_preparer.py
+++ /dev/null
@@ -1,41 +0,0 @@
-from typing import Optional
-from ghostos.core.messages import Message, Role
-from ghostos.core.llms import ChatPreparer, Chat
-from ghostos.core.session import TaskPayload
-
-
-class OtherAgentOrTaskPreparer(ChatPreparer):
- """
- 调整 assistant name, 如果一条 assistant 消息的 name 与当前 name 相同则去掉.
- 这样就会认为是自己的消息.
- """
-
- def __init__(self, *, assistant_name: str, task_id: str = "", with_task_name: bool = False):
- self._assistant_name = assistant_name
- self._task_id = task_id
- self._with_task_name = with_task_name
-
- def prepare_chat(self, chat: Chat) -> Chat:
- def filter_fn(message: Message) -> Optional[Message]:
- if message.role != Role.ASSISTANT.value:
- return message
-
- copy = None
- if message.name != self._assistant_name:
- copy = message.get_copy()
- copy.name = ""
-
- task_payload = TaskPayload.read(message)
- # 判断是否要做任务信息的改造.
- if task_payload is None or message.memory is None or task_payload.task_id == self._task_id:
- return copy if copy else message
-
- copy = copy if copy else message.get_copy()
- # 对齐用户所见的消息体.
- copy.memory = None
- if self._with_task_name:
- copy.name = "task." + task_payload.name
- return copy
-
- chat.filter_messages(filter_fn)
- return chat
diff --git a/ghostos/framework/configs/__init__.py b/ghostos/framework/configs/__init__.py
index 44bc2642..b1f7fdb8 100644
--- a/ghostos/framework/configs/__init__.py
+++ b/ghostos/framework/configs/__init__.py
@@ -1,2 +1,3 @@
from ghostos.contracts.configs import Configs
+from ghostos.framework.configs.memimpl import MemoryConfigs
from ghostos.framework.configs.storageimpl import ConfigsByStorageProvider, WorkspaceConfigsProvider
diff --git a/ghostos/framework/configs/basic.py b/ghostos/framework/configs/basic.py
new file mode 100644
index 00000000..1d785651
--- /dev/null
+++ b/ghostos/framework/configs/basic.py
@@ -0,0 +1,39 @@
+from typing import Type, Optional
+from ghostos.contracts.configs import Configs, Config, C
+from abc import ABC, abstractmethod
+
+
+class BasicConfigs(Configs, ABC):
+ """
+ A Configs(repository) based on Storage, no matter what the Storage is.
+ """
+
+ def get(self, conf_type: Type[C], relative_path: Optional[str] = None) -> C:
+ path = conf_type.conf_path()
+ relative_path = relative_path if relative_path else path
+ content = self._get(relative_path)
+ return conf_type.unmarshal(content)
+
+ def get_or_create(self, conf: C) -> C:
+ path = conf.conf_path()
+ if not self._exists(path):
+ self._put(path, conf.marshal())
+ return conf
+ return self.get(type(conf))
+
+ @abstractmethod
+ def _get(self, relative_path: str) -> bytes:
+ pass
+
+ @abstractmethod
+ def _put(self, relative_path: str, content: bytes) -> None:
+ pass
+
+ @abstractmethod
+ def _exists(self, relative_path: str) -> bool:
+ pass
+
+ def save(self, conf: Config, relative_path: Optional[str] = None) -> None:
+ marshaled = conf.marshal()
+ relative_path = relative_path if relative_path else conf.conf_path()
+ self._put(relative_path, marshaled)
diff --git a/ghostos/framework/configs/memimpl.py b/ghostos/framework/configs/memimpl.py
new file mode 100644
index 00000000..867e8592
--- /dev/null
+++ b/ghostos/framework/configs/memimpl.py
@@ -0,0 +1,20 @@
+from typing import Dict, Optional
+from .basic import BasicConfigs
+
+
+class MemoryConfigs(BasicConfigs):
+
+ def __init__(self, defaults: Optional[Dict] = None):
+ defaults = defaults or {}
+ self._cache: Dict[str, bytes] = defaults
+
+ def _get(self, relative_path: str) -> bytes:
+ if relative_path not in self._cache:
+ raise FileNotFoundError(f'{relative_path} is not in cache')
+ return self._cache.get(relative_path)
+
+ def _put(self, relative_path: str, content: bytes) -> None:
+ self._cache[relative_path] = content
+
+ def _exists(self, relative_path: str) -> bool:
+ return relative_path in self._cache
diff --git a/ghostos/framework/configs/storageimpl.py b/ghostos/framework/configs/storageimpl.py
index b7f3bebb..f54cb1df 100644
--- a/ghostos/framework/configs/storageimpl.py
+++ b/ghostos/framework/configs/storageimpl.py
@@ -1,23 +1,24 @@
-from typing import Type, Optional
-from ghostos.contracts.configs import Configs, C
+from typing import Optional, Dict
+from ghostos.contracts.configs import Configs
from ghostos.contracts.storage import Storage
-from ghostos.container import Provider, Container, ABSTRACT
-from ghostos.core.ghosts import Workspace
+from ghostos.container import Provider, Container
+from ghostos.contracts.workspace import Workspace
+from .basic import BasicConfigs
-class StorageConfigs(Configs):
- """
- 基于 storage 实现的 configs.
- """
+class StorageConfigs(BasicConfigs):
def __init__(self, storage: Storage, conf_dir: str):
self._storage = storage.sub_storage(conf_dir)
- def get(self, conf_type: Type[C], file_name: Optional[str] = None) -> C:
- path = conf_type.conf_path()
- file_name = file_name if file_name else path
- content = self._storage.get(file_name)
- return conf_type.load(content)
+ def _get(self, relative_path: str) -> bytes:
+ return self._storage.get(relative_path)
+
+ def _put(self, relative_path: str, content: bytes) -> None:
+ self._storage.put(relative_path, content)
+
+ def _exists(self, relative_path: str) -> bool:
+ return self._storage.exists(relative_path)
class ConfigsByStorageProvider(Provider[Configs]):
@@ -28,22 +29,19 @@ def __init__(self, conf_dir: str):
def singleton(self) -> bool:
return True
- def contract(self) -> Type[Configs]:
- return Configs
-
def factory(self, con: Container) -> Optional[Configs]:
storage = con.force_fetch(Storage)
return StorageConfigs(storage, self._conf_dir)
class WorkspaceConfigsProvider(Provider[Configs]):
+ """
+ the Configs repository located at storage - workspace.configs()
+ """
def singleton(self) -> bool:
return True
- def contract(self) -> Type[ABSTRACT]:
- return Configs
-
- def factory(self, con: Container) -> Optional[ABSTRACT]:
+ def factory(self, con: Container) -> Optional[Configs]:
workspace = con.force_fetch(Workspace)
return StorageConfigs(workspace.configs(), "")
diff --git a/ghostos/framework/documents/__init__.py b/ghostos/framework/documents/__init__.py
new file mode 100644
index 00000000..8709040e
--- /dev/null
+++ b/ghostos/framework/documents/__init__.py
@@ -0,0 +1,2 @@
+from ghostos.contracts.documents import DocumentRegistry
+from .storage_impl import ConfiguredDocumentRegistryProvider, StorageDocumentsConfig
diff --git a/ghostos/framework/documents/storage_impl.py b/ghostos/framework/documents/storage_impl.py
new file mode 100644
index 00000000..eccc490e
--- /dev/null
+++ b/ghostos/framework/documents/storage_impl.py
@@ -0,0 +1,139 @@
+from typing import Iterable, List, Dict, Optional
+from typing_extensions import Self
+
+from ghostos.identifier import Identifier
+from ghostos.contracts.storage import FileStorage
+from ghostos.contracts.documents import Documents, DocumentRegistry
+from ghostos.container import Provider, Container
+from ghostos.contracts.configs import Configs, YamlConfig
+from ghostos.contracts.workspace import Workspace
+from pydantic import BaseModel, Field
+from os.path import join
+
+
+class StorageDocuments(Documents):
+
+ def __init__(
+ self,
+ storage: FileStorage,
+ *,
+ domain: str,
+ description: str,
+ default_lang: str,
+ ext: str,
+ lang: str = "",
+ ):
+ self._domain = domain
+ self._storage = storage
+ self._description = description
+ self._default_lang = default_lang
+ self._ext = ext
+ self._lang = lang or default_lang
+
+ def with_lang(self, lang: str) -> Self:
+ return StorageDocuments(
+ self._storage,
+ domain=self._domain,
+ description=self._description,
+ default_lang=self._default_lang,
+ ext=self._ext,
+ lang=lang,
+ )
+
+ def domain(self) -> str:
+ return self._domain
+
+ def directory(self) -> str:
+ return self._storage.abspath()
+
+ def description(self) -> str:
+ return self._description
+
+ def default_lang(self) -> str:
+ return self._default_lang
+
+ def langs(self) -> List[str]:
+ # todo
+ raise NotImplemented("todo")
+
+ def make_path(self, locale: str, filename: str) -> str:
+ return join(self.domain(), locale, filename + self._ext)
+
+ def read(self, filename: str, lang: str = "") -> str:
+ if not lang:
+ lang = self._default_lang
+ return self._read(lang, filename)
+
+ def _read(self, locale: str, filename: str) -> str:
+ path = self.make_path(locale, filename)
+ if not self._storage.exists(path):
+ path = self.make_path(self.default_lang(), filename)
+ content = self._storage.get(path)
+ return content.decode('utf-8')
+
+ def iterate(self, depth: int = -1) -> Iterable[str]:
+ raise NotImplemented("todo")
+
+
+class StorageDocumentsRegistry(DocumentRegistry):
+
+ def __init__(self):
+ self._documents: Dict[str, Documents] = {}
+
+ def get_domain(self, domain: str, lang: str = "") -> Documents:
+ if domain in self._documents:
+ docs = self._documents[domain]
+ return docs.with_lang(lang)
+ raise FileNotFoundError(f"documents domain not found: {domain}")
+
+ def register(self, domain: Documents) -> None:
+ self._documents[domain.domain()] = domain
+
+ def list_domains(self) -> List[Identifier]:
+ for domain in self._documents.values():
+ yield domain.__identifier__()
+
+
+class StorageDocumentsConfig(YamlConfig):
+ relative_path = "documents_registry.yml"
+
+ class DocConf(BaseModel):
+ directory: str = Field(description="sub directory to the assets directory")
+ domain: str = Field(description="Domain name")
+ extension: str = Field(description="File extension")
+ default_lang: str = Field(description="Default locale language name")
+ description: str = Field(default="", description="Description")
+
+ docs: List[DocConf] = Field(default_factory=list)
+
+
+class ConfiguredDocumentRegistryProvider(Provider[DocumentRegistry]):
+
+ def __init__(self, config_file: str = "documents_registry.yml"):
+ self._config_file = config_file
+
+ def singleton(self) -> bool:
+ return True
+
+ def factory(self, con: Container) -> Optional[DocumentRegistry]:
+ class Conf(StorageDocumentsConfig):
+ relative_path = self._config_file
+
+ configs = con.force_fetch(Configs)
+ conf = configs.get_or_create(Conf())
+
+ workspace = con.force_fetch(Workspace)
+ assets = workspace.assets()
+
+ registry = StorageDocumentsRegistry()
+
+ for c in conf.docs:
+ doc = StorageDocuments(
+ assets.sub_storage(c.directory),
+ domain=c.domain,
+ description=c.description,
+ default_lang=c.default_lang,
+ ext=c.extension,
+ )
+ registry.register(doc)
+ return registry
diff --git a/ghostos/framework/eventbuses/__init__.py b/ghostos/framework/eventbuses/__init__.py
index 2d4e0f66..6bc71a52 100644
--- a/ghostos/framework/eventbuses/__init__.py
+++ b/ghostos/framework/eventbuses/__init__.py
@@ -1 +1,2 @@
-from ghostos.framework.eventbuses.memimpl import MemEventBusImplProvider
+from ghostos.core.runtime import EventBus
+from ghostos.framework.eventbuses.memimpl import MemEventBusImplProvider, MemEventBusImpl
diff --git a/ghostos/framework/eventbuses/memimpl.py b/ghostos/framework/eventbuses/memimpl.py
index a02c3271..c61130e0 100644
--- a/ghostos/framework/eventbuses/memimpl.py
+++ b/ghostos/framework/eventbuses/memimpl.py
@@ -1,9 +1,10 @@
from typing import Optional, Dict, Type
+from typing_extensions import Self
-from ghostos.core.session import Event
-from ghostos.core.session.events import EventBus
+from ghostos.core.runtime import Event
+from ghostos.core.runtime.events import EventBus
from queue import Queue, Empty
-from ghostos.container import Provider, Container, BootstrappingProvider
+from ghostos.container import Provider, Container, BootstrapProvider
from ghostos.contracts.shutdown import Shutdown
@@ -14,20 +15,25 @@ def __init__(self):
self._task_notification_queue = Queue()
self._task_queues: Dict[str, Queue] = {}
+ def with_process_id(self, process_id: str) -> Self:
+ return self
+
+ def clear_all(self):
+ pass
+
def send_event(self, e: Event, notify: bool) -> None:
self._send_task_event(e)
if notify:
self.notify_task(e.task_id)
def _send_task_event(self, e: Event) -> None:
- event_id = e.id
+ event_id = e.event_id
task_id = e.task_id
self._events[event_id] = e
if task_id not in self._task_queues:
self._task_queues[task_id] = Queue()
queue = self._task_queues[task_id]
queue.put(event_id)
- queue.task_done()
def pop_task_event(self, task_id: str) -> Optional[Event]:
if task_id not in self._task_queues:
@@ -66,7 +72,7 @@ def shutdown(self) -> None:
del self._task_queues
-class MemEventBusImplProvider(BootstrappingProvider[EventBus]):
+class MemEventBusImplProvider(BootstrapProvider[EventBus]):
"""
mem event bus provider
"""
diff --git a/ghostos/framework/ghostos/__init__.py b/ghostos/framework/ghostos/__init__.py
index fa56baf1..c11509ec 100644
--- a/ghostos/framework/ghostos/__init__.py
+++ b/ghostos/framework/ghostos/__init__.py
@@ -1,5 +1,3 @@
-from ghostos.framework.ghostos.basic import BasicGhostOS
-from ghostos.framework.ghostos.demo_os import DemoGhostOS, DemoGhostOSConf
-
-demo_ghostos = DemoGhostOS()
-""" demo ghost os"""
+from ghostos.framework.ghostos.ghostos_impl import GhostOS, GhostOSImpl, GhostOSConfig, GhostOSProvider
+from ghostos.framework.ghostos.shell_impl import ShellImpl, ShellConf, Shell
+from ghostos.framework.ghostos.conversation_impl import Conversation, ConversationImpl, ConversationConf
diff --git a/ghostos/framework/ghostos/basic.py b/ghostos/framework/ghostos/basic.py
deleted file mode 100644
index c6ca9600..00000000
--- a/ghostos/framework/ghostos/basic.py
+++ /dev/null
@@ -1,143 +0,0 @@
-from typing import Optional, List
-from abc import ABC, abstractmethod
-from os.path import join, dirname
-import yaml
-from logging.config import dictConfig
-from ghostos.container import Container
-from ghostos.core.ghostos import AbsGhostOS
-from ghostos.core.ghosts import Ghost
-from ghostos.core.messages import Stream
-from ghostos.core.session import Process, Task
-from ghostos.contracts.shutdown import ShutdownProvider
-from ghostos.contracts.modules import Modules, DefaultModulesProvider
-from ghostos.framework.storage import FileStorageProvider
-from ghostos.framework.logger import NamedLoggerProvider
-from ghostos.framework.workspaces import BasicWorkspaceProvider
-from ghostos.framework.configs import WorkspaceConfigsProvider
-from ghostos.framework.threads import WorkspaceThreadsProvider
-from ghostos.framework.processes import WorkspaceProcessesProvider
-from ghostos.framework.tasks import WorkspaceTasksProvider
-from ghostos.framework.llms import ConfigBasedLLMsProvider
-from ghostos.framework.eventbuses import MemEventBusImplProvider
-from ghostos.contracts.pool import DefaultPoolProvider
-from ghostos.entity import EntityFactory, EntityFactoryImpl
-from ghostos.container import Provider, Bootstrapper
-
-project_dir = dirname(dirname(dirname(__file__)))
-demo_dir = join(project_dir, "demo")
-logging_conf_path = join(demo_dir, "configs/logging.yml")
-
-__all__ = ['BasicGhostOS']
-
-
-class BasicGhostOS(AbsGhostOS, ABC):
-
- def __init__(
- self, *,
- root_dir: str = demo_dir,
- logger_conf_path: str = logging_conf_path,
- logger_name: str = "debug",
- config_path: str = "configs",
- runtime_path: str = "runtime",
- source_path: str = "src",
- processes_path: str = "processes",
- tasks_path: str = "tasks",
- threads_path: str = "threads",
- llm_config_path: str = "llms_conf.yml",
- container: Optional[Container] = None,
- providers: Optional[List[Provider]] = None,
- bootstrapper: Optional[List[Bootstrapper]] = None,
- ):
- self._root_dir = root_dir
- self._processes_path = processes_path
- self._tasks_path = tasks_path
- self._threads_path = threads_path
- self._llm_config_path = llm_config_path
- self._config_path = config_path
- self._runtime_path = runtime_path
- self._source_path = source_path
-
- # container
- self._container = container if container else Container()
- self._prepare_logger(logger_conf_path, logger_name)
- self._prepare_container(providers, bootstrapper)
-
- modules = self._container.force_fetch(Modules)
- # register entity factory
- self._entity_factory = EntityFactoryImpl(modules.import_module)
- self._container.set(EntityFactory, self._entity_factory)
- self._container.bootstrap()
- self._on_initialized()
-
- @abstractmethod
- def _on_initialized(self):
- """
- callback on initialized the ghost os
- """
- pass
-
- @abstractmethod
- def make_ghost(
- self, *,
- upstream: Stream,
- process: Process,
- task: Optional[Task] = None,
- task_id: Optional[str] = None,
- ) -> Ghost:
- pass
-
- @abstractmethod
- def on_error(self, error: Exception) -> bool:
- pass
-
- def _default_providers(self) -> List[Provider]:
- return [
- FileStorageProvider(self._root_dir),
- BasicWorkspaceProvider(
- root_dir="",
- configs_path=self._config_path,
- runtime_path=self._runtime_path,
- source_path=self._source_path,
- ),
- DefaultModulesProvider(),
- WorkspaceConfigsProvider(),
- WorkspaceProcessesProvider(self._processes_path),
- WorkspaceTasksProvider(self._tasks_path),
- WorkspaceThreadsProvider(self._threads_path),
- DefaultPoolProvider(100),
- ConfigBasedLLMsProvider(self._llm_config_path),
- MemEventBusImplProvider(),
- ShutdownProvider(),
- ]
-
- def _prepare_container(
- self,
- providers: Optional[List[Provider]] = None,
- bootstrapper: Optional[List[Bootstrapper]] = None,
- ):
- if providers:
- for provider in providers:
- self._container.register(provider)
- if bootstrapper:
- for b in bootstrapper:
- self._container.add_bootstrapper(b)
- # register default providers.
- for provider in self._default_providers():
- contract = provider.contract()
- # 只有未绑定的, 才会使用默认的去绑定.
- if not self._container.bound(contract):
- self._container.register(provider)
-
- def _prepare_logger(self, logger_conf_path: str, logger_name: str):
- with open(logger_conf_path, "rb") as f:
- content = f.read()
- data = yaml.safe_load(content)
- dictConfig(data)
- self._container.register(NamedLoggerProvider(logger_name))
-
- def container(self) -> Container:
- return self._container
-
- def destroy(self) -> None:
- self._container.destroy()
- del self._container
diff --git a/ghostos/framework/ghostos/conversation_impl.py b/ghostos/framework/ghostos/conversation_impl.py
new file mode 100644
index 00000000..4492e2e2
--- /dev/null
+++ b/ghostos/framework/ghostos/conversation_impl.py
@@ -0,0 +1,331 @@
+from typing import Optional, Iterable, List, TypeVar, Tuple, Union, Callable
+
+from ghostos.container import Container
+from ghostos.abcd import Conversation, Scope, Ghost, Context
+from ghostos.abcd import run_session_event
+from ghostos.errors import SessionError
+from ghostos.contracts.variables import Variables
+from ghostos.core.messages import (
+ Message, Role, MessageKind, MessageKindParser,
+ Stream, Receiver, new_basic_connection,
+)
+from ghostos.core.runtime import (
+ Event, EventTypes, EventBus,
+ GoTaskStruct, TaskLocker, GoTasks, TaskState,
+ GoThreadInfo, GoThreads,
+)
+from ghostos.core.llms import LLMFunc
+from ghostos.contracts.pool import Pool
+from ghostos.contracts.logger import LoggerItf, wrap_logger
+from ghostos.entity import to_entity_meta, get_entity
+from pydantic import BaseModel, Field
+from .session_impl import SessionImpl
+from threading import Lock, Thread
+
+__all__ = ["ConversationImpl", "ConversationConf", "Conversation"]
+
+
+class ConversationConf(BaseModel):
+ message_receiver_idle: float = Field(
+ 0.05,
+ description="The time in seconds to wait between retrievals",
+ )
+ max_session_step: int = Field(
+ 10,
+ description="The maximum number of steps to run session event",
+ )
+ max_task_errors: int = Field(
+ 3,
+ description="The maximum error number of task",
+ )
+
+
+G = TypeVar("G", bound=Ghost)
+
+
+class ConversationImpl(Conversation[G]):
+
+ def __init__(
+ self,
+ conf: ConversationConf,
+ container: Container,
+ task: GoTaskStruct,
+ task_locker: TaskLocker,
+ is_background: bool,
+ shell_closed: Callable[[], bool],
+ username: str = "",
+ user_role: str = Role.USER.value,
+ ):
+ self._closed = False
+ self._conf = conf
+ self.task_id = task.task_id
+ self._container = Container(parent=container, name="conversation")
+ self._username = username
+ self._user_role = user_role
+ variables = self._container.force_fetch(Variables)
+ self._message_parser = MessageKindParser(
+ variables,
+ name=self._username,
+ role=self._user_role,
+ )
+
+ self.scope = Scope(
+ shell_id=task.shell_id,
+ process_id=task.process_id,
+ task_id=task.task_id,
+ parent_task_id=task.parent,
+ )
+ self.logger = wrap_logger(
+ self._container.force_fetch(LoggerItf),
+ dict(scope=self.scope.model_dump(exclude_defaults=True)),
+ )
+
+ self._pool = self._container.force_fetch(Pool)
+ self._is_background = is_background
+ self._ctx: Optional[Context] = None
+ self._task_locker = task_locker
+ self._tasks = container.force_fetch(GoTasks)
+ self._threads = container.force_fetch(GoThreads)
+ self._eventbus = container.force_fetch(EventBus)
+ self._submit_session_thread: Optional[Thread] = None
+ self._handling_event = False
+ self._mutex = Lock()
+ self._shell_closed = shell_closed
+ self._bootstrap()
+
+ def _bootstrap(self):
+ providers = self.get_ghost_driver().providers()
+ # bind self
+ self._container.set(Conversation, self)
+ for provider in providers:
+ self._container.register(provider)
+ self._container.bootstrap()
+
+ def container(self) -> Container:
+ self._validate_closed()
+ return self._container
+
+ def get_task(self) -> GoTaskStruct:
+ self._validate_closed()
+ return self._tasks.get_task(self.scope.task_id)
+
+ def get_thread(self, truncated: bool = False) -> GoThreadInfo:
+ self._validate_closed()
+ task = self.get_task()
+ if not truncated:
+ thread_id = task.thread_id
+ return self._threads.get_thread(thread_id, create=True)
+ session = self._create_session(task, None)
+ return session.get_truncated_thread()
+
+ def update_thread(self, thread: GoThreadInfo) -> None:
+ self.refresh()
+ self._validate_closed()
+ task = self.get_task()
+ thread.id = task.thread_id
+ self._threads.save_thread(thread)
+
+ def get_ghost(self) -> Ghost:
+ self._validate_closed()
+ task = self.get_task()
+ return get_entity(task.meta, Ghost)
+
+ def get_context(self) -> Optional[Context]:
+ self._validate_closed()
+ task = self.get_task()
+ if task.context is None:
+ return None
+ return get_entity(task.context, Context)
+
+ def get_functions(self) -> List[LLMFunc]:
+ self.refresh()
+ self._validate_closed()
+ session = self._create_session(self.get_task(), None)
+ actions = self.get_ghost_driver().actions(session)
+ functions = []
+ for action in actions:
+ function = action.as_function()
+ if function is not None:
+ functions.append(function)
+ return functions
+
+ def get_instructions(self) -> str:
+ self.refresh()
+ self._validate_closed()
+ session = self._create_session(self.get_task(), None)
+ try:
+ instructions = session.get_instructions()
+ return instructions
+ finally:
+ session.destroy()
+
+ def refresh(self) -> bool:
+ self._validate_closed()
+ ok = self._task_locker.refresh()
+ if not ok:
+ self.close()
+ return ok
+
+ def get_artifact(self) -> Tuple[Union[Ghost.ArtifactType, None], TaskState]:
+ self._validate_closed()
+ task = self.get_task()
+ session = self._create_session(task, None)
+ with session:
+ return session.get_artifact(), TaskState(session.task.state)
+
+ def talk(self, query: str, user_name: str = "", context: Optional[Ghost.ContextType] = None) -> Tuple[Event, Receiver]:
+ self._validate_closed()
+ self.logger.debug("talk to user %s", user_name)
+ message = Role.USER.new(content=query, name=user_name)
+ return self.respond([message], context)
+
+ def update_context(self, context: Context) -> None:
+ self._validate_closed()
+ self._ctx = context
+
+ def respond(
+ self,
+ inputs: Iterable[MessageKind],
+ context: Optional[Ghost.ContextType] = None,
+ streaming: bool = True,
+ ) -> Tuple[Event, Receiver]:
+ self._validate_closed()
+ if self._submit_session_thread:
+ self._submit_session_thread.join()
+ self._submit_session_thread = None
+ messages = list(self._message_parser.parse(inputs))
+ context_meta = to_entity_meta(context) if context is not None else None
+ if self._ctx is not None:
+ context_meta = to_entity_meta(self._ctx)
+ self._ctx = None
+ event = EventTypes.INPUT.new(
+ task_id=self.scope.task_id,
+ messages=messages,
+ context=context_meta,
+ )
+ return event, self.respond_event(event, streaming=streaming)
+
+ def respond_event(
+ self,
+ event: Event,
+ timeout: float = 0.0,
+ streaming: bool = True,
+ ) -> Receiver:
+ self.refresh()
+ self._validate_closed()
+ if self._handling_event:
+ raise RuntimeError("conversation is handling event")
+ # complete task_id
+ if not event.task_id:
+ event.task_id = self.scope.task_id
+ self.logger.debug("start to respond event %s", event.event_id)
+
+ stream, retriever = new_basic_connection(
+ timeout=timeout,
+ idle=self._conf.message_receiver_idle,
+ complete_only=self._is_background or not streaming,
+ )
+ if self._submit_session_thread:
+ self._submit_session_thread.join()
+ self._submit_session_thread = None
+ self._submit_session_thread = Thread(target=self._submit_session_event, args=(event, stream,))
+ self._submit_session_thread.start()
+ return retriever
+
+ def _validate_closed(self):
+ # todo: change error to defined error
+ if self._closed:
+ raise RuntimeError(f"Conversation is closed")
+ if self._shell_closed():
+ raise RuntimeError(f"Shell is closed")
+
+ def _submit_session_event(self, event: Event, stream: Stream) -> None:
+ with self._mutex:
+ self._handling_event = True
+ self.logger.debug("submit session event")
+ try:
+ with stream:
+ task = self._tasks.get_task(event.task_id)
+ session = self._create_session(task, stream)
+ self.logger.debug(
+ f"create session from event id %s, task_id is %s",
+ event.event_id, task.task_id,
+ )
+ with session:
+ run_session_event(session, event, self._conf.max_session_step)
+ except Exception as e:
+ if not self.fail(error=e):
+ raise
+ finally:
+ if task and task.shall_notify():
+ self._eventbus.notify_task(event.task_id)
+ self._handling_event = False
+ self._submit_session_thread = None
+
+ def _create_session(
+ self,
+ task: GoTaskStruct,
+ stream: Optional[Stream],
+ ) -> SessionImpl:
+ return SessionImpl(
+ container=self.container(),
+ logger=self.logger,
+ scope=self.scope,
+ stream=stream,
+ task=task,
+ refresh_callback=self.refresh,
+ alive_check=self.is_alive,
+ max_errors=self._conf.max_task_errors,
+ )
+
+ def pop_event(self) -> Optional[Event]:
+ if self.available():
+ return self._eventbus.pop_task_event(self.scope.task_id)
+ return None
+
+ def send_event(self, event: Event) -> None:
+ self._validate_closed()
+ task = self._tasks.get_task(event.task_id)
+ notify = True
+ if task:
+ notify = task.depth > 0
+ self._eventbus.send_event(event, notify)
+
+ def fail(self, error: Exception) -> bool:
+ if self._closed:
+ return False
+ if isinstance(error, SessionError):
+ self.logger.info(f"conversation {self.task_id} receive session stop error: {error}")
+ return False
+ elif isinstance(error, IOError):
+ self.logger.exception(f"conversation {self.task_id} receive IO error: {error}")
+ return False
+ # otherwise, close the whole thing.
+ self.logger.exception(f"conversation {self.task_id} receive runtime error: {error}")
+ self.close()
+ return False
+
+ def close(self):
+ if self._closed:
+ return
+ self._closed = True
+ self.logger.info("conversation %s is closing", self.task_id)
+ self._handling_event = False
+ if self._submit_session_thread:
+ self._submit_session_thread = None
+ self.logger.info("conversation %s is destroying", self.task_id)
+ self._container.shutdown()
+ self._container = None
+ if self._task_locker.acquired():
+ self._task_locker.release()
+
+ def is_closed(self) -> bool:
+ return self._closed or self._shell_closed()
+
+ def is_alive(self) -> bool:
+ return not self._closed
+
+ def available(self) -> bool:
+ if self.is_closed() or self._shell_closed() or self._handling_event:
+ return False
+ return True
diff --git a/ghostos/framework/ghostos/demo_os.py b/ghostos/framework/ghostos/demo_os.py
deleted file mode 100644
index 88b80122..00000000
--- a/ghostos/framework/ghostos/demo_os.py
+++ /dev/null
@@ -1,76 +0,0 @@
-from typing import Optional, ClassVar, Dict
-
-from ghostos.core.ghosts import Ghost, GhostConf, Workspace, Shell
-from ghostos.core.messages import Stream
-from ghostos.core.session import Process, Task
-from ghostos.contracts.logger import LoggerItf
-from ghostos.contracts.configs import Configs, YamlConfig
-
-from ghostos.entity import EntityMeta
-from ghostos.framework.shells import EmptyShell
-from ghostos.framework.ghostos.basic import BasicGhostOS
-from ghostos.framework.ghosts import DemoGhostConf, DemoGhost
-from pydantic import Field
-
-
-class DemoGhostOSConf(YamlConfig):
- relative_path: ClassVar[str] = "ghosts.yml"
- ghosts: Dict[str, EntityMeta] = Field(default_factory=dict, description="ghost conf entity metas, key is ghost id")
-
-
-class DemoGhostOS(BasicGhostOS):
-
- def _on_initialized(self):
- configs = self.container().force_fetch(Configs)
- ghosts_confs = configs.get(DemoGhostOSConf)
- self._ghosts_metas = ghosts_confs.ghosts
-
- def make_ghost(
- self, *,
- upstream: Stream,
- process: Process,
- task: Optional[Task] = None,
- task_id: Optional[str] = None,
- ) -> Ghost:
- conf = self._entity_factory.force_new_entity(process.ghost_meta, GhostConf)
- return self._make_ghost_instance(conf, upstream, process, task, task_id)
-
- def register(self, ghost_conf: GhostConf) -> None:
- ghost_id = ghost_conf.identifier().id
- self._ghosts_metas[ghost_id] = ghost_conf.to_entity_meta()
-
- def _make_ghost_instance(
- self,
- conf: GhostConf,
- upstream: Stream,
- process: Process,
- task: Optional[Task] = None,
- task_id: Optional[str] = None,
- ) -> Ghost:
- if isinstance(conf, DemoGhostConf):
- return DemoGhost(
- conf=conf,
- container=self.container(),
- entity_factory=self._entity_factory,
- workspace=self.container().force_fetch(Workspace),
- shell=self._make_shell(conf),
- process=process,
- upstream=upstream,
- task=task,
- task_id=task_id,
- )
- else:
- raise NotImplementedError(f"GhostOS {conf} is not supported yet.")
-
- def _make_shell(self, ghost_conf: GhostConf) -> Shell:
- return EmptyShell()
-
- def on_error(self, error: Exception) -> bool:
- logger = self.container().force_fetch(LoggerItf)
- logger.error(str(error))
- return True
-
- def get_ghost_meta(self, ghost_id: str) -> Optional[EntityMeta]:
- if ghost_id in self._ghosts_metas:
- return self._ghosts_metas[ghost_id]
- return None
diff --git a/ghostos/framework/ghostos/ghostos_impl.py b/ghostos/framework/ghostos/ghostos_impl.py
new file mode 100644
index 00000000..edc964b7
--- /dev/null
+++ b/ghostos/framework/ghostos/ghostos_impl.py
@@ -0,0 +1,94 @@
+from typing import Optional, Dict, List
+
+from ghostos.abcd import GhostOS, Shell
+from ghostos.core.runtime import GoProcesses, GoProcess, GoThreads, GoTasks, EventBus
+from ghostos.container import Container, Provider, Contracts, INSTANCE
+from ghostos.contracts.configs import Configs, YamlConfig
+from ghostos.contracts.modules import Modules
+from ghostos.contracts.variables import Variables
+from ghostos.contracts.workspace import Workspace
+from ghostos.contracts.logger import LoggerItf, get_ghostos_logger
+from pydantic import Field
+from .shell_impl import ShellImpl, ShellConf
+
+__all__ = ['GhostOS', "GhostOSImpl", "GhostOSConfig", "GhostOSProvider"]
+
+
+class GhostOSConfig(YamlConfig):
+ relative_path = "ghostos_conf.yml"
+ shells: Dict[str, ShellConf] = Field(
+ description="the shell configurations",
+ )
+
+
+class GhostOSImpl(GhostOS):
+ contracts: Contracts = Contracts([
+ GoProcesses,
+ GoTasks,
+ GoThreads,
+ EventBus,
+ LoggerItf,
+ Configs,
+ Modules,
+ Variables,
+ Workspace,
+ ])
+
+ def __init__(
+ self,
+ container: Container,
+ ):
+ self.contracts.validate(container)
+ self._container = container
+ self._processes = container.force_fetch(GoProcesses)
+ self._configs = container.force_fetch(Configs)
+ self._ghostos_config = self._configs.get(GhostOSConfig)
+
+ @property
+ def logger(self) -> LoggerItf:
+ return get_ghostos_logger()
+
+ def container(self) -> Container:
+ return self._container
+
+ def create_shell(
+ self,
+ name: str,
+ *,
+ shell_id: str = "",
+ providers: Optional[List[Provider]] = None,
+ process_id: Optional[str] = None,
+ ) -> Shell:
+ if name not in self._ghostos_config.shells:
+ shell_conf = ShellConf()
+ else:
+ shell_conf = self._ghostos_config.shells[name]
+ if not shell_id:
+ shell_id = name
+ process = self._processes.get_process(shell_id)
+ if process is None:
+ process = GoProcess.new(shell_id=shell_id, process_id=process_id)
+ self.logger.debug(f"Created shell `{shell_id}` process `{process_id}`")
+ elif process_id is not None and process.process_id != process_id:
+ process = GoProcess.new(shell_id=shell_id, process_id=process_id)
+ self.logger.debug(f"Created shell `{shell_id}` new process `{process_id}`")
+ else:
+ self.logger.debug(f"get shell `{shell_id}` new process `{process.process_id}`")
+ self._processes.save_process(process)
+
+ # prepare container
+ providers = providers or []
+ return ShellImpl(
+ config=shell_conf,
+ container=self._container,
+ process=process,
+ providers=providers,
+ )
+
+
+class GhostOSProvider(Provider[GhostOS]):
+ def singleton(self) -> bool:
+ return True
+
+ def factory(self, con: Container) -> Optional[INSTANCE]:
+ return GhostOSImpl(con)
diff --git a/ghostos/framework/ghostos/session_impl.py b/ghostos/framework/ghostos/session_impl.py
new file mode 100644
index 00000000..5dc569d7
--- /dev/null
+++ b/ghostos/framework/ghostos/session_impl.py
@@ -0,0 +1,466 @@
+from typing import Optional, List, Iterable, Tuple, TypeVar, Dict, Union, Any, Callable
+from ghostos.errors import StreamingError
+from ghostos.abcd import (
+ Session, Ghost, GhostDriver, Shell, Scope, Taskflow, Operator, Subtasks,
+ Messenger,
+)
+from ghostos.abcd import get_ghost_driver
+from ghostos.core.messages import (
+ MessageKind, Message, FunctionCaller, Stream, Role, MessageKindParser, MessageType
+)
+from ghostos.core.messages.message_classes import FunctionCallMessage
+from ghostos.core.runtime import (
+ TaskBrief, GoTaskStruct, TaskPayload, GoTasks, TaskState,
+ EventBus, Event, EventTypes,
+ GoThreads,
+ GoThreadInfo,
+)
+from ghostos.prompter import Prompter
+from ghostos.contracts.logger import LoggerItf
+from ghostos.contracts.variables import Variables
+from ghostos.container import Container, provide, Contracts
+from ghostos.entity import to_entity_meta, from_entity_meta, get_entity, EntityType
+from ghostos.identifier import get_identifier
+from ghostos.framework.messengers import DefaultMessenger
+from .taskflow_impl import TaskflowImpl
+from .subtasks_impl import SubtasksImpl
+from threading import Lock
+
+from ...errors import SessionError
+
+G = TypeVar("G", bound=Ghost)
+
+
+class EmptyOperator(Operator):
+
+ def run(self, session: Session) -> Union[Operator, None]:
+ return None
+
+ def destroy(self):
+ pass
+
+
+class SessionImpl(Session[Ghost]):
+ contracts = Contracts([
+ GoThreads,
+ GoTasks,
+ EventBus,
+ ])
+
+ def __init__(
+ self,
+ container: Container,
+ logger: LoggerItf,
+ scope: Scope,
+ stream: Optional[Stream],
+ task: GoTaskStruct,
+ refresh_callback: Callable[[], bool],
+ alive_check: Callable[[], bool],
+ max_errors: int,
+ ):
+ # session level container
+ self.container = Container(parent=container, name="session")
+
+ self.upstream = stream
+ self.task = task
+ self.logger = logger
+ self._refresh_callback = refresh_callback
+ self._alive_check = alive_check
+ threads = container.force_fetch(GoThreads)
+ thread = threads.get_thread(task.thread_id, create=True)
+ self.thread = thread
+ self.scope = scope
+
+ self.ghost: G = get_entity(self.task.meta, Ghost)
+ self.ghost_driver: GhostDriver[G] = get_ghost_driver(self.ghost)
+ identifier = get_identifier(self.ghost)
+ variables = container.force_fetch(Variables)
+ self._message_parser = MessageKindParser(
+ variables,
+ name=identifier.name,
+ role=Role.ASSISTANT.value,
+ )
+ self.state = self.unmarshal_state(task)
+ self._max_errors = max_errors
+ self._fetched_task_briefs: Dict[str, TaskBrief] = {}
+ self._creating_tasks: Dict[str, GoTaskStruct] = {}
+ self._firing_events: List[Event] = []
+ self._saving_threads: Dict[str, GoThreadInfo] = {}
+ self._system_logs: List[str] = []
+ self._failed = False
+ self._done = False
+ self._destroyed = False
+ self._saved = False
+ self._bootstrap()
+ self._respond_lock = Lock()
+ Session.instance_count += 1
+
+ def __del__(self):
+ # for gc test
+ Session.instance_count -= 1
+ self.destroy()
+
+ def _bootstrap(self):
+ self.contracts.validate(self.container)
+ self.container.set(Session, self)
+ self.container.set(Scope, self.scope)
+ self.container.set(MessageKindParser, self._message_parser)
+ self.container.register(provide(GoTaskStruct, False)(lambda c: self.task))
+ self.container.register(provide(GoThreadInfo, False)(lambda c: self.thread))
+ self.container.register(provide(Taskflow, False)(lambda c: self.taskflow()))
+ self.container.register(provide(Subtasks, False)(lambda c: self.subtasks()))
+ self.container.register(provide(Messenger, False)(lambda c: self.messenger()))
+ self.container.bootstrap()
+ # truncate thread.
+
+ def get_truncated_thread(self) -> GoThreadInfo:
+ thread = self.ghost_driver.truncate(self)
+ return thread
+
+ @staticmethod
+ def unmarshal_state(task: GoTaskStruct) -> Dict[str, EntityType]:
+ state_values = {}
+ for key, entity_meta in task.state_values.items():
+ entity = from_entity_meta(entity_meta)
+ state_values[key] = entity
+ return state_values
+
+ def alive(self) -> bool:
+ if self._failed or self._destroyed:
+ return False
+ return self._alive_check() and (self.upstream is None or self.upstream.alive())
+
+ def _validate_alive(self):
+ if not self.alive():
+ raise RuntimeError(f"Session is not alive")
+
+ def to_messages(self, values: Iterable[Union[MessageKind, Any]]) -> List[Message]:
+ return list(self._message_parser.parse(values))
+
+ def parse_event(self, event: Event) -> Tuple[Optional[Event], Optional[Operator]]:
+ self._validate_alive()
+ driver = get_ghost_driver(self.ghost)
+ # if the task is new, initialize the task.
+ if self.task.state == TaskState.NEW.value:
+ driver.on_creating(self)
+ self.task.state = TaskState.RUNNING.value
+
+ # always let ghost driver decide event handling logic first.
+ event = driver.parse_event(self, event)
+ if event is None:
+ return None, None
+
+ if EventTypes.ACTION_CALL.value == event.type:
+ self.thread.new_turn(event)
+ actions = self.ghost_driver.actions(self)
+ actions_map = {action.name(): action for action in actions}
+ for message in event.messages:
+ fc = FunctionCallMessage.from_message(message)
+ if fc is None:
+ continue
+ if fc.caller.name in actions_map:
+ action = actions_map[fc.caller.name]
+ op = action.run(self, fc.caller)
+ if op is not None:
+ return None, op
+ return None, None
+
+ # notification do not trigger the handling
+ if EventTypes.NOTIFY.value == event.type:
+ self.thread.new_turn(event)
+ return None, None
+
+ if EventTypes.INPUT.value == event.type:
+ # only input event can reset errors.
+ self.task.errors = 0
+ self.task.state = TaskState.RUNNING.value
+ if event.context is not None:
+ self.task.context = event.context
+
+ # other event
+ elif self.task.is_dead():
+ # dead task can only respond event from parent input.
+ self.thread.new_turn(event)
+ self.logger.info(
+ "task %s is dead, save event %s without run", self.scope.task_id, event.event_id,
+ )
+ return None, EmptyOperator()
+
+ if EventTypes.ERROR.value == event.type:
+ self.task.errors += 1
+ if self.task.errors > self._max_errors:
+ # if reach max errors, fail the task
+ return None, self.taskflow().fail("task failed too much, exceeds max errors")
+
+ if EventTypes.CANCEL.value == event.type:
+ # cancel self and all subtasks.
+ self.task.errors = 0
+ self.thread.new_turn(event)
+ self.task.state = TaskState.CANCELLED.value
+ # cancel children.
+ if self.task.children:
+ for child_id in self.task.children:
+ event = EventTypes.CANCEL.new(
+ task_id=child_id,
+ messages=[],
+ from_task_id=self.task.task_id,
+ from_task_name=self.task.name,
+ reason="parent task is canceled",
+ instruction="cancel what you are doing",
+ )
+ self.fire_events(event)
+ return None, EmptyOperator()
+
+ event.context = None
+ return event, None
+
+ def system_log(self, log: str) -> None:
+ self._system_logs.append(log)
+
+ def taskflow(self) -> Taskflow:
+ self._validate_alive()
+ return TaskflowImpl(self, self._message_parser)
+
+ def subtasks(self) -> Subtasks:
+ return SubtasksImpl(self)
+
+ def get_context(self) -> Optional[Prompter]:
+ if self.task.context is None:
+ return None
+ return get_entity(self.task.context, Prompter)
+
+ def get_artifact(self) -> Ghost.ArtifactType:
+ return self.ghost_driver.get_artifact(self)
+
+ def get_instructions(self) -> str:
+ return self.ghost_driver.get_instructions(self)
+
+ def refresh(self, throw: bool = False) -> bool:
+ if self._failed:
+ if throw:
+ raise RuntimeError(f"Session is already failed")
+ return False
+ if self._destroyed:
+ if throw:
+ raise RuntimeError(f"Session is already destroyed")
+ return False
+
+ if not self.alive():
+ if throw:
+ raise RuntimeError(f"Session is not alive")
+ return False
+ if self._refresh_callback():
+ self._saved = False
+ return True
+ elif throw:
+ raise RuntimeError(f"session refresh callback failed")
+ else:
+ return False
+
+ def _reset(self):
+ self._fetched_task_briefs = {}
+ self._firing_events = []
+ self._creating_tasks = {}
+ self._saving_threads = {}
+ self._system_logs = []
+ self.task = self.task.new_turn()
+
+ def messenger(self, stage: str = "") -> Messenger:
+ self._validate_alive()
+ task_payload = TaskPayload.from_task(self.task)
+ identity = get_identifier(self.ghost)
+ return DefaultMessenger(
+ upstream=self.upstream,
+ name=identity.name,
+ role=Role.ASSISTANT.value,
+ payloads=[task_payload],
+ stage=stage,
+ )
+
+ def respond(self, messages: Iterable[MessageKind], stage: str = "") -> Tuple[List[Message], List[FunctionCaller]]:
+ self._validate_alive()
+ messages = self._message_parser.parse(messages)
+ with self._respond_lock:
+ messenger = self.messenger(stage)
+ try:
+ messenger.send(messages)
+ except StreamingError as e:
+ raise SessionError(f"session failed during streaming: {e}")
+
+ buffer, callers = messenger.flush()
+ self.logger.debug("append messages to thread: %s", buffer)
+ self.thread.append(*buffer)
+ return buffer, callers
+
+ def cancel_subtask(self, ghost: G, reason: str = "") -> None:
+ self._validate_alive()
+ driver = get_ghost_driver(ghost)
+ task_id = driver.make_task_id(self.scope)
+ tasks = self.container.force_fetch(GoTasks)
+ subtask = tasks.get_task(task_id)
+ if subtask is None:
+ return
+ event = EventTypes.CANCEL.new(
+ task_id=task_id,
+ reason=reason,
+ messages=[],
+ from_task_id=self.task.task_id,
+ from_task_name=self.task.name,
+ )
+ self.fire_events(event)
+
+ def create_tasks(self, *tasks: GoTaskStruct) -> None:
+ self._validate_alive()
+ for task in tasks:
+ self._creating_tasks[task.task_id] = task
+
+ def create_threads(self, *threads: GoThreadInfo) -> None:
+ self._validate_alive()
+ for t in threads:
+ self._saving_threads[t.id] = t
+
+ def call(self, ghost: Ghost, ctx: Ghost.ContextType) -> Ghost.ArtifactType:
+ self._validate_alive()
+ shell = self.container.force_fetch(Shell)
+ return shell.call(ghost, ctx)
+
+ def fire_events(self, *events: "Event") -> None:
+ self._validate_alive()
+ firing = list(events)
+ self.logger.debug("session fire events: %d", len(firing))
+ self._firing_events.extend(firing)
+
+ def get_task_briefs(self, *task_ids: str) -> Dict[str, TaskBrief]:
+ self._validate_alive()
+ ids = set(task_ids)
+ result = {}
+ fetch = []
+ for task_id in ids:
+ if task_id in self._fetched_task_briefs:
+ result[task_id] = self._fetched_task_briefs[task_id]
+ else:
+ fetch.append(task_id)
+ if fetch:
+ tasks = self.container.force_fetch(GoTasks)
+ briefs = tasks.get_task_briefs(fetch)
+ for task_brief in briefs.values():
+ result[task_brief.task_id] = task_brief
+ self._fetched_task_briefs[task_brief.task_id] = task_brief
+ return result
+
+ def save(self) -> None:
+ if self._saved or self._destroyed:
+ return
+ self._saved = True
+ self.logger.info("saving session on %s", self.scope.model_dump())
+ self._validate_alive()
+ self._update_subtasks()
+ self._update_state_changes()
+ self._do_create_tasks()
+ self._do_save_threads()
+ self._do_fire_events()
+ self._reset()
+
+ def _update_subtasks(self):
+ children = self.task.children
+ if len(children) == 0:
+ return
+ tasks = self.get_task_briefs(*children)
+ for tid, tb in tasks:
+ if TaskState.is_dead(tb.state_name):
+ continue
+ children.append(tid)
+ self.task.children = children
+
+ def _update_state_changes(self) -> None:
+ task = self.task
+ task.meta = to_entity_meta(self.ghost)
+ state_values = {}
+ for name, value in self.state.items():
+ state_values[name] = to_entity_meta(value)
+ thread = self.thread
+ # update system log
+ if len(self._system_logs) > 0:
+ content = "\n".join(self._system_logs)
+ message = Role.SYSTEM.new(content=content)
+ thread.append(message)
+ self._system_logs = []
+
+ task.thread_id = thread.id
+ task.state_values = state_values
+ if task.state == TaskState.RUNNING.value:
+ task.state = TaskState.WAITING.value
+
+ tasks = self.container.force_fetch(GoTasks)
+ threads = self.container.force_fetch(GoThreads)
+ self.logger.debug("task info %s", task.model_dump())
+ tasks.save_task(task)
+ threads.save_thread(thread)
+
+ def _do_create_tasks(self) -> None:
+ tasks = self.container.force_fetch(GoTasks)
+ if self._creating_tasks:
+ tasks.save_task(*self._creating_tasks.values())
+ self._creating_tasks = []
+
+ def _do_save_threads(self) -> None:
+ threads = self.container.force_fetch(GoThreads)
+ if self._saving_threads:
+ threads.save_thread(*self._saving_threads.values())
+ self._saving_threads = []
+
+ def _do_fire_events(self) -> None:
+ if not self._firing_events:
+ return
+ logger = self.logger
+ bus = self.container.force_fetch(EventBus)
+ for e in self._firing_events:
+ # all the sub-tasks need notification
+ notify = True
+ if e.task_id == self.task.parent:
+ notify = self.task.depth - 1 == 0
+ bus.send_event(e, notify)
+ logger.debug("session fired event %s", {e.event_id})
+ self._firing_events = []
+
+ def __enter__(self):
+ if not self.refresh(throw=True):
+ raise RuntimeError(f"Failed to start session")
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ self.logger.debug("session exited")
+ if exc_val is not None:
+ intercepted = self.fail(exc_val)
+ self.destroy()
+ return intercepted
+ elif not self._destroyed:
+ self.save()
+ self.destroy()
+ return None
+
+ def fail(self, err: Optional[Exception]) -> bool:
+ if self._failed:
+ return False
+ self._failed = True
+ self.logger.exception("Session failed: %s", err)
+ if self.upstream is not None and self.upstream.alive():
+ message = MessageType.ERROR.new(content=str(err))
+ self.upstream.deliver(message)
+ return False
+
+ def destroy(self) -> None:
+ if self._destroyed:
+ return
+ self._destroyed = True
+ self.container.shutdown()
+ del self._alive_check
+ del self.container
+ del self._firing_events
+ del self.task
+ del self.thread
+ del self._fetched_task_briefs
+ del self.state
+ del self.ghost
+ del self.ghost_driver
+ del self.scope
diff --git a/ghostos/framework/ghostos/shell_impl.py b/ghostos/framework/ghostos/shell_impl.py
new file mode 100644
index 00000000..b7d9896a
--- /dev/null
+++ b/ghostos/framework/ghostos/shell_impl.py
@@ -0,0 +1,352 @@
+import time
+from typing import Union, Optional, Iterable, List, Tuple, TypeVar, Callable
+from ghostos.contracts.logger import LoggerItf, get_ghostos_logger
+from ghostos.contracts.pool import Pool, DefaultPool
+from ghostos.container import Container, Provider
+from ghostos.abcd import Shell, Conversation, Ghost, Scope, Background
+from ghostos.abcd.utils import get_ghost_driver
+from ghostos.core.messages import Message, Receiver, Role
+from ghostos.core.runtime import (
+ Event, GoProcess, EventBus,
+ GoTasks, TaskState, GoTaskStruct,
+)
+from ghostos.core.messages import Stream
+from ghostos.helpers import uuid, Timeleft, import_from_path
+from ghostos.identifier import get_identifier
+from ghostos.entity import to_entity_meta
+from threading import Lock
+from pydantic import BaseModel, Field
+from .conversation_impl import ConversationImpl, ConversationConf
+
+__all__ = ['ShellConf', 'ShellImpl', 'Shell']
+
+
+class ShellConf(BaseModel):
+ max_session_steps: int = Field(
+ default=10,
+ )
+ max_task_errors: int = Field(
+ default=3,
+ )
+ pool_size: int = 100
+ background_idle_time: float = Field(1)
+ task_lock_overdue: float = Field(
+ default=10.0
+ )
+ providers: List[str] = []
+
+
+G = TypeVar("G", bound=Ghost)
+
+
+class ShellImpl(Shell):
+
+ def __init__(
+ self,
+ config: ShellConf,
+ container: Container,
+ process: GoProcess,
+ providers: List[Provider],
+ ):
+ self._conversation_mutex = Lock()
+ self._conf = config
+ self._container = Container(parent=container, name="shell")
+ # prepare container
+ for provider in providers:
+ self._container.register(provider)
+ for provider_name in config.providers:
+ p = import_from_path(provider_name)
+ if isinstance(p, Provider):
+ self._container.register(p)
+ elif issubclass(p, Provider):
+ self._container.register(p())
+
+ self._shell_id = process.shell_id
+ self._process_id = process.process_id
+ self._scope = Scope(
+ shell_id=self._shell_id,
+ process_id=self._process_id,
+ task_id=self._process_id,
+ )
+ self._pool = DefaultPool(config.pool_size)
+ self._container.set(Pool, self._pool)
+ self._eventbus = self._container.force_fetch(EventBus)
+ self._tasks = self._container.force_fetch(GoTasks)
+ self._closed = False
+ self._background_started = False
+ # bootstrap the container.
+ # bind self
+ self._container.set(Shell, self)
+ self._container.set(ShellImpl, self)
+ self._container.set(ShellConf, config)
+ self._container.bootstrap()
+ self._conversations: List[Conversation] = []
+
+ @property
+ def logger(self) -> LoggerItf:
+ return get_ghostos_logger()
+
+ def container(self) -> Container:
+ return self._container
+
+ def send_event(self, event: Event) -> None:
+ task_id = event.task_id
+ task = self._tasks.get_task(task_id)
+ notify = True
+ if task:
+ notify = task.depth > 0
+ self._eventbus.send_event(event, notify)
+
+ def sync(
+ self,
+ ghost: Ghost,
+ context: Optional[Ghost.ContextType] = None,
+ *,
+ username: str = "",
+ user_role: str = Role.USER.value,
+ force: bool = False,
+ ) -> Conversation:
+ driver = get_ghost_driver(ghost)
+ task_id = driver.make_task_id(self._scope)
+ self.logger.debug("sync ghost with task id %s", task_id)
+
+ task = self._tasks.get_task(task_id)
+ if task is None:
+ task = self.create_root_task(task_id, ghost, context)
+ self.logger.debug("create root task task id %s for ghost", task_id)
+
+ task.meta = to_entity_meta(ghost)
+ if context is not None:
+ task.context = to_entity_meta(context)
+ conversation = self.sync_task(
+ task,
+ throw=True,
+ is_background=False,
+ username=username,
+ user_role=user_role,
+ force=force,
+ )
+ return conversation
+
+ def sync_task(
+ self,
+ task: GoTaskStruct,
+ *,
+ throw: bool,
+ is_background: bool,
+ username: str = "",
+ user_role: str = "",
+ force: bool = False,
+ ) -> Optional[Conversation]:
+ locker = self._tasks.lock_task(task.task_id, self._conf.task_lock_overdue, force)
+ if locker.acquire():
+ conf = ConversationConf(
+ max_session_steps=self._conf.max_session_steps,
+ max_task_errors=self._conf.max_task_errors,
+ )
+ self._tasks.save_task(task)
+ conversation = ConversationImpl(
+ conf=conf,
+ container=self._container,
+ task=task,
+ task_locker=locker,
+ is_background=is_background,
+ shell_closed=self.closed,
+ username=username,
+ user_role=user_role,
+ )
+ self._join_conversation(conversation)
+ return conversation
+ elif throw:
+ raise RuntimeError(f'create conversation failed, Task {task.task_id} already locked')
+ return None
+
+ def _join_conversation(self, conversation: Conversation):
+ with self._conversation_mutex:
+ exists = self._conversations
+ running = []
+ # remove closed ones
+ for c in exists:
+ if c.is_closed():
+ continue
+ running.append(c)
+ running.append(conversation)
+ self._conversations = running
+
+ def call(
+ self,
+ ghost: Ghost,
+ context: Optional[Ghost.ContextType] = None,
+ instructions: Optional[Iterable[Message]] = None,
+ timeout: float = 0.0,
+ stream: Optional[Stream] = None,
+ ) -> Tuple[Union[Ghost.ArtifactType, None], TaskState]:
+
+ def send_message(receiver: Receiver):
+ with receiver:
+ if stream:
+ stream.send(receiver.recv())
+ else:
+ receiver.wait()
+
+ timeleft = Timeleft(timeout)
+ task_id = uuid()
+ task = self.create_root_task(task_id, ghost, context)
+ conversation = self.sync_task(task, throw=True, is_background=False)
+ if conversation is None:
+ raise RuntimeError('create conversation failed')
+ with conversation:
+ e, r = conversation.respond(instructions)
+ send_message(r)
+
+ while timeleft.alive():
+ task = conversation.get_task()
+ if task.is_dead():
+ break
+ e = conversation.pop_event()
+ if e is not None:
+ r = conversation.respond_event(e)
+ send_message(r)
+ else:
+ conversation.talk("continue to fulfill your task or fail")
+ return conversation.get_artifact()
+
+ def create_root_task(
+ self,
+ task_id: str,
+ ghost: Ghost,
+ context: Optional[Ghost.ContextType],
+ ) -> GoTaskStruct:
+ id_ = get_identifier(ghost)
+ context_meta = to_entity_meta(context) if context else None
+ task = GoTaskStruct.new(
+ task_id=task_id,
+ shell_id=self._scope.shell_id,
+ process_id=self._scope.process_id,
+ depth=0,
+ name=id_.name,
+ description=id_.description,
+ meta=to_entity_meta(ghost),
+ context=context_meta,
+ parent_task_id=None,
+ )
+ self._tasks.save_task(task)
+ return task
+
+ def run_background_event(
+ self,
+ background: Optional[Background] = None,
+ ) -> Union[Event, None]:
+ self._validate_closed()
+ task_id = self._eventbus.pop_task_notification()
+ if task_id is None:
+ return None
+
+ task = self._tasks.get_task(task_id)
+ if task is None:
+ self._eventbus.clear_task(task_id)
+ return None
+
+ conversation = self.sync_task(task, throw=False, is_background=True)
+ if conversation is None:
+ return None
+
+ def on_event(e: Event, r: Receiver) -> None:
+ if background:
+ messages = r.wait()
+ tails = []
+ for message in messages:
+ if message.is_complete():
+ tails.append(message)
+ background.on_event(e, tails)
+
+ with conversation:
+ event = conversation.pop_event()
+ if event is None:
+ return None
+ try:
+ receiver = conversation.respond_event(event)
+ with receiver:
+ on_event(event, receiver)
+ receiver.wait()
+ return event
+ except Exception as err:
+ if background:
+ intercepted = background.on_error(err)
+ if not intercepted:
+ raise
+ finally:
+ self._eventbus.notify_task(self._scope.task_id)
+
+ def submit(self, caller: Callable, *args, **kwargs):
+ pool = self.container().force_fetch(Pool)
+ pool.submit(caller, *args, **kwargs)
+
+ def background_run(self, worker: int = 4, background: Optional[Background] = None) -> None:
+ self._validate_closed()
+ if self._background_started:
+ raise RuntimeError(f'background run already started')
+
+ for i in range(worker):
+ self._pool.submit(self._run_background_worker, background)
+
+ def _run_background_worker(self, background: Optional[Background] = None):
+ def is_stopped() -> bool:
+ if self._closed:
+ return True
+ if background:
+ return not background.alive()
+ return False
+
+ def idle():
+ time.sleep(self._conf.background_idle_time)
+
+ def halt() -> int:
+ if background:
+ return background.halt()
+ return 0
+
+ while not is_stopped():
+ if halt_time := halt():
+ time.sleep(halt_time)
+ continue
+ try:
+ handled_event = self.run_background_event(background)
+ if handled_event:
+ continue
+ except Exception as err:
+ self.logger.exception(err)
+ if background and not background.on_error(err):
+ self.logger.info("stop shell due to background not catch error")
+ break
+ idle()
+ self.logger.info("shut down background worker")
+ self.close()
+
+ def _validate_closed(self):
+ if self._closed:
+ raise RuntimeError(f'Shell is closed')
+
+ def closed(self) -> bool:
+ return self._closed
+
+ def close(self):
+ if self._closed:
+ return
+ self._closed = True
+ self.logger.info(
+ "start closing shell %s, conversations %d",
+ self._shell_id,
+ len(self._conversations)
+ )
+ for conversation in self._conversations:
+ if conversation.is_closed():
+ continue
+ self.logger.info("closing shell conversation %s", conversation.task_id)
+ conversation.close()
+ self.logger.info("shell conversations are closed")
+ self._container.shutdown()
+ self.logger.info("shell container destroyed")
+ self.logger.info("shutting down shell pool")
+ self._pool.shutdown(cancel_futures=True)
+ self.logger.info("shell pool is shut")
diff --git a/ghostos/framework/ghostos/subtasks_impl.py b/ghostos/framework/ghostos/subtasks_impl.py
new file mode 100644
index 00000000..be783919
--- /dev/null
+++ b/ghostos/framework/ghostos/subtasks_impl.py
@@ -0,0 +1,150 @@
+from typing import Optional, Dict, List
+
+from ghostos.container import Container
+from ghostos.abcd import Subtasks, Session, Ghost
+from ghostos.abcd import get_ghost_driver
+from ghostos.core.runtime import GoTaskStruct, GoTasks, EventTypes, TaskBrief, TaskState
+from ghostos.identifier import get_identifier
+from ghostos.helpers import yaml_pretty_dump
+from ghostos.entity import to_entity_meta
+
+
+class SubtasksImpl(Subtasks):
+
+ def __init__(self, session: Session, max_subtasks: int = 20):
+ self.session = session
+ self.max_shown_subtasks = max_subtasks
+
+ def get_subtasks(self) -> Dict[str, TaskBrief]:
+ children = self.session.task.children
+ if len(children) == 0:
+ return {}
+ tasks = self.session.get_task_briefs(*children)
+ return {t.name: t for t in tasks.values()}
+
+ def cancel(self, name: str, reason: str = "") -> None:
+ subtasks = self.get_subtasks()
+ if name not in subtasks:
+ raise NameError(f"Subtask {name} does not exist")
+ subtask_brief = subtasks[name]
+ task_id = subtask_brief.task_id
+ self_task = self.session.task
+ event = EventTypes.CANCEL.new(
+ task_id=task_id,
+ reason=reason,
+ messages=[],
+ from_task_id=self_task.task_id,
+ from_task_name=self_task.name,
+ )
+ self.session.fire_events(event)
+
+ def send(
+ self,
+ name: str,
+ *messages: Subtasks.MessageKind,
+ ctx: Optional[Ghost.ContextType] = None,
+ ) -> None:
+ subtasks = self.get_subtasks()
+ if name not in subtasks:
+ raise NameError(f"Subtask {name} does not exist")
+ subtask_brief = subtasks[name]
+ task_id = subtask_brief.task_id
+ event_messages = self.session.to_messages(messages)
+ self_task = self.session.task
+ event = EventTypes.INPUT.new(
+ task_id=task_id,
+ messages=event_messages,
+ from_task_id=self_task.task_id,
+ from_task_name=self_task.name,
+ )
+ self.session.fire_events(event)
+
+ def create(
+ self,
+ ghost: Ghost,
+ instruction: str = "",
+ ctx: Optional[Ghost.ContextType] = None,
+ task_name: Optional[str] = None,
+ task_description: Optional[str] = None,
+ ) -> None:
+ driver = get_ghost_driver(ghost)
+ task_id = driver.make_task_id(self.session.scope)
+ tasks = self.session.container.force_fetch(GoTasks)
+ task = tasks.get_task(task_id)
+ self_task = self.session.task
+ if not task:
+ identifier = get_identifier(ghost)
+ task_name = task_name or identifier.name
+ task_description = task_description or identifier.description
+ context_meta = to_entity_meta(ctx) if ctx is not None else None
+ task = GoTaskStruct.new(
+ task_id=task_id,
+ shell_id=self_task.shell_id,
+ process_id=self_task.process_id,
+ depth=self_task.depth + 1,
+ name=task_name,
+ description=task_description,
+ meta=to_entity_meta(ghost),
+ context=context_meta,
+ parent_task_id=self_task.task_id,
+ )
+ self.session.create_tasks(task)
+ event = EventTypes.CREATED.new(
+ task_id=task_id,
+ messages=[],
+ reason=f"receive task from parent task {self_task.name}",
+ from_task_id=self_task.task_id,
+ from_task_name=self_task.name,
+ instruction=instruction,
+ )
+ self.session.fire_events(event)
+
+ def self_prompt(self, container: Container) -> str:
+ subtasks = self.get_subtasks()
+ total = len(subtasks)
+ if total == 0:
+ return "There are no subtasks yet. You can create any by Subtasks lib if you need"
+
+ tasks = subtasks.values()
+ sorted_tasks = sort_tasks(list(tasks))
+ prior_tasks = sorted_tasks[:5]
+ tasks = sorted_tasks[5:]
+ wait_tasks = []
+ dead_tasks = []
+ other_tasks = []
+ for subtask in tasks:
+ if subtask.state == TaskState.WAITING.value:
+ wait_tasks.append(subtask)
+ elif TaskState.is_dead(subtask.state):
+ dead_tasks.append(subtask)
+ else:
+ other_tasks.append(subtask)
+ wait_tasks = sort_tasks(wait_tasks)
+ dead_tasks = sort_tasks(dead_tasks)
+ other_tasks = sort_tasks(other_tasks)
+
+ blocks = []
+ count = 0
+ for order_tasks in [prior_tasks, wait_tasks, other_tasks, dead_tasks]:
+ for task in order_tasks:
+ if count > self.max_shown_subtasks:
+ break
+ blocks.append(task.model_dump(exclude={"task_id"}, exclude_defaults=True))
+ count += 1
+
+ shown_tasks = len(blocks)
+ tasks_content = yaml_pretty_dump(shown_tasks)
+ return f"""
+There are {total} subtasks, first {shown_tasks} tasks brief are:
+
+```yaml
+{tasks_content.strip()}
+```
+"""
+
+ def get_title(self) -> str:
+ return "Subtasks"
+
+
+def sort_tasks(tasks: List[TaskBrief]) -> List[TaskBrief]:
+ return sorted(tasks, key=lambda t: t.updated, reverse=True)
diff --git a/ghostos/framework/ghostos/taskflow_impl.py b/ghostos/framework/ghostos/taskflow_impl.py
new file mode 100644
index 00000000..66523167
--- /dev/null
+++ b/ghostos/framework/ghostos/taskflow_impl.py
@@ -0,0 +1,240 @@
+from __future__ import annotations
+
+from typing import Union, List, Self
+from abc import ABC
+
+from ghostos.container import Container
+from ghostos.abcd import Taskflow, Session, Operator
+from ghostos.abcd import fire_session_event
+from ghostos.core.runtime import TaskState, EventTypes, TaskBrief
+from ghostos.core.moss import Injection, MossRuntime
+from ghostos.core.messages import MessageKind, MessageKindParser, Message, Role, MessageType
+from pprint import pprint
+from contextlib import redirect_stdout
+from io import StringIO
+from ghostos.prompter import Prompter
+
+
+class TaskflowImpl(Taskflow, Prompter, Injection):
+
+ def __init__(self, session: Session, parser: MessageKindParser):
+ self.task = session.task
+ self.container = session.container
+ self.parser = parser
+ self._destroyed = False
+
+ def self_prompt(self, container: Container) -> str:
+ brief = TaskBrief.from_task(self.task)
+ return f"""
+You are handling a task `{brief.name}`:
+
+description: {brief.description}
+state: {brief.state}
+status_desc: {brief.status_desc}
+
+use Taskflow to change the task state if you need.
+"""
+
+ def get_title(self) -> str:
+ return "Taskflow"
+
+ def on_inject(self, runtime: MossRuntime, property_name: str) -> Self:
+ self.container = runtime.container()
+ return self
+
+ def on_destroy(self) -> None:
+ self.destroy()
+
+ def finish(self, status: str = "", *replies: MessageKind) -> Operator:
+ messages = self.parser.parse(replies)
+ return FinishOperator(status, list(messages))
+
+ def fail(self, reason: str = "", *replies: MessageKind) -> Operator:
+ messages = self.parser.parse(replies)
+ return FailOperator(reason, list(messages))
+
+ def wait(self, status: str = "", *replies: MessageKind) -> Operator:
+ messages = self.parser.parse(replies)
+ return WaitOperator(status, list(messages))
+
+ def think(self, *messages: MessageKind, instruction: str = "", sync: bool = False) -> Operator:
+ messages = self.parser.parse(messages)
+ return RotateOperator(list(messages), instruction, sync)
+
+ def observe(self, **kwargs) -> Operator:
+ task = self.task
+ observation = f"## observation on turn {task.turns}\n"
+ for key, value in kwargs.items():
+ observation += f"\n### `{key}`\n"
+ if isinstance(value, Prompter):
+ content = value.get_prompt(self.container, depth=3)
+ else:
+ buffer = StringIO()
+ with redirect_stdout(buffer):
+ pprint(value)
+ content = str(buffer.getvalue())
+ observation += f"\n```\n{content}\n```"
+ message = Role.SYSTEM.new(content="", memory=observation)
+ return RotateOperator(
+ messages=[message],
+ instruction="",
+ sync=False,
+ )
+
+ def error(self, *messages: MessageKind) -> Operator:
+ messages = self.parser.parse(messages)
+ return ErrorOperator(list(messages))
+
+ def destroy(self):
+ if self._destroyed:
+ return
+ self._destroyed = True
+ del self.container
+ del self.parser
+ del self.task
+
+
+class AbcOperator(Operator, ABC):
+
+ def __init__(
+ self,
+ status: str,
+ messages: List[Message],
+ ):
+ self.status = status
+ self.messages = messages
+
+ def destroy(self):
+ del self.messages
+
+
+class ErrorOperator(Operator):
+
+ def __init__(self, messages: List[Message]):
+ self.messages = messages
+
+ def run(self, session: Session) -> Union[Operator, None]:
+ task = session.task
+ event = EventTypes.ERROR.new(
+ task_id=task.task_id,
+ messages=self.messages,
+ from_task_id=task.task_id,
+ from_task_name=task.name,
+ )
+ session.fire_events(event)
+ return None
+
+ def destroy(self):
+ del self.messages
+
+
+class RotateOperator(Operator):
+
+ def __init__(self, messages: List[Message], instruction: str, sync: bool):
+ self.messages = messages
+ self.instruction = instruction
+ self.sync: bool = sync
+
+ def run(self, session: Session) -> Union[Operator, None]:
+ task = session.task
+ event = EventTypes.ROTATE.new(
+ task_id=task.task_id,
+ messages=self.messages,
+ from_task_id=task.task_id,
+ from_task_name=task.name,
+ instruction=self.instruction,
+ )
+ if self.sync:
+ return fire_session_event(session, event)
+ else:
+ # msg = Role.SYSTEM.new(content=f"issue observation at turn {task.turns}")
+ # session.thread.append(msg)
+ event.reason = f"receive observation at turn {task.turns}"
+ session.fire_events(event)
+ session.task.state = TaskState.WAITING.value
+ return None
+
+ def destroy(self):
+ del self.messages
+
+
+class FailOperator(Operator):
+ def __init__(
+ self,
+ reason: str,
+ messages: List[Message],
+ ):
+ self.reason = reason
+ self.messages = messages
+
+ def run(self, session: Session) -> Union[Operator, None]:
+ task = session.task
+ session.task.state = TaskState.FAILED.value
+ session.task.status_desc = f"[FAILED] {self.reason}"
+ if task.parent:
+ event = EventTypes.FAILURE_CALLBACK.new(
+ task_id=task.parent,
+ messages=self.messages,
+ from_task_id=task.task_id,
+ from_task_name=task.name,
+ reason=f"task {task.name} is failed: {self.reason}",
+ )
+ session.fire_events(event)
+ messages = []
+ if self.reason:
+ messages = [Role.SYSTEM.new(content=self.reason)]
+ if self.messages:
+ messages.extend(self.messages)
+ if messages:
+ session.respond(messages)
+ return None
+
+ def destroy(self):
+ del self.messages
+
+
+class WaitOperator(AbcOperator, ABC):
+
+ def run(self, session: Session) -> Union[Operator, None]:
+ task = session.task
+ task.state = TaskState.WAITING.value
+ if len(self.messages) > 0:
+ task.status_desc = self.status
+ if task.parent:
+ event = EventTypes.WAIT_CALLBACK.new(
+ task_id=task.parent,
+ messages=self.messages,
+ from_task_id=task.task_id,
+ from_task_name=task.name,
+ )
+ session.fire_events(event)
+ else:
+ session.respond(self.messages)
+ return None
+
+
+class FinishOperator(AbcOperator):
+
+ def run(self, session: Session) -> Union[Operator, None]:
+ task = session.task
+ session.task.state = TaskState.FINISHED.value
+ session.task.status_desc = self.status
+ messages = list(self.messages)
+ artifact = session.get_artifact()
+ if artifact is not None:
+ # send artifact
+ artifact_message = session.to_messages([artifact])
+ messages.extend(artifact_message)
+
+ if task.parent:
+ event = EventTypes.FINISH_CALLBACK.new(
+ task_id=task.parent,
+ messages=messages,
+ from_task_id=task.task_id,
+ from_task_name=task.name,
+ reason=f"task {task.name} is finished."
+ )
+ session.fire_events(event)
+ elif self.messages:
+ session.respond(self.messages)
+ return None
diff --git a/ghostos/framework/ghosts/__init__.py b/ghostos/framework/ghosts/__init__.py
deleted file mode 100644
index 4a47d6d1..00000000
--- a/ghostos/framework/ghosts/__init__.py
+++ /dev/null
@@ -1,2 +0,0 @@
-from ghostos.framework.ghosts.basic import BasicGhost, InputsPipe
-from ghostos.framework.ghosts.demo import DemoGhost, DemoGhostConf
diff --git a/ghostos/framework/ghosts/basic.py b/ghostos/framework/ghosts/basic.py
deleted file mode 100644
index bcb17b90..00000000
--- a/ghostos/framework/ghosts/basic.py
+++ /dev/null
@@ -1,384 +0,0 @@
-from typing import Optional, Tuple, List, Dict, Type, ClassVar
-from abc import ABC, abstractmethod
-
-from ghostos.container import provide
-from ghostos.contracts.storage import Storage
-from ghostos.contracts.logger import LoggerItf
-from ghostos.contracts.shutdown import Shutdown
-from ghostos.contracts.pool import Pool
-from ghostos.core.ghosts import (
- Ghost, GhostConf, Operator, Inputs, Shell, Mindset, Thought,
- MultiTask, Taskflow, Utils, Workspace
-)
-from ghostos.core.ghosts.schedulers import Replier
-from ghostos.core.llms import LLMs
-from ghostos.core.moss import MossCompiler
-from ghostos.core.messages import Caller
-from ghostos.core.session import (
- Session, Event, DefaultEventType,
- EventBus, Tasks, Processes, Threads, Messenger,
- Process, Task, MsgThread,
-)
-from ghostos.framework.operators import OnEventOperator
-from ghostos.framework.multitasks import MultiTaskBasicImpl
-from ghostos.framework.taskflow import TaskflowBasicImpl
-from ghostos.framework.repliers import ReplierImpl
-from ghostos.framework.session import BasicSession
-from ghostos.core.moss.impl import MossCompilerImpl
-from ghostos.contracts.modules import Modules
-from ghostos.core.messages import Stream
-from ghostos.framework.mindsets import WorkspaceMindsetProvider
-from ghostos.framework.configs import Configs
-from ghostos.container import Container, Provider
-from ghostos.entity import EntityFactory
-
-__all__ = ['InputsPipe', 'BasicGhost']
-
-
-class InputsPipe:
- def __init__(self, ghost: Ghost):
- self.ghost = ghost
-
- @abstractmethod
- def intercept(self, inputs: Inputs) -> Optional[Inputs]:
- pass
-
-
-class BasicGhost(Ghost, ABC):
- """
- Basic ghost implementation.
- """
-
- inputs_pipes: List[Type[InputsPipe]] = []
- """inputs pipes that can intercept inputs"""
-
- providers: List[Provider] = []
- """ providers that ghost container shall register"""
-
- depend_contracts: ClassVar[List[Type]] = [
- Modules,
- LoggerItf,
- Storage,
- Configs,
- EventBus,
- Processes,
- Tasks,
- Threads,
- Pool,
- LLMs,
- Shutdown,
- ]
-
- ghost_contracts: ClassVar[List[Type]] = [
- Session,
- Shell,
- Ghost,
- Mindset,
- EntityFactory,
- MultiTask,
- Taskflow,
- Replier,
- Workspace,
- MossCompiler,
- Utils,
- Messenger,
- EntityFactory,
- ]
- """default contracts that ghost container shall validate before start."""
-
- def __init__(
- self, *,
- conf: GhostConf,
- container: Container,
- shell: Shell,
- workspace: Workspace,
- entity_factory: EntityFactory,
- upstream: Stream,
- process: Process,
- max_operator_runs: int,
- task: Optional[Task] = None,
- task_id: Optional[str] = None,
- ):
- # init ghost container, validate it first
- self._validate_parent_container_contracts(container)
- container = Container(parent=container)
- self._container = container
- # other properties
- self._origin_event: Optional[Event] = None
- # workspace
- self._workspace = workspace
- # config
- self._conf = conf
- self._max_operator_runs = max_operator_runs
- # init shell.
- self._shell = shell
- # entity factory
- self._entity_factory = entity_factory
- # root thought
- root_thought_meta = conf.root_thought_meta()
- root_thought = entity_factory.force_new_entity(root_thought_meta, Thought)
- self._root_thought = root_thought
- logger = container.force_fetch(LoggerItf)
- # instance session.
- self._session = self.make_session(
- upstream=upstream,
- root_thought=root_thought,
- process=process,
- task=task,
- task_id=task_id,
- logger=logger,
- )
- # prepare ghost logger
- trace = self.trace()
- ghost_logger = logger.with_trace(trace)
- self._logger = ghost_logger
- # 初始化 container 的相关绑定.
- self._bootstrap_ghost_container()
- # 检查所有必须绑定的对象.
- self._validate_default_contracts()
-
- def _bootstrap_ghost_container(self):
- # init shell
- # storage provider
- container = self._container
- # init mindset
- if not container.bound(Mindset):
- mindset_provider = WorkspaceMindsetProvider()
- container.register(mindset_provider)
-
- self._container.set(Ghost, self)
- self._container.set(Shell, self._shell)
- self._container.set(Session, self._session)
- self._container.set(LoggerItf, self._logger)
- self._container.set(Workspace, self._workspace)
- self._container.set(EntityFactory, self._entity_factory)
-
- # register ghost self modules.
- ghost_function_providers = {
- MultiTask: self.multitasks,
- Taskflow: self.taskflow,
- Replier: self.replier,
- MossCompiler: self.moss,
- Utils: self.utils,
- }
- for contract, maker in ghost_function_providers.items():
- _maker = maker
- self._container.register_maker(contract, _maker)
-
- # register session drivers:
- session_function_providers = {
- Tasks: self._session.tasks,
- Processes: self._session.processes,
- Messenger: self._session.messenger,
- Threads: self._session.threads,
- EventBus: self._session.eventbus,
- }
- for contract, maker in session_function_providers.items():
- _maker = maker
- self._container.register_maker(contract, _maker)
-
- # register shell drivers
- for driver in self._shell.drivers():
- _driver = driver
-
- def maker(c):
- return self._shell.get_driver(_driver)
-
- provider = provide(driver, False)(maker)
- self._container.register(provider)
-
- # register ghost providers
- for provider in self.providers:
- self._container.register(provider)
- self._container.bootstrap()
-
- def make_session(
- self,
- logger: LoggerItf,
- upstream: Stream,
- process: Process,
- root_thought: Thought,
- task: Optional[Task] = None,
- task_id: Optional[str] = None,
- ) -> Session:
- container = self.container()
- identifier = self.conf().identifier()
- processes = container.force_fetch(Processes)
- tasks = container.force_fetch(Tasks)
- threads = container.force_fetch(Threads)
- pool = container.force_fetch(Pool)
- eventbus = container.force_fetch(EventBus)
- # task and thread init.
- if task is None:
- if task_id is not None:
- task = tasks.get_task(task_id, True)
- if not task:
- raise RuntimeError(f"Task {task_id} not found")
- else:
- task_id = process.main_task_id
- task = tasks.get_task(task_id, False)
- if not task:
- identifier = root_thought.identifier()
- meta = root_thought.to_entity_meta()
- task = Task.new(
- task_id=task_id,
- session_id=process.session_id,
- process_id=process.process_id,
- name=identifier.name,
- description=identifier.description,
- meta=meta,
- )
- # 生成锁.
- task.lock = tasks.refresh_task_lock(task.task_id, "")
- logger = logger.with_trace({
- "process_id": process.process_id,
- "session_id": process.session_id,
- "task_id": task.task_id,
- "thread_id": task.thread_id,
- })
- thread = threads.get_thread(task.thread_id)
- if thread is None:
- thread = MsgThread.new(None, thread_id=task.thread_id)
- return BasicSession(
- ghost_name=identifier.name,
- ghost_role=self.role(),
- upstream=upstream,
- eventbus=eventbus,
- pool=pool,
- processes=processes,
- tasks=tasks,
- threads=threads,
- logger=logger,
- process=process,
- task=task,
- thread=thread,
- )
-
- @abstractmethod
- def meta_prompt(self) -> str:
- pass
-
- def mindset(self) -> "Mindset":
- return self._container.force_fetch(Mindset)
-
- def modules(self) -> "Modules":
- return self._container.force_fetch(Modules)
-
- def workspace(self) -> Workspace:
- return self._workspace
-
- def configs(self) -> Configs:
- return self._container.force_fetch(Configs)
-
- def entity_factory(self) -> EntityFactory:
- return self._entity_factory
-
- def _validate_default_contracts(self):
- for contract in self.ghost_contracts:
- if not self._container.bound(contract):
- raise NotImplementedError(f"Contract {contract} not bound to ghost container")
-
- @classmethod
- def _validate_parent_container_contracts(cls, container: Container):
- for contract in cls.depend_contracts:
- if not container.bound(contract):
- raise NotImplementedError(f"Contract {contract} not bound to the container")
-
- def on_inputs(self, inputs: Inputs) -> Optional["Event"]:
- for pipe_type in self.inputs_pipes:
- pipe = pipe_type(self)
- inputs = pipe.intercept(inputs)
- if inputs is None:
- return None
- task = self.session().task()
- if task.is_new():
- event = DefaultEventType.CREATED.new(
- task_id=self.session().task().task_id,
- messages=inputs.messages,
- )
- else:
- event = DefaultEventType.INPUT.new(
- task_id=self.session().task().task_id,
- messages=inputs.messages,
- )
- return event
-
- def init_operator(self, event: "Event") -> Tuple["Operator", int]:
- # set origin event
- self._origin_event = event
- return OnEventOperator(event), self._max_operator_runs
-
- def init_event(self) -> Optional["Event"]:
- return self._origin_event
-
- def container(self) -> Container:
- return self._container
-
- def session(self) -> Session:
- return self._session
-
- def shell(self) -> "Shell":
- return self._shell
-
- def root_thought(self) -> "Thought":
- return self._root_thought
-
- def logger(self) -> "LoggerItf":
- return self._logger
-
- def llms(self) -> LLMs:
- return self._container.force_fetch(LLMs)
-
- def multitasks(self) -> "MultiTask":
- return MultiTaskBasicImpl(self)
-
- def taskflow(self) -> "Taskflow":
- return TaskflowBasicImpl()
-
- def replier(self) -> "Replier":
- e = self.init_event()
- event_from_task = e.from_task_id if e else None
- task = self.session().task()
- return ReplierImpl(task, event_from_task)
-
- def moss(self) -> "MossCompiler":
- return MossCompilerImpl(container=self._container)
-
- def utils(self) -> "Utils":
- return Utils(self)
-
- def trace(self) -> Dict[str, str]:
- return self._make_trace(self._session, self._shell)
-
- def _make_trace(self, session: Session, shell: Shell) -> Dict:
- session_id = session.id()
- process_id = session.process().process_id
- task_id = session.task().task_id
- identifier = self.conf().identifier()
- return {
- "ghost_id": identifier.id,
- "ghost_name": identifier.name,
- "shell_id": shell.id(),
- "session_id": session_id,
- "process_id": process_id,
- "task_id": task_id,
- }
-
- def save(self) -> None:
- self._logger.info(f"save ghost")
- self._session.save()
-
- def done(self) -> None:
- self._logger.info(f"ghost is done")
- self._session.save()
- self._session.done()
-
- def fail(self, err: Optional[Exception]) -> None:
- self._logger.error(f"ghost run failed: {err}")
-
- def destroy(self) -> None:
- self._container.destroy()
- del self._container
- del self._session
- del self._logger
- del self._shell
diff --git a/ghostos/framework/ghosts/demo.py b/ghostos/framework/ghosts/demo.py
deleted file mode 100644
index 678fca46..00000000
--- a/ghostos/framework/ghosts/demo.py
+++ /dev/null
@@ -1,103 +0,0 @@
-from typing import Optional, List
-from ghostos.abc import Identifier
-from ghostos.core.ghosts import GhostConf, Shell, Workspace
-from ghostos.core.session import Process, Task
-from ghostos.contracts.modules import Modules
-from ghostos.core.messages import Stream
-from ghostos.framework.ghosts.basic import BasicGhost, InputsPipe
-from ghostos.framework.streams import EmptyStream
-from ghostos.framework.shells import EmptyShell
-from ghostos.container import Container, Provider
-from ghostos.entity import EntityMeta, EntityFactory
-from ghostos.helpers import import_from_path
-from pydantic import Field
-
-__all__ = ['DemoGhost', 'DemoGhostConf']
-
-
-class DemoGhostConf(GhostConf):
- """
- configration of simple ghost implementation
- """
-
- id: str = Field(description="id of the ghost")
- name: str = Field(description="name of the ghost")
- description: str = Field(default="", description="description of the ghost")
-
- # prompt
- meta_prompt: str = Field(description="raw meta prompt")
-
- # meta
- thought_meta: EntityMeta = Field(description="root thought meta entity")
-
- # importing
- input_pipes: List[str] = Field(default_factory=list, description="import path for input pipes")
- providers: List[str] = Field(default_factory=list, description="import path for providers")
-
- # system conf
- max_operators_run: int = Field(default=10, description="max operators run")
-
- def identifier(self) -> Identifier:
- return Identifier(
- id=self.id,
- name=self.name,
- description=self.description,
- )
-
- def root_thought_meta(self) -> EntityMeta:
- return self.thought_meta
-
-
-class DemoGhost(BasicGhost):
- """
- simple implementation of a ghost
- """
-
- def __init__(
- self,
- conf: DemoGhostConf,
- container: Container,
- entity_factory: EntityFactory,
- workspace: Workspace,
- process: Process,
- upstream: Optional[Stream] = None,
- shell: Optional[Shell] = None,
- task: Optional[Task] = None,
- task_id: Optional[str] = None,
- ):
- self._conf = conf
- shell = shell if shell is None else EmptyShell()
- upstream = upstream if upstream else EmptyStream()
- modules = container.force_fetch(Modules)
-
- # importing
- for provider_path in conf.providers:
- provider = import_from_path(provider_path, modules.import_module)
- if not isinstance(provider, Provider):
- raise ValueError(f"provider {provider_path} is not an instance of {Provider}")
- self.providers.append(provider)
-
- for input_pipe_path in conf.input_pipes:
- pipe = import_from_path(input_pipe_path, modules.import_module)
- if not issubclass(pipe, InputsPipe):
- raise ValueError(f"pipe {input_pipe_path} is not an subclass of {InputsPipe}")
- self.inputs_pipes.append(pipe)
-
- super().__init__(
- conf=conf,
- container=container,
- shell=shell,
- workspace=workspace,
- entity_factory=entity_factory,
- upstream=upstream,
- process=process,
- task=task,
- task_id=task_id,
- max_operator_runs=conf.max_operators_run,
- )
-
- def meta_prompt(self) -> str:
- return self._conf.meta_prompt
-
- def conf(self) -> DemoGhostConf:
- return self._conf
diff --git a/ghostos/framework/libraries/auto_memory.py b/ghostos/framework/libraries/auto_memory.py
deleted file mode 100644
index bb499de0..00000000
--- a/ghostos/framework/libraries/auto_memory.py
+++ /dev/null
@@ -1,141 +0,0 @@
-import os
-
-from abc import ABC, abstractmethod
-from typing import List, Dict
-
-from pydantic import BaseModel, Field
-
-
-class DBConfig(BaseModel):
- path: str = Field(description="Path to the database file", default="memory.db")
-
-
-class VectorDBConfig(BaseModel):
- provider: str = Field(description="Provider of the vector store (e.g., 'qdrant', 'chroma')", default="qdrant")
-
-
-class ProxyConfig(BaseModel):
- proxy_url: str = Field(description="Proxy URL", default=None)
-
-
-class TextMemory(ABC):
- """
- TextMemory Storage enhances AI assistants and agents with an intelligent memory layer, enabling personalized AI interactions
- """
-
- def __init__(self, proxy_config: ProxyConfig):
- if proxy_config and proxy_config.proxy_url:
- os.environ['http_proxy'] = proxy_config.proxy_url
- os.environ['https_proxy'] = proxy_config.proxy_url
-
- @abstractmethod
- def add(self, data: str, agent_id=None, run_id=None, metadata=None, filters=None, prompt=None):
- """
- Create a new memory.
-
- Args:
- data (str): Data to store in the memory.
- agent_id (str, optional): ID of the agent creating the memory. Defaults to None.
- run_id (str, optional): ID of the run creating the memory. Defaults to None.
- metadata (dict, optional): Metadata to store with the memory. Defaults to None.
- filters (dict, optional): Filters to apply to the search. Defaults to None.
- prompt (str, optional): Prompt to use for memory deduction. Defaults to None.
-
- Returns: None
- """
- pass
-
- @abstractmethod
- def search(self, query, agent_id=None, run_id=None, limit=100, filters=None) -> List[Dict]:
- """
- Search for memories.
-
- Args:
- query (str): Query to search for.
- agent_id (str, optional): ID of the agent to search for. Defaults to None.
- run_id (str, optional): ID of the run to search for. Defaults to None.
- limit (int, optional): Limit the number of results. Defaults to 100.
- filters (dict, optional): Filters to apply to the search. Defaults to None.
-
- Returns:
- list: List of search results.
- """
- pass
-
- @abstractmethod
- def get(self, memory_id):
- """
- Retrieve a memory by ID.
-
- Args:
- memory_id (str): ID of the memory to retrieve.
-
- Returns:
- dict: Retrieved memory.
- """
- pass
-
- @abstractmethod
- def get_all(self, agent_id=None, run_id=None, limit=100):
- """
- List all memories.
-
- Returns:
- list: List of all memories.
- """
- pass
-
- @abstractmethod
- def update(self, memory_id, data):
- """
- Update a memory by ID.
-
- Args:
- memory_id (str): ID of the memory to update.
- data (dict): Data to update the memory with.
-
- Returns:
- dict: Updated memory.
- """
- pass
-
- @abstractmethod
- def delete(self, memory_id):
- """
- Delete a memory by ID.
-
- Args:
- memory_id (str): ID of the memory to delete.
- """
- pass
-
- @abstractmethod
- def delete_all(self, agent_id=None, run_id=None):
- """
- Delete all memories.
-
- Args:
- agent_id (str, optional): ID of the agent to delete memories for. Defaults to None.
- run_id (str, optional): ID of the run to delete memories for. Defaults to None.
- """
- pass
-
- @abstractmethod
- def history(self, memory_id):
- """
- Get the history of changes for a memory by ID.
-
- Args:
- memory_id (str): ID of the memory to get history for.
-
- Returns:
- list: List of changes for the memory.
- """
- pass
-
- @abstractmethod
- def clear(self):
- """
- Clear the memory store.
- """
- pass
diff --git a/ghostos/framework/llms/__init__.py b/ghostos/framework/llms/__init__.py
index e3a4af97..d10ea522 100644
--- a/ghostos/framework/llms/__init__.py
+++ b/ghostos/framework/llms/__init__.py
@@ -1,9 +1,5 @@
+from ghostos.core.llms import LLMs, Prompt, PromptStorage
from ghostos.framework.llms.llms import LLMsImpl
from ghostos.framework.llms.openai_driver import OpenAIDriver, OpenAIAdapter, LitellmAdapter
-from ghostos.framework.llms.providers import ConfigBasedLLMsProvider
-
-
-default_llms_provider = ConfigBasedLLMsProvider("llms/llms_conf.yaml")
-"""default llms provider based by configs contract """
-
-
+from ghostos.framework.llms.providers import ConfigBasedLLMsProvider, PromptStorageInWorkspaceProvider, LLMsYamlConfig
+from ghostos.framework.llms.prompt_storage_impl import PromptStorageImpl
diff --git a/ghostos/framework/llms/llms.py b/ghostos/framework/llms/llms.py
index e2fcbe8c..05b26962 100644
--- a/ghostos/framework/llms/llms.py
+++ b/ghostos/framework/llms/llms.py
@@ -23,7 +23,7 @@ def __init__(
self._llm_models: Dict[str, ModelConf] = {}
self._default_driver = default_driver
self._apis: Dict[str, LLMApi] = {}
- self._default_llm_model: ModelConf = conf.default
+ self._default_llm_model: ModelConf = conf.models[conf.default]
if self._default_llm_model is None:
raise AttributeError("llms conf must contains default model conf")
diff --git a/ghostos/framework/llms/openai_driver.py b/ghostos/framework/llms/openai_driver.py
index 2b56152c..ada8d4ea 100644
--- a/ghostos/framework/llms/openai_driver.py
+++ b/ghostos/framework/llms/openai_driver.py
@@ -1,28 +1,29 @@
-import os
from typing import List, Iterable, Union, Optional
-
from openai import OpenAI
from httpx import Client
from httpx_socks import SyncProxyTransport
from openai import NOT_GIVEN
from openai.types.chat import ChatCompletion
from openai.types.chat.chat_completion_stream_options_param import ChatCompletionStreamOptionsParam
+from openai.types.chat.chat_completion_message_param import ChatCompletionMessageParam
from openai.types.chat.chat_completion_chunk import ChatCompletionChunk
-
+from ghostos.contracts.logger import LoggerItf, get_ghostos_logger
+from ghostos.helpers import timestamp
from ghostos.core.messages import (
- Message, OpenAIMessageParser, DefaultOpenAIMessageParser, DefaultMessageTypes,
- CompletionUsagePayload,
+ Message, OpenAIMessageParser, DefaultOpenAIMessageParser,
+ CompletionUsagePayload, Role,
)
from ghostos.core.llms import (
- LLMs, LLMDriver, LLMApi, ModelConf, ServiceConf, OPENAI_DRIVER_NAME,
- Chat,
+ LLMs, LLMDriver, LLMApi, ModelConf, ServiceConf, OPENAI_DRIVER_NAME, LITELLM_DRIVER_NAME,
+ Prompt, PromptPayload, PromptStorage,
FunctionalToken,
)
from ghostos.container import Bootstrapper, Container
-import litellm
-
-__all__ = ['OpenAIDriver', 'OpenAIAdapter', 'OpenAIDriverBootstrapper', 'LitellmAdapter']
+__all__ = [
+ 'OpenAIDriver', 'OpenAIAdapter',
+ 'LitellmAdapter', 'LiteLLMDriver',
+]
class FunctionalTokenPrompt(str):
@@ -73,10 +74,15 @@ def __init__(
service_conf: ServiceConf,
model_conf: ModelConf,
parser: OpenAIMessageParser,
+ storage: PromptStorage,
+ logger: LoggerItf,
+ # deprecated:
functional_token_prompt: Optional[str] = None,
):
self._service = service_conf
self._model = model_conf
+ self._storage: PromptStorage = storage
+ self._logger = logger
http_client = None
if service_conf.proxy:
transport = SyncProxyTransport.from_url(service_conf.proxy)
@@ -102,80 +108,89 @@ def get_model(self) -> ModelConf:
def text_completion(self, prompt: str) -> str:
raise NotImplemented("text_completion is deprecated, implement it later")
- # def get_embeddings(self, texts: List[str]) -> Embeddings:
- # try:
- # model = self._model.model
- # resp = self._client.embeddings.create(
- # input=texts,
- # model=model,
- # # todo: 未来再做更多细节.
- # )
- # result = []
- # for i, text in enumerate(texts):
- # embedding = resp.embeddings[i]
- # result.append(Embedding(
- # text=text,
- # embedding=embedding
- # ))
- # return Embeddings(result=result)
- #
- # except Exception as e:
- # # todo: log
- # raise GhostOSIOError("failed to get text embedding", e)
-
- def _chat_completion(self, chat: Chat, stream: bool) -> Union[ChatCompletion, Iterable[ChatCompletionChunk]]:
- # todo: try catch
- chat = self.parse_chat(chat)
+ def parse_message_params(self, messages: List[Message]) -> List[ChatCompletionMessageParam]:
+ return list(self._parser.parse_message_list(messages, self._model.message_types))
+
+ def _chat_completion(self, prompt: Prompt, stream: bool) -> Union[ChatCompletion, Iterable[ChatCompletionChunk]]:
+ prompt = self.parse_prompt(prompt)
+ self._logger.info(f"start chat completion for prompt %s", prompt.id)
include_usage = ChatCompletionStreamOptionsParam(include_usage=True) if stream else NOT_GIVEN
- messages = chat.get_messages()
- messages = self._parser.parse_message_list(messages)
+ messages = prompt.get_messages()
+ messages = self.parse_message_params(messages)
if not messages:
raise AttributeError("empty chat!!")
- return self._client.chat.completions.create(
- messages=messages,
- model=self._model.model,
- function_call=chat.get_openai_function_call(),
- functions=chat.get_openai_functions(),
- tools=chat.get_openai_tools(),
- max_tokens=self._model.max_tokens,
- temperature=self._model.temperature,
- n=self._model.n,
- timeout=self._model.timeout,
- stream=stream,
- stream_options=include_usage,
- **self._model.kwargs,
- )
-
- def chat_completion(self, chat: Chat) -> Message:
- message: ChatCompletion = self._chat_completion(chat, stream=False)
- pack = self._parser.from_chat_completion(message.choices[0].message)
- # add completion usage
- if message.usage:
- usage = CompletionUsagePayload.from_usage(message.usage)
- usage.set(pack)
-
- if not pack.is_tail():
- pack.pack = False
- return pack
-
- def chat_completion_chunks(self, chat: Chat) -> Iterable[Message]:
- chunks: Iterable[ChatCompletionChunk] = self._chat_completion(chat, stream=True)
- messages = self._parser.from_chat_completion_chunks(chunks)
- first = True
- for chunk in messages:
- if first:
- self._model.set(chunk)
- first = False
- yield chunk
-
- def parse_chat(self, chat: Chat) -> Chat:
- if not chat.functional_tokens:
- return chat
- prompt = FunctionalTokenPrompt(self._functional_token_prompt)
- content = prompt.format_tokens(chat.functional_tokens)
- message = DefaultMessageTypes.DEFAULT.new_system(content=content)
- chat.system.append(message)
- return chat
+ try:
+ prompt.run_start = timestamp()
+ self._logger.debug(f"start chat completion messages %s", messages)
+ functions = prompt.get_openai_functions()
+ tools = prompt.get_openai_tools()
+ if self._model.use_tools:
+ functions = NOT_GIVEN
+ else:
+ tools = NOT_GIVEN
+ return self._client.chat.completions.create(
+ messages=messages,
+ model=self._model.model,
+ function_call=prompt.get_openai_function_call(),
+ functions=functions,
+ tools=tools,
+ max_tokens=self._model.max_tokens,
+ temperature=self._model.temperature,
+ n=self._model.n,
+ timeout=self._model.timeout,
+ stream=stream,
+ stream_options=include_usage,
+ **self._model.kwargs,
+ )
+ except Exception as e:
+ self._logger.error(f"error chat completion for prompt {prompt.id}: {e}")
+ raise
+ finally:
+ self._logger.debug(f"end chat completion for prompt {prompt.id}")
+ prompt.run_end = timestamp()
+
+ def chat_completion(self, prompt: Prompt) -> Message:
+ try:
+ message: ChatCompletion = self._chat_completion(prompt, stream=False)
+ prompt.added = [message]
+ pack = self._parser.from_chat_completion(message.choices[0].message)
+ # add completion usage
+ self._model.set_payload(pack)
+ if message.usage:
+ usage = CompletionUsagePayload.from_usage(message.usage)
+ usage.set_payload(pack)
+
+ if not pack.is_complete():
+ pack.chunk = False
+ return pack
+ except Exception as e:
+ prompt.error = str(e)
+ raise
+ finally:
+ self._storage.save(prompt)
+
+ def chat_completion_chunks(self, prompt: Prompt) -> Iterable[Message]:
+ try:
+ chunks: Iterable[ChatCompletionChunk] = self._chat_completion(prompt, stream=True)
+ messages = self._parser.from_chat_completion_chunks(chunks)
+ prompt_payload = PromptPayload.from_prompt(prompt)
+ output = []
+ for chunk in messages:
+ yield chunk
+ if chunk.is_complete():
+ self._model.set_payload(chunk)
+ prompt_payload.set_payload(chunk)
+ output.append(chunk)
+ prompt.added = output
+ except Exception as e:
+ prompt.error = str(e)
+ raise
+ finally:
+ self._storage.save(prompt)
+
+ def parse_prompt(self, prompt: Prompt) -> Prompt:
+ prompt.model = self._model
+ return prompt
class OpenAIDriver(LLMDriver):
@@ -183,20 +198,19 @@ class OpenAIDriver(LLMDriver):
adapter
"""
- def __init__(self, parser: Optional[OpenAIMessageParser] = None):
+ def __init__(self, storage: PromptStorage, logger: LoggerItf, parser: Optional[OpenAIMessageParser] = None):
if parser is None:
- parser = DefaultOpenAIMessageParser()
+ parser = DefaultOpenAIMessageParser(None, None)
+ self._logger = logger
self._parser = parser
+ self._storage = storage
def driver_name(self) -> str:
return OPENAI_DRIVER_NAME
def new(self, service: ServiceConf, model: ModelConf) -> LLMApi:
- # todo: 不能这么 hack.
- if service.name in ("anthropic", "deepseek"):
- return LitellmAdapter(service, model, self._parser)
-
- return OpenAIAdapter(service, model, self._parser)
+ get_ghostos_logger().debug(f"new llm api %s at service %s", model.model, service.name)
+ return OpenAIAdapter(service, model, self._parser, self._storage, self._logger)
class LitellmAdapter(OpenAIAdapter):
@@ -204,9 +218,10 @@ class LitellmAdapter(OpenAIAdapter):
adapter class wrap openai api to ghostos.blueprint.kernel.llms.LLMApi
"""
- def _chat_completion(self, chat: Chat, stream: bool) -> ChatCompletion:
+ def _chat_completion(self, chat: Prompt, stream: bool) -> ChatCompletion:
+ import litellm
messages = chat.get_messages()
- messages = self._parser.parse_message_list(messages)
+ messages = self.parse_message_params(messages)
response = litellm.completion(
model=self._model.model,
messages=list(messages),
@@ -219,9 +234,24 @@ def _chat_completion(self, chat: Chat, stream: bool) -> ChatCompletion:
)
return response.choices[0].message
+ def parse_message_params(self, messages: List[Message]) -> List[ChatCompletionMessageParam]:
+ parsed = super().parse_message_params(messages)
+ outputs = []
+ count = 0
+ for message in parsed:
+ # filter all the system message to __system__ user message.
+ if count > 0 and "role" in message and message["role"] == Role.SYSTEM.value:
+ message["role"] = Role.USER.value
+ message["name"] = "__system__"
+ outputs.append(message)
+ count += 1
+ return outputs
-class OpenAIDriverBootstrapper(Bootstrapper):
- def bootstrap(self, container: Container) -> None:
- llms = container.force_fetch(LLMs)
- llms.register_driver(OpenAIDriver())
+class LiteLLMDriver(OpenAIDriver):
+
+ def driver_name(self) -> str:
+ return LITELLM_DRIVER_NAME
+
+ def new(self, service: ServiceConf, model: ModelConf) -> LLMApi:
+ return LitellmAdapter(service, model, self._parser, self._storage, self._logger)
diff --git a/ghostos/framework/llms/prompt_storage_impl.py b/ghostos/framework/llms/prompt_storage_impl.py
new file mode 100644
index 00000000..92bea9a6
--- /dev/null
+++ b/ghostos/framework/llms/prompt_storage_impl.py
@@ -0,0 +1,32 @@
+from typing import Optional
+
+from ghostos.contracts.storage import Storage
+from ghostos.core.llms import Prompt
+from ghostos.core.llms.prompt import PromptStorage
+from ghostos.helpers import yaml_pretty_dump
+import yaml
+
+
+class PromptStorageImpl(PromptStorage):
+
+ def __init__(self, storage: Storage):
+ self._storage = storage
+
+ @staticmethod
+ def _get_filename(prompt_id: str) -> str:
+ filename = f"{prompt_id}.prompt.yml"
+ return filename
+
+ def save(self, prompt: Prompt) -> None:
+ data = prompt.model_dump(exclude_defaults=True)
+ filename = self._get_filename(prompt.id)
+ content = yaml_pretty_dump(data)
+ self._storage.put(filename, content.encode())
+
+ def get(self, prompt_id: str) -> Optional[Prompt]:
+ filename = self._get_filename(prompt_id)
+ if self._storage.exists(filename):
+ content = self._storage.get(filename)
+ data = yaml.safe_load(content)
+ return Prompt(**data)
+ return None
diff --git a/ghostos/framework/llms/providers.py b/ghostos/framework/llms/providers.py
index 1c33535e..2b19889e 100644
--- a/ghostos/framework/llms/providers.py
+++ b/ghostos/framework/llms/providers.py
@@ -1,12 +1,22 @@
from typing import Type, Optional
-
from ghostos.contracts.configs import YamlConfig, Configs
from ghostos.container import Provider, Container
-from ghostos.core.llms import LLMs, LLMsConfig
+from ghostos.core.llms import LLMs, LLMsConfig, PromptStorage
+from ghostos.core.messages.openai import OpenAIMessageParser
from ghostos.framework.llms.llms import LLMsImpl
-from ghostos.framework.llms.openai_driver import OpenAIDriver
+from ghostos.framework.llms.openai_driver import OpenAIDriver, LiteLLMDriver
+from ghostos.framework.llms.prompt_storage_impl import PromptStorageImpl
+from ghostos.contracts.workspace import Workspace
+from ghostos.contracts.logger import LoggerItf
+
+__all__ = ['ConfigBasedLLMsProvider', 'PromptStorageInWorkspaceProvider', 'LLMsYamlConfig']
-__all__ = ['ConfigBasedLLMsProvider']
+
+class LLMsYamlConfig(YamlConfig, LLMsConfig):
+ """
+ LLMs Service and Models configurations.
+ """
+ relative_path = "llms_conf.yml"
class ConfigBasedLLMsProvider(Provider[LLMs]):
@@ -14,9 +24,6 @@ class ConfigBasedLLMsProvider(Provider[LLMs]):
基于 Config 来读取
"""
- def __init__(self, llm_conf_path: str):
- self.llm_conf_path = llm_conf_path
-
def singleton(self) -> bool:
return True
@@ -24,16 +31,31 @@ def contract(self) -> Type[LLMs]:
return LLMs
def factory(self, con: Container) -> Optional[LLMs]:
- class LLMsYamlConfig(YamlConfig, LLMsConfig):
- """
- 配置项存储位置.
- 详细配置项见 LLMsConfig
- """
- relative_path = self.llm_conf_path
-
configs = con.force_fetch(Configs)
+ storage = con.force_fetch(PromptStorage)
+ parser = con.get(OpenAIMessageParser)
+ logger: LoggerItf = con.force_fetch(LoggerItf)
+
conf = configs.get(LLMsYamlConfig)
- openai_driver = OpenAIDriver()
+ openai_driver = OpenAIDriver(storage, logger, parser)
+ lite_llm_driver = LiteLLMDriver(storage, logger, parser)
+
+ # register default drivers.
llms = LLMsImpl(conf=conf, default_driver=openai_driver)
llms.register_driver(openai_driver)
+ llms.register_driver(lite_llm_driver)
+
return llms
+
+
+class PromptStorageInWorkspaceProvider(Provider[PromptStorage]):
+ def __init__(self, relative_path: str = "prompts"):
+ self._relative_path = relative_path
+
+ def singleton(self) -> bool:
+ return True
+
+ def factory(self, con: Container) -> Optional[PromptStorage]:
+ ws = con.force_fetch(Workspace)
+ storage = ws.runtime().sub_storage(self._relative_path)
+ return PromptStorageImpl(storage)
diff --git a/ghostos/framework/llms/test_case.py b/ghostos/framework/llms/test_case.py
index a9115c8d..9cbb0830 100644
--- a/ghostos/framework/llms/test_case.py
+++ b/ghostos/framework/llms/test_case.py
@@ -3,8 +3,8 @@
import datetime
from typing import List, Optional, Dict
from pydantic import BaseModel, Field
-from ghostos.core.llms import LLMs, Chat, ModelConf, ServiceConf
-from ghostos.core.messages import Message, DefaultMessageTypes
+from ghostos.core.llms import LLMs, Prompt, ModelConf, ServiceConf
+from ghostos.core.messages import Message, MessageType
# 测试用, 不直接对外开放.
@@ -21,7 +21,7 @@ class ChatCompletionTestResult(BaseModel):
class ChatCompletionTestCase(BaseModel):
- chat: Chat
+ chat: Prompt
apis: List[APIInfo]
results: List[ChatCompletionTestResult] = Field(default_factory=list)
@@ -39,7 +39,7 @@ def run_test_cases(cases: ChatCompletionTestCase, llms: LLMs) -> Dict[str, Messa
return result
-def run_test_case(api_info: APIInfo, chat: Chat, llms: LLMs, result: Dict[str, Message]) -> None:
+def run_test_case(api_info: APIInfo, chat: Prompt, llms: LLMs, result: Dict[str, Message]) -> None:
api = None
if api_info.api:
api = llms.get_api(api_info.api)
@@ -58,7 +58,7 @@ def run_test_case(api_info: APIInfo, chat: Chat, llms: LLMs, result: Dict[str, M
try:
message = api.chat_completion(chat)
except Exception as e:
- message = DefaultMessageTypes.ERROR.new(content=str(e))
+ message = MessageType.ERROR.new(content=str(e))
finally:
end = time.time()
duration = end - start
diff --git a/ghostos/framework/logger/__init__.py b/ghostos/framework/logger/__init__.py
index 362e3eed..10b4abe4 100644
--- a/ghostos/framework/logger/__init__.py
+++ b/ghostos/framework/logger/__init__.py
@@ -1,2 +1,2 @@
-from ghostos.framework.logger.named import NamedLoggerProvider
-from ghostos.framework.logger.fake import FakeLogger
+from ghostos.contracts.logger import LoggerItf, FakeLogger
+from ghostos.framework.logger.named import DefaultLoggerProvider
diff --git a/ghostos/framework/logger/fake.py b/ghostos/framework/logger/fake.py
deleted file mode 100644
index a7feeb11..00000000
--- a/ghostos/framework/logger/fake.py
+++ /dev/null
@@ -1,32 +0,0 @@
-from typing import Dict
-
-from ghostos.contracts.logger import LoggerItf
-
-
-class FakeLogger(LoggerItf):
- def debug(self, msg, *args, **kwargs):
- pass
-
- def info(self, msg, *args, **kwargs):
- pass
-
- def warning(self, msg, *args, **kwargs):
- pass
-
- def error(self, msg, *args, **kwargs):
- pass
-
- def exception(self, msg, *args, exc_info=True, **kwargs):
- pass
-
- def critical(self, msg, *args, **kwargs):
- pass
-
- def fatal(self, msg, *args, **kwargs):
- pass
-
- def log(self, level, msg, *args, **kwargs):
- pass
-
- def with_trace(self, trace: Dict) -> "LoggerItf":
- return self
diff --git a/ghostos/framework/logger/named.py b/ghostos/framework/logger/named.py
index a978da7a..f25d4f14 100644
--- a/ghostos/framework/logger/named.py
+++ b/ghostos/framework/logger/named.py
@@ -1,32 +1,45 @@
from typing import Optional, Type
from ghostos.container import Provider, Container
-from ghostos.contracts.logger import LoggerItf, LoggerWrapper
+from ghostos.contracts.logger import LoggerItf, get_ghostos_logger
+from ghostos.contracts.workspace import Workspace
from os.path import join
import logging
+from logging.handlers import TimedRotatingFileHandler
-__all__ = ['NamedLoggerProvider']
+__all__ = ['DefaultLoggerProvider']
-class NamedLoggerProvider(Provider[LoggerItf]):
+class DefaultLoggerProvider(Provider[LoggerItf]):
"""
basic logger
+ todo: make logger configurable
"""
- def __init__(
- self,
- logger_name: str = "ghostos",
- ):
- self.logger_name = logger_name
-
def singleton(self) -> bool:
- return True
+ return False
def contract(self) -> Type[LoggerItf]:
return LoggerItf
def factory(self, con: Container) -> Optional[LoggerItf]:
logging.captureWarnings(True)
- origin = logging.getLogger(self.logger_name)
- adapter = LoggerWrapper(origin)
- return adapter
+ ws = con.force_fetch(Workspace)
+ logger = get_ghostos_logger()
+ if not logger.hasHandlers():
+ path = ws.runtime().abspath()
+ logfile = join(path, "logs/ghostos.log")
+ handler = TimedRotatingFileHandler(
+ logfile,
+ when="midnight",
+ interval=1,
+ backupCount=10
+ )
+ handler.setLevel(logging.DEBUG)
+ formatter = logging.Formatter(
+ fmt="%(asctime)s - %(levelname)s - %(message)s (%(filename)s:%(lineno)d)",
+ )
+ handler.setFormatter(formatter)
+ logger.addHandler(handler)
+ logger.setLevel(logging.DEBUG)
+ return logger
diff --git a/ghostos/framework/messages/__init__.py b/ghostos/framework/messages/__init__.py
index d09cea48..573407dd 100644
--- a/ghostos/framework/messages/__init__.py
+++ b/ghostos/framework/messages/__init__.py
@@ -1,6 +1,9 @@
+# todo: remove buffer some day
+from ghostos.core.messages import Buffer
from ghostos.framework.messages.buffers import DefaultBuffer
# default payloads
-from ghostos.core.session import TaskPayload
+from ghostos.core.runtime import TaskPayload
from ghostos.core.messages.openai import CompletionUsagePayload
+from ghostos.core.llms import PromptPayload
diff --git a/ghostos/framework/messages/buffers.py b/ghostos/framework/messages/buffers.py
index 323db150..c373b4db 100644
--- a/ghostos/framework/messages/buffers.py
+++ b/ghostos/framework/messages/buffers.py
@@ -1,13 +1,14 @@
import time
from typing import Iterable, Optional, List, Dict, Set
-from ghostos.core.messages import Message, Caller, DefaultMessageTypes, Role, Payload, Attachment, Buffer, Flushed
+from ghostos.core.messages import Message, FunctionCaller, MessageType, Role, Payload, Buffer, Flushed
from ghostos.core.llms import FunctionalToken
from ghostos.helpers import uuid
__all__ = ['DefaultBuffer']
+# deprecated
class DefaultBuffer(Buffer):
"""
基于 Message 标准协议的默认 buffer.
@@ -19,7 +20,6 @@ def __init__(
name: Optional[str] = None,
role: str = Role.ASSISTANT.value,
payloads: Optional[Iterable[Payload]] = None,
- attachments: Optional[Iterable[Attachment]] = None,
functional_tokens: Optional[Iterable[FunctionalToken]] = None,
):
self._default_name = name
@@ -28,15 +28,11 @@ def __init__(
"""默认的角色"""
self._payloads = list(payloads) if payloads else None
"""默认的 payloads"""
- self._attachments = list(attachments) if attachments else None
- """默认的 attachments"""
self._buffering_message: Optional[Message] = None
"""正在 buff 的消息体. """
self._buffed_messages: List[Message] = []
- """发送出去的完整消息体. """
- self._buffed_callers: List[Caller] = []
"""过程中 buff 的 caller. """
self._origin_functional_tokens = functional_tokens
@@ -48,6 +44,8 @@ def __init__(
self._functional_token_chars: Dict[int, Set[str]] = {}
""" functional token 的字符组.. """
+ self._destroyed = False
+
if functional_tokens:
for ft in functional_tokens:
start = ft.token
@@ -87,13 +85,13 @@ def match(self, message: Message) -> bool:
# 默认可以匹配任何一种 message 消息体.
return True
- def buff(self, pack: "Message") -> List[Message]:
+ def add(self, pack: "Message") -> List[Message]:
# 获取buff 后需要发送的包.
items = self._buff(pack)
result = []
for item in items:
# 如果是尾包, 对尾包进行必要的处理.
- is_tail = item.is_tail()
+ is_tail = item.is_complete()
if is_tail:
self._buff_tail_pack(item)
result.append(item)
@@ -104,11 +102,11 @@ def _buff(self, pack: "Message") -> Iterable[Message]:
return []
# 不深拷贝的话, 加工逻辑就会交叉污染?
# pack = origin.model_copy(deep=True)
- if DefaultMessageTypes.is_protocol_type(pack):
+ if MessageType.is_protocol_message(pack):
# final 包不进行 buffer.
yield pack
return
- if pack.is_tail():
+ if pack.is_complete():
# 如果收到了一个尾包, 则走尾包逻辑.
yield from self._receive_tail_pack(pack)
return
@@ -217,7 +215,7 @@ def _parse_content_by_functional_token(self, pack: "Message") -> "Message":
# 输出的消息会缓存到一起.
self._buffering_message_delivered_content += deliver_content
# 结算环节, 变更 pack 可以输出的 content.
- if pack.is_tail() and pack.content != self._buffering_message_delivered_content:
+ if pack.is_complete() and pack.content != self._buffering_message_delivered_content:
pack.memory = pack.content
pack.content = deliver_content
return pack
@@ -253,11 +251,6 @@ def _buff_tail_pack(self, tail: Message) -> None:
# 剥离所有的 callers.
self._buffed_messages.append(tail)
- # 从标准的 payload 和 attachments 里读取 caller.
- if tail.callers:
- for caller in tail.callers:
- self._buffed_callers.append(caller)
-
def _wrap_first_pack(self, pack: Message) -> Message:
# 首包强拷贝, 用来做一个 buffer.
pack = pack.model_copy(deep=True)
@@ -278,13 +271,9 @@ def _wrap_first_pack(self, pack: Message) -> Message:
# 添加默认的 payloads.
if self._payloads:
for payload in self._payloads:
- if not payload.exists(pack):
- payload.set(pack)
+ if not payload.payload_exists(pack):
+ payload.set_payload(pack)
- # 添加默认的 attachments.
- if self._attachments:
- for attachment in self._attachments:
- attachment.add(pack)
return pack
def _receive_head_pack(self, pack: "Message") -> Iterable[Message]:
@@ -300,7 +289,7 @@ def _clear_tail_pack(self) -> Optional[Message]:
return None
buffering = self._buffering_message
- buffering.pack = False
+ buffering = buffering.as_tail()
if self._functional_token_starts:
if self._buffering_token:
@@ -330,11 +319,13 @@ def _reset_buffering(self) -> None:
self._current_functional_token = ""
self._current_functional_token_content = ""
- def _generate_current_caller(self) -> Optional[Caller]:
+ def _generate_current_caller(self) -> Optional[FunctionCaller]:
if not self._current_functional_token:
return None
functional_token = self._functional_token_starts[self._current_functional_token]
- return functional_token.new_caller(self._current_functional_token_content)
+ caller = functional_token.new_caller(self._current_functional_token_content)
+ self._current_functional_token = ""
+ return caller
def new(self) -> "DefaultBuffer":
return DefaultBuffer(
@@ -352,8 +343,26 @@ def flush(self) -> Flushed:
self._buff_tail_pack(unsent)
deliver.append(unsent)
- flushed = Flushed(unsent=deliver, messages=self._buffed_messages, callers=self._buffed_callers)
+ callers = []
+ messages = self._buffed_messages
+ for item in messages:
+ callers.extend(item.callers)
+ flushed = Flushed(
+ unsent=deliver,
+ messages=messages,
+ callers=callers,
+ )
self._reset_buffering()
self._buffed_messages = []
- self._buffed_callers = []
return flushed
+
+ def destroy(self) -> None:
+ if self._destroyed:
+ return
+ self._destroyed = True
+ del self._buffering_message
+ del self._buffering_message_delivered_content
+ del self._buffering_token
+ del self._functional_token_starts
+ del self._origin_functional_tokens
+ del self._functional_token_ends
diff --git a/ghostos/framework/messengers/__init__.py b/ghostos/framework/messengers/__init__.py
index 13f547a9..f162f3cd 100644
--- a/ghostos/framework/messengers/__init__.py
+++ b/ghostos/framework/messengers/__init__.py
@@ -1 +1,2 @@
-from ghostos.framework.messengers.defaults import DefaultMessenger, TestMessengerProvider
+from ghostos.abcd import Messenger
+from ghostos.framework.messengers.defaults import DefaultMessenger
diff --git a/ghostos/framework/messengers/defaults.py b/ghostos/framework/messengers/defaults.py
index 19667504..ae3ee5be 100644
--- a/ghostos/framework/messengers/defaults.py
+++ b/ghostos/framework/messengers/defaults.py
@@ -1,200 +1,125 @@
-from typing import Optional, Iterable, TYPE_CHECKING, Type
-from ghostos.container import Container, Provider
-from ghostos.core.session.messenger import Messenger, Buffed
+from typing import Optional, Iterable, List, Tuple
+from ghostos.abcd.concepts import Messenger
from ghostos.core.messages import (
- Message, Payload, Attachment, Role, DefaultMessageTypes,
- Buffer, Stream,
+ Message, Payload, Role, MessageType,
+ Stream, FunctionCaller,
)
-from ghostos.core.session.threads import MsgThread
-from ghostos.core.llms import FunctionalToken
-from ghostos.framework.messages.buffers import DefaultBuffer
-
-if TYPE_CHECKING:
- from ghostos.contracts.logger import LoggerItf
+from ghostos.core.messages.pipeline import SequencePipe
__all__ = [
- 'DefaultMessenger', 'TestMessengerProvider'
+ 'DefaultMessenger'
]
-class DefaultMessenger(Messenger, Stream):
- """
- 默认的 Deliver, 支持消息的各种工具.
- """
+class DefaultMessenger(Messenger):
def __init__(
- self, *,
- upstream: Optional[Stream] = None,
- thread: Optional["MsgThread"] = None,
+ self,
+ upstream: Optional[Stream],
+ *,
name: Optional[str] = None,
role: Optional[str] = None,
- buffer: Optional[Buffer] = None,
payloads: Optional[Iterable[Payload]] = None,
- attachments: Optional[Iterable[Attachment]] = None,
- functional_tokens: Optional[Iterable[FunctionalToken]] = None,
- saving: bool = True,
- logger: Optional["LoggerItf"] = None,
+ stage: str = "",
):
- """
- 初始化一个 Messenger.
- :param upstream: 如果为 None 的话, 不会对上游发送消息.
- :param thread: 如果不为 None, 会把发送的尾包记录到 thread 里.
- :param name: 消息体的名字.
- :param role: 消息体的角色, 默认设定为 Assistant
- :param buffer: 是否传入自定义的 buffer.
- :param payloads: 每条消息都必须添加的 payload.
- :param attachments: 每条消息都必须添加的 attachments.
- :param functional_tokens: 是否有需要处理的 functional tokens
- :param logger:
- """
- self._thread: Optional[MsgThread] = thread
- self._name = name
- self._logger = logger
+ self._upstream = upstream
+ self._assistant_name = name
self._role = role if role else Role.ASSISTANT.value
- self._upstream: Optional[upstream] = upstream
- self._stopped: bool = False
- self._saving: bool = saving
- self._payloads: Optional[Iterable[Payload]] = payloads
- """默认的 payloads"""
- self._attachments: Optional[Iterable[Attachment]] = attachments
- """消息体默认的附件. """
- self._functional_tokens = functional_tokens
- if buffer is None:
- buffer = DefaultBuffer(
- name=self._name,
- role=self._role,
- payloads=self._payloads,
- attachments=self._attachments,
- functional_tokens=self._functional_tokens,
- )
- self._buffer: Buffer = buffer
-
- def new(
- self, *,
- sending: bool = True,
- thread: Optional[MsgThread] = None,
- name: Optional[str] = None,
- buffer: Optional[Buffer] = None,
- payloads: Optional[Iterable[Payload]] = None,
- attachments: Optional[Iterable[Attachment]] = None,
- functional_tokens: Optional[Iterable[FunctionalToken]] = None,
- ) -> "Messenger":
- # payloads 完成初始化.
- _payloads = None
- if self._payloads is not None or payloads is not None:
- payloads_map = {}
- if self._payloads:
- for payload in self._payloads:
- payloads_map[payload.key] = payload
- if payloads:
- for payload in payloads:
- payloads_map[payload.key] = payload
- _payloads = payloads_map.values()
-
- # attachments 初始化.
- _attachments = None
- if self._attachments is not None or attachments is not None:
- _attachments = []
- if self._attachments:
- _attachments.extend(self._attachments)
- if attachments:
- _attachments.extend(attachments)
-
- # 如果能传输数据, 则传递上游的 upstream.
- upstream = self._upstream if sending else None
- thread = self._thread
- functional_tokens = functional_tokens if functional_tokens else self._functional_tokens
- messenger = DefaultMessenger(
- upstream=upstream,
- thread=thread,
- name=self._name,
- role=self._role,
- buffer=buffer,
- payloads=_payloads,
- attachments=_attachments,
- functional_tokens=functional_tokens,
- )
- return messenger
-
- def is_streaming(self) -> bool:
- if self._upstream is None:
- return False
- return self._upstream.is_streaming()
-
- def deliver(self, pack: "Message") -> bool:
- if self.stopped():
- return False
- if not pack:
- return False
- # 下游返回 error, 会导致全链路的 messenger 因为 error 而停止.
- # 所以 error 类型的消息, 链路里只能有一个.
- if DefaultMessageTypes.ERROR.match(pack):
- self._stop(pack)
- return True
-
- if DefaultMessageTypes.is_final(pack):
- # 下游发送的 final 包, 上游会装作已经发送成功.
- return True
- delivery = self._buffer.buff(pack)
- return self._deliver(delivery)
-
- def _deliver(self, delivery: Iterable[Message]) -> bool:
- for item in delivery:
- if (
- self._saving
- and self._thread is not None # thread exists.
- and not DefaultMessageTypes.is_protocol_type(item) # not a protocol type message.
- and not item.pack
- ): # is tail package.
- # append tail message to thread.
- self._thread.append(item)
- if self._upstream:
- # 如果发送不成功, 直接中断.
- success = self._upstream.deliver(item)
- if not success:
- return False
- return True
+ self._payloads = payloads
+ self._sent_message_ids = []
+ self._sent_messages = {}
+ self._sent_callers = []
+ self._stage = stage
+ self._destroyed = False
+
+ def flush(self) -> Tuple[List[Message], List[FunctionCaller]]:
+ messages = []
+ callers = []
+ done = set()
+ for msg_id in self._sent_message_ids:
+ if msg_id in done:
+ continue
+ else:
+ done.add(msg_id)
+
+ message = self._sent_messages[msg_id]
+ messages.append(message)
+ if message.type == MessageType.FUNCTION_CALL:
+ callers.append(FunctionCaller(
+ id=message.call_id,
+ name=message.name,
+ arguments=message.content,
+ ))
+ elif message.callers:
+ callers.extend(message.callers)
+ self.destroy()
+ return messages, callers
+
+ def __del__(self):
+ self.destroy()
+
+ def destroy(self):
+ if self._destroyed:
+ return
+ self._destroyed = True
+ del self._upstream
+ del self._sent_messages
+ del self._sent_message_ids
+ del self._sent_callers
+ del self._payloads
def send(self, messages: Iterable[Message]) -> bool:
- for item in messages:
- success = self.deliver(item)
- if not success:
- return False
+ messages = self.buffer(messages)
+ if self._upstream is not None:
+ return self._upstream.send(messages)
+ list(messages)
return True
- def flush(self) -> "Buffed":
- if self._stopped:
- return Buffed(messages=[], callers=[])
-
- buffed = self._buffer.flush()
- if buffed.unsent:
- self._deliver(buffed.unsent)
- return Buffed(messages=buffed.messages, callers=buffed.callers)
-
- def _stop(self, final: Optional[Message]) -> None:
- """
- 停止并且发送指定的 final 包. 如果没有指定, 则发送 DefaultTypes.final()
- """
- self._stopped = True
- if final is None or not DefaultMessageTypes.is_protocol_type(final):
- final = DefaultMessageTypes.final()
- if self._upstream and not self._upstream.stopped():
- self._upstream.deliver(final)
-
- def stopped(self) -> bool:
- return self._stopped or (self._upstream is not None and self._upstream.stopped())
-
-
-class TestMessengerProvider(Provider[Messenger]):
- """
- for test only
- """
-
- def singleton(self) -> bool:
- return True
-
- def contract(self) -> Type[Messenger]:
- return Messenger
-
- def factory(self, con: Container) -> Messenger:
- return DefaultMessenger()
+ def buffer(self, messages: Iterable[Message]) -> Iterable[Message]:
+ messages = SequencePipe().across(messages)
+ for item in messages:
+ # add message info
+ if item.is_complete() or item.is_head():
+ if not item.name and MessageType.is_text(item):
+ item.name = self._assistant_name
+ if not item.stage:
+ item.stage = self._stage
+ if not item.role:
+ item.role = self._role
+ # create buffer in case upstream is cancel
+ if item.is_complete():
+ # add payload to complete one
+ if self._payloads:
+ for payload in self._payloads:
+ payload.set_payload(item)
+
+ # buffer outputs
+ self._sent_message_ids.append(item.msg_id)
+ self._sent_messages[item.msg_id] = item
+
+ # skip chunk
+ if self._upstream and self._upstream.completes_only() and not item.is_complete():
+ continue
+ yield item
+
+ def completes_only(self) -> bool:
+ return self._upstream is not None and self._upstream.completes_only()
+
+ def alive(self) -> bool:
+ return self._upstream is None or self._upstream.alive()
+
+ def close(self):
+ return
+
+ def fail(self, error: str) -> bool:
+ if self._upstream is not None:
+ return self._upstream.fail(error)
+ return False
+
+ def error(self) -> Optional[Message]:
+ if self._upstream is not None:
+ return self._upstream.error()
+ return None
+
+ def closed(self) -> bool:
+ return self._upstream is None or self._upstream.closed()
diff --git a/ghostos/framework/mindsets/__init__.py b/ghostos/framework/mindsets/__init__.py
deleted file mode 100644
index 718ad82d..00000000
--- a/ghostos/framework/mindsets/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-from ghostos.framework.mindsets.storage_impl import StorageMindsetProvider, WorkspaceMindsetProvider
diff --git a/ghostos/framework/mindsets/storage_impl.py b/ghostos/framework/mindsets/storage_impl.py
deleted file mode 100644
index 78ba0181..00000000
--- a/ghostos/framework/mindsets/storage_impl.py
+++ /dev/null
@@ -1,108 +0,0 @@
-from typing import Dict, Iterable, Type, Optional
-import yaml
-from ghostos.core.ghosts import Thought, ThoughtDriver, Mindset, Workspace, get_thought_driver_type
-from ghostos.contracts.storage import Storage
-from ghostos.contracts.modules import Modules
-from ghostos.helpers import generate_import_path, import_from_path
-from ghostos.container import Provider, Container
-
-__all__ = ['StorageMindset', 'StorageMindsetProvider', 'WorkspaceMindsetProvider']
-
-
-class StorageMindset(Mindset):
- """
- 基于 storage 来实现.
- """
-
- def __init__(self, storage: Storage, modules: Modules, namespace: str):
- self._modules = modules
- self._storage = storage
- self._cache_file = f"mindsets_{namespace}.cache.yml"
- self._thought_driver_map: Dict[Type[Thought], Type[ThoughtDriver]] = {}
- self._thought_path_driver_path_map: Dict[str, str] = {}
- if self._storage.exists(self._cache_file):
- content = storage.get(self._cache_file)
- data = yaml.safe_load(content)
- for key, val in data.items():
- if not isinstance(key, str) or not isinstance(val, str):
- continue
- self._thought_path_driver_path_map[key] = val
-
- def register_thought_type(self, cls: Type[Thought], driver: Optional[Type[ThoughtDriver]] = None) -> None:
- if driver is None:
- driver = get_thought_driver_type(cls)
- self._thought_driver_map[cls] = driver
- thought_type_path = generate_import_path(cls)
- thought_driver_type_path = generate_import_path(driver)
- self._thought_path_driver_path_map[thought_type_path] = thought_driver_type_path
- self._save_map()
-
- def _save_map(self) -> None:
- content = yaml.safe_dump(self._thought_path_driver_path_map)
- self._storage.put(self._cache_file, content.encode('utf-8'))
-
- def get_thought_driver_type(self, thought_cls: Type[Thought]) -> Type[ThoughtDriver]:
- if thought_cls in self._thought_driver_map:
- return self._thought_driver_map.get(thought_cls)
- thought_type_path = generate_import_path(thought_cls)
- if thought_type_path in self._thought_path_driver_path_map:
- driver_type_path = self._thought_path_driver_path_map[thought_type_path]
- result = import_from_path(driver_type_path, self._modules.import_module)
- if result is not None:
- return result
- return get_thought_driver_type(thought_cls)
-
- def thought_types(self) -> Iterable[Type[Thought]]:
- done = set()
- for thought_type in self._thought_driver_map:
- thought_type_path = generate_import_path(thought_type)
- done.add(thought_type_path)
- yield thought_type
-
- for thought_type_path in self._thought_path_driver_path_map:
- if thought_type_path not in done:
- done.add(thought_type_path)
- thought_type = import_from_path(thought_type_path, self._modules.import_module)
- yield thought_type
-
-
-class StorageMindsetProvider(Provider):
- """
- mindset based by storage
- """
-
- def __init__(self, relative_path: str = "runtime/cache"):
- self._relative_path = relative_path
-
- def singleton(self) -> bool:
- return True
-
- def contract(self) -> Type[Mindset]:
- return Mindset
-
- def factory(self, con: Container) -> Optional[Mindset]:
- storage = con.force_fetch(Storage)
- modules = con.force_fetch(Modules)
- cache_storage = storage.sub_storage(self._relative_path)
- return StorageMindset(cache_storage, modules, "")
-
-
-class WorkspaceMindsetProvider(Provider[Mindset]):
- """
- mindset based by workspace
- """
-
- def __init__(self, relative_path: str = "cache"):
- self._relative_path = relative_path
-
- def singleton(self) -> bool:
- return True
-
- def contract(self) -> Type[Mindset]:
- return Mindset
-
- def factory(self, con: Container) -> Optional[Mindset]:
- workspace = con.force_fetch(Workspace)
- modules = con.force_fetch(Modules)
- cache_storage = workspace.runtime().sub_storage(self._relative_path)
- return StorageMindset(cache_storage, modules, "")
diff --git a/ghostos/framework/multitasks/__init__.py b/ghostos/framework/multitasks/__init__.py
deleted file mode 100644
index 710a6e75..00000000
--- a/ghostos/framework/multitasks/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-from ghostos.framework.multitasks.basic import MultiTaskBasicImpl
diff --git a/ghostos/framework/multitasks/basic.py b/ghostos/framework/multitasks/basic.py
deleted file mode 100644
index edf5142b..00000000
--- a/ghostos/framework/multitasks/basic.py
+++ /dev/null
@@ -1,89 +0,0 @@
-from typing import Tuple
-from ghostos.core.ghosts import MultiTask, Operator, Ghost, Thought, NewTask
-from ghostos.core.llms import Chat
-from ghostos.core.messages import MessageKind, Role
-from ghostos.core.session.events import DefaultEventType
-from ghostos.framework.operators import WaitOnTasksOperator
-from ghostos.helpers import yaml_pretty_dump
-
-
-class MultiTaskBasicImpl(MultiTask):
-
- def __init__(self, ghost: Ghost):
- self._ghost = ghost
-
- def prepare_chat(self, chat: Chat) -> Chat:
- children = self._ghost.session().get_task_briefs(children=True)
- if not children:
- return chat
- prompt = """
-# MultiTask
-
-You are equipped with MultiTask library. You have created the async tasks below:
-```yaml
-{tasks}
-```
-"""
- data = []
- for task in children:
- data.append(task.model_dump(exclude_defaults=True, exclude={"task_id"}))
- tasks = yaml_pretty_dump(data)
- content = prompt.format(tasks=tasks)
- chat.system.append(Role.SYSTEM.new(content=content))
- return chat
-
- def wait_on_tasks(self, *new_tasks: Tuple[str, str, Thought, str]) -> Operator:
- tasks = []
- for task in new_tasks:
- task_name, task_desc, thought, instruction = task
- tasks.append(NewTask(
- task_name=task_name,
- task_desc=task_desc,
- thought=thought,
- instruction=instruction,
- ))
- return WaitOnTasksOperator(
- new_tasks=tasks,
- )
-
- def run_tasks(self, *new_tasks: Tuple[str, str, Thought, str]) -> None:
- tasks = []
- for task in new_tasks:
- task_name, task_desc, thought, instruction = task
- tasks.append(NewTask(
- task_name=task_name,
- task_desc=task_desc,
- thought=thought,
- instruction=instruction,
- ))
- self._ghost.utils().create_child_tasks(depend=False, new_tasks=tasks)
-
- def send_task(self, task_name: str, *messages: MessageKind) -> None:
- messages = list(messages)
- if not messages:
- return
- session = self._ghost.session()
- from_task_id = session.task().task_id
- tasks = session.get_task_briefs(children=True)
- for task in tasks:
- if task.name == task_name:
- event = DefaultEventType.INPUT.new(
- task_id=task.id,
- from_task_id=from_task_id,
- messages=messages,
- )
- session.fire_events(event)
-
- def cancel_task(self, task_name: str, reason: str) -> None:
- session = self._ghost.session()
- from_task_id = session.task().task_id
- tasks = session.get_task_briefs(children=True)
- for task in tasks:
- if task.name == task_name:
- event = DefaultEventType.CANCELING.new(
- task_id=task.id,
- from_task_id=from_task_id,
- messages=[],
- reason=reason,
- )
- session.fire_events(event)
diff --git a/ghostos/framework/openai_realtime/__init__.py b/ghostos/framework/openai_realtime/__init__.py
new file mode 100644
index 00000000..2dfc3f24
--- /dev/null
+++ b/ghostos/framework/openai_realtime/__init__.py
@@ -0,0 +1,30 @@
+from ghostos.abcd import Conversation
+from ghostos.abcd.realtime import RealtimeApp
+from ghostos.abcd.realtime import Speaker, Listener
+from ghostos.framework.openai_realtime.driver import OpenAIRealtimeDriver
+
+
+def get_openai_realtime_app(
+ conversation: Conversation,
+ *,
+ vad_mode: bool,
+ speaker: Speaker,
+ listener: Listener,
+) -> RealtimeApp:
+
+ from ghostos.framework.openai_realtime.app import RealtimeAppImpl
+ from ghostos.framework.openai_realtime.configs import OpenAIRealtimeAppConf
+ from ghostos.framework.openai_realtime.ws import OpenAIWSConnection
+ from ghostos.bootstrap import get_container
+ from ghostos.contracts.configs import Configs
+
+ _container = get_container()
+ _configs = _container.force_fetch(Configs)
+ _conf = _configs.get(OpenAIRealtimeAppConf)
+ return RealtimeAppImpl(
+ conf=_conf,
+ vad_mode=vad_mode,
+ conversation=conversation,
+ listener=listener,
+ speaker=speaker,
+ )
diff --git a/ghostos/framework/openai_realtime/app.py b/ghostos/framework/openai_realtime/app.py
new file mode 100644
index 00000000..008687f0
--- /dev/null
+++ b/ghostos/framework/openai_realtime/app.py
@@ -0,0 +1,259 @@
+import threading
+from typing import Optional, Tuple, List, Iterable
+from ghostos.framework.openai_realtime.configs import OpenAIRealtimeAppConf
+from ghostos.framework.openai_realtime.client import (
+ AppClient
+)
+from ghostos.framework.openai_realtime.state_of_client import (
+ SynchronizingState, StateOfClient, AppState,
+)
+from ghostos.framework.openai_realtime.output import OutputBuffer, DefaultOutputBuffer
+from collections import deque
+from ghostos.abcd import Conversation
+from ghostos.core.messages import ReceiverBuffer, Message
+from ghostos.abcd.realtime import RealtimeApp, Listener, Speaker, Operator
+from ghostos.helpers import Timeleft
+from threading import Thread
+from queue import Empty
+import time
+
+__all__ = ['RealtimeAppImpl']
+
+
+class RealtimeAppImpl(RealtimeApp):
+
+ def __init__(
+ self,
+ *,
+ conf: OpenAIRealtimeAppConf,
+ vad_mode: bool,
+ conversation: Conversation,
+ listener: Listener,
+ speaker: Speaker,
+ ):
+ self.conversation = conversation
+ self._config = conf
+ self._listener = listener
+ self._speaker = speaker
+ self._started: bool = False
+ self._closed: bool = False
+ self._client: Optional[AppClient] = None
+ self._state: Optional[StateOfClient] = None
+ self._output: OutputBuffer = self._create_output_buffer()
+ self._operators: deque = deque()
+ self._threads: List[Thread] = []
+ self._client = AppClient(self._config, vad_mode, self._output, self.conversation)
+ self._stopped = threading.Event()
+ self._close_check = Thread(target=self._close_check)
+
+ def _create_output_buffer(self) -> OutputBuffer:
+ return DefaultOutputBuffer(self.is_closed, logger=self.conversation.logger)
+
+ def _close_check(self):
+ while not self._stopped.is_set():
+ time.sleep(0.5)
+ self.close()
+
+ @property
+ def vad_mode(self) -> bool:
+ return self._client.vad_mode()
+
+ @property
+ def listen_mode(self) -> bool:
+ return self._client.listen_mode()
+
+ def start(self, vad_mode: bool = True, listen_mode: bool = True):
+ if self.is_closed():
+ # todo
+ return
+ if self._started:
+ return
+ self._started = True
+ self.set_mode(vad_mode=vad_mode, listen_mode=listen_mode)
+
+ self._threads.append(Thread(target=self._main_state_thread))
+ self._threads.append(Thread(target=self._speaking_thread))
+ self._threads.append(Thread(target=self._listening_thread))
+ for t in self._threads:
+ t.start()
+
+ def add_message(self, message: Message, previous_message_id: Optional[str] = None):
+ self._client.server_ctx.add_message_to_server(message, previous_message_id)
+
+ def _main_state_thread(self):
+ try:
+ interval = 0.1
+ timeleft = Timeleft(5)
+ while not self.is_closed() and not self._stopped.is_set():
+ # check conversation, refresh it.
+ if not timeleft.alive():
+ if not self.conversation.refresh():
+ break
+ timeleft = Timeleft(5)
+ self._client.logger.debug("realtime main thread refresh conversation")
+
+ state = self._state
+ if state is None:
+ self._client.logger.debug("synchronize state")
+ state = SynchronizingState(self._client)
+ state.on_init()
+ self._state = state
+ continue
+ state: StateOfClient = state
+
+ # run operators
+ if len(self._operators) > 0:
+ op = self._operators.popleft()
+ next_state = state.operate(op)
+ else:
+ # tick frame
+ next_state = state.tick_frame()
+
+ # init next state
+ if next_state is not None:
+ self._client.logger.debug("change state from %s to %s", state, next_state)
+ self._operators.clear()
+ next_state.on_init()
+ self._state = next_state
+ state.destroy()
+ continue
+
+ if state.recv_server_event():
+ self._client.logger.debug("handled server event")
+ continue
+ elif event := self._client.conversation.pop_event():
+ # handle ghostos event if server event is missing.
+ self._client.logger.debug("handle ghostos event")
+ self._client.handle_ghostos_event(event)
+ continue
+
+ time.sleep(interval)
+ except Exception as e:
+ self._client.logger.exception(e)
+ self._stopped.set()
+
+ def _speaking_thread(self):
+ try:
+ while not self.is_closed() and not self._stopped.is_set():
+ response_id = self._output.get_response_id()
+ if response_id is None:
+ time.sleep(0.1)
+ continue
+ self._client.logger.debug("start speaking. respond id is %s", response_id)
+ self._run_speaking_loop(response_id)
+ self._output.stop_output(response_id)
+ self._client.logger.debug("stop speaking. responding is %r", self._client.is_responding())
+ except Exception as e:
+ self._client.logger.exception(e)
+ self._stopped.set()
+
+ def _run_speaking_loop(self, response_id: str):
+ output_buffer = self._output
+ q = output_buffer.speaking_queue(response_id)
+ if q is None:
+ return
+
+ client = self._client
+ client.logger.debug("start speaking loop")
+
+ def receive():
+ try:
+ item = q.get(block=True)
+ return item
+ except Empty:
+ time.sleep(0.2)
+
+ with self._speaker.speak(receive) as speaking:
+ while not speaking.done():
+ if self.is_closed() or not self._output.is_speaking():
+ break
+ if self._output.get_response_id() != response_id:
+ break
+ time.sleep(0.1)
+ client.logger.debug("end speaking loop")
+
+ def _listening_thread(self):
+ try:
+ while not self.is_closed() and not self._stopped.is_set():
+ client = self._client
+ if not client.is_listening():
+ time.sleep(0.1)
+ continue
+ session_id = client.get_session_id()
+ self._run_listening_loop(session_id)
+ except Exception as e:
+ self._client.logger.exception(e)
+ self._stopped.set()
+
+ def _run_listening_loop(self, session_id: str):
+ client = self._client
+ client.logger.debug("start listening loop")
+ with self._listener.listen(client.audio_buffer_append):
+ while not self.is_closed():
+ if not client.is_listening():
+ client.logger.debug("stop listening loop")
+ break
+ time.sleep(0.1)
+ client.logger.debug("end listening loop")
+
+ def close(self):
+ if self._closed:
+ return
+ self._closed = True
+ for t in self._threads:
+ t.join()
+ self._client.server_ctx.update_local_conversation()
+ self._client.close()
+ del self.conversation
+
+ def is_closed(self) -> bool:
+ return self._closed or self.conversation.is_closed() or self._stopped.is_set()
+
+ def history_messages(self) -> Iterable[Message]:
+ return self._output.get_outputted_messages()
+
+ def set_mode(
+ self,
+ *,
+ vad_mode: Optional[bool] = None,
+ listen_mode: Optional[bool] = None,
+ ):
+
+ if listen_mode is not None:
+ if listen_mode:
+ self._client.start_listening()
+ else:
+ self._client.stop_listening()
+
+ if vad_mode is not None:
+ self._client.set_vad_mode(vad_mode=vad_mode)
+
+ def state(self) -> Tuple[str, List[Operator]]:
+ if self.is_closed():
+ return str(AppState.closed.value), []
+ elif self._state is None:
+ return str(AppState.created.value), []
+
+ state_name = self._state.state_name()
+ if isinstance(state_name, AppState):
+ state_name = state_name.value
+ operators = self._state.operators()
+ return str(state_name), operators
+
+ def operate(self, operator: Operator) -> bool:
+ if self.is_closed():
+ return False
+ if self._state is None:
+ return False
+ if self._state.allow(operator):
+ self._operators.append(operator)
+ return True
+ return False
+
+ def fail(self, error: Exception) -> bool:
+ self._client.logger.exception(error)
+ self.close()
+ return True
+
+ def output(self) -> Optional[ReceiverBuffer]:
+ return self._output.output_received()
diff --git a/ghostos/framework/openai_realtime/client.py b/ghostos/framework/openai_realtime/client.py
new file mode 100644
index 00000000..4e5b4f78
--- /dev/null
+++ b/ghostos/framework/openai_realtime/client.py
@@ -0,0 +1,501 @@
+import base64
+from typing import Dict, Optional, List
+from ghostos.abcd import Conversation
+from ghostos.contracts.logger import LoggerItf
+from ghostos.contracts.assets import AudioAssets
+from ghostos.core.messages import Message, MessageType
+from ghostos.core.runtime import Turn, Event as GhostOSEvent, EventTypes as GhostOSEventTypes
+from io import BytesIO
+from ghostos.framework.openai_realtime.configs import OpenAIRealtimeAppConf
+from ghostos.framework.openai_realtime.ws import OpenAIWSConnection
+from ghostos.framework.openai_realtime.event_data_objects import MessageItem, SessionObject, Tool
+from ghostos.framework.openai_realtime.event_from_server import ServerSessionCreated
+from ghostos.framework.openai_realtime.event_from_client import (
+ SessionUpdate,
+ ClientEvent,
+ ConversationItemCreate,
+ ResponseCancel,
+ ResponseCreate,
+ InputAudioBufferCommit,
+ InputAudioBufferClear,
+ InputAudioBufferAppend,
+)
+from ghostos.framework.openai_realtime.state_of_server import ServerContext, SessionState
+from ghostos.framework.openai_realtime.state_of_client import Client
+from ghostos.framework.openai_realtime.output import OutputBuffer
+from threading import Lock
+import wave
+
+RATE = 24000
+
+
+class Context(ServerContext):
+
+ def __init__(
+ self,
+ *,
+ conversation: Conversation,
+ connection: OpenAIWSConnection,
+ listening: bool,
+ output: OutputBuffer,
+ logger: Optional[LoggerItf] = None,
+
+ ):
+ self.conversation: Conversation = conversation
+ self.connection = connection
+ self.thread = conversation.get_thread(truncated=True)
+ self.history_messages: Dict[str, Message] = {}
+ self.history_message_order: List[str] = []
+ self._reset_history_messages()
+
+ if logger is None:
+ logger = conversation.logger
+ self.logger = logger
+
+ self.buffer_message_ids: List[str] = []
+ self.buffer_messages: Dict[str, Message] = {}
+ self.output_buffer: OutputBuffer = output
+ self.listening: bool = listening
+ self.response_id: Optional[str] = None
+ self.response_audio_buffer: Dict[str, BytesIO] = {}
+
+ self.client_audio_buffer: BytesIO = BytesIO()
+ self.client_audio_buffer_locker: Lock = Lock()
+ self.update_history_locker: Lock = Lock()
+ self._destroyed = False
+
+ def destroy(self):
+ if self._destroyed:
+ return
+ self._destroyed = True
+ self.connection = None
+ self.conversation = None
+ self._reset_buffer_messages()
+ self._reset_history_messages()
+
+ def get_server_response_id(self) -> Optional[str]:
+ return self.response_id
+
+ def _add_history_message(self, message: Message):
+ if message.msg_id not in self.history_messages:
+ self.history_message_order.append(message.msg_id)
+ self.history_messages[message.msg_id] = message
+
+ def _reset_history_messages(self):
+ self.history_messages: Dict[str, Message] = {}
+ self.history_message_order: List[str] = []
+ for message in self.thread.get_messages(True):
+ self.history_messages[message.msg_id] = message
+ self.history_message_order.append(message.msg_id)
+
+ def _reset_buffer_messages(self):
+ self.buffer_message_ids = []
+ self.buffer_messages: Dict[str, Message] = {}
+
+ def update_local_conversation(self) -> None:
+ buffered = []
+ function_call = False
+ for msg_id in self.buffer_message_ids:
+ if msg_id in self.history_messages:
+ # already updated.
+ continue
+ message = self.buffer_messages[msg_id]
+ if not message.is_complete():
+ continue
+ buffered.append(message)
+ if message.type == MessageType.FUNCTION_CALL:
+ function_call = True
+ self._reset_buffer_messages()
+ if not buffered:
+ return
+
+ event_type = GhostOSEventTypes.ACTION_CALL if function_call else GhostOSEventTypes.INPUT
+ event = event_type.new(
+ task_id=self.conversation.task_id,
+ messages=buffered
+ )
+ if not function_call:
+ self.thread.new_turn(event)
+ self.conversation.update_thread(self.thread)
+ self.thread = self.conversation.get_thread(True)
+ else:
+ receiver = self.conversation.respond_event(event, streaming=False)
+ with receiver:
+ for item in receiver.recv():
+ if item.is_complete():
+ self.logger.info("receive conversation message: %s, %s", item.type, item.msg_id)
+ self.add_message_to_server(item)
+
+ self._reset_history_messages()
+ # update audio buffer
+ for item_id in self.response_audio_buffer:
+ buffer = self.response_audio_buffer[item_id]
+ buffer.seek(0)
+ data = buffer.getvalue()
+ buffer.close()
+ self.save_audio_data(item_id, data)
+ self.response_audio_buffer = {}
+
+ def respond_message_chunk(self, response_id: str, chunk: Optional[Message]) -> bool:
+ if chunk is None:
+ return False
+ ok = self.output_buffer.add_response_chunk(response_id, chunk)
+ if not ok:
+ self.logger.debug(f"Failed to add response chunk: {response_id}")
+ return ok
+
+ def respond_error_message(self, error: str) -> None:
+ message = MessageType.ERROR.new(content=error)
+ self.output_buffer.add_error_message(message)
+
+ def save_audio_item(self, item: MessageItem) -> None:
+ if not item.has_audio():
+ return
+ audio_data = item.get_audio_bytes()
+ self.save_audio_data(item.id, audio_data)
+
+ def save_audio_data(self, item_id: str, audio_data: bytes) -> None:
+ if not audio_data:
+ return
+ buffer = BytesIO()
+ with wave.open(buffer, 'wb') as f:
+ f.setnchannels(1)
+ f.setsampwidth(2)
+ f.setframerate(24000)
+ f.writeframes(audio_data)
+
+ saving = buffer.getvalue()
+
+ asserts = self.conversation.container().force_fetch(AudioAssets)
+ fileinfo = asserts.get_fileinfo(item_id)
+ if fileinfo is None:
+ fileinfo = asserts.new_fileinfo(
+ fileid=item_id,
+ filename=f"{item_id}.wav",
+ )
+ asserts.save(fileinfo, saving)
+
+ def update_history_message(self, message: Optional[Message]) -> None:
+ if message is None:
+ return
+ if not message.is_complete():
+ # only complete message is useful
+ return
+ if not message.content or not message.msg_id:
+ return
+
+ if message.msg_id in self.history_messages:
+ # if the history message already exists, update it
+ self.history_messages[message.msg_id] = message
+ self.thread.update_message(message)
+ else:
+ # otherwise, add the message to buffer message and try to send it.
+ if message.msg_id not in self.buffer_messages:
+ self.buffer_message_ids.append(message.msg_id)
+ self.buffer_messages[message.msg_id] = message
+
+ def add_message_item(self, item: MessageItem, previous_item_id: Optional[str] = None) -> None:
+ if item is None:
+ return
+ message = item.to_complete_message()
+ if message is not None:
+ self.update_history_message(message)
+ self.output_buffer.add_message(message, previous_item_id)
+
+ def start_server_response(self, response_id: str) -> None:
+ self.response_id = response_id
+ self.output_buffer.start_output(response_id)
+
+ def is_server_responding(self) -> bool:
+ return self.response_id is not None
+
+ def end_server_response(self, response_id: str) -> bool:
+ match = response_id == self.response_id
+ self.response_id = None
+ return self.output_buffer.end_output(response_id) and match
+
+ def stop_listening(self) -> bool:
+ if self.listening:
+ self.listening = False
+ self.clear_client_audio_buffer()
+ return True
+ return False
+
+ def respond_audio_chunk(self, response_id: str, item_id: str, data: bytes) -> bool:
+ if response_id == self.response_id:
+ if item_id not in self.response_audio_buffer:
+ self.response_audio_buffer[item_id] = BytesIO()
+ buffer = self.response_audio_buffer[item_id]
+ buffer.write(data)
+ return self.output_buffer.add_audio_output(response_id, data)
+
+ def add_message_to_server(self, message: Message, previous_item_id: Optional[str] = None) -> bool:
+ item = MessageItem.from_message(message)
+ if item is not None:
+ ce = ConversationItemCreate(
+ previous_item_id=previous_item_id,
+ item=item,
+ )
+ self.send_client_event(ce)
+ return True
+ return False
+
+ def send_client_event(self, event: ClientEvent, exclude_none=True):
+ data = event.to_event_dict(exclude_none=exclude_none)
+ self.logger.debug("send client event type %s", type(event))
+ self.connection.send(data)
+
+ def audio_buffer_append(self, buffer: bytes) -> None:
+ with self.client_audio_buffer_locker:
+ content = base64.b64encode(buffer)
+ self.client_audio_buffer.write(buffer)
+ ce = InputAudioBufferAppend(
+ audio=content
+ )
+ self.send_client_event(ce)
+
+ def clear_client_audio_buffer(self) -> None:
+ with self.client_audio_buffer_locker:
+ self.client_audio_buffer.close()
+ self.client_audio_buffer = BytesIO()
+ if self.listening:
+ ce = InputAudioBufferClear()
+ self.send_client_event(ce)
+
+ def set_client_audio_buffer_start(self, cut_ms: int) -> None:
+ pass
+ # with self.client_audio_buffer_locker:
+ # data = self.client_audio_buffer.getvalue()
+ # cut = int((RATE / 1000) * cut_ms)
+ # if cut < len(data):
+ # cut_data = data[cut:]
+ # self.client_audio_buffer = BytesIO(cut_data)
+
+ def set_client_audio_buffer_stop(self, item_id: str, end_ms: int) -> None:
+ with self.client_audio_buffer_locker:
+ data = self.client_audio_buffer.getvalue()
+ # cut = int((RATE / 1000) * end_ms)
+ # cut_data = data[:cut]
+ # if cut < len(data):
+ # left_data = data[cut:]
+ # self.client_audio_buffer = BytesIO(left_data)
+ self.save_audio_data(item_id, data)
+ self.client_audio_buffer.close()
+ self.client_audio_buffer = BytesIO()
+
+
+class AppClient(Client):
+
+ def __init__(
+ self,
+ conf: OpenAIRealtimeAppConf,
+ vad_mode: bool,
+ output_buffer: OutputBuffer,
+ conversation: Conversation,
+ ):
+ self._closed: bool = False
+ self._vad_mode = vad_mode
+ self.conf: OpenAIRealtimeAppConf = conf
+ self.conversation: Conversation = conversation
+ self.logger = conversation.logger
+ self.connection: OpenAIWSConnection = self.connect()
+ self.server_ctx: Context = Context(
+ conversation=conversation,
+ connection=self.connection,
+ listening=self._vad_mode,
+ logger=conversation.logger,
+ output=output_buffer,
+ )
+ self.session_state: SessionState = self._create_session_state()
+ self.synchronized: bool = False
+ self._sync_history: Optional[List[str]] = None
+
+ def set_vad_mode(self, vad_mode: bool) -> None:
+ if vad_mode == self._vad_mode:
+ return
+ self._vad_mode = vad_mode
+ self.update_session()
+
+ def vad_mode(self) -> bool:
+ return self._vad_mode
+
+ def connect(self) -> OpenAIWSConnection:
+ self._validate_closed()
+ self._sync_history = None
+ self.synchronized = False
+ return OpenAIWSConnection(
+ self.conf.ws_conf,
+ logger=self.logger,
+ )
+
+ def _validate_closed(self):
+ if self._closed:
+ raise RuntimeError("App Client is closed")
+
+ def close(self) -> None:
+ if self._closed:
+ return
+ self._closed = True
+ if self.connection:
+ self.connection.close()
+ if self.session_state:
+ self.session_state.destroy()
+ if self.server_ctx:
+ self.server_ctx.destroy()
+ del self.connection
+ del self.session_state
+ del self.server_ctx
+
+ def get_session_id(self) -> str:
+ self._validate_closed()
+ if self.session_state:
+ return self.session_state.session_id
+ return ""
+
+ def reconnect(self) -> None:
+ self._validate_closed()
+ if self.connection is not None:
+ connection = self.connection
+ connection.close()
+ if self.session_state is not None:
+ self.session_state.destroy()
+ self.session_state = None
+
+ self.connection: OpenAIWSConnection = self.connect()
+ self.session_state: SessionState = self._create_session_state()
+ self.synchronized = False
+
+ def audio_buffer_append(self, buffer: bytes) -> None:
+ if not self.is_listening():
+ # listening change is quick then sending
+ return
+ self.server_ctx.audio_buffer_append(buffer)
+
+ def is_listening(self) -> bool:
+ if not self.synchronized:
+ return False
+ if self.server_ctx.is_server_responding():
+ return False
+ if self.server_ctx.output_buffer.is_speaking():
+ return False
+ return self.server_ctx.listening
+
+ def listen_mode(self) -> bool:
+ return self.server_ctx.listening
+
+ def _create_session_state(self) -> SessionState:
+ e = self.connection.recv(timeout=self.conf.session_created_timeout, timeout_error=True)
+ se = ServerSessionCreated(**e)
+ return SessionState(self.server_ctx, se)
+
+ def update_session(self):
+ session_obj = self.get_session_obj()
+ ce = SessionUpdate(
+ session=session_obj.model_dump(),
+ )
+ self.logger.debug("client update session: %r", ce)
+ self.server_ctx.send_client_event(ce, exclude_none=False)
+
+ def get_sync_history(self):
+ if self._sync_history is not None:
+ return self._sync_history
+ history = []
+ for msg_id in self.server_ctx.history_message_order:
+ message = self.server_ctx.history_messages[msg_id]
+ if not message.content:
+ continue
+ history.append(message.role + ": " + message.content)
+ self._sync_history = history
+ return self._sync_history
+
+ def get_session_obj(self) -> SessionObject:
+ session_obj = self._get_session_obj(self._vad_mode)
+ history = self.get_sync_history()
+ if history:
+ history_content = "\n\n---\n\n".join(history)
+ session_obj.instructions += "\n\n# history messages\n\n" + history_content
+ return session_obj
+
+ def synchronize_server_session(self):
+ if self.synchronized:
+ return
+ count = 0
+ self.update_session()
+ self.logger.info("Synchronizing server session done with item %d", count)
+ self.synchronized = True
+
+ def cancel_responding(self) -> bool:
+ # cancel local response first.
+ self.server_ctx.output_buffer.stop_output(None)
+ if self.server_ctx.is_server_responding():
+ ce = ResponseCancel()
+ self.server_ctx.send_client_event(ce)
+ self.clear_audio_input()
+ return True
+ return False
+
+ def start_listening(self) -> bool:
+ if not self.server_ctx.listening:
+ self.server_ctx.listening = True
+ self.server_ctx.output_buffer.stop_output(None)
+ if self.server_ctx.is_server_responding():
+ self.cancel_responding()
+ return True
+ return False
+
+ def stop_listening(self) -> bool:
+ return self.server_ctx.stop_listening()
+
+ def commit_audio_input(self) -> bool:
+ if self.server_ctx.listening:
+ ce = InputAudioBufferCommit()
+ self.server_ctx.send_client_event(ce)
+ return True
+ return False
+
+ def clear_audio_input(self) -> bool:
+ if self.server_ctx.listening:
+ ce = InputAudioBufferClear()
+ self.server_ctx.send_client_event(ce)
+ return True
+ return False
+
+ def _get_session_obj(self, vad_mode: bool) -> SessionObject:
+ session_obj = self.conf.get_session_obj(vad_mode)
+ session_obj.instructions = self.conversation.get_instructions()
+ tools = []
+ for fn in self.conversation.get_functions():
+ tool = Tool(**fn.to_dict())
+ tools.append(tool)
+ session_obj.tools = tools
+ return session_obj
+
+ def create_response(self) -> bool:
+ session_obj = self.get_session_obj()
+ ce = ResponseCreate(
+ response=session_obj.model_dump(),
+ )
+ self.server_ctx.send_client_event(ce)
+ return True
+
+ def is_server_responding(self) -> bool:
+ return self.server_ctx.is_server_responding()
+
+ def is_speaking(self) -> bool:
+ return self.server_ctx.output_buffer.is_speaking()
+
+ def receive_server_event(self) -> bool:
+ data = self.connection.recv(timeout=0.1)
+ if data:
+ self.logger.debug("got received server event")
+ self.session_state.recv(data)
+ return True
+ return False
+
+ def handle_ghostos_event(self, event: GhostOSEvent):
+ # send message to server, let the realtime server handle the new message items.
+ for msg in Turn.iter_event_message(event):
+ self.server_ctx.add_message_to_server(msg)
+
+ def respond_error_message(self, error: str) -> None:
+ self.server_ctx.respond_error_message(error)
diff --git a/ghostos/framework/openai_realtime/configs.py b/ghostos/framework/openai_realtime/configs.py
new file mode 100644
index 00000000..f30d6894
--- /dev/null
+++ b/ghostos/framework/openai_realtime/configs.py
@@ -0,0 +1,65 @@
+from typing import ClassVar, Optional, Self
+from ghostos.abcd.realtime import RealtimeAppConfig
+from ghostos.contracts.configs import YamlConfig
+from ghostos.framework.openai_realtime.event_data_objects import SessionObject
+from pydantic import BaseModel, Field
+
+__all__ = ['OPENAI_REALTIME_DRIVER_NAME', 'OpenAIRealtimeAppConf', 'OpenAIWebsocketsConf']
+
+OPENAI_REALTIME_DRIVER_NAME = "openai_realtime_driver"
+
+
+# 拆一个 base model 方便未来做成表单.
+class OpenAIWebsocketsConf(BaseModel):
+ api_key: str = Field(
+ default="$OPENAI_API_KEY",
+ description="The OpenAI key used to authenticate with WebSockets.",
+ )
+ proxy: Optional[str] = Field(
+ default="$OPENAI_PROXY",
+ description="The proxy to connect to. only support socket v5 now",
+ )
+ uri: str = Field("wss://api.openai.com/v1/realtime?model=gpt-4o-mini-realtime-preview-2024-12-17")
+ close_check: float = Field(
+ default=0.5,
+ description="check if the connection is still going while sending event to server",
+ )
+
+ def load_from_env(self) -> Self:
+ from os import environ
+ copied = self.model_copy(deep=True)
+ if copied.api_key and copied.api_key.startswith("$"):
+ copied.api_key = environ[copied.api_key[1:]]
+ if copied.proxy and copied.proxy.startswith("$"):
+ copied.proxy = environ[copied.proxy[1:]]
+ return copied
+
+
+class OpenAIRealtimeAppConf(YamlConfig, RealtimeAppConfig):
+ """
+ OpenAI Realtime Beta API configuration
+ """
+ relative_path: ClassVar[str] = "openai_realtime_config.yml"
+
+ ws_conf: OpenAIWebsocketsConf = Field(
+ default_factory=OpenAIWebsocketsConf,
+ description="OpenAI Websockets configuration",
+ )
+ session: SessionObject = Field(
+ default_factory=SessionObject,
+ description="basic session settings, if None, use openai default session",
+ )
+ session_created_timeout: int = Field(10, description="session created timeout")
+
+ def get_session_obj(self, vad_mode: bool) -> SessionObject:
+ """
+ get session object
+ :return:
+ """
+ session = self.session.model_copy(deep=True)
+ if not vad_mode:
+ session.turn_detection = None
+ return session
+
+ def driver_name(self) -> str:
+ return OPENAI_REALTIME_DRIVER_NAME
diff --git a/ghostos/framework/openai_realtime/driver.py b/ghostos/framework/openai_realtime/driver.py
new file mode 100644
index 00000000..0f4fe975
--- /dev/null
+++ b/ghostos/framework/openai_realtime/driver.py
@@ -0,0 +1,30 @@
+from typing import Optional
+
+from ghostos.abcd import Conversation
+from ghostos.abcd.realtime import RealtimeDriver, Listener, Speaker, RealtimeApp
+from ghostos.framework.openai_realtime.configs import OPENAI_REALTIME_DRIVER_NAME, OpenAIRealtimeAppConf
+from ghostos.framework.openai_realtime.app import RealtimeAppImpl
+
+__all__ = ['OpenAIRealtimeDriver']
+
+
+class OpenAIRealtimeDriver(RealtimeDriver[OpenAIRealtimeAppConf]):
+
+ def driver_name(self) -> str:
+ return OPENAI_REALTIME_DRIVER_NAME
+
+ def create(
+ self,
+ config: OpenAIRealtimeAppConf,
+ conversation: Conversation,
+ listener: Optional[Listener] = None,
+ speaker: Optional[Speaker] = None,
+ vad_mode: bool = False,
+ ) -> RealtimeApp:
+ return RealtimeAppImpl(
+ conf=config,
+ vad_mode=vad_mode,
+ conversation=conversation,
+ listener=listener,
+ speaker=speaker,
+ )
diff --git a/ghostos/framework/openai_realtime/event_data_objects.py b/ghostos/framework/openai_realtime/event_data_objects.py
new file mode 100644
index 00000000..d3bf8666
--- /dev/null
+++ b/ghostos/framework/openai_realtime/event_data_objects.py
@@ -0,0 +1,382 @@
+from __future__ import annotations
+
+import base64
+
+from pydantic import BaseModel, Field
+from typing import Optional, Literal, List, Union, Dict
+from io import BytesIO
+from ghostos.core.messages import (
+ MessageType, Message, AudioMessage, FunctionCallMessage, FunctionCallOutputMessage,
+ FunctionCaller, Role,
+)
+from ghostos.helpers import md5
+from enum import Enum
+
+
+class RateLimit(BaseModel):
+ name: str = Field("", description="Name of the rate-limited event.", enum={"requests", "tokens"})
+ limit: int = Field(0, description="The maximum allowed value for the rate limit.")
+ remaining: int = Field(0, description="The remaining value before the limit is reached.")
+ reset_seconds: float = Field(0.0, description="Seconds until the rate limit resets.")
+
+
+class TokensDetails(BaseModel):
+ cached_tokens: int = Field(default=0)
+ text_tokens: int = Field(default=0)
+ audio_tokens: int = Field(default=0)
+
+
+class Usage(BaseModel):
+ total_tokens: int = Field()
+ input_tokens: int = Field()
+ output_tokens: int = Field()
+ input_token_details: Optional[TokensDetails] = Field(None)
+ output_token_details: Optional[TokensDetails] = Field(None)
+
+
+class Error(BaseModel):
+ type: str = Field("")
+ code: Optional[str] = Field(None)
+ message: Optional[str] = Field(None)
+ param: Optional[str] = Field(None)
+
+
+class ResponseStatusDetails(BaseModel):
+ type: str = Field("")
+ reason: str = Field("")
+ error: Optional[Error] = Field(None)
+
+
+class Response(BaseModel):
+ id: str = Field()
+ object: str = Field("realtime.response")
+ status: Literal["completed", "cancelled", "failed", "incomplete", "in_progress"] = Field()
+ status_details: Optional[ResponseStatusDetails] = Field(None)
+ output: List[MessageItem] = Field(default_factory=list)
+ usage: Optional[Usage] = Field(None)
+
+
+class Content(BaseModel):
+ """
+ The content of the message, applicable for message items.
+ Message items with a role of system support only input_text content,
+ message items of role user support input_text and input_audio content,
+ and message items of role assistant support text content.
+ """
+ type: Literal["input_text", "input_audio", "text", "audio"] = Field()
+ text: Optional[str] = Field(None)
+ audio: Optional[str] = Field(None)
+ transcript: Optional[str] = Field(None)
+
+
+class MessageItem(BaseModel):
+ """
+ The item to add to the conversation.
+ """
+ id: Optional[str] = Field(None)
+ type: Literal["message", "function_call", "function_call_output"] = Field("")
+ status: Optional[str] = Field(None, enum={"completed", "incomplete"})
+ role: Optional[str] = Field(default=None, enum={"assistant", "user", "system"})
+ content: Optional[List[Content]] = Field(None)
+ call_id: Optional[str] = Field(None)
+ name: Optional[str] = Field(None, description="The name of the function being called (for function_call items).")
+ arguments: Optional[str] = Field(None, description="The arguments of the function call (for function_call items).")
+ output: Optional[str] = Field(None, description="The output of the function call (for function_call_output items).")
+
+ @classmethod
+ def from_message(cls, message: Message) -> Optional[MessageItem]:
+ if message is None or not message.content:
+ return None
+ id_ = message.msg_id
+ call_id = None
+ output = None
+ arguments = None
+ content = None
+ role = message.role
+ if not role:
+ role = Role.ASSISTANT.value
+
+ if message.type == MessageType.FUNCTION_CALL.value:
+ type_ = "function_call"
+ call_id = message.call_id
+ arguments = message.content
+ role = None
+ elif message.type == MessageType.FUNCTION_OUTPUT.value:
+ type_ = "function_call_output"
+ call_id = message.call_id
+ output = message.content
+ role = None
+ else:
+
+ type_ = "message"
+ if role == Role.ASSISTANT.value:
+ content_type = "text"
+ content = [
+ Content(type=content_type, text=message.content),
+ ]
+ elif role == Role.USER.value:
+ content_type = "input_text"
+ content = [
+ Content(type=content_type, text=message.content),
+ ]
+ elif role == Role.SYSTEM.value:
+ content_type = "input_text"
+ content = [
+ Content(type=content_type, text=message.content),
+ ]
+ else:
+ content_type = "input_text"
+ content = [
+ Content(type=content_type, text=message.content),
+ ]
+ return cls(
+ id=id_,
+ type=type_,
+ role=role,
+ content=content,
+ arguments=arguments,
+ call_id=call_id,
+ output=output,
+ )
+
+ @classmethod
+ def from_audio_message(cls, message: Message) -> MessageItem:
+ pass
+
+ def has_audio(self) -> bool:
+ if self.content and len(self.content) > 0:
+ for c in self.content:
+ if c.type == "input_audio":
+ return True
+ return False
+
+ def get_audio_bytes(self) -> bytes:
+ buffer = BytesIO()
+ for c in self.content:
+ if c.audio:
+ data = base64.b64decode(c.audio)
+ buffer.write(data)
+ return buffer.getvalue()
+
+ def to_message_head(self) -> Message:
+
+ if self.type == "function_call_output":
+ return Message.new_head(
+ typ_=MessageType.FUNCTION_OUTPUT.value,
+ role=self.role or Role.ASSISTANT.value,
+ content=self.output,
+ msg_id=self.id,
+ call_id=self.call_id,
+ )
+ elif self.type == "function_call":
+ return Message.new_head(
+ typ_=MessageType.FUNCTION_CALL.value,
+ msg_id=self.id,
+ role=self.role or Role.ASSISTANT.value,
+ name=self.name,
+ call_id=self.call_id,
+ content=self.arguments,
+ )
+ elif self.type == "message":
+ content = ""
+ for c in self.content:
+ if c.text:
+ content += c.text
+ elif c.transcript:
+ content += c.transcript
+
+ typ_ = MessageType.DEFAULT.value
+ if self.role == Role.ASSISTANT.value:
+ typ_ = MessageType.AUDIO.value
+
+ return Message.new_head(
+ typ_=typ_,
+ msg_id=self.id,
+ role=self.role or "",
+ content=content,
+ )
+
+ else:
+ return Message.new_head(
+ msg_id=self.id,
+ role=self.role or "",
+ )
+
+ def to_complete_message(self) -> Optional[Message]:
+ if self.status == "incomplete":
+ return None
+ if self.type == "function_call_output":
+ return FunctionCallOutputMessage(
+ msg_id=self.id,
+ name=self.name,
+ call_id=self.call_id,
+ content=self.output,
+ ).to_message()
+ elif self.type == "function_call":
+ return FunctionCallMessage(
+ msg_id=self.id,
+ role=self.role or "",
+ caller=FunctionCaller(
+ id=self.call_id,
+ name=self.name,
+ arguments=self.arguments,
+ )
+ ).to_message()
+ elif self.type == "message":
+ parsed_type = MessageType.TEXT
+ if self.role == Role.ASSISTANT.value or self.has_audio():
+ parsed_type = MessageType.AUDIO
+
+ parsed_content = ""
+ for c in self.content:
+ if c.text:
+ parsed_content = parsed_content + c.text
+ elif c.transcript:
+ parsed_content = parsed_content + c.transcript
+ if not parsed_content:
+ return None
+
+ if parsed_type is MessageType.AUDIO:
+ return AudioMessage(
+ msg_id=self.id,
+ role=self.role or "",
+ content=parsed_content,
+ ).to_message()
+ else:
+ return Message.new_tail(
+ msg_id=self.id,
+ role=self.role or "",
+ content=parsed_content,
+ )
+ else:
+ return None
+
+
+class DeltaIndex(BaseModel):
+ response_id: str = Field("")
+ item_id: str = Field("")
+ output_index: int = Field(0)
+ content_index: int = Field(0)
+
+
+class ConversationObject(BaseModel):
+ id: str = Field("")
+ object: str = Field("realtime.conversation")
+
+
+class TurnDetection(BaseModel):
+ type: str = Field("server_vad")
+ threshold: float = Field(
+ 0.5,
+ description="Activation threshold for VAD (0.0 to 1.0), "
+ "this defaults to 0.5. A higher threshold will require louder audio to activate the model, "
+ "and thus might perform better in noisy environments.",
+ )
+ prefix_padding_ms: int = Field(
+ default=300,
+ description="Amount of audio to include before the VAD detected speech (in milliseconds). Defaults to 300ms."
+ )
+ silence_duration_ms: int = Field(
+ default=500,
+ description="Duration of silence to detect speech stop (in milliseconds). "
+ "Defaults to 500ms. "
+ "With shorter values the model will respond more quickly, "
+ "but may jump in on short pauses from the user."
+ )
+
+
+class InputAudioTranscription(BaseModel):
+ model: str = Field("whisper-1")
+
+
+class Voice(str, Enum):
+ alloy = "alloy"
+ echo = "echo"
+ shimmer = "shimmer"
+ ash = "ash"
+ ballad = "ballad"
+ coral = "coral"
+ sage = "sage"
+ verse = "verse"
+
+
+class OpenAIRealtimeModel(str, Enum):
+ gpt_4o_realtime_preview_2024_10_01 = "gpt-4o-realtime-preview-2024-10-01"
+ gpt_4o_mini_realtime_preview_2024_12_17 = "gpt-4o-mini-realtime-preview-2024-12-17"
+ gpt_4o_realtime_preview_2024_12_17 = "gpt-4o-realtime-preview-2024-12-17"
+
+
+class Tool(BaseModel):
+ type: Literal["function"] = "function"
+ name: str = Field()
+ description: str = Field()
+ parameters: Dict = Field(default_factory=dict)
+
+
+class SessionObjectBase(BaseModel):
+ """
+ immutable configuration for the openai session object
+ """
+ model: OpenAIRealtimeModel = Field(OpenAIRealtimeModel.gpt_4o_realtime_preview_2024_12_17)
+ modalities: List[str] = Field(default_factory=lambda: ["audio", "text"], enum={"text", "audio"})
+ voice: Voice = Field(
+ default="coral",
+ description="Voice to use",
+ )
+ input_audio_format: str = Field(
+ default="pcm16",
+ enum={"pcm16", "g711_ulaw", "g711_alaw"},
+ description="only support pcm16 yet",
+ )
+ output_audio_format: str = Field(
+ default="pcm16",
+ enum={"pcm16", "g711_ulaw", "g711_alaw"},
+ description="only support pcm16 yet",
+ )
+ turn_detection: Union[TurnDetection, None] = Field(
+ default_factory=TurnDetection,
+ description="Configuration for turn detection. "
+ "Can be set to null to turn off. "
+ "Server VAD means that the model will detect the start and end of speech based on audio volume "
+ "and respond at the end of user speech."
+ )
+ input_audio_transcription: Optional[InputAudioTranscription] = Field(
+ default_factory=InputAudioTranscription,
+ description="Configuration for input audio transcription."
+ )
+ instructions: str = Field(default="", description="instructions of the session")
+ tools: List[Tool] = Field(default_factory=list)
+ tool_choice: str = Field(default="auto")
+ temperature: float = Field(default=0.8)
+ max_response_output_tokens: Union[int, Literal['inf']] = Field(default='inf')
+
+
+class ResponseSettings(BaseModel):
+ modalities: List[str] = Field(default_factory=lambda: ["audio", "text"], enum={"text", "audio"})
+ voice: Voice = Field(
+ default="coral",
+ description="Voice to use",
+ )
+ output_audio_format: str = Field(
+ default="pcm16",
+ enum={"pcm16", "g711_ulaw", "g711_alaw"},
+ description="only support pcm16 yet",
+ )
+ instructions: str = Field(default="", description="instructions of the session")
+ tools: List[dict] = Field(default_factory=list)
+ tool_choice: str = Field(default="auto")
+ temperature: float = Field(default=0.8)
+
+ # max_response_output_tokens: Union[int, Literal['inf']] = Field(default='inf')
+
+ @classmethod
+ def from_session(cls, session: SessionObjectBase):
+ return cls(**session.model_dump())
+
+
+class SessionObject(SessionObjectBase):
+ """
+ full data model for openai realtime-api session object
+ """
+ id: str = Field(default="", description="id of the session")
+ object: Literal["realtime.session"] = "realtime.session"
diff --git a/ghostos/framework/openai_realtime/event_from_client.py b/ghostos/framework/openai_realtime/event_from_client.py
new file mode 100644
index 00000000..cb59b301
--- /dev/null
+++ b/ghostos/framework/openai_realtime/event_from_client.py
@@ -0,0 +1,138 @@
+from typing import Optional, ClassVar, Self
+from abc import ABC
+from enum import Enum
+from pydantic import BaseModel, Field
+from ghostos.framework.openai_realtime.event_data_objects import ResponseSettings, SessionObjectBase, MessageItem
+
+__all__ = [
+ 'ClientEventType',
+ 'ClientEvent',
+
+ # 9 client events
+
+ 'SessionUpdate',
+ 'ConversationItemCreate',
+ 'ConversationItemDelete',
+ 'ConversationItemTruncate',
+
+ 'InputAudioBufferClear',
+ 'InputAudioBufferAppend',
+ 'InputAudioBufferCommit',
+
+ 'ResponseCreate',
+ 'ResponseCancel'
+]
+
+
+class ClientEventType(str, Enum):
+ session_update = "session.update"
+ input_audio_buffer_append = "input_audio_buffer.append"
+ input_audio_buffer_commit = "input_audio_buffer.commit"
+ """
+ 1. This event will produce an error if the input audio buffer is empty.
+ 2. When in Server VAD mode, the client does not need to send this event.
+ 3. Committing the input audio buffer will trigger input audio transcription.
+ (if enabled in session configuration)
+ 4. it will not create a response from the model.
+ 5. The server will respond with an input_audio_buffer.committed event.
+ """
+
+ input_audio_buffer_clear = "input_audio_buffer.clear"
+
+ conversation_item_create = "conversation.item.create"
+ conversation_item_truncate = "conversation.item.truncate"
+ conversation_item_delete = "conversation.item.delete"
+
+ response_create = "response.create"
+ """
+ 1. When in Server VAD mode, the server will create Responses automatically.
+ 2. A Response will include at least one Item, and may have two, in which case the second will be a function call.
+ 3. These Items will be appended to the conversation history.
+ 4. The server will respond with:
+ 1) a response.created event,
+ 2) events for Items and content created,
+ 3) and finally a response.done event to indicate the Response is complete.
+ 5. The response.create event includes inference configuration like instructions, and temperature.
+ 6. These fields will override the Session's configuration for **this Response only**.
+ """
+
+ response_cancel = "response.cancel"
+ """
+ 1. The server will respond with a response.cancelled event
+ 2. or an error if there is no response to cancel.
+ """
+
+
+# ---- client side events ---- #
+
+
+class ClientEvent(BaseModel, ABC):
+ type: ClassVar[str]
+ event_id: Optional[str] = Field(
+ default=None,
+ description="Optional client-generated ID used to identify this event.",
+ )
+
+ def to_event_dict(self, exclude_none: bool = True) -> dict:
+ data = self.model_dump(exclude_none=exclude_none)
+ data["type"] = self.type
+ return data
+
+
+class SessionUpdate(ClientEvent):
+ type: ClassVar[str] = ClientEventType.session_update.value
+ session: SessionObjectBase
+
+
+class InputAudioBufferAppend(ClientEvent):
+ type: ClassVar[str] = ClientEventType.input_audio_buffer_append.value
+ audio: str = Field()
+
+
+class InputAudioBufferCommit(ClientEvent):
+ """
+ Send this event to commit the user input audio buffer,
+ which will create a new user message item in the conversation.
+ This event will produce an error if the input audio buffer is empty.
+ When in Server VAD mode, the client does not need to send this event,
+ the server will commit the audio buffer automatically.
+ Committing the input audio buffer will trigger input audio transcription (if enabled in session configuration),
+ but it will not create a response from the model.
+ The server will respond with an input_audio_buffer.committed event.
+ """
+ type: ClassVar[str] = ClientEventType.input_audio_buffer_commit.value
+
+
+class InputAudioBufferClear(ClientEvent):
+ """
+ Send this event to clear the audio bytes in the buffer.
+ The server will respond with an input_audio_buffer.cleared event.
+ """
+ type: ClassVar[str] = ClientEventType.input_audio_buffer_clear.value
+
+
+class ConversationItemCreate(ClientEvent):
+ type: ClassVar[str] = ClientEventType.conversation_item_create.value
+ previous_item_id: Optional[str] = Field(None)
+ item: MessageItem = Field()
+
+
+class ConversationItemTruncate(ClientEvent):
+ type: ClassVar[str] = ClientEventType.conversation_item_truncate.value
+ item_id: str = Field()
+ content_index: int = Field(0)
+ audio_end_ms: int = Field()
+
+
+class ConversationItemDelete(ClientEvent):
+ type: ClassVar[str] = ClientEventType.conversation_item_delete.value
+ item_id: str = Field()
+
+
+class ResponseCreate(ClientEvent):
+ type: ClassVar[str] = ClientEventType.response_create.value
+ response: Optional[ResponseSettings] = Field(None)
+
+
+class ResponseCancel(ClientEvent):
+ type: ClassVar[str] = ClientEventType.response_cancel.value
diff --git a/ghostos/framework/openai_realtime/event_from_server.py b/ghostos/framework/openai_realtime/event_from_server.py
new file mode 100644
index 00000000..bfebf2f8
--- /dev/null
+++ b/ghostos/framework/openai_realtime/event_from_server.py
@@ -0,0 +1,472 @@
+import base64
+from typing import Self, List, Optional, Union, ClassVar
+from abc import ABC
+from enum import Enum
+from pydantic import BaseModel, Field
+from ghostos.core.messages import Message as GhostOSMessage, MessageType
+from ghostos.framework.openai_realtime.event_data_objects import (
+ RateLimit, Response, MessageItem,
+ DeltaIndex, ConversationObject, Error, SessionObject,
+ ResponseSettings,
+ Content,
+)
+
+__all__ = [
+ 'ServerEventType',
+ 'ServerEvent',
+ # 28 server events
+
+ 'ServerError',
+
+ # 2 session events
+ 'ServerSessionUpdated',
+ 'ServerSessionCreated',
+
+ 'ConversationCreated',
+
+ # rate event
+ 'RateLimitsUpdated',
+
+ # 4 input audio
+ 'InputAudioBufferSpeechStarted',
+ 'InputAudioBufferCommitted',
+ 'InputAudioBufferCleared',
+ 'InputAudioBufferSpeechStopped',
+
+ # 5 conversation item events
+ 'ConversationItemCreated',
+ 'ConversationItemTruncated',
+ 'ConversationInputAudioTranscriptionCompleted',
+ 'ConversationInputAudioTranscriptionFailed',
+ 'ConversationItemDeleted',
+
+ # 14 response events
+ 'ResponseCreated',
+ 'ResponseDone',
+
+ 'ResponseOutputItemAdded',
+ 'ResponseOutputItemDone',
+
+ 'ResponseContentPartAdded',
+ 'ResponseContentPartDone',
+ # delta
+ 'ResponseAudioDelta',
+ 'ResponseAudioDone',
+
+ 'ResponseAudioTranscriptDelta',
+ 'ResponseAudioTranscriptDone',
+
+ 'ResponseTextDelta',
+ 'ResponseTextDone',
+
+ 'ResponseFunctionCallArgumentsDelta',
+ 'ResponseFunctionCallArgumentsDone',
+
+]
+
+
+class ServerEventType(str, Enum):
+ # recover-able error
+ error = "error"
+
+ # non-block inform
+ session_created = "session.created"
+ session_updated = "session.updated"
+
+ conversation_created = "conversation.created"
+
+ # streaming items
+
+ # complete message item alignments
+ conversation_item_created = "conversation.item.created"
+ conversation_item_input_audio_transcription_completed = "conversation.item.input_audio_transcription.completed"
+ conversation_item_input_audio_transcription_failed = "conversation.item.input_audio_transcription.failed"
+ conversation_item_truncated = "conversation.item.truncated"
+ conversation_item_deleted = "conversation.item.deleted"
+
+ input_audio_buffer_committed = "input_audio_buffer.committed"
+ input_audio_buffer_cleared = "input_audio_buffer.cleared"
+ input_audio_buffer_speech_started = "input_audio_buffer.speech_started"
+ input_audio_buffer_speech_stopped = "input_audio_buffer.speech_stopped"
+
+ # 14 response events
+ response_created = "response.created"
+ response_done = "response.done"
+
+ response_output_item_added = "response.output_item.added"
+ response_output_item_done = "response.output_item.done"
+
+ response_content_part_added = "response.content_part.added"
+ response_content_part_done = "response.content_part.done"
+
+ response_text_delta = "response.text.delta"
+ response_text_done = "response.text.done"
+
+ response_audio_transcript_delta = "response.audio_transcript.delta"
+ response_audio_transcript_done = "response.audio_transcript.done"
+
+ response_audio_delta = "response.audio.delta"
+ response_audio_done = "response.audio.done"
+
+ response_function_call_arguments_delta = "response.function_call_arguments.delta"
+ response_function_call_arguments_done = "response.function_call_arguments.done"
+
+ # system
+ rate_limits_updated = "rate_limits.updated"
+
+ @classmethod
+ def get_type(cls, event: dict) -> Self:
+ return cls(event.get("type", ""))
+
+ @classmethod
+ def is_session_event(cls, event: dict, e_type: Optional[str] = None) -> bool:
+ if e_type is None:
+ e_type = event.get("type", "")
+ return e_type.startswith("session.")
+
+ @classmethod
+ def is_input_audio_event(cls, event: dict, e_type: Optional[str] = None) -> bool:
+ if e_type is None:
+ e_type = event.get("type", "")
+ return e_type.startswith("input_audio_buffer.")
+
+ @classmethod
+ def is_respond_event(cls, event: dict, e_type: Optional[str] = None) -> bool:
+ if e_type is None:
+ e_type = event.get("type", "")
+ return e_type.startswith("response.")
+
+ @classmethod
+ def is_conversation_event(cls, event: dict, e_type: Optional[str] = None) -> bool:
+ if e_type is None:
+ e_type = event.get("type", "")
+ return e_type.startswith("conversation.")
+
+ @classmethod
+ def is_conversation_item_event(cls, event: dict, e_type: Optional[str] = None) -> bool:
+ if e_type is None:
+ e_type = event.get("type", "")
+ return e_type.startswith("conversation.item")
+
+ @classmethod
+ def get_event_id(cls, event: dict) -> str:
+ return event.get("event_id", "")
+
+ @classmethod
+ def get_response_id(cls, event: dict) -> Union[str, None]:
+ if "response" in event:
+ return event["response"].get("id", None)
+ if "response_id" in event:
+ return event["response_id"]
+ return None
+
+ @classmethod
+ def get_item_id(cls, event: dict) -> Optional[str]:
+ if "item_id" in event:
+ return event["item_id"]
+ elif "item" in event:
+ item = event['item']
+ if isinstance(item, dict) and "item_id" in item:
+ return item["item_id"]
+ return None
+
+ def match(self, event: dict) -> bool:
+ return "type" in event and event["type"] == self.value
+
+
+# ---- server side events ---- #
+
+class ServerEvent(BaseModel, ABC):
+ type: ClassVar[str]
+ event_id: str = Field(description="Optional client-generated ID used to identify this event.")
+
+
+class ServerError(ServerEvent):
+ type: ClassVar[str] = ServerEventType.error.value
+ error: Error
+
+
+class ServerSessionCreated(ServerEvent):
+ type: ClassVar[str] = ServerEventType.session_created.value
+ session: SessionObject
+
+
+class ServerSessionUpdated(ServerEvent):
+ type: ClassVar[str] = ServerEventType.session_updated.value
+ session: SessionObject
+
+
+class ConversationCreated(ServerEvent):
+ type: ClassVar[str] = ServerEventType.conversation_created.value
+ conversation: ConversationObject
+
+
+class ConversationItemCreated(ServerEvent):
+ """
+ Returned when a conversation item is created.
+ There are several scenarios that produce this event:
+
+ 1. The server is generating a Response,
+ which if successful will produce either one or two Items,
+ which will be of type message (role assistant) or type function_call.
+ 2. The input audio buffer has been committed,
+ either by the client or the server (in server_vad mode).
+ The server will take the content of the input audio buffer and add it to a new user message Item.
+ 3. The client has sent a conversation.item.create event to add a new Item to the Conversation.
+ """
+ type: ClassVar[str] = ServerEventType.conversation_item_created.value
+ previous_item_id: Optional[str] = Field(None)
+ item: MessageItem = Field()
+
+
+class ConversationItemDeleted(ServerEvent):
+ type: ClassVar[str] = ServerEventType.conversation_item_deleted.value
+ item_id: str = Field("")
+
+
+class ConversationInputAudioTranscriptionCompleted(ServerEvent):
+ """
+ This event is the output of audio transcription for user audio written to the user audio buffer.
+ Transcription begins when the input audio buffer is committed by the client or server
+ (in server_vad mode).
+ Transcription runs asynchronously with Response creation,
+ so this event may come before or after the Response events.
+ Realtime API models accept audio natively,
+ and thus input transcription is a separate process run on a separate ASR (Automatic Speech Recognition) model,
+ currently always whisper-1.
+ Thus the transcript may diverge somewhat from the model's interpretation,
+ and should be treated as a rough guide.
+ """
+ type: ClassVar[str] = ServerEventType.conversation_item_input_audio_transcription_completed.value
+ item_id: str = Field("")
+ content_index: int = Field(default=0)
+ transcript: str = Field(default="")
+
+
+class ConversationInputAudioTranscriptionFailed(ServerEvent):
+ type: ClassVar[str] = ServerEventType.conversation_item_input_audio_transcription_failed.value
+ item_id: str = Field("")
+ content_index: int = Field(default=0)
+ error: Error = Field()
+
+
+class ConversationItemTruncated(ServerEvent):
+ """
+ Returned when an earlier assistant audio message item is truncated by the client with
+ a conversation.item.truncate event.
+ This event is used to synchronize the server's understanding of the audio with the client's playback.
+ This action will truncate the audio and remove the server-side text transcript
+ to ensure there is no text in the context that hasn't been heard by the user.
+ """
+ type: ClassVar[str] = ServerEventType.conversation_item_truncated.value
+ item_id: str = Field("")
+ content_index: int = Field(default=0)
+ audio_end_ms: int = Field(default=0)
+
+
+class InputAudioBufferCommitted(ServerEvent):
+ type: ClassVar[str] = ServerEventType.input_audio_buffer_committed.value
+ previous_item_id: Optional[str] = Field(None)
+ item_id: str = Field("")
+
+
+class InputAudioBufferSpeechStarted(ServerEvent):
+ type: ClassVar[str] = ServerEventType.input_audio_buffer_speech_started.value
+ audio_start_ms: int = Field(default=0)
+ item_id: str = Field("")
+
+
+class InputAudioBufferSpeechStopped(ServerEvent):
+ type: ClassVar[str] = ServerEventType.input_audio_buffer_speech_stopped.value
+ audio_end_ms: int = Field(default=0)
+ item_id: str = Field("")
+
+
+class InputAudioBufferCleared(ServerEvent):
+ type: ClassVar[str] = ServerEventType.input_audio_buffer_cleared.value
+
+
+class ResponseCreated(ServerEvent):
+ type: ClassVar[str] = ServerEventType.response_created.value
+ response: Response = Field()
+
+
+class ResponseDone(ServerEvent):
+ type: ClassVar[str] = ServerEventType.response_done.value
+ response: Response = Field()
+
+
+class ResponseOutputItemAdded(ServerEvent):
+ type: ClassVar[str] = ServerEventType.response_output_item_added.value
+ response_id: str = Field("")
+ output_index: int = Field(0)
+ item: MessageItem = Field()
+
+
+class ResponseOutputItemDone(ServerEvent):
+ type: ClassVar[str] = ServerEventType.response_output_item_done.value
+ response_id: str = Field("")
+ output_index: int = Field(0)
+ item: MessageItem = Field()
+
+
+class ResponseContentPartAdded(ServerEvent):
+ type: ClassVar[str] = ServerEventType.response_content_part_added.value
+ response_id: str = Field("")
+ content_index: int = Field(0)
+ output_index: int = Field(0)
+ item_id: str = Field("")
+ part: Content = Field(None, description="The content part that was added. shall not be None.")
+
+ def as_message_chunk(self) -> Optional[GhostOSMessage]:
+ if self.part is None:
+ return None
+ if self.part.transcript:
+ return GhostOSMessage.new_chunk(
+ msg_id=self.item_id,
+ content=self.part.transcript,
+ )
+ elif self.part.text:
+ return GhostOSMessage.new_chunk(
+ msg_id=self.item_id,
+ content=self.part.text,
+ )
+ else:
+ return None
+
+
+class ResponseContentPartDone(ServerEvent):
+ type: ClassVar[str] = ServerEventType.response_content_part_done.value
+ response_id: str = Field("")
+ output_index: int = Field(0)
+ item_id: str = Field("")
+ part: Content = Field()
+
+ def as_message_chunk(self) -> Optional[GhostOSMessage]:
+ if self.part is None:
+ return None
+ if self.part.transcript:
+ return GhostOSMessage.new_chunk(
+ msg_id=self.item_id,
+ content=self.part.transcript,
+ )
+ elif self.part.text:
+ return GhostOSMessage.new_chunk(
+ msg_id=self.item_id,
+ content=self.part.text,
+ )
+ else:
+ return None
+
+
+class ResponseTextDelta(DeltaIndex, ServerEvent):
+ type: ClassVar[str] = ServerEventType.response_text_delta.value
+ delta: str = Field("")
+
+ def as_content(self) -> Content:
+ return Content(
+ type="text",
+ text=self.delta,
+ )
+
+ def as_message_chunk(self) -> Optional[GhostOSMessage]:
+ if self.delta:
+ return GhostOSMessage.new_chunk(
+ msg_id=self.item_id,
+ content=self.delta,
+ )
+ return None
+
+
+class ResponseTextDone(DeltaIndex, ServerEvent):
+ type: ClassVar[str] = ServerEventType.response_text_done.value
+ text: str = Field("")
+
+ def as_content(self) -> Content:
+ return Content(
+ type="text",
+ text=self.text,
+ )
+
+
+class ResponseAudioTranscriptDelta(DeltaIndex, ServerEvent):
+ type: ClassVar[str] = ServerEventType.response_audio_transcript_delta.value
+ delta: str = Field("")
+
+ def as_content(self) -> Content:
+ return Content(
+ type="audio",
+ transcript=self.delta,
+ )
+
+ def as_message_chunk(self) -> Optional[GhostOSMessage]:
+ if self.delta:
+ return GhostOSMessage.new_chunk(
+ msg_id=self.item_id,
+ content=self.delta,
+ )
+ return None
+
+
+class ResponseAudioTranscriptDone(DeltaIndex, ServerEvent):
+ type: ClassVar[str] = ServerEventType.response_audio_transcript_done.value
+ transcript: str = Field("")
+
+ def as_content(self) -> Content:
+ return Content(
+ type="audio",
+ transcript=self.delta,
+ )
+
+
+class ResponseAudioDelta(DeltaIndex, ServerEvent):
+ type: ClassVar[str] = ServerEventType.response_audio_delta.value
+ delta: str = Field("")
+
+ def get_audio_bytes(self) -> bytes:
+ return base64.b64decode(self.delta)
+
+
+class ResponseAudioDone(DeltaIndex, ServerEvent):
+ type: ClassVar[str] = ServerEventType.response_audio_transcript_done.value
+
+
+class ResponseFunctionCallArgumentsDelta(DeltaIndex, ServerEvent):
+ type: ClassVar[str] = ServerEventType.response_function_call_arguments_delta.value
+ delta: str = Field("")
+
+ def as_message_chunk(self) -> Optional[GhostOSMessage]:
+ if self.delta:
+ return GhostOSMessage.new_chunk(
+ typ_=MessageType.FUNCTION_CALL.value,
+ msg_id=self.item_id,
+ content=self.delta,
+ )
+ return None
+
+
+class ResponseFunctionCallArgumentsDone(DeltaIndex, ServerEvent):
+ type: ClassVar[str] = ServerEventType.response_function_call_arguments_done.value
+ call_id: str = Field("")
+ arguments: str = Field("")
+ name: str = Field("")
+
+ def as_message(self) -> GhostOSMessage:
+ return GhostOSMessage.new_tail(
+ msg_id=self.item_id,
+ type_=MessageType.FUNCTION_CALL.value,
+ name=self.name,
+ content=self.arguments,
+ call_id=self.call_id,
+ )
+
+
+class RateLimitsUpdated(ServerEvent):
+ """
+ Emitted at the beginning of a Response to indicate the updated rate limits.
+ When a Response is created some tokens will be "reserved" for the output tokens,
+ the rate limits shown here reflect that reservation,
+ which is then adjusted accordingly once the Response is completed.
+ """
+ type: ClassVar[str] = ServerEventType.rate_limits_updated.value
+ rate_limits: List[RateLimit] = Field(default_factory=list)
diff --git a/ghostos/framework/openai_realtime/output.py b/ghostos/framework/openai_realtime/output.py
new file mode 100644
index 00000000..d1c8461f
--- /dev/null
+++ b/ghostos/framework/openai_realtime/output.py
@@ -0,0 +1,316 @@
+import time
+from abc import ABC, abstractmethod
+from typing import List, Optional, Dict, Iterable, Callable, Set
+from queue import Queue
+from ghostos.contracts.logger import LoggerItf
+from ghostos.core.messages import Message, ReceiverBuffer, SequencePipe
+
+
+class OutputBuffer(ABC):
+ @abstractmethod
+ def stop_output(self, response_id: Optional[str]):
+ """
+ stop the current response.
+ """
+ pass
+
+ @abstractmethod
+ def end_output(self, response_id: str):
+ pass
+
+ @abstractmethod
+ def start_output(self, response_id: str):
+ """
+ start a new response
+ :param response_id:
+ :return:
+ """
+ pass
+
+ @abstractmethod
+ def stop_speaking(self):
+ pass
+
+ @abstractmethod
+ def is_speaking(self):
+ pass
+
+ @abstractmethod
+ def add_response_chunk(self, response_id: str, chunk: Message) -> bool:
+ """
+ add a response chunk to certain response.
+ :param response_id:
+ :param chunk:
+ :return:
+ """
+ pass
+
+ @abstractmethod
+ def add_message(self, message: Message, previous_item_id: Optional[str]) -> bool:
+ """
+ add complete message to the output. the already sent message will not be sent again.
+ :param message:
+ :param previous_item_id:
+ :return:
+ """
+ pass
+
+ @abstractmethod
+ def get_outputted_messages(self) -> List[Message]:
+ """
+ get already outputted messages.
+ :return:
+ """
+ pass
+
+ @abstractmethod
+ def get_response_id(self) -> Optional[str]:
+ """
+ get current response id.
+ :return:
+ """
+ pass
+
+ @abstractmethod
+ def add_audio_output(self, response_id: str, data: Optional[bytes], filetype: str = "wav") -> bool:
+ """
+ send an audio message to output.
+ :param response_id:
+ :param data:
+ :param filetype:
+ :return:
+ """
+ pass
+
+ @abstractmethod
+ def add_error_message(self, error: Message):
+ """
+ add error message
+ :param error:
+ :return:
+ """
+ pass
+
+ @abstractmethod
+ def output_received(self) -> Optional[ReceiverBuffer]:
+ """
+ :return:
+ """
+ pass
+
+ @abstractmethod
+ def speaking_queue(self, response_id: str) -> Optional[Queue]:
+ """
+ get uncanceled response speaking queue.
+ :param response_id:
+ :return:
+ """
+ pass
+
+
+class DefaultOutputBuffer(OutputBuffer):
+
+ def __init__(
+ self,
+ is_close_check: Callable[[], bool],
+ logger: LoggerItf,
+ ):
+ self.is_close_check = is_close_check
+ self.logger = logger
+
+ # response stream
+ self.response_id: Optional[str] = None
+ self.response_item_ids: Optional[List[str]] = None
+ self.responding_item_id: Optional[str] = None
+ self.response_chunks: Optional[Dict[str, List[Message]]] = None
+
+ # speaking
+ self.speak_queue: Optional[Queue] = None
+
+ self.outputted_message_ids: List[str] = []
+ self.outputted_messages: Dict[str, Message] = {}
+ """the outputted messages in order"""
+
+ self.error_messages: List[Message] = []
+ """unsent error messages"""
+ self._is_speaking: bool = False
+
+ self.unsent_message_ids: List[str] = []
+ self.sent_message_ids: Set[str] = set()
+
+ def stop_output(self, response_id: Optional[str]):
+ self.logger.debug("start output")
+ if response_id is None or response_id == self.response_id:
+ self.response_id = None
+ self.response_chunks = None
+ self.response_item_ids = None
+ self.responding_item_id = None
+ self.stop_speaking()
+
+ def end_output(self, response_id: str):
+ # self.response_id = None
+ # self.response_chunks = None
+ # self.response_item_ids = None
+ # self.responding_item_id = None
+ if response_id == self.response_id and self.speak_queue is not None:
+ self.logger.debug("send none to speaking queue but not stop speaking")
+ self.speak_queue.put(None, block=False)
+
+ def start_output(self, response_id: str):
+ self.stop_output(None)
+ self.logger.debug("start output")
+ self.response_id = response_id
+ self.response_chunks = {}
+ self.response_item_ids = []
+ self.responding_item_id = None
+ self.start_speaking()
+
+ def start_speaking(self):
+ self._is_speaking = True
+ if self.speak_queue is not None:
+ self.speak_queue.put(None, block=False)
+ self.speak_queue = Queue()
+ self.logger.debug("start output speaking")
+
+ def stop_speaking(self):
+ self.logger.debug("stop output speaking")
+ self._is_speaking = False
+ if self.speak_queue is not None:
+ self.speak_queue.put(None, block=False)
+ self.logger.debug("speaking queue send none")
+ self.speak_queue = None
+
+ def is_speaking(self):
+ return self._is_speaking
+
+ def add_message(self, message: Message, previous_item_id: Optional[str]) -> bool:
+ if message is None or not message.is_complete():
+ return False
+ if not message.content:
+ return False
+ msg_id = message.msg_id
+ # the message is a new item.
+ if msg_id not in self.outputted_message_ids:
+ self.outputted_messages[msg_id] = message
+ self.outputted_message_ids.append(msg_id)
+ self.unsent_message_ids.append(msg_id)
+ else:
+ self.outputted_messages[msg_id] = message
+ # re-range messages
+ if previous_item_id is not None:
+ outputted_message_ids = []
+ current_message_id = msg_id
+ inserted = False
+ for msg_id in self.outputted_message_ids:
+ if msg_id == current_message_id:
+ continue
+ outputted_message_ids.append(msg_id)
+ if msg_id == previous_item_id:
+ outputted_message_ids.append(current_message_id)
+ inserted = True
+ if not inserted:
+ outputted_message_ids.append(current_message_id)
+ self.outputted_message_ids = outputted_message_ids
+
+ return True
+
+ def add_response_chunk(self, response_id: str, chunk: Message) -> bool:
+ if chunk is None:
+ return False
+ if response_id != self.response_id:
+ return False
+ if self.response_chunks is None:
+ self.response_chunks = {}
+ if chunk.msg_id:
+ self.responding_item_id = chunk.msg_id
+ if not self.responding_item_id:
+ self.responding_item_id = ""
+ if self.responding_item_id not in self.response_chunks:
+ self.response_chunks[self.responding_item_id] = []
+ self.response_item_ids.append(self.responding_item_id)
+ chunks = self.response_chunks[self.responding_item_id]
+ chunks.append(chunk)
+ return True
+
+ def get_outputted_messages(self) -> List[Message]:
+ messages = []
+ for msg_id in self.outputted_message_ids:
+ message = self.outputted_messages[msg_id]
+ messages.append(message)
+ return messages
+
+ def get_response_id(self) -> Optional[str]:
+ return self.response_id
+
+ def add_audio_output(self, response_id: str, data: Optional[bytes], filetype: str = "wav") -> bool:
+ if response_id != self.response_id:
+ return False
+ queue = self.speak_queue
+ if queue is None:
+ return False
+ queue.put(data)
+ return True
+
+ def add_error_message(self, error: Message):
+ self.error_messages.append(error)
+
+ def speaking_queue(self, response_id: str) -> Optional[Queue]:
+ return self.speak_queue
+
+ def output_received(self) -> Optional[ReceiverBuffer]:
+ chunks = self._output_chunks()
+ if chunks is None:
+ return None
+
+ sent = SequencePipe().across(chunks)
+ return ReceiverBuffer.new(sent)
+
+ def _output_chunks(self) -> Optional[Iterable[Message]]:
+ # first of all, the error message is priory
+ if len(self.error_messages) > 0:
+ error = self.error_messages.pop(0)
+ if error.msg_id not in self.sent_message_ids:
+ yield from [error]
+ self.sent_message_ids.add(error.msg_id)
+ return
+
+ # if there are unsent complete message, send it.
+ if len(self.unsent_message_ids) > 0:
+ msg_id = self.unsent_message_ids.pop(0)
+ if msg_id in self.outputted_messages and msg_id not in self.sent_message_ids:
+ message = self.outputted_messages[msg_id]
+ yield from [message]
+ self.sent_message_ids.add(msg_id)
+ return
+
+ # output current responding
+ if self.response_id is None:
+ return None
+
+ output_item_id = ""
+ response_id = self.response_id
+ if len(self.response_item_ids) > 0:
+ output_item_id = self.response_item_ids.pop(0)
+ if not output_item_id or output_item_id in self.sent_message_ids:
+ return None
+
+ while not self.is_close_check():
+ if response_id != self.response_id or self.response_chunks is None:
+ # stream canceled
+ break
+ if self.response_chunks is None:
+ break
+ if output_item_id in self.outputted_messages:
+ break
+
+ chunks = self.response_chunks[output_item_id]
+
+ if len(chunks) > 0:
+ first = chunks.pop(0)
+ yield first
+ else:
+ time.sleep(0.1)
+
+ if output_item_id in self.outputted_messages:
+ yield self.outputted_messages[output_item_id]
+ self.sent_message_ids.add(output_item_id)
diff --git a/ghostos/framework/openai_realtime/state_of_client.py b/ghostos/framework/openai_realtime/state_of_client.py
new file mode 100644
index 00000000..b407fb5e
--- /dev/null
+++ b/ghostos/framework/openai_realtime/state_of_client.py
@@ -0,0 +1,389 @@
+from __future__ import annotations
+from typing import List, Optional, Tuple, Protocol, Self
+from abc import ABC, abstractmethod
+from ghostos.framework.openai_realtime.configs import OpenAIRealtimeAppConf
+from ghostos.core.runtime import Event as GhostOSEvent
+from ghostos.abcd.realtime import Operator, OperatorName
+from ghostos.contracts.logger import LoggerItf
+from enum import Enum
+
+
+class AppState(str, Enum):
+ created = "created"
+ closed = "closed"
+
+ connecting = "connecting"
+ """connecting to the websocket server"""
+
+ synchronizing = "synchronizing"
+ """synchronizing conversation with the websocket server"""
+
+ waiting_response = "waiting response"
+
+ updating = "updating"
+ """updating the local conversation from server state"""
+
+ listening = "listening"
+ """listening on the local audio inputs"""
+
+ responding = "responding"
+ """responding from the server"""
+
+ idle = "idle"
+
+ function_call = "function calling"
+ """local conversation function call"""
+
+
+listen_op = OperatorName.listen.new("start listening and sending audio buffer")
+respond_op = OperatorName.respond.new("create a response")
+clear_audio_op = OperatorName.clear_audio.new("clear input audio buffer")
+stop_listen_op = OperatorName.stop_listen.new("stop listening but do nothing.")
+cancel_response_op = OperatorName.cancel_responding.new("cancel current responding")
+
+
+class Client(Protocol):
+ conf: OpenAIRealtimeAppConf
+ logger: LoggerItf
+
+ @abstractmethod
+ def reconnect(self) -> None:
+ """
+ recreate the ws connection.
+ :return:
+ """
+ pass
+
+ @abstractmethod
+ def vad_mode(self) -> bool:
+ pass
+
+ @abstractmethod
+ def update_session(self):
+ pass
+
+ @abstractmethod
+ def synchronize_server_session(self):
+ """
+ :return:
+ """
+ pass
+
+ @abstractmethod
+ def cancel_responding(self) -> bool:
+ """
+ cancel server responding and local output
+ :return:
+ """
+ pass
+
+ @abstractmethod
+ def start_listening(self) -> bool:
+ pass
+
+ @abstractmethod
+ def stop_listening(self) -> bool:
+ pass
+
+ @abstractmethod
+ def is_listening(self) -> bool:
+ pass
+
+ @abstractmethod
+ def listen_mode(self) -> bool:
+ pass
+
+ @abstractmethod
+ def is_server_responding(self) -> bool:
+ pass
+
+ def is_responding(self) -> bool:
+ return self.is_server_responding() or self.is_speaking()
+
+ @abstractmethod
+ def is_speaking(self) -> bool:
+ pass
+
+ @abstractmethod
+ def audio_buffer_append(self, buffer: bytes) -> None:
+ pass
+
+ @abstractmethod
+ def commit_audio_input(self) -> bool:
+ """
+ and stop listening.
+ :return:
+ """
+ pass
+
+ @abstractmethod
+ def clear_audio_input(self) -> bool:
+ pass
+
+ @abstractmethod
+ def create_response(self) -> bool:
+ pass
+
+ @abstractmethod
+ def receive_server_event(self) -> bool:
+ pass
+
+ @abstractmethod
+ def respond_error_message(self, error: str) -> None:
+ pass
+
+ @abstractmethod
+ def handle_ghostos_event(self, event: GhostOSEvent):
+ pass
+
+
+class StateOfClient(ABC):
+
+ def __init__(
+ self,
+ client: Client,
+ ):
+ self.client = client
+
+ @abstractmethod
+ def on_init(self):
+ """
+ 同步阻塞逻辑.
+ """
+ pass
+
+ @abstractmethod
+ def state_name(self) -> str:
+ """
+
+ :return:
+ """
+ pass
+
+ def recv_server_event(self) -> bool:
+ return self.client.receive_server_event()
+
+ @abstractmethod
+ def operate(self, operator: Operator) -> Optional[Self]:
+ pass
+
+ def allow(self, operator: Operator) -> bool:
+ operators = self.operators()
+ return operator.name in {op.name for op in operators}
+
+ @abstractmethod
+ def operators(self) -> List[Operator]:
+ pass
+
+ @abstractmethod
+ def tick_frame(self) -> Optional[Self]:
+ pass
+
+ def destroy(self):
+ self.client = None
+
+ def default_mode(self) -> Self:
+ if self.client.listen_mode():
+ # start vad mode and listening to anything.
+ return ListeningState(self.client)
+ else:
+ # wait for user's operator.
+ return IdleState(self.client)
+
+
+class ConnectingState(StateOfClient):
+ """
+ connecting the websocket server
+ """
+
+ def state_name(self) -> str:
+ # when connecting nothing is able to do.
+ return AppState.connecting.value
+
+ def on_init(self):
+ self.client.reconnect()
+
+ def allow(self, operator: str) -> bool:
+ return False
+
+ def operate(self, operator: str) -> Optional[Self]:
+ return None
+
+ def recv_server_event(self) -> bool:
+ return False
+
+ def operators(self) -> List[str]:
+ return []
+
+ def tick_frame(self) -> Optional[Self]:
+ return SynchronizingState(self.client)
+
+
+class SynchronizingState(StateOfClient):
+ """
+ synchronizing conversation history to the websocket server
+ """
+
+ def state_name(self) -> str:
+ return AppState.synchronizing.value
+
+ def allow(self, operator: str) -> bool:
+ return False
+
+ def recv_server_event(self) -> bool:
+ return False
+
+ def operate(self, operator: str) -> Optional[Self]:
+ return None
+
+ def operators(self) -> List[str]:
+ return []
+
+ def on_init(self):
+ self.client.synchronize_server_session()
+
+ def tick_frame(self) -> Optional[Self]:
+ return self.default_mode()
+
+
+class ListeningState(StateOfClient):
+
+ def on_init(self):
+ self.client.start_listening()
+
+ def state_name(self) -> str:
+ return AppState.listening.value
+
+ def operators(self) -> List[str]:
+ return [
+ respond_op,
+ stop_listen_op,
+ clear_audio_op,
+ ]
+
+ def operate(self, operator: Operator) -> Optional[Self]:
+ name = operator.name
+ if name == OperatorName.respond.value:
+ # commit and create response.
+ return CreateResponseState(self.client)
+
+ elif name == OperatorName.stop_listen:
+ # stop listening, and do nothing.
+ # do not clear the input audio buffer automatically.
+ self.client.stop_listening()
+ return IdleState(self.client)
+
+ elif name == OperatorName.clear_audio:
+ self.client.clear_audio_input()
+ # clear and stay idle
+ return IdleState(self.client)
+
+ else:
+ return None
+
+ def tick_frame(self) -> Optional[Self]:
+ if self.client.is_server_responding():
+ # responding not cancel listening
+ return RespondingState(self.client)
+ if not self.client.listen_mode():
+ return self.default_mode()
+ return None
+
+
+class CreateResponseState(StateOfClient):
+
+ def on_init(self):
+ if self.client.is_server_responding():
+ # if create response while responding, cancel current one first.
+ self.client.cancel_responding()
+ elif self.client.is_listening():
+ # if create response while listening, commit it
+ self.client.commit_audio_input()
+
+ # create response.
+ self.client.create_response()
+ return
+
+ def state_name(self) -> str:
+ return str(AppState.waiting_response.value)
+
+ def operate(self, operator: Operator) -> Optional[Self]:
+ return None
+
+ def operators(self) -> List[Operator]:
+ # when creating responding, no operators allowed.
+ return []
+
+ def tick_frame(self) -> Optional[Self]:
+ if self.client.is_server_responding():
+ return RespondingState(self.client)
+ return None
+
+
+class RespondingState(StateOfClient):
+
+ def on_init(self):
+ if not self.client.is_server_responding():
+ self.client.respond_error_message("enter responding state but server is not responding")
+ return
+
+ def state_name(self) -> str:
+ return str(AppState.responding.value)
+
+ def operate(self, operator: Operator) -> Optional[Self]:
+ name = operator.name
+ if name == OperatorName.cancel_responding.value:
+ # cancel current responding.
+ if self.client.is_responding():
+ self.client.cancel_responding()
+ return self.default_mode()
+
+ elif name == OperatorName.listen.value:
+ if self.client.is_responding():
+ self.client.cancel_responding()
+ return ListeningState(self.client)
+ else:
+ return None
+
+ def operators(self) -> List[Operator]:
+ return [
+ cancel_response_op,
+ listen_op,
+ ]
+
+ def tick_frame(self) -> Optional[Self]:
+ if self.client.is_responding():
+ return None
+ else:
+ self.client.logger.debug("responding state return default mode")
+ return self.default_mode()
+
+
+class IdleState(StateOfClient):
+
+ def on_init(self):
+ if self.client.is_listening():
+ self.client.stop_listening()
+ elif self.client.is_responding():
+ self.client.cancel_responding()
+
+ def state_name(self) -> str:
+ return AppState.idle.value
+
+ def operate(self, operator: Operator) -> Optional[Self]:
+ if operator.name == OperatorName.listen.value:
+ return ListeningState(self.client)
+ elif operator.name == OperatorName.respond.value:
+ return CreateResponseState(self.client)
+ return None
+
+ def operators(self) -> List[Operator]:
+ return [
+ listen_op,
+ respond_op,
+ ]
+
+ def tick_frame(self) -> Optional[Self]:
+ if self.client.is_listening():
+ return ListeningState(self.client)
+ return None
diff --git a/ghostos/framework/openai_realtime/state_of_server.py b/ghostos/framework/openai_realtime/state_of_server.py
new file mode 100644
index 00000000..ad728c4c
--- /dev/null
+++ b/ghostos/framework/openai_realtime/state_of_server.py
@@ -0,0 +1,697 @@
+from __future__ import annotations
+from abc import ABC, abstractmethod
+from typing import Protocol, Optional, Dict, Self, List, Union
+from ghostos.framework.openai_realtime.event_from_server import *
+from ghostos.framework.openai_realtime.event_data_objects import (
+ MessageItem,
+ RateLimit,
+ SessionObject,
+)
+from pydantic import ValidationError
+from ghostos.core.messages import Message, MessageType
+from ghostos.contracts.logger import LoggerItf
+from ghostos.container import get_caller_info
+
+
+class ServerContext(Protocol):
+ logger: LoggerItf
+
+ @abstractmethod
+ def respond_message_chunk(self, response_id: str, chunk: Union[Message, None]) -> bool:
+ """
+ respond a message chunk usually a text chunk.
+ :param response_id:
+ :param chunk:
+ :return:
+ """
+ pass
+
+ @abstractmethod
+ def respond_error_message(self, error: str) -> None:
+ """
+ output error message
+ :param error:
+ :return:
+ """
+ pass
+
+ @abstractmethod
+ def update_history_message(self, message: Message) -> None:
+ """
+ update history message
+ :param message:
+ :return:
+ """
+ pass
+
+ @abstractmethod
+ def update_local_conversation(self) -> None:
+ """
+ update realtime conversation with local conversation
+ """
+ pass
+
+ @abstractmethod
+ def add_message_item(self, item: MessageItem, previous_item_id: Optional[str] = None) -> None:
+ """
+ add realtime message item to update history message
+ :param item:
+ :param previous_item_id:
+ :return:
+ """
+ pass
+
+ @abstractmethod
+ def respond_audio_chunk(self, response_id: str, item_id: str, data: bytes) -> bool:
+ """
+ respond
+ :param response_id:
+ :param item_id:
+ :param data:
+ :return:
+ """
+ pass
+
+ @abstractmethod
+ def save_audio_item(self, item: MessageItem) -> None:
+ """
+ save audio data to local storage
+ :param item:
+ :return:
+ """
+ pass
+
+ @abstractmethod
+ def start_server_response(self, response_id: str) -> None:
+ """
+ start server response
+ :param response_id:
+ :return:
+ """
+ pass
+
+ @abstractmethod
+ def get_server_response_id(self) -> Optional[str]:
+ pass
+
+ @abstractmethod
+ def end_server_response(self, response_id: str) -> bool:
+ """
+ end but not cancel a response.
+ :param response_id:
+ :return:
+ """
+ pass
+
+ @abstractmethod
+ def clear_client_audio_buffer(self) -> None:
+ pass
+
+ @abstractmethod
+ def set_client_audio_buffer_start(self, cut_ms: int) -> None:
+ pass
+
+ @abstractmethod
+ def set_client_audio_buffer_stop(self, item_id: str, end_ms: int) -> None:
+ pass
+
+
+class StateOfServer(ABC):
+
+ def __init__(self, ctx: ServerContext):
+ self.ctx = ctx
+ self._destroyed = False
+
+ @classmethod
+ def recv_event(cls, state: Self, event: dict) -> None:
+ try:
+ state.receive(event)
+ except ValidationError as e:
+ state.ctx.logger.error("unwrap event failed: %r, origin event: %r", e, event)
+ raise e
+
+ @abstractmethod
+ def recv(self, event: dict) -> None:
+ """
+ recv an openai realtime server event, and handle it.
+ recv one server event at a time globally.
+ :param event:
+ :return:
+ """
+ pass
+
+ def __del__(self):
+ self.destroy()
+
+ def destroy(self) -> None:
+ if self._destroyed:
+ return
+ self._destroyed = True
+ self._destroy()
+
+ @abstractmethod
+ def _destroy(self):
+ pass
+
+ def recv_invalid_event(self, event: dict):
+ type_ = ServerEventType.get_type(event)
+ if ServerEventType.error.value == type_:
+ se = ServerError(**event)
+ # send error message.
+ self.ctx.logger.error("state %s recv server error: %r", type(self), se.error)
+ return self.ctx.respond_error_message(se.error.message)
+ else:
+ self.ctx.logger.error("state %s receive invalid event: %s", type(self), str(event)[:100])
+
+ def ack_server_event(self, event: ServerEvent):
+ line = get_caller_info(2, with_full_file=False)
+ self.ctx.logger.debug(
+ "ack event: event `%s` id `%s' by state %s at line `%s`",
+ event.type, event.event_id, type(self), line,
+ )
+
+
+class SessionState(StateOfServer):
+ """
+ session is the root of server state
+ """
+
+ def __init__(
+ self,
+ ctx: ServerContext,
+ session_created: ServerSessionCreated,
+ ):
+ super().__init__(ctx)
+ self.conversation = ConversationState(ctx, session_id="", conversation_id="")
+ self.session_id = session_created.session.id
+ self.session_obj: SessionObject = session_created.session
+ self.input_audio = InputAudioState(ctx)
+ self.tokens_rate_limit: Optional[RateLimit] = None
+ self.requests_rate_limit: Optional[RateLimit] = None
+
+ def is_responding(self) -> bool:
+ return self.conversation.responding_id is not None
+
+ def recv(self, event: dict):
+ type_name = ServerEventType.get_type(event)
+ if ServerEventType.error.value == type_name:
+ return self.recv_invalid_event(event)
+ elif ServerEventType.is_session_event(event, type_name):
+ return self._recv_session_event(event, type_name)
+
+ elif ServerEventType.rate_limits_updated.value == type_name:
+ return self._update_rate_limit(event)
+
+ # input audio event
+ elif ServerEventType.is_input_audio_event(event, type_name):
+ return self._recv_input_audio_event(event)
+
+ # conversation event
+ elif ServerEventType.is_conversation_event(event, type_name):
+ return self._recv_conversation_event(event)
+
+ # response event
+ elif ServerEventType.is_respond_event(event, type_name):
+ return self._recv_response_event(event)
+ else:
+ return self.recv_invalid_event(event)
+
+ def _destroy(self):
+ self.conversation.destroy()
+ self.input_audio.destroy()
+ del self.conversation
+ del self.input_audio
+ del self.session_obj
+ del self.ctx
+
+ def _recv_session_event(self, event: dict, e_type: str):
+ if e_type == ServerSessionCreated.type:
+ obj = ServerSessionCreated(**event)
+ self.session_id = obj.session.id
+ self.session_obj = obj.session
+
+ elif e_type == ServerSessionUpdated.type:
+ obj = ServerSessionUpdated(**event)
+ if self.session_id and obj.session.id != self.session_id:
+ # recv other session event, which is not possible.
+ return self.recv_invalid_event(event)
+ self.ctx.logger.info("realtime session updated: %r", obj.session)
+ self.session_obj = obj.session
+ else:
+ return self.recv_invalid_event(event)
+
+ def _recv_response_event(self, event: dict):
+ # let conversation handle response event
+ return self.conversation.recv(event)
+
+ def _recv_conversation_event(self, event: dict):
+ return self.conversation.recv(event)
+
+ def _recv_input_audio_event(self, event: dict):
+ return self.input_audio.recv(event)
+
+ def _update_rate_limit(self, event: dict):
+ # todo: use rate limit in future.
+ rlu = RateLimitsUpdated(**event)
+ for limit in rlu.rate_limits:
+ if limit.name == "requests":
+ self.requests_rate_limit = limit
+ elif limit.name == "tokens":
+ self.tokens_rate_limit = limit
+ self.ctx.logger.info(f"Rate limit updated {rlu}")
+ self.ack_server_event(rlu)
+
+
+class ConversationState(StateOfServer):
+
+ def __init__(
+ self,
+ ctx: ServerContext,
+ session_id: str,
+ conversation_id: str,
+ ):
+ super().__init__(ctx)
+ self.session_id = session_id
+ self.conversation_id = conversation_id
+ # session-conversation completed item.
+ self.conversation_item_states: dict[str, ConversationItemState] = {}
+ self.responses: Dict[str, ResponseState] = {}
+ self.responding_id: Optional[str] = None
+
+ def _destroy(self):
+ for item in self.conversation_item_states.values():
+ item.destroy()
+ self.conversation_item_states = {}
+ for item in self.responses.values():
+ item.destroy()
+ self.responses = {}
+ self.responding_id = None
+
+ def get_conversation_items(self) -> List[ConversationItemState]:
+ """
+ return the conversation items in orders.
+ """
+ items = []
+ start_item_id = ""
+ item_ids = set()
+ next_item_trace = {}
+ for key, item in self.conversation_item_states.items():
+ item_ids.add(key)
+ if not item.previous_item_id:
+ start_item_id = key
+ else:
+ next_item_trace[item.previous_item_id] = key
+ if not start_item_id:
+ for item_id in item_ids:
+ if item_id not in next_item_trace:
+ start_item_id = item_id
+ break
+
+ current_item_id = start_item_id
+ while current_item_id in self.conversation_item_states:
+ item = self.conversation_item_states[current_item_id]
+ items.append(item)
+ current_item_id = next_item_trace[current_item_id]
+ return items
+
+ def recv(self, event: dict):
+ type_name = ServerEventType.get_type(event)
+ # conversation events
+ if ServerEventType.conversation_created.match(event):
+ return self._conversation_created(event)
+ elif ServerEventType.conversation_item_created.value == type_name:
+ return self._conversation_item_created(event)
+ elif ServerEventType.conversation_item_deleted.value == type_name:
+ return self._delete_item(event)
+
+ # item event
+ elif ServerEventType.conversation_item_truncated.value == type_name:
+ return self._send_event_to_item(event)
+ elif ServerEventType.conversation_item_input_audio_transcription_completed.value == type_name:
+ return self._send_event_to_item(event)
+ elif ServerEventType.conversation_item_input_audio_transcription_failed.value == type_name:
+ return self._send_event_to_item(event)
+
+ # response event
+ elif ServerEventType.is_respond_event(event):
+ return self._on_response_event(event)
+ else:
+ return self.recv_invalid_event(event)
+
+ def _conversation_created(self, event: dict):
+ cic = ConversationItemCreated(**event)
+ self.conversation_id = cic.conversation_id
+ return self.ack_server_event(cic)
+
+ def _conversation_item_created(self, event: dict):
+ server_event = ConversationItemCreated(**event)
+ item = server_event.item
+ if item.id not in self.conversation_item_states:
+ conversation_item_state = ConversationItemState(
+ ctx=self.ctx,
+ created_event=server_event,
+ )
+ self.conversation_item_states[item.id] = conversation_item_state
+ if len(self.conversation_item_states) > 4:
+ first = list(self.conversation_item_states.keys())[0]
+ if first != item.id:
+ del self.conversation_item_states[first]
+ else:
+ # let conversation_item_state handle the item event.
+ state = self.conversation_item_states[item.id]
+ return state.recv(event)
+
+ def _delete_item(self, event: dict):
+ cid = ConversationItemDeleted(**event)
+ item_id = cid.item_id
+ # delete exists conversation item.
+ if item_id in self.conversation_item_states:
+ del self.conversation_item_states[item_id]
+ self.ctx.logger.info(f"Deleted item {item_id}")
+ return self.ack_server_event(cid)
+ return self.recv_invalid_event(event)
+
+ def _on_response_event(self, event: dict):
+ response_id = ServerEventType.get_response_id(event)
+ if not response_id:
+ # response_id exists in protocol
+ return self.recv_invalid_event(event)
+
+ if response_id not in self.responses:
+ if not ServerEventType.response_created.match(event):
+ self.ctx.logger.error("Response is not created")
+ else:
+ rc = ResponseCreated(**event)
+ response = self._create_response(rc)
+ self.responses[response.response_id] = response
+ return None
+ # response exists.
+ response = self.responses[response_id]
+ response.recv(event)
+ if response.is_done():
+ # if response is done, reset current responding id.
+ self.responding_id = None
+
+ def _send_event_to_item(self, event: dict):
+ item_id = ServerEventType.get_item_id(event)
+ # todo:
+ if item_id in self.conversation_item_states:
+ return self.conversation_item_states[item_id].recv(event)
+ return self.recv_invalid_event(event)
+
+ def _create_response(self, event: ResponseCreated) -> ResponseState:
+ # return response state
+ return ResponseState(
+ ctx=self.ctx,
+ event=event,
+ )
+
+
+class ConversationItemState(StateOfServer):
+ def __init__(
+ self,
+ ctx: ServerContext,
+ created_event: ConversationItemCreated,
+ ):
+ super().__init__(ctx)
+ self.previous_item_id: Optional[str] = created_event.previous_item_id
+ self.item: MessageItem = created_event.item
+ self.message = created_event.item.to_message_head().as_tail(copy=True)
+ self._on_conversation_item_created(created_event)
+
+ def _destroy(self):
+ self.item = None
+
+ def recv(self, event: dict):
+ type_name = ServerEventType.get_type(event)
+
+ # conversation item is created yet.
+ if ServerEventType.conversation_item_created.value == type_name:
+ obj = ConversationItemCreated(**event)
+ return self._on_conversation_item_created(obj)
+
+ elif ServerEventType.conversation_item_truncated.value == type_name:
+ obj = ConversationItemTruncated(**event)
+ # todo truncate audio file
+ return self.ack_server_event(obj)
+
+ elif ServerEventType.conversation_item_input_audio_transcription_completed.value == type_name:
+ obj = ConversationInputAudioTranscriptionCompleted(**event)
+ # update transcription.
+ self.message.content = obj.transcript
+ self.message.type = MessageType.AUDIO.value
+ self.ctx.update_history_message(self.message)
+ return self.ack_server_event(obj)
+
+ elif ServerEventType.conversation_item_input_audio_transcription_failed.value == type_name:
+ obj = ConversationInputAudioTranscriptionFailed(**event)
+ # todo
+ self.ctx.logger.error(f"Conversation item {self.item.id} transcription failed: %r", obj.error)
+ return self.ack_server_event(obj)
+ else:
+ return self.recv_invalid_event(event)
+
+ def _on_conversation_item_created(self, server_event: ConversationItemCreated):
+ self.previous_item_id = server_event.previous_item_id
+ self.item = server_event.item
+ self.message = self.item.to_message_head().as_tail(copy=True)
+ # add new message item.
+ self.ctx.add_message_item(server_event.item, server_event.previous_item_id)
+ if self.item is None:
+ self.ctx.logger.info("receive conversation item created but is None: %r", server_event)
+ if self.item and self.item.has_audio():
+ # save audio.
+ self.ctx.save_audio_item(self.item)
+ return self.ack_server_event(server_event)
+
+
+class InputAudioState(StateOfServer):
+
+ def recv(self, event: dict):
+ type_name = ServerEventType.get_type(event)
+ if ServerEventType.input_audio_buffer_cleared == type_name:
+ return self._on_input_audio_buffer_cleared(event)
+ elif ServerEventType.input_audio_buffer_committed == type_name:
+ return self._on_input_audio_buffer_committed(event)
+ elif ServerEventType.input_audio_buffer_speech_started == type_name:
+ return self._on_input_audio_buffer_started(event)
+ elif ServerEventType.input_audio_buffer_speech_stopped == type_name:
+ return self._on_input_audio_buffer_stopped(event)
+
+ def _on_input_audio_buffer_stopped(self, event: dict):
+ se = InputAudioBufferSpeechStopped(**event)
+ self.ctx.set_client_audio_buffer_stop(se.item_id, se.audio_end_ms)
+ return self.ack_server_event(se)
+
+ def _on_input_audio_buffer_started(self, event: dict):
+ """
+ the input audio started.
+ :param event
+ :return:
+ """
+ se = InputAudioBufferSpeechStarted(**event)
+ self.ctx.set_client_audio_buffer_start(se.audio_start_ms)
+ return self.ack_server_event(se)
+
+ def _on_input_audio_buffer_committed(self, event: dict):
+ se = InputAudioBufferCommitted(**event)
+ # todo:
+ return self.ack_server_event(se)
+
+ def _on_input_audio_buffer_cleared(self, event: dict):
+ se = InputAudioBufferCleared(**event)
+ return self.ack_server_event(se)
+
+ def _destroy(self):
+ pass
+
+
+class ResponseState(StateOfServer):
+ """
+ handle all response events.
+ """
+
+ def __init__(
+ self,
+ ctx: ServerContext,
+ event: ResponseCreated,
+ ):
+ super().__init__(ctx)
+ self.response_id = event.response.id
+ self.response_obj = event.response
+ self.item_states: dict[str, ResponseItemState] = {}
+ self.responding_item_id: Optional[str] = None
+ self._on_response_created(event)
+
+ def _destroy(self):
+ for item in self.item_states.values():
+ item.destroy()
+ self.item_states = {}
+
+ def is_done(self) -> bool:
+ return self.response_obj.status in {"completed", "cancelled", "failed"}
+
+ def recv(self, event: dict) -> None:
+ type_name = ServerEventType.get_type(event)
+ response_id = ServerEventType.get_response_id(event)
+ # receive current response event only
+ if response_id != self.response_id:
+ return self.recv_invalid_event(event)
+
+ if ServerEventType.response_created.value == type_name:
+ rc = ResponseCreated(**event)
+ return self.ack_server_event(rc)
+
+ elif ServerEventType.response_done.value == type_name:
+ return self._on_response_done(event)
+
+ elif ServerEventType.response_output_item_added.value == type_name:
+ # send head package
+ return self._on_response_output_item_added(event)
+
+ elif self.responding_item_id:
+ item_state = self.item_states[self.responding_item_id]
+ # let the item state handle event
+ item_state.recv(event)
+ # update the status after item is added
+ if item_state.is_done():
+ self.responding_item_id = None
+
+ else:
+ return self.recv_invalid_event(event)
+
+ def _on_response_created(self, event: ResponseCreated):
+ self.response_obj = event.response
+ self.response_id = event.response.id
+
+ # start response
+ self.ctx.start_server_response(self.response_id)
+ return self.ack_server_event(event)
+
+ def _on_response_done(self, event: dict) -> None:
+ """
+ response is done, update the
+ """
+ rd = ResponseDone(**event)
+ # update message item
+ self.response_obj = rd.response
+ self.ctx.end_server_response(rd.response.id)
+ if rd.response.status not in ["completed", "cancelled"] and rd.response.status_details:
+ error = rd.response.status_details.error
+ if error:
+ self.ctx.logger.error("response done with error: %s", error)
+ self.ctx.respond_error_message(repr(error))
+
+ elif rd.response.output:
+ # update history messages again
+ for item in rd.response.output:
+ self.ctx.update_history_message(item.to_complete_message())
+ # update local conversation when response is done.
+ self.ctx.update_local_conversation()
+ return self.ack_server_event(rd)
+
+ def _on_response_output_item_added(self, event: dict) -> None:
+ se = ResponseOutputItemAdded(**event)
+ item_id = se.item.id
+ if not item_id:
+ return self.recv_invalid_event(event)
+ # create response item state
+ state = ResponseItemState(self.ctx, se)
+ self.item_states[item_id] = state
+ self.responding_item_id = item_id
+
+
+class ResponseItemState(StateOfServer):
+
+ def __init__(self, ctx: ServerContext, event: ResponseOutputItemAdded):
+ super().__init__(ctx)
+ self.item = event.item
+ self.response_id = event.response_id
+ self.output_index = event.output_index
+ # send head
+ self._on_response_output_item_added(event)
+
+ def _destroy(self):
+ pass
+
+ def is_done(self) -> bool:
+ return self.item.status in {"completed"}
+
+ def _on_response_output_item_added(self, event: ResponseOutputItemAdded):
+ self.ctx.respond_message_chunk(event.response_id, event.item.to_message_head())
+ return self.ack_server_event(event)
+
+ def recv(self, event: dict) -> None:
+ type_name = ServerEventType.get_type(event)
+ if ServerEventType.response_output_item_added.value == type_name:
+ se = ResponseOutputItemAdded(**event)
+ return self._on_response_output_item_added(se)
+
+ elif ServerEventType.response_output_item_done.value == type_name:
+ # update message item.
+ se = ResponseOutputItemDone(**event)
+ return self._on_response_output_item_done(se)
+
+ elif ServerEventType.response_content_part_added.value == type_name:
+ se = ResponseContentPartAdded(**event)
+ return self._on_response_content_part_added(se)
+
+ elif ServerEventType.response_content_part_done.value == type_name:
+ se = ResponseContentPartDone(**event)
+ return self._on_response_content_part_done(se)
+
+ elif ServerEventType.response_text_delta.value == type_name:
+ se = ResponseTextDelta(**event)
+ self.ctx.respond_message_chunk(se.response_id, se.as_message_chunk())
+ return self.ack_server_event(se)
+
+ elif ServerEventType.response_text_done.value == type_name:
+ se = ResponseTextDone(**event)
+ return self.ack_server_event(se)
+
+ elif ServerEventType.response_audio_delta.value == type_name:
+ se = ResponseAudioDelta(**event)
+ self.ctx.respond_audio_chunk(se.response_id, se.item_id, se.get_audio_bytes())
+ return self.ack_server_event(se)
+
+ elif ServerEventType.response_audio_done.value == type_name:
+ se = ResponseAudioDone(**event)
+ return self.ack_server_event(se)
+
+ elif ServerEventType.response_audio_transcript_delta.value == type_name:
+ se = ResponseAudioTranscriptDelta(**event)
+ self.ctx.respond_message_chunk(se.response_id, se.as_message_chunk())
+ return self.ack_server_event(se)
+
+ elif ServerEventType.response_audio_transcript_done.value == type_name:
+ se = ResponseAudioTranscriptDone(**event)
+ return self.ack_server_event(se)
+
+ elif ServerEventType.response_function_call_arguments_delta.value == type_name:
+ se = ResponseFunctionCallArgumentsDelta(**event)
+ self.ctx.respond_message_chunk(se.response_id, se.as_message_chunk())
+ return self.ack_server_event(se)
+
+ elif ServerEventType.response_function_call_arguments_done.value == type_name:
+ se = ResponseFunctionCallArgumentsDone(**event)
+ chunk = se.as_message()
+ self.ctx.respond_message_chunk(se.response_id, chunk)
+ self.ctx.logger.info("receive function call done: %s", repr(se))
+
+ return self.ack_server_event(se)
+
+ else:
+ return self.recv_invalid_event(event)
+
+ def _on_response_output_item_done(self, event: ResponseOutputItemDone) -> None:
+ self.item = event.item
+ done = event.item.to_complete_message()
+ self.ctx.logger.info("response done with event: %s, message is %s", repr(event), repr(done))
+ self.ctx.update_history_message(done)
+ return self.ack_server_event(event)
+
+ def _on_response_content_part_added(self, event: ResponseContentPartAdded) -> None:
+ return self.ack_server_event(event)
+
+ def _on_response_content_part_done(self, event: ResponseContentPartDone) -> None:
+ return self.ack_server_event(event)
diff --git a/ghostos/framework/openai_realtime/ws.py b/ghostos/framework/openai_realtime/ws.py
new file mode 100644
index 00000000..8bfe25d0
--- /dev/null
+++ b/ghostos/framework/openai_realtime/ws.py
@@ -0,0 +1,169 @@
+from __future__ import annotations
+import time
+
+import socks
+from typing import Union
+
+import urllib3.util
+import websockets
+import json
+import logging
+from websockets.sync.client import connect as ws_connect, ClientConnection
+from ghostos.contracts.logger import LoggerItf, get_console_logger
+from ghostos.framework.openai_realtime.configs import OpenAIWebsocketsConf
+
+__all__ = ['OpenAIWSConnection']
+
+
+class OpenAIWSConnection:
+ """
+ websocket adapter, provides:
+ 1. connect config adaption
+ 2. event marshal and unmarshal
+ 3. exception catch
+ """
+
+ def __init__(
+ self,
+ conf: OpenAIWebsocketsConf,
+ *,
+ logger: LoggerItf = None,
+ ):
+ """
+ :param conf:
+ :param logger:
+ """
+ self._running = False
+ self._closed = False
+ self._logger = logger if logger else logging.getLogger()
+ conf = conf.load_from_env()
+ self._conf = conf
+ sock = None
+ if conf.proxy is not None:
+ sock = self._create_socket(conf.proxy, conf.uri)
+ # 同步创建 connection.
+ self._logger.info("connecting openai realtime api")
+ self._ws = ws_connect(
+ uri=self._conf.uri,
+ additional_headers={
+ "Authorization": "Bearer " + self._conf.api_key,
+ "OpenAI-Beta": "realtime=v1",
+ },
+ sock=sock,
+ )
+ self._logger.info("connected openai realtime api")
+
+ def _create_socket(self, proxy: str, uri: str):
+ parsed = urllib3.util.parse_url(proxy)
+ if parsed.scheme != "socks5":
+ raise NotImplementedError(f"Only socks5 is supported, got {parsed.scheme}")
+ host = parsed.hostname
+ port = parsed.port
+ s = socks.socksocket()
+ s.set_proxy(socks.SOCKS5, host, port)
+
+ uri_parsed = urllib3.util.parse_url(uri)
+ s.connect((uri_parsed.hostname, 443))
+ return s
+
+ def client(self) -> ClientConnection:
+ if self._closed:
+ raise RuntimeError("Connection was already stopped")
+ return self._ws
+
+ def send(self, event: dict) -> None:
+ if self._closed:
+ raise RuntimeError("Connection was already stopped")
+ try:
+ data = json.dumps(event)
+ # last check
+ if self._closed:
+ return
+ self._ws.send(data)
+ self._logger.debug(f"[OpenAIWSConnection] send data to server: %s", data[:300])
+ except websockets.exceptions.ConnectionClosedOK:
+ self.close()
+
+ def recv(self, timeout: Union[float, None] = None, timeout_error: bool = False) -> Union[dict, None]:
+ if self._closed:
+ return None
+ try:
+ data = self._ws.recv(timeout=timeout)
+ if not data:
+ self._logger.error(f"[OpenAIWSConnection] receive empty data: {data}")
+ return None
+ if data:
+ self._logger.debug(f"[OpenAIWSConnection] receive data: %s", data[:300])
+ event = json.loads(data)
+ return event
+ return None
+ except websockets.exceptions.ConnectionClosed:
+ self.close()
+ return None
+ except TimeoutError:
+ if timeout == 0:
+ # return None as expected
+ return None
+ if timeout_error:
+ raise
+ return None
+
+ def close(self):
+ if self._closed:
+ return
+ self._closed = True
+ if self._ws is not None:
+ self._ws.close()
+ self._ws = None
+ self._logger.debug("[OpenAIWSConnection] stop the connection")
+
+ def closed(self) -> bool:
+ return self._closed
+
+
+connect = OpenAIWSConnection
+
+# some local tests
+if __name__ == "__main__":
+ import os
+ from ghostos.helpers import Timeleft
+
+ _token = os.environ["OPENAI_API_KEY"]
+ print("+++++ token", _token)
+ _conf = OpenAIWebsocketsConf(token=_token)
+ _conf.proxy = "socks5://127.0.0.1:1080"
+ _c = connect(
+ _conf,
+ logger=get_console_logger(debug=True),
+ )
+
+ # test parallel actions
+ _c.send({
+ "type": "conversation.item.create",
+ "previous_item_id": None,
+ "item": {
+ "type": "message",
+ "role": "user",
+ "content": [
+ {
+ "type": "input_text",
+ "text": "Hello, how are you?"
+ }
+ ]
+ }
+ })
+ _c.send({
+ "type": "response.create",
+ "response": {},
+ })
+
+ left = Timeleft(10)
+ while left.alive():
+ _data = _c.recv(timeout=0)
+ if _data:
+ print("+++++", _data)
+ time.sleep(0.2)
+ print("+++++ timeleft", left.left())
+ _c.close()
+
+ print("done")
diff --git a/ghostos/framework/operators/__init__.py b/ghostos/framework/operators/__init__.py
deleted file mode 100644
index bed12ddf..00000000
--- a/ghostos/framework/operators/__init__.py
+++ /dev/null
@@ -1,28 +0,0 @@
-from ghostos.framework.operators.action_ops import (
- WaitsOperator,
- FailOperator,
- FinishOperator,
- ThinkOperator,
- WaitOnTasksOperator,
-)
-from ghostos.framework.operators.event_ops import (
- # 统一的事件状态机.
- OnEventOperator,
-
- # 上游相关事件.
- OnUpstreamEventOperator,
- OnInputOperator,
- OnCancelingOperator,
- OnCreatedOperator,
-
- # 自身的事件.
- OnSelfEventOperator,
- OnObserveOperator,
-
- # 下游的 callback 事件.
- OnCallbackEventOperator,
- OnFinishCallbackOperator,
- OnNotifyCallbackOperator,
- OnWaitCallbackOperator,
- OnFailureCallbackOperator,
-)
diff --git a/ghostos/framework/operators/action_ops.py b/ghostos/framework/operators/action_ops.py
deleted file mode 100644
index 53a3fa2e..00000000
--- a/ghostos/framework/operators/action_ops.py
+++ /dev/null
@@ -1,268 +0,0 @@
-from typing import Optional, List, ClassVar
-
-from ghostos.core.ghosts import (
- Operator, Ghost, NewTask,
-)
-from ghostos.core.messages import (
- MessageKind, MessageKindParser, Role,
-)
-from ghostos.core.session import (
- DefaultEventType,
- TaskState,
- Task,
-)
-
-__all__ = [
- 'ActionOperator',
- 'WaitsOperator',
- 'FailOperator',
- 'FinishOperator',
- 'ThinkOperator',
- 'WaitOnTasksOperator',
-]
-
-
-class ActionOperator(Operator):
- """
- 当前任务进入等待状态. 需要执行的流程:
- 1. 发送存在的消息.
- 2. 更新当前 task 的状态, 添加日志.
- 3. 如果父任务存在, 向父任务发送消息.
- """
- task_state: ClassVar[str] = TaskState.WAITING.value
- callback_event_type: ClassVar[str] = DefaultEventType.WAIT_CALLBACK.value
-
- def __init__(
- self, *,
- messages: List[MessageKind],
- reason: str = "",
- instruction: str = "",
- callback_task_id: Optional[str] = None,
- ):
- self.reason = reason
- self.instruction = instruction
- self.messages = messages
- self.callback_task_id = callback_task_id
-
- def send_replies(self, g: "Ghost") -> None:
- if self.messages:
- session = g.session()
- self.messages = session.send_messages(*self.messages)
-
- def get_callback_task_id(self, task: Task) -> Optional[str]:
- if self.callback_task_id is not None:
- return self.callback_task_id
- return task.parent
-
- def change_task_state(self, g: "Ghost") -> None:
- session = g.session()
- task = session.task()
- thread = session.thread()
- task.state = self.task_state
- if self.reason:
- task.logs.append(f"{self.task_state}: {self.reason}")
- # 状态判断.
- # 消息不为空的时候才发送.
- callbacks = self.messages
- callback_task_id = self.get_callback_task_id(task)
- if callback_task_id:
- utils = g.utils()
- # 发送消息给父任务.
- utils.send_task_event(
- task_id=callback_task_id,
- event_type=self.callback_event_type,
- messages=callbacks,
- reason=self.reason,
- self_task=task,
- )
-
- # 更新当前 session, 主要更新 thread 的状态.
- session.update_task(task, thread, True)
-
- def send_children_events(self, g: "Ghost") -> None:
- return
-
- def next_operator(self, g: "Ghost") -> Optional[Operator]:
- return None
-
- def run(self, g: "Ghost") -> Optional["Operator"]:
- self.send_replies(g)
- self.send_children_events(g)
- self.change_task_state(g)
- self.send_children_events(g)
- return self.next_operator(g)
-
- def destroy(self) -> None:
- del self.messages
-
-
-class WaitsOperator(ActionOperator):
- task_state: ClassVar[str] = TaskState.WAITING.value
- callback_event_type: ClassVar[str] = DefaultEventType.WAIT_CALLBACK.value
-
- def __init__(self, *, reason: str, messages: List[MessageKind], callback_task_id: Optional[str] = None):
- super().__init__(reason=reason, messages=messages, callback_task_id=callback_task_id)
-
-
-class FailOperator(ActionOperator):
- """
- 结束当前任务.
- 1. 通过 session 发送消息, 同时消息保存到 Thread 里.
- 2. 变更当前 Task 的状态, 并保存.
- 3. 反馈 fail_callback 事件给父 task, 如果存在的话.
- 4. 取消未完成的子任务. 向它们发送取消事件.
- 5. 自己继续执行 on_finished 事件, 可以创建独立的任务去理解.
- """
- task_state: ClassVar[str] = TaskState.FAILED.value
- callback_event_type: ClassVar[str] = DefaultEventType.FAILURE_CALLBACK.value
-
- def __init__(
- self, *,
- reason: str,
- messages: List[MessageKind],
- callback_task_id: Optional[str] = None,
- ):
- super().__init__(reason=reason, messages=messages, callback_task_id=callback_task_id)
-
- def format_cancel_children_reason(self) -> str:
- if not self.reason:
- return ""
- return f"task is canceled cause parent task is failed: {self.reason}"
-
- def send_children_events(self, g: "Ghost") -> None:
- # 取消所有的子任务.
- reason = self.format_cancel_children_reason()
- utils = g.utils()
- utils.cancel_children_tasks(reason=reason, includes=None)
-
- def next_operator(self, g: "Ghost") -> Optional[Operator]:
- session = g.session()
- task = session.task()
- # finish 没有后续. 但还是要执行一个反思事件.
- event = DefaultEventType.FAILED.new(
- task_id=task.task_id,
- messages=[],
- )
- # 立刻执行 on_finished 事件,
- return g.utils().handle_event(event)
-
-
-class FinishOperator(ActionOperator):
- """
- 结束当前任务.
- 1. 通过 session 发送消息, 同时消息保存到 Thread 里.
- 2. 变更当前 Task 的状态, 并保存.
- 3. 反馈 finish_callback 事件给父 task, 如果存在的话.
- 4. 取消未完成的子任务. 向它们发送取消事件.
- 5. 自己继续执行 on_finished 事件, 可以创建独立的任务去理解.
- """
- task_state: ClassVar[str] = TaskState.FINISHED.value
- callback_event_type: ClassVar[str] = DefaultEventType.FINISH_CALLBACK.value
-
- def __init__(
- self, *,
- reason: str,
- messages: List[MessageKind],
- ):
- super().__init__(
- messages=messages,
- reason=reason,
- )
-
- def format_cancel_children_reason(self) -> str:
- if not self.reason:
- return ""
- return f"task is canceled cause parent task is finished: {self.reason}"
-
- def send_children_events(self, g: "Ghost") -> None:
- utils = g.utils()
- reason = self.format_cancel_children_reason()
- utils.cancel_children_tasks(reason=reason, includes=None)
-
- def next_operator(self, g: "Ghost") -> Optional[Operator]:
- # finish 没有后续. 但还是要执行一个反思事件.
- session = g.session()
- task = session.task()
- event = DefaultEventType.FINISHED.new(
- task_id=task.task_id,
- messages=[],
- )
-
- # 立刻执行 on_finished 事件,
- return g.utils().handle_event(event)
-
-
-class WaitOnTasksOperator(ActionOperator):
- """
- wait on children tasks
- """
- task_state: ClassVar[str] = TaskState.RUNNING.value
- callback_event_type: ClassVar[str] = DefaultEventType.NOTIFY_CALLBACK.value
-
- def __init__(
- self, *,
- new_tasks: List[NewTask],
- ):
- for item in new_tasks:
- if not isinstance(item, NewTask):
- raise TypeError(f'new_tasks must be a NewTask instance, got {type(item)}')
- self.new_tasks = new_tasks
- super().__init__(
- messages=[],
- reason="",
- instruction="",
- )
-
- def send_children_events(self, g: "Ghost") -> None:
- g.utils().create_child_tasks(
- depend=True,
- new_tasks=self.new_tasks,
- )
-
- def destroy(self) -> None:
- del self.new_tasks
- del self.messages
-
-
-class ThinkOperator(ActionOperator):
- """
- 运行下一轮思考.
- """
- task_state: ClassVar[str] = TaskState.RUNNING.value
- callback_event_type: ClassVar[str] = DefaultEventType.NOTIFY_CALLBACK.value
-
- def __init__(
- self, *,
- reason: str,
- instruction: str,
- observation: List[MessageKind],
- caller_id: str = "",
- ):
- self.observation = observation
- self.caller_id = caller_id
- super().__init__(
- messages=[],
- reason=reason,
- instruction=instruction,
- )
-
- def next_operator(self, g: "Ghost") -> Optional["Operator"]:
- observations = []
- if self.observation:
- parser = MessageKindParser(role=Role.TOOL, ref_id=self.caller_id)
- observations = parser.parse(self.observation)
- task = g.session().task()
- # 执行下一轮思考.
- utils = g.utils()
- utils.send_task_event(
- task_id=task.task_id,
- event_type=DefaultEventType.OBSERVE.value,
- reason=self.reason,
- instruction=self.instruction,
- messages=observations,
- self_task=task,
- )
- return None
-
- def destroy(self) -> None:
- del self.observation
diff --git a/ghostos/framework/operators/event_ops.py b/ghostos/framework/operators/event_ops.py
deleted file mode 100644
index 24109209..00000000
--- a/ghostos/framework/operators/event_ops.py
+++ /dev/null
@@ -1,253 +0,0 @@
-from typing import Optional, ClassVar
-
-from ghostos.core.ghosts import (
- EventOperator, Ghost, Operator, get_event_operator
-)
-from ghostos.core.session import (
- TaskState,
- DefaultEventType,
-)
-
-__all__ = [
- 'OnEventOperator',
-
- # 上游相关事件.
- 'OnUpstreamEventOperator',
- 'OnInputOperator',
- 'OnCancelingOperator',
- 'OnCreatedOperator',
-
- # 自身的事件.
- 'OnSelfEventOperator',
- 'OnObserveOperator',
-
- # 下游的 callback 事件.
- 'OnCallbackEventOperator',
- 'OnFinishCallbackOperator',
- 'OnNotifyCallbackOperator',
- 'OnWaitCallbackOperator',
- 'OnFailureCallbackOperator',
-]
-
-
-class OnEventOperator(EventOperator):
- """
- 标准的事件状态机. 区分上游事件, 自身事件, 和下游 callback 事件.
- """
- event_type: ClassVar[str] = ""
-
- def run(self, g: "Ghost") -> Optional["Operator"]:
- task = g.session().task()
- if self.event.from_self():
- op = get_event_operator(
- {
- "": OnSelfEventOperator,
- OnObserveOperator.event_type: OnObserveOperator,
- },
- self.event,
- )
- return op
- elif self.event.callback or self.event.from_task_id in task.children:
- op = get_event_operator(
- {
- "": OnCallbackEventOperator,
- OnFinishCallbackOperator.event_type: OnFinishCallbackOperator,
- OnFailureCallbackOperator.event_type: OnFailureCallbackOperator,
- OnWaitCallbackOperator.event_type: OnWaitCallbackOperator,
- OnNotifyCallbackOperator.event_type: OnNotifyCallbackOperator,
- },
- self.event,
- )
- return op
- else:
- op = get_event_operator(
- {
- "": OnUpstreamEventOperator,
- OnCancelingOperator.event_type: OnCancelingOperator,
- OnInputOperator.event_type: OnInputOperator,
- OnCreatedOperator.event_type: OnCreatedOperator,
- },
- self.event,
- )
- return op
-
-
-class OnUpstreamEventOperator(EventOperator):
- """
- 上游事件的默认处理逻辑.
- """
-
- event_type: ClassVar[str] = ""
- default_state: ClassVar[str] = TaskState.WAITING.value
-
- def handle_event(self, g: "Ghost") -> Optional["Operator"]:
- session = g.session()
- task = session.task()
- # 思考轮次设置为 0.
- task.think_turns = 0
- thread = session.thread()
- thread.new_turn(self.event)
- session.update_task(task, thread, update_history=False)
- return g.utils().handle_event(self.event)
-
- def default_action(self, g: "Ghost") -> Optional["Operator"]:
- session = g.session()
- task = session.task()
- # 默认 await.
- task.state = self.default_state
- session.update_task(task, None, update_history=True)
- return None
-
- def run(self, g: "Ghost") -> Optional["Operator"]:
- op = self.handle_event(g)
- if op is not None:
- return op
- return self.default_action(g)
-
-
-class OnSelfEventOperator(EventOperator):
- """
- 自己触发的事件的处理逻辑.
- """
- event_type: ClassVar[str] = ""
- default_state: ClassVar[str] = TaskState.WAITING.value
-
- def handle_event(self, g: "Ghost") -> Optional["Operator"]:
- session = g.session()
- task = session.task()
- # 思考轮次设置为 0.
- task.think_turns += 1
- thread = session.thread()
- thread.new_turn(self.event)
- if task.think_too_much():
- session.update_task(task, thread, update_history=True)
- # 不再运行思考. 只是追加信息. 必须等待上游的输入才能继续运行.
- # todo: 是否要通知上游, ask to continue.
- return None
- session.update_task(task, thread, update_history=False)
- return g.utils().handle_event(self.event)
-
- def default_action(self, g: "Ghost") -> Optional["Operator"]:
- session = g.session()
- task = session.task()
- # 默认 await.
- task.state = self.default_state
- session.update_task(task, None, update_history=True)
- return None
-
- def run(self, g: "Ghost") -> Optional["Operator"]:
- op = self.handle_event(g)
- if op is not None:
- return op
- return self.default_action(g)
-
-
-class OnCallbackEventOperator(EventOperator):
- """
- 子任务触发的事件.
- """
- event_type: ClassVar[str] = ""
-
- def receive_event(self, g: "Ghost") -> bool:
- session = g.session()
- task = session.task()
- # 思考轮次设置为 0.
- thread = session.thread()
- thread.new_turn(self.event)
- session.update_task(task, thread, update_history=False)
- return task.is_dead()
-
- def handle_event(self, g: "Ghost") -> Optional["Operator"]:
- return g.utils().handle_event(self.event)
-
- def run(self, g: "Ghost") -> Optional["Operator"]:
- # 接受事件.
- dead = self.receive_event(g)
- if dead:
- # 不处理逻辑了.
- return None
- # 处理事件.
- op = self.handle_event(g)
- if op is not None:
- return op
- # 不变更任何状态.
- return None
-
-
-# --- upstream event operators --- #
-
-class OnInputOperator(OnUpstreamEventOperator):
- """
- 接受到上游的输入.
- """
- event_type: ClassVar[str] = DefaultEventType.INPUT.value
- default_state: ClassVar[str] = TaskState.WAITING.value
-
-
-class OnCreatedOperator(OnUpstreamEventOperator):
- """
- 接受到创建任务的消息.
- """
- event_type: ClassVar[str] = DefaultEventType.CREATED.value
- default_state: ClassVar[str] = TaskState.WAITING.value
-
-
-class OnCancelingOperator(OnUpstreamEventOperator):
- event_type = DefaultEventType.CANCELING.value
- default_state: ClassVar[str] = TaskState.CANCELLED.value
-
- def handle_event(self, g: "Ghost") -> Optional["Operator"]:
- session = g.session()
- task = session.task()
- if task.is_dead():
- # 不做额外处理.
- return None
- return super().handle_event(g)
-
- def default_action(self, g: "Ghost") -> Optional["Operator"]:
- session = g.session()
- task = session.task()
- # 取消所有的子任务.
- if task.children:
- g.utils().cancel_children_tasks(
- reason=f"parent task {task.task_id} is canceled",
- self_task=task,
- )
- return super().default_action(g)
-
-
-# --- self event operators --- #
-
-class OnObserveOperator(OnSelfEventOperator):
- event_type: ClassVar[str] = DefaultEventType.OBSERVE.value
- default_state: ClassVar[str] = TaskState.WAITING.value
-
-
-# --- call back operators --- #
-
-class OnFinishCallbackOperator(OnCallbackEventOperator):
- event_type: ClassVar[str] = DefaultEventType.FINISH_CALLBACK
-
- def handle_event(self, g: "Ghost") -> Optional["Operator"]:
- session = g.session()
- task = session.task()
- from_task_id = self.event.from_task_id
- group = None
- if from_task_id:
- group = task.on_callback_task(from_task_id)
- session.update_task(task, None, update_history=False)
- if group is not None:
- return g.utils().handle_event(self.event)
- return None
-
-
-class OnFailureCallbackOperator(OnCallbackEventOperator):
- event_type: ClassVar[str] = DefaultEventType.FAILURE_CALLBACK
-
-
-class OnWaitCallbackOperator(OnCallbackEventOperator):
- event_type: ClassVar[str] = DefaultEventType.WAIT_CALLBACK
-
-
-class OnNotifyCallbackOperator(OnCallbackEventOperator):
- event_type: ClassVar[str] = DefaultEventType.NOTIFY_CALLBACK
diff --git a/ghostos/framework/processes/__init__.py b/ghostos/framework/processes/__init__.py
index 0ea9b26e..7492f429 100644
--- a/ghostos/framework/processes/__init__.py
+++ b/ghostos/framework/processes/__init__.py
@@ -1 +1,2 @@
+from ghostos.core.runtime import GoProcesses
from ghostos.framework.processes.storage_processes import StorageProcessImplProvider, WorkspaceProcessesProvider
diff --git a/ghostos/framework/processes/storage_processes.py b/ghostos/framework/processes/storage_processes.py
index bb128e0b..72393ca3 100644
--- a/ghostos/framework/processes/storage_processes.py
+++ b/ghostos/framework/processes/storage_processes.py
@@ -1,23 +1,22 @@
from typing import Optional, Dict, Type
import yaml
-from ghostos.core.session import Process
-from ghostos.core.session.processes import Processes
+from ghostos.core.runtime import GoProcess
+from ghostos.core.runtime.processes import GoProcesses
from ghostos.contracts.storage import Storage
from ghostos.contracts.logger import LoggerItf
-from ghostos.core.ghosts.workspace import Workspace
-from threading import Lock
+from ghostos.contracts.workspace import Workspace
from ghostos.container import Provider, Container
+from ghostos.helpers import yaml_pretty_dump
-__all__ = ['StorageProcessesImpl', 'StorageProcessImplProvider', 'WorkspaceProcessesProvider']
+__all__ = ['StorageGoProcessesImpl', 'StorageProcessImplProvider', 'WorkspaceProcessesProvider']
-class StorageProcessesImpl(Processes):
+class StorageGoProcessesImpl(GoProcesses):
session_map_name = "sessions.yml"
def __init__(self, storage: Storage, logger: LoggerItf):
self._storage = storage
self._logger = logger
- self._lock = Lock()
def _get_session_process_map(self) -> Dict[str, str]:
filename = self.session_map_name
@@ -28,71 +27,54 @@ def _get_session_process_map(self) -> Dict[str, str]:
return {}
@staticmethod
- def _get_process_filename(process_id: str) -> str:
- return f"{process_id}.process.yml"
+ def _get_process_filename(shell_id: str) -> str:
+ return f"{shell_id}.process.yml"
- def get_process(self, process_id: str) -> Optional[Process]:
- filename = self._get_process_filename(process_id)
- if self._storage.exists(filename):
- content = self._storage.get(filename)
- data = yaml.safe_load(content)
- process = Process(**data)
- return process
- return None
-
- def _save_session_process_map(self, session_map: Dict[str, str]) -> None:
- content = yaml.safe_dump(session_map)
- filename = self.session_map_name
- self._storage.put(filename, content.encode("utf-8"))
-
- def get_session_process(self, session_id: str) -> Optional[Process]:
- m = self._get_session_process_map()
- process_id = m.get(session_id, None)
- if process_id is None:
+ def get_process(self, shell_id: str) -> Optional[GoProcess]:
+ filename = self._get_process_filename(shell_id)
+ if not self._storage.exists(filename):
return None
- return self.get_process(process_id)
+ content = self._storage.get(filename)
+ data = yaml.safe_load(content)
+ process = GoProcess(**data)
+ return process
- def save_process(self, process: Process) -> None:
- session_id = process.session_id
- process_id = process.process_id
- with self._lock:
- m = self._get_session_process_map()
- m[session_id] = process_id
- self._save_session_process_map(m)
- content = yaml.safe_dump(process.model_dump(exclude_defaults=True))
- filename = self._get_process_filename(process_id)
- self._storage.put(filename, content.encode("utf-8"))
+ def save_process(self, process: GoProcess) -> None:
+ filename = self._get_process_filename(process.shell_id)
+ data = process.model_dump(exclude_defaults=True)
+ content = yaml_pretty_dump(data)
+ self._storage.put(filename, content.encode())
-class StorageProcessImplProvider(Provider[Processes]):
+class StorageProcessImplProvider(Provider[GoProcesses]):
def __init__(self, process_dir: str = "runtime/processes"):
self.process_dir = process_dir
def singleton(self) -> bool:
return True
- def contract(self) -> Type[Processes]:
- return Processes
+ def contract(self) -> Type[GoProcesses]:
+ return GoProcesses
- def factory(self, con: Container) -> Optional[Processes]:
+ def factory(self, con: Container) -> Optional[GoProcesses]:
storage = con.force_fetch(Storage)
logger = con.force_fetch(LoggerItf)
processes_storage = storage.sub_storage(self.process_dir)
- return StorageProcessesImpl(processes_storage, logger)
+ return StorageGoProcessesImpl(processes_storage, logger)
-class WorkspaceProcessesProvider(Provider[Processes]):
+class WorkspaceProcessesProvider(Provider[GoProcesses]):
def __init__(self, process_dir: str = "processes"):
self.process_dir = process_dir
def singleton(self) -> bool:
return True
- def contract(self) -> Type[Processes]:
- return Processes
+ def contract(self) -> Type[GoProcesses]:
+ return GoProcesses
- def factory(self, con: Container) -> Optional[Processes]:
+ def factory(self, con: Container) -> Optional[GoProcesses]:
workspace = con.force_fetch(Workspace)
logger = con.force_fetch(LoggerItf)
processes_storage = workspace.runtime().sub_storage(self.process_dir)
- return StorageProcessesImpl(processes_storage, logger)
+ return StorageGoProcessesImpl(processes_storage, logger)
diff --git a/ghostos/framework/realtime/__init__.py b/ghostos/framework/realtime/__init__.py
new file mode 100644
index 00000000..4c48f0ec
--- /dev/null
+++ b/ghostos/framework/realtime/__init__.py
@@ -0,0 +1,2 @@
+from ghostos.abcd.realtime import Realtime, RealtimeConfig
+from ghostos.framework.realtime.defaults import ConfigBasedRealtimeProvider
diff --git a/ghostos/framework/realtime/defaults.py b/ghostos/framework/realtime/defaults.py
new file mode 100644
index 00000000..dc9d273e
--- /dev/null
+++ b/ghostos/framework/realtime/defaults.py
@@ -0,0 +1,60 @@
+from __future__ import annotations
+from typing import Optional, Dict
+
+from ghostos.abcd import Conversation
+from ghostos.abcd.realtime import (
+ Realtime, RealtimeConfig, RealtimeDriver, Listener, Speaker, RealtimeAppConfig,
+ RealtimeApp,
+)
+from ghostos.contracts.configs import YamlConfig, Configs
+from ghostos.container import Container, Provider, INSTANCE
+from ghostos.framework.openai_realtime import OpenAIRealtimeDriver
+
+
+class BasicRealtimeConfig(RealtimeConfig, YamlConfig):
+ relative_path = "realtime_conf.yml"
+
+
+class ConfigsBasedRealtime(Realtime):
+
+ def __init__(self, configs: Configs):
+ self._drivers: Dict[str: RealtimeDriver] = {}
+ self._configs = configs
+
+ def create(
+ self,
+ conversation: Conversation,
+ listener: Listener,
+ speaker: Speaker,
+ app_name: str = "",
+ config: Optional[RealtimeAppConfig] = None,
+ ) -> RealtimeApp:
+ realtime_conf = self.get_config()
+ if config is None:
+ if not app_name:
+ app_name = realtime_conf.default
+ config = realtime_conf.get_app_conf(app_name)
+ if config is None:
+ raise NotImplementedError(f"No config for {app_name}")
+ driver: RealtimeDriver | None = self._drivers.get(config.driver_name())
+ if driver is None:
+ raise NotImplementedError(f"No driver for {config.driver_name()}")
+ return driver.create(config, conversation, listener, speaker)
+
+ def get_config(self) -> RealtimeConfig:
+ return self._configs.get(BasicRealtimeConfig)
+
+ def register(self, driver: RealtimeDriver):
+ self._drivers[driver.driver_name()] = driver
+
+
+class ConfigBasedRealtimeProvider(Provider[Realtime]):
+
+ def singleton(self) -> bool:
+ return True
+
+ def factory(self, con: Container) -> Optional[INSTANCE]:
+ configs = con.get(Configs)
+ realtime = ConfigsBasedRealtime(configs)
+ realtime.register(OpenAIRealtimeDriver())
+ return realtime
diff --git a/ghostos/framework/repliers/__init__.py b/ghostos/framework/repliers/__init__.py
deleted file mode 100644
index 0c9f8827..00000000
--- a/ghostos/framework/repliers/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-from ghostos.framework.repliers.basic import ReplierImpl
\ No newline at end of file
diff --git a/ghostos/framework/repliers/basic.py b/ghostos/framework/repliers/basic.py
deleted file mode 100644
index 2693aaeb..00000000
--- a/ghostos/framework/repliers/basic.py
+++ /dev/null
@@ -1,71 +0,0 @@
-from typing import Optional, Dict, Any
-
-from ghostos.core.ghosts import Operator
-from ghostos.core.ghosts.schedulers import Replier
-from ghostos.core.messages import Role
-from ghostos.core.session import Task
-from ghostos.framework.operators import WaitsOperator, ThinkOperator, FinishOperator
-from ghostos.helpers import yaml_pretty_dump
-
-
-class ReplierImpl(Replier):
-
- def __init__(self, task: Task, event_from_task: Optional[str] = None):
- callback_task_id = task.parent
- if event_from_task and event_from_task != task.task_id:
- callback_task_id = event_from_task
- self.callback_task_id = callback_task_id
-
- def finish(self, reply: str) -> Operator:
- if not reply:
- raise AttributeError(f'finish reply shall not be empty ')
- return FinishOperator(
- reason="",
- messages=[reply],
- )
-
- def reply(self, content: str) -> Operator:
- if not content:
- raise ValueError("reply Content cannot be empty")
- return WaitsOperator(
- reason="",
- messages=[content],
- callback_task_id=self.callback_task_id,
- )
-
- def ask_clarification(self, question: str) -> Operator:
- if not question:
- raise ValueError("ask clarification question cannot be empty")
- return WaitsOperator(
- reason="",
- messages=[question],
- callback_task_id=self.callback_task_id,
- )
-
- def fail(self, reply: str) -> Operator:
- if not reply:
- raise ValueError("fail reply cannot be empty")
- return WaitsOperator(
- reason="",
- messages=[reply],
- callback_task_id=self.callback_task_id,
- )
-
- def think(self, observations: Optional[Dict[str, Any]] = None, instruction: str = "") -> Operator:
- messages = []
- if observations:
- values = {name: str(value) for name, value in observations.items()}
- content = yaml_pretty_dump(values)
-
- # 用什么协议没想明白, function ? tool? system ?
- content = "# observe values: \n" + content
- msg = Role.new_assistant_system(
- content=content,
- )
- messages.append(msg)
-
- return ThinkOperator(
- observation=messages,
- reason="",
- instruction=instruction,
- )
diff --git a/ghostos/framework/session/__init__.py b/ghostos/framework/session/__init__.py
deleted file mode 100644
index 5f1e2176..00000000
--- a/ghostos/framework/session/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-from ghostos.framework.session.basic import BasicSession
\ No newline at end of file
diff --git a/ghostos/framework/session/basic.py b/ghostos/framework/session/basic.py
deleted file mode 100644
index 871292a3..00000000
--- a/ghostos/framework/session/basic.py
+++ /dev/null
@@ -1,324 +0,0 @@
-from typing import Optional, Callable, List, Iterable, Dict
-from ghostos.core.messages import (
- MessageKind, Role, Stream, MessageKindParser, DefaultMessageTypes,
- Buffer, Payload, Attachment, Message,
-)
-from ghostos.core.session import (
- Session,
- Process, Processes,
- MsgThread, Threads,
- Task, Tasks, TaskPayload, TaskState,
- Messenger,
- Event, EventBus, DefaultEventType,
- TaskBrief,
-)
-from ghostos.core.llms import FunctionalToken
-from ghostos.framework.messengers import DefaultMessenger
-from ghostos.helpers import uuid
-from ghostos.contracts.logger import LoggerItf
-from ghostos.contracts.pool import Pool
-
-
-class Future:
- def __init__(self, future_id: str, bus: EventBus, event: Event):
- self.bus = bus
- self.event = event
- self.future_id = future_id
-
- def run(self, callback: Callable[[], Iterable[MessageKind]]) -> None:
- messages = list(callback())
- if len(messages) > 0:
- self.event.messages = messages
- self.bus.send_event(self.event, notify=True)
- del self.bus
- del self.event
-
-
-class BasicSession(Session):
-
- def __init__(
- self, *,
- ghost_name: str,
- ghost_role: str,
- upstream: Stream,
- eventbus: EventBus,
- pool: Pool,
- processes: Processes,
- tasks: Tasks,
- threads: Threads,
- logger: LoggerItf,
- # 当前任务信息.
- process: Process,
- task: Task,
- thread: MsgThread,
- ):
- self._pool = pool
- self._upstream = upstream
- self._logger = logger
- self._tasks: Tasks = tasks
- self._processes: Processes = processes
- self._ghost_name: str = ghost_name
- self._message_role: str = ghost_role
- self._threads: Threads = threads
- self._eventbus: EventBus = eventbus
- # 需要管理的状态.
- self._task: Task = task
- self._process: Process = process
- self._creating: List[Task] = []
- self._thread: MsgThread = thread
- self._firing_events: List[Event] = []
- self._fetched_task_briefs: Dict[str, TaskBrief] = {}
-
- def id(self) -> str:
- return self._task.session_id
-
- def alive(self) -> bool:
- return (
- not self._upstream.stopped()
- and self._task.lock is not None
- )
-
- def refresh_lock(self) -> bool:
- lock = self._task.lock if self._task.lock else ""
- lock = self._tasks.refresh_task_lock(self._task.task_id, lock)
- if lock:
- self._task.lock = lock
- return True
- return False
-
- def process(self) -> "Process":
- return self._process
-
- def task(self) -> "Task":
- return self._task
-
- def thread(self) -> "MsgThread":
- return self._thread
-
- def messenger(
- self, *,
- sending: bool = True,
- saving: bool = True,
- thread: Optional[MsgThread] = None,
- name: Optional[str] = None,
- buffer: Optional[Buffer] = None,
- payloads: Optional[Iterable[Payload]] = None,
- attachments: Optional[Iterable[Attachment]] = None,
- functional_tokens: Optional[Iterable[FunctionalToken]] = None
- ) -> "Messenger":
- payload = TaskPayload.from_task(self._task)
- if payloads is None:
- payloads = []
- payloads.append(payload)
- name = name if name else self._assistant_name()
- thread = thread if thread else self._thread
-
- messenger = DefaultMessenger(
- upstream=self._upstream,
- saving=saving,
- thread=thread,
- buffer=buffer,
- name=name,
- payloads=payloads,
- attachments=attachments,
- role=self._message_role,
- logger=self._logger,
- functional_tokens=functional_tokens,
- )
- return messenger
-
- def _assistant_name(self) -> str:
- if self._task.assistant:
- return self._task.assistant.name
- return self._ghost_name
-
- def send_messages(self, *messages: MessageKind, role: str = Role.ASSISTANT.value) -> List[Message]:
- parser = MessageKindParser(self._message_role)
- outputs = parser.parse(messages)
- messenger = self.messenger()
- messenger.send(outputs)
- sent, callers = messenger.flush()
- self._logger.info(f"send message by session [send_messages], sent: {len(sent)}, callers: {len(callers)}")
- return sent
-
- def update_task(self, task: "Task", thread: Optional["MsgThread"], update_history: bool) -> None:
- self._task = task
- if thread is not None:
- self._task.thread_id = thread.id
- self._thread = thread.update_history()
- if update_history:
- self._thread = self._thread.update_history()
-
- def update_thread(self, thread: "MsgThread", update_history: bool) -> None:
- if update_history:
- thread = thread.update_history()
- self._thread = thread
-
- def create_tasks(self, *tasks: "Task") -> None:
- self._creating.extend(tasks)
-
- def fire_events(self, *events: "Event") -> None:
- extending = []
- from_task_name = self._task.name
- from_task_id = self._task.task_id
- for e in events:
- if e.task_id == self._task.parent:
- e.callback = True
- e.from_task_id = from_task_id
- e.from_task_name = from_task_name
- extending.append(e)
- self._firing_events.extend(extending)
-
- def future(self, name: str, call: Callable[[], Iterable[MessageKind]], reason: str) -> None:
- future_id = uuid()
- # 增加一个消息.
- system = DefaultMessageTypes.DEFAULT.new_system(
- content=f"async call `{name}` with id `{future_id}`, wait for future callback.",
- )
- self.send_messages(system)
- event = DefaultEventType.OBSERVE.new(
- task_id=self._task.task_id,
- from_task_id=self._task.task_id,
- messages=[],
- )
- # 让异步任务全局执行.
- future = Future(future_id, self._eventbus, event)
- self._pool.submit(future.run)
-
- def _do_quit(self) -> None:
- main_task_id = self._process.main_task_id
- task = self._tasks.get_task(main_task_id, False)
- self._firing_events = []
- for task_id in task.children:
- event = DefaultEventType.KILLING.new(
- task_id=task_id,
- messages=[],
- from_task_id=self._task.task_id,
- reason="the process is quited"
- )
- self._firing_events.append(event)
- self._do_fire_events()
-
- def _do_create_tasks(self) -> None:
- if self._creating:
- self._tasks.save_task(*self._creating)
- self._creating = []
-
- def _do_fire_events(self) -> None:
- if not self._firing_events:
- return
- process = self._process
- bus = self._eventbus
- main_task_id = process.main_task_id
- for e in self._firing_events:
- # 异步进程需要通知.
- notify = not self._upstream.is_streaming() or e.task_id != main_task_id
- self._logger.info(f"fire event {e.type}: eid {e.id}; task_id {e.task_id}")
- bus.send_event(e, notify)
- self._firing_events = []
-
- def _do_finish_task_and_thread(self) -> None:
- self._task.thread_id = self._thread.id
- task = self._task
- # 回收掉完成的任务.
- if task.children and task.too_much_children():
- children = self.get_task_briefs(children=True)
- idx = 0
- max_idx = len(children) - 1
- while task.too_much_children() and idx < max_idx:
- idx += 1
- child = children[idx]
- if child.is_overdue() or TaskState.is_dead(child.task_state):
- task.remove_child(child.task_id)
- task.update_turn()
- self._task = task
- self._fetched_task_briefs = {}
- self._thread = self._thread.update_history()
- self._tasks.save_task(task)
- self._threads.save_thread(self._thread)
-
- def get_task_briefs(self, *task_ids, children: bool = False) -> "List[TaskBrief]":
- ids = set(task_ids)
- result = []
- if children:
- for task_id in self._task.children:
- ids.add(task_id)
- if not ids:
- return result
-
- fetch = []
- for task_id in ids:
- if task_id in self._fetched_task_briefs:
- result.append(self._fetched_task_briefs[task_id])
- else:
- fetch.append(task_id)
- if fetch:
- briefs = self._tasks.get_task_briefs(fetch)
- for task_brief in briefs:
- result.append(task_brief)
- self._fetched_task_briefs[task_brief.task_id] = task_brief
- return result
-
- def tasks(self) -> Tasks:
- return self._tasks
-
- def processes(self) -> Processes:
- return self._processes
-
- def threads(self) -> Threads:
- return self._threads
-
- def eventbus(self) -> EventBus:
- return self._eventbus
-
- def update_process(self, process: "Process") -> None:
- self._process = process
-
- def quit(self) -> None:
- if self._process.main_task_id != self._task.task_id:
- raise RuntimeError(
- f"only main task {self._process.main_task_id} is able to quit process, not {self._task.task_id}"
- )
- self._process.quited = True
-
- def destroy(self) -> None:
- del self._upstream
- del self._logger
- del self._task
- del self._tasks
- del self._thread
- del self._threads
- del self._process
- del self._processes
- del self._firing_events
- del self._fetched_task_briefs
- del self._pool
-
- def save(self) -> None:
- with self._eventbus.transaction():
- with self._tasks.transaction():
- with self._threads.transaction():
- with self._processes.transaction():
- if self._process.quited:
- self._do_quit()
- else:
- self._do_create_tasks()
- self._do_finish_task_and_thread()
- self._do_fire_events()
- if self._process.main_task_id == self._task.task_id:
- self._processes.save_process(process=self._process)
-
- def fail(self, err: Optional[Exception]) -> None:
- # 暂时只做解开锁.
- locked = self._task.lock
- if locked:
- self._tasks.unlock_task(self._task.task_id, locked)
- self._task.lock = None
- self._upstream.deliver(DefaultMessageTypes.ERROR.new(content=str(err)))
- self._logger.error(err)
-
- def done(self) -> None:
- locked = self._task.lock
- if locked:
- self._tasks.unlock_task(self._task.task_id, locked)
- self._upstream.deliver(DefaultMessageTypes.final())
diff --git a/ghostos/framework/shells/__init__.py b/ghostos/framework/shells/__init__.py
deleted file mode 100644
index 63107798..00000000
--- a/ghostos/framework/shells/__init__.py
+++ /dev/null
@@ -1,2 +0,0 @@
-from ghostos.framework.shells.basic import BasicShell
-from ghostos.framework.shells.empty import EmptyShell
\ No newline at end of file
diff --git a/ghostos/framework/shells/basic.py b/ghostos/framework/shells/basic.py
deleted file mode 100644
index c1c7d602..00000000
--- a/ghostos/framework/shells/basic.py
+++ /dev/null
@@ -1,38 +0,0 @@
-from typing import Type, Iterable, List, Tuple
-
-from ghostos.container import ABSTRACT
-from ghostos.core.ghosts import Action
-from ghostos.core.ghosts.shells import Shell
-
-
-class BasicShell(Shell):
-
- def __init__(
- self, *,
- shell_id: str,
- prompt: str,
- actions: List[Action],
- drivers: List[Tuple[Type, object]]
- ):
- self._id = shell_id
- self._actions = actions
- self._prompt = prompt
- self._drivers = {t: i for t, i in drivers}
-
- def id(self) -> str:
- return self._id
-
- def shell_prompt(self) -> str:
- return self._prompt
-
- def actions(self) -> Iterable[Action]:
- return self._actions
-
- def drivers(self) -> Iterable[Type[ABSTRACT]]:
- for driver in self._drivers:
- yield driver
-
- def get_driver(self, driver: Type[ABSTRACT]) -> ABSTRACT:
- if driver not in self._drivers:
- raise KeyError(f"Driver {driver} not supported")
- return self._drivers[driver]
diff --git a/ghostos/framework/shells/empty.py b/ghostos/framework/shells/empty.py
deleted file mode 100644
index 7c2b53e9..00000000
--- a/ghostos/framework/shells/empty.py
+++ /dev/null
@@ -1,12 +0,0 @@
-from ghostos.framework.shells.basic import BasicShell
-
-
-class EmptyShell(BasicShell):
-
- def __init__(self):
- super().__init__(
- shell_id="empty_shell",
- prompt="",
- actions=[],
- drivers=[]
- )
diff --git a/ghostos/framework/storage/__init__.py b/ghostos/framework/storage/__init__.py
index 58f1cbac..aa087a8a 100644
--- a/ghostos/framework/storage/__init__.py
+++ b/ghostos/framework/storage/__init__.py
@@ -1,2 +1,3 @@
-from ghostos.framework.storage.filestorage import FileStorageProvider, FileStorage
+from ghostos.contracts.storage import Storage, FileStorage
+from ghostos.framework.storage.filestorage import FileStorageProvider, FileStorageImpl
from ghostos.framework.storage.memstorage import MemStorage
diff --git a/ghostos/framework/storage/filestorage.py b/ghostos/framework/storage/filestorage.py
index d7f4675b..dc99acf4 100644
--- a/ghostos/framework/storage/filestorage.py
+++ b/ghostos/framework/storage/filestorage.py
@@ -1,20 +1,33 @@
import os
import re
-from typing import Optional, AnyStr, Type, Iterable
-from ghostos.container import Provider, Container
-from ghostos.contracts.storage import Storage
+from typing import Optional, Iterable
+from ghostos.container import Provider, Container, ABSTRACT
+from ghostos.contracts.storage import Storage, FileStorage
+__all__ = ["FileStorageProvider", "FileStorageImpl"]
-class FileStorage(Storage):
+
+class FileStorageImpl(FileStorage):
+ """
+ FileStorage implementation based on python filesystem.
+ Simplest implementation.
+ """
def __init__(self, dir_: str):
self._dir: str = os.path.abspath(dir_)
+ def abspath(self) -> str:
+ return self._dir
+
def get(self, file_path: str) -> bytes:
file_path = self._join_file_path(file_path)
with open(file_path, 'rb') as f:
return f.read()
+ def remove(self, file_path: str) -> None:
+ file_path = self._join_file_path(file_path)
+ os.remove(file_path)
+
def exists(self, file_path: str) -> bool:
file_path = self._join_file_path(file_path)
return os.path.exists(file_path)
@@ -36,11 +49,11 @@ def put(self, file_path: str, content: bytes) -> None:
with open(file_path, 'wb') as f:
f.write(content)
- def sub_storage(self, relative_path: str) -> "Storage":
+ def sub_storage(self, relative_path: str) -> "FileStorage":
if not relative_path:
return self
dir_path = self._join_file_path(relative_path)
- return FileStorage(dir_path)
+ return FileStorageImpl(dir_path)
def dir(self, prefix_dir: str, recursive: bool, patten: Optional[str] = None) -> Iterable[str]:
dir_path = self._join_file_path(prefix_dir)
@@ -66,7 +79,7 @@ def _match_file_pattern(filename: str, pattern: Optional[str]) -> bool:
return r is not None
-class FileStorageProvider(Provider[Storage]):
+class FileStorageProvider(Provider[FileStorage]):
def __init__(self, dir_: str):
self._dir: str = dir_
@@ -74,8 +87,8 @@ def __init__(self, dir_: str):
def singleton(self) -> bool:
return True
- def contract(self) -> Type[Storage]:
- return Storage
+ def aliases(self) -> Iterable[ABSTRACT]:
+ yield Storage
def factory(self, con: Container) -> Optional[Storage]:
- return FileStorage(self._dir)
+ return FileStorageImpl(self._dir)
diff --git a/ghostos/framework/storage/memstorage.py b/ghostos/framework/storage/memstorage.py
index 1a2ce997..64cb2ea1 100644
--- a/ghostos/framework/storage/memstorage.py
+++ b/ghostos/framework/storage/memstorage.py
@@ -1,17 +1,20 @@
-from typing import Optional, Iterable, AnyStr, Dict
+from typing import Optional, Iterable, Dict
-from ghostos.contracts.storage import Storage
+from ghostos.contracts.storage import FileStorage
from os.path import join
__all__ = ["MemStorage"]
-class MemStorage(Storage):
+class MemStorage(FileStorage):
def __init__(self, saved: Dict[str, bytes] = None, namespace: str = ""):
self._namespace = namespace
self._saved: Dict[str, bytes] = saved if saved else {}
+ def abspath(self) -> str:
+ return "/test/mem/"
+
def sub_storage(self, relative_path: str) -> "Storage":
namespace = self._namespace
if relative_path:
@@ -20,16 +23,24 @@ def sub_storage(self, relative_path: str) -> "Storage":
def get(self, file_path: str) -> bytes:
key = join(self._namespace, file_path)
+ key = key.lstrip('/')
if key not in self._saved:
raise KeyError(key)
return self._saved.get(key)
def exists(self, file_path: str) -> bool:
key = join(self._namespace, file_path)
+ key = key.lstrip('/')
return key in self._saved
+ def remove(self, file_path: str) -> None:
+ key = join(self._namespace, file_path)
+ key = key.lstrip('/')
+ del self._saved[key]
+
def put(self, file_path: str, content: bytes) -> None:
key = join(self._namespace, file_path)
+ key = key.lstrip('/')
self._saved[key] = content
def dir(self, prefix_dir: str, recursive: bool, patten: Optional[str] = None) -> Iterable[str]:
diff --git a/ghostos/framework/streams/__init__.py b/ghostos/framework/streams/__init__.py
deleted file mode 100644
index 86186b94..00000000
--- a/ghostos/framework/streams/__init__.py
+++ /dev/null
@@ -1,2 +0,0 @@
-from ghostos.framework.streams.queuestream import QueueStream
-from ghostos.framework.streams.empty import EmptyStream
\ No newline at end of file
diff --git a/ghostos/framework/streams/empty.py b/ghostos/framework/streams/empty.py
deleted file mode 100644
index eccd7659..00000000
--- a/ghostos/framework/streams/empty.py
+++ /dev/null
@@ -1,29 +0,0 @@
-from typing import Iterable
-
-from ghostos.core.messages import Stream, Message, DefaultMessageTypes
-
-
-class EmptyStream(Stream):
-
- def __init__(self, max_final: int = 0):
- self._max_final = max_final
- self._final_count = 0
-
- def deliver(self, pack: "Message") -> bool:
- if self.stopped():
- return False
- if DefaultMessageTypes.is_final(pack):
- self._final_count += 1
- return True
-
- def is_streaming(self) -> bool:
- return False
-
- def send(self, messages: Iterable[Message]) -> bool:
- for item in messages:
- if not self.deliver(item):
- return False
- return True
-
- def stopped(self) -> bool:
- return self._final_count > self._max_final
diff --git a/ghostos/framework/streams/queuestream.py b/ghostos/framework/streams/queuestream.py
deleted file mode 100644
index 5d147215..00000000
--- a/ghostos/framework/streams/queuestream.py
+++ /dev/null
@@ -1,51 +0,0 @@
-from typing import Iterable
-
-from ghostos.core.messages import Stream, Message, DefaultMessageTypes
-from queue import Queue
-
-__all__ = ["QueueStream"]
-
-
-class QueueStream(Stream):
-
- def __init__(self, queue: Queue, streaming: bool = True, max_final: int = 1):
- self._queue = queue
- self._streaming = streaming
- self._stopped = False
- self._max_final = max_final
- self._final_count = 0
-
- def deliver(self, pack: "Message") -> bool:
- if self._stopped:
- return False
- if DefaultMessageTypes.is_protocol_type(pack):
- if DefaultMessageTypes.ERROR.match(pack):
- self._queue.put(pack, block=True)
- self._queue.task_done()
- self._stopped = True
- elif DefaultMessageTypes.FINAL.match(pack):
- self._final_count += 1
- if self._final_count >= self._max_final:
- self._stopped = True
- self._queue.task_done()
- self._queue.put(pack, block=True)
- return True
- elif self._streaming and not pack.is_tail():
- # 不发送间包, 只发送尾包.
- return True
- else:
- self._queue.put(pack, block=True)
- return True
-
- def is_streaming(self) -> bool:
- return self._streaming
-
- def send(self, messages: Iterable[Message]) -> bool:
- for item in messages:
- ok = self.deliver(item)
- if not ok:
- return False
- return True
-
- def stopped(self) -> bool:
- return self._stopped
diff --git a/ghostos/framework/taskflow/basic.py b/ghostos/framework/taskflow/basic.py
index 9b965594..84b95db8 100644
--- a/ghostos/framework/taskflow/basic.py
+++ b/ghostos/framework/taskflow/basic.py
@@ -1,6 +1,6 @@
from typing import Dict, Any
from ghostos.core.ghosts import Taskflow, Operator
-from ghostos.core.messages import DefaultMessageTypes
+from ghostos.core.messages import MessageType
from ghostos.framework.operators import (
ThinkOperator,
FinishOperator,
@@ -27,8 +27,8 @@ def observe(self, objects: Dict[str, Any], reason: str = "", instruction: str =
content = yaml_pretty_dump(values)
# 用什么协议没想明白, function ? tool? system ?
- content = "# observe values: \n" + content
- msg = DefaultMessageTypes.DEFAULT.new_system(
+ content = "observe values: \n" + content
+ msg = MessageType.DEFAULT.new_system(
content=content,
)
observation.append(msg)
diff --git a/ghostos/framework/tasks/__init__.py b/ghostos/framework/tasks/__init__.py
index 9fec82f5..0ff1e27a 100644
--- a/ghostos/framework/tasks/__init__.py
+++ b/ghostos/framework/tasks/__init__.py
@@ -1 +1,2 @@
+from ghostos.core.runtime import GoTasks
from ghostos.framework.tasks.storage_tasks import StorageTasksImplProvider, WorkspaceTasksProvider
diff --git a/ghostos/framework/tasks/storage_tasks.py b/ghostos/framework/tasks/storage_tasks.py
index 83a1c38b..245a9797 100644
--- a/ghostos/framework/tasks/storage_tasks.py
+++ b/ghostos/framework/tasks/storage_tasks.py
@@ -1,64 +1,119 @@
-from typing import Optional, List, Iterable, Dict, Type
+import time
+from typing import Optional, List, Iterable, Type, TypedDict
import yaml
-from ghostos.core.session import TaskState, TaskBrief, Task, Tasks
-from ghostos.core.ghosts import Workspace
+from ghostos.core.runtime import TaskState, TaskBrief, GoTaskStruct, GoTasks
+from ghostos.contracts.workspace import Workspace
from ghostos.contracts.logger import LoggerItf
from ghostos.contracts.storage import Storage
from ghostos.container import Provider, Container
-from ghostos.helpers import uuid
+from ghostos.core.runtime.tasks import TaskLocker
+from ghostos.helpers import uuid, timestamp
+
+__all__ = ['StorageGoTasksImpl', 'StorageTasksImplProvider', 'WorkspaceTasksProvider']
+
+
+class SimpleStorageLocker(TaskLocker):
+ class LockData(TypedDict):
+ lock_id: str
+ overdue: float
+
+ def __init__(self, storage: Storage, task_id: str, overdue: float, force: bool = False):
+ self.task_id = task_id
+ self.storage = storage
+ self.lock_id = uuid()
+ self._acquired = False
+ self._overdue = overdue
+ self._force = force
+
+ def acquire(self) -> bool:
+ filename = self.locker_file_name()
+ if self.storage.exists(filename):
+ content = self.storage.get(filename)
+ data = None
+ if content:
+ data = yaml.safe_load(content)
+ if data:
+ lock = self.LockData(**data)
+ now = time.time()
+ if lock['lock_id'] == self.lock_id or now - float(lock["overdue"]) > 0:
+ self.create_lock()
+ return True
+ elif not self._acquired and self._force:
+ self.create_lock()
+ return True
+ else:
+ return False
+ self.create_lock()
+ return True
+
+ def acquired(self) -> bool:
+ return self._acquired
+
+ def create_lock(self) -> None:
+ filename = self.locker_file_name()
+ overdue_at = time.time() + self._overdue
+ lock = self.LockData(lock_id=self.lock_id, overdue=overdue_at)
+ content = yaml.safe_dump(lock)
+ self.storage.put(filename, content.encode())
+ self._acquired = True
+
+ def locker_file_name(self) -> str:
+ return f'{self.task_id}.lock'
+
+ def refresh(self) -> bool:
+ if not self._acquired:
+ return False
+ return self.acquire()
-__all__ = ['StorageTasksImpl', 'StorageTasksImplProvider', 'WorkspaceTasksProvider']
+ def release(self) -> bool:
+ if not self._acquired:
+ return False
+ filename = self.locker_file_name()
+ if self.refresh():
+ self.storage.remove(filename)
+ self._acquired = False
+ return True
+ return False
-class StorageTasksImpl(Tasks):
+class StorageGoTasksImpl(GoTasks):
def __init__(self, storage: Storage, logger: LoggerItf):
self._storage = storage
self._logger = logger
- self._locks: Dict[str, str] = {}
- def save_task(self, *tasks: Task) -> None:
+ def save_task(self, *tasks: GoTaskStruct) -> None:
for task in tasks:
filename = self._get_task_filename(task.task_id)
- content = yaml.safe_dump(task.model_dump(exclude_defaults=True))
- # todo: 正确的做法要先 check lock.
+ data = task.model_dump(exclude_defaults=True)
+ content = yaml.safe_dump(data)
+ task.updated = timestamp()
self._storage.put(filename, content.encode('utf-8'))
@staticmethod
def _get_task_filename(task_id: str) -> str:
return f"{task_id}.task.yml"
- def _get_task(self, task_id: str) -> Optional[Task]:
+ def _get_task(self, task_id: str) -> Optional[GoTaskStruct]:
filename = self._get_task_filename(task_id)
if not self._storage.exists(filename):
return None
content = self._storage.get(filename)
data = yaml.safe_load(content)
- task = Task(**data)
+ task = GoTaskStruct(**data)
return task
def exists(self, task_id: str) -> bool:
filename = self._get_task_filename(task_id)
return self._storage.exists(filename)
- def get_task(self, task_id: str, lock: bool) -> Optional[Task]:
- task = self._get_task(task_id)
- if task is None:
- return None
- if lock:
- if task.lock:
- return None
- task.lock = uuid()
- self.save_task(task)
- return task
- else:
- task.lock = None
- return task
-
- def get_tasks(self, task_ids: List[str], states: Optional[List[TaskState]] = None) -> Iterable[Task]:
+ def get_task(self, task_id: str) -> Optional[GoTaskStruct]:
+ return self._get_task(task_id)
+
+ def get_tasks(self, task_ids: List[str], states: Optional[List[TaskState]] = None) -> Iterable[GoTaskStruct]:
states = set(states) if states else None
for task_id in task_ids:
- task = self.get_task(task_id, lock=False)
+ task = self.get_task(task_id)
if states and task.state not in states:
continue
yield task
@@ -67,27 +122,11 @@ def get_task_briefs(self, task_ids: List[str], states: Optional[List[TaskState]]
for task in self.get_tasks(task_ids, states):
yield TaskBrief.from_task(task)
- def unlock_task(self, task_id: str, lock: str) -> None:
- task = self._get_task(task_id)
- if task is None:
- return
- if task.lock == lock:
- task.lock = None
- self.save_task(task)
-
- def refresh_task_lock(self, task_id: str, lock: str) -> Optional[str]:
- task = self._get_task(task_id)
- if task is None:
- return uuid()
- if task.lock or task.lock == lock:
- lock = uuid()
- task.lock = lock
- self.save_task(task)
- return lock
- return None
-
-
-class StorageTasksImplProvider(Provider[Tasks]):
+ def lock_task(self, task_id: str, overdue: float = 30, force: bool = False) -> TaskLocker:
+ return SimpleStorageLocker(self._storage, task_id, overdue, force)
+
+
+class StorageTasksImplProvider(Provider[GoTasks]):
"""
provide storage based Tasks
"""
@@ -98,17 +137,17 @@ def __init__(self, tasks_dir: str = "runtime/tasks"):
def singleton(self) -> bool:
return True
- def contract(self) -> Type[Tasks]:
- return Tasks
+ def contract(self) -> Type[GoTasks]:
+ return GoTasks
- def factory(self, con: Container) -> Optional[Tasks]:
+ def factory(self, con: Container) -> Optional[GoTasks]:
logger = con.force_fetch(LoggerItf)
storage = con.force_fetch(Storage)
tasks_storage = storage.sub_storage(self.tasks_dir)
- return StorageTasksImpl(tasks_storage, logger)
+ return StorageGoTasksImpl(tasks_storage, logger)
-class WorkspaceTasksProvider(Provider[Tasks]):
+class WorkspaceTasksProvider(Provider[GoTasks]):
def __init__(self, namespace: str = "tasks"):
self.namespace = namespace
@@ -116,12 +155,12 @@ def __init__(self, namespace: str = "tasks"):
def singleton(self) -> bool:
return True
- def contract(self) -> Type[Tasks]:
- return Tasks
+ def contract(self) -> Type[GoTasks]:
+ return GoTasks
- def factory(self, con: Container) -> Optional[Tasks]:
+ def factory(self, con: Container) -> Optional[GoTasks]:
workspace = con.force_fetch(Workspace)
runtime_storage = workspace.runtime()
tasks_storage = runtime_storage.sub_storage(self.namespace)
logger = con.force_fetch(LoggerItf)
- return StorageTasksImpl(tasks_storage, logger)
+ return StorageGoTasksImpl(tasks_storage, logger)
diff --git a/ghostos/framework/threads/__init__.py b/ghostos/framework/threads/__init__.py
index a8284341..75553330 100644
--- a/ghostos/framework/threads/__init__.py
+++ b/ghostos/framework/threads/__init__.py
@@ -1 +1,2 @@
-from ghostos.framework.threads.storage_threads import StorageThreadsProvider, WorkspaceThreadsProvider
+from ghostos.core.runtime import GoThreads, GoThreadInfo
+from ghostos.framework.threads.storage_threads import MsgThreadRepoByStorageProvider, MsgThreadsRepoByWorkSpaceProvider
diff --git a/ghostos/framework/threads/storage_threads.py b/ghostos/framework/threads/storage_threads.py
index 0b72dce0..a56ca22d 100644
--- a/ghostos/framework/threads/storage_threads.py
+++ b/ghostos/framework/threads/storage_threads.py
@@ -1,6 +1,6 @@
from typing import Optional, Type
-from ghostos.core.session import MsgThread, Threads, SimpleMsgThread
-from ghostos.core.ghosts import Workspace
+from ghostos.core.runtime import GoThreadInfo, GoThreads, ThreadHistory
+from ghostos.contracts.workspace import Workspace
from ghostos.contracts.storage import Storage
from ghostos.contracts.logger import LoggerItf
from ghostos.helpers import yaml_pretty_dump
@@ -8,10 +8,10 @@
import yaml
import os
-__all__ = ['StorageThreads', 'StorageThreadsProvider', 'WorkspaceThreadsProvider']
+__all__ = ['GoThreadsByStorage', 'MsgThreadRepoByStorageProvider', 'MsgThreadsRepoByWorkSpaceProvider']
-class StorageThreads(Threads):
+class GoThreadsByStorage(GoThreads):
def __init__(
self, *,
@@ -23,46 +23,37 @@ def __init__(
self._logger = logger
self._allow_saving_file = allow_saving_file
- def get_thread(self, thread_id: str, create: bool = False) -> Optional[MsgThread]:
+ def get_thread(self, thread_id: str, create: bool = False) -> Optional[GoThreadInfo]:
path = self._get_thread_filename(thread_id)
if not self._storage.exists(path):
+ if create:
+ thread = GoThreadInfo(id=thread_id)
+ self.save_thread(thread)
+ return thread
return None
content = self._storage.get(path)
data = yaml.safe_load(content)
- thread = MsgThread(**data)
+ thread = GoThreadInfo(**data)
return thread
- def save_thread(self, thread: MsgThread) -> None:
+ def save_thread(self, thread: GoThreadInfo) -> None:
data = thread.model_dump(exclude_defaults=True)
data_content = yaml_pretty_dump(data)
path = self._get_thread_filename(thread.id)
saving = data_content.encode('utf-8')
self._storage.put(path, saving)
- # saving to special file
- if thread.save_file and self._allow_saving_file:
- simple = SimpleMsgThread.from_thread(thread)
- simple_data = simple.model_dump(exclude_defaults=True)
- content = yaml_pretty_dump(simple_data)
- if thread.save_file.startswith('/'):
- # saving to absolute path
- saving_dir = os.path.dirname(thread.save_file)
- if not os.path.exists(saving_dir):
- os.makedirs(saving_dir)
- with open(thread.save_file, 'wb') as f:
- f.write(content.encode('UTF-8'))
- else:
- # saving to relative path
- self._storage.put(thread.save_file, content.encode('UTF-8'))
@staticmethod
def _get_thread_filename(thread_id: str) -> str:
return thread_id + ".thread.yml"
- def fork_thread(self, thread: MsgThread) -> MsgThread:
- return thread.fork()
+ def fork_thread(self, thread: GoThreadInfo) -> GoThreadInfo:
+ fork = thread.fork()
+ self.save_thread(fork)
+ return fork
-class StorageThreadsProvider(Provider[Threads]):
+class MsgThreadRepoByStorageProvider(Provider[GoThreads]):
def __init__(self, threads_dir: str = "runtime/threads"):
self._threads_dir = threads_dir
@@ -70,17 +61,17 @@ def __init__(self, threads_dir: str = "runtime/threads"):
def singleton(self) -> bool:
return True
- def contract(self) -> Type[Threads]:
- return Threads
+ def contract(self) -> Type[GoThreads]:
+ return GoThreads
- def factory(self, con: Container) -> Optional[Threads]:
+ def factory(self, con: Container) -> Optional[GoThreads]:
storage = con.force_fetch(Storage)
threads_storage = storage.sub_storage(self._threads_dir)
logger = con.force_fetch(LoggerItf)
- return StorageThreads(storage=threads_storage, logger=logger)
+ return GoThreadsByStorage(storage=threads_storage, logger=logger)
-class WorkspaceThreadsProvider(Provider[Threads]):
+class MsgThreadsRepoByWorkSpaceProvider(Provider[GoThreads]):
def __init__(self, namespace: str = "threads"):
self._namespace = namespace
@@ -88,11 +79,11 @@ def __init__(self, namespace: str = "threads"):
def singleton(self) -> bool:
return True
- def contract(self) -> Type[Threads]:
- return Threads
+ def contract(self) -> Type[GoThreads]:
+ return GoThreads
- def factory(self, con: Container) -> Optional[Threads]:
+ def factory(self, con: Container) -> Optional[GoThreads]:
workspace = con.force_fetch(Workspace)
logger = con.force_fetch(LoggerItf)
threads_storage = workspace.runtime().sub_storage(self._namespace)
- return StorageThreads(storage=threads_storage, logger=logger)
+ return GoThreadsByStorage(storage=threads_storage, logger=logger)
diff --git a/ghostos/framework/translation/__init__.py b/ghostos/framework/translation/__init__.py
new file mode 100644
index 00000000..7ed2135b
--- /dev/null
+++ b/ghostos/framework/translation/__init__.py
@@ -0,0 +1,2 @@
+from ghostos.contracts.translation import Translation
+from ghostos.framework.translation.dict_impl import WorkspaceTranslationProvider
diff --git a/ghostos/framework/translation/dict_impl.py b/ghostos/framework/translation/dict_impl.py
new file mode 100644
index 00000000..cb9b5a1f
--- /dev/null
+++ b/ghostos/framework/translation/dict_impl.py
@@ -0,0 +1,120 @@
+from typing import Dict, List, Optional, Iterable
+from abc import ABC, abstractmethod
+from ghostos.contracts.translation import Translator, Translation, DomainTranslator, TransItem
+from ghostos.contracts.storage import FileStorage
+from ghostos.contracts.workspace import Workspace
+from ghostos.container import Provider, Container, INSTANCE
+from ghostos.helpers import yaml_pretty_dump, generate_import_path
+from pydantic import BaseModel, Field
+import yaml
+
+
+class DomainTranslationData(BaseModel):
+ domain: str = Field(description="the target domain")
+ langs: List[str] = Field(default_factory=lambda: ["zh", "en"], description="the target langs")
+ default_lang: str = "en"
+ items: Dict[str, TransItem] = Field(default_factory=dict)
+
+
+class BasicDomainTranslator(DomainTranslator, Translator, ABC):
+
+ def __init__(self, data: DomainTranslationData):
+ self.data = data
+
+ def gettext(self, message: str, lang: str = "", **kwargs: str) -> str:
+ item = self.get_item(message)
+ if not lang:
+ lang = self.default_lang
+ return item.gettext(lang, **kwargs)
+
+ def get_item(self, item_id: str) -> TransItem:
+ if item_id not in self.data.items:
+ self.data.items[item_id] = TransItem(id=item_id)
+ self.save()
+ item = self.data.items.get(item_id)
+ return item
+
+ def domain(self) -> str:
+ return self.data.domain
+
+ def langs(self) -> List[str]:
+ return self.data.langs
+
+ def default_lang(self) -> str:
+ return self.data.default_lang
+
+ def update(self, lang: str, text: str, value: str):
+ item = self.data.items.get(text)
+ if item is None:
+ item = TransItem(id=text)
+ item.translations[lang] = value
+ self.data.items[item.id] = item
+ self.save()
+
+
+class YamlDomainTranslator(BasicDomainTranslator):
+
+ def __init__(self, storage: FileStorage, domain: str):
+ self.storage = storage
+ filename = self.domain_yaml_name(domain)
+ if not storage.exists(filename):
+ data = DomainTranslationData(domain=domain)
+ self.do_save(data)
+ else:
+ content = storage.get(filename)
+ unmarshal = yaml.safe_load(content)
+ data = DomainTranslationData(**unmarshal)
+ super().__init__(data)
+
+ def get_translator(self, lang: str = "") -> Translator:
+ if lang and lang != self.data.default_lang:
+ self.data.default_lang = lang
+ self.save()
+ return self
+
+ @staticmethod
+ def domain_yaml_name(domain: str) -> str:
+ return f"{domain}.yml"
+
+ def items(self) -> Iterable[TransItem]:
+ yield from self.data.items.values()
+
+ def save(self) -> None:
+ self.do_save(self.data)
+
+ def do_save(self, data_obj: DomainTranslationData) -> None:
+ data = data_obj.model_dump()
+ content = yaml_pretty_dump(data)
+ content = f"# model is {generate_import_path(DomainTranslationData)} \n" + content
+ filename = f"{data_obj.domain}.yml"
+ self.storage.put(filename, content.encode())
+
+
+class YamlAssetTranslation(Translation):
+
+ def __init__(self, asset_storage: FileStorage):
+ self.asset_storage = asset_storage
+ self.domain_translators = {}
+
+ def get_domain(self, domain: str) -> DomainTranslator:
+ if domain not in self.domain_translators:
+ translator = YamlDomainTranslator(self.asset_storage, domain)
+ self.domain_translators[domain] = translator
+ return self.domain_translators[domain]
+
+
+class WorkspaceTranslationProvider(Provider[Translation]):
+
+ def __init__(
+ self,
+ translation_dir: str = "translations",
+ ):
+ self.translation_dir = translation_dir
+
+ def singleton(self) -> bool:
+ return True
+
+ def factory(self, con: Container) -> Optional[INSTANCE]:
+ workspace = con.force_fetch(Workspace)
+ storage = workspace.assets().sub_storage(self.translation_dir)
+ return YamlAssetTranslation(storage)
diff --git a/ghostos/framework/variables/__init__.py b/ghostos/framework/variables/__init__.py
new file mode 100644
index 00000000..c3f8aa38
--- /dev/null
+++ b/ghostos/framework/variables/__init__.py
@@ -0,0 +1,7 @@
+from ghostos.contracts.variables import Variables
+from ghostos.framework.variables.variables_impl import VariablesImpl, WorkspaceVariablesProvider
+from ghostos.framework.storage import MemStorage
+
+test_variables = VariablesImpl(MemStorage())
+
+__all__ = ("Variables", "VariablesImpl", "WorkspaceVariablesProvider", "test_variables")
diff --git a/ghostos/framework/variables/variables_impl.py b/ghostos/framework/variables/variables_impl.py
new file mode 100644
index 00000000..c08f8b9b
--- /dev/null
+++ b/ghostos/framework/variables/variables_impl.py
@@ -0,0 +1,77 @@
+from typing import Optional, Type, Union, Any, TypeVar
+
+from pydantic import BaseModel
+
+from ghostos.contracts.variables import Variables
+from ghostos.contracts.storage import Storage
+from ghostos.contracts.workspace import Workspace
+from ghostos.entity import EntityType, to_entity_meta, from_entity_meta, EntityMeta
+from ghostos.identifier import try_get_identifier
+from ghostos.helpers import md5, generate_import_path, uuid
+from ghostos.container import Provider, Container
+import json
+
+T = TypeVar("T")
+
+
+class VariablesImpl(Variables):
+
+ def __init__(self, storage: Storage):
+ self.storage = storage
+
+ def save(
+ self,
+ val: Union[BaseModel, dict, list, str, int, float, bool, EntityType, Any],
+ desc: str = "",
+ ) -> Variables.Var:
+ if isinstance(val, Variables.Var):
+ return val
+ entity_meta = to_entity_meta(val)
+ type_ = generate_import_path(type(val))
+ id_ = try_get_identifier(val)
+ if id_ is not None and id_.id:
+ vid = md5(type_ + "::" + id_.id)
+ else:
+ vid = uuid()
+ var = Variables.Var(
+ vid=vid,
+ type=type_,
+ desc=desc,
+ )
+ content = json.dumps(entity_meta)
+ filename = self._get_filename(vid)
+ self.storage.put(filename, content.encode())
+ return var
+
+ @staticmethod
+ def _get_filename(vid: str) -> str:
+ return f"{vid}.var.json"
+
+ def load(self, vid: str, expect: Optional[Type[T]] = None, force: bool = False) -> Optional[T]:
+ filename = self._get_filename(vid)
+ if not self.storage.exists(filename):
+ if not force:
+ return None
+ else:
+ raise FileNotFoundError(f"variable {vid} not found at: {filename}")
+ content = self.storage.get(filename)
+ data = json.loads(content)
+ entity_meta = EntityMeta(**data)
+ entity = from_entity_meta(entity_meta)
+ if expect and not isinstance(entity, expect):
+ raise ValueError(f"variable {vid} expect {expect} but got {type(entity)}")
+ return entity
+
+
+class WorkspaceVariablesProvider(Provider[Variables]):
+
+ def __init__(self, relative_path: str = "variables"):
+ self.relative_path = relative_path
+
+ def singleton(self) -> bool:
+ return True
+
+ def factory(self, con: Container) -> Optional[Variables]:
+ ws = con.force_fetch(Workspace)
+ storage = ws.runtime().sub_storage(self.relative_path)
+ return VariablesImpl(storage)
diff --git a/ghostos/framework/workspaces/__init__.py b/ghostos/framework/workspaces/__init__.py
index 43cc7c6a..328db359 100644
--- a/ghostos/framework/workspaces/__init__.py
+++ b/ghostos/framework/workspaces/__init__.py
@@ -1 +1 @@
-from ghostos.framework.workspaces.basic import BasicWorkspaceProvider
+from ghostos.framework.workspaces.basic import BasicWorkspaceProvider, BasicWorkspace
diff --git a/ghostos/framework/workspaces/basic.py b/ghostos/framework/workspaces/basic.py
index 2ddbd397..4258e947 100644
--- a/ghostos/framework/workspaces/basic.py
+++ b/ghostos/framework/workspaces/basic.py
@@ -1,32 +1,38 @@
from typing import Optional, Type
-from ghostos.core.ghosts.workspace import Workspace
-from ghostos.contracts.storage import Storage
-from ghostos.container import Provider, Container, ABSTRACT
+from ghostos.contracts.workspace import Workspace
+from ghostos.framework.storage import FileStorage, FileStorageImpl
+from ghostos.container import Provider, Container, INSTANCE
class BasicWorkspace(Workspace):
def __init__(
self,
- root_storage: Storage,
+ workspace_storage: FileStorage,
runtime_path: str = "runtime",
- configs_path="configs",
- source_path="sources",
+ configs_path: str = "configs",
+ assets_path: str = "assets",
):
- self._root_storage = root_storage
- self._runtime_storage = root_storage.sub_storage(runtime_path)
- self._configs_storage = root_storage.sub_storage(configs_path)
- self._source_storage = root_storage.sub_storage(source_path)
+ self._storage: FileStorage = workspace_storage
+ self._runtime_storage = workspace_storage.sub_storage(runtime_path)
+ self._configs_storage = workspace_storage.sub_storage(configs_path)
+ self._assets_storage = workspace_storage.sub_storage(assets_path)
+
+ def root(self) -> FileStorage:
+ return self._storage
- def runtime(self) -> Storage:
+ def runtime(self) -> FileStorage:
return self._runtime_storage
- def configs(self) -> Storage:
- return self._configs_storage
+ def runtime_cache(self) -> FileStorage:
+ return self._runtime_storage.sub_storage("cache")
- def source(self) -> Storage:
- return self._source_storage
+ def assets(self) -> FileStorage:
+ return self._assets_storage
+
+ def configs(self) -> FileStorage:
+ return self._configs_storage
class BasicWorkspaceProvider(Provider):
@@ -36,28 +42,32 @@ class BasicWorkspaceProvider(Provider):
def __init__(
self,
- root_dir: str = "",
+ workspace_dir: str,
runtime_path: str = "runtime",
configs_path="configs",
- source_path="src",
+ assets_path: str = "assets",
):
- self._root_path = root_dir
+ """
+ :param workspace_dir: relative workspace dir to the root path
+ :param runtime_path: relative runtime path to the workspace dir
+ :param configs_path: relative configs path to the workspace dir
+ """
+ self._root_path = workspace_dir
self._runtime_path = runtime_path
self._configs_path = configs_path
- self._source_path = source_path
+ self._assets_path = assets_path
def singleton(self) -> bool:
return True
- def contract(self) -> Type[ABSTRACT]:
+ def contract(self) -> Type[INSTANCE]:
return Workspace
- def factory(self, con: Container) -> Optional[ABSTRACT]:
- storage = con.force_fetch(Storage)
- root_storage = storage.sub_storage(self._root_path)
+ def factory(self, con: Container) -> Optional[INSTANCE]:
+ root_storage = FileStorageImpl(self._root_path)
return BasicWorkspace(
root_storage,
runtime_path=self._runtime_path,
configs_path=self._configs_path,
- source_path=self._source_path,
+ assets_path=self._assets_path,
)
diff --git a/ghostos/ghosts/__init__.py b/ghostos/ghosts/__init__.py
new file mode 100644
index 00000000..3e7446d8
--- /dev/null
+++ b/ghostos/ghosts/__init__.py
@@ -0,0 +1,2 @@
+from ghostos.ghosts.chatbot import Chatbot
+from ghostos.ghosts.moss_agent import MossAgent
diff --git a/ghostos/ghosts/chatbot/__init__.py b/ghostos/ghosts/chatbot/__init__.py
new file mode 100644
index 00000000..bff5386d
--- /dev/null
+++ b/ghostos/ghosts/chatbot/__init__.py
@@ -0,0 +1 @@
+from ghostos.ghosts.chatbot.simplest import Chatbot
diff --git a/ghostos/ghosts/chatbot/chatbots.py b/ghostos/ghosts/chatbot/chatbots.py
new file mode 100644
index 00000000..72fa27f9
--- /dev/null
+++ b/ghostos/ghosts/chatbot/chatbots.py
@@ -0,0 +1,18 @@
+from abc import ABC, abstractmethod
+from typing import Optional, List
+from ghostos.ghosts.chatbot.simplest import Chatbot
+from ghostos.identifier import Identifier
+
+
+class Chatbots(ABC):
+ @abstractmethod
+ def save(self, bot: Chatbot) -> None:
+ pass
+
+ @abstractmethod
+ def find(self, bot_name: str) -> Optional[Chatbot]:
+ pass
+
+ @abstractmethod
+ def search(self, query: str) -> List[Identifier]:
+ pass
diff --git a/ghostos/ghosts/chatbot/simplest.py b/ghostos/ghosts/chatbot/simplest.py
new file mode 100644
index 00000000..095d9e2d
--- /dev/null
+++ b/ghostos/ghosts/chatbot/simplest.py
@@ -0,0 +1,102 @@
+from typing import Union, Iterable, ClassVar, List
+
+from ghostos.abcd import Agent, GhostDriver, Session, Operator
+from ghostos.abcd.thoughts import LLMThought, Thought
+from ghostos.container import Provider
+from ghostos.core.runtime import Event, GoThreadInfo
+from ghostos.core.messages import Role
+from ghostos.core.llms import Prompt, LLMFunc
+from ghostos.entity import ModelEntity
+from ghostos.prompter import TextPrmt, Prompter
+from ghostos.identifier import Identifier
+from pydantic import BaseModel, Field
+
+
+class Chatbot(ModelEntity, Agent):
+ """
+ simplest chatbot that can chat only
+ """
+ name: str = Field(description="name of the chatbot")
+ description: str = Field(description="description of the chatbot")
+ persona: str = Field(description="persona of the chatbot")
+ instruction: str = Field(description="instruction of the chatbot")
+ llm_api: str = Field(default="", description="llm api of the chatbot")
+ history_turns: int = Field(default=20, description="history turns of thread max turns")
+
+ ArtifactType: ClassVar = None
+ ContextType: ClassVar = None
+ DriverType: ClassVar = None
+
+ def __identifier__(self) -> Identifier:
+ return Identifier(
+ id=None,
+ name=self.name,
+ description=self.description,
+ )
+
+
+class ChatbotDriver(GhostDriver[Chatbot]):
+
+ def get_artifact(self, session: Session) -> None:
+ return None
+
+ def get_instructions(self, session: Session) -> str:
+ return self.get_system_prompter().get_prompt(session.container)
+
+ def actions(self, session: Session) -> List[LLMFunc]:
+ return []
+
+ def providers(self) -> Iterable[Provider]:
+ return []
+
+ def parse_event(self, session: Session, event: Event) -> Union[Event, None]:
+ return event
+
+ def get_system_prompter(self) -> Prompter:
+ return TextPrmt().with_children(
+ TextPrmt(title="Persona", content=self.ghost.persona),
+ TextPrmt(title="Instruction", content=self.ghost.instruction),
+ )
+
+ def on_event(self, session: Session, event: Event) -> Union[Operator, None]:
+ method = getattr(self, f"on_{event.type}", None)
+ if method is not None:
+ return method(session, event)
+ return self.default_handle_event(session, event)
+
+ def on_creating(self, session: Session) -> None:
+ return
+
+ def thought(self, session: Session) -> Thought:
+ thought = LLMThought(llm_api=self.ghost.llm_api)
+ return thought
+
+ def prompt(self, session: Session) -> Prompt:
+ system_prompter = self.get_system_prompter()
+ system_message = Role.SYSTEM.new(content=system_prompter.get_prompt(session.container))
+ prompt = session.thread.to_prompt([system_message])
+ return prompt
+
+ def truncate(self, session: Session) -> GoThreadInfo:
+ thread = session.thread
+ if 0 < self.ghost.history_turns < len(thread.history):
+ thread.history[-self.ghost.history_turns].summary = ""
+ elif self.ghost.history_turns == 0:
+ thread.history[-1].summary = ""
+
+ thread.history = thread.history[-self.ghost.history_turns:]
+ return thread
+
+ def default_handle_event(self, session: Session, event: Event) -> Union[Operator, None]:
+ # update session thread
+ session.thread.new_turn(event)
+ # get thought
+ thought = self.thought(session)
+ # get prompt
+ prompt = self.prompt(session)
+
+ # take action
+ prompt, op = thought.think(session, prompt)
+ if op is not None:
+ return op
+ return session.taskflow().wait()
diff --git a/ghostos/ghosts/moss_agent/__init__.py b/ghostos/ghosts/moss_agent/__init__.py
new file mode 100644
index 00000000..50efa1e8
--- /dev/null
+++ b/ghostos/ghosts/moss_agent/__init__.py
@@ -0,0 +1,36 @@
+from ghostos.ghosts.moss_agent.agent import MossAgent, MossAgentDriver, MossAction
+
+
+def new_moss_agent(
+ modulename: str,
+ *,
+ name: str = None,
+ description: str = None,
+ persona: str = None,
+ instruction: str = None,
+ llm_api: str = "",
+) -> MossAgent:
+ if persona is None:
+ persona = f"""
+You are an Agent created from python module file.
+Your goal is helping user to:
+- understand the python code.
+- interact with the python code that provided to you.
+"""
+ if instruction is None:
+ instruction = """
+- you are kind, helpful agent.
+- you are master of python coding.
+"""
+ if name is None:
+ name = modulename
+ if description is None:
+ description = f"default moss agent built from python module `{modulename}`."
+ return MossAgent(
+ moss_module=modulename,
+ persona=persona,
+ instructions=instruction,
+ name=name,
+ description=description,
+ llm_api=llm_api,
+ )
diff --git a/ghostos/ghosts/moss_agent/agent.py b/ghostos/ghosts/moss_agent/agent.py
new file mode 100644
index 00000000..e8fb8b8f
--- /dev/null
+++ b/ghostos/ghosts/moss_agent/agent.py
@@ -0,0 +1,413 @@
+from typing import Union, Optional, Dict, List, Self, Iterable, Tuple, ClassVar
+from types import ModuleType
+
+from ghostos.identifier import Identifier
+from pydantic import BaseModel, Field
+
+from ghostos.helpers import import_from_path
+from ghostos.prompter import TextPrmt, Prompter
+from ghostos.abcd import GhostDriver, Operator, Agent, Session, StateValue, Action, Thought, Ghost
+from ghostos.core.runtime import Event, GoThreadInfo
+from ghostos.core.moss import MossCompiler, PyContext, MossRuntime
+from ghostos.entity import ModelEntity
+from ghostos.core.messages import FunctionCaller, Role
+from ghostos.core.llms import (
+ Prompt, PromptPipe, AssistantNamePipe, run_prompt_pipeline,
+ LLMFunc,
+)
+from .instructions import (
+ GHOSTOS_INTRODUCTION, MOSS_INTRODUCTION, AGENT_META_INTRODUCTION, MOSS_FUNCTION_DESC,
+ get_moss_context_prompter, get_agent_identity,
+)
+import json
+
+from ghostos.container import Provider
+
+__all__ = ['MossAgent', 'MossAgentDriver', 'MossAction']
+
+
+class MossAgent(ModelEntity, Agent):
+ """
+ Basic Agent that turn a python module into a conversational agent.
+ """
+
+ """ subclass of MossAgent could have a GoalType, default is None"""
+
+ moss_module: str = Field(description="Moss module name for the agent")
+ persona: str = Field(description="Persona for the agent, if not given, use global persona")
+ instructions: str = Field(description="The instruction that the agent should follow")
+
+ # optional configs
+ name: str = Field(default="", description="name of the agent")
+ description: str = Field(default="", description="description of the agent")
+ code: Optional[str] = Field(default=None, description="code override the module")
+ compile_module: Optional[str] = Field(None, description="Compile module name for the agent")
+ llm_api: str = Field(default="", description="name of the llm api, if none, use default one")
+ truncate_at_turns: int = Field(default=40, description="when history turns reach the point, truncate")
+ truncate_to_turns: int = Field(default=20, description="when truncate the history, left turns")
+
+ def __identifier__(self) -> Identifier:
+ name = self.name if self.name else self.moss_module
+ return Identifier(
+ id=self.moss_module,
+ name=name,
+ description=self.description,
+ )
+
+
+class MossAgentDriver(GhostDriver[MossAgent]):
+
+ def get_module(self) -> ModuleType:
+ m = import_from_path(self.ghost.moss_module)
+ return m
+
+ def providers(self) -> Iterable[Provider]:
+ """
+ ghost session level providers
+ """
+ from ghostos.ghosts.moss_agent.for_developer import __moss_agent_providers__
+ m = self.get_module()
+ fn = __moss_agent_providers__
+ if __moss_agent_providers__.__name__ in m.__dict__:
+ fn = m.__dict__[__moss_agent_providers__.__name__]
+ return fn(self.ghost)
+
+ def parse_event(self, session: Session, event: Event) -> Union[Event, None]:
+ """
+ parse the event before handle it. if return None, the event is ignored.
+ :param session:
+ :param event:
+ :return:
+ """
+ from ghostos.ghosts.moss_agent.for_developer import __moss_agent_parse_event__
+ fn = __moss_agent_parse_event__
+ if __moss_agent_parse_event__.__name__ in event.__dict__:
+ fn = event.__dict__[__moss_agent_parse_event__.__name__]
+ return fn(self.ghost, session, event)
+
+ def truncate(self, session: Session) -> GoThreadInfo:
+ from ghostos.ghosts.moss_agent.for_developer import __moss_agent_truncate__
+ fn = __moss_agent_truncate__
+ if __moss_agent_truncate__.__name__ in session.__dict__:
+ fn = session.__dict__[__moss_agent_truncate__.__name__]
+ return fn(self.ghost, session)
+
+ def get_artifact(self, session: Session) -> Optional[MossAgent.ArtifactType]:
+ from .for_meta_ai import __moss_agent_artifact__
+ m = self.get_module()
+ if __moss_agent_artifact__.__name__ not in m.__dict__:
+ return None
+ fn = getattr(m, __moss_agent_artifact__.__name__)
+ compiler = self._get_moss_compiler(session)
+ with compiler:
+ runtime = compiler.compile(self.ghost.moss_module)
+ with runtime:
+ moss = runtime.moss()
+ return fn(self.ghost, moss)
+
+ def get_instructions(self, session: Session) -> str:
+ compiler = self._get_moss_compiler(session)
+ with compiler:
+ rtm = compiler.compile(self.ghost.compile_module)
+ with rtm:
+ return self._get_instructions(session, rtm)
+
+ def actions(self, session: Session) -> List[Action]:
+ compiler = self._get_moss_compiler(session)
+ with compiler:
+ runtime = compiler.compile(self.ghost.moss_module)
+ actions = self.get_actions(session, runtime)
+ return list(actions)
+
+ def thought(self, session: Session, runtime: MossRuntime) -> Thought:
+ from .for_meta_ai import __moss_agent_thought__ as fn
+ compiled = runtime.module()
+ if fn.__name__ in compiled.__dict__:
+ fn = compiled.__dict__[fn.__name__]
+ return fn(self.ghost, runtime.moss(), *self.get_actions(session, runtime))
+
+ def get_actions(self, session: Session, runtime: MossRuntime) -> Iterable[Action]:
+ """
+ get moss agent's actions. default is moss action.
+ """
+ from .for_meta_ai import __moss_agent_actions__ as fn
+ compiled = runtime.module()
+ if fn.__name__ in compiled.__dict__:
+ fn = compiled.__dict__[fn.__name__]
+ yield from fn(self.ghost, runtime.moss())
+ # moss action at last
+ moss_action = MossAction(runtime)
+ yield moss_action
+
+ def on_creating(self, session: Session) -> None:
+ from ghostos.ghosts.moss_agent.for_developer import __moss_agent_creating__ as fn
+ m = self.get_module()
+ if fn.__name__ in m.__dict__:
+ fn = m.__dict__[fn.__name__]
+ fn(self.ghost, session)
+ return
+
+ def on_event(self, session: Session, event: Event) -> Union[Operator, None]:
+ compiler = self._get_moss_compiler(session)
+ with compiler:
+ rtm = compiler.compile(self.ghost.compile_module)
+ with rtm:
+ # prepare instructions.
+ op, ok = self._on_custom_event_handler(session, rtm, event)
+ if ok:
+ return op or session.taskflow().wait()
+
+ # prepare thread
+ thread = session.thread
+ thread.new_turn(event)
+
+ # prepare prompt
+ instructions = self._get_instructions(session, rtm)
+ prompt = thread.to_prompt([Role.SYSTEM.new(content=instructions)], truncate=True)
+ pipes = self._get_prompt_pipes(session, rtm)
+ prompt = run_prompt_pipeline(prompt, pipes)
+
+ # prepare actions
+ thought = self.thought(session, rtm)
+ prompt, op = thought.think(session, prompt)
+ if op is not None:
+ return op
+ return session.taskflow().wait()
+
+ def _on_custom_event_handler(
+ self,
+ session: Session,
+ runtime: MossRuntime,
+ event: Event,
+ ) -> Tuple[Optional[Event], bool]:
+ method_name = "__moss_agent_on_" + event.type + "__"
+ compiled = runtime.module()
+ if method_name in compiled.__dict__:
+ fn = compiled.__dict__[method_name]
+ return fn(self.ghost, session, runtime, event), True
+ return None, False
+
+ def _get_instructions(self, session: Session, runtime: MossRuntime) -> str:
+ """
+ generate moss agent's instruction
+ :param session:
+ :param runtime:
+ :return:
+ """
+ prompter = self._get_instruction_prompter(session, runtime)
+ instruction = prompter.get_prompt(session.container, depth=0)
+ return instruction
+
+ def _get_instruction_prompter(self, session: Session, runtime: MossRuntime) -> Prompter:
+ agent = self.ghost
+ return TextPrmt().with_children(
+ # system meta prompt
+ TextPrmt(
+ title="Meta Instruction",
+ content=AGENT_META_INTRODUCTION,
+ ).with_children(
+ TextPrmt(title="GhostOS", content=GHOSTOS_INTRODUCTION),
+ TextPrmt(title="MOSS", content=MOSS_INTRODUCTION),
+ # code context
+ get_moss_context_prompter("Code Context", runtime),
+ ),
+ # agent prompt
+ TextPrmt(
+ title="Agent Info",
+ content="The Agent info about who you are and what you are doing: ",
+ ).with_children(
+ get_agent_identity("Identity", agent.__identifier__()),
+ TextPrmt(title="Persona", content=self._get_agent_persona(session, runtime)),
+ TextPrmt(title="Instruction", content=self._get_agent_instruction(session, runtime)),
+ ),
+ TextPrmt(
+ title="Context",
+ content="",
+ ).with_children(
+ self._get_context_prompter(session),
+ )
+ )
+
+ def _get_agent_persona(self, session: Session, runtime: MossRuntime) -> str:
+ from .for_meta_ai import __moss_agent_persona__ as fn
+ compiled = runtime.module()
+ if fn.__name__ in compiled.__dict__:
+ fn = compiled.__dict__[fn.__name__]
+ return fn(self.ghost, runtime.moss())
+
+ def _get_agent_instruction(self, session: Session, runtime: MossRuntime) -> str:
+ from .for_meta_ai import __moss_agent_instruction__ as fn
+ compiled = runtime.module()
+ if fn.__name__ in compiled.__dict__:
+ fn = compiled.__dict__[fn.__name__]
+ return fn(self.ghost, runtime.moss())
+
+ def _get_context_prompter(self, session: Session) -> Optional[Prompter]:
+ ctx = session.get_context()
+ if ctx is None:
+ return None
+ return ctx
+
+ def _get_prompt_pipes(self, session: Session, runtime: MossRuntime) -> Iterable[PromptPipe]:
+ yield AssistantNamePipe(self.ghost.name)
+
+ def _get_moss_compiler(self, session: Session) -> MossCompiler:
+ from ghostos.ghosts.moss_agent.for_developer import __moss_agent_injections__
+ pycontext = self.get_pycontext(session)
+
+ compiler = session.container.force_fetch(MossCompiler)
+ container = compiler.container()
+
+ compiler = compiler.join_context(pycontext)
+ compiler = compiler.with_locals(Optional=Optional)
+
+ # register self
+ container.set(Ghost, self.ghost)
+
+ # bind moss agent itself
+ compiler.bind(type(self.ghost), self.ghost)
+
+ # bind agent level injections.
+ fn = __moss_agent_injections__
+ module = self.get_module()
+ # if magic function __agent_moss_injections__ exists, use it to get some instance level injections to moss.
+ if __moss_agent_injections__.__name__ in module.__dict__:
+ fn = module.__dict__[__moss_agent_injections__.__name__]
+ injections = fn(self.ghost, session)
+ if injections:
+ compiler = compiler.injects(**injections)
+ return compiler
+
+ def get_pycontext(self, session: Session) -> PyContext:
+ """
+ get moss pycontext. moss pycontext is bind to session.state as default.
+ :param session:
+ :return:
+ """
+ pycontext = SessionPyContext(
+ module=self.ghost.moss_module,
+ code=self.ghost.code,
+ )
+ return pycontext.get_or_bind(session)
+
+
+class SessionPyContext(PyContext, StateValue):
+ """
+ bind pycontext to session.state
+ """
+
+ def get(self, session: Session) -> Optional[Self]:
+ data = session.state.get(SessionPyContext.__name__, None)
+ if data is not None:
+ if isinstance(data, Dict):
+ return SessionPyContext(**data)
+ elif isinstance(data, SessionPyContext):
+ return data
+ return None
+
+ def bind(self, session: Session) -> None:
+ session.state[SessionPyContext.__name__] = self
+
+
+class MossAction(Action, PromptPipe):
+ DEFAULT_NAME: ClassVar[str] = "moss"
+
+ class Argument(BaseModel):
+ code: str = Field(description="the python code you want to execute. never quote them with ```")
+
+ def __init__(self, runtime: MossRuntime, name: str = DEFAULT_NAME):
+ self.runtime: MossRuntime = runtime
+ self._name = name
+
+ def name(self) -> str:
+ return self._name
+
+ def as_function(self) -> Optional[LLMFunc]:
+ parameters = self.Argument.model_json_schema()
+ llm_func = LLMFunc(
+ name=self.name(),
+ description=MOSS_FUNCTION_DESC,
+ parameters=parameters,
+ )
+ return llm_func
+
+ def update_prompt(self, prompt: Prompt) -> Prompt:
+ llm_func = self.as_function()
+ if llm_func is not None:
+ prompt.functions.append(llm_func)
+ return prompt
+
+ @classmethod
+ def unmarshal_arguments(cls, arguments: str) -> str:
+ try:
+ if arguments.startswith("{"):
+ if not arguments.endswith("}"):
+ arguments += "}"
+ data = json.loads(arguments)
+ args = cls.Argument(**data)
+ else:
+ args = cls.Argument(code=arguments)
+ except Exception:
+ args = cls.Argument(code=arguments)
+ code = args.code.strip()
+ if code.startswith("```python"):
+ code.lstrip("```python")
+ if code.startswith("```"):
+ code.lstrip("```")
+ if code.endswith("```"):
+ code.rstrip("```")
+ return code.strip()
+
+ def run(self, session: Session, caller: FunctionCaller) -> Union[Operator, None]:
+ session.logger.debug("MossAction receive caller: %s", caller)
+ # prepare arguments.
+ arguments = caller.arguments
+ code = self.unmarshal_arguments(arguments)
+ if code.startswith("{") and code.endswith("}"):
+ # unmarshal again.
+ code = self.unmarshal_arguments(code)
+
+ # if code is not exists, inform the llm
+ if not code:
+ return self.fire_error(session, caller, "the moss code is empty")
+ session.logger.debug("moss action code: %s", code)
+
+ error = self.runtime.lint_exec_code(code)
+ if error:
+ return self.fire_error(session, caller, f"the moss code has syntax errors:\n{error}")
+
+ moss = self.runtime.moss()
+ try:
+ result = self.runtime.execute(target="run", code=code, args=[moss])
+ op = result.returns
+ if op is not None and not isinstance(op, Operator):
+ return self.fire_error(session, caller, "result of moss code is not None or Operator")
+ pycontext = result.pycontext
+ # rebind pycontext to bind session
+ pycontext = SessionPyContext(**pycontext.model_dump(exclude_defaults=True))
+ pycontext.bind(session)
+
+ # handle std output
+ std_output = result.std_output
+ session.logger.debug("moss action std_output: %s", std_output)
+ if std_output:
+ output = f"Moss output:\n{std_output}"
+ message = caller.new_output(output)
+ session.respond([message])
+ if op is None:
+ # if std output is not empty, and op is none, observe the output as default.
+ return session.taskflow().think()
+ else:
+ output = caller.new_output("executed")
+ session.respond([output])
+ return None
+
+ except Exception as e:
+ session.logger.exception(e)
+ return self.fire_error(session, caller, f"error during executing moss code: {e}")
+
+ @staticmethod
+ def fire_error(session: Session, caller: FunctionCaller, error: str) -> Operator:
+ message = caller.new_output(error)
+ session.respond([message])
+ return session.taskflow().error()
diff --git a/ghostos/ghosts/moss_agent/for_developer.py b/ghostos/ghosts/moss_agent/for_developer.py
new file mode 100644
index 00000000..67807aee
--- /dev/null
+++ b/ghostos/ghosts/moss_agent/for_developer.py
@@ -0,0 +1,112 @@
+from typing import Optional, TypeVar, Dict, Any, Iterable
+from .agent import MossAgent
+from ghostos.core.moss import MossRuntime
+from ghostos.abcd.concepts import Session, Operator
+from ghostos.core.runtime import GoThreadInfo, Event
+from ghostos.container import Provider
+
+A = TypeVar("A")
+
+
+#
+# lifecycle methods for developer.
+# agent need not know details about these methods, but also ok.
+# hide these methods because too much ghostos designing patterns are needed.
+
+def __moss_agent_providers__(agent: A) -> Iterable[Provider]:
+ """
+ return conversation level providers that specially required by the Agent.
+ the conversation container will automatically register the providers and run them.
+
+ :param agent: the moss agent instance.
+ :return: providers that register to the session container.
+ """
+ return []
+
+
+def __shell_providers__() -> Iterable[Provider]:
+ """
+ return shell level providers that specially required by the Agent.
+ if the shell is running by `ghostos web` or `ghostos console`,
+ the script will detect the __shell_providers__ attribute and register them into shell level container.
+
+ You can consider the Shell is the body of an agent.
+ So shell level providers usually register the body parts singletons, bootstrap them and register shutdown functions.
+ """
+ return []
+
+
+def __moss_agent_creating__(agent: A, session: Session) -> None:
+ """
+ once a moss agent is creating,
+ this function will be called.
+ you can do something here to initialize Thread, Pycontext or Other things.
+ """
+ pass
+
+
+def __moss_agent_truncate__(agent: MossAgent, session: Session) -> GoThreadInfo:
+ """
+ default history messages truncate logic of the agent
+ :param agent:
+ :param session:
+ :return:
+ """
+ from ghostos.abcd.thoughts import SummaryThought
+ from ghostos.core.llms import Prompt
+
+ thread = session.thread
+ turns = thread.get_history_turns(True)
+ # do the truncate
+ if len(turns) > agent.truncate_at_turns:
+ # the history turns to remove
+ truncated = agent.truncate_at_turns - agent.truncate_to_turns
+ if truncated <= 0:
+ return thread
+ turns = turns[:truncated]
+ # last turn of the truncated turns
+ if len(turns) < 1:
+ return thread
+ target = turns[-1]
+ messages = []
+ for turn in turns:
+ messages.extend(turn.messages(False))
+ prompt = Prompt(history=messages)
+ _, summary = SummaryThought(llm_api=agent.llm_api).think(session, prompt)
+ if summary:
+ target.summary = summary
+ return session.thread
+
+
+def __moss_agent_parse_event__(agent: MossAgent, session: Session, event: Event) -> Optional[Event]:
+ """
+ when moss agent receive an event, will check it first before handle it.
+ for example, if the user input is way too big, agent can ignore or reject the event here.
+ :return: if None, the event will be ignored.
+ """
+ return event
+
+
+def __moss_agent_injections__(agent: A, session: Session[A]) -> Dict[str, Any]:
+ """
+ manually define some of the injections to the Moss Class.
+ if a property of Moss is not injected here, the session container will inject it by typehint.
+ """
+ return {
+ }
+
+
+def __moss_agent_on_event_type__(
+ agent: MossAgent,
+ session: Session[A],
+ runtime: MossRuntime,
+ event: Event,
+) -> Optional[Operator]:
+ """
+ define customized event handler for each event type.
+ the method name is the pattern `__moss_agent_on_[event_type]__`
+ you may create any event type handler, instead of default logic in MossAgentDriver.
+ """
+ pass
+
+#
diff --git a/ghostos/ghosts/moss_agent/for_meta_ai.py b/ghostos/ghosts/moss_agent/for_meta_ai.py
new file mode 100644
index 00000000..adba5d44
--- /dev/null
+++ b/ghostos/ghosts/moss_agent/for_meta_ai.py
@@ -0,0 +1,67 @@
+from typing import TypeVar, Iterable
+from .agent import MossAgent
+from ghostos.core.moss import Moss
+from ghostos.abcd import Action, Thought, LLMThought
+
+A = TypeVar("A")
+
+
+# --- lifecycle methods for meta agent --- #
+
+
+def __moss_agent_artifact__(agent: MossAgent, moss: Moss):
+ """
+ get the agent artifact, default is None
+
+ Artifact is what an Agent is producing. Like return value of a function.
+ But I want the production of an agent is altering during the runtime,
+ So I can see what going on, and intercept an automatic agent if necessary.
+ """
+ return None
+
+
+def __moss_agent_actions__(agent: MossAgent, moss: Moss) -> Iterable[Action]:
+ """
+ define the actions that agent equipped.
+ but I prefer let the agent do everything with code,
+ So the Only needed Action is MossAction.
+ """
+ yield from []
+
+
+def __moss_agent_persona__(agent: MossAgent, moss: Moss) -> str:
+ """
+ return the persona of the agent, which is showed in agent's system instruction.
+ consider this, if moss.persona is a defined str attr of Moss, the Agent can modify it by MossAction.
+ so the moss agent can easily modify itself persona.
+ """
+ return agent.persona
+
+
+def __moss_agent_instruction__(agent: MossAgent, moss: Moss) -> str:
+ """
+ return the instruction string of the agent.
+ """
+ return agent.instructions
+
+
+def __moss_agent_thought__(agent: MossAgent, moss: Moss, *actions: Action) -> Thought:
+ """
+ GhostOS separate single turn `Thought` and Multi-turns `Agent`.
+ So the Agent (or Ghost) can conversation in multiple turns with the GoThreadInfo which saved history.
+ But in a single turn, the GoThreadInfo will convert to LLM Prompt instance, and run by Thought.
+
+ We can combine multiple Thought to a pipeline, graph or parallel tree to implements Single-turn Chain of Thought.
+
+ take moderation for example:
+ 1. first thought: quick thought to judge the user input is legal to respond.
+ 2. second thought: think about what to reply
+ 3. end thought: reflect the reply, if illegal, re-think it.
+
+ we can easily build this pipeline by code.
+ """
+ return LLMThought(
+ llm_api=agent.llm_api,
+ actions=actions,
+ message_stage="",
+ )
diff --git a/ghostos/ghosts/moss_agent/instructions.py b/ghostos/ghosts/moss_agent/instructions.py
new file mode 100644
index 00000000..85884b56
--- /dev/null
+++ b/ghostos/ghosts/moss_agent/instructions.py
@@ -0,0 +1,104 @@
+from ghostos.core.moss import MossRuntime
+from ghostos.prompter import Prompter, TextPrmt
+from ghostos.identifier import Identifier
+
+AGENT_META_INTRODUCTION = """
+You are the mind of an AI Agent driven by `GhostOS` framework.
+Here are some basic information you might expect:
+"""
+
+GHOSTOS_INTRODUCTION = """
+`GhostOS` is an AI Agent framework written in Python,
+providing llm connections, body shell, tools, memory etc and specially the `MOSS` for you.
+"""
+
+MOSS_INTRODUCTION = """
+You are equipped with the MOSS (Model-oriented Operating System Simulator).
+Which provides you a way to control your body / tools / thoughts through Python code.
+
+basic usage:
+1. you will get the python code context that MOSS provide to you below.
+2. you can generate code by `moss` tool, then the `GhostOS` will execute them for you.
+3. if you print anything in your generated code, the output will be shown in further messages.
+
+"""
+
+MOSS_CONTEXT_TEMPLATE = """
+The python context `{modulename}` that MOSS provides to you are below:
+
+```python
+{code_context}
+```
+
+Notices:
+* the imported functions are only shown with signature, the source code is omitted.
+* the properties on moss instance, will keep existence.
+* You can bind variables of type int/float/bool/str/list/dict/BaseModel to moss instance if you need them for next turn.
+
+You are able to call the `moss` tool, generate code to fulfill your will.
+the python code you generated, must include a `run` function, follow the pattern:
+
+```python
+def run(moss: Moss):
+ \"""
+ :param moss: instance of the class `Moss`, the properties on it will be injected with runtime implementations.
+ :return: Optional[Operator]
+ if return None, the outer system will perform default action, or observe the values you printed.
+ Otherwise, the outer system will execute the operator.
+ You shall only return operator by the libraries provided on `moss`.
+ \"""
+```
+
+Then the `GhostOS` system will add your code to the python module provided to you,
+and execute the `run` function.
+
+Notices:
+* Your code will **APPEND** to the code of `{modulename}` then execute, so **DO NOT REPEAT THE DEFINED CODE IN THE MODULE**.
+* if the python code context can not fulfill your will, do not use the `moss` tool.
+* you can reply as usual without calling the tool `moss`. use it only when you know what you're doing.
+* don't copy the main function's __doc__, they are instruction to you only.
+* in your code generation, comments is not required, comment only when necessary.
+"""
+
+MOSS_FUNCTION_DESC = """
+useful to call MOSS system to execute the code. The code must include a `run` function.
+"""
+
+
+def get_moss_context_prompter(title: str, runtime: MossRuntime) -> Prompter:
+ code_context = runtime.prompter().dump_module_prompt()
+
+ injections = runtime.moss_injections()
+ children = []
+ container = runtime.container()
+
+ for name, injection in injections.items():
+ if isinstance(injection, Prompter):
+ prompter = TextPrmt(
+ title=f"property moss.{name}",
+ content=injection.self_prompt(container),
+ )
+ children.append(prompter)
+
+ content = MOSS_CONTEXT_TEMPLATE.format(
+ modulename=runtime.module().__name__,
+ code_context=code_context,
+ )
+
+ return TextPrmt(
+ title=title,
+ content=content,
+ ).with_children(*children)
+
+
+def get_agent_identity(title: str, id_: Identifier) -> Prompter:
+ from ghostos.helpers import yaml_pretty_dump
+ value = id_.model_dump(exclude_defaults=True)
+ return TextPrmt(
+ title=title,
+ content=f"""
+```yaml
+{yaml_pretty_dump(value)}
+```
+"""
+ )
diff --git a/ghostos/demo/src/examples/__init__.py b/ghostos/ghosts/moss_agent/template.py
similarity index 100%
rename from ghostos/demo/src/examples/__init__.py
rename to ghostos/ghosts/moss_agent/template.py
diff --git a/ghostos/helpers/__init__.py b/ghostos/helpers/__init__.py
index 5b7789e0..66be41f3 100644
--- a/ghostos/helpers/__init__.py
+++ b/ghostos/helpers/__init__.py
@@ -1,34 +1,44 @@
+from typing import TYPE_CHECKING
from ghostos.helpers.dictionary import (dict_without_none, dict_without_zero)
from ghostos.helpers.string import camel_to_snake
from ghostos.helpers.yaml import yaml_pretty_dump, yaml_multiline_string_pipe
from ghostos.helpers.modules import (
import_from_path,
- parse_import_module_and_spec,
+ import_class_from_path,
+ parse_import_path_module_and_attr_name,
join_import_module_and_spec,
- get_module_spec,
- generate_module_spec,
+ get_module_attr,
+ generate_module_and_attr_name,
generate_import_path,
Importer,
is_method_belongs_to_class,
get_calling_modulename,
rewrite_module,
rewrite_module_by_path,
+ create_module,
+ create_and_bind_module,
)
from ghostos.helpers.io import BufferPrint
-from ghostos.helpers.time import Timeleft
-from ghostos.helpers.hashes import md5
+from ghostos.helpers.timeutils import Timeleft, timestamp_datetime, timestamp
+from ghostos.helpers.hashes import md5, sha1, sha256
+from ghostos.helpers.trans import gettext, ngettext, get_current_locale, GHOSTOS_DOMAIN
-from typing import Callable
+from ghostos.helpers.coding import reflect_module_code, unwrap
+from ghostos.helpers.openai import get_openai_key
+from ghostos.helpers.tree_sitter import tree_sitter_parse, code_syntax_check
+
+if TYPE_CHECKING:
+ from typing import Callable
# --- private methods --- #
def __uuid() -> str:
from uuid import uuid4
- return str(uuid4())
+ # keep uuid in 32 chars
+ return md5(str(uuid4()))
# --- facade --- #
-uuid: Callable[[], str] = __uuid
+uuid: "Callable[[], str]" = __uuid
""" patch this method to change global uuid generator"""
-
diff --git a/ghostos/helpers/coding.py b/ghostos/helpers/coding.py
new file mode 100644
index 00000000..54c22a25
--- /dev/null
+++ b/ghostos/helpers/coding.py
@@ -0,0 +1,15 @@
+from typing import TypeVar, Union, Callable
+from types import ModuleType
+
+T = TypeVar("T")
+
+
+def unwrap(value: Union[T, Callable[[], T]]) -> T:
+ if isinstance(value, Callable):
+ return value()
+ return value
+
+
+def reflect_module_code(module: ModuleType) -> str:
+ with open(module.__file__) as f:
+ return f.read()
diff --git a/ghostos/helpers/hashes.py b/ghostos/helpers/hashes.py
index 11f5adab..c83d4365 100644
--- a/ghostos/helpers/hashes.py
+++ b/ghostos/helpers/hashes.py
@@ -9,3 +9,17 @@ def md5(input_string: str) -> str:
# 获取16进制的哈希值
hash_value = md5_obj.hexdigest()
return hash_value
+
+
+def sha1(input_string: str) -> str:
+ sha1_obj = hashlib.sha1()
+ sha1_obj.update(input_string.encode('utf-8'))
+ hash_value = sha1_obj.hexdigest()
+ return hash_value
+
+
+def sha256(input_string: str) -> str:
+ sha256_obj = hashlib.sha256()
+ sha256_obj.update(input_string.encode('utf-8'))
+ hash_value = sha256_obj.hexdigest()
+ return hash_value
diff --git a/ghostos/helpers/modules.py b/ghostos/helpers/modules.py
index 54350cc6..359bce40 100644
--- a/ghostos/helpers/modules.py
+++ b/ghostos/helpers/modules.py
@@ -1,23 +1,35 @@
import inspect
-from typing import Any, Tuple, Optional, Dict, Callable, Type
+from typing import Any, Tuple, Optional, Dict, Callable, Type, TypeVar
from types import ModuleType
__all__ = [
'Importer',
'import_from_path',
+ 'import_class_from_path',
'get_calling_modulename',
- 'get_module_spec',
+ 'get_module_attr',
'generate_import_path',
- 'generate_module_spec',
+ 'generate_module_and_attr_name',
'join_import_module_and_spec',
'is_method_belongs_to_class',
- 'parse_import_module_and_spec',
+ 'parse_import_path_module_and_attr_name',
'rewrite_module',
'rewrite_module_by_path',
+ 'create_module',
+ 'create_and_bind_module',
]
Importer = Callable[[str], ModuleType]
+T = TypeVar('T', bound=type)
+
+
+def import_class_from_path(path: str, parent: Optional[T]) -> T:
+ imported = import_from_path(path)
+ if parent and not issubclass(imported, parent):
+ raise TypeError(f'{path} is not a subclass of {parent}')
+ return imported
+
def import_from_path(module_spec: str, importer: Optional[Importer] = None) -> Any:
if importer is None:
@@ -28,18 +40,18 @@ def import_from_path(module_spec: str, importer: Optional[Importer] = None) -> A
spec = parts[1] if len(parts) > 1 else None
imported_module = importer(module)
if spec:
- return get_module_spec(imported_module.__dict__, spec)
+ return get_module_attr(imported_module.__dict__, spec)
return imported_module
-def get_module_spec(module, spec: str) -> Optional[Any]:
- parts = spec.split('.')
+def get_module_attr(module, attr_name: str) -> Optional[Any]:
+ parts = attr_name.split('.')
value = module
for part in parts:
if value is None:
- raise AttributeError(f'Module has no attribute {spec}')
+ raise AttributeError(f'Module has no attribute {attr_name}')
if part == "":
- raise AttributeError(f'local attribute {spec} is not support yet')
+ raise AttributeError(f'local attribute {attr_name} is not support yet')
if isinstance(value, Dict):
value = value.get(part)
else:
@@ -47,18 +59,20 @@ def get_module_spec(module, spec: str) -> Optional[Any]:
return value
-def generate_module_spec(value: Any) -> Tuple[str, Optional[str]]:
+def generate_module_and_attr_name(value: Any) -> Tuple[str, Optional[str]]:
if inspect.ismodule(value):
return value.__name__, None
- elif inspect.isclass(value):
- module = getattr(value, '__module__', '')
- spec = getattr(value, '__qualname__', getattr(value, '__name__', ""))
- return module, spec
+ elif inspect.isclass(value) or inspect.isfunction(value):
+ modulename = getattr(value, '__module__', '')
+ if modulename.endswith(".__init__"):
+ modulename = modulename[:-len(".__init__")]
+ attr_name = getattr(value, '__qualname__', getattr(value, '__name__', ""))
+ return modulename, attr_name
else:
raise AttributeError(f'value {value} should be module or class to generate module spec')
-def parse_import_module_and_spec(import_path: str) -> Tuple[str, Optional[str]]:
+def parse_import_path_module_and_attr_name(import_path: str) -> Tuple[str, Optional[str]]:
"""
parse import_path to modulename and spec
:param import_path: pattern is `module:spec`
@@ -70,8 +84,28 @@ def parse_import_module_and_spec(import_path: str) -> Tuple[str, Optional[str]]:
return parts[0], parts[1]
+def create_module(module_name: str, file_path: str):
+ from importlib import util
+
+ # 加载模块
+ spec = util.spec_from_file_location(module_name, file_path)
+ module = util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+
+ return module
+
+
+def create_and_bind_module(modulename: str, filename: str, force: bool = False):
+ from sys import modules
+ if not force and modulename in modules:
+ return modules[modulename]
+ module = create_module(modulename, filename)
+ modules[modulename] = module
+ return module
+
+
def generate_import_path(value: Any) -> str:
- module, spec = generate_module_spec(value)
+ module, spec = generate_module_and_attr_name(value)
return join_import_module_and_spec(module, spec)
diff --git a/ghostos/helpers/openai.py b/ghostos/helpers/openai.py
new file mode 100644
index 00000000..d5b6eaf6
--- /dev/null
+++ b/ghostos/helpers/openai.py
@@ -0,0 +1,3 @@
+def get_openai_key() -> str:
+ import os
+ return os.environ.get("OPENAI_API_KEY", "")
diff --git a/ghostos/helpers/time.py b/ghostos/helpers/time.py
deleted file mode 100644
index c4c00395..00000000
--- a/ghostos/helpers/time.py
+++ /dev/null
@@ -1,18 +0,0 @@
-from typing import Tuple
-import time
-
-__all__ = ['Timeleft']
-
-
-class Timeleft:
-
- def __init__(self, timeout: float):
- self.timeout = timeout
- self.start = time.time()
-
- def left(self) -> float:
- if self.timeout <= 0.0:
- return 0.0
- now = time.time()
- timeleft = self.timeout - (now - self.start)
- return timeleft
diff --git a/ghostos/helpers/timeutils.py b/ghostos/helpers/timeutils.py
new file mode 100644
index 00000000..8b2fc0d1
--- /dev/null
+++ b/ghostos/helpers/timeutils.py
@@ -0,0 +1,31 @@
+from datetime import datetime
+import time
+
+__all__ = ['Timeleft', 'timestamp_datetime', 'timestamp']
+
+
+class Timeleft:
+
+ def __init__(self, timeout: float):
+ self.timeout = timeout
+ self.start = time.time()
+
+ def left(self) -> float:
+ passed = self.passed()
+ timeleft = self.timeout - passed
+ return timeleft if timeleft > 0 else 0
+
+ def alive(self) -> bool:
+ return self.timeout <= 0 or self.passed() < self.timeout
+
+ def passed(self) -> float:
+ now = time.time()
+ return round(now - self.start, 4)
+
+
+def timestamp_datetime() -> datetime:
+ return datetime.fromtimestamp(int(time.time()))
+
+
+def timestamp() -> int:
+ return int(time.time())
diff --git a/ghostos/helpers/trans.py b/ghostos/helpers/trans.py
new file mode 100644
index 00000000..a7ff3f11
--- /dev/null
+++ b/ghostos/helpers/trans.py
@@ -0,0 +1,22 @@
+from gettext import dgettext, dngettext
+
+__all__ = ['GHOSTOS_DOMAIN', 'gettext', 'ngettext', 'get_current_locale']
+
+GHOSTOS_DOMAIN = 'ghostos'
+
+
+def gettext(message):
+ return dgettext(GHOSTOS_DOMAIN, message)
+
+
+def ngettext(singular, plural, n):
+ return dngettext(GHOSTOS_DOMAIN, singular, plural, n)
+
+
+def get_current_locale() -> str:
+ from babel import default_locale
+ return default_locale()
+
+
+if __name__ == '__main__':
+ print(get_current_locale())
\ No newline at end of file
diff --git a/ghostos/helpers/tree_sitter.py b/ghostos/helpers/tree_sitter.py
index 11e182b8..30e492c6 100644
--- a/ghostos/helpers/tree_sitter.py
+++ b/ghostos/helpers/tree_sitter.py
@@ -1,4 +1,4 @@
-from typing import Optional, Iterable, List, Set, Dict, Type, ClassVar
+from typing import Optional, Iterable, List, Set, Dict, Type, ClassVar, Generator
from abc import ABC, abstractmethod
from tree_sitter_languages import get_parser
from tree_sitter import (
@@ -6,11 +6,66 @@
)
from enum import Enum
-_PythonParser = get_parser('python')
+_PythonParser = None
+__all__ = ['tree_sitter_parse', 'code_syntax_check']
-def parse(code: str) -> Tree:
- return _PythonParser.parse(code)
+
+def tree_sitter_parse(code: str) -> Tree:
+ global _PythonParser
+ if _PythonParser is None:
+ _PythonParser = get_parser('python')
+ return _PythonParser.parse(code.encode())
+
+
+def code_syntax_check(code: str) -> Optional[str]:
+ try:
+ tree = tree_sitter_parse(code)
+ except Exception as e:
+ return f"parse code failed: {e}"
+
+ errors = []
+ travel_node_error(code, tree.root_node, errors)
+ if errors:
+ return "- " + "\n- ".join(errors)
+ return None
+
+
+def traverse_tree(tree: Tree) -> Generator[TreeSitterNode, None, None]:
+ cursor = tree.walk()
+
+ visited_children = False
+ while True:
+ if not visited_children:
+ yield cursor.node
+ if not cursor.goto_first_child():
+ visited_children = True
+ elif cursor.goto_next_sibling():
+ visited_children = False
+ elif not cursor.goto_parent():
+ break
+
+
+def travel_node_error(code: str, node: TreeSitterNode, errors: List[str]) -> None:
+ error = get_node_error(code, node)
+ if error is not None:
+ errors.append(error)
+ return
+ for child in node.children:
+ travel_node_error(code, child, errors)
+
+
+def get_node_error(code: str, node: TreeSitterNode) -> Optional[str]:
+ """
+ get all the errors when traversing a node
+ """
+ if node.is_error:
+ start_point_row, col = node.start_point
+ line_number = start_point_row + 1
+ line_content = code.splitlines()[line_number - 1]
+ # 这里假设错误分析就是节点的类型和文本内容
+ return f"Syntax Error at line {line_number}: `{line_content}`"
+ return None
def get_error_nodes(node: TreeSitterNode) -> Iterable[TreeSitterNode]:
diff --git a/ghostos/identifier.py b/ghostos/identifier.py
new file mode 100644
index 00000000..9432f5d9
--- /dev/null
+++ b/ghostos/identifier.py
@@ -0,0 +1,167 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Optional, Dict, Union, Callable, Any
+from pydantic import BaseModel, Field
+from ghostos.helpers import generate_import_path
+from typing_extensions import Protocol
+import inspect
+
+__all__ = [
+ 'get_identifier', 'try_get_identifier',
+ 'identify_class',
+ 'identify_class_id',
+
+ 'Identifier', 'Identifiable',
+ 'Identical', 'IdenticalClass',
+ 'IdenticalObject',
+
+]
+
+
+def get_identifier(value: Any) -> Identifier:
+ """
+ get identifier or not from any value
+ """
+ id_ = try_get_identifier(value)
+ if id_ is None:
+ raise AttributeError(f'{value} is not an identifier')
+ return id_
+
+
+def try_get_identifier(value: Any, throw: bool = False) -> Union[Identifier, None]:
+ try:
+ if value is None:
+ return None
+ # identifier it self
+ if isinstance(value, Identifier):
+ return value
+ # explicit identifiable object
+ elif isinstance(value, Identical):
+ return value.__identifier__()
+ # explicit identifiable class
+ elif issubclass(value, IdenticalClass):
+ return value.class_identifier()
+ # function is always identifiable
+ elif inspect.isfunction(value):
+ return Identifier(
+ id=generate_import_path(value),
+ name=value.__name__,
+ description=value.__doc__,
+ )
+ # method just like function
+ elif inspect.ismethod(value):
+ return Identifier(
+ id=generate_import_path(value.__class__) + ":" + value.__name__,
+ name=value.__name__,
+ description=value.__doc__,
+ )
+ # class is special at runtime.
+ # notice the id of a class is alternative at different project due to python import by relative path.
+ elif isinstance(value, type):
+ return identify_class(value)
+ # a dict
+ elif isinstance(value, Dict) and "name" in value and "description" in value:
+ return Identifier(
+ id=value.get("id", None),
+ name=value["name"],
+ description=value["description"],
+ )
+ elif hasattr(value, "__identifier__"):
+ identifier = getattr(value, "identifier")
+ if isinstance(identifier, Identifier):
+ return identifier
+ elif isinstance(identifier, Callable):
+ return identifier()
+
+ elif hasattr(value, 'name') and hasattr(value, 'description'):
+ return Identifier(
+ id=getattr(value, 'id', None),
+ name=getattr(value, 'name'),
+ description=getattr(value, 'description'),
+ )
+ return None
+ except Exception:
+ if throw:
+ raise
+ return None
+
+
+class Identifier(BaseModel):
+ """
+ a simplest model identify an object
+ """
+ id: Optional[str] = Field(default=None, description="Unique id")
+ name: str = Field(
+ default="",
+ description="Name of the object, name has it meaning only for the subject who named it",
+ )
+ description: str = Field(default="", description="Description of the object")
+
+ def match_keyword(self, keyword: str) -> bool:
+ keyword = keyword.strip()
+ if not keyword:
+ return True
+ return (
+ keyword.lower() in self.name.lower()
+ or keyword.lower() in self.description.lower()
+ or keyword.lower() in self.id.lower()
+ )
+
+
+class Identical(ABC):
+ """
+ abstract class that identifiable class can extend it.
+ """
+
+ @abstractmethod
+ def __identifier__(self) -> Identifier:
+ pass
+
+
+class IdenticalObject(Protocol):
+ """
+ less invasive way to describe an identifiable object.
+ when we need to hide the complexity of a class to someone, especially the AI model,
+ we need to use protocol to do implicit implementation sometimes. duck type is useful.
+ """
+
+ @abstractmethod
+ def __identifier__(self) -> Identifier:
+ pass
+
+
+class IdenticalClass(ABC):
+ """
+ class is identifiable, but sometimes we need to specific the identifier.
+ """
+
+ @classmethod
+ @abstractmethod
+ def class_identifier(cls) -> Identifier:
+ pass
+
+
+Identifiable = Union[Identical, IdenticalObject]
+
+
+def identify_class(cls: type) -> Identifier:
+ """
+ 一个默认的用来描述类的方法.
+ :param cls: 目标类.
+ :return: 返回一个 identifier.
+ """
+ if issubclass(cls, IdenticalClass):
+ return cls.class_identifier()
+ id_ = identify_class_id(cls)
+ name = cls.__name__
+ desc = cls.__doc__
+ return Identifier(
+ id=id_,
+ name=name,
+ description=desc or "",
+ )
+
+
+def identify_class_id(cls: type) -> str:
+ return generate_import_path(cls)
diff --git a/ghostos/libraries/.dir_index.yml b/ghostos/libraries/.dir_index.yml
deleted file mode 100644
index ef67b4bd..00000000
--- a/ghostos/libraries/.dir_index.yml
+++ /dev/null
@@ -1,4 +0,0 @@
-files:
- file_editor.py:
- summary: defines FileEditor and DirectoryEditor abstract class with implementations
- filename: file_editor.py
diff --git a/ghostos/libraries/rag/llamaindex.py b/ghostos/libraries/rag/llamaindex.py
deleted file mode 100644
index 1ea84f9f..00000000
--- a/ghostos/libraries/rag/llamaindex.py
+++ /dev/null
@@ -1,10 +0,0 @@
-
-from llama_index.core import VectorStoreIndex, SimpleDirectoryReader
-
-documents = SimpleDirectoryReader("YOUR_DATA_DIRECTORY").load_data()
-index = VectorStoreIndex.from_documents(documents)
-index.update()
-index.insert()
-engine = index.as_query_engine()
-engine.query()
-index.storage_context.persist()
\ No newline at end of file
diff --git a/ghostos/mocks/libraries/auto_text_memory.py b/ghostos/mocks/libraries/auto_text_memory.py
deleted file mode 100644
index 305f4598..00000000
--- a/ghostos/mocks/libraries/auto_text_memory.py
+++ /dev/null
@@ -1,48 +0,0 @@
-from typing import Dict
-
-from mem0 import Memory
-from mem0.configs.base import MemoryConfig, LlmConfig, VectorStoreConfig
-from ghostos.framework.libraries.auto_memory import TextMemory, ProxyConfig, VectorDBConfig, DBConfig
-
-
-class Mem0TextMemory(TextMemory):
-
- def __init__(self, proxy_config: ProxyConfig = None, llm_config: Dict = None, vector_config: VectorDBConfig = None, db_config: DBConfig = None):
- super().__init__(proxy_config)
-
- conf = MemoryConfig()
- if llm_config:
- conf.llm = LlmConfig(provider=llm_config["llm_provider"])
- if vector_config:
- conf.vector_store = VectorStoreConfig(provider=vector_config.provider)
- if db_config:
- conf.history_db_path = db_config.path
- self.memory = Memory(conf)
-
- def add(self, data, agent_id=None, run_id=None, metadata=None, filters=None, prompt=None):
- # Implement the add method
- self.memory.add(data, agent_id=agent_id, run_id=run_id, metadata=metadata, filters=filters, prompt=prompt)
-
- def search(self, query, agent_id=None, run_id=None, limit=100, filters=None):
- return self.memory.search(query, agent_id=agent_id, run_id=run_id, limit=limit, filters=filters)
-
- def get(self, memory_id):
- return self.memory.get(memory_id)
-
- def get_all(self, agent_id=None, run_id=None, limit=100):
- return self.memory.get_all(agent_id=agent_id, run_id=run_id, limit=limit)
-
- def update(self, memory_id, data):
- return self.memory.update(memory_id, data)
-
- def delete(self, memory_id):
- return self.memory.delete(memory_id)
-
- def delete_all(self, agent_id=None, run_id=None):
- return self.memory.delete_all(agent_id=agent_id, run_id=run_id)
-
- def history(self, memory_id):
- return self.memory.history(memory_id)
-
- def clear(self):
- return self.memory.reset()
diff --git a/ghostos/prompter.py b/ghostos/prompter.py
new file mode 100644
index 00000000..56802c38
--- /dev/null
+++ b/ghostos/prompter.py
@@ -0,0 +1,341 @@
+from __future__ import annotations
+
+import inspect
+from typing import (
+ List, Self, Union, Callable, Any, Protocol, Optional, Dict, TypeVar, Type, Generic, Set,
+)
+from abc import ABC, abstractmethod
+from types import ModuleType
+from ghostos.container import Container
+from ghostos.helpers import generate_import_path, import_class_from_path, import_from_path
+from pydantic import BaseModel, Field
+from ghostos.entity import EntityMeta, from_entity_meta, to_entity_meta
+
+import json
+
+__all__ = [
+ 'get_defined_prompt',
+ 'set_prompt', 'set_class_prompt',
+ 'Prompter', 'DataPrompter', 'DataPrompterDriver',
+ 'TextPrmt',
+ 'InspectPrmt',
+ 'PromptAbleObj', 'PromptAbleClass',
+]
+
+
+def get_defined_prompt(value: Any, container: Optional[Container] = None) -> Union[str, None]:
+ attr = get_defined_prompt_attr(value, container)
+ if attr is None:
+ return None
+ if isinstance(attr, str):
+ return attr
+ return attr()
+
+
+def get_defined_prompt_attr(value: Any, container: Optional[Container] = None) -> Union[None, str, Callable[[], str]]:
+ if value is None:
+ return None
+ elif isinstance(value, PromptAbleObj):
+ return value.__prompt__
+ elif isinstance(value, Prompter) and container is not None:
+ return value.get_prompt(container)
+
+ elif isinstance(value, type):
+ if issubclass(value, PromptAbleClass):
+ return value.__class_prompt__
+ # class without __class_prompt__ is not defined as prompter
+ if hasattr(value, "__class_prompt__"):
+ return getattr(value, "__class_prompt__")
+
+ elif hasattr(value, "__prompt__"):
+ prompter = getattr(value, "__prompt__")
+ if inspect.isfunction(value) or inspect.ismethod(value) or hasattr(prompter, '__self__'):
+ return prompter
+ elif isinstance(value, ModuleType) and '__prompt__' in value.__dict__:
+ prompter = value.__dict__['__prompt__']
+ return prompter
+ return None
+
+
+def set_prompt(obj: Any, prompter: Union[Callable[[], str], str], force: bool = False) -> None:
+ if force or not hasattr(obj, '__prompt__'):
+ setattr(obj, '__prompt__', prompter)
+
+
+def set_class_prompt(cls: type, prompter: Union[Callable[[], str], str], force: bool = False) -> None:
+ if hasattr(cls, '__class__prompt__'):
+ fn = getattr(cls, '__class_prompt__')
+ cls_name = generate_import_path(cls)
+ if force or fn.__class_name__ != cls_name:
+ pass
+ else:
+ return
+ prompter.__class_name__ = generate_import_path(cls)
+ setattr(cls, '__class_prompt__', prompter)
+
+
+# ---- prompter ---- #
+
+class Prompter(ABC):
+ """
+ is strong-typed model for runtime alternative properties of a ghost.
+ """
+
+ priority: int = Field(default=0, description='Priority of this prompter.')
+
+ __children__: Optional[List[Prompter]] = None
+ """ children is fractal sub context nodes"""
+
+ __self_prompt__: Optional[str] = None
+
+ def with_children(self, *children: Prompter) -> Self:
+ children = list(children)
+ if len(children) > 0:
+ for child in children:
+ if child is None:
+ continue
+ self.add_child(child)
+ return self
+
+ def add_child(self, *prompters: Prompter) -> Self:
+ if self.__children__ is None:
+ self.__children__ = []
+ for prompter in prompters:
+ self.__children__.append(prompter)
+ return self
+
+ @abstractmethod
+ def self_prompt(self, container: Container) -> str:
+ """
+ generate prompt by self, without children
+ :param container:
+ :return:
+ """
+ pass
+
+ @abstractmethod
+ def get_title(self) -> str:
+ """
+ the title of the prompt
+ """
+ pass
+
+ def get_priority(self) -> int:
+ return self.priority
+
+ def get_prompt(self, container: Container, depth: int = 0) -> str:
+ """
+ get prompt with container which provides libraries to generate prompt
+ :param container:
+ :param depth:
+ :return:
+ """
+ if self.__self_prompt__ is not None:
+ return self.__self_prompt__
+
+ title = self.get_title()
+ depth = depth
+ if title:
+ title = '#' * (depth + 1) + ' ' + title
+ depth = depth + 1
+
+ self_prompt = self.self_prompt(container)
+ prompts = []
+ if self_prompt:
+ prompts.append(self_prompt)
+
+ if self.__children__ is not None:
+ for child in self.__children__:
+ child_prompt = child.get_prompt(container, depth=depth)
+ if child_prompt:
+ prompts.append(child_prompt)
+ # empty prompts
+ if not prompts:
+ return ""
+
+ # generate output prompt
+ if title:
+ prompts.insert(0, title)
+ output = ""
+ for paragraph in prompts:
+ paragraph = paragraph.strip()
+ if paragraph:
+ output += "\n\n" + paragraph
+ self.__self_prompt__ = output.strip()
+ return self.__self_prompt__
+
+ def flatten(self, index: str = "") -> Dict[str, Self]:
+ if not index:
+ index = "0"
+ result = {index: self}
+ idx = 0
+ for child in self.__children__:
+ if not child:
+ continue
+ sub_index = index + "." + str(idx)
+ sub_flatten = child.flatten(sub_index)
+ for key in sub_flatten:
+ result[key] = sub_flatten[key]
+ return result
+
+
+class ModelPrompter(BaseModel, Prompter, ABC):
+
+ def __to_entity_meta__(self) -> EntityMeta:
+ type_ = generate_import_path(self.__class__)
+ ctx_data = self.model_dump(exclude_defaults=True)
+ children_data = []
+ if self.__children__ is not None:
+ for child in self.__children__:
+ children_data.append(to_entity_meta(child))
+ data = {"ctx": ctx_data, "children": children_data}
+ content = json.dumps(data)
+ return EntityMeta(type=type_, content=content)
+
+ @classmethod
+ def __from_entity_meta__(cls, meta: EntityMeta) -> Self:
+ data = json.loads(meta["content"])
+ ctx_data = data["ctx"]
+ children_data = data["children"]
+ result = cls(**ctx_data)
+ children = []
+ for child in children_data:
+ children.append(from_entity_meta(child))
+ return result.with_children(*children)
+
+
+class DataPrompter(ModelPrompter, ABC):
+ __driver__: Optional[Type[DataPrompterDriver]] = None
+
+ def get_driver(self) -> DataPrompterDriver:
+ driver = self.__driver__
+ if driver is None:
+ driver_path = generate_import_path(self.__class__) + "Driver"
+ driver = import_class_from_path(driver_path, DataPrompterDriver)
+ return driver(self)
+
+ def self_prompt(self, container: Container) -> str:
+ """
+ generate prompt from model values with libraries that container provides.
+ :param container: IoC container provides library implementation.
+ :return: natural language prompt
+ """
+ return self.get_driver().self_prompt(container)
+
+ def get_title(self) -> str:
+ return self.get_driver().get_title()
+
+
+D = TypeVar("D", bound=DataPrompter)
+
+
+class DataPrompterDriver(Generic[D], ABC):
+
+ def __init__(self, data: D):
+ self.data = data
+
+ @abstractmethod
+ def self_prompt(self, container: Container) -> str:
+ """
+ generate prompt from model values with libraries that container provides.
+ :param container: IoC container provides library implementation.
+ :return: natural language prompt
+ """
+ pass
+
+ @abstractmethod
+ def get_title(self) -> str:
+ pass
+
+
+class TextPrmt(ModelPrompter):
+ title: str = ""
+ content: str = ""
+
+ def self_prompt(self, container: Container) -> str:
+ return self.content
+
+ def get_title(self) -> str:
+ return self.title
+
+
+class InspectPrmt(DataPrompter):
+ title: str = Field(
+ default="Code Inspection",
+ description="The title of the inspect prompt.",
+ )
+ source_target: List[str] = Field(
+ default_factory=list,
+ description="Inspect source code of these targets. ",
+ )
+
+ def inspect_source(self, target: Union[type, Callable, str]) -> Self:
+ if not isinstance(target, str):
+ target = generate_import_path(target)
+ self.source_target.append(target)
+ return self
+
+
+class InspectPrmtDriver(DataPrompterDriver[InspectPrmt]):
+
+ def self_prompt(self, container: Container) -> str:
+ prompts = {}
+ for target in self.data.source_target:
+ got = import_from_path(target)
+ source = inspect.getsource(got)
+ prompts[target] = source
+
+ result = ""
+ for target, source in prompts.items():
+ source = source.strip()
+ if not source:
+ continue
+ result += f"""
+
+source code of `{target}`:
+```python
+{source}
+```
+"""
+ return result.strip()
+
+ def get_title(self) -> str:
+ pass
+
+
+# ---- prompt-able ---- #
+
+class PromptAbleObj(ABC):
+ """
+ 拥有 __prompt__ 方法的类.
+ 这里只是一个示范, 并不需要真正继承这个类, 只需要有 __prompt__ 方法或属性.
+ """
+
+ @abstractmethod
+ def __prompt__(self) -> str:
+ pass
+
+
+class PromptAbleProtocol(Protocol):
+ @abstractmethod
+ def __prompt__(self) -> str:
+ pass
+
+
+class PromptAbleClass(ABC):
+
+ @classmethod
+ @abstractmethod
+ def __class_prompt__(cls) -> str:
+ pass
+
+
+class PromptAbleClassProtocol(Protocol):
+
+ @classmethod
+ @abstractmethod
+ def __class_prompt__(cls) -> str:
+ pass
+
+
+PromptAble = Union[PromptAbleClass, PromptAbleObj, PromptAbleProtocol, PromptAbleClassProtocol]
diff --git a/ghostos/prototypes/README.md b/ghostos/prototypes/README.md
new file mode 100644
index 00000000..2847e1bd
--- /dev/null
+++ b/ghostos/prototypes/README.md
@@ -0,0 +1,3 @@
+# Prototypes
+
+The prototypes of `GhostOS` applications.
\ No newline at end of file
diff --git a/ghostos/demo/src/examples/code_edits/__init__.py b/ghostos/prototypes/__init__.py
similarity index 100%
rename from ghostos/demo/src/examples/code_edits/__init__.py
rename to ghostos/prototypes/__init__.py
diff --git a/ghostos/prototypes/aifunc/__init__.py b/ghostos/prototypes/aifunc/__init__.py
deleted file mode 100644
index 5635de75..00000000
--- a/ghostos/prototypes/aifunc/__init__.py
+++ /dev/null
@@ -1,20 +0,0 @@
-from ghostos.prototypes.aifunc.app import run_aifunc
-from ghostos.core.aifunc import AIFunc, AIFuncResult
-from os.path import dirname
-
-__all__ = ["run_aifunc", "quick_run_aifunc"]
-
-
-def quick_run_aifunc(
- aifunc: AIFunc,
- current_path: str,
- dirname_times: int = 0,
- debug: bool = True,
-) -> AIFuncResult:
- """
- create aifunc runtime with default paths and run it
- """
- root_dir = current_path
- for i in range(dirname_times):
- root_dir = dirname(root_dir)
- return run_aifunc(root_dir, aifunc, debug=debug)
diff --git a/ghostos/prototypes/aifunc/app.py b/ghostos/prototypes/aifunc/app.py
deleted file mode 100644
index 354440cb..00000000
--- a/ghostos/prototypes/aifunc/app.py
+++ /dev/null
@@ -1,110 +0,0 @@
-import argparse
-import sys
-import os
-import yaml
-from typing import List, Dict
-
-from ghostos.core.session import MsgThread
-from logging.config import dictConfig
-from ghostos.core.llms import Chat
-from ghostos.core.messages import Message
-from ghostos.core.moss import test_container
-from ghostos.core.aifunc import (
- DefaultAIFuncManagerImpl, AIFunc, DefaultAIFuncDriverImpl, AIFuncManager,
- AIFuncResult,
-)
-from ghostos.framework.logger import NamedLoggerProvider
-from ghostos.framework.storage import FileStorageProvider
-from ghostos.framework.llms import ConfigBasedLLMsProvider
-from ghostos.framework.threads import StorageThreadsProvider
-from ghostos.framework.configs import ConfigsByStorageProvider
-from rich.console import Console
-from rich.panel import Panel
-from rich.markdown import Markdown
-from rich.prompt import Prompt
-
-__all__ = ['run_aifunc']
-
-console = Console()
-
-
-def init_logger(conf_path: str):
- with open(conf_path) as f:
- content = f.read()
- data = yaml.safe_load(content)
- dictConfig(data)
-
-
-def run_aifunc(
- root_dir: str,
- aifunc: AIFunc,
- logger_conf_path: str = "configs/logging.yml",
- logger_name: str = "debug",
- threads_path: str = "runtime/threads",
- configs_path: str = "configs",
- llm_conf_path: str = "llms_conf.yml",
- llm_api_name: str = "",
- debug: bool = True,
-) -> AIFuncResult:
- # prepare logger
- absolute_logger_conf = os.path.join(root_dir, logger_conf_path)
- init_logger(absolute_logger_conf)
-
- # prepare container
- container = test_container()
- container.register(FileStorageProvider(root_dir))
- container.register(NamedLoggerProvider(logger_name=logger_name))
- container.register(StorageThreadsProvider(threads_dir=threads_path))
- container.register(ConfigsByStorageProvider(configs_path))
- container.register(ConfigBasedLLMsProvider(llm_conf_path))
-
- class TestDriverImpl(DefaultAIFuncDriverImpl):
- console = console
-
- def on_message(self, message: Message) -> None:
- self.console.print(
- Panel(
- Markdown(message.get_content()),
- title=f"generated message ({self.name()})",
- )
- )
- if debug:
- value = Prompt.ask("Continue?", choices=["y", "n"], default="y")
- if value != "y":
- exit(0)
-
- def on_chat(self, chat: Chat) -> None:
- for message in chat.get_messages():
- self.console.print(Panel(
- Markdown(message.get_content()),
- title=f"chat_info ({self.name()})",
- ))
- if debug:
- value = Prompt.ask("Continue?", choices=["y", "n"], default="y")
- if value != "y":
- exit(0)
-
- def on_system_messages(self, messages: List[Message]) -> None:
- pass
-
- def on_save(self, manager: AIFuncManager, thread: MsgThread) -> None:
- current = thread.current
- if current:
- for message in current.messages():
- self.console.print(
- Panel(
- Markdown(message.get_content()),
- title="thread new round message",
- )
- )
- super().on_save(manager, thread)
-
- manager = DefaultAIFuncManagerImpl(
- container=container,
- llm_api_name=llm_api_name,
- default_driver=TestDriverImpl,
- )
- try:
- return manager.execute(aifunc)
- finally:
- manager.destroy()
diff --git a/ghostos/prototypes/console/__init__.py b/ghostos/prototypes/console/__init__.py
index 4b2e7c00..d921a401 100644
--- a/ghostos/prototypes/console/__init__.py
+++ b/ghostos/prototypes/console/__init__.py
@@ -1,194 +1 @@
-from os.path import join, dirname
-from typing import Optional, List
-from ghostos.core import GhostOS
-from ghostos.framework.ghostos import demo_ghostos, DemoGhostOS
-from ghostos.framework.ghosts.demo import DemoGhostConf
-from ghostos.prototypes.console.app import ConsolePrototype
-from ghostos.core.ghosts import Thought
-from ghostos.helpers import get_calling_modulename, import_from_path, md5, uuid
-from ghostos.container import Provider
-
-__all__ = [
- 'ConsoleApp',
- 'ConsolePrototype',
- 'quick_new_console_app',
- 'demo_console_app',
-]
-
-
-class ConsoleApp:
- """
- Create a GhostOS Console app with a ghostos.
- New demo from specific directory as default.
- """
-
- def __init__(
- self,
- ghostos: GhostOS = None,
- ):
- self._ghostos = ghostos
- # status
- self._ran_thought = False
- self._ran_console = False
-
- @classmethod
- def new_demo(
- cls,
- *,
- root_dir: str,
- logger_conf_path: str,
- logger_name: str = "debug",
- conf_path: str = "configs",
- runtime_path: str = "runtime",
- processes_path: str = "processes",
- tasks_path: str = "tasks",
- threads_path: str = "threads",
- llm_conf_path: str = "llms_conf.yml",
- source_path: str = "src",
- ):
- ghostos = DemoGhostOS(
- root_dir=root_dir,
- logger_conf_path=logger_conf_path,
- logger_name=logger_name,
- config_path=conf_path,
- runtime_path=runtime_path,
- source_path=source_path,
- processes_path=processes_path,
- tasks_path=tasks_path,
- threads_path=threads_path,
- llm_config_path=llm_conf_path,
- )
- return cls(ghostos)
-
- def with_providers(self, providers: List[Provider]):
- """
- register global providers
- :param providers: provide contracts and implementations.
- """
- for provider in providers:
- self._ghostos.container().register(provider)
-
- def run_console(
- self,
- ghost_id: str,
- *,
- username: str = "BrightRed",
- on_create_message: Optional[str] = None,
- debug: bool = False,
- session_id: Optional[str] = None,
- welcome_user_message: Optional[str] = None,
- ):
- """
- :param ghost_id: should exist in configs/ghosts.yml
- :param username: the username, default is my name haha.
- :param on_create_message: the message to send to the assistant as default.
- :param debug: if debug is True, render more verbosely.
- :param session_id: if given, the console will start in the same session by the id.
- :param welcome_user_message: if on_create_message is None, use welcome_user_message let agent welcome first
- """
- if self._ran_console:
- return
- self._ran_console = True
- ghostos = demo_ghostos
- console_impl = ConsolePrototype(
- ghostos=ghostos,
- ghost_id=ghost_id,
- username=username,
- debug=debug,
- on_create_message=on_create_message,
- session_id=session_id,
- welcome_user_message=welcome_user_message,
- )
- console_impl.run()
- self._ran_console = False
-
- def run_thought(
- self,
- thought: Thought,
- *,
- instruction: Optional[str] = None,
- username: str = "BrightRed",
- ghost_name: str = "GhostOSDemo",
- debug: bool = False,
- meta_prompt: str = "",
- long_term_session: bool = False,
- welcome_user_message: Optional[str] = None,
- ):
- """
- Run a thought instead of run a defined Ghost.
- :param thought: root thought of the ghost
- :param instruction: the init instruction send to assistant
- :param username: name of the console's user. default is my name hahaha.
- :param ghost_name: ghost name
- :param debug: if debug is True, render more verbosely.
- :param meta_prompt: define the meta prompt of the ghost. I leave it empty.
- :param long_term_session: if true, session id is always related to the calling module.
- :param welcome_user_message: if on_create_message is None, use welcome_user_message let agent welcome first
- :return:
- """
- if self._ran_thought:
- return
- self._ran_thought = True
- modulename = get_calling_modulename(1)
- module = import_from_path(modulename)
- file = module.__file__
- ghost_id = md5(file)
- ghost_conf = DemoGhostConf(
- id=ghost_id,
- name=ghost_name,
- thought_meta=thought.to_entity_meta(),
- meta_prompt=meta_prompt,
- )
- self._ghostos.register(ghost_conf)
- if long_term_session:
- session_id = ghost_id
- else:
- session_id = uuid()
- console_impl = ConsolePrototype(
- ghostos=self._ghostos,
- ghost_id=ghost_id,
- username=username,
- on_create_message=instruction,
- debug=debug,
- session_id=session_id,
- welcome_user_message=welcome_user_message,
- )
- console_impl.run()
- self._ran_thought = False
-
-
-demo_dir = join(dirname(dirname(dirname(__file__))), "demo")
-
-demo_console_app = ConsoleApp.new_demo(
- root_dir=demo_dir,
- logger_conf_path=join(demo_dir, "configs/logging.yml"),
-)
-""" default app instance for testing convenient"""
-
-__app__ = None
-
-
-def quick_new_console_app(
- current_file: str,
- dirname_times: int = 0,
- logging_conf: str = "configs/logging.yml",
- **kwargs,
-) -> ConsoleApp:
- """
- quick to create a console app based on root_dir. only once shall be called globally.
- :param current_file: current file name, usually __file__
- :param dirname_times: depth from current_file to root dir
- :param logging_conf:
- :return:
- """
- global __app__
- if __app__ is not None:
- return __app__
- root_dir = current_file
- for i in range(dirname_times):
- root_dir = dirname(root_dir)
-
- logger_conf_path = join(root_dir, logging_conf)
- app = ConsoleApp.new_demo(root_dir=root_dir, logger_conf_path=logger_conf_path, **kwargs)
- __app__ = app
- return app
+from ghostos.prototypes.console.app import ConsoleApp
diff --git a/ghostos/prototypes/console/app.py b/ghostos/prototypes/console/app.py
index 00450752..eee61cbf 100644
--- a/ghostos/prototypes/console/app.py
+++ b/ghostos/prototypes/console/app.py
@@ -1,119 +1,164 @@
from __future__ import annotations
-from ghostos.core.ghostos import GhostOS
-import time
import asyncio
from typing import Optional, List
-from ghostos.core.messages import Message, Role, DefaultMessageTypes
-from ghostos.core.ghosts import Inputs
-from ghostos.framework.streams import QueueStream
+from ghostos.abcd import GhostOS, Ghost, Background
+from ghostos.container import Provider
+from ghostos.contracts.logger import get_console_logger
+from ghostos.core.messages import Message, Role, MessageType, Receiver
from ghostos.framework.messages import TaskPayload
-from ghostos.helpers import uuid
-from threading import Thread
+from ghostos.core.runtime import Event
from queue import Queue, Empty
from prompt_toolkit.key_binding import KeyBindings
from prompt_toolkit.patch_stdout import patch_stdout
from prompt_toolkit.shortcuts import PromptSession
+from threading import Lock
from rich.console import Console
from rich.panel import Panel
from rich.markdown import Markdown
+from rich.status import Status
-__all__ = ['ConsolePrototype']
+__all__ = ['ConsoleApp']
-class ConsolePrototype:
+class ConsoleApp(Background):
def __init__(
self, *,
ghostos: GhostOS,
- ghost_id: str,
+ ghost: Ghost,
username: str,
debug: bool = False,
- on_create_message: Optional[str] = None,
- session_id: Optional[str] = None,
+ shell_name: str = "console",
+ shell_id: Optional[str] = None,
process_id: Optional[str] = None,
- task_id: Optional[str] = None,
- background_num: int = 4,
+ worker_num: int = 4,
welcome_user_message: Optional[str] = None,
+ on_create_message: Optional[str] = None,
+ providers: Optional[List[Provider]] = None,
):
self._os = ghostos
- self._ghost_id = ghost_id
- self._on_create_message = on_create_message
+ self._ghost = ghost
self._username = username
+ self._shell_name = shell_name
+ self._shell_id = shell_id if shell_id else shell_name
self._process_id = process_id
- self._task_id = task_id
- self._session_id = session_id if session_id else uuid()
- session = PromptSession("\n\n<<< ", )
- self._prompt_session = session
self._console = Console()
+ self._logger = get_console_logger()
+ self._closed = False
self._stopped = False
self._main_queue = Queue()
+ self._thread_locker = Lock()
self._debug = debug
- self._threads: List[Thread] = []
- self._background_num = background_num
+ self._worker_num = worker_num
if not welcome_user_message:
- welcome_user_message = "the conversation is going to begin, please welcome user and introduce your self"
+ welcome_user_message = "the conversation is going to begin, please welcome user"
self._welcome_user_message = welcome_user_message
+ self._on_create_message = on_create_message
self._main_task_id = ""
+ session = PromptSession("\n\n<<< ", )
+ self._prompt_session = session
+ self._shell = self._os.create_shell(
+ self._shell_name,
+ process_id=self._process_id,
+ providers=providers,
+ )
+ self._conversation = self._shell.sync(self._ghost)
+
+ def __del__(self):
+ self.close()
+ self._conversation.close()
+ self._shell.close()
def run(self):
- for i in range(self._background_num):
- background_run_task = Thread(target=self._start_background)
- background_run_task.start()
- self._threads.append(background_run_task)
- print_output_task = Thread(target=self._print_output)
- print_output_task.start()
- self._threads.append(print_output_task)
+ # self._shell.background_run(self._worker_num)
+ self._shell.submit(self._print_output)
+ self._shell.background_run(self._worker_num, self)
asyncio.run(self._main())
+ def close(self):
+ if self._closed:
+ return
+ self._closed = True
+ self._stopped = True
+ self._main_queue.put(None)
+ self._console.print("start exiting")
+ self._conversation.close()
+ self._console.print("conversation closed")
+ self._shell.close()
+ self._console.print("ghostos shell shutdown")
+ self._console.print("Exit, Bye!")
+ exit(0)
+
+ async def _main(self):
+ self._welcome()
+ self._console.print("waiting for agent say hi...")
+ message = Role.new_system(
+ self._welcome_user_message,
+ )
+ event, receiver = self._conversation.respond([message])
+ self.output_receiver(receiver)
+
+ with patch_stdout(raw=True):
+ await self._loop()
+ self._console.print("Quitting event loop. Bye.")
+
def _print_output(self):
while not self._stopped:
try:
- message = self._main_queue.get(block=True, timeout=1)
+ message = self._main_queue.get(block=True)
+ if message is None:
+ self.close()
+ return
if not isinstance(message, Message):
raise ValueError(f"Expected Message, got {message}")
self._print_message(message)
except Empty:
continue
- def _start_background(self):
- while not self._stopped:
- stream = self._stream()
- handled = self._os.background_run(stream)
- if not handled:
- time.sleep(1)
- elif not self._debug:
- self._console.print(f"handled event {handled.type}: task_id {handled.task_id}; event_id {handled.id};")
- else:
- self._console.print(Panel(
- Markdown(f"```json\n{handled.model_dump_json(indent=2)}\n```"),
- title="handle event",
- border_style="yellow",
- ))
-
- def _stream(self) -> QueueStream:
- return QueueStream(self._main_queue, streaming=False)
+ def output_receiver(self, receiver: Receiver):
+ with self._thread_locker:
+ status = Status("receiving", console=self._console)
+ with status:
+ with receiver:
+ buffer = None
+ for message in receiver.recv():
+ if self._stopped:
+ return
- async def _main(self):
- self._welcome()
- if self._on_create_message:
- self._console.print(
- Panel(
- Markdown(self._on_create_message),
- title="on_created instruction",
- border_style="green",
- )
- )
- self._main_task_id = self._on_input(self._on_create_message)
- else:
- self._console.print("waiting for agent say hi...")
- message = Role.new_assistant_system(
- self._welcome_user_message,
- )
- self._main_task_id = self._on_message_input(message)
- with patch_stdout(raw=True):
- await self._loop()
- self._console.print("Quitting event loop. Bye.")
+ if message.is_complete():
+ buffer = None
+ self._main_queue.put(message)
+ elif buffer is None:
+ buffer = message.as_head()
+ else:
+ patched = buffer.patch(message)
+ if patched:
+ buffer = patched
+ else:
+ buffer = message.as_head()
+ if buffer:
+ status.update(buffer.content[-30:])
+ else:
+ status.update("")
+
+ def output_event(self, event: Event):
+ self._json_output(event.model_dump_json(indent=2, exclude_defaults=True))
+
+ def on_error(self, error: Exception) -> bool:
+ self._logger.exception(error)
+ self.close()
+ return False
+
+ def on_event(self, event: Event, messages: List[Message]) -> None:
+ self._logger.debug(f"Received event {event.event_id} for task {event.task_id}")
+ # self.output_receiver(retriever)
+
+ def alive(self) -> bool:
+ return not self._stopped
+
+ def halt(self) -> int:
+ return 0
async def _loop(self):
session = self._prompt_session
@@ -126,12 +171,12 @@ async def _loop(self):
self._console.print(Markdown("\n----\n"))
self._on_input(text)
except (EOFError, KeyboardInterrupt):
- self._exit()
+ self.close()
except Exception:
self._console.print_exception()
- self._exit()
+ self.close()
- def _on_input(self, text: str) -> str:
+ def _on_input(self, text: str) -> None:
"""
:return: task_id
"""
@@ -139,34 +184,18 @@ def _on_input(self, text: str) -> str:
content=text,
name=self._username,
)
- return self._on_message_input(message)
+ self._on_message_input(message)
- def _on_message_input(self, message: Message) -> str:
+ def _on_message_input(self, message: Message) -> None:
"""
:return: task_id
"""
- inputs_ = Inputs(
- trace_id=uuid(),
- session_id=self._session_id,
- ghost_id=self._ghost_id,
- messages=[message],
- process_id=self._process_id,
- task_id=self._task_id,
- )
- stream = self._stream()
- if not self._debug:
- self._console.print(f"push input event id: {inputs_.trace_id}")
- else:
- self._console.print(Panel(
- Markdown(f"```json\n{inputs_.model_dump_json(indent=2)}\n```"),
- title="push input event",
- border_style="yellow",
- ))
- return self._os.on_inputs(inputs_, stream, is_async=True)
+ event, receiver = self._conversation.respond([message])
+ self.output_receiver(receiver)
def _intercept_text(self, text: str) -> bool:
if text == "/exit":
- self._exit()
+ self.close()
return False
@staticmethod
@@ -185,27 +214,14 @@ def _welcome(self) -> None:
----
"""))
- def _exit(self):
- self._stopped = True
- _continue = True
- self._console.print("start exiting")
- while _continue:
- try:
- self._main_queue.get_nowait()
- except Empty:
- break
- self._console.print("stop queue")
- self._console.print("queue closed")
- for t in self._threads:
- t.join()
- self._console.print("threads joined")
- self._os.shutdown()
- self._console.print("ghostos shutdown")
- self._console.print("Exit, Bye!")
- exit(0)
-
def _print_message(self, message: Message):
- if self._debug:
+ if not message.is_complete():
+ return
+
+ if message.is_empty():
+ return
+
+ if not MessageType.is_text(message):
self._console.print(
Panel(
self._json_output(message.model_dump_json(exclude_defaults=True, indent=2)),
@@ -213,13 +229,13 @@ def _print_message(self, message: Message):
border_style="green",
)
)
- if message.is_empty():
return
+
content = message.content
# some message is not visible to user
if not content:
return
- payload = TaskPayload.read(message)
+ payload = TaskPayload.read_payload(message)
title = "receive message"
# markdown content
prefix = ""
@@ -229,13 +245,10 @@ def _print_message(self, message: Message):
f"> thread_id: {payload.thread_id}",
f"> task_name: {payload.task_name}\n\n",
])
- if "" in content:
- content = content.replace("", "\n```python\n# \n", )
- if " " in content:
- content = content.replace(" ", "\n# \n```\n", )
+
markdown = self._markdown_output(prefix + content)
# border style
- if DefaultMessageTypes.ERROR.match(message):
+ if MessageType.ERROR.match(message):
border_style = "red"
elif payload is not None and payload.task_id == self._main_task_id:
border_style = "blue"
diff --git a/ghostos/prototypes/ghostfunc/__init__.py b/ghostos/prototypes/ghostfunc/__init__.py
index 8d28f14a..eb2bb1a3 100644
--- a/ghostos/prototypes/ghostfunc/__init__.py
+++ b/ghostos/prototypes/ghostfunc/__init__.py
@@ -1,30 +1,12 @@
-from os.path import dirname, join
from ghostos.prototypes.ghostfunc.decorator import GhostFunc
-from ghostos.prototypes.ghostfunc.prepare import init_ghost_func_container
+from ghostos.prototypes.ghostfunc.prepare import init_ghost_func, init_ghost_func_container
"""
this is a toy that using MOSS to implement a light-weight dynamic function based by llm code generation.
"""
-__all__ = ['ghost_func', 'GhostFunc', 'init_ghost_func_container', 'init_ghost_func']
-
-demo_dir = join(dirname(dirname(dirname(__file__))), 'demo')
-_container = init_ghost_func_container(demo_dir)
-
-ghost_func = GhostFunc(_container)
-
-
-def init_ghost_func(
- root_dir: str,
- configs_path: str = "configs",
- llm_conf_path: str = "llms_conf.yml",
-) -> GhostFunc:
- """
- init ghost func instance from a dir keeping configs.
- :param root_dir: root dir of runtime and configs.
- :param configs_path: configs dir path from root dir
- :param llm_conf_path: llm config path in configs dir
- :return: instance of GhostFunc, with decorators.
- """
- ghost_func_container = init_ghost_func_container(root_dir, configs_path, llm_conf_path)
- return GhostFunc(ghost_func_container)
+__all__ = [
+ 'GhostFunc',
+ 'init_ghost_func_container',
+ 'init_ghost_func',
+]
diff --git a/ghostos/prototypes/ghostfunc/decorator.py b/ghostos/prototypes/ghostfunc/decorator.py
index 50f4a4b5..b947c328 100644
--- a/ghostos/prototypes/ghostfunc/decorator.py
+++ b/ghostos/prototypes/ghostfunc/decorator.py
@@ -1,6 +1,5 @@
import inspect
from typing import Callable, Optional, Dict
-from abc import ABC, abstractmethod
from ghostos.container import Container
from ghostos.prototypes.ghostfunc.driver import (
GhostFuncDriver, GhostFuncCache, get_ghost_func_cache, save_ghost_func_cache,
@@ -16,6 +15,7 @@
class GhostFunc:
def __init__(self, container: Container):
self._container = container
+ self._container.bootstrap()
self._caches: Dict[str, GhostFuncCache] = {}
self._compiled = set()
diff --git a/ghostos/prototypes/ghostfunc/driver.py b/ghostos/prototypes/ghostfunc/driver.py
index f9099257..c4987bcf 100644
--- a/ghostos/prototypes/ghostfunc/driver.py
+++ b/ghostos/prototypes/ghostfunc/driver.py
@@ -2,8 +2,9 @@
import os
import yaml
import importlib
+
from ghostos.container import Container
-from ghostos.core.session import MsgThread, DefaultEventType, thread_to_chat
+from ghostos.core.runtime import GoThreadInfo, EventTypes, thread_to_prompt
from ghostos.core.moss import MossRuntime, MossCompiler, PyContext
from ghostos.core.llms import LLMs, LLMApi
from ghostos.core.messages import Role, Message
@@ -21,7 +22,7 @@ class GhostFuncCache(BaseModel):
"""
modulename: str = Field(description="the module name that decorated function located")
filename: Optional[str] = Field(default=None, description="the filename that decorated function located")
- threads: Dict[str, MsgThread] = Field(
+ threads: Dict[str, GoThreadInfo] = Field(
default_factory=dict,
description="a map of function.__qualname__ to thread instance",
)
@@ -130,7 +131,7 @@ def execute(self, args: List[Any], kwargs: Dict[str, Any]) -> Any:
thread = self._init_thread()
return self._run(thread, args, kwargs)
- def _run(self, thread: MsgThread, args: List[Any], kwargs: Dict[str, Any]) -> Any:
+ def _run(self, thread: GoThreadInfo, args: List[Any], kwargs: Dict[str, Any]) -> Any:
"""
run the ghost func with the origin function's args and kwargs.
:param thread:
@@ -140,7 +141,7 @@ def _run(self, thread: MsgThread, args: List[Any], kwargs: Dict[str, Any]) -> An
"""
# get generated code from history, run it.
pycontext = thread.last_turn().pycontext
- generated = pycontext.generated
+ generated = pycontext.execute_code
if self._caching and generated and pycontext.executed:
thread, result = self._start_with_generated_code(generated, thread, pycontext, args, kwargs)
else:
@@ -162,14 +163,14 @@ def _init_prompt(self, context_code: str) -> str:
target_source=self._target_source,
)
- def _init_thread(self) -> MsgThread:
+ def _init_thread(self) -> GoThreadInfo:
pycontext = self._init_pycontext()
moss_runtime = self._moss_runtime(pycontext)
- context_code = moss_runtime.prompter().dump_context_prompt()
+ context_code = moss_runtime.prompter().dump_module_prompt()
instruction = self._init_prompt(context_code)
system = Role.SYSTEM.new(content=instruction)
- e = DefaultEventType.OBSERVE.new(task_id="", messages=[system], from_task_id="")
- return MsgThread.new(
+ e = EventTypes.ROTATE.new(task_id="", messages=[system], from_task_id="")
+ return GoThreadInfo.new(
event=e,
pycontext=pycontext,
)
@@ -193,17 +194,17 @@ def _get_llm_api(self) -> LLMApi:
def _start_with_generated_code(
self,
generated: str,
- thread: MsgThread,
+ thread: GoThreadInfo,
pycontext: PyContext,
args: List[Any],
kwargs: Dict[str, Any],
- ) -> Tuple[MsgThread, Any]:
+ ) -> Tuple[GoThreadInfo, Any]:
result, ok = self._run_code(generated, thread, pycontext, args, kwargs)
if ok:
return thread, result
return self._think(thread, args, kwargs)
- def _think(self, thread: MsgThread, args: List[Any], kwargs: Dict[str, Any]) -> Tuple[MsgThread, Any]:
+ def _think(self, thread: GoThreadInfo, args: List[Any], kwargs: Dict[str, Any]) -> Tuple[GoThreadInfo, Any]:
turns = 0
while True:
result, ok = self._run_turn(thread, args, kwargs)
@@ -213,9 +214,9 @@ def _think(self, thread: MsgThread, args: List[Any], kwargs: Dict[str, Any]) ->
if turns > self._max_turns:
raise RuntimeError(f"Exceed max turns {self._max_turns} turns, still not success")
- def _run_turn(self, thread: MsgThread, args: List[Any], kwargs: Dict[str, Any]) -> Tuple[Any, bool]:
+ def _run_turn(self, thread: GoThreadInfo, args: List[Any], kwargs: Dict[str, Any]) -> Tuple[Any, bool]:
pycontext = thread.last_turn().pycontext
- chat = thread_to_chat(thread.id, [], thread)
+ chat = thread_to_prompt(thread.id, [], thread)
llm_api = self._get_llm_api()
message = llm_api.chat_completion(chat)
thread.append(message)
@@ -223,7 +224,7 @@ def _run_turn(self, thread: MsgThread, args: List[Any], kwargs: Dict[str, Any])
code, ok = self._unwrap_message_code(message)
if not ok:
thread.new_turn(
- event=DefaultEventType.OBSERVE.new(
+ event=EventTypes.ROTATE.new(
task_id="",
from_task_id="",
messages=[Role.SYSTEM.new(content=code)],
@@ -235,13 +236,13 @@ def _run_turn(self, thread: MsgThread, args: List[Any], kwargs: Dict[str, Any])
def _run_code(
self,
code: str,
- thread: MsgThread,
+ thread: GoThreadInfo,
pycontext: PyContext,
args: List[Any],
kwargs: Dict[str, Any],
) -> Tuple[Any, bool]:
runtime = self._moss_runtime(pycontext)
- pycontext.generated = code
+ pycontext.execute_code = code
pycontext.executed = True
executed = None
try:
@@ -250,24 +251,24 @@ def _run_code(
if not self._ask_confirm_error(thread, e):
message = Role.SYSTEM.new(content=f"Error occur: {e}")
thread.new_turn(
- event=DefaultEventType.OBSERVE.new(task_id="", messages=[message], from_task_id="")
+ event=EventTypes.ROTATE.new(task_id="", messages=[message], from_task_id="")
)
return None, False
finally:
- runtime.destroy()
+ runtime.close()
result, ok = executed.returns
if not ok:
message = Role.SYSTEM.new(content=executed.std_output)
thread.new_turn(
- event=DefaultEventType.OBSERVE.new(task_id="", messages=[message], from_task_id=""),
+ event=EventTypes.ROTATE.new(task_id="", messages=[message], from_task_id=""),
)
return None, False
return result, True
- def _ask_confirm_error(self, thread: MsgThread, error: Exception) -> bool:
- chat = thread_to_chat(thread.id, [], thread)
- chat.appending.append(
+ def _ask_confirm_error(self, thread: GoThreadInfo, error: Exception) -> bool:
+ chat = thread_to_prompt(thread.id, [], thread)
+ chat.added.append(
Role.SYSTEM.new(
content=f"Catch Error: {error} \nIf the error is expected, return `ok`, otherwise return `false`"
)
@@ -286,7 +287,7 @@ def _unwrap_message_code(message: Message) -> Tuple[str, bool]:
splits = code.split(' ', 2)
return splits[0], True
- def _save_thread(self, thread: MsgThread) -> None:
+ def _save_thread(self, thread: GoThreadInfo) -> None:
self._cache.threads[self._target_qualname] = thread
def destroy(self) -> None:
diff --git a/ghostos/prototypes/ghostfunc/prepare.py b/ghostos/prototypes/ghostfunc/prepare.py
index 5dfe96b0..5e6f7d38 100644
--- a/ghostos/prototypes/ghostfunc/prepare.py
+++ b/ghostos/prototypes/ghostfunc/prepare.py
@@ -1,19 +1,48 @@
+from typing import Optional
from ghostos.container import Container
-from ghostos.core.moss import test_container
+from ghostos.core.moss import moss_container, MossCompiler
+
+from ghostos.core.llms import LLMs
from ghostos.framework.configs import ConfigsByStorageProvider
from ghostos.framework.storage import FileStorageProvider
from ghostos.framework.llms import ConfigBasedLLMsProvider
+from ghostos.prototypes.ghostfunc.decorator import GhostFunc
+from ghostos.container import Contracts
+
+__all__ = ["init_ghost_func_container", "init_ghost_func", 'ghost_func_contracts']
-__all__ = ["init_ghost_func_container"]
+ghost_func_contracts = Contracts([
+ LLMs,
+ MossCompiler,
+])
def init_ghost_func_container(
- root_path: str,
- configs_path: str = "configs",
- llm_conf_path: str = "llms_conf.yml",
+ workspace_dir: str,
+ configs_dir: str = "configs",
+ container: Optional[Container] = None,
) -> Container:
- container = test_container()
- container.register(FileStorageProvider(root_path))
- container.register(ConfigsByStorageProvider(configs_path))
- container.register(ConfigBasedLLMsProvider(llm_conf_path))
+ """
+ init ghost_func's container
+ :param workspace_dir:
+ :param configs_dir: relative directory from workspace
+ :param container: parent container.
+ """
+ if container is None:
+ container = moss_container()
+ container.register(FileStorageProvider(workspace_dir))
+ container.register(ConfigsByStorageProvider(configs_dir))
+ container.register(ConfigBasedLLMsProvider())
return container
+
+
+def init_ghost_func(
+ container: Container,
+) -> GhostFunc:
+ """
+ return ghost func instance
+ :param container: application container.
+ """
+ ghost_func_contracts.validate(container)
+ self_container = Container(parent=container)
+ return GhostFunc(self_container)
diff --git a/ghostos/prototypes/mosstemp/__init__.py b/ghostos/prototypes/mosstemp/__init__.py
deleted file mode 100644
index 90de905c..00000000
--- a/ghostos/prototypes/mosstemp/__init__.py
+++ /dev/null
@@ -1,17 +0,0 @@
-import inspect
-from typing import Optional
-
-from ghostos.prototypes.mosstemp import template
-from ghostos.helpers import get_calling_modulename, rewrite_module_by_path
-
-__all__ = ['init_moss_module']
-
-
-def init_moss_module(modulename: Optional[str] = None):
- """
- init moss file with default template
- """
- if not modulename:
- modulename = get_calling_modulename(1)
- source = inspect.getsource(template)
- rewrite_module_by_path(modulename, source)
diff --git a/ghostos/prototypes/mosstemp/template.py b/ghostos/prototypes/mosstemp/template.py
deleted file mode 100644
index 5f9a203c..00000000
--- a/ghostos/prototypes/mosstemp/template.py
+++ /dev/null
@@ -1,66 +0,0 @@
-from typing import Optional
-from ghostos.core.ghosts import Operator
-from ghostos.core.moss import Moss as Parent
-
-
-# todo: import necessary libraries and methods
-
-
-class Moss(Parent):
- """
- todo: define attrs and dependency injection
- """
- pass
-
-
-# todo: can write in-context learning cases for llm
-if __name__ == "__examples__":
- def example_hello_world_main(moss: Moss) -> Optional[Operator]:
- """
- todo: use docstring to describe the user query and planning thought of this example case
- """
- # todo: the example codes
- pass
-
-# the content between mark are not visible in the prompt for LLM
-
-from typing import TYPE_CHECKING
-
-if TYPE_CHECKING:
- # todo: these libraries are useful for lifecycle functions
- pass
-
-# todo: can define these OPTIONAL lifecycle hooks
-from ghostos.core.moss.lifecycle import (
- __moss_compile__ as __default_moss_compile__,
- __moss_attr_prompts__ as __default_moss_attr_prompts__,
- __moss_prompt__ as __default_moss_prompt__,
- __moss_exec__ as __default_moss_exec__,
-)
-
-# todo: define or remove this __moss_compile__
-__moss_compile__ = __default_moss_compile__
-""" do something before MossCompiler.compile() """
-
-# todo: define or remove this __moss_attr_prompts__
-__moss_attr_prompts__ = __default_moss_attr_prompts__
-""" define prompt for the module attr name. set [attr_name] to '' means not to prompt it. """
-
-# todo: define or remove this __moss_prompt__
-__moss_prompt__ = __default_moss_prompt__
-""" define prompt generation """
-
-# todo: define or remove this __moss_exec__
-__moss_exec__ = __default_moss_exec__
-""" redefine the moss exec function. not recommended"""
-
-# todo: can define a moss thought in a moss file
-from ghostos.thoughts.moss_thought import MossThought
-
-thought = MossThought(
- instruction="???",
- moss_modulename=__name__,
- llm_api_name="",
-)
-
-#
diff --git a/ghostos/demo/src/examples/moss_codes/__init__.py b/ghostos/prototypes/realtime_console/__init__.py
similarity index 100%
rename from ghostos/demo/src/examples/moss_codes/__init__.py
rename to ghostos/prototypes/realtime_console/__init__.py
diff --git a/ghostos/demo/src/examples/thoughts/__init__.py b/ghostos/prototypes/realtime_console/console.py
similarity index 100%
rename from ghostos/demo/src/examples/thoughts/__init__.py
rename to ghostos/prototypes/realtime_console/console.py
diff --git a/ghostos/prototypes/realtime_console/vad_test_script.py b/ghostos/prototypes/realtime_console/vad_test_script.py
new file mode 100644
index 00000000..f8da654c
--- /dev/null
+++ b/ghostos/prototypes/realtime_console/vad_test_script.py
@@ -0,0 +1,57 @@
+from ghostos.framework.openai_realtime.app import RealtimeAppImpl
+from ghostos.framework.openai_realtime.configs import OpenAIRealtimeAppConf
+from ghostos.bootstrap import get_ghostos
+from ghostos.contracts.configs import Configs
+from ghostos.contracts.logger import LoggerItf, get_console_logger
+from ghostos.ghosts import Chatbot
+from ghostos.framework.audio import get_pyaudio_pcm16_speaker, get_pyaudio_pcm16_listener
+from rich.console import Console
+import time
+
+console = Console()
+
+if __name__ == "__main__":
+ ghostos = get_ghostos()
+ logger = get_console_logger(debug=True)
+ ghostos.container().set(LoggerItf, logger)
+ configs = ghostos.container().force_fetch(Configs)
+ app_conf = configs.get(OpenAIRealtimeAppConf)
+ # app_conf.listening = False
+ app_conf.ws_conf.proxy = "socks5://127.0.0.1:1080"
+ jojo = Chatbot(
+ name="jojo",
+ description="a chatbot for baseline test",
+ persona="you are an LLM-driven cute girl, named jojo",
+ instruction="remember talk to user with user's language."
+ )
+ shell = ghostos.create_shell("realtime_test")
+ conversation = shell.sync(jojo)
+ realtime_app = RealtimeAppImpl(
+ conf=app_conf,
+ vad_mode=True,
+ conversation=conversation,
+ listener=get_pyaudio_pcm16_listener(),
+ speaker=get_pyaudio_pcm16_speaker(),
+ )
+ listening = False
+
+ with realtime_app:
+ messages = realtime_app.history_messages()
+ logger.info("render history messages")
+ for message in messages:
+ logger.info("render message %r", message)
+
+ while not realtime_app.is_closed():
+ state, operators = realtime_app.state()
+ logger.info("state: %s, operators: %r", state, operators)
+ buffer = realtime_app.output()
+ if buffer is None:
+ time.sleep(0.5)
+ continue
+ logger.info("receive buffer")
+ while buffer is not None:
+ for chunk in buffer.chunks():
+ logger.info("receive chunk %s", chunk.content)
+ tail = buffer.tail()
+ logger.info("receive tail %r", tail)
+ buffer = buffer.next()
diff --git a/ghostos/framework/libraries/__init__.py b/ghostos/prototypes/spherogpt/__init__.py
similarity index 100%
rename from ghostos/framework/libraries/__init__.py
rename to ghostos/prototypes/spherogpt/__init__.py
diff --git a/ghostos/prototypes/spherogpt/bolt/__init__.py b/ghostos/prototypes/spherogpt/bolt/__init__.py
new file mode 100644
index 00000000..94dab8e3
--- /dev/null
+++ b/ghostos/prototypes/spherogpt/bolt/__init__.py
@@ -0,0 +1,16 @@
+try:
+ import bleak
+ import spherov2
+except ImportError:
+ raise ImportError(f"Package bleak or spherov2 not installed. please run `pip install ghostos[sphero]` first")
+from ghostos.prototypes.spherogpt.bolt.ball_impl import SpheroBoltBallAPIProvider
+from ghostos.prototypes.spherogpt.bolt.runtime_impl import ShellSpheroBoltRuntimeProvider
+from ghostos.prototypes.spherogpt.bolt.led_matrix_impl import SpheroBoltLedMatrixProvider
+
+from ghostos.prototypes.spherogpt.bolt.bolt_shell import (
+ RollFunc,
+ Ball,
+ Move,
+ LedMatrix,
+ Animation,
+)
diff --git a/ghostos/prototypes/spherogpt/bolt/ball_impl.py b/ghostos/prototypes/spherogpt/bolt/ball_impl.py
new file mode 100644
index 00000000..61b86665
--- /dev/null
+++ b/ghostos/prototypes/spherogpt/bolt/ball_impl.py
@@ -0,0 +1,330 @@
+from typing import Literal, Optional, Callable, Self, Dict, Tuple
+
+from spherov2.commands.io import FrameRotationOptions
+
+from ghostos.contracts.storage import FileStorage
+from ghostos.contracts.workspace import Workspace
+from ghostos.abcd import Conversation
+from ghostos.entity import ModelEntityMeta, from_entity_model_meta, to_entity_model_meta
+from ghostos.helpers import yaml_pretty_dump
+from ghostos.prompter import Prompter
+from ghostos.container import Container, Provider
+from ghostos.core.moss import Injection, MossRuntime
+from pydantic import BaseModel, Field
+from ghostos.prototypes.spherogpt.bolt.sphero_edu_api_patch import SpheroEventType
+from ghostos.prototypes.spherogpt.bolt.bolt_shell import Ball, Move, RollFunc, Animation
+from ghostos.prototypes.spherogpt.bolt.runtime import SpheroBoltRuntime, BoltBallMovement
+from ghostos.prototypes.spherogpt.bolt.movements import (
+ GroupMovement,
+ RunAPIMovement,
+ CurveRollMovement,
+)
+import yaml
+
+__all__ = ['SpheroBoltBallAPIProvider', 'BallImpl']
+
+
+class SavedMove(BaseModel):
+ name: str = Field(description="move name")
+ description: str = Field(description="move description")
+ move_meta: ModelEntityMeta = Field(description="move meta")
+ generated_code: str = Field(default="", description="the code creating this move")
+
+ @classmethod
+ def new(cls, name: str, description: str, move: BoltBallMovement) -> Self:
+ return SavedMove(
+ name=name,
+ description=description,
+ move_meta=to_entity_model_meta(move),
+ )
+
+ def get_move(self) -> BoltBallMovement:
+ return from_entity_model_meta(self.move_meta)
+
+
+class MovesMemoryCache(BaseModel):
+ moves: Dict[str, SavedMove] = Field(default_factory=dict)
+
+ def add_saved(self, saved: SavedMove):
+ self.moves[saved.name] = saved
+
+ def get_move(self, name: str) -> Optional[BoltBallMovement]:
+ got = self.moves.get(name, None)
+ if got is None:
+ return None
+ return from_entity_model_meta(got.move_meta)
+
+ @staticmethod
+ def filename(unique_id: str) -> str:
+ return f"{unique_id}_sphero_moves.yml"
+
+ def to_content(self) -> str:
+ return yaml_pretty_dump(self.model_dump())
+
+
+class MoveAdapter(Move):
+
+ def __init__(
+ self,
+ runtime: SpheroBoltRuntime,
+ run_immediately: bool,
+ animation: Optional[Animation] = None,
+ event_desc: str = "",
+ buffer: Optional[GroupMovement] = None,
+ ):
+ self._runtime = runtime
+ self._run_immediately = run_immediately
+ self._move_added: int = 0
+ if buffer is None:
+ buffer = GroupMovement(desc="move", event_desc=event_desc or "", animation=animation)
+ if animation is not None:
+ buffer.animation = animation
+ self.buffer: GroupMovement = buffer
+
+ def _add_move(self, movement: BoltBallMovement):
+ if self._run_immediately:
+ movement.stop_at_first = self._move_added == 0
+ self._runtime.add_movement(movement)
+
+ self.buffer.add_child(movement)
+ self._move_added += 1
+
+ def roll(self, heading: int, speed: int, duration: float) -> Self:
+ roll_fn = RollFunc(
+ heading=heading,
+ speed=speed,
+ duration=duration,
+ code="",
+ )
+ move = CurveRollMovement(
+ desc="roll",
+ curve=roll_fn,
+ )
+ self._add_move(move)
+ return self
+
+ def spin(self, angle: int, duration: float) -> Self:
+ self._add_move(RunAPIMovement(
+ desc="spin",
+ method="spin",
+ duration=duration,
+ args=[angle, duration],
+ ))
+ return self
+
+ def set_waddle(self, waddle: bool) -> Self:
+ self._add_move(RunAPIMovement(
+ desc="set_waddle",
+ method="set_waddle",
+ duration=0.0,
+ args=[waddle],
+ ))
+ return self
+
+ def roll_by_func(self, fn: RollFunc) -> Self:
+ self._add_move(CurveRollMovement(
+ desc="roll_curve",
+ curve=fn,
+ ))
+ return self
+
+ def stop_roll(self, heading: int = None) -> Self:
+ self._add_move(RunAPIMovement(
+ desc="stop_roll",
+ method="stop_roll",
+ duration=0.0,
+ args=[heading],
+ ))
+ return self
+
+ def reset_aim(self) -> Self:
+ self._add_move(RunAPIMovement(
+ desc="reset_aim",
+ method="reset_aim",
+ duration=0.0,
+ args=[],
+ ))
+ return self
+
+ def set_compass_direction(self, direction: int = 0) -> Self:
+ self._add_move(RunAPIMovement(
+ desc="reset_aim",
+ method="reset_aim",
+ duration=0.0,
+ args=[],
+ ))
+ return self
+
+ def on_collision(
+ self,
+ callback: Optional[Callable[[Self], None]] = None,
+ *,
+ log: str = "feeling collision",
+ ) -> None:
+ self._add_event_callback(SpheroEventType.on_collision.name, log, callback)
+
+ def _add_event_callback(
+ self,
+ event_name: str,
+ log: str,
+ callback: Optional[Callable[[Self], None]] = None,
+ ) -> None:
+ sub_move = MoveAdapter(
+ runtime=self._runtime,
+ run_immediately=False,
+ event_desc=log,
+ )
+ if callback is not None:
+ callback(sub_move)
+ event_move = sub_move.buffer
+ event_move.stop_at_first = True
+ self.buffer.add_event_move(event_name, event_move)
+
+ def on_freefall(
+ self,
+ log: str = "feeling freefall",
+ *,
+ callback: Optional[Callable[[Self], None]] = None,
+ ) -> None:
+ self._add_event_callback(SpheroEventType.on_freefall.name, log, callback)
+
+ def on_landing(
+ self,
+ callback: Optional[Callable[[Self], None]] = None,
+ *,
+ log: str = "feeling landing",
+ ) -> None:
+ self._add_event_callback(SpheroEventType.on_landing.name, log, callback)
+
+
+class BallImpl(Ball, Injection, Prompter):
+
+ def __init__(
+ self,
+ runtime: SpheroBoltRuntime,
+ memory_cache: FileStorage,
+ executing_code: Optional[str] = None,
+ ):
+ self._runtime = runtime
+ self._executing_code = executing_code
+ self._memory_cache_storage = memory_cache
+ self._memory_cache_file = MovesMemoryCache.filename(self._runtime.get_task_id())
+ if self._memory_cache_storage.exists(self._memory_cache_file):
+ content = self._memory_cache_storage.get(self._memory_cache_file)
+ data = yaml.safe_load(content)
+ self._memory_cache = MovesMemoryCache(**data)
+ else:
+ self._memory_cache = MovesMemoryCache()
+
+ def _save_cache(self):
+ content = self._memory_cache.to_content()
+ self._memory_cache_storage.put(self._memory_cache_file, content.encode())
+
+ def on_inject(self, runtime: MossRuntime, property_name: str) -> Self:
+ self._executing_code = runtime.moss().executing_code
+ return self
+
+ def on_destroy(self) -> None:
+ return None
+
+ def new_move(
+ self,
+ *,
+ animation: Optional[Animation] = None,
+ run: bool = False,
+ ) -> Move:
+ return MoveAdapter(self._runtime, run, animation=animation)
+
+ def run(self, move: Move, stop_at_first: bool = True) -> None:
+ if not isinstance(move, MoveAdapter):
+ raise TypeError(f"move instance must be created by this api new_move()")
+ movement = move.buffer
+ if movement.animation is not None:
+ self._runtime.add_animation(movement.animation)
+ movement.stop_at_first = stop_at_first
+ self._runtime.add_movement(movement)
+
+ def save_move(self, name: str, description: str, move: Move, animation: Optional[Animation] = None) -> None:
+ if not isinstance(move, MoveAdapter):
+ raise TypeError(f"move instance must be created by this api new_move()")
+ if animation:
+ move.buffer.animation = animation
+ saved_move = SavedMove.new(name=name, description=description, move=move.buffer)
+ saved_move.generated_code = self._executing_code or ""
+ self._memory_cache.add_saved(saved_move)
+ self._save_cache()
+
+ def delete_move(self, name: str) -> None:
+ if name in self._memory_cache.moves:
+ del self._memory_cache.moves[name]
+ self._save_cache()
+
+ def set_matrix_rotation(self, rotation: Literal[0, 90, 180, 270] = 0) -> None:
+ rotations = {
+ 0: FrameRotationOptions.NORMAL,
+ 90: FrameRotationOptions.ROTATE_90_DEGREES,
+ 180: FrameRotationOptions.ROTATE_180_DEGREES,
+ 270: FrameRotationOptions.ROTATE_270_DEGREES,
+ }
+ move = RunAPIMovement(
+ desc="set_matrix_rotation",
+ method="set_matrix_rotation",
+ args=[rotations.get(rotation, FrameRotationOptions.NORMAL)]
+ )
+ self._runtime.add_movement(move)
+
+ def run_move(self, name: str) -> None:
+ got = self._memory_cache.get_move(name)
+ if got is None:
+ raise NotImplementedError(f"move {name} not implemented")
+ got.stop_at_first = True
+ self._runtime.add_movement(got)
+
+ def read_move(self, name: str) -> Tuple[Move, str]:
+ saved = self._memory_cache.moves.get(name, None)
+ if saved is None:
+ raise NotImplementedError(f"move {name} not implemented")
+ got = self._memory_cache.get_move(name)
+ move = MoveAdapter(
+ self._runtime,
+ run_immediately=False,
+ buffer=got,
+ )
+ return move, saved.generated_code
+
+ def on_charging(self, log: str = "feeling at charging") -> None:
+ self._runtime.set_charging_callback(log)
+
+ def on_not_charging(self, log: str = "feeling stop charging") -> None:
+ self._runtime.set_off_charging_callback(log)
+
+ def self_prompt(self, container: Container) -> str:
+ if len(self._memory_cache.moves) == 0:
+ return ""
+ lines = []
+ for move in self._memory_cache.moves.values():
+ line = f"- `{move.name}`: {move.description}"
+ lines.append(line)
+ saved_content = "\n".join(lines)
+ return f"""
+your saved moves, from name to description are below:
+{saved_content}
+
+you can run the saved move by it's name.
+"""
+
+ def get_title(self) -> str:
+ return "SpheroBolt Ball saved moves"
+
+
+class SpheroBoltBallAPIProvider(Provider[Ball]):
+
+ def singleton(self) -> bool:
+ return False
+
+ def factory(self, con: Container) -> Optional[Ball]:
+ runtime = con.force_fetch(SpheroBoltRuntime)
+ workspace = con.force_fetch(Workspace)
+ conversation = con.force_fetch(Conversation)
+ runtime.connect(conversation.task_id, False)
+ return BallImpl(runtime, workspace.runtime_cache())
diff --git a/ghostos/prototypes/spherogpt/bolt/bolt_shell.py b/ghostos/prototypes/spherogpt/bolt/bolt_shell.py
new file mode 100644
index 00000000..13db6c40
--- /dev/null
+++ b/ghostos/prototypes/spherogpt/bolt/bolt_shell.py
@@ -0,0 +1,341 @@
+from typing import Self, Optional, Callable, List, Literal, Tuple
+from abc import ABC, abstractmethod
+from pydantic import BaseModel, Field
+
+
+class RollFunc(BaseModel):
+ """
+ to define a curve rolling frame by frame.
+ """
+ heading: int = Field(0, description="Heading angle of the sphero bolt in degrees from -180 ~ 180", ge=-360, le=360)
+ speed: int = Field(90, description="speed of the sphero bolt rolling", ge=0, le=255)
+ duration: float = Field(1, description="duration of the rolling, if 0, means forever")
+ code: str = Field(
+ default="",
+ description="the python code to change self heading, speed, stop at each frame of rolling."
+ "if empty, means run straight",
+ )
+
+ def run_frame(self, passed: float) -> bool:
+ """
+ real logic of the curve rolling.
+ sphero runtime will call `run_frame` method at each frame (for example: 0.01 second)
+ this method shall change the speed and heading at each frame.
+
+ :param passed: the time in seconds that passed since the curve rolling started
+ :return: shall stop?
+ """
+ # the real logic is eval the python code here, change the heading and spead to complete a curve.
+ # for example, a sin cure:
+ # self.speed = 90
+ # self.heading = int(math.sin(passed % 3) * 180)) % 360
+ if self.code:
+ code = "\n".join([line.lstrip() for line in self.code.splitlines()])
+ exec(code)
+ return not (self.duration == 0 or passed < self.duration)
+
+
+class Straight(RollFunc):
+ heading: int = Field(0)
+ speed: int = Field(90)
+ duration: float = Field(1)
+ code: str = ""
+
+ def run_frame(self, passed: float) -> bool:
+ return not (self.duration == 0 or passed < self.duration)
+
+
+class Move(ABC):
+ """
+ to define a sequence of sphero bolt ball movements.
+ you can call several methods in order to define a sequence.
+ the move instance do not execute until it is run by `Ball` interface.
+ """
+
+ @abstractmethod
+ def roll(self, heading: int, speed: int, duration: float) -> Self:
+ """Combines heading(0-360°), speed(-255-255), and duration to make the robot roll with one line of code.
+ For example, to have the robot roll at 90°, at speed 200 for 2s, use ``roll(90, 200, 2)``"""
+ pass
+
+ @abstractmethod
+ def spin(self, angle: int, duration: float) -> Self:
+ """Spins the robot for a given number of degrees over time, with 360° being a single revolution.
+ For example, to spin the robot 360° over 1s, use: ``spin(360, 1)``.
+ Use :func:`set_speed` prior to :func:`spin` to have the robot move in circle or an arc or circle.
+
+ Note: Unlike official API, performance of spin is guaranteed, but may be longer than the specified duration."""
+ pass
+
+ @abstractmethod
+ def set_waddle(self, waddle: bool) -> Self:
+ """Turns the waddle walk on using `set_waddle(True)`` and off using ``set_waddle(False)``."""
+ pass
+
+ @abstractmethod
+ def roll_by_func(self, fn: RollFunc) -> Self:
+ """
+ run a curve rolling frame by frame until it reach the duration.
+ """
+ pass
+
+ @abstractmethod
+ def stop_roll(self, heading: int = None) -> Self:
+ """Sets the speed to zero to stop the robot, effectively the same as the ``set_speed(0)`` command."""
+ pass
+
+ @abstractmethod
+ def reset_aim(self) -> Self:
+ """Resets the heading calibration (aim) angle to use the current direction of the robot as 0°."""
+ pass
+
+ @abstractmethod
+ def set_compass_direction(self, direction: int = 0) -> Self:
+ """
+ Sets the direction relative to compass zero
+ """
+ pass
+
+ # below are events methods. only need call them for certain and clear purpose.
+
+ @abstractmethod
+ def on_collision(
+ self,
+ callback: Optional[Callable[[Self], None]] = None,
+ ) -> None:
+ """
+ when the bolt feeling collision. default is stop.
+ for example:
+ `move.on_collision(lambda m: m.spin(180, 1))` means when collision, spin 180 degree in 1 second.
+ """
+ pass
+
+ @abstractmethod
+ def on_freefall(
+ self,
+ callback: Optional[Callable[[Self], None]] = None,
+ ) -> None:
+ """
+ when the bolt feeling free fall. default is stop.
+ """
+ pass
+
+ @abstractmethod
+ def on_landing(
+ self,
+ callback: Optional[Callable[[Self], None]] = None,
+ ) -> None:
+ """
+ when the bolt feeling landing. default is stop.
+ """
+ pass
+
+
+class Animation(BaseModel):
+ """
+ to define an animation by sequence of frame.
+ the animation will be played on Sphero Bolt 8*8 led matrix.
+ """
+ fps: int = Field(1, description="frames per second", ge=1, le=30),
+ transition: bool = Field(True, description="if true, fade between frames"),
+ palette: List[str] = Field(
+ default_factory=lambda: ["000000", "ff0000", "00ff00", "0000ff", "ffffff"],
+ description="define color palette, the index is the color id. "
+ "in default case: 0 is black, 1 is red, 2 is green, 3 is blue, 4 is white",
+ ),
+ loop: bool = Field(
+ default=True,
+ description="loop count for animation",
+ ),
+ duration: float = Field(default=0.0, description="duration of animation in seconds, clear matrix after animation"),
+ frames: List[List[List[int]]] = Field(
+ default_factory=lambda: [
+ [
+ # a simple smile
+ [0, 0, 0, 0, 0, 0, 0, 0],
+ [0, 1, 1, 0, 0, 1, 1, 0],
+ [1, 0, 0, 0, 0, 0, 0, 1],
+ [0, 0, 0, 0, 0, 0, 0, 0],
+ [0, 0, 0, 0, 0, 0, 0, 0],
+ [0, 1, 0, 0, 0, 0, 1, 0],
+ [0, 0, 1, 1, 1, 1, 0, 0],
+ [0, 0, 0, 0, 0, 0, 0, 0],
+ ]
+ ],
+ description="list of animation frame, every frame is a 8*8 matrix, each element is an palette color index",
+ )
+
+ def add_frame(self, frame: List[List[int]]) -> None:
+ self.frames.append(frame)
+
+ def add_frame_by_node(
+ self,
+ nodes: List[Tuple[int, int, int]],
+ background_color: int = 0,
+ ):
+ """
+ add a frame by declare several nodes only.
+ :param nodes: list of nodes. [(row, col, color), ...]
+ :param background_color: color index from palette
+ """
+ row = [background_color] * 8
+ frame = [row] * 8 # create an empty
+ for node in nodes:
+ row_idx, col_idx, color_idx = node
+ target_row = frame[row_idx]
+ target_row[col_idx] = color_idx
+ self.add_frame(frame)
+
+
+class Ball(ABC):
+ """
+ Sphero bolt body (which is a rolling ball) control interface.
+ """
+
+ @abstractmethod
+ def new_move(
+ self,
+ *,
+ run: bool = False,
+ animation: Optional[Animation] = None,
+ ) -> Move:
+ """
+ create a new Move instance, to define a sequence of movements.
+ :param run: run immediately if True, otherwise the move will not execute until run it.
+ :param animation: if animation is not none, it will be played while run the move.
+ """
+ pass
+
+ @abstractmethod
+ def run(self, move: Move, stop_at_first: bool = True) -> None:
+ """
+ run the bolt ball movement
+ :param move: the Move instance that defined the movements by calling it methods one by one.
+ :param stop_at_first: shall stop any movement of the ball before executing the new move?
+ """
+ pass
+
+ @abstractmethod
+ def save_move(self, name: str, description: str, move: Move, animation: Optional[Animation] = None) -> None:
+ """
+ define a move that you can call it anytime with the name only.
+ **remember** only save the important one
+ :param name: move name
+ :param description: describe the move, in less than 100 words
+ :param move: the Move instance.
+ :param animation: if animation is not none, it will be played while run the move.
+ """
+ pass
+
+ @abstractmethod
+ def read_move(self, name: str) -> Tuple[Move, str]:
+ """
+ read a saved move with the code that generated it.
+ print the code to see details.
+ :param name: move name
+ :return: (move instance, the code that generated it.)
+ """
+ pass
+
+ @abstractmethod
+ def delete_move(self, name: str) -> None:
+ """
+ delete move by name
+ """
+ pass
+
+ @abstractmethod
+ def run_move(self, name: str) -> None:
+ """
+ run a defined move
+ :param name: the name of the move. make sure you have run save_move() before calling it.
+ :raise: NotImplementedError if move is not defined
+ """
+ pass
+
+ @abstractmethod
+ def set_matrix_rotation(self, rotation: Literal[0, 90, 180, 360] = 0) -> None:
+ """
+ Rotates the LED matrix
+ :param rotation: 0 to 90, 180, 360 degrees
+ """
+ pass
+
+
+class LedMatrix(ABC):
+
+ @abstractmethod
+ def pause_animation(self) -> None:
+ """
+ pause the playing animation
+ """
+ pass
+
+ @abstractmethod
+ def resume_animation(self):
+ """
+ resume the playing animation
+ """
+ pass
+
+ @abstractmethod
+ def clear_matrix(self):
+ """
+ clear the matrix.
+ """
+ pass
+
+ @abstractmethod
+ def scroll_matrix_text(self, text: str, color: str = 'ffffff', fps: int = 1, wait: bool = True) -> Self:
+ """
+ Scrolls text on the matrix, with specified color.
+ *this is a better way to print character on matrix*, with it, you do not need to write matrix frame yourself.
+
+ :param text: max 25 characters, only allow char byte in 0~256
+ :param color: color of the char
+ :param fps: 1 to 30
+ :param wait: if the programs wait until completion
+ """
+ pass
+
+ @abstractmethod
+ def set_matrix_character(self, character: str, color: str):
+ """
+ Sets a character on the matrix with color specified
+ :param character: output character
+ :param color: 6 digit hex RGB color, e.g. "ffffff", '00ff00'
+ """
+ pass
+
+
+class SpheroBoltGPT(ABC):
+ """
+ the sphero bolt robot api
+ """
+ body: Ball
+ """your ball body"""
+
+ face: LedMatrix
+ """your ball face"""
+
+ def save_expression(
+ self,
+ name: str,
+ desc: str,
+ builder: Callable[[Ball, LedMatrix], None]
+ ) -> None:
+ """
+ create a named expression that express your feelings and can call it by name later.
+ :param name: name of the expression
+ :param desc: desc of the expression
+ :param builder: define the movement and the animation combined that express your feeling.
+ :return:
+ """
+ pass
+
+ def run(self, expression_name: str) -> None:
+ """
+ run a defined expression
+ :param expression_name: saved expression name
+ """
+ pass
diff --git a/ghostos/prototypes/spherogpt/bolt/led_matrix_impl.py b/ghostos/prototypes/spherogpt/bolt/led_matrix_impl.py
new file mode 100644
index 00000000..7c55fd79
--- /dev/null
+++ b/ghostos/prototypes/spherogpt/bolt/led_matrix_impl.py
@@ -0,0 +1,141 @@
+from typing import List, Dict, Self, Optional, AnyStr
+
+from .sphero_edu_api_patch import SpheroEduAPI
+
+from ghostos.prototypes.spherogpt.bolt.bolt_shell import LedMatrix, Animation
+from ghostos.prototypes.spherogpt.bolt.runtime import BoltLedMatrixCommand, SpheroBoltRuntime
+from ghostos.prototypes.spherogpt.bolt.sphero_edu_api_patch import Color
+from ghostos.container import Container, Provider
+from pydantic import BaseModel, Field
+
+
+def parse_str_to_color(s: AnyStr) -> Color:
+ color = str(s).lower()
+ if not color.startswith("0x"):
+ color = "0x" + color
+ digits = int(color, 0)
+ return Color(digits >> 16, (digits >> 8) % 256, digits % 256)
+
+
+class AnimationMemoryCache(BaseModel):
+ palette: Dict[str, List[int]] = Field(default_factory=dict, description="Palette of colors. name to (r, g, b)")
+
+ def add_palette(self, name: str, color: Color):
+ self.palette[name] = [color.r, color.g, color.b]
+
+
+class ResumeAnimation(BoltLedMatrixCommand):
+
+ def start(self, api: SpheroEduAPI) -> None:
+ api.resume_matrix_animation()
+
+
+class ClearMatrix(BoltLedMatrixCommand):
+
+ def start(self, api: SpheroEduAPI) -> None:
+ api.clear_matrix()
+
+
+class PauseAnimation(BoltLedMatrixCommand):
+
+ def start(self, api: SpheroEduAPI) -> None:
+ api.pause_matrix_animation()
+
+
+class ScrollMatrixText(BoltLedMatrixCommand):
+ text: str = Field(description="the outputting text")
+ color: str = Field(default="ffffff", description="the palette color of the text")
+ fps: int = Field(default=1, description="the fps of the animation")
+ wait: bool = Field(default=True, description="wait for the animation to finish")
+
+ def start(self, api: SpheroEduAPI) -> None:
+ rgb = parse_str_to_color(str(self.color))
+ api.scroll_matrix_text(self.text, rgb, self.fps, self.wait)
+
+
+class SetMatrixChar(BoltLedMatrixCommand):
+ character: str = Field(description="the charactor")
+ color: str = Field(default="ffffff", description="the palette color of the text")
+
+ def start(self, api: SpheroEduAPI) -> None:
+ color = parse_str_to_color(self.color)
+ api.set_matrix_character(self.character, color)
+
+
+class PlayAnimation(BoltLedMatrixCommand):
+ animation: Animation
+
+ def start(self, api: SpheroEduAPI) -> None:
+ frames = self.animation.frames
+ fps = int(self.animation.fps)
+ palette = []
+ for color in self.animation.palette:
+ rgb = parse_str_to_color(color)
+ palette.append(rgb)
+
+ api.register_matrix_animation(
+ frames,
+ fps=fps,
+ palette=palette,
+ transition=bool(self.animation.transition),
+ )
+ aid = api.get_animation_id()
+ api.play_matrix_animation(aid, self.animation.loop)
+
+ def end(self, api: SpheroEduAPI, passed: float) -> bool:
+ duration = self.animation.duration
+ if 0 < duration <= passed:
+ api.clear_matrix()
+ return True
+ return False
+
+
+class LedMatrixImpl(LedMatrix):
+
+ def __init__(self, runtime: SpheroBoltRuntime):
+ self._runtime = runtime
+ self.last_command: Optional[BoltLedMatrixCommand] = None
+
+ def _add_command(self, command: BoltLedMatrixCommand):
+ self._runtime.add_matrix_command(command)
+ self.last_command = command
+
+ def play_animation(self, animation: Animation) -> None:
+ pa = PlayAnimation(animation=animation)
+ self._runtime.add_matrix_command(pa)
+
+ def scroll_matrix_text(self, text: str, color: str = 'ffffff', fps: int = 1, wait: bool = True) -> Self:
+ if len(text) > 25:
+ raise AttributeError("Text length must be less than 25 characters")
+ for char in text:
+ if ord(char) > 255:
+ raise AttributeError("Character must be in range(0, 256)")
+
+ s = ScrollMatrixText(text=text, color_name=color, fps=fps, wait=wait)
+ self._add_command(s)
+ return self
+
+ def set_matrix_character(self, character: str, color: str):
+ s = SetMatrixChar(character=character, color=color)
+ self._add_command(s)
+
+ def pause_animation(self) -> None:
+ self._add_command(PauseAnimation())
+
+ def resume_animation(self):
+ self._add_command(ResumeAnimation())
+
+ def clear_matrix(self):
+ self._add_command(ClearMatrix())
+
+
+class SpheroBoltLedMatrixProvider(Provider[LedMatrix]):
+
+ def singleton(self) -> bool:
+ return False
+
+ def factory(self, con: Container) -> Optional[LedMatrix]:
+ runtime = con.force_fetch(SpheroBoltRuntime)
+ return LedMatrixImpl(
+ runtime=runtime,
+ )
diff --git a/ghostos/prototypes/spherogpt/bolt/movements.py b/ghostos/prototypes/spherogpt/bolt/movements.py
new file mode 100644
index 00000000..a76959af
--- /dev/null
+++ b/ghostos/prototypes/spherogpt/bolt/movements.py
@@ -0,0 +1,97 @@
+from typing import List, Union, Dict, Optional, Self
+from pydantic import BaseModel, Field
+from ghostos.entity import ModelEntityMeta, to_entity_model_meta, from_entity_model_meta
+
+from .runtime import BoltBallMovement
+from .sphero_edu_api_patch import SpheroEduAPI
+from .bolt_shell import RollFunc, Animation
+
+
+class RunAPIMovement(BoltBallMovement):
+ desc: str = Field(description="desc of the movement")
+ method: str = Field(description="sphero edu api name")
+ args: List[Union[str, int, float, None]] = Field(default_factory=list, description="args")
+ kwargs: Dict[str, Union[str, int, float, None]] = Field(default_factory=dict, description="kwargs")
+ duration: float = Field(0.0, description="duration of the movement")
+
+ def start(self, api: SpheroEduAPI) -> None:
+ method = getattr(api, self.method)
+ method(*self.args, **self.kwargs)
+
+ def run_frame(self, api: SpheroEduAPI, passed: float) -> bool:
+ return passed > self.duration
+
+ def on_event(self, event_type: str) -> Optional[Self]:
+ return None
+
+
+class CurveRollMovement(BoltBallMovement):
+ desc: str = Field(description="desc of the movement")
+ curve: RollFunc = Field(description="curve roll")
+ stopped: bool = Field(default=False)
+ error: str = Field("")
+
+ def start(self, api: SpheroEduAPI) -> None:
+ self.run_frame(api, 0)
+
+ def run_frame(self, api: SpheroEduAPI, passed: float) -> bool:
+ self.stopped = self.curve.run_frame(passed)
+ if not self.stopped:
+ api.set_speed(self.curve.speed)
+ api.set_heading(self.curve.heading)
+ return self.stopped
+
+ def on_event(self, event_type: str) -> Optional[Self]:
+ return None
+
+
+class GroupMovement(BoltBallMovement):
+ animation: Optional[Animation] = Field(None)
+ children: List[ModelEntityMeta] = Field(default_factory=list)
+ event_desc: Optional[str] = Field(default=None)
+ event_moves: Dict[str, ModelEntityMeta] = Field(default_factory=dict)
+ __iter_idx__: int = 0
+ __new_child_started__: bool = False
+ __new_child_start_at__: float = 0.0
+
+ def add_child(self, move: BoltBallMovement):
+ meta = to_entity_model_meta(move)
+ self.children.append(meta)
+
+ def get_child(self, idx: int) -> BoltBallMovement:
+ meta = self.children[idx]
+ return from_entity_model_meta(meta)
+
+ def start(self, api: SpheroEduAPI) -> None:
+ if len(self.children) > 0:
+ self.__iter_idx__ = 0
+ child = self.get_child(self.__iter_idx__)
+ child.start(api)
+ self.__new_child_started__ = True
+
+ def run_frame(self, api: SpheroEduAPI, passed: float) -> bool:
+ if self.__iter_idx__ >= len(self.children):
+ return True
+ child = self.get_child(self.__iter_idx__)
+ # start if not started
+ if not self.__new_child_started__:
+ child.start(api)
+ self.__new_child_started__ = True
+
+ child_passed = passed - self.__new_child_start_at__
+ stopped = child.run_frame(api, child_passed)
+ if stopped:
+ self.__iter_idx__ += 1
+ self.__new_child_start_at__ = passed
+ self.__new_child_started__ = False
+ return self.run_frame(api, passed)
+ return False
+
+ def on_event(self, event_type: str) -> Optional[Self]:
+ if event_type in self.event_moves:
+ meta = self.event_moves[event_type]
+ return from_entity_model_meta(meta)
+ return None
+
+ def add_event_move(self, event_type: str, move: BoltBallMovement):
+ self.event_moves[event_type] = to_entity_model_meta(move)
diff --git a/ghostos/prototypes/spherogpt/bolt/runtime.py b/ghostos/prototypes/spherogpt/bolt/runtime.py
new file mode 100644
index 00000000..a5970bc3
--- /dev/null
+++ b/ghostos/prototypes/spherogpt/bolt/runtime.py
@@ -0,0 +1,85 @@
+from abc import ABC, abstractmethod
+from typing import Optional, Self
+from ghostos.entity import ModelEntity
+from pydantic import BaseModel, Field
+from .sphero_edu_api_patch import SpheroEduAPI
+
+from ghostos.prototypes.spherogpt.bolt.bolt_shell import Animation
+
+_STOPPED = bool
+
+
+class BoltBallMovement(BaseModel, ABC):
+ animation: Optional[Animation] = Field(None)
+ desc: str = Field("", description="description of the command")
+ stop_at_first: bool = Field(default=False, description="stop the world at first")
+
+ @abstractmethod
+ def start(self, api: SpheroEduAPI) -> None:
+ pass
+
+ @abstractmethod
+ def run_frame(self, api: SpheroEduAPI, passed: float) -> _STOPPED:
+ pass
+
+ def succeed_log(self, passed: float) -> str:
+ if not self.desc:
+ return ""
+ return f"done `{self.desc}` after {round(passed, 4)} seconds"
+
+ def interrupt_log(self, reason: str, passed: float) -> str:
+ desc = self.desc or str(type(self))
+ return f"interrupt `{desc}` running because `{reason}` after {round(passed, 4)} seconds"
+
+ @abstractmethod
+ def on_event(self, event_type: str) -> Optional[Self]:
+ pass
+
+
+class BoltLedMatrixCommand(ModelEntity, ABC):
+
+ @abstractmethod
+ def start(self, api: SpheroEduAPI) -> None:
+ pass
+
+ def end(self, api: SpheroEduAPI, passed: float) -> bool:
+ return True
+
+
+class SpheroBoltRuntime(ABC):
+
+ @abstractmethod
+ def get_task_id(self) -> str:
+ pass
+
+ @abstractmethod
+ def connect(self, task_id: str, shall_notify: bool):
+ pass
+
+ @abstractmethod
+ def add_movement(self, command: BoltBallMovement):
+ pass
+
+ @abstractmethod
+ def add_animation(self, animation: Animation) -> None:
+ pass
+
+ @abstractmethod
+ def set_charging_callback(self, event: str):
+ pass
+
+ @abstractmethod
+ def set_off_charging_callback(self, event: str):
+ pass
+
+ @abstractmethod
+ def add_matrix_command(self, command: BoltLedMatrixCommand):
+ pass
+
+ @abstractmethod
+ def bootstrap(self):
+ pass
+
+ @abstractmethod
+ def close(self):
+ pass
diff --git a/ghostos/prototypes/spherogpt/bolt/runtime_impl.py b/ghostos/prototypes/spherogpt/bolt/runtime_impl.py
new file mode 100644
index 00000000..823db7a2
--- /dev/null
+++ b/ghostos/prototypes/spherogpt/bolt/runtime_impl.py
@@ -0,0 +1,328 @@
+from typing import Optional, Callable, Type, ClassVar, Self
+import time
+
+from ghostos.contracts.logger import LoggerItf
+from ghostos.core.messages import MessageType
+from ghostos.core.runtime import EventBus, Event, EventTypes as GhostOSEventTypes
+from ghostos.container import Container, BootstrapProvider, INSTANCE
+from ghostos.abcd import Conversation
+from ghostos.helpers import Timeleft
+from threading import Thread, Event, Lock
+from collections import deque
+
+from .bolt_shell import Animation
+from .sphero_edu_api_patch import SpheroEduAPI, Color, SpheroEventType, scanner
+from .runtime import SpheroBoltRuntime, BoltLedMatrixCommand, BoltBallMovement
+from .led_matrix_impl import PlayAnimation
+
+__all__ = ['SpheroBoltRuntimeImpl', 'ShellSpheroBoltRuntimeProvider']
+
+
+class SpheroBoltRuntimeImpl(SpheroBoltRuntime):
+ __instance__: ClassVar[Optional[Self]] = None
+
+ def __init__(
+ self,
+ *,
+ eventbus: EventBus,
+ logger: LoggerItf,
+ ):
+ self._task_id = ""
+ self._shall_notify = False
+ self._eventbus = eventbus
+ self._logger = logger
+ self._stopped = Event()
+ self._bootstrapped: Event = Event()
+ self._closed = False
+ self._error: Optional[str] = None
+ self._main_thread = Thread(target=self._main_thread)
+ self._move_queue = deque()
+ self._animation_queue = deque()
+ self._current_animation: Optional[BoltLedMatrixCommand] = None
+ self._current_animation_timeleft: Optional[Timeleft] = None
+ self._clear_matrix_timeleft: Optional[Timeleft] = None
+ self._current_movement: Optional[BoltBallMovement] = None
+ self._current_movement_timeleft: Optional[Timeleft] = None
+ self._movement_mutex = Lock()
+ self._charging_callback: str = "feeling at charging"
+ self._breathing: bool = False
+ self._moving: bool = False
+ self._off_charging_callback: str = "feeling stop charging"
+ SpheroBoltRuntimeImpl.__instance__ = self
+
+ @classmethod
+ def singleton(cls) -> Optional[Self]:
+ if cls.__instance__ and not cls.__instance__._closed:
+ return cls.__instance__
+ return None
+
+ def _reset_all_state(self):
+ self._current_animation = None
+ self._current_animation_timeleft = None
+ self._current_movement = None
+ self._current_movement_timeleft = None
+ self._animation_queue.clear()
+ self._move_queue.clear()
+
+ def bootstrap(self):
+ if self._bootstrapped.is_set():
+ self._logger.error(f"SpheroBolt Runtime already bootstrapped")
+ return
+ self._main_thread.start()
+ self._bootstrapped.wait(10)
+ if not self._bootstrapped.is_set():
+ raise RuntimeError(f'SpheroBolt Runtime bootstrap failed')
+
+ def _main_thread(self):
+ connected_error = 0
+ while not self._stopped.is_set():
+ try:
+ self._logger.info("SpheroBolt Bootstrap started")
+ _bolt = scanner.find_BOLT()
+ self._logger.info("SpheroBolt Bootstrap connected")
+ connected_error = 0
+ # run the loop until errors.
+ try:
+ if not self._bootstrapped.is_set():
+ # make sure no errors
+ self._bootstrapped.wait(2)
+ self._bootstrapped.set()
+ self._run_bolt_loop(_bolt)
+ except Exception as exc:
+ self._logger.exception(exc)
+ self._send_event(GhostOSEventTypes.ERROR, "error occur during runtime: %s" % str(exc))
+ self._reset_all_state()
+ continue
+
+ except Exception as e:
+ self._logger.exception(e)
+ self._logger.info("SpheroBolt Bootstrap failed")
+ connected_error += 1
+ if connected_error > 3:
+ self._stopped.set()
+ self._error = 'failed to connected SpheroBolt'
+ self._send_event(GhostOSEventTypes.ERROR, "sphero bolt failed to connected")
+ raise RuntimeError(self._error)
+
+ def _strobe(self, api: SpheroEduAPI, passed: float):
+ if self._moving:
+ return
+ if int(passed) % 6 < 3:
+ if not self._breathing:
+ api.set_front_led(Color(0, 5, 0))
+ api.set_back_led(Color(0, 5, 0))
+ self._breathing = True
+ elif self._breathing:
+ api.set_front_led(Color(0, 0, 0))
+ api.set_back_led(Color(0, 0, 0))
+ self._breathing = False
+
+ def _set_current_animation(self, animation: BoltLedMatrixCommand, api: SpheroEduAPI):
+ animation.start(api)
+ self._current_animation = animation
+ self._current_animation_timeleft = Timeleft(0)
+
+ def _check_end_of_animation(self, api: SpheroEduAPI):
+ if self._current_animation is None or self._current_animation_timeleft is None:
+ return
+ if self._current_animation.end(api, self._current_animation_timeleft.passed()):
+ self._current_animation = None
+ self._current_animation_timeleft = None
+
+ def _run_bolt_loop(self, _bolt):
+ start_at = Timeleft(0)
+ with SpheroEduAPI(_bolt) as api:
+ self._init_sphero_edu_api(api)
+ while not self._stopped.is_set():
+ if len(self._animation_queue) > 0:
+ self._current_animation = None
+ self._current_animation_timeleft = None
+ animation_command: Optional[BoltLedMatrixCommand] = self._animation_queue.popleft()
+ # animation command execute immediately
+ self._set_current_animation(animation_command, api)
+
+ # trigger end of animation.
+ self._check_end_of_animation(api)
+
+ if self._current_movement is None:
+ movement = self._get_new_movement()
+ if movement is not None:
+ self._set_current_movement(movement, api)
+ else:
+ self._strobe(api, start_at.passed())
+ time.sleep(0.5)
+ continue
+ else:
+ passed = self._current_movement_timeleft.passed()
+ stopped = self._current_movement.run_frame(api, passed)
+ if stopped:
+ self._clear_current_movement(api, notify=False)
+
+ def _init_sphero_edu_api(self, api):
+ api.register_event(SpheroEventType.on_landing, self._on_landing)
+ api.register_event(SpheroEventType.on_freefall, self._on_freefall)
+ api.register_event(SpheroEventType.on_collision, self._on_collision)
+ api.register_event(SpheroEventType.on_charging, self._on_charging)
+ api.register_event(SpheroEventType.on_not_charging, self._on_off_charging)
+
+ def _on_collision(self, api: SpheroEduAPI, *args, **kwargs):
+ self._on_event_handler(api, SpheroEventType.on_collision.name)
+
+ def _on_event_handler(self, api: SpheroEduAPI, event_name: str):
+ api.stop_roll()
+ try:
+ if self._current_movement is not None:
+ move = self._current_movement.on_event(event_name)
+ self._clear_current_movement(api, event_name, notify=True)
+ if move is not None:
+ self._send_event(GhostOSEventTypes.NOTIFY, move.event_desc)
+ self._set_current_movement(move, api)
+ except Exception as e:
+ self._logger.exception(e)
+ api.stop_roll()
+
+ def _on_landing(self, api: SpheroEduAPI, *args, **kwargs):
+ self._on_event_handler(api, SpheroEventType.on_landing.name)
+
+ def _on_freefall(self, api: SpheroEduAPI):
+ self._on_event_handler(api, SpheroEventType.on_freefall.name)
+
+ def _on_charging(self, api: SpheroEduAPI):
+ api.stop_roll()
+ self._clear_current_movement(api, SpheroEventType.on_charging.name, notify=False)
+ self._send_event(GhostOSEventTypes.NOTIFY, self._charging_callback)
+
+ def _on_off_charging(self, api: SpheroEduAPI):
+ api.stop_roll()
+ self._clear_current_movement(api, SpheroEventType.on_not_charging.name, notify=False)
+ self._send_event(GhostOSEventTypes.NOTIFY, self._off_charging_callback)
+
+ def _default_on_event(self, event_type: SpheroEventType, api: SpheroEduAPI):
+ api.stop_roll()
+ api.clear_matrix()
+ if event_type == SpheroEventType.on_charging:
+ self._send_event(GhostOSEventTypes.NOTIFY, self._charging_callback)
+ elif event_type == SpheroEventType.on_not_charging:
+ self._send_event(GhostOSEventTypes.NOTIFY, self._off_charging_callback)
+ return
+
+ def _set_current_movement(self, movement: BoltBallMovement, api: SpheroEduAPI):
+ if movement is None:
+ return
+
+ with self._movement_mutex:
+ self._current_movement = movement
+ self._current_movement_timeleft = Timeleft(0)
+ self._moving = True
+ # always clear matrix at first.
+ if movement.animation is not None:
+ api.clear_matrix()
+ pa = PlayAnimation(animation=movement.animation)
+ self._set_current_animation(pa, api)
+
+ api.set_front_led(Color(0, 10, 0))
+ self._logger.debug("start new movement %r", movement)
+ movement.start(api)
+
+ def _clear_current_movement(self, api: SpheroEduAPI, interrupt: Optional[str] = None, notify: bool = True):
+ with self._movement_mutex:
+ self._moving = False
+ if self._current_movement is None or self._current_movement_timeleft is None:
+ return
+ animation = self._current_movement.animation
+ movement = self._current_movement
+ timeleft = self._current_movement_timeleft
+ self._current_movement = None
+ self._current_movement_timeleft = None
+ api.stop_roll()
+ api.set_front_led(Color(0, 0, 0))
+ if animation is not None:
+ api.clear_matrix()
+ if notify:
+ if not interrupt:
+ log = movement.succeed_log(timeleft.passed())
+ if log:
+ self._send_event(GhostOSEventTypes.NOTIFY, log)
+ else:
+ log = movement.interrupt_log(interrupt, timeleft.passed())
+ if log:
+ self._send_event(GhostOSEventTypes.NOTIFY, log)
+
+ def _get_new_movement(self) -> Optional[BoltBallMovement]:
+ if len(self._move_queue) == 0:
+ return None
+ movement: BoltBallMovement = self._move_queue.popleft()
+ return movement
+
+ def _send_event(self, event_type: GhostOSEventTypes, content: str):
+ if not self._task_id:
+ return
+ event = event_type.new(
+ task_id=self._task_id,
+ messages=[MessageType.TEXT.new_system(content=content)],
+ callback=True,
+ )
+ self._eventbus.send_event(event, self._shall_notify)
+
+ def add_movement(self, move: BoltBallMovement):
+ if move.stop_at_first:
+ self._move_queue.clear()
+ self._move_queue.append(move)
+ else:
+ self._move_queue.append(move)
+
+ def add_animation(self, animation: Animation) -> None:
+ pa = PlayAnimation(animation=animation)
+ self.add_matrix_command(pa)
+
+ def add_matrix_command(self, command: BoltLedMatrixCommand):
+ self._animation_queue.append(command)
+
+ def close(self):
+ if self._closed:
+ return
+ self._closed = True
+ self._stopped.set()
+
+ def get_task_id(self) -> str:
+ return self._task_id
+
+ def connect(self, task_id: str, shall_notify: bool):
+ if self._task_id and task_id != self._task_id:
+ raise RuntimeError(f"Sphero already connected to task {task_id}, one conversation at a time!")
+ self._task_id = task_id
+ self._shall_notify = shall_notify
+
+ def set_charging_callback(self, event: str):
+ self._charging_callback = event
+
+ def set_off_charging_callback(self, event: str):
+ self._off_charging_callback = event
+
+
+class ShellSpheroBoltRuntimeProvider(BootstrapProvider):
+
+ def contract(self) -> Type[INSTANCE]:
+ return SpheroBoltRuntime
+
+ def bootstrap(self, container: Container) -> None:
+ runtime = container.force_fetch(SpheroBoltRuntime)
+ runtime.bootstrap()
+ container.add_shutdown(runtime.close)
+
+ def singleton(self) -> bool:
+ return True
+
+ def inheritable(self) -> bool:
+ return False
+
+ def factory(self, con: Container) -> Optional[SpheroBoltRuntime]:
+ if singleton := SpheroBoltRuntimeImpl.singleton():
+ return singleton
+ logger = con.force_fetch(LoggerItf)
+ logger.error("runtime bootstrap at container %s", con.bloodline)
+ eventbus = con.force_fetch(EventBus)
+ return SpheroBoltRuntimeImpl(
+ eventbus=eventbus,
+ logger=logger,
+ )
diff --git a/ghostos/prototypes/spherogpt/bolt/sphero_edu_api_patch.py b/ghostos/prototypes/spherogpt/bolt/sphero_edu_api_patch.py
new file mode 100644
index 00000000..8c1188c7
--- /dev/null
+++ b/ghostos/prototypes/spherogpt/bolt/sphero_edu_api_patch.py
@@ -0,0 +1,30 @@
+import struct
+from spherov2.commands.sensor import Sensor, CollisionDetected
+
+__all__ = ["SpheroEduAPI", "SpheroEventType", "Color", "scanner"]
+
+
+def __collision_detected_notify_helper(listener, packet):
+ """
+ 解决 Spherov2 解码 bolt 的 bug?
+ """
+ unpacked = struct.unpack('>3hB3hBH', packet.data)
+ listener(CollisionDetected(acceleration_x=unpacked[0] / 4096, acceleration_y=unpacked[1] / 4096,
+ acceleration_z=unpacked[2] / 4096, x_axis=bool(unpacked[3] & 1),
+ y_axis=bool(unpacked[3] & 2), power_x=unpacked[4], power_y=unpacked[5],
+ power_z=unpacked[6], speed=unpacked[7], time=unpacked[8] / 1000))
+
+
+Sensor.collision_detected_notify = (24, 18, 0xff), __collision_detected_notify_helper
+
+from spherov2 import scanner
+from spherov2.sphero_edu import SpheroEduAPI as Api, EventType as SpheroEventType, Color
+
+
+class SpheroEduAPI(Api):
+
+ def get_animation_id(self) -> int:
+ _id = self.__animation_index - 1
+ if _id < 0:
+ return 0
+ return _id
diff --git a/ghostos/prototypes/spherogpt/bolt/test_main.py b/ghostos/prototypes/spherogpt/bolt/test_main.py
new file mode 100644
index 00000000..8237cb03
--- /dev/null
+++ b/ghostos/prototypes/spherogpt/bolt/test_main.py
@@ -0,0 +1,41 @@
+from ghostos.prototypes.spherogpt.bolt.runtime import SpheroBoltRuntime
+from ghostos.prototypes.spherogpt.bolt.runtime_impl import SpheroBoltRuntimeImpl
+from ghostos.prototypes.spherogpt.bolt.bolt_shell import Ball, RollFunc
+from ghostos.prototypes.spherogpt.bolt.ball_impl import SpheroBoltBallAPIProvider
+from ghostos.framework.eventbuses import MemEventBusImpl, EventBus
+from ghostos.framework.workspaces import BasicWorkspace
+from ghostos.framework.storage import MemStorage
+from ghostos.contracts.logger import get_console_logger
+from ghostos.contracts.workspace import Workspace
+from ghostos.container import Container
+
+if __name__ == "__main__":
+ eventbus = MemEventBusImpl()
+ logger = get_console_logger()
+ _runtime = SpheroBoltRuntimeImpl(task_id="task_id", eventbus=eventbus, logger=logger)
+ container = Container()
+ storage = MemStorage()
+ _workspace = BasicWorkspace(storage)
+ container.set(EventBus, eventbus)
+ container.set(SpheroBoltRuntime, _runtime)
+ container.set(Workspace, _workspace)
+ container.register(SpheroBoltBallAPIProvider())
+ container.bootstrap()
+ _runtime.bootstrap()
+
+ ball = container.get(Ball)
+
+ # test command
+
+ move = ball.new_move()
+ curve = RollFunc(
+ heading=0,
+ speed=90,
+ duration=6,
+ code="""
+ self.speed = 90
+ self.heading = int((passed % 6) * 60) # Change heading to create a circular path
+ """
+ )
+ move.roll_by_func(curve)
+ ball.run(move)
diff --git a/ghostos/prototypes/spherogpt/bolt_command_control.py b/ghostos/prototypes/spherogpt/bolt_command_control.py
new file mode 100644
index 00000000..2e6efdf3
--- /dev/null
+++ b/ghostos/prototypes/spherogpt/bolt_command_control.py
@@ -0,0 +1,297 @@
+import time
+
+try:
+ import spherov2
+except ImportError:
+ exit("This script requires the spherov2 to be installed.")
+
+from spherov2 import scanner
+from spherov2.types import Color, ToyType
+
+from abc import ABC, abstractmethod
+from typing import Optional, List
+from spherov2.sphero_edu import SpheroEduAPI
+from pydantic import BaseModel, Field
+from ghostos.helpers import Timeleft
+from ghostos.abcd import Conversation
+from ghostos.contracts.logger import LoggerItf
+from ghostos.core.runtime import EventBus, EventTypes
+from ghostos.core.messages import MessageType
+from ghostos.core.moss.prompts import reflect_class_with_methods, get_prompt
+from ghostos.container import BootstrapProvider, Container
+from threading import Thread
+
+__all__ = [
+ 'Command',
+ 'SpheroBolt',
+ 'SpheroEduAPI',
+ 'SpheroBoltProvider',
+ 'exports',
+]
+
+
+class Command(BaseModel):
+ """
+ Sphero Bolt Command that execute frame by frame in time.
+ """
+ name: str = Field("", description="aim of the command in simple words")
+ duration: float = Field(
+ default=0.0,
+ description="the command running duration in seconds. "
+ "after the duration is reached, next command will be executed."
+ )
+ run_every: bool = Field(False, description="if True, the command run every frame")
+ code: str = Field(description="the command code to execute in the sphero bolt runtime.")
+
+ def run_frame(self, api: SpheroEduAPI, passed: float, frame: int) -> None:
+ """
+ run a single frame every tick
+ :param api: SpheroEduAPI that control the sphero bolt
+ :param passed: the passed time from command start to now
+ :param frame: the frame number, frame == 0 means the command is starting
+
+ for example, if you want roll at a special curve,
+ you shall change head angle by passed time at each frame,
+ """
+ # import types in case you need.
+ from spherov2.types import Color, ToyType
+ # eval the python code defined in the command.
+ # this is how the command work
+ for line in self.code.splitlines():
+ line = line.strip()
+ if line:
+ eval(line)
+
+ @classmethod
+ def once(cls, name: str, code: str, duration: float):
+ """
+ run only once, wait until duration is out
+ """
+ return cls(name=name, code=code, duration=duration, run_every=False)
+
+
+class SpheroBolt(ABC):
+ """
+ Sphero Bolt interface
+ Notice you can only run sphero by Command.
+ """
+
+ @abstractmethod
+ def run(self, *commands: Command) -> None:
+ """
+ run command on sphero bolt. will always stop movement at beginning and end of the execution time.
+ :param commands: the commands to execute in order
+ """
+ pass
+
+
+class SpheroBoltImpl(SpheroBolt):
+
+ def __init__(
+ self,
+ logger: LoggerItf,
+ eventbus: EventBus,
+ task_id: str,
+ notify: bool,
+ tick_interval: float = 0.01,
+ ):
+ self._logger: LoggerItf = logger
+ self._executing_command: Optional[Command] = None
+ self._command_stack: List[Command] = []
+ self._timeleft: Optional[Timeleft] = None
+ self._executing: bool = False
+ self._task_id: str = task_id
+ self._notify: bool = notify
+ self._eventbus = eventbus
+ self._destroyed = False
+ self._main_thread: Optional[Thread] = None
+ self._tick_interval = tick_interval
+ self._ticked_frames: int = 0
+ self._error = None
+
+ def bootstrap(self):
+ try:
+ self._logger.info("SpheroBolt Bootstrap started")
+ _bolt = scanner.find_BOLT()
+ self._main_thread = Thread(target=self._main, args=(_bolt,))
+ self._main_thread.start()
+ except Exception as e:
+ raise NotImplementedError("Could not find the Bolt device. " + str(e))
+
+ def destroy(self) -> None:
+ if self._destroyed:
+ return
+ self._destroyed = True
+ self._logger.info("SpheroBolt Bootstrap destroying")
+ if self._main_thread is not None:
+ self._main_thread.join()
+ del self._logger
+ del self._eventbus
+
+ def __del__(self):
+ self.destroy()
+
+ def _clear_command(self, clear_all: bool):
+ self._executing_command = None
+ self._timeleft = None
+ self._ticked_frames = 0
+ self._executing = False
+ if clear_all:
+ self._command_stack = []
+
+ def _command_succeeded(self):
+ self._reset_command_at("succeeded", successful=True, clear_all=False)
+
+ def _reset_command_at(self, action: str, successful: bool, clear_all: bool):
+ if self._executing_command is None or self._timeleft is None:
+ return
+ name = self._executing_command.name
+ passed = self._timeleft.passed()
+ content = f"sphero bolt: command `{name}` {action} after running `{round(passed, 4)}` second"
+ self._clear_command(clear_all)
+ event_type = EventTypes.NOTIFY if successful else EventTypes.ERROR
+ event = event_type.new(
+ task_id=self._task_id,
+ messages=[MessageType.TEXT.new_system(content=content)],
+ callback=True,
+ )
+ self._eventbus.send_event(event, self._notify)
+
+ def _main(self, bolt) -> None:
+ while not self._destroyed:
+ self._logger.info("SpheroBolt toy connected")
+ try:
+ self._run_toy(bolt)
+ except Exception as e:
+ self._logger.error(str(e))
+ self._logger.info("SpheroBolt toy reconnecting")
+ self.destroy()
+
+ def _run_toy(self, toy) -> None:
+ with SpheroEduAPI(toy) as api:
+ while not self._destroyed:
+ try:
+ if self._executing_command and self._timeleft:
+ api.set_front_led(Color(0, 100, 0))
+ has_duration = self._executing_command.duration > 0
+ must_run = self._ticked_frames == 0
+ run_every = self._executing_command.run_every
+ if must_run or (self._timeleft.left() > 0 and run_every):
+ self._executing_command.run_frame(
+ api,
+ self._timeleft.passed(),
+ self._ticked_frames,
+ )
+ self._ticked_frames += 1
+ if has_duration:
+ time.sleep(self._tick_interval)
+ continue
+ else:
+ self._command_succeeded()
+ api.set_front_led(Color(0, 0, 0))
+ continue
+ elif len(self._command_stack) > 0:
+ current: Command = self._command_stack.pop(0)
+ self._executing = True
+ self._executing_command = current
+ self._timeleft = Timeleft(current.duration)
+ self._ticked_frames = 0
+ else:
+ time.sleep(0.5)
+ except Exception as e:
+ self._logger.exception(e)
+ self._reset_command_at(
+ f"stopped because of error {e}",
+ successful=False,
+ clear_all=True,
+ )
+ self._logger.info("SpheroBolt start to stop")
+ self._logger.info("SpheroBolt stopped")
+
+ def run(self, *commands: Command) -> None:
+ if self._error:
+ raise RuntimeError(self._error)
+ if self._executing:
+ self._reset_command_at("stop during new command", successful=True, clear_all=True)
+ commands = list(commands)
+ if len(commands) == 0:
+ return
+ self._command_stack = commands
+
+
+class SpheroBoltProvider(BootstrapProvider):
+ """
+ Sphero Bolt Provider interface
+ """
+
+ def singleton(self) -> bool:
+ return True
+
+ def contract(self):
+ return SpheroBolt
+
+ def factory(self, con: Container) -> Optional[SpheroBolt]:
+ conversation = con.force_fetch(Conversation)
+ eventbus = con.force_fetch(EventBus)
+ logger = con.force_fetch(LoggerItf)
+ task = conversation.get_task()
+ return SpheroBoltImpl(
+ logger,
+ eventbus,
+ task_id=task.task_id,
+ notify=task.shall_notify(),
+ tick_interval=0.01,
+ )
+
+ @staticmethod
+ def bootstrap(container: Container) -> None:
+ sphero_bolt = container.force_fetch(SpheroBolt)
+ if isinstance(sphero_bolt, SpheroBoltImpl):
+ container.add_shutdown(sphero_bolt.destroy)
+ sphero_bolt.bootstrap()
+
+
+exports = {
+ Command.__name__: get_prompt(Command),
+ SpheroBolt.__name__: get_prompt(SpheroBolt),
+ SpheroEduAPI.__name__: reflect_class_with_methods(SpheroEduAPI),
+}
+
+if __name__ == "__exports__":
+ from ghostos.helpers import yaml_pretty_dump
+
+ print(yaml_pretty_dump(exports))
+
+if __name__ == "__main__":
+ from ghostos.framework.eventbuses import MemEventBusImpl
+ from ghostos.contracts.logger import get_console_logger
+
+ _logger = get_console_logger()
+ _eventbus = MemEventBusImpl()
+ sb = SpheroBoltImpl(_logger, _eventbus, "task_id", False)
+ sb.bootstrap()
+
+ # class TestCommand(Command):
+ # code: str = ""
+ #
+ # def run_frame(self, api: SpheroEduAPI, passed: float, frame: int) -> None:
+ # api.roll(0, 100, 1)
+ # api.roll(90, 100, 1)
+ # api.roll(180, 100, 1)
+ # api.roll(270, 100, 1)
+ # api.roll(0, 100, 1)
+ c = Command(
+ name="roll in a circle",
+ code="""
+api.set_speed(100)
+api.roll(0, 100, 1)
+api.roll(90, 100, 1)
+api.roll(180, 100, 1)
+api.roll(270, 100, 1)
+api.roll(360, 100, 1)
+api.set_speed(0)
+""",
+ duration=5
+ )
+
+ sb.run(c)
diff --git a/ghostos/prototypes/streamlitapp/__init__.py b/ghostos/prototypes/streamlitapp/__init__.py
new file mode 100644
index 00000000..0162a5fd
--- /dev/null
+++ b/ghostos/prototypes/streamlitapp/__init__.py
@@ -0,0 +1,3 @@
+from ghostos.prototypes.streamlitapp.main import main_run
+from ghostos.prototypes.streamlitapp.utils.session import Singleton
+from ghostos.prototypes.streamlitapp.pages.router import *
diff --git a/ghostos/framework/runtimes/__init__.py b/ghostos/prototypes/streamlitapp/cli/__init__.py
similarity index 100%
rename from ghostos/framework/runtimes/__init__.py
rename to ghostos/prototypes/streamlitapp/cli/__init__.py
diff --git a/ghostos/prototypes/streamlitapp/cli/helloworld.py b/ghostos/prototypes/streamlitapp/cli/helloworld.py
new file mode 100644
index 00000000..dea0fcb7
--- /dev/null
+++ b/ghostos/prototypes/streamlitapp/cli/helloworld.py
@@ -0,0 +1,11 @@
+import streamlit as st
+from ghostos.prototypes.streamlitapp.main import main_run, Singleton
+from sys import argv
+
+if len(argv) < 2:
+ raise SystemExit("invalid arguments")
+
+argument = argv[1]
+
+st.title("Hello World")
+st.write(argument)
diff --git a/ghostos/prototypes/streamlitapp/cli/run_aifunc_app.py b/ghostos/prototypes/streamlitapp/cli/run_aifunc_app.py
new file mode 100644
index 00000000..3515822a
--- /dev/null
+++ b/ghostos/prototypes/streamlitapp/cli/run_aifunc_app.py
@@ -0,0 +1,44 @@
+from ghostos.helpers import create_and_bind_module
+from ghostos.scripts.cli.run_aifunc import RunAIFuncApp
+from ghostos.bootstrap import get_container
+from ghostos.prototypes.streamlitapp.main import main_run
+from ghostos.prototypes.streamlitapp.pages.router import default_router, AIFuncDetailRoute
+from ghostos.prototypes.streamlitapp.utils.session import Singleton
+from ghostos.contracts.logger import get_console_logger
+import streamlit as st
+import sys
+import json
+
+if len(sys.argv) < 2:
+ raise SystemExit(f"invalid RunAIFuncApp arguments")
+
+
+def bootstrap():
+ logger = get_console_logger()
+ run_aifunc_app_arg = sys.argv[1]
+ data = json.loads(run_aifunc_app_arg)
+
+ app_arg = RunAIFuncApp(**data)
+
+ if app_arg.is_temp:
+ # create temp module
+ logger.debug(f"Create Temp module {app_arg.modulename}")
+ create_and_bind_module(app_arg.modulename, app_arg.filename)
+
+ # bootstrap container
+ logger.debug(f"generate ghostos app container at workspace {app_arg.workspace_dir}")
+ container = get_container()
+
+ # bound route.
+ page_route = AIFuncDetailRoute(aifunc_id=app_arg.import_path)
+ # initialize router and set aifunc is default
+ router = default_router().with_current(page_route)
+
+ st.session_state["hello"] = "world"
+ return [
+ Singleton(container),
+ Singleton(router),
+ ]
+
+
+main_run(bootstrap)
diff --git a/ghostos/prototypes/streamlitapp/cli/run_configs.py b/ghostos/prototypes/streamlitapp/cli/run_configs.py
new file mode 100644
index 00000000..a95fb952
--- /dev/null
+++ b/ghostos/prototypes/streamlitapp/cli/run_configs.py
@@ -0,0 +1,13 @@
+from ghostos.prototypes.streamlitapp import main_run, Singleton, default_router, ConfigsRoute
+from ghostos.bootstrap import get_container
+
+
+def bootstrap():
+ router = default_router().with_current(ConfigsRoute())
+ return [
+ Singleton(get_container()),
+ Singleton(router),
+ ]
+
+
+main_run(bootstrap)
diff --git a/ghostos/prototypes/streamlitapp/cli/run_ghost_chat.py b/ghostos/prototypes/streamlitapp/cli/run_ghost_chat.py
new file mode 100644
index 00000000..837f20a3
--- /dev/null
+++ b/ghostos/prototypes/streamlitapp/cli/run_ghost_chat.py
@@ -0,0 +1,93 @@
+from typing import List
+
+from ghostos.core.messages import Message
+from ghostos.core.runtime import Event
+from ghostos.contracts.logger import get_ghostos_logger
+from ghostos.helpers import create_and_bind_module
+from ghostos.scripts.cli.run_streamlit_app import RunGhostChatApp
+from ghostos.bootstrap import get_ghostos, get_container
+from ghostos.prototypes.streamlitapp.main import main_run
+from ghostos.prototypes.streamlitapp.pages.router import default_router, GhostChatRoute
+from ghostos.prototypes.streamlitapp.utils.session import Singleton
+from ghostos.abcd import Shell, Background
+from ghostos.abcd.utils import get_module_magic_shell_providers
+import importlib
+import streamlit as st
+import sys
+import json
+
+if len(sys.argv) < 2:
+ raise SystemExit(f"invalid RunAIFuncApp arguments")
+
+logger = get_ghostos_logger()
+
+
+class StreamlitBackgroundApp(Background):
+
+ def on_error(self, error: Exception) -> bool:
+ logger.exception(error)
+ return True
+
+ def on_event(self, event: Event, messages: List[Message]) -> None:
+ pass
+
+ def alive(self) -> bool:
+ from streamlit.runtime import Runtime, RuntimeState
+ state = Runtime.instance().state
+ # if streamlit is closed, close ghostos shell
+ return state not in {RuntimeState.STOPPING, RuntimeState.STOPPED}
+
+ def halt(self) -> int:
+ return 0
+
+
+def bootstrap():
+ run_aifunc_app_arg = sys.argv[1]
+ data = json.loads(run_aifunc_app_arg)
+
+ app_arg = RunGhostChatApp(**data)
+
+ if app_arg.is_temp:
+ # create temp module
+ logger.debug(f"Create Temp module {app_arg.modulename}")
+ started_module = create_and_bind_module(app_arg.modulename, app_arg.filename)
+ else:
+ started_module = importlib.import_module(app_arg.modulename)
+
+ providers = get_module_magic_shell_providers(started_module)
+
+ # bootstrap container
+ logger.debug(f"generate ghostos app container at workspace {app_arg.workspace_dir}")
+
+ # bound route.
+ page_route = GhostChatRoute(
+ ghost_meta=app_arg.ghost_meta,
+ context_meta=app_arg.context_meta,
+ filename=app_arg.filename,
+ )
+ page_route = page_route.get_or_bind(st.session_state)
+ # initialize router and set aifunc is default
+ router = default_router().with_current(page_route)
+
+ ghostos = get_ghostos()
+ shell = Singleton.get(Shell, st.session_state, force=False)
+ container = get_container()
+ if shell is None:
+ logger.debug("start shell background run")
+ shell = ghostos.create_shell(
+ "ghostos_streamlit_app",
+ providers=providers,
+ )
+ shell.background_run(4, StreamlitBackgroundApp())
+ Singleton(shell, Shell).bind(st.session_state)
+
+ return [
+ Singleton(container),
+ Singleton(router),
+ ]
+
+
+try:
+ main_run(bootstrap)
+except RuntimeError as e:
+ SystemExit(e)
diff --git a/ghostos/prototypes/streamlitapp/main.py b/ghostos/prototypes/streamlitapp/main.py
new file mode 100644
index 00000000..237f0110
--- /dev/null
+++ b/ghostos/prototypes/streamlitapp/main.py
@@ -0,0 +1,100 @@
+import streamlit as st
+from typing import Callable, List
+from ghostos.container import Container, Contracts
+from ghostos.contracts.configs import Configs
+from ghostos.prototypes.streamlitapp.utils.session import expect, SingletonContracts, Singleton
+from ghostos.prototypes.streamlitapp.utils.route import Router
+from ghostos.prototypes.streamlitapp.resources import AppConf
+from ghostos.helpers import gettext as _
+
+__all__ = [
+ "SINGLETONS", "BOOTSTRAP", "BOOTSTRAPPED_KEY",
+ "streamlit_contracts",
+ "container_contracts",
+ "SingletonContracts",
+ "Singleton",
+ "main_run",
+]
+
+st.set_page_config(page_title='GhostOS')
+
+SINGLETONS = List[Singleton]
+
+BOOTSTRAP = Callable[[], SINGLETONS]
+
+BOOTSTRAPPED_KEY = "ghostos.streamlit.app.bootstrapped"
+
+container_contracts = Contracts([
+ Configs,
+])
+
+streamlit_contracts = SingletonContracts([
+ Container,
+ Router,
+])
+
+
+def boot(fn: BOOTSTRAP) -> None:
+ if expect(st.session_state, BOOTSTRAPPED_KEY, True):
+ return
+ singletons = fn()
+ for s in singletons:
+ s.bind(st.session_state, force=False)
+
+ # validate streamlit bounds
+ unbound = streamlit_contracts.validate(st.session_state)
+ if unbound:
+ error = ",".join([str(c) for c in unbound])
+ raise NotImplementedError(f'GhostOS Streamlit app unbound contracts: {error}')
+
+ # validate container bounds
+ container = Singleton.get(Container, st.session_state)
+ container_contracts.validate(container)
+ # validate the container bootstrapped outside.
+ st.session_state[BOOTSTRAPPED_KEY] = True
+
+
+def main_run(bootstrap: BOOTSTRAP) -> None:
+ """
+ run streamlit application with outside bootstrap function.
+ :param bootstrap: a bootstrap function defined outside the streamlit app run
+
+ Why we need bootstrap argument when there is streamlit.cache_resource?
+ 1. I need a streamlit app func which can run everywhere
+ 2. The ghostos streamlit app need some contracts (for example, workspace), bootstrapped outside cause 1.
+ 3. @st.cache_resource wrap a function who control the resources lifecycle, conflict to 2.
+ """
+ # bootstrap once
+ boot(bootstrap)
+ # load pages
+ router = Singleton.get(Router, st.session_state)
+ pgs = st.navigation(router.pages(), position="hidden")
+ # define sidebar
+ with st.sidebar:
+ # router.render_homepage()
+ # render page links
+ with st.expander(label=_("Navigator"), expanded=False, icon=":material/menu:"):
+ router.render_navigator(use_container_width=True)
+ # with helper mode toggle
+ # open_navigator = st.button(
+ # label=_("GhostOS Navigator"),
+ # help=_("the navigations"),
+ # icon=":material/menu:",
+ # use_container_width=True,
+ # )
+ with st.expander(label="Options", expanded=True, icon=":material/settings:"):
+ AppConf.BoolOpts.HELP_MODE.render_toggle(
+ label=_("Help Mode"),
+ tips=_("switch help mode at every page"),
+ )
+ AppConf.BoolOpts.DEBUG_MODE.render_toggle(
+ label=_("Debug Mode"),
+ tips=_("switch debug mode at every page"),
+ )
+ st.subheader(_("Menu"))
+
+ # global navigator dialog
+ # if open_navigator:
+ # app_navigator_dialog()
+
+ pgs.run()
diff --git a/ghostos/prototypes/streamlitapp/pages/aifuncs/detail.py b/ghostos/prototypes/streamlitapp/pages/aifuncs/detail.py
new file mode 100644
index 00000000..dcf161fe
--- /dev/null
+++ b/ghostos/prototypes/streamlitapp/pages/aifuncs/detail.py
@@ -0,0 +1,248 @@
+from typing import Type
+import streamlit as st
+import streamlit_react_jsonschema as srj
+from ghostos.prototypes.streamlitapp.pages.router import AIFuncListRoute, AIFuncDetailRoute
+from ghostos.prototypes.streamlitapp.resources import (
+ get_container,
+)
+from ghostos.prototypes.streamlitapp.widgets.docs import (
+ help_document, markdown_document,
+)
+from ghostos.prototypes.streamlitapp.widgets.exec_frame import (
+ render_exec_frame_tree,
+ flatten_exec_frame_tree,
+)
+from ghostos.prototypes.streamlitapp.widgets.moss import render_pycontext
+from ghostos.prototypes.streamlitapp.widgets.messages import (
+ render_message_item,
+ render_messages,
+)
+from ghostos.core.messages import new_basic_connection
+from ghostos.core.aifunc import (
+ AIFunc,
+ AIFuncExecutor,
+ get_aifunc_result_type,
+ ExecFrame, ExecStep,
+)
+from ghostos.identifier import Identifier, identify_class
+from ghostos.helpers import (
+ uuid,
+ gettext as _,
+ import_from_path,
+ import_class_from_path, generate_import_path,
+ parse_import_path_module_and_attr_name,
+ Timeleft,
+)
+import inspect
+import webbrowser
+from threading import Thread
+
+
+def render_sidebar():
+ with st.sidebar:
+ AIFuncListRoute().render_page_link(use_container_width=True)
+
+
+def render_header(fn: Type[AIFunc]) -> Identifier:
+ idt = identify_class(fn)
+ # 渲染全局信息.
+ st.title(idt.name)
+ st.caption(idt.id)
+ st.markdown(idt.description)
+ return idt
+
+
+def render_source(route: AIFuncDetailRoute, fn: Type[AIFunc]):
+ # prepare
+ module_name, attr_name = parse_import_path_module_and_attr_name(route.aifunc_id)
+ mod = import_from_path(module_name)
+ result_type = get_aifunc_result_type(fn)
+ idt = identify_class(fn)
+
+ # open source code
+ if st.button("Open The Source File"):
+ webbrowser.open(f"file://{mod.__file__}")
+
+ # func code panel
+ st.subheader(_("Func Request"))
+ st.caption(idt.id)
+ source = inspect.getsource(fn)
+ st.code(source, line_numbers=True, wrap_lines=True)
+ help_document("aifunc/request_info")
+ st.divider()
+
+ # result code panel
+ st.subheader(_("Func Result"))
+ st.caption(generate_import_path(result_type))
+ source = inspect.getsource(result_type)
+ st.code(source, line_numbers=True, wrap_lines=True)
+ st.divider()
+
+ # run
+ st.subheader(_("Usage Example"))
+ markdown_document("aifunc/usage_example")
+
+ # full context
+ st.subheader(_("AIFunc Full Context"))
+ with st.expander(module_name):
+ source = inspect.getsource(mod)
+ st.code(source, line_numbers=True, wrap_lines=True)
+
+
+def render_aifunc_execute_stream(route: AIFuncDetailRoute, fn: Type[AIFunc]):
+ route.clear_execution()
+ # render form
+ with st.expander(_("Request"), expanded=True):
+ args, submitted = srj.pydantic_form(fn, key=route.aifunc_id)
+ if not submitted or not args:
+ return
+ if not isinstance(args, AIFunc):
+ st.error(f"Expected an AIFunc instance, got {type(args)}")
+ return
+
+ executor = get_container().force_fetch(AIFuncExecutor)
+ stream, receiver = new_basic_connection(timeout=route.timeout, idle=route.exec_idle, complete_only=True)
+ frame, caller = executor.new_exec_frame(args, stream)
+ # save status
+ route.frame = frame
+ route.executed = True
+ route.bind(st.session_state)
+
+ # render
+ timeleft = Timeleft(route.timeout)
+ t = Thread(target=caller)
+ t.start()
+ with st.status(_("executing...")):
+ with receiver:
+ for item in receiver.recv():
+ if not item.is_complete():
+ continue
+ route.received.append(item)
+ render_message_item(item, debug=False)
+ st.write(f"executed in {round(timeleft.passed(), 2)} seconds")
+
+
+def render_aifunc_frame_tail(frame: ExecFrame):
+ if not frame:
+ return
+ # error
+ if frame.error:
+ st.error(frame.error.get_content())
+
+ result = frame.get_result()
+ if result:
+ with st.expander("Exec Result", expanded=True):
+ key = "AIFuncResult_" + uuid()
+ srj.pydantic_instance_form(result, readonly=True, key=key)
+
+ if frame.steps:
+ st.caption(f"{len(frame.steps)} steps ran")
+
+ with st.expander("origin data", expanded=False):
+ st.json(frame.model_dump_json(indent=2, exclude_defaults=True))
+
+
+def render_aifunc_frame_stack(frame: ExecFrame):
+ if not frame:
+ return
+ selected = render_exec_frame_tree(_("Exec Stack"), frame)
+ # open dialog
+ if selected:
+ mapping = flatten_exec_frame_tree(frame)
+ selected_item = mapping.get(selected, None)
+ if isinstance(selected_item, ExecFrame):
+ open_exec_frame_dialog(selected_item)
+ elif isinstance(selected_item, ExecStep):
+ open_exec_step_dialog(selected_item)
+
+
+def render_aifunc_executed_frame_head(frame: ExecFrame):
+ if not frame:
+ return
+ args = frame.get_args()
+ # render form
+ with st.expander(_("Request"), expanded=True):
+ key = "AIFuncRequest_" + uuid()
+ srj.pydantic_instance_form(args, readonly=True, key=key)
+
+
+def render_aifunc_exec_step(step: ExecStep):
+ if step.pycontext:
+ render_pycontext(step.pycontext)
+
+ if step.error:
+ with st.expander(_("Error"), expanded=True):
+ st.error(step.error.get_content())
+
+ with st.expander(label=_("History"), expanded=True):
+ render_messages(step.iter_messages(), debug=False, in_expander=True)
+
+ if step.frames:
+ st.caption(f"{len(step.frames)} frames called")
+
+ with st.expander("origin data", expanded=False):
+ st.json(step.model_dump_json(indent=2, exclude_defaults=True))
+
+
+@st.dialog(title=_("ExecStep"), width="large")
+def open_exec_step_dialog(step: ExecStep):
+ st.subheader(step.func_name())
+ st.caption(f"step_id: {step.step_id}")
+ render_aifunc_exec_step(step)
+
+
+@st.dialog(title=_("ExecFrame"), width="large")
+def open_exec_frame_dialog(exec_frame: ExecFrame):
+ st.subheader(exec_frame.func_name())
+ st.caption(f"frame_id: {exec_frame.frame_id}")
+ render_aifunc_executed_frame_head(exec_frame)
+
+ with st.expander(label=_("History"), expanded=False):
+ idx = 0
+ for step in exec_frame.steps:
+ st.caption(f"step {idx}")
+ render_messages(step.iter_messages(), debug=False, in_expander=True)
+ idx = idx + 1
+
+ render_aifunc_frame_tail(exec_frame)
+
+
+def main():
+ route = AIFuncDetailRoute().get_or_bind(st.session_state)
+ render_sidebar()
+
+ if not route.aifunc_id:
+ st.error(f"AI Function {route.aifunc_id} found")
+ return
+ try:
+ fn = import_class_from_path(route.aifunc_id, AIFunc)
+ except TypeError as e:
+ st.error(e)
+ return
+
+ # render header
+ render_header(fn)
+
+ tab_exec, tab_source = st.tabs([_("Execute AIFuncs"), _("Source Code")])
+ with tab_exec:
+ if not route.executed:
+ render_aifunc_execute_stream(route, fn)
+ render_aifunc_frame_tail(route.frame)
+ render_aifunc_frame_stack(route.frame)
+ elif route.frame:
+ render_aifunc_executed_frame_head(route.frame)
+ with st.expander(label=_("messages"), expanded=True):
+ render_messages(route.received, debug=False, in_expander=True)
+
+ if route.frame:
+ render_aifunc_frame_tail(route.frame)
+ render_aifunc_frame_stack(route.frame)
+
+ if route.executed:
+ if st.button(_("rerun?")):
+ route.clear_execution()
+ route.bind(st.session_state)
+ st.rerun()
+
+ with tab_source:
+ render_source(route, fn)
diff --git a/ghostos/prototypes/streamlitapp/pages/aifuncs/index.py b/ghostos/prototypes/streamlitapp/pages/aifuncs/index.py
new file mode 100644
index 00000000..859a16cf
--- /dev/null
+++ b/ghostos/prototypes/streamlitapp/pages/aifuncs/index.py
@@ -0,0 +1,82 @@
+from typing import Iterable
+from ghostos.prototypes.streamlitapp.pages.router import AIFuncListRoute, AIFuncDetailRoute
+from ghostos.prototypes.streamlitapp.resources import (
+ get_container,
+ get_app_conf,
+ get_app_docs,
+)
+from ghostos.prototypes.streamlitapp.widgets.dialogs import (
+ open_code_dialog
+)
+from ghostos.core.aifunc import (
+ AIFuncRepository
+)
+import ghostos.core.aifunc.func as func
+import ghostos.core.aifunc.interfaces as interfaces
+from ghostos.identifier import Identifier
+from ghostos.helpers import (
+ gettext as _,
+ reflect_module_code,
+)
+import streamlit as st
+
+
+def render_aifuncs(items: Iterable[Identifier], keyword: str = "") -> None:
+ for idt in items:
+ if not idt.match_keyword(keyword):
+ continue
+ with st.container(border=True):
+ st.subheader(idt.name)
+ st.caption(idt.id)
+ st.markdown(idt.description)
+ key = idt.id + ":run"
+ if st.button("run", key=key):
+ route = AIFuncDetailRoute(
+ aifunc_id=idt.id,
+ )
+ route.switch_page()
+
+
+def main():
+ # bind if route value not bind before
+ route = AIFuncListRoute().get_or_bind(st.session_state)
+ app_conf = get_app_conf()
+ # render title and description.
+ st.title(_("AI Functions"))
+ with st.expander(_("introduce"), expanded=app_conf.BoolOpts.HELP_MODE.get()):
+ doc = get_app_docs().read("aifunc/introduction")
+ st.markdown(doc)
+
+ code_pattern = "AIFunc Code Pattern"
+ contracts_pattern = "AIFunc Interfaces"
+
+ if st.button(code_pattern):
+ file_code = reflect_module_code(func)
+ open_code_dialog(code_pattern, file_code)
+ if st.button(contracts_pattern):
+ file_code = reflect_module_code(interfaces)
+ open_code_dialog(contracts_pattern, file_code)
+
+ # scan
+ repo = get_container().force_fetch(AIFuncRepository)
+ with st.expander(_("tools"), expanded=app_conf.BoolOpts.DEBUG_MODE.get()):
+ do_scan = st.text_input(
+ label=_("scan ai funcs"),
+ value="",
+ placeholder=_("input python module name, scan all the funcs recursively"),
+ )
+
+ if do_scan:
+ found = repo.scan(do_scan, recursive=True, save=True)
+ st.write(f"Found {len(found)} AIFuncs")
+ render_aifuncs(found)
+ else:
+ funcs = list(repo.list())
+ col1, col2 = st.columns([2, 1])
+ with col2:
+ keyword = st.text_input(
+ label=_("filter by keyword"),
+ help=_("filter the funcs list by keyword"),
+ value=route.search,
+ )
+ render_aifuncs(funcs, keyword)
diff --git a/ghostos/prototypes/streamlitapp/pages/chat_with_ghost.py b/ghostos/prototypes/streamlitapp/pages/chat_with_ghost.py
new file mode 100644
index 00000000..41896092
--- /dev/null
+++ b/ghostos/prototypes/streamlitapp/pages/chat_with_ghost.py
@@ -0,0 +1,509 @@
+import streamlit as st
+import streamlit_react_jsonschema as srj
+import streamlit_paste_button as spb
+import time
+from PIL.Image import Image
+from typing import Iterable, List
+from ghostos.prototypes.streamlitapp.pages.router import (
+ GhostChatRoute, GhostTaskRoute,
+)
+from ghostos.prototypes.streamlitapp.utils.session import Singleton
+from ghostos.prototypes.streamlitapp.widgets.messages import (
+ render_message_in_content, render_messages,
+)
+from ghostos.prototypes.streamlitapp.widgets.renderer import (
+ render_object, render_event, render_turn,
+ render_empty,
+)
+from ghostos.prototypes.streamlitapp.resources import (
+ get_app_conf, save_uploaded_image, save_pil_image,
+)
+
+from ghostos.core.runtime import GoThreadInfo, Event, GoTaskStruct
+from ghostos.core.messages import (
+ Receiver, Role, ReceiverBuffer, MessageType, Message,
+ ImageAssetMessage,
+)
+from streamlit.logger import get_logger
+from ghostos.framework.openai_realtime import RealtimeApp
+from ghostos.abcd.realtime import OperatorName
+from ghostos.abcd import Shell, Conversation, Context
+from ghostos.identifier import get_identifier
+from ghostos.entity import to_entity_meta
+from ghostos.helpers import gettext as _
+from ghostos.helpers import generate_import_path, yaml_pretty_dump
+from ghostos.scripts.cli.utils import GhostsConf, GhostInfo
+from streamlit.runtime.uploaded_file_manager import UploadedFile
+from pydantic import BaseModel
+import inspect
+
+logger = get_logger("ghostos")
+
+
+def main_chat():
+ # create shell
+ route = GhostChatRoute.get(st.session_state)
+ if route is None:
+ st.error("No route found for session state")
+ return
+
+ # get ghost and context
+ conversation = get_conversation(route)
+ if not conversation.refresh():
+ st.error("Conversation is Closed")
+ return
+ realtime_app = Singleton.get(RealtimeApp, st.session_state, force=False)
+ # run the pages
+ with st.sidebar:
+ # other pages
+ with st.container(border=True):
+ GhostTaskRoute().render_page_link(use_container_width=True)
+ if st.button("Clear Messages", use_container_width=True):
+ thread = conversation.get_thread()
+ fork = thread.fork()
+ thread = fork.reset_history([])
+ conversation.update_thread(thread)
+ route.link.switch_page()
+
+ st.subheader("chat options")
+ with st.container(border=True):
+ route.realtime = st.toggle(
+ _("realtime chat"),
+ help=_("chat with agent by voice"),
+ value=route.realtime
+ )
+ if route.realtime:
+ route.auto_run = False
+ route.camera_input = False
+ route.image_input = False
+ created = False
+ with st.container():
+ route.vad_mode = st.toggle(
+ _("vad mod"),
+ help=_("the realtime api auto detected your input"),
+ value=route.vad_mode,
+ )
+ route.listen_mode = st.toggle(
+ _("listening"),
+ help=_("turn on or turn off listening"),
+ value=route.listen_mode,
+ )
+
+ if realtime_app is None:
+ realtime_app = get_realtime_app(conversation)
+ created = True
+ elif realtime_app.conversation is not conversation:
+ realtime_app.close()
+ realtime_app = get_realtime_app(conversation)
+ created = True
+ Singleton(realtime_app, RealtimeApp).bind(st.session_state)
+ if created:
+ realtime_app.start(vad_mode=route.vad_mode, listen_mode=route.listen_mode)
+ else:
+ realtime_app.set_mode(vad_mode=route.vad_mode, listen_mode=route.listen_mode)
+
+ canceled = get_response_button_count()
+ if not route.vad_mode:
+ if st.button("response", key=f"create_realtime_response_{canceled}"):
+ incr_response_button_count()
+ realtime_app.operate(OperatorName.respond.new(""))
+ st.rerun()
+ if st.button("cancel response", key=f"cancel_realtime_response_{canceled}"):
+ incr_response_button_count()
+ realtime_app.operate(OperatorName.cancel_responding.new(""))
+ st.rerun()
+
+ else:
+ if realtime_app is not None:
+ # if bind realtime app, release it.
+ realtime_app.close()
+ Singleton.release(RealtimeApp, st.session_state)
+
+ with st.container():
+ route.auto_run = st.toggle(
+ _("auto run event"),
+ help=_("automatic run background event"),
+ value=route.auto_run,
+ )
+ route.camera_input = st.toggle(
+ _("camera"),
+ help=_("take picture from camera, the model shall support image type"),
+ value=route.camera_input,
+ key=route.generate_key(st.session_state, "camera_input"),
+ )
+ route.image_input = st.toggle(
+ "upload image",
+ help=_("upload picture, the model shall support image type"),
+ value=route.image_input,
+ key=route.generate_key(st.session_state, "image input"),
+ )
+ route.bind(st.session_state)
+
+ # header
+ st.title("Ghost")
+ with st.container(border=True):
+ ghost = route.get_ghost()
+ id_ = get_identifier(ghost)
+ import_path = generate_import_path(ghost.__class__)
+ data = {
+ _("name"): id_.name,
+ _("desc"): id_.description,
+ _("class"): import_path,
+ }
+ if route.filename:
+ data[_("from")] = route.filename
+ # description
+ st.markdown(f"""
+```yaml
+{yaml_pretty_dump(data)}
+```
+""")
+ col1, col2, col3, col4 = st.columns([1, 1, 1, 1])
+ with col1:
+ show_ghost_settings = st.toggle("settings")
+ with col2:
+ show_instruction = st.toggle("instructions")
+ with col3:
+ show_context = st.toggle("context")
+ with col4:
+ pass
+
+ # render ghost settings
+ if show_ghost_settings:
+ render_ghost_settings(route)
+ if show_instruction:
+ render_instruction(conversation)
+ if show_context:
+ render_context_settings(conversation)
+
+ # inputs
+ if not route.realtime:
+ st.subheader("Chat")
+ chatting(route, conversation)
+ else:
+ st.subheader("Realtime Chat")
+ run_realtime(route, realtime_app)
+
+
+def run_realtime(route: GhostChatRoute, app: RealtimeApp):
+ try:
+ _run_realtime(route, app)
+ except Exception as e:
+ app.close()
+ raise e
+
+
+def get_response_button_count() -> int:
+ if "real_time_canceled" not in st.session_state:
+ st.session_state["real_time_canceled"] = 0
+ return st.session_state["real_time_canceled"]
+
+
+def incr_response_button_count():
+ if "real_time_canceled" not in st.session_state:
+ st.session_state["real_time_canceled"] = 0
+ st.session_state["real_time_canceled"] += 1
+
+
+def _run_realtime(route: GhostChatRoute, app: RealtimeApp):
+ thread = app.conversation.get_thread()
+ render_thread_messages(thread)
+ messages = app.history_messages()
+ render_messages(messages, debug=False, in_expander=False)
+ while not app.is_closed():
+ state = "waiting"
+ buffer = None
+ with st.empty():
+ with st.status(state) as status:
+ while buffer is None:
+ state, operators = app.state()
+ status.update(label=state)
+ buffer = app.output()
+ if buffer is None:
+ time.sleep(0.5)
+ continue
+
+ while buffer is not None:
+ head = buffer.head()
+ role = head.role
+ role = "user" if role == Role.USER.value else "assistant"
+ with st.chat_message(role):
+ with st.empty():
+ with st.container():
+ chunks = chunks_to_st_stream(buffer.chunks())
+ st.write_stream(chunks)
+ with st.container():
+ render_message_in_content(buffer.tail(), False, False)
+ buffer = buffer.next()
+ st.write("app is close")
+
+
+def get_realtime_app(conversation: Conversation) -> RealtimeApp:
+ from ghostos.framework.audio import get_pyaudio_pcm16_speaker, get_pyaudio_pcm16_listener
+ from ghostos.framework.openai_realtime import get_openai_realtime_app
+ speaker = get_pyaudio_pcm16_speaker()
+ listener = get_pyaudio_pcm16_listener()
+ vad_mode = True
+ return get_openai_realtime_app(conversation, vad_mode=vad_mode, listener=listener, speaker=speaker)
+
+
+def get_conversation(route: GhostChatRoute) -> Conversation:
+ if "chat_rendered" not in st.session_state:
+ st.session_state["chat_rendered"] = 0
+ else:
+ st.session_state["chat_rendered"] += 1
+ force_create_conversation = st.session_state["chat_rendered"] < 2
+
+ conversation = Singleton.get(Conversation, st.session_state, force=False)
+ if not conversation or conversation.is_closed():
+ shell = Singleton.get(Shell, st.session_state)
+ # create conversation
+ conversation = shell.sync(route.get_ghost(), route.get_context(), force=force_create_conversation)
+ Singleton(conversation, Conversation).bind(st.session_state)
+ return conversation
+
+
+def main_task():
+ route = GhostChatRoute.get(st.session_state)
+ with st.sidebar:
+ # other pages
+ with st.container(border=True):
+ route.render_page_link(use_container_width=True)
+ conversation = get_conversation(route)
+ task = conversation.get_task()
+ thread = conversation.get_thread()
+ st.title("Ghost Task Info")
+ render_task_info_settings(task, thread)
+
+
+def chatting(route: GhostChatRoute, conversation: Conversation):
+ if "rerun_chat" not in st.session_state:
+ st.session_state["rerun_chat"] = 0
+ st.session_state["rerun_chat"] += 1
+ for i in range(st.session_state["rerun_chat"]):
+ st.empty()
+ _chatting(route, conversation)
+
+
+def _chatting(route: GhostChatRoute, conversation: Conversation):
+ chat_input = st.chat_input("message")
+ thread = conversation.get_thread()
+ render_thread_messages(thread, max_turn=20, truncate=True)
+ debug = get_app_conf().BoolOpts.DEBUG_MODE.get()
+
+ pics: List[UploadedFile] = []
+ if route.camera_input:
+ if pic := st.camera_input(_("Task picture")):
+ pics.append(pic)
+ else:
+ st.empty()
+
+ if route.image_input:
+ if pic := st.file_uploader(_("Choose a picture"), type=["png", "jpg", "jpeg"]):
+ pics.append(pic)
+ paste = spb.paste_image_button(_("Paste a picture"))
+ if paste.image_data is not None:
+ st.image(paste.image_data, width=300)
+ pics.append(paste.image_data)
+ else:
+ st.empty()
+
+ inputs = st.session_state["ghostos_inputs"] if "ghostos_inputs" in st.session_state else []
+ st.session_state["ghostos_inputs"] = []
+ if chat_input:
+ if pics:
+ saved_images = []
+ for p in pics:
+ if isinstance(p, UploadedFile):
+ image_info = save_uploaded_image(p)
+ saved_images.append(image_info)
+ elif isinstance(p, Image):
+ image_info = save_pil_image(p, "user paste image")
+ saved_images.append(image_info)
+
+ message = ImageAssetMessage.from_image_asset(
+ name="",
+ content=chat_input,
+ images=saved_images,
+ )
+ inputs.append(message)
+ st.session_state["ghostos_inputs"] = inputs
+ route.new_render_turn(st.session_state)
+ st.rerun()
+ else:
+ inputs.append(Role.USER.new(chat_input))
+
+ if inputs:
+ event, receiver = conversation.respond(inputs)
+ with st.container():
+ render_event(event, debug)
+ render_receiver(receiver, debug)
+
+ interval = 0.1
+ while not route.media_input() and route.auto_run and conversation.available():
+ if event := conversation.pop_event():
+ with st.container():
+ render_event(event, debug)
+ receiver = conversation.respond_event(event)
+ render_receiver(receiver, debug)
+ interval = 0.1
+ else:
+ time.sleep(interval)
+ interval = 1
+
+
+@st.dialog("Textarea")
+def video_input_dialog(route: GhostChatRoute):
+ text = st.text_area("You message", value="")
+ if text:
+ st.session_state["chat_text_input"] = text
+ logger.debug("end of text area input")
+
+
+def render_receiver(receiver: Receiver, debug: bool):
+ try:
+ with receiver:
+ with st.container():
+ with st.empty():
+ with st.status("thinking"):
+ buffer = ReceiverBuffer.new(receiver.recv())
+ st.empty()
+ if buffer is None:
+ return
+ with st.chat_message("assistant"):
+ render_receive_buffer(buffer, debug)
+
+ except Exception as e:
+ st.error(str(e))
+ st.exception(e)
+
+
+def render_receive_buffer(buffer: ReceiverBuffer, debug: bool):
+ while buffer is not None:
+ st.logger.get_logger("ghostos").info("receive buffer head: %s", buffer.head())
+ if MessageType.is_text(buffer.head()):
+ with st.empty():
+ contents = chunks_to_st_stream(buffer.chunks())
+ st.write_stream(contents)
+ with st.container():
+ render_message_in_content(buffer.tail(), debug, in_expander=False)
+
+ elif MessageType.FUNCTION_CALL.match(buffer.head()):
+ contents = chunks_to_st_stream(buffer.chunks())
+ with st.empty():
+ st.write_stream(contents)
+ with st.container():
+ render_message_in_content(buffer.tail(), debug, in_expander=False)
+ else:
+ render_message_in_content(buffer.tail(), debug, in_expander=False)
+ # render next item
+ buffer = buffer.next()
+
+
+def chunks_to_st_stream(chunks: Iterable[Message]) -> Iterable[str]:
+ for chunk in chunks:
+ if chunk.content:
+ yield chunk.content
+
+
+def render_ghost_settings(route: GhostChatRoute):
+ st.subheader(_("Settings"))
+ with st.container(border=True):
+ if route is None:
+ st.error("page is not configured")
+ return
+ ghost = route.get_ghost()
+ # render ghost info
+ if isinstance(ghost, BaseModel):
+ data, submitted = srj.pydantic_instance_form(ghost)
+ if route.filename and submitted:
+ ghosts_conf = GhostsConf.load_from(route.filename)
+ key = ghosts_conf.file_ghost_key(route.filename)
+ info = GhostInfo(ghost=to_entity_meta(data))
+ ghosts_conf.ghosts[key] = info
+ ghosts_conf.save(route.filename)
+ st.write("saved")
+
+ else:
+ st.write(ghost)
+ source = inspect.getsource(ghost.__class__)
+ if st.toggle("source code", key="ghost_source_code"):
+ st.caption(generate_import_path(ghost.__class__))
+ st.code(source)
+ render_empty()
+
+
+def render_instruction(conversation: Conversation):
+ st.subheader("Instructions")
+ instructions = conversation.get_instructions()
+ with st.container(border=True):
+ st.markdown(instructions)
+
+
+def render_context_settings(conversation: Conversation):
+ st.subheader("Context")
+ with st.container(border=True):
+ ctx = conversation.get_context()
+ ghost = conversation.get_ghost()
+ if ctx is None and ghost.ContextType is None:
+ st.info("No specific Context for this Ghost")
+ return
+ if ctx is None:
+ if ghost.ContextType is not None:
+ data, submitted = srj.pydantic_form(ghost.ContextType)
+ if submitted and isinstance(data, Context):
+ conversation.update_context(data)
+ ctx = data
+ else:
+ data, changed = render_object(ctx, immutable=False)
+ if changed and isinstance(data, Context):
+ conversation.update_context(data)
+ ctx = data
+
+ # render prompt
+ if ctx is not None:
+ st.subheader(_("Context prompt"))
+ try:
+ prompt = ctx.get_prompt(conversation.container())
+ st.markdown(prompt)
+ except Exception as e:
+ st.error(e)
+ # render artifact
+ if ghost.ArtifactType:
+ st.subheader(_("Artifact"))
+ artifact = conversation.get_artifact()
+ render_object(artifact)
+
+
+def render_task_info_settings(task: GoTaskStruct, thread: GoThreadInfo):
+ from ghostos.core.runtime.tasks import TaskBrief
+ brief = TaskBrief.from_task(task)
+ srj.pydantic_instance_form(brief, readonly=True)
+
+ with st.expander(_("Detail"), expanded=False):
+ st.write(task.model_dump(exclude_defaults=True))
+
+ st.subheader("Thread Info")
+ render_thread_messages(thread, max_turn=0, truncate=False)
+
+
+def render_thread_messages(thread: GoThreadInfo, max_turn: int = 20, truncate: bool = True):
+ turns = list(thread.turns(truncate=truncate))
+ if max_turn > 0:
+ turns = turns[-max_turn:]
+ debug = get_app_conf().BoolOpts.DEBUG_MODE.get()
+ count = 0
+ for turn in turns:
+ count += render_turn(turn, debug)
+ if count == 0:
+ st.info("No thread messages yet")
+ else:
+ st.empty()
+
+
+def render_event_object(event: Event, debug: bool):
+ if event is None:
+ return
+ from_task_name = event.from_task_name
+ if debug and from_task_name is not None:
+ st.button(f"from task {from_task_name}", key=f"from task {event.event_id}")
diff --git a/ghostos/prototypes/streamlitapp/pages/configs.py b/ghostos/prototypes/streamlitapp/pages/configs.py
new file mode 100644
index 00000000..5f69967b
--- /dev/null
+++ b/ghostos/prototypes/streamlitapp/pages/configs.py
@@ -0,0 +1,56 @@
+import inspect
+
+import streamlit as st
+import streamlit_react_jsonschema as srj
+from typing import Type
+from ghostos.prototypes.streamlitapp.pages.router import ConfigsRoute
+from ghostos.prototypes.streamlitapp.resources import get_container
+from ghostos.contracts.configs import Configs, Config
+from ghostos.contracts.workspace import Workspace
+from ghostos.helpers import import_from_path
+from ghostos.identifier import identify_class
+from os.path import join
+
+
+def main():
+ route = ConfigsRoute.get(st.session_state)
+ classes = []
+ container = get_container()
+ ws = container.force_fetch(Workspace)
+ configs_dir = ws.configs().abspath()
+
+ for cls_name in route.config_classes:
+ cls = import_from_path(cls_name)
+ classes.append(cls)
+ identity = identify_class(cls)
+ filename = ""
+ if issubclass(cls, Config):
+ filename = join(configs_dir, cls.conf_path())
+
+ with st.container(border=True):
+ st.subheader(cls.__name__)
+ st.caption(identity.description)
+ if filename:
+ st.caption(f"at: `{filename}`\n")
+ st.caption(f"class: `{identity.id}`")
+ col1, col2, col3 = st.columns([1, 1, 2])
+ with col1:
+ open_form = st.toggle("edit", key="open_cls_form:" + identity.id)
+ with col2:
+ show_source = st.toggle("code", key="show_source:" + identity.id)
+
+ if open_form:
+ open_config_update_dialog(cls)
+ if show_source:
+ code = inspect.getsource(cls)
+ st.code(code)
+
+
+def open_config_update_dialog(cls: Type[Config]):
+ container = get_container()
+ configs = container.force_fetch(Configs)
+ data = configs.get(cls)
+ updated, submitted = srj.pydantic_instance_form(data)
+ if submitted and isinstance(updated, Config):
+ configs.save(updated)
+ st.write("Config saved")
diff --git a/ghostos/prototypes/streamlitapp/pages/homepage.py b/ghostos/prototypes/streamlitapp/pages/homepage.py
new file mode 100644
index 00000000..2ffea97d
--- /dev/null
+++ b/ghostos/prototypes/streamlitapp/pages/homepage.py
@@ -0,0 +1,37 @@
+from ghostos.helpers import gettext as _
+
+
+def home():
+ import streamlit as st
+ from ghostos.prototypes.streamlitapp.widgets.navigators import application_navigator_menu
+
+ st.title(_("GhostOS Homepage"))
+ with st.expander(_("App Menu"), expanded=True):
+ application_navigator_menu()
+
+
+def helloworld():
+ import streamlit as st
+ from ghostos.container import Container
+ from ghostos.prototypes.streamlitapp.utils.session import Singleton
+
+ st.write("hello world!")
+ container = Singleton.get(Container, st.session_state)
+ st.write(str(container))
+
+
+def navigator():
+ import streamlit as st
+ from ghostos.prototypes.streamlitapp.utils.route import Router
+ from ghostos.prototypes.streamlitapp.utils.session import Singleton
+
+ router = Singleton.get(Router, st.session_state)
+ menu = router.default_antd_menu_items()
+ route = router.render_antd_menu(menu)
+ if route is not None:
+ route.switch_page()
+
+
+def ghostos_host():
+ import streamlit as st
+ st.title("GhostOS Host")
diff --git a/ghostos/prototypes/streamlitapp/pages/router.py b/ghostos/prototypes/streamlitapp/pages/router.py
new file mode 100644
index 00000000..ab401dd7
--- /dev/null
+++ b/ghostos/prototypes/streamlitapp/pages/router.py
@@ -0,0 +1,231 @@
+from typing import Optional, List
+from ghostos.prototypes.streamlitapp.utils.route import Route, Router, Link
+from ghostos.core.messages import Message
+from ghostos.core.aifunc import ExecFrame
+from ghostos.abcd import Ghost, Context
+from ghostos.entity import EntityMeta, get_entity
+from ghostos.helpers import generate_import_path
+from enum import Enum
+from pydantic import Field
+from ghostos.framework.llms import LLMsYamlConfig
+from ghostos.framework.openai_realtime.configs import OpenAIRealtimeAppConf
+
+__all__ = [
+ "Route",
+ "Router",
+ "default_router",
+ "PagePath",
+ "GhostTaskRoute",
+ "ConfigsRoute",
+ "GhostChatRoute",
+ "AIFuncListRoute",
+ "AIFuncDetailRoute",
+]
+
+
+class PagePath(str, Enum):
+ HOMEPAGE = "ghostos.prototypes.streamlitapp.pages.homepage"
+ AIFUNCS = "ghostos.prototypes.streamlitapp.pages.aifuncs"
+ GHOSTS = "ghostos.prototypes.streamlitapp.pages.chat_with_ghost"
+ CONFIGS = "ghostos.prototypes.streamlitapp.pages.configs"
+
+ def suffix(self, attr_name: str):
+ return self.value + attr_name
+
+
+# --- ghosts --- #
+
+class GhostTaskRoute(Route):
+ link = Link(
+ name="Task Info",
+ import_path=PagePath.GHOSTS.suffix(":main_task"),
+ streamlit_icon=":material/smart_toy:",
+ button_help="todo",
+ antd_icon="robot",
+ )
+
+
+class GhostChatRoute(Route):
+ link = Link(
+ name="Chat",
+ import_path=PagePath.GHOSTS.suffix(":main_chat"),
+ streamlit_icon=":material/smart_toy:",
+ button_help="todo",
+ antd_icon="robot",
+ )
+ task_id: str = Field(default="", description="Ghost Task ID")
+ ghost_meta: Optional[EntityMeta] = Field(default=None, description="ghost meta")
+ context_meta: Optional[EntityMeta] = Field(default=None, description="context meta")
+ filename: Optional[str] = Field(default=None, description="filename to lunch the ghost")
+ camera_input: bool = Field(default=False, description="camera input")
+ image_input: bool = Field(default=False, description="image input")
+ auto_run: bool = Field(default=True, description="auto run")
+ realtime: bool = Field(default=False, description="realtime")
+ vad_mode: bool = Field(default=True, description="vad mode")
+ listen_mode: bool = Field(default=True, description="listening")
+
+ __ghost__ = None
+
+ def generate_key(self, session_state, key: str) -> str:
+ turn = self.get_render_turn(session_state)
+ return generate_import_path(self.__class__) + "-turn-" + str(turn) + "-key-" + key
+
+ def media_input(self) -> bool:
+ return self.camera_input or self.image_input
+
+ def get_render_turn(self, session_state) -> int:
+ key = generate_import_path(self.__class__) + ":turn"
+ if key not in session_state:
+ session_state[key] = 0
+ return session_state[key]
+
+ def new_render_turn(self, session_state) -> int:
+ key = generate_import_path(self.__class__) + ":turn"
+ if key not in session_state:
+ session_state[key] = 0
+ session_state[key] += 1
+ return session_state[key]
+
+ def get_ghost(self) -> Ghost:
+ if self.__ghost__ is None:
+ self.__ghost__ = get_entity(self.ghost_meta, Ghost)
+ return self.__ghost__
+
+ __context__ = None
+
+ def get_context(self) -> Optional[Context]:
+ if self.context_meta is None:
+ return None
+ if self.__context__ is None:
+ self.__context__ = get_entity(self.context_meta, Context)
+ return self.__context__
+
+
+# --- configs --- #
+
+class ConfigsRoute(Route):
+ link = Link(
+ name="Configs",
+ import_path=PagePath.CONFIGS.suffix(":main"),
+ streamlit_icon=":material/settings:",
+ button_help="todo",
+ antd_icon="settings",
+ )
+ config_classes: List[str] = Field(
+ default_factory=lambda: [
+ generate_import_path(LLMsYamlConfig),
+ generate_import_path(OpenAIRealtimeAppConf),
+ ],
+ description="config classes"
+ )
+
+
+# --- home --- #
+
+class Home(Route):
+ link = Link(
+ name="GhostOS",
+ import_path=PagePath.HOMEPAGE.suffix(":home"),
+ streamlit_icon=":material/home:",
+ button_help="help",
+ antd_icon="house-fill",
+ )
+
+
+class Navigator(Route):
+ link = Link(
+ name="Navigator",
+ import_path=PagePath.HOMEPAGE.suffix(":navigator"),
+ streamlit_icon=":material/home:",
+ antd_icon="box-fill",
+ )
+
+
+class GhostOSHost(Route):
+ link = Link(
+ name="GhostOS Host",
+ import_path=PagePath.HOMEPAGE.suffix(":ghostos_host"),
+ streamlit_icon=":material/smart_toy:",
+ )
+
+
+class Helloworld(Route):
+ """
+ test only
+ """
+ link = Link(
+ name="Hello World",
+ import_path=PagePath.HOMEPAGE.suffix(":helloworld"),
+ streamlit_icon=":material/home:",
+ )
+
+
+# --- ai functions --- #
+
+class AIFuncListRoute(Route):
+ link = Link(
+ name="AIFunc List",
+ import_path=PagePath.AIFUNCS.suffix(".index:main"),
+ streamlit_icon=":material/functions:",
+ )
+ search: str = Field(
+ default="",
+ description="search ai functions with keyword",
+ )
+
+
+class AIFuncDetailRoute(Route):
+ link = Link(
+ name="AIFunc Detail",
+ import_path=PagePath.AIFUNCS.suffix(".detail:main"),
+ streamlit_icon=":material/functions:",
+ )
+ aifunc_id: str = Field(
+ default="",
+ description="AIFunc ID, which is import path of it",
+ )
+ frame: Optional[ExecFrame] = Field(
+ default=None,
+ description="current execution frame",
+ )
+ executed: bool = False
+ received: List[Message] = Field(
+ default_factory=list,
+ description="list of execution messages",
+ )
+ timeout: float = 40
+ exec_idle: float = 0.2
+
+ def clear_execution(self):
+ self.executed = False
+ self.received = []
+ self.frame = None
+
+
+# --- routers --- #
+
+def default_router() -> Router:
+ return Router(
+ [
+ Home(),
+ Helloworld(),
+ Navigator(),
+ GhostOSHost(),
+ AIFuncListRoute(),
+ AIFuncDetailRoute(),
+ # ghosts
+ GhostChatRoute(),
+ GhostTaskRoute(),
+
+ ConfigsRoute(),
+ ],
+ home=Home.label(),
+ navigator_page_names=[
+ ConfigsRoute.label(),
+ ],
+ default_menu={
+ Home.label(): None,
+ },
+ default_sidebar_buttons=[
+ ],
+ )
diff --git a/ghostos/prototypes/streamlitapp/resources.py b/ghostos/prototypes/streamlitapp/resources.py
new file mode 100644
index 00000000..f700c33a
--- /dev/null
+++ b/ghostos/prototypes/streamlitapp/resources.py
@@ -0,0 +1,137 @@
+from typing import Optional, Dict, Tuple, List
+
+from enum import Enum
+from pydantic import Field
+import streamlit as st
+from ghostos.container import Container
+from ghostos.prototypes.streamlitapp.utils.session import Singleton
+from ghostos.contracts.configs import YamlConfig, Configs
+from ghostos.contracts.assets import ImageAssets, FileInfo, AudioAssets
+from ghostos.contracts.documents import DocumentRegistry, Documents
+from PIL.Image import Image as ImageType
+from ghostos.core.messages.message_classes import ImageAssetMessage
+from ghostos.helpers import GHOSTOS_DOMAIN, uuid
+from streamlit.runtime.uploaded_file_manager import DeletedFile, UploadedFile
+
+
+@st.cache_resource
+def get_container() -> Container:
+ return Singleton.get(Container, st.session_state)
+
+
+class AppConf(YamlConfig):
+ relative_path = "streamlit_app.yml"
+
+ domain: str = GHOSTOS_DOMAIN
+ lang: str = Field("zh", description="lang of the app")
+
+ bool_options: Dict[str, bool] = Field(
+ default_factory=dict,
+ )
+
+ class BoolOpts(str, Enum):
+ HELP_MODE = "ghostos.streamlit.app.help_mode"
+ """global help mode"""
+
+ DEBUG_MODE = "ghostos.streamlit.app.debug_mode"
+
+ def get(self) -> bool:
+ return get_app_conf().bool_options.get(self.name, True)
+
+ def render_toggle(
+ self,
+ label: str, *,
+ tips: Optional[str] = None,
+ disabled: bool = False,
+ ) -> None:
+ key = self.value
+ val = self.get()
+
+ def on_change():
+ """
+ change the config
+ """
+ value = st.session_state[key]
+ conf = get_app_conf()
+ conf.bool_options[self.name] = value
+ configs = get_container().force_fetch(Configs)
+ configs.save(conf)
+
+ st.toggle(
+ label,
+ key=self.value,
+ value=val,
+ disabled=disabled,
+ help=tips,
+ on_change=on_change,
+ )
+
+
+@st.cache_resource
+def get_app_conf() -> AppConf:
+ from ghostos.contracts.configs import Configs
+ configs = get_container().force_fetch(Configs)
+ return configs.get(AppConf)
+
+
+@st.cache_resource
+def get_app_docs() -> Documents:
+ conf = get_app_conf()
+ registry = get_container().force_fetch(DocumentRegistry)
+ return registry.get_domain(conf.domain, conf.lang)
+
+
+@st.cache_resource
+def get_images_assets() -> ImageAssets:
+ container = get_container()
+ return container.force_fetch(ImageAssets)
+
+
+@st.cache_resource
+def get_audio_assets() -> AudioAssets:
+ container = get_container()
+ return container.force_fetch(AudioAssets)
+
+
+def save_uploaded_image(file: UploadedFile) -> FileInfo:
+ image_info = FileInfo(
+ fileid=file.file_id,
+ filename=file.name,
+ description="streamlit camera input",
+ filetype=file.type,
+ )
+ binary = file.getvalue()
+ save_image_info(image_info, binary)
+ return image_info
+
+
+def save_image_info(image_info: FileInfo, binary: bytes) -> None:
+ assets = get_images_assets()
+ assets.save(image_info, binary)
+
+
+def save_pil_image(image: ImageType, desc: str) -> FileInfo:
+ from io import BytesIO
+ file_id = uuid()
+ img_bytes = BytesIO()
+ image.save(img_bytes, format='PNG')
+ binary = img_bytes.getvalue()
+ image_info = FileInfo(
+ image_id=file_id,
+ filename=file_id + ".png",
+ filetype="image/png",
+ description=desc
+ )
+ save_image_info(image_info, binary)
+ return image_info
+
+
+def get_images_from_image_asset(image_ids: List[str]) -> Dict[str, Tuple[FileInfo, Optional[bytes]]]:
+ result = {}
+ assets = get_images_assets()
+ for image_id in image_ids:
+ data = assets.get_file_and_binary_by_id(image_id)
+ if data is None:
+ continue
+ result[image_id] = data
+ return result
diff --git a/ghostos/prototypes/streamlitapp/tests/README.md b/ghostos/prototypes/streamlitapp/tests/README.md
new file mode 100644
index 00000000..a95dfb52
--- /dev/null
+++ b/ghostos/prototypes/streamlitapp/tests/README.md
@@ -0,0 +1,4 @@
+# About this directory
+
+test the basic features of the streamlit.
+remove later.
diff --git a/ghostos/libraries/__init__.py b/ghostos/prototypes/streamlitapp/tests/__init__.py
similarity index 100%
rename from ghostos/libraries/__init__.py
rename to ghostos/prototypes/streamlitapp/tests/__init__.py
diff --git a/ghostos/prototypes/streamlitapp/tests/aifunc/aifunc_elements.py b/ghostos/prototypes/streamlitapp/tests/aifunc/aifunc_elements.py
new file mode 100644
index 00000000..f03184c3
--- /dev/null
+++ b/ghostos/prototypes/streamlitapp/tests/aifunc/aifunc_elements.py
@@ -0,0 +1,37 @@
+import streamlit as st
+import streamlit_react_jsonschema as srj
+import inspect
+
+from ghostos.core.aifunc import get_aifunc_result_type
+from ghostos.demo.aifuncs.weather import WeatherAIFunc, WeatherAIFuncResult
+
+# source code
+st.title("Source Code")
+filepath = inspect.getmodule(WeatherAIFunc).__file__
+with open(filepath, "r") as f:
+ code = f.read()
+ st.code(code, language="python", line_numbers=True)
+
+# AiFunc code
+aifunc_code = inspect.getsource(WeatherAIFunc)
+st.title("AiFunc code")
+st.code(aifunc_code, language="python", line_numbers=True)
+
+# AiFunc result type Code
+aifunc_result_type = get_aifunc_result_type(WeatherAIFunc)
+aifunc_result_code = inspect.getsource(aifunc_result_type)
+st.title("AiFunc result code")
+st.code(aifunc_result_code, language="python", line_numbers=True)
+
+# json schema
+st.title("AiFuncs JSON Schema")
+st.json(WeatherAIFunc.model_json_schema())
+
+# aifunc pydantic output
+request = WeatherAIFunc()
+st.title("AiFunc output")
+srj.pydantic_instance_form(request)
+
+# aifunc result pydantic output
+st.title("AiFuncs Result output")
+srj.jsonschema_form("test", schema=WeatherAIFuncResult.model_json_schema(), default={})
diff --git a/ghostos/prototypes/streamlitapp/tests/async_test.py b/ghostos/prototypes/streamlitapp/tests/async_test.py
new file mode 100644
index 00000000..aa21c5d8
--- /dev/null
+++ b/ghostos/prototypes/streamlitapp/tests/async_test.py
@@ -0,0 +1,24 @@
+import streamlit as st
+import time
+
+st.title("Test")
+
+
+@st.fragment
+def messages():
+ count = 0
+ while "run" not in st.session_state or st.session_state["run"] is True:
+ count += 1
+ with st.chat_message("ai"):
+ st.write(f"Hello world! {count}")
+ time.sleep(1)
+
+
+# st.toggle(label="run", key="run", value=True)
+
+def callback():
+ st.session_state["run"] = False
+
+
+st.chat_input("test", on_submit=callback)
+messages()
diff --git a/ghostos/prototypes/streamlitapp/tests/audio_output_test.py b/ghostos/prototypes/streamlitapp/tests/audio_output_test.py
new file mode 100644
index 00000000..7c66c935
--- /dev/null
+++ b/ghostos/prototypes/streamlitapp/tests/audio_output_test.py
@@ -0,0 +1,51 @@
+import streamlit as st
+from typing import Iterable
+import time
+
+if "run" not in st.session_state:
+ st.session_state["run"] = 1
+run = st.session_state["run"]
+st.write(run)
+
+
+class Output:
+ def __init__(self):
+ self.content: str = ""
+
+ def iter_content(self) -> Iterable[str]:
+ passed = 0
+ while True:
+ if passed > 500:
+ break
+ self.content += "hello"
+ yield "hello"
+ time.sleep(0.1)
+ passed += 1
+
+
+if "output" not in st.session_state:
+ st.session_state["output"] = Output()
+output = st.session_state["output"]
+
+with st.chat_message("ai"):
+ st.write(output.content)
+
+recorded = 0
+if "recorded" in st.session_state:
+ recorded = st.session_state["recorded"]
+st.write("recorded: %d" % recorded)
+if recorded == 0:
+ if audio := st.audio_input("Audio file", key=f"{run}-audio-input"):
+ st.write(audio)
+ recorded = len(audio.getvalue())
+ st.session_state["recorded"] = recorded
+ st.rerun()
+else:
+ if st.button("stop"):
+ run += 1
+ st.session_state["run"] = run
+ st.session_state["recorded"] = 0
+ st.rerun()
+ with st.empty():
+ st.write(output.iter_content())
+ st.write("done")
diff --git a/ghostos/prototypes/streamlitapp/tests/audio_test.py b/ghostos/prototypes/streamlitapp/tests/audio_test.py
new file mode 100644
index 00000000..efda6faa
--- /dev/null
+++ b/ghostos/prototypes/streamlitapp/tests/audio_test.py
@@ -0,0 +1,60 @@
+import streamlit as st
+from threading import Thread
+from io import BytesIO, BufferedReader
+from os import path
+from pyaudio import PyAudio, paInt16
+
+pyaudio = PyAudio()
+import time
+
+if "run" not in st.session_state:
+ st.session_state["run"] = 0
+st.session_state["run"] += 1
+st.write(st.session_state["run"])
+
+
+def write_audio_to_bytes(io_in: BytesIO, filename: str):
+ with open(filename, "wb+") as fl:
+ while True:
+ r = io_in.read(1024)
+ if not r:
+ break
+ fl.write(r)
+ time.sleep(0.01)
+
+
+t = None
+io_output = None
+if audio := st.audio_input("Audio file"):
+ st.write(audio)
+ st.write(len(audio.getvalue()))
+ io_input = BytesIO(audio.getvalue())
+ io_output = path.abspath("test.wav")
+
+ t = Thread(target=write_audio_to_bytes, args=(io_input, io_output))
+ t.start()
+ time.sleep(0.1)
+
+if io_output is not None:
+ st.write("output start")
+ now = time.time()
+ with open(io_output, "rb+") as f:
+ stream = pyaudio.open(
+ format=paInt16,
+ channels=1,
+ rate=44100,
+ output=True,
+ )
+ while True:
+ got = f.read(1024)
+ if not got:
+ break
+ stream.write(got)
+ stream.stop_stream()
+ stream.close()
+ end = time.time()
+ st.write(round(end - now, 2))
+ st.write("output end")
+
+if t is not None:
+ t.join()
diff --git a/ghostos/prototypes/streamlitapp/tests/chat_render_by_messages.py b/ghostos/prototypes/streamlitapp/tests/chat_render_by_messages.py
new file mode 100644
index 00000000..a4c341cd
--- /dev/null
+++ b/ghostos/prototypes/streamlitapp/tests/chat_render_by_messages.py
@@ -0,0 +1,50 @@
+from typing import List, Any, Dict
+import streamlit as st
+import pandas as pd
+import numpy as np
+from pydantic import BaseModel, Field
+
+st.session_state.received = []
+
+chart_data = pd.DataFrame(
+ {
+ "col1": np.random.randn(20),
+ "col2": np.random.randn(20),
+ "col3": np.random.choice(["A", "B", "C"], 20),
+ }
+)
+
+
+class Item(BaseModel):
+ method: str = "write"
+ args: List[Any] = Field(default_factory=list)
+ kwargs: Dict[str, Any] = Field(default_factory=dict)
+
+
+st.session_state.received.append(Item(
+ method="write",
+ args=["hello world!"],
+))
+
+st.session_state.received.append(Item(
+ method="area_chart",
+ args=[chart_data],
+ kwargs=dict(x="col1", y="col2", color="col3"),
+))
+
+messages: List[Item] = st.session_state.received
+
+for item in messages:
+ with st.chat_message("assistant"):
+ method = getattr(st, item.method)
+ method(*item.args, **item.kwargs)
+
+
+# 可以用数据结构来定义, 但并不优雅.
+# 更好的办法是, 数据保存在文件, 同时定义一个函数. 函数接受回调.
+# like:
+
+
+class RenderObject(BaseModel):
+ data_path: str = Field(description="the path to save the data")
+ data_type: str = Field(description="the data type that follow with a callback func to render the data")
diff --git a/ghostos/prototypes/streamlitapp/tests/container_test/index.py b/ghostos/prototypes/streamlitapp/tests/container_test/index.py
new file mode 100644
index 00000000..1f0556a5
--- /dev/null
+++ b/ghostos/prototypes/streamlitapp/tests/container_test/index.py
@@ -0,0 +1,2 @@
+import streamlit as st
+
diff --git a/ghostos/prototypes/streamlitapp/tests/container_test/main.py b/ghostos/prototypes/streamlitapp/tests/container_test/main.py
new file mode 100644
index 00000000..162f73b7
--- /dev/null
+++ b/ghostos/prototypes/streamlitapp/tests/container_test/main.py
@@ -0,0 +1,10 @@
+import streamlit as st
+from ghostos.container import Container
+
+
+
+
+def main(con: Container):
+ st.navigation([
+ st.Page('page.py', title='Page'),
+ ])
diff --git a/ghostos/libraries/rag/__init__.py b/ghostos/prototypes/streamlitapp/tests/container_test/page.py
similarity index 100%
rename from ghostos/libraries/rag/__init__.py
rename to ghostos/prototypes/streamlitapp/tests/container_test/page.py
diff --git a/ghostos/mocks/__init__.py b/ghostos/prototypes/streamlitapp/tests/design2/__init__.py
similarity index 100%
rename from ghostos/mocks/__init__.py
rename to ghostos/prototypes/streamlitapp/tests/design2/__init__.py
diff --git a/ghostos/mocks/libraries/__init__.py b/ghostos/prototypes/streamlitapp/tests/design2/aifuncs/__init__.py
similarity index 100%
rename from ghostos/mocks/libraries/__init__.py
rename to ghostos/prototypes/streamlitapp/tests/design2/aifuncs/__init__.py
diff --git a/ghostos/prototypes/streamlitapp/tests/design2/aifuncs/details.py b/ghostos/prototypes/streamlitapp/tests/design2/aifuncs/details.py
new file mode 100644
index 00000000..70110d42
--- /dev/null
+++ b/ghostos/prototypes/streamlitapp/tests/design2/aifuncs/details.py
@@ -0,0 +1,53 @@
+import streamlit as st
+from streamlit_react_jsonschema import pydantic_form
+import inspect
+from ghostos.demo.aifuncs.weather import WeatherAIFunc
+from ghostos.core.aifunc import get_aifunc_result_type
+
+
+def show_tab_detail():
+ st.subheader("Import Path")
+ st.code(f"from {WeatherAIFunc.__module__} import {WeatherAIFunc.__qualname__}", language="python")
+
+ st.subheader("Request Type of the Func")
+ instance = pydantic_form(WeatherAIFunc)
+
+ st.subheader("Result Type of the Func")
+ result_type = get_aifunc_result_type(WeatherAIFunc)
+ instance = pydantic_form(result_type)
+
+ st.subheader("Source Code")
+ with st.expander("Full Code", expanded=False):
+ mod = inspect.getmodule(WeatherAIFunc)
+ codes = inspect.getsource(mod)
+ st.code(codes, language="python")
+
+ with st.expander("Request Code", expanded=False):
+ codes = inspect.getsource(WeatherAIFunc)
+ st.code(codes, language="python")
+
+ with st.expander("Result code", expanded=False):
+ result_type = get_aifunc_result_type(WeatherAIFunc)
+ codes = inspect.getsource(result_type)
+ st.code(codes, language="python")
+
+
+with st.sidebar:
+ st.page_link("aifuncs/index.py", label="AI Function List", icon=":material/list:")
+
+st.title(WeatherAIFunc.__name__)
+st.markdown(WeatherAIFunc.__doc__)
+
+tab_detail, tab_run, tab_test, tab_history = st.tabs(["Detail", "Run", "Tests", "History"])
+
+with tab_detail:
+ show_tab_detail()
+
+with tab_run:
+ st.caption("run")
+
+with tab_test:
+ st.caption("test")
+
+with tab_history:
+ st.caption("history")
diff --git a/ghostos/prototypes/streamlitapp/tests/design2/aifuncs/index.py b/ghostos/prototypes/streamlitapp/tests/design2/aifuncs/index.py
new file mode 100644
index 00000000..47262753
--- /dev/null
+++ b/ghostos/prototypes/streamlitapp/tests/design2/aifuncs/index.py
@@ -0,0 +1,28 @@
+import streamlit as st
+
+AIFUNC_SEARCH_MESSAGES = "aifunc.index.messages"
+
+with st.sidebar:
+ if search_aifunc := st.text_input(
+ "Search AIFUNC",
+ placeholder="description of the AIFunc you want",
+ ):
+ st.write(search_aifunc)
+
+st.title("AI Functions")
+
+with st.expander("introduction"):
+ st.write("hello world")
+
+pressed = False
+for i in range(5):
+ with st.container(border=True):
+ st.subheader("AIFuncName")
+ st.caption("from foo.bar.zoo import xxx")
+ st.text("description of the AIFunc __doc__")
+ hit = st.button("enter", key=f"button-{i}")
+ if not pressed and hit:
+ pressed = True
+
+if pressed:
+ st.switch_page("aifuncs/details.py")
diff --git a/ghostos/scripts/__init__.py b/ghostos/prototypes/streamlitapp/tests/design2/homepage/__init__.py
similarity index 100%
rename from ghostos/scripts/__init__.py
rename to ghostos/prototypes/streamlitapp/tests/design2/homepage/__init__.py
diff --git a/ghostos/prototypes/streamlitapp/tests/design2/homepage/applications.py b/ghostos/prototypes/streamlitapp/tests/design2/homepage/applications.py
new file mode 100644
index 00000000..9ef314ae
--- /dev/null
+++ b/ghostos/prototypes/streamlitapp/tests/design2/homepage/applications.py
@@ -0,0 +1,19 @@
+import streamlit as st
+
+st.title("Applications")
+
+with st.expander("description"):
+ st.write("""
+hello world
+""")
+
+with st.container(border=True):
+ st.subheader("Chatbots")
+ st.text("chatbots that predefined")
+ col1, col2, col3 = st.columns([1,2,3])
+ with col1:
+ st.button("all", help="show all the AIFuncs or search one", type="primary")
+ with col2:
+ st.button("WeatherAIFunc", help="show weather AIFuncs")
+ with col3:
+ st.button("AgenticAIFunc", help="show weather AIFuncs")
diff --git a/ghostos/prototypes/streamlitapp/tests/design2/homepage/home.py b/ghostos/prototypes/streamlitapp/tests/design2/homepage/home.py
new file mode 100644
index 00000000..b1c2dc6f
--- /dev/null
+++ b/ghostos/prototypes/streamlitapp/tests/design2/homepage/home.py
@@ -0,0 +1,44 @@
+import streamlit as st
+import streamlit_antd_components as sac
+
+st.title("GhostOS")
+with st.expander(label="Introduction"):
+ st.markdown("hello world")
+with st.expander(label="How to"):
+ st.markdown("hello world")
+
+with st.container(border=True):
+ st.subheader("Page links for test")
+ st.page_link(
+ "aifuncs/index.py",
+ label="AI Functions",
+ use_container_width=True,
+ )
+
+with st.container(border=True):
+ st.subheader("Navigation")
+ label = sac.menu([
+ sac.MenuItem(
+ 'Home',
+ icon='house-fill',
+ children=[
+ sac.MenuItem("Host", icon="robot", description="GhostOS official chatbot"),
+ sac.MenuItem("Documents", icon="book-fill", description="guide book"),
+ sac.MenuItem("Settings", icon="gear-fill", description="configs"),
+ sac.MenuItem("Tools", icon="hammer", description="System scripts"),
+ ],
+ ),
+ sac.MenuItem('Applications', icon='box-fill', children=[
+ sac.MenuItem('ChatBots', icon='chat-dots'),
+ sac.MenuItem('AIFuncs', icon='code-square'),
+ ]),
+ sac.MenuItem('Resources', icon='database-fill-gear', children=[
+ sac.MenuItem('LLMs', icon='heart-fill'),
+ sac.MenuItem('Moss Files', icon='heart-fill'),
+ sac.MenuItem('Thoughts', icon='heart-fill'),
+ sac.MenuItem('Registry', icon='heart-fill'),
+ sac.MenuItem('Knowledge', icon='heart-fill'),
+ sac.MenuItem('Data Objects', icon='heart-fill'),
+ ]),
+ ], open_all=True, index=0, variant="left-bar")
+ st.write(label)
diff --git a/ghostos/prototypes/streamlitapp/tests/design2/homepage/host.py b/ghostos/prototypes/streamlitapp/tests/design2/homepage/host.py
new file mode 100644
index 00000000..cc7e36b8
--- /dev/null
+++ b/ghostos/prototypes/streamlitapp/tests/design2/homepage/host.py
@@ -0,0 +1,3 @@
+import streamlit as st
+
+st.title("home")
diff --git a/ghostos/prototypes/streamlitapp/tests/design2/index.py b/ghostos/prototypes/streamlitapp/tests/design2/index.py
new file mode 100644
index 00000000..b33e379a
--- /dev/null
+++ b/ghostos/prototypes/streamlitapp/tests/design2/index.py
@@ -0,0 +1,33 @@
+import streamlit as st
+
+pages = st.navigation(
+ [
+ st.Page('homepage/home.py', title="Home", default=True),
+ st.Page('homepage/host.py'),
+ st.Page('homepage/applications.py'),
+ st.Page('aifuncs/index.py'),
+ st.Page('aifuncs/details.py'),
+ ],
+ position="hidden",
+)
+
+
+with st.sidebar:
+ st.page_link(
+ "homepage/home.py",
+ label="Home",
+ icon=":material/home:",
+ )
+# st.page_link(
+# "homepage/home.py",
+# label="Home",
+# icon=":material/home:",
+# help="GhostOS homepage",
+# )
+# st.page_link(
+# "homepage/applications.py",
+# label="Applications",
+# icon=":material/apps:",
+# )
+
+pages.run()
diff --git a/ghostos/libraries/rag/abc.py b/ghostos/prototypes/streamlitapp/tests/design2/router.py
similarity index 100%
rename from ghostos/libraries/rag/abc.py
rename to ghostos/prototypes/streamlitapp/tests/design2/router.py
diff --git a/ghostos/prototypes/streamlitapp/tests/duplex.py b/ghostos/prototypes/streamlitapp/tests/duplex.py
new file mode 100644
index 00000000..5554a117
--- /dev/null
+++ b/ghostos/prototypes/streamlitapp/tests/duplex.py
@@ -0,0 +1,77 @@
+import streamlit as st
+import time
+from typing import List, Iterator
+from ghostos.prototypes.streamlitapp.utils.route import Route, Link
+
+
+class ChatRoute(Route):
+ link = Link(
+ name="chat",
+ import_path="path",
+ )
+ messages: List[str] = []
+ input: str = ""
+ disabled: bool = False
+ buffer: str = ""
+
+ def set_input(self):
+ i = st.session_state["t_inputs"]
+ self.input = i
+
+ def iter_messages(self) -> Iterator[str]:
+ for line in self.messages:
+ yield line
+ if self.buffer:
+ self.messages.append(self.buffer)
+ buffer = self.buffer
+ self.buffer = ""
+ yield buffer
+
+ def iter_output(self, line: str) -> Iterator[str]:
+ self.buffer = ""
+ for c in line:
+ self.buffer += c
+ yield c
+ if self.input:
+ break
+ time.sleep(0.2)
+
+
+chat = ChatRoute().get_or_bind(st.session_state)
+
+
+@st.fragment
+def run_messages():
+ count = 0
+ for msg in chat.iter_messages():
+ role = "ai"
+ if msg.startswith("user:"):
+ role = "user"
+ with st.chat_message(role):
+ st.write(msg)
+ with st.expander("debug mode", expanded=False):
+ st.write("hello")
+
+ while True:
+ if i := chat.input:
+ content = f"user:{i}"
+ chat.input = ""
+ with st.chat_message("user"):
+ st.write(content)
+ chat.messages.append(content)
+ count += 1
+ if count % 30 == 0:
+ chat.disabled = True
+ content = f"another round {count}"
+ with st.chat_message("ai"):
+ items = chat.iter_output(content)
+ st.write_stream(items)
+ chat.messages.append(content)
+ chat.disabled = False
+ time.sleep(0.1)
+
+
+if i := st.chat_input("input"):
+ chat.input = i
+
+run_messages()
diff --git a/ghostos/prototypes/streamlitapp/tests/image_test.py b/ghostos/prototypes/streamlitapp/tests/image_test.py
new file mode 100644
index 00000000..826d72b9
--- /dev/null
+++ b/ghostos/prototypes/streamlitapp/tests/image_test.py
@@ -0,0 +1,8 @@
+import streamlit as st
+
+if pic := st.camera_input("You photo"):
+ st.write(pic)
+ data = pic.getvalue()
+ with open("pic.jpg", "wb") as f:
+ f.write(data)
+ st.write("write to file")
diff --git a/ghostos/prototypes/streamlitapp/tests/messages_in_empty.py b/ghostos/prototypes/streamlitapp/tests/messages_in_empty.py
new file mode 100644
index 00000000..eff95e79
--- /dev/null
+++ b/ghostos/prototypes/streamlitapp/tests/messages_in_empty.py
@@ -0,0 +1,18 @@
+from typing import Iterable
+import streamlit as st
+import time
+
+
+def get_content() -> Iterable[str]:
+ content = "hello world"
+ for c in content:
+ yield c
+ time.sleep(0.05)
+
+
+for i in range(5):
+ with st.empty():
+ with st.chat_message("ai"):
+ st.write_stream(get_content())
+ with st.chat_message("ai"):
+ st.write("hello world!!")
diff --git a/ghostos/prototypes/streamlitapp/tests/pages_test.py b/ghostos/prototypes/streamlitapp/tests/pages_test.py
new file mode 100644
index 00000000..aeabcd0a
--- /dev/null
+++ b/ghostos/prototypes/streamlitapp/tests/pages_test.py
@@ -0,0 +1,56 @@
+import streamlit as st
+
+if "logged_in" not in st.session_state:
+ st.session_state.logged_in = False
+
+
+def login():
+ if st.button("Log in"):
+ st.session_state.logged_in = True
+ st.rerun()
+
+
+def logout():
+ if st.button("Log out"):
+ st.session_state.logged_in = False
+ # 关键的逻辑, 触发重新渲染.
+ st.rerun()
+
+
+login_page = st.Page(login, title="Log in", icon=":material/login:")
+logout_page = st.Page(logout, title="Log out", icon=":material/logout:")
+
+dashboard = st.Page(
+ "reports/dashboard.py", title="Dashboard", icon=":material/dashboard:", default=True
+)
+bugs = st.Page("reports/bugs.py", title="Bug reports", icon=":material/bug_report:")
+alerts = st.Page(
+ "reports/alerts.py", title="System alerts", icon=":material/notification_important:"
+)
+
+search = st.Page("tools/search.py", title="Search", icon=":material/search:")
+history = st.Page(
+ "tools/history.py",
+ title="History",
+ icon=":material/history:",
+ url_path="history",
+)
+history2 = st.Page(
+ "tools/history.py",
+ title="History2",
+ icon=":material/history:",
+ url_path="history2?hello=world",
+)
+
+if st.session_state.logged_in or True:
+ pg = st.navigation(
+ {
+ "Account": [logout_page],
+ "Reports": [dashboard, bugs, alerts],
+ "Tools": [search, history, history2],
+ }
+ )
+else:
+ pg = st.navigation([login_page])
+
+pg.run()
diff --git a/ghostos/prototypes/streamlitapp/tests/reports/alerts.py b/ghostos/prototypes/streamlitapp/tests/reports/alerts.py
new file mode 100644
index 00000000..78d76467
--- /dev/null
+++ b/ghostos/prototypes/streamlitapp/tests/reports/alerts.py
@@ -0,0 +1,4 @@
+import streamlit as st
+
+x = st.slider("Select a value")
+st.write(x, "squared is", x * x)
diff --git a/ghostos/prototypes/streamlitapp/tests/reports/bugs.py b/ghostos/prototypes/streamlitapp/tests/reports/bugs.py
new file mode 100644
index 00000000..78d76467
--- /dev/null
+++ b/ghostos/prototypes/streamlitapp/tests/reports/bugs.py
@@ -0,0 +1,4 @@
+import streamlit as st
+
+x = st.slider("Select a value")
+st.write(x, "squared is", x * x)
diff --git a/ghostos/prototypes/streamlitapp/tests/reports/dashboard.py b/ghostos/prototypes/streamlitapp/tests/reports/dashboard.py
new file mode 100644
index 00000000..78d76467
--- /dev/null
+++ b/ghostos/prototypes/streamlitapp/tests/reports/dashboard.py
@@ -0,0 +1,4 @@
+import streamlit as st
+
+x = st.slider("Select a value")
+st.write(x, "squared is", x * x)
diff --git a/ghostos/prototypes/streamlitapp/tests/session_render_by_set.py b/ghostos/prototypes/streamlitapp/tests/session_render_by_set.py
new file mode 100644
index 00000000..4c3402ae
--- /dev/null
+++ b/ghostos/prototypes/streamlitapp/tests/session_render_by_set.py
@@ -0,0 +1,9 @@
+import streamlit as st
+from random import randint
+
+st.write(randint(0, 1000))
+
+if value := st.button("test render"):
+ # always set same value, see if the session trigger rendering
+ st.write("button: " + str(value))
+ st.session_state["some"] = dict(foo="bar")
diff --git a/ghostos/prototypes/streamlitapp/tests/sidebar/echo_spinner.py b/ghostos/prototypes/streamlitapp/tests/sidebar/echo_spinner.py
new file mode 100644
index 00000000..73a60937
--- /dev/null
+++ b/ghostos/prototypes/streamlitapp/tests/sidebar/echo_spinner.py
@@ -0,0 +1,10 @@
+import streamlit as st
+import time
+
+with st.sidebar:
+ with st.echo():
+ st.write("This code will be printed to the sidebar.")
+
+ with st.spinner("Loading..."):
+ time.sleep(5)
+ st.success("Done!")
diff --git a/ghostos/prototypes/streamlitapp/tests/slider.py b/ghostos/prototypes/streamlitapp/tests/slider.py
new file mode 100644
index 00000000..78d76467
--- /dev/null
+++ b/ghostos/prototypes/streamlitapp/tests/slider.py
@@ -0,0 +1,4 @@
+import streamlit as st
+
+x = st.slider("Select a value")
+st.write(x, "squared is", x * x)
diff --git a/ghostos/prototypes/streamlitapp/tests/srj_test.py b/ghostos/prototypes/streamlitapp/tests/srj_test.py
new file mode 100644
index 00000000..8654f49a
--- /dev/null
+++ b/ghostos/prototypes/streamlitapp/tests/srj_test.py
@@ -0,0 +1,15 @@
+import streamlit as st
+import streamlit_react_jsonschema as srj
+from pydantic import BaseModel
+
+
+class Foo(BaseModel):
+ foo: str = "foo"
+
+
+args, submitted = srj.pydantic_instance_form(Foo(), key="hello")
+
+st.write(submitted)
+
+if value := st.button("hello"):
+ st.write(value)
diff --git a/ghostos/prototypes/streamlitapp/tests/status_elements.py b/ghostos/prototypes/streamlitapp/tests/status_elements.py
new file mode 100644
index 00000000..1f41faf3
--- /dev/null
+++ b/ghostos/prototypes/streamlitapp/tests/status_elements.py
@@ -0,0 +1,8 @@
+import streamlit as st
+
+st.success('This is a success message!', icon="✅")
+st.info('This is a purely informational message', icon="ℹ️")
+st.warning('This is a warning', icon="⚠️")
+st.error('This is an error', icon="🚨")
+e = RuntimeError("This is an exception of type RuntimeError")
+st.exception(e)
diff --git a/ghostos/prototypes/streamlitapp/tests/sub_run/main.py b/ghostos/prototypes/streamlitapp/tests/sub_run/main.py
new file mode 100644
index 00000000..0a58a984
--- /dev/null
+++ b/ghostos/prototypes/streamlitapp/tests/sub_run/main.py
@@ -0,0 +1,12 @@
+from streamlit.web.cli import main_run
+from os.path import dirname, join
+import streamlit as st
+
+hello = "world"
+
+if __name__ == "__main__":
+ hello = "hello"
+ st.session_state["hello"] = "hello"
+ filename = join(dirname(__file__), "page.py")
+ print("++++++++++++", filename)
+ main_run([filename, "hello world", "who are your daddy", "--logger.enableRich=True"] )
diff --git a/ghostos/prototypes/streamlitapp/tests/sub_run/page.py b/ghostos/prototypes/streamlitapp/tests/sub_run/page.py
new file mode 100644
index 00000000..30c07035
--- /dev/null
+++ b/ghostos/prototypes/streamlitapp/tests/sub_run/page.py
@@ -0,0 +1,8 @@
+import streamlit as st
+from ghostos.prototypes.streamlitapp.tests.sub_run.main import hello
+import sys
+
+st.write(sys.argv)
+st.write(hello)
+if "hello" in st.session_state:
+ st.title(st.session_state["hello"])
diff --git a/ghostos/prototypes/streamlitapp/tests/test_markdown.py b/ghostos/prototypes/streamlitapp/tests/test_markdown.py
new file mode 100644
index 00000000..c50b9104
--- /dev/null
+++ b/ghostos/prototypes/streamlitapp/tests/test_markdown.py
@@ -0,0 +1,9 @@
+import streamlit as st
+
+st.markdown(f"""
+```moss
+def foo():
+ return "hello world"
+
+```
+""")
diff --git a/ghostos/prototypes/streamlitapp/tests/text_input.py b/ghostos/prototypes/streamlitapp/tests/text_input.py
new file mode 100644
index 00000000..37d93ac1
--- /dev/null
+++ b/ghostos/prototypes/streamlitapp/tests/text_input.py
@@ -0,0 +1,19 @@
+import streamlit as st
+
+
+@st.fragment
+def show_text_input():
+ text_input = st.session_state['text_input']
+ with st.empty():
+ st.write(text_input)
+
+
+title = st.text_input(
+ "Movie title",
+ "Life of Brian",
+ key="text_input",
+)
+
+show_text_input()
+
+st.write("The current movie title is", title)
diff --git a/ghostos/prototypes/streamlitapp/tests/tools/history.py b/ghostos/prototypes/streamlitapp/tests/tools/history.py
new file mode 100644
index 00000000..3580ce82
--- /dev/null
+++ b/ghostos/prototypes/streamlitapp/tests/tools/history.py
@@ -0,0 +1,6 @@
+import streamlit as st
+
+x = st.slider("Select a value")
+st.write(x, "squared is", x * x)
+values = {k: v for k, v in st.query_params.conversation_item_states()}
+st.write(values)
diff --git a/ghostos/prototypes/streamlitapp/tests/tools/search.py b/ghostos/prototypes/streamlitapp/tests/tools/search.py
new file mode 100644
index 00000000..78d76467
--- /dev/null
+++ b/ghostos/prototypes/streamlitapp/tests/tools/search.py
@@ -0,0 +1,4 @@
+import streamlit as st
+
+x = st.slider("Select a value")
+st.write(x, "squared is", x * x)
diff --git a/ghostos/prototypes/streamlitapp/tests/write_stream_char.py b/ghostos/prototypes/streamlitapp/tests/write_stream_char.py
new file mode 100644
index 00000000..2e1cd6f0
--- /dev/null
+++ b/ghostos/prototypes/streamlitapp/tests/write_stream_char.py
@@ -0,0 +1,15 @@
+from typing import Iterable
+import streamlit as st
+import time
+
+
+def get_content() -> Iterable[str]:
+ content = "hello world"
+ for c in content:
+ yield c
+ time.sleep(0.05)
+
+
+with st.chat_message("ai"):
+ for c in get_content():
+ st.write_stream([c])
diff --git a/tests/helpers/__init__.py b/ghostos/prototypes/streamlitapp/utils/__init__.py
similarity index 100%
rename from tests/helpers/__init__.py
rename to ghostos/prototypes/streamlitapp/utils/__init__.py
diff --git a/ghostos/prototypes/streamlitapp/utils/route.py b/ghostos/prototypes/streamlitapp/utils/route.py
new file mode 100644
index 00000000..92e75f22
--- /dev/null
+++ b/ghostos/prototypes/streamlitapp/utils/route.py
@@ -0,0 +1,299 @@
+from __future__ import annotations
+
+from typing import ClassVar, Callable, Optional, MutableMapping, TypeVar, List, Dict, Set, Union
+from abc import ABC
+from typing_extensions import Self
+from ghostos.prototypes.streamlitapp.utils.session import SessionStateValue
+from ghostos.helpers import generate_import_path, import_from_path
+from pydantic import BaseModel, Field
+import streamlit as st
+from ghostos.helpers import gettext as _
+import streamlit_antd_components as sac
+
+__all__ = ["Router", 'Route', 'Link']
+
+T = TypeVar("T")
+
+
+class Link:
+ """
+ wrap streamlit page functions
+ """
+
+ def __init__(
+ self,
+ name: str,
+ import_path: str,
+ *,
+ button_help: Optional[str] = None,
+ menu_desc: Optional[str] = None,
+ url_path: str | None = None,
+ streamlit_icon: str = ":material/box:",
+ antd_icon: str = "box-fill",
+ ):
+ self.name = name
+ self.import_path = import_path
+ self.streamlit_icon = streamlit_icon
+ self.antd_icon = antd_icon
+ self.button_help = button_help
+ self.menu_desc = menu_desc
+ self.url_path = url_path if url_path else name.lower().replace(" ", '_')
+
+ def st_page(
+ self, *,
+ default: bool = False,
+ title: Optional[str] = None,
+ url_path: Optional[str] = None,
+ ) -> st.Page:
+ title = _(title) if title is not None else None
+ # function
+ if ':' in self.import_path:
+ page = import_from_path(self.import_path)
+ else:
+ page = self.import_path
+
+ return st.Page(
+ page=page,
+ title=title,
+ icon=self.streamlit_icon,
+ url_path=url_path,
+ default=default,
+ )
+
+ def switch_page(self, url_path: Optional[str] = None) -> None:
+ st.switch_page(self.st_page(url_path=url_path))
+
+
+class Route(SessionStateValue, BaseModel, ABC):
+ """
+ wrap the basic methods:
+ 1. the data useful to render a streamlit page
+ 2. switch to a streamlit page
+ 3. render a navigation
+ 4. render a page switch button
+ 5. render a page switch dialog
+ """
+
+ link: ClassVar[Link]
+ url_query: str = Field("", description="urlpath query")
+
+ def page(self, default: bool = False) -> st.Page:
+ url_path = self.full_url_path()
+ return self.link.st_page(url_path=url_path, default=default)
+
+ @classmethod
+ def label(cls) -> str:
+ return _(cls.link.name)
+
+ def full_url_path(self) -> str:
+ url_path = self.link.url_path
+ if self.url_query:
+ url_path += "?" + self.url_query
+ return url_path
+
+ def switch_page(self) -> None:
+ """
+ bind self to the session state and switch the page.
+ """
+ # bind the route value to the session state
+ url_path = self.full_url_path()
+ self.bind(st.session_state)
+ self.link.switch_page(url_path=url_path)
+
+ def rerun(self) -> None:
+ self.bind(st.session_state)
+ st.rerun()
+
+ def render_page_link(
+ self, *,
+ disabled: bool = False,
+ use_container_width: bool = False,
+ ):
+ """
+ shall run under `with st.sidebar`
+ """
+ label = self.label()
+ help_ = self.link.button_help
+ if help_ is not None:
+ help_ = _(help_)
+ self.bind(st.session_state)
+ st.page_link(
+ self.page(),
+ label=label,
+ help=help_,
+ icon=self.link.streamlit_icon,
+ disabled=disabled,
+ use_container_width=use_container_width,
+ )
+
+ def antd_menu_item(self, children: Optional[List[sac.MenuItem]] = None) -> sac.MenuItem:
+ """
+ generate menu item
+ """
+ menu_desc = self.link.menu_desc
+ if menu_desc is not None:
+ menu_desc = _(menu_desc)
+ return sac.MenuItem(
+ label=self.label(),
+ description=menu_desc,
+ children=children,
+ icon=self.link.antd_icon,
+ )
+
+ @classmethod
+ def session_state_key(cls) -> str:
+ return generate_import_path(cls)
+
+ @classmethod
+ def get(cls, session_state: MutableMapping) -> Optional[Self]:
+ key = cls.session_state_key()
+ if key in session_state:
+ data = session_state[key]
+ return cls(**data)
+ return None
+
+ def bind(self, session_state: MutableMapping) -> None:
+ from ghostos.container import get_caller_info
+ key = self.session_state_key()
+ session_state[key] = self.model_dump(exclude_defaults=True)
+
+ @classmethod
+ def label_of_current_page(cls) -> str:
+ current = generate_import_path(Route)
+ if current in st.session_state:
+ return st.session_state[current]
+ return ""
+
+ @classmethod
+ def get_route_bound(cls, value: T, key: str = "") -> T:
+ if not key:
+ key = generate_import_path(type(value))
+ session_key = cls.session_state_key() + ":" + key
+ if session_key in st.session_state:
+ return st.session_state[session_key]
+ st.session_state[session_key] = value
+ return value
+
+
+class Router:
+
+ def __init__(
+ self,
+ routes: List[Route], *,
+ home: str,
+ navigator_page_names: List[str],
+ default_menu: Dict[str, Union[sac.MenuItem, Dict, None]],
+ default_sidebar_buttons: List[str],
+ current_page: str = None,
+ ):
+ self.routes: Dict[str, Route] = {}
+ self.routes_order = []
+ self.home = home
+ self.append(*routes)
+ self.default_menu_tree = default_menu
+ self.default_sidebar_buttons = default_sidebar_buttons
+ self.default_navigator_names = navigator_page_names
+ self.current_page: str = current_page if current_page is not None else self.home
+
+ def with_current(self, route: Route) -> Self:
+ self.current_page = route.label()
+ self.routes[route.label()] = route
+ return self
+
+ def append(self, *routes: Route):
+ for route in routes:
+ name = route.label()
+ if name in self.routes:
+ raise KeyError(f"Duplicate route name: {name}")
+ self.routes[name] = route
+ self.routes_order.append(name)
+
+ def render_homepage(self) -> None:
+ route = self.routes[self.home]
+ route.render_page_link(use_container_width=True)
+
+ def pages(self, default: Optional[str] = None, names: Optional[List[str]] = None) -> List[st.Page]:
+ """
+ render sidebar pages
+ :param default:
+ :param names:
+ :return:
+ """
+ pages = []
+ if names is None:
+ names = self.routes_order
+ idx = 0
+ if default is None:
+ default = self.current_page
+
+ for name in names:
+ route = self.routes[name]
+ is_default = name == default
+ idx += 1
+ if is_default:
+ route.bind(st.session_state)
+ st.session_state["hello"] = route.model_dump()
+ page = route.page(default=is_default)
+ pages.append(page)
+ return pages
+
+ def render_page_links(
+ self, *,
+ names: Optional[List[str]],
+ disabled: Optional[Set[str]] = None,
+ use_container_width: bool = True,
+ ) -> None:
+ """
+ render streamlit page link buttons
+ """
+ for name in names:
+ route = self.routes[name]
+ is_disabled = disabled is not None and name in disabled
+ route.render_page_link(
+ disabled=is_disabled,
+ use_container_width=use_container_width,
+ )
+
+ def render_navigator(
+ self,
+ disabled: Optional[Set[str]] = None,
+ use_container_width: bool = True,
+ ):
+ """
+ render default page links built buttons
+ """
+ self.render_page_links(
+ names=self.default_navigator_names,
+ disabled=disabled,
+ use_container_width=use_container_width,
+ )
+
+ def _antd_menu_items(self, node_tree: Dict[str, Union[sac.MenuItem, Dict, None]]) -> List[sac.MenuItem]:
+ """
+ return antd menu items from routes.
+ """
+ result = []
+ for label in node_tree:
+ item = node_tree[label]
+ if isinstance(item, sac.MenuItem):
+ item.label = label
+ result.append(item)
+ else:
+ if label not in self.routes:
+ raise KeyError(f"menu label : {label} not found in Route")
+ route = self.routes[label]
+ children = None
+ if isinstance(item, dict) and len(item) > 0:
+ children = self._antd_menu_items(item)
+ menu_item = route.antd_menu_item(children)
+ result.append(menu_item)
+ return result
+
+ def default_antd_menu_items(self) -> List[sac.MenuItem]:
+ return self._antd_menu_items(self.default_menu_tree)
+
+ def render_antd_menu(self, items: List[sac.MenuItem]) -> Optional[Route]:
+ choose = sac.menu(items, index=-1)
+ if choose in self.routes:
+ return self.routes[choose]
+ return None
diff --git a/ghostos/prototypes/streamlitapp/utils/session.py b/ghostos/prototypes/streamlitapp/utils/session.py
new file mode 100644
index 00000000..df1fd8fe
--- /dev/null
+++ b/ghostos/prototypes/streamlitapp/utils/session.py
@@ -0,0 +1,139 @@
+from abc import ABC, abstractmethod
+from typing import MutableMapping, Optional, ClassVar, Any, TypeVar, Type, List, Callable
+from typing_extensions import Self
+from pydantic import BaseModel
+from ghostos.helpers import generate_import_path
+import streamlit as st
+
+__all__ = [
+ 'SessionStateValue', 'ModelSingleton',
+ 'SingletonContracts',
+ 'Singleton',
+ # functions
+ 'expect',
+]
+
+
+class SessionStateValue(ABC):
+ """
+ Value that bind to streamlit.session_state
+ """
+
+ @classmethod
+ @abstractmethod
+ def get(cls, session_state: MutableMapping) -> Optional[Self]:
+ """
+ load self value from session_state
+ :param session_state: the streamlit session state
+ :return: None if not bound yet
+ """
+ pass
+
+ def get_or_bind(self, session_state: MutableMapping) -> Self:
+ value = self.get(session_state)
+ cls = self.__class__
+ if value is None:
+ self.bind(session_state)
+ return self
+ if not isinstance(value, cls):
+ raise ValueError(f"type {cls} can not find self in streamlit.session_state, {value} found")
+ return value
+
+ @abstractmethod
+ def bind(self, session_state: MutableMapping) -> None:
+ """
+ bind self to session_state
+ :param session_state: streamlit.session_state
+ """
+ pass
+
+
+class ModelSingleton(BaseModel, SessionStateValue, ABC):
+ """
+ use pydantic.BaseModel to define state value
+ """
+
+ @classmethod
+ def get(cls, session_state: MutableMapping) -> Optional[Self]:
+ """
+ load self value from session_state
+ :param session_state: the streamlit session state
+ :return: None if not bound yet
+ """
+ key = cls.session_key()
+ if key not in session_state:
+ return None
+ return session_state.get(key, None)
+
+ @classmethod
+ def session_key(cls) -> str:
+ return generate_import_path(cls)
+
+ def bind(self, session_state: MutableMapping) -> None:
+ key = self.session_key()
+ session_state[key] = self
+
+
+T = TypeVar('T')
+
+
+class Singleton:
+ """
+ session state singleton, key is the class type
+ """
+
+ def __init__(self, value: object, abstract: Optional[Type] = None):
+ self.value = value
+ if abstract is None:
+ abstract = type(value)
+ self.key = self.gen_key(abstract)
+
+ def bind(self, session_state: MutableMapping, force: bool = False) -> None:
+ """
+ :param session_state: streamlit session state
+ :param force: if False, only bind when target is not exists.
+ """
+ if force or self.key not in session_state:
+ session_state[self.key] = self.value
+
+ @classmethod
+ def get(cls, t: Type[T], session_state: MutableMapping, force: bool = True) -> T:
+ key = cls.gen_key(t)
+ if key not in session_state:
+ if force:
+ raise KeyError(f'key {key} not found in session state')
+ return None
+ value = session_state[key]
+ return value
+
+ @classmethod
+ def gen_key(cls, t: Type) -> str:
+ return generate_import_path(t)
+
+ @classmethod
+ def bound(cls, t: Type, session_state: MutableMapping) -> bool:
+ key = cls.gen_key(t)
+ return key in session_state
+
+ @classmethod
+ def release(cls, t: Type, session_state: MutableMapping) -> None:
+ key = cls.gen_key(t)
+ del session_state[key]
+
+
+class SingletonContracts:
+ def __init__(self, types: List[Type]):
+ self.types = types
+
+ def validate(self, session_state: MutableMapping) -> List[Type]:
+ unbound = []
+ for typ in self.types:
+ if not Singleton.bound(typ, session_state):
+ unbound.append(typ)
+ return unbound
+
+
+def expect(session_state: MutableMapping, key: str, value: Any) -> bool:
+ if key not in session_state:
+ return False
+ return value == session_state[key]
diff --git a/ghostos/prototypes/streamlitapp/widgets/__init__.py b/ghostos/prototypes/streamlitapp/widgets/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/ghostos/prototypes/streamlitapp/widgets/dialogs.py b/ghostos/prototypes/streamlitapp/widgets/dialogs.py
new file mode 100644
index 00000000..e1116d3e
--- /dev/null
+++ b/ghostos/prototypes/streamlitapp/widgets/dialogs.py
@@ -0,0 +1,96 @@
+import streamlit as st
+from ghostos.helpers import gettext as _, yaml_pretty_dump
+from ghostos.framework.messages import CompletionUsagePayload
+from ghostos.core.messages import Message
+from ghostos.prototypes.streamlitapp.widgets.renderer import render_empty
+
+
+@st.dialog(title=_("Code"), width="large")
+def open_code_dialog(title: str, code: str):
+ st.subheader(title)
+ st.code(code, line_numbers=True, wrap_lines=True)
+ render_empty()
+
+
+@st.dialog(title=_("Task Info"), width="large")
+def open_task_info_dialog(task_id: str):
+ from ghostos.prototypes.streamlitapp.widgets.renderer import render_task_by_id
+ render_task_by_id(task_id)
+ render_empty()
+
+
+@st.dialog(title=_("Token Usage"), width="large")
+def open_completion_usage_dialog(completion: CompletionUsagePayload):
+ import streamlit_react_jsonschema as srj
+ srj.pydantic_instance_form(completion, readonly=True)
+ render_empty()
+
+
+@st.dialog(title=_("Message Detail"), width="large")
+def open_message_dialog(message: Message):
+ st.json(message.model_dump_json(indent=2))
+
+
+@st.dialog(title=_("Prompt Info"), width="large")
+def open_prompt_info_dialog(prompt_id: str):
+ import streamlit_react_jsonschema as srj
+ from ghostos.prototypes.streamlitapp.utils.session import Singleton
+ from ghostos.prototypes.streamlitapp.widgets.messages import render_messages
+ from ghostos.container import Container
+ from ghostos.core.llms import PromptStorage
+
+ prefix = "prompt-info"
+ container = Singleton.get(Container, st.session_state)
+ storage = container.get(PromptStorage)
+ if storage is None:
+ st.error(f"Prompt storage is not initialized")
+ return
+ prompt = storage.get(prompt_id)
+ if prompt is None:
+ st.error(f"Prompt {prompt_id} not found")
+ return
+
+ # description
+ desc = prompt.model_dump(include={"id", "description"})
+ st.markdown(f"""
+```yaml
+{yaml_pretty_dump(desc)}
+```
+""")
+
+ # model info
+ if prompt.model:
+ st.subheader("Model Info")
+ srj.pydantic_instance_form(prompt.model, readonly=True)
+
+ # prompt error
+ if prompt.error:
+ st.subheader("Prompt error")
+ st.error(prompt.error)
+
+ # prompt functions
+ if prompt.functions:
+ st.subheader(_("Functions"))
+ with st.container(border=True):
+ for func in prompt.functions:
+ with st.expander(func.name):
+ st.write(func.model_dump())
+
+ system_prompt = prompt.system_prompt()
+ st.subheader("System Prompt")
+ with st.container(border=True):
+ st.markdown(system_prompt)
+
+ if prompt.history:
+ st.subheader(_("History"))
+ with st.container(border=True):
+ render_messages(prompt.history, False, False, prefix=prefix)
+ if prompt.inputs:
+ st.subheader(_("Input"))
+ with st.container(border=True):
+ render_messages(prompt.inputs, False, False, prefix)
+ if prompt.added:
+ st.subheader(_("Added"))
+ with st.container(border=True):
+ render_messages(prompt.added, False, False, prefix)
+ render_empty()
diff --git a/ghostos/prototypes/streamlitapp/widgets/docs.py b/ghostos/prototypes/streamlitapp/widgets/docs.py
new file mode 100644
index 00000000..3f287cab
--- /dev/null
+++ b/ghostos/prototypes/streamlitapp/widgets/docs.py
@@ -0,0 +1,16 @@
+import streamlit as st
+from ghostos.prototypes.streamlitapp.resources import get_app_docs, get_app_conf
+
+
+def help_document(doc_name: str, label="help"):
+ is_helping = get_app_conf().BoolOpts.HELP_MODE.get()
+ with st.expander(label=label, expanded=is_helping):
+ doc = get_app_docs().read(doc_name)
+ st.markdown(doc, unsafe_allow_html=True)
+
+
+def markdown_document(doc_name: str, **kwargs):
+ doc = get_app_docs().read(doc_name)
+ if kwargs:
+ doc = doc.format(**kwargs)
+ st.markdown(doc, unsafe_allow_html=True)
diff --git a/ghostos/prototypes/streamlitapp/widgets/exec_frame.py b/ghostos/prototypes/streamlitapp/widgets/exec_frame.py
new file mode 100644
index 00000000..580f1a6b
--- /dev/null
+++ b/ghostos/prototypes/streamlitapp/widgets/exec_frame.py
@@ -0,0 +1,90 @@
+from typing import List, Union, Dict, Iterable, Tuple
+import streamlit_antd_components as sac
+from ghostos.core.aifunc import ExecFrame, ExecStep
+
+
+def flatten_exec_frame_tree(frame: ExecFrame) -> Dict[str, Union[ExecFrame, ExecStep]]:
+ def iter_frame(fr: ExecFrame, bloodline: List[int]) -> Iterable[Tuple[str, Union[ExecFrame, ExecStep]]]:
+ yield __frame_label(fr, bloodline), fr
+ idx = 0
+ for step in fr.steps:
+ idx += 1
+ next_bloodline = bloodline.copy()
+ next_bloodline.append(idx)
+ yield from iter_step(step, next_bloodline)
+
+ def iter_step(step: ExecStep, bloodline: List[int]) -> Iterable[Tuple[str, Union[ExecStep, ExecFrame]]]:
+ yield __step_label(step, bloodline), step
+ idx = 0
+ for fra in step.frames:
+ idx += 1
+ next_bloodline = bloodline.copy()
+ next_bloodline.append(idx)
+ yield from iter_frame(fra, next_bloodline)
+
+ result = {}
+ for key, value in iter_frame(frame.model_copy(), []):
+ result[key] = value
+ return result
+
+
+def render_exec_frame_tree(label: str, frame: ExecFrame):
+ root = build_exec_frame_tree_node(frame.model_copy(), [])
+ return sac.tree(
+ [root],
+ label=label,
+ size="lg",
+ open_all=True,
+ show_line=True,
+ )
+
+
+def build_exec_frame_tree_node(frame: ExecFrame, bloodline: List[int]) -> sac.TreeItem:
+ children = []
+ if len(bloodline) < 20:
+ steps = frame.steps
+ idx = 0
+ for step in steps:
+ idx += 1
+ next_bloodline = bloodline.copy()
+ next_bloodline.append(idx)
+ step_node = build_exec_step_tree_node(step, next_bloodline)
+ children.append(step_node)
+ return sac.TreeItem(
+ label=__frame_label(frame, bloodline),
+ icon="stack",
+ tooltip=f"click to see the frame details",
+ children=children,
+ )
+
+
+def build_exec_step_tree_node(step: ExecStep, bloodline: List[int]) -> sac.TreeItem:
+ children = []
+ if len(bloodline) < 20:
+ idx = 0
+ for frame in step.frames:
+ idx += 1
+ next_bloodline = bloodline.copy()
+ next_bloodline.append(idx)
+ frame_node = build_exec_frame_tree_node(frame, next_bloodline)
+ children.append(frame_node)
+ return sac.TreeItem(
+ __step_label(step, bloodline),
+ icon="circle" if len(children) == 0 else "plus-circle",
+ tooltip=f"click to see the step details",
+ children=children,
+ )
+
+
+def __frame_label(frame: ExecFrame, bloodline: List[int]) -> str:
+ suffix = ""
+ if len(bloodline) > 0:
+ suffix = "__" + "_".join([str(c) for c in bloodline])
+ return frame.func_name() + suffix
+
+
+def __step_label(step: ExecStep, bloodline: List[int]) -> str:
+ suffix = ""
+ if len(bloodline) > 0:
+ suffix = "__" + "_".join([str(c) for c in bloodline])
+ return step.func_name() + suffix
diff --git a/ghostos/prototypes/streamlitapp/widgets/messages.py b/ghostos/prototypes/streamlitapp/widgets/messages.py
new file mode 100644
index 00000000..47fb2017
--- /dev/null
+++ b/ghostos/prototypes/streamlitapp/widgets/messages.py
@@ -0,0 +1,261 @@
+import streamlit_antd_components as sac
+import streamlit as st
+from typing import Iterable, List, NamedTuple
+from ghostos.core.messages import (
+ Message, Role, MessageType, FunctionCaller,
+ ImageAssetMessage,
+)
+from ghostos.framework.messages import CompletionUsagePayload, TaskPayload, PromptPayload
+from ghostos.helpers import gettext as _
+
+
+class MessageGroup(NamedTuple):
+ msg_name: str
+ msg_role: str
+ stage: str
+ messages: List[Message]
+
+
+def render_messages(messages: Iterable[Message], debug: bool, in_expander: bool, prefix: str = ""):
+ groups: List[MessageGroup] = []
+ group = MessageGroup("", "", "", [])
+
+ for msg in messages:
+ if not msg.is_complete():
+ continue
+ if msg.name != group.msg_name or msg.role != group.msg_role or msg.stage != group.stage:
+ if group.messages:
+ groups.append(group)
+ group = MessageGroup(msg.name or "", msg.role, msg.stage, [])
+ group.messages.append(msg)
+
+ if group.messages:
+ groups.append(group)
+ for group in groups:
+ render_message_group(group, debug, in_expander, prefix)
+
+
+def render_message_group(group: MessageGroup, debug: bool, in_expander: bool, prefix: str = ""):
+ role = group.msg_role
+ name = group.msg_name
+ stage = group.stage
+ caption = f"{role}: {name}" if name else role
+ render_role = "user" if role == Role.USER.value else "assistant"
+ with st.container():
+ if stage:
+ with st.expander(stage, expanded=False):
+ with st.chat_message(render_role):
+ st.caption(caption)
+ for msg in group.messages:
+ render_message_in_content(msg, debug, prefix=prefix, in_expander=True)
+ else:
+ with st.chat_message(render_role):
+ st.caption(caption)
+ for msg in group.messages:
+ render_message_in_content(msg, debug, prefix=prefix, in_expander=in_expander)
+
+
+def render_message_payloads(message: Message, debug: bool, prefix: str = ""):
+ from ghostos.prototypes.streamlitapp.widgets.dialogs import (
+ open_task_info_dialog, open_completion_usage_dialog, open_prompt_info_dialog,
+ open_message_dialog,
+ )
+
+ if not debug:
+ st.empty()
+ return
+ msg_id = message.msg_id
+ with st.container():
+ col1, col2, col3, col4 = st.columns(4)
+ with col1:
+ if st.button(label="Detail", key="Detail" + msg_id):
+ open_message_dialog(message)
+
+ with col2:
+ task_payload = TaskPayload.read_payload(message)
+ if task_payload and st.button(label="Task Info", key="Task Info" + msg_id):
+ open_task_info_dialog(task_payload.task_id)
+
+ with col3:
+ completion_usage = CompletionUsagePayload.read_payload(message)
+ if completion_usage and st.button(label="Token usage", key="token usage" + msg_id):
+ open_completion_usage_dialog(completion_usage)
+
+ with col4:
+ prompt_payload = PromptPayload.read_payload(message)
+ if prompt_payload and st.button(label="Prompt Info", key="Prompt Info" + msg_id):
+ open_prompt_info_dialog(prompt_payload.prompt_id)
+
+
+def render_message_in_content(message: Message, debug: bool, in_expander: bool, *, prefix: str = ""):
+ if message.type == MessageType.ERROR.value:
+ st.error(f"Error: {message.content}")
+
+ elif MessageType.is_text(message):
+ st.markdown(message.content)
+
+ elif MessageType.FUNCTION_CALL.match(message):
+ callers = FunctionCaller.from_message(message)
+ render_message_caller(callers, debug, in_expander)
+
+ elif MessageType.FUNCTION_OUTPUT.match(message):
+ render_message_caller_output(message, debug, in_expander)
+ # todo: more types
+ elif MessageType.IMAGE.match(message):
+ # render image type message
+ render_image_message(message)
+ elif MessageType.AUDIO.match(message):
+ render_audio_message(message)
+ else:
+ st.write(message.model_dump(exclude_defaults=True))
+ if message.callers:
+ render_message_caller(message.callers, debug, in_expander)
+
+ render_message_payloads(message, debug, prefix)
+ st.empty()
+
+
+def render_audio_message(message: Message):
+ from ghostos.prototypes.streamlitapp.resources import get_audio_assets
+ if message.content:
+ st.markdown(message.content)
+
+ assets = get_audio_assets()
+ file, data = assets.get_file_and_binary_by_id(message.msg_id)
+ if data is not None:
+ # st.write(data)
+ st.audio(data)
+
+
+def render_image_message(message: Message):
+ from ghostos.prototypes.streamlitapp.resources import get_images_from_image_asset
+ if message.type != MessageType.IMAGE.value:
+ return
+ image_msg = ImageAssetMessage.from_message(message)
+ content = image_msg.content
+ # render content first
+ st.markdown(content)
+ image_ids = [image_id.image_id for image_id in image_msg.attrs.images]
+ got = get_images_from_image_asset(image_ids)
+ for image_info, binary in got.values():
+ if binary:
+ st.image(binary)
+ elif image_info.url:
+ st.image(image_info.url)
+
+
+def render_message_caller_output(message: Message, debug: bool, in_expander: bool):
+ if not in_expander:
+ with st.expander("Caller Output", expanded=debug):
+ st.caption(f"function {message.name or message.call_id} output:")
+ st.write(message.content)
+ else:
+ st.caption(f"function {message.name or message.call_id} output:")
+ st.write(message.content)
+
+
+def render_message_caller(callers: Iterable[FunctionCaller], debug: bool, in_expander: bool):
+ if not in_expander:
+ with st.expander("Callers", expanded=debug):
+ _render_message_caller(callers)
+ else:
+ _render_message_caller(callers)
+
+
+def _render_message_caller(callers: Iterable[FunctionCaller]):
+ from ghostos.ghosts.moss_agent import MossAction
+ for caller in callers:
+ if caller.name == MossAction.DEFAULT_NAME:
+ st.caption(f"function call: {caller.name}")
+ code = MossAction.unmarshal_arguments(caller.arguments)
+ st.code(code)
+ else:
+ st.caption(f"function call: {caller.name}")
+ st.write(caller.arguments)
+
+
+def render_message_item(msg: Message, debug: bool):
+ if not msg.is_complete():
+ return
+ if MessageType.ERROR.match(msg):
+ with st.chat_message("user"):
+ st.caption(_("Error"))
+ st.error(msg.get_content())
+ return
+ if msg.role == Role.ASSISTANT.value:
+ render_ai_message(msg, debug)
+ elif msg.role == Role.USER.value:
+ render_user_message(msg, debug)
+ elif msg.role == Role.SYSTEM.value:
+ render_sys_message(msg, debug)
+ elif msg.role == Role.FUNCTION.value:
+ render_func_message(msg, debug)
+ else:
+ render_other_message(msg, debug)
+
+
+def render_ai_message(msg: Message, debug: bool):
+ content = msg.content
+ if not content:
+ return
+ replacements = {
+ "": "\n```python\n",
+ "": "\n```\n",
+ "": "\n```python\n",
+ " ": "\n```\n",
+ }
+ for key, value in replacements.items():
+ content = content.replace(key, value)
+
+ with st.chat_message("ai"):
+ if msg.type:
+ st.caption(msg.type)
+ if msg.name:
+ st.caption(msg.name)
+ st.markdown(content, unsafe_allow_html=True)
+ if debug:
+ render_msg_debug(msg)
+
+
+def render_msg_debug(msg: Message):
+ with st.expander(label=_("debug"), expanded=False):
+ st.json(msg.model_dump_json(exclude_defaults=True, indent=2))
+
+
+def render_user_message(msg: Message, debug: bool):
+ content = msg.get_content()
+ with st.chat_message("user"):
+ if msg.name:
+ st.caption(msg.name)
+ if msg.type:
+ st.caption(msg.type)
+ st.markdown(content, unsafe_allow_html=True)
+
+
+def render_sys_message(msg: Message, debug: bool):
+ content = msg.content
+ with st.chat_message("user"):
+ st.caption("system message")
+ st.markdown(content, unsafe_allow_html=True)
+ if debug:
+ render_msg_debug(msg)
+
+
+def render_func_message(msg: Message, debug: bool):
+ content = msg.content
+ with st.expander(_("function"), expanded=False):
+ if msg.name:
+ st.caption(msg.name)
+ st.markdown(content, unsafe_allow_html=True)
+ if debug:
+ render_msg_debug(msg)
+
+
+def render_other_message(msg: Message, debug: bool):
+ content = msg.content
+ with st.expander(_("other"), expanded=False):
+ if msg.name:
+ st.caption(msg.name)
+ st.markdown(content, unsafe_allow_html=True)
+ if debug:
+ render_msg_debug(msg)
diff --git a/ghostos/prototypes/streamlitapp/widgets/moss.py b/ghostos/prototypes/streamlitapp/widgets/moss.py
new file mode 100644
index 00000000..599ae697
--- /dev/null
+++ b/ghostos/prototypes/streamlitapp/widgets/moss.py
@@ -0,0 +1,19 @@
+import streamlit as st
+from ghostos.core.moss import PyContext
+from ghostos.helpers import gettext as _
+
+
+def render_pycontext(pycontext: PyContext):
+ if not pycontext:
+ return
+ st.subheader("PyContext")
+ if pycontext.module:
+ st.caption(f"module: {pycontext.module}")
+ if pycontext.code:
+ with st.expander(_("Code"), expanded=True):
+ st.code(pycontext.code)
+ if pycontext.execute_code:
+ with st.expander(_("Execute"), expanded=True):
+ st.code(pycontext.execute_code)
+ st.write(f"executed: {pycontext.executed}")
+ st.divider()
diff --git a/ghostos/prototypes/streamlitapp/widgets/navigators.py b/ghostos/prototypes/streamlitapp/widgets/navigators.py
new file mode 100644
index 00000000..8a8f719e
--- /dev/null
+++ b/ghostos/prototypes/streamlitapp/widgets/navigators.py
@@ -0,0 +1,11 @@
+import streamlit as st
+from ghostos.prototypes.streamlitapp.utils.route import Router
+from ghostos.prototypes.streamlitapp.utils.session import Singleton
+
+
+def application_navigator_menu():
+ router = Singleton.get(Router, st.session_state)
+ menu = router.default_antd_menu_items()
+ route = router.render_antd_menu(menu)
+ if route and route.label() != route.label_of_current_page():
+ route.switch_page()
diff --git a/ghostos/prototypes/streamlitapp/widgets/renderer.py b/ghostos/prototypes/streamlitapp/widgets/renderer.py
new file mode 100644
index 00000000..318b395d
--- /dev/null
+++ b/ghostos/prototypes/streamlitapp/widgets/renderer.py
@@ -0,0 +1,179 @@
+from typing import TypeVar, Tuple
+import streamlit as st
+import streamlit_react_jsonschema as srj
+from pydantic import BaseModel
+from ghostos.helpers import generate_import_path, yaml_pretty_dump
+from ghostos.streamlit import render_streamlit_object, StreamlitRenderer
+from ghostos.core.runtime import (
+ TaskBrief, GoTasks, GoTaskStruct,
+ GoThreads, GoThreadInfo, Turn,
+ Event,
+)
+from ghostos.prototypes.streamlitapp.utils.session import Singleton
+from ghostos.container import Container
+from ghostos.helpers import gettext as _
+import inspect
+
+__all__ = [
+ 'render_object',
+ 'render_task', 'render_task_by_id',
+ 'render_thread', 'render_event', 'render_turn', 'render_event_object',
+ 'render_empty',
+]
+
+T = TypeVar('T')
+
+
+def render_task_by_id(task_id: str):
+ container = Singleton.get(Container, st.session_state)
+ tasks = container.force_fetch(GoTasks)
+ task = tasks.get_task(task_id)
+ if task is None:
+ st.info(f"Task {task_id} not found")
+ st.empty()
+ return
+ render_task(task)
+
+
+def render_empty():
+ for i in range(20):
+ st.empty()
+
+
+def render_task(task: GoTaskStruct):
+ brief = TaskBrief.from_task(task)
+ srj.pydantic_instance_form(brief, readonly=True)
+
+ with st.expander(_("Detail"), expanded=False):
+ st.write(task.model_dump(exclude_defaults=True))
+
+ if task.children:
+ st.subheader("Subtasks")
+ st.write("todo")
+
+ container = Singleton.get(Container, st.session_state)
+ threads = container.force_fetch(GoThreads)
+ thread = threads.get_thread(task.thread_id)
+ st.subheader("Thread Info")
+ if thread is None:
+ st.info(f"Thread {task.thread_id} is not created yet")
+ st.empty()
+ return
+ with st.container(border=True):
+ render_thread(thread, prefix="render_thread_in_task", debug=False)
+
+
+def render_thread(thread: GoThreadInfo, max_turn: int = 20, prefix: str = "", debug: bool = False):
+ st.subheader("Thread Info")
+ turns = list(thread.turns(truncate=True))
+ turns = turns[-max_turn:]
+ count = 0
+ for turn in turns:
+ count += render_turn(turn, debug, prefix)
+ if count == 0:
+ st.info("No thread messages yet")
+
+
+def render_turn(turn: Turn, debug: bool, prefix: str = "") -> int:
+ from ghostos.prototypes.streamlitapp.widgets.messages import render_messages
+ if turn.summary is not None:
+ st.info("summary:\n" + turn.summary)
+
+ if turn.is_from_inputs() or turn.is_from_self():
+ messages = list(turn.messages(False))
+ render_messages(messages, debug, in_expander=False, prefix=prefix)
+ return len(messages)
+ # from other task
+ else:
+ event = turn.event
+ sub_title = _("background run")
+ if event is not None:
+ sub_title = _("background event: ") + event.type
+ with st.expander(sub_title, expanded=False):
+ event_messages = turn.event_messages()
+ render_messages(event_messages, debug, in_expander=True)
+ render_event_object(event, debug)
+ if turn.added:
+ render_messages(turn.added, debug, in_expander=False)
+ messages = list(turn.messages(False))
+ return len(messages)
+
+
+def render_event(event: Event, debug: bool):
+ from ghostos.prototypes.streamlitapp.widgets.messages import render_messages
+ if event is None:
+ return
+ if event.callback:
+ sub_title = _("background event: ") + event.type
+ with st.expander(sub_title, expanded=False):
+ messages = event.iter_message(show_instruction=True)
+ render_messages(messages, debug, in_expander=True)
+ else:
+ messages = event.iter_message(show_instruction=True)
+ with st.container():
+ render_messages(messages, debug, in_expander=False)
+
+
+def render_event_object(event: Event, debug: bool):
+ if event is None:
+ return
+ from_task_name = event.from_task_name
+ if debug and from_task_name is not None:
+ st.button(f"from task {from_task_name}", key=f"from task {event.event_id}")
+
+
+def render_object(obj: T, immutable: bool = False) -> Tuple[T, bool]:
+ """
+ render an object in a streamlit compatible way.
+ :param obj: the object to render
+ :param immutable: is the object immutable?
+ :return: [value: obj after rendering, changed: is object changed]
+ """
+ if obj is None:
+ st.info("None")
+
+ container = Singleton.get(Container, st.session_state)
+ renderer = container.get(StreamlitRenderer)
+ if renderer:
+ r = renderer.render(obj)
+ if r is not None:
+ return r.value, r.changed
+
+ if r := render_streamlit_object(obj):
+ return r.value, r.changed
+
+ if inspect.isclass(obj):
+ source = inspect.getsource(obj)
+ st.subheader(f"Class {generate_import_path(obj)}")
+ st.code(source)
+ return obj, False
+ elif inspect.isfunction(obj):
+ source = inspect.getsource(obj)
+ st.subheader(f"Function {generate_import_path(obj)}")
+ st.code(source)
+ return obj, False
+ elif isinstance(obj, BaseModel):
+ obj, submitted = srj.pydantic_instance_form(obj, readonly=immutable)
+ return obj, submitted
+ elif isinstance(obj, dict):
+ st.subheader("Dictionary")
+ st.markdown(f"""
+```yaml
+{yaml_pretty_dump(obj)}
+```
+""")
+ return obj, False
+ elif isinstance(obj, list):
+ st.subheader("List")
+ st.markdown(f"""
+```yaml
+{yaml_pretty_dump(obj)}
+```
+""")
+ return obj, False
+ else:
+ type_ = type(obj)
+ st.subheader(f"Type {generate_import_path(type_)}")
+ with st.container(border=True):
+ st.write(obj)
+ return obj, False
diff --git a/ghostos/scripts/aifunc_test.py b/ghostos/scripts/aifunc_test.py
deleted file mode 100644
index 92402a0a..00000000
--- a/ghostos/scripts/aifunc_test.py
+++ /dev/null
@@ -1,154 +0,0 @@
-import argparse
-import sys
-import os
-import yaml
-from typing import List, Dict
-
-from ghostos.core.session import MsgThread
-from ghostos.scripts.logconf import prepare_logger
-from ghostos.core.llms import Chat
-from ghostos.core.messages import Message
-from ghostos.core.moss import test_container
-from ghostos.core.aifunc import DefaultAIFuncManagerImpl, AIFunc, DefaultAIFuncDriverImpl, AIFuncManager
-from ghostos.framework.logger import NamedLoggerProvider
-from ghostos.framework.storage import FileStorageProvider
-from ghostos.framework.llms import ConfigBasedLLMsProvider
-from ghostos.framework.threads import StorageThreadsProvider
-from ghostos.container import Container
-from ghostos.contracts.modules import Modules
-from ghostos.contracts.storage import Storage
-from ghostos.framework.configs import ConfigsByStorageProvider
-from ghostos.helpers import import_from_path, yaml_pretty_dump
-from rich.console import Console
-from rich.panel import Panel
-from rich.markdown import Markdown
-from rich.prompt import Prompt
-
-console = Console()
-
-prepare_logger()
-
-
-def prepare_container(root_dir: str) -> Container:
- container = test_container()
- container.register(FileStorageProvider(root_dir))
- container.register(NamedLoggerProvider(logger_name="debug"))
- container.register(StorageThreadsProvider(threads_dir='runtime/threads'))
- container.register(ConfigsByStorageProvider("configs"))
- container.register(ConfigBasedLLMsProvider("llms_conf.yml"))
- return container
-
-
-def main() -> None:
- parser = argparse.ArgumentParser(
- description="run ghostos aifunc test cases, show results",
- )
- parser.add_argument(
- "--case", '-c',
- help="ghostos aifunc test case name in demo/tests/aifunc_tests.yml",
- type=str,
- default="swe_bench_lite",
- )
- parser.add_argument(
- "--import_path", '-i',
- help="the import path of the AIFunc instance, such as foo.bar:baz",
- type=str,
- # 默认使用专门测试 MossTestSuite 的文件.
- # default="ghostos.core.aifunc.examples.agentic:example",
- default="",
- )
- parser.add_argument(
- "--llm_api", '-l',
- help="the llm api name",
- type=str,
- # 默认使用专门测试 MossTestSuite 的文件.
- default="",
- )
- parser.add_argument(
- "--auto", '-a',
- help="auto run the test or stop at each generations",
- action="store_true",
- # 默认使用专门测试 MossTestSuite 的文件.
- default=True,
- )
-
- parsed = parser.parse_args(sys.argv[1:])
- llm_api = parsed.llm_api
- demo_dir = os.path.abspath(os.path.dirname(__file__) + "/../../demo")
- container = prepare_container(demo_dir)
- import_path = parsed.import_path
- if parsed.case:
- storage = container.force_fetch(Storage)
- cases_content_file = storage.get("tests/aifunc_tests.yml")
- cases: Dict[str, str] = yaml.safe_load(cases_content_file)
- import_path = cases.get(parsed.case, import_path)
- if not import_path:
- raise Exception("no aifunc test cases found. use -c or -i to specify aifunc test case")
-
- class TestDriverImpl(DefaultAIFuncDriverImpl):
- console = console
-
- def on_message(self, message: Message) -> None:
- self.console.print(
- Panel(
- Markdown(message.get_content()),
- title=f"generated message ({self.name()})",
- )
- )
- if not parsed.auto:
- value = Prompt.ask("Continue?", choices=["y", "n"], default="y")
- if value != "y":
- exit(0)
-
- def on_chat(self, chat: Chat) -> None:
- for message in chat.get_messages():
- self.console.print(Panel(
- Markdown(message.get_content()),
- title=f"chat_info ({self.name()})",
- ))
- if not parsed.auto:
- value = Prompt.ask("Continue?", choices=["y", "n"], default="y")
- if value != "y":
- exit(0)
-
- def on_system_messages(self, messages: List[Message]) -> None:
- pass
-
- def on_save(self, manager: AIFuncManager, thread: MsgThread) -> None:
- current = thread.current
- if current:
- for message in current.messages():
- self.console.print(
- Panel(
- Markdown(message.get_content()),
- title="thread new round message",
- )
- )
- super().on_save(manager, thread)
-
- manager_ = DefaultAIFuncManagerImpl(
- container=container,
- llm_api_name=llm_api,
- default_driver=TestDriverImpl,
- )
- modules = container.force_fetch(Modules)
- aifunc = import_from_path(import_path, modules.import_module)
- if not isinstance(aifunc, AIFunc):
- raise AttributeError(f'aifunc must be an instance of {AIFunc}, {aifunc} given')
-
- driver = manager_.get_driver(aifunc)
- # print initialized thread.
- thread_ = driver.initialize()
- thread_content = yaml_pretty_dump(thread_.model_dump(exclude_defaults=True))
- console.print(Panel(
- Markdown(f"```markdown\n{thread_content}\n```"),
- title="initialized thread",
- ))
-
- result = manager_.execute(aifunc)
- console.print(result)
- manager_.destroy()
-
-
-if __name__ == "__main__":
- main()
diff --git a/ghostos/scripts/clear.py b/ghostos/scripts/clear.py
new file mode 100644
index 00000000..42e1a7e0
--- /dev/null
+++ b/ghostos/scripts/clear.py
@@ -0,0 +1,86 @@
+from os.path import join
+import argparse
+import sys
+import os
+
+"""
+this script is used to clear the local file cache in runtime directory
+"""
+
+__all__ = ['clear_directory', 'clear_runtime', 'clear_assets']
+
+ignore_patterns = ['.gitignore']
+
+
+def clear_directory(directory: str, recursive=True, depth: int = 0) -> int:
+ """
+ clear all files in directory recursively except the files match any of ignore_patterns
+ :param directory: the target directory
+ :param recursive: recursively clear all files in directory
+ :param depth: the depth of recursion
+ :return: number of files cleared
+ """
+
+ cleared_files_count = 0
+
+ print("search file at %s" % directory)
+ for root, dirs, files in os.walk(directory):
+ for name in files:
+ if name not in ignore_patterns:
+ file_path = os.path.join(root, name)
+ try:
+ print(f"- remove file: {file_path}")
+ os.remove(file_path)
+ cleared_files_count += 1
+ except Exception as e:
+ print(f"Error deleting file {file_path}: {e}")
+
+ if not recursive:
+ break
+ for dir_path in dirs:
+ real_dir_path = os.path.join(root, dir_path)
+ clear_directory(real_dir_path, recursive=recursive, depth=depth + 1)
+ # os.rmdir(real_dir_path)
+
+ return cleared_files_count
+
+
+def clear_assets(sub_path: str) -> int:
+ from ghostos.bootstrap import get_bootstrap_config
+ bootstrap_config = get_bootstrap_config()
+ asserts_dir = bootstrap_config.abs_asserts_dir()
+
+ target_dir = asserts_dir
+ if sub_path:
+ target_dir = join(target_dir, sub_path)
+
+ cleared = clear_directory(target_dir, recursive=True)
+ return cleared
+
+
+def clear_runtime(sub_path: str) -> int:
+ from ghostos.bootstrap import get_bootstrap_config
+ bootstrap_config = get_bootstrap_config()
+ runtime_dir = bootstrap_config.abs_runtime_dir()
+
+ target_dir = runtime_dir
+ if sub_path:
+ target_dir = join(target_dir, sub_path)
+
+ cleared = clear_directory(target_dir, recursive=True)
+ return cleared
+
+
+def main():
+ parser = argparse.ArgumentParser(
+ description="clear temp files in workspace runtime directories",
+ )
+ parser.add_argument(
+ "--path", "-p",
+ help="the target directory path, if not given, clear every sub directory in runtime.",
+ type=str,
+ required=False,
+ default=None,
+ )
+ parsed = parser.parse_args(sys.argv[1:])
+ clear_runtime(parsed.path)
diff --git a/ghostos/scripts/cli/__init__.py b/ghostos/scripts/cli/__init__.py
new file mode 100644
index 00000000..fa885751
--- /dev/null
+++ b/ghostos/scripts/cli/__init__.py
@@ -0,0 +1,168 @@
+from __future__ import annotations
+
+import click
+import sys
+from os import getcwd
+from os.path import join, abspath
+from rich.console import Console
+from rich.markdown import Markdown
+from rich.panel import Panel
+from rich.prompt import Prompt
+
+
+@click.group()
+def main():
+ pass
+
+
+@main.command("web")
+@click.argument("python_file_or_module")
+def start_streamlit_web(python_file_or_module: str):
+ """
+ turn a python file or module into a streamlit web agent
+ """
+ from ghostos.scripts.cli.run_streamlit_app import start_ghost_app
+ from ghostos.scripts.cli.utils import find_ghost_by_file_or_module
+ ghost_info, module, filename, is_temp = find_ghost_by_file_or_module(python_file_or_module)
+ start_ghost_app(ghost_info, module.__name__, filename, is_temp)
+
+
+@main.command("console")
+@click.argument("python_file_or_module")
+def start_console_app(python_file_or_module: str):
+ """
+ turn a python file or module into a console agent
+ """
+ from ghostos.scripts.cli.run_console import run_console_app
+ run_console_app(python_file_or_module)
+
+
+@main.command("config")
+def start_web_config():
+ """
+ config the ghostos in streamlit web app
+ """
+ from ghostos.scripts.cli.run_streamlit_app import start_streamlit_prototype_cli
+ from ghostos.bootstrap import get_bootstrap_config
+ start_streamlit_prototype_cli("run_configs.py", "", get_bootstrap_config().workspace_dir)
+
+
+@main.command("clear-runtime")
+@click.option("--path", default="", show_default=True)
+def clear_runtime(path: str):
+ """
+ clear workspace runtime files
+ """
+ from ghostos.scripts.clear import clear_runtime
+ from ghostos.bootstrap import get_bootstrap_config
+ conf = get_bootstrap_config()
+ confirm = Prompt.ask(
+ f"Will clear all workspace runtime files at {conf.abs_runtime_dir()}\n\nWould you like to proceed? [y/N]",
+ choices=["y", "n"],
+ default="y",
+ )
+ if confirm == "y":
+ clear_runtime(path)
+ else:
+ print("Aborted")
+
+
+@main.command("init")
+@click.option("--path", default="", show_default=True)
+def init_app(path: str):
+ """
+ init ghostos workspace
+ """
+ from ghostos.scripts.copy_workspace import copy_workspace
+ from ghostos.bootstrap import expect_workspace_dir, app_stub_dir, get_bootstrap_config
+ console = Console()
+ console.print(Panel(
+ Markdown("""
+`GhostOS` need an `app` directory as workspace.
+
+The Workspace meant to save local files such as configs, logs, cache files.
+"""),
+ title="Initialize GhostOS",
+ ))
+
+ conf = get_bootstrap_config(local=False)
+ workspace_dir, ok = expect_workspace_dir()
+ app_dir = workspace_dir.rstrip('/').split("/")[-1]
+ result = Prompt.ask(
+ f"\n>> will init ghostos workspace at `{getcwd()}`. input directory name:",
+ default=app_dir,
+ )
+ source_dir = join(conf.ghostos_dir, "ghostos/app")
+ real_workspace_dir = abspath(result)
+ console.print("start to init ghostos workspace")
+ copy_workspace(source_dir, real_workspace_dir)
+ console.print("ghostos workspace copied")
+
+ if conf.workspace_dir != real_workspace_dir:
+ conf.workspace_dir = real_workspace_dir
+ conf.save(getcwd())
+ console.print("save .ghostos.yml")
+ console.print(Panel(Markdown(f"""
+Done create workspace!
+
+`GhostOS` use OpenAI model service and `gpt-4o` model as default LLM model.
+It need environment variable `OPENAI_API_KEY` to call the service.
+
+You can provide your OpenAI API key by:
+
+```bash
+export OPENAI_API_KEY=
+```
+
+Or:
+1. copy `{conf.env_example_file()}` to `{conf.env_file()}, and offer the necessary env configuration. (Optional)
+2. run `ghostos configs` to configure them manually. (Optional)
+
+Finally you can run `ghostos web ghostos.demo.agents.jojo` to test if everything is working.
+""")))
+
+
+@main.command("docs")
+def open_docs():
+ """See GhostOS Docs"""
+ from ghostos.bootstrap import get_bootstrap_config
+ conf = get_bootstrap_config()
+ console = Console()
+ m = Markdown(f"""
+You can open ghostos documentation:
+
+* zh-cn: https://github.com/ghost-in-moss/GhostOS/docs/zh-cn/README.md
+* en: https://github.com/ghost-in-moss/GhostOS/docs/en/README.md
+
+or `git clone https://github.com/ghost-in-moss/GhostOS` then `docsify serve`
+"""
+ )
+ console.print(Panel(
+ m,
+ title="GhostOS docs",
+ ))
+
+
+@main.command("help")
+def ghostos_help():
+ """Print this help message."""
+ _get_command_line_as_string()
+
+ assert len(sys.argv) == 2 # This is always true, but let's assert anyway.
+ # Pretend user typed 'streamlit --help' instead of 'streamlit help'.
+ sys.argv[1] = "--help"
+ main(prog_name="main")
+
+
+# copy from streamlit app, thanks
+def _get_command_line_as_string() -> str | None:
+ """Print this help message."""
+ import sys
+ import subprocess
+ parent = click.get_current_context().parent
+ if parent is None:
+ return None
+
+ cmd_line_as_list = [parent.command_path]
+ cmd_line_as_list.extend(sys.argv[1:])
+ return subprocess.list2cmdline(cmd_line_as_list)
diff --git a/ghostos/scripts/cli/run_aifunc.py b/ghostos/scripts/cli/run_aifunc.py
new file mode 100644
index 00000000..6cadf2d0
--- /dev/null
+++ b/ghostos/scripts/cli/run_aifunc.py
@@ -0,0 +1,77 @@
+from sys import argv
+from os import path
+from typing import Optional, NamedTuple
+from ghostos.helpers import import_from_path
+from ghostos.scripts.cli.utils import check_ghostos_workspace_exists
+from ghostos.prototypes.streamlitapp import cli
+from ghostos.core.aifunc import AIFunc
+from pydantic import BaseModel, Field
+from ghostos.helpers import create_module, generate_import_path
+from streamlit.web.cli import main_run
+
+import inspect
+import sys
+
+
+class RunAIFuncApp(BaseModel):
+ modulename: str = Field(description="expect aifunc modulename")
+ filename: str = Field(description="expect aifunc filename")
+ import_path: str = Field(description="expect aifunc import path")
+ is_temp: bool = Field(description="if the modulename is temp module")
+ workspace_dir: str = Field(description="the ghostos dir")
+
+
+class FoundAIFunc(NamedTuple):
+ aifunc: Optional[AIFunc]
+ filename: str
+ modulename: str
+ is_temp: bool
+
+
+def find_aifunc_by_name(filename_or_modulename: str) -> FoundAIFunc:
+ if filename_or_modulename.endswith(".py"):
+ filename = path.abspath(filename_or_modulename)
+ module = create_module("ghostos_app.temp.aifunc", filename)
+ is_temp = True
+ else:
+ module = import_from_path(filename_or_modulename)
+ filename = module.__file__
+ is_temp = False
+
+ aifunc = None
+ for name, value in module.__dict__.conversation_item_states():
+ if name.startswith("_"):
+ continue
+ if not inspect.isclass(value):
+ continue
+ if value.__module__ != module.__name__:
+ continue
+ if issubclass(value, AIFunc):
+ aifunc = value
+ break
+ return FoundAIFunc(aifunc, filename, module.__name__, is_temp)
+
+
+def main():
+ # path
+ workspace_dir = check_ghostos_workspace_exists()
+
+ # argument check
+ args = argv[1:]
+ if len(args) <= 0:
+ raise ValueError("At least one argument (python filename or modulename) is required")
+ filename_or_modulename = args[0]
+ found = find_aifunc_by_name(filename_or_modulename)
+ if found.aifunc is None:
+ raise ValueError(f"No aifunc in module named {filename_or_modulename}")
+
+ run_aifunc_app_arg = RunAIFuncApp(
+ modulename=found.modulename,
+ filename=found.filename,
+ import_path=generate_import_path(found.aifunc),
+ is_temp=found.is_temp,
+ workspace_dir=workspace_dir,
+ )
+ script_path = path.join(path.dirname(cli.__file__), "run_aifunc_app.py")
+ args = [script_path, run_aifunc_app_arg.model_dump_json(), *sys.argv[1:]]
+ main_run(args)
diff --git a/ghostos/scripts/cli/run_console.py b/ghostos/scripts/cli/run_console.py
new file mode 100644
index 00000000..612a91ac
--- /dev/null
+++ b/ghostos/scripts/cli/run_console.py
@@ -0,0 +1,17 @@
+from ghostos.abcd import Ghost
+from ghostos.abcd.utils import get_module_magic_shell_providers
+from ghostos.scripts.cli.utils import (
+ find_ghost_by_file_or_module,
+)
+from ghostos.bootstrap import get_ghostos
+from ghostos.prototypes.console import ConsoleApp
+from ghostos.entity import get_entity
+
+
+def run_console_app(file_or_module: str):
+ ghost_info, module, filename, is_temp = find_ghost_by_file_or_module(file_or_module)
+ providers = get_module_magic_shell_providers(module)
+ ghostos = get_ghostos()
+ ghost = get_entity(ghost_info.ghost, Ghost)
+ app = ConsoleApp(ghostos=ghostos, ghost=ghost, username="", providers=providers)
+ app.run()
diff --git a/ghostos/scripts/cli/run_helloworld.py b/ghostos/scripts/cli/run_helloworld.py
new file mode 100644
index 00000000..0c2f754a
--- /dev/null
+++ b/ghostos/scripts/cli/run_helloworld.py
@@ -0,0 +1,15 @@
+from sys import argv
+from os.path import dirname, join
+
+
+def main():
+ """
+ test start streamlit and pass value
+ :return:
+ """
+ from ghostos.prototypes.streamlitapp import cli
+ from streamlit.web.cli import main_run
+ args = argv[1:]
+ filename = join(dirname(cli.__file__), "helloworld.py")
+ args.insert(0, filename)
+ main_run(args)
diff --git a/ghostos/scripts/cli/run_streamlit_app.py b/ghostos/scripts/cli/run_streamlit_app.py
new file mode 100644
index 00000000..633a8f5d
--- /dev/null
+++ b/ghostos/scripts/cli/run_streamlit_app.py
@@ -0,0 +1,66 @@
+from typing import Optional, List
+from ghostos.scripts.cli.utils import (
+ GhostInfo,
+)
+from streamlit.web.cli import main_run as run_streamlit_web
+from ghostos.prototypes.streamlitapp import cli
+from ghostos.bootstrap import get_bootstrap_config
+from ghostos.entity import EntityMeta
+from pydantic import BaseModel, Field
+from os import path
+
+__all__ = [
+ "run_streamlit_app.py",
+ "start_streamlit_prototype_cli",
+ "RunGhostChatApp",
+ "get_config_flag_options",
+ "start_ghost_app",
+]
+
+
+class RunGhostChatApp(BaseModel):
+ modulename: str = Field(description="expect ghost modulename")
+ filename: str = Field(description="expect ghost filename")
+ is_temp: bool = Field(description="if the modulename is temp module")
+ workspace_dir: str = Field(description="the ghostos dir")
+ ghost_meta: EntityMeta
+ context_meta: Optional[EntityMeta] = Field(default=None)
+
+
+def get_config_flag_options(workspace_dir: str) -> List[str]:
+ from os.path import join
+ from toml import loads
+ filename = join(workspace_dir, ".streamlit/config.toml")
+ with open(filename, "r") as f:
+ data = loads(f.read())
+ flags = []
+ for key in data:
+ attrs = data[key]
+ for attr_name in attrs:
+ value = attrs[attr_name]
+ flags.append(f"--{key}.{attr_name}={value}")
+ return flags
+
+
+def start_ghost_app(ghost_info: GhostInfo, modulename: str, filename: str, is_temp: bool):
+ # path
+ conf = get_bootstrap_config(local=True)
+ workspace_dir = conf.workspace_dir
+ args = RunGhostChatApp(
+ modulename=modulename,
+ filename=filename,
+ is_temp=is_temp,
+ workspace_dir=workspace_dir,
+ ghost_meta=ghost_info.ghost,
+ context_meta=ghost_info.context,
+ )
+ start_streamlit_prototype_cli("run_ghost_chat.py", args.model_dump_json(), conf.workspace_dir)
+
+
+def start_streamlit_prototype_cli(filename: str, cli_args: str, workspace_dir: str):
+ script_path = path.join(path.dirname(cli.__file__), filename)
+ args = [script_path, cli_args]
+
+ flags = get_config_flag_options(workspace_dir)
+ args.extend(flags)
+ run_streamlit_web(args)
diff --git a/ghostos/scripts/cli/utils.py b/ghostos/scripts/cli/utils.py
new file mode 100644
index 00000000..43dc191d
--- /dev/null
+++ b/ghostos/scripts/cli/utils.py
@@ -0,0 +1,161 @@
+from __future__ import annotations
+import sys
+from typing import Tuple, List, NamedTuple, Any, Optional, Dict, Self
+from types import ModuleType
+
+from ghostos.bootstrap import expect_workspace_dir
+from ghostos.contracts.logger import get_console_logger
+from ghostos.helpers import create_module, import_from_path
+from ghostos.abcd import Ghost
+from pydantic import BaseModel, Field
+from ghostos.entity import EntityMeta, to_entity_meta
+from ghostos.ghosts.moss_agent import new_moss_agent
+import inspect
+import json
+
+__all__ = [
+ 'get_ghost_by_cli_argv',
+ 'get_or_create_module_from_name',
+ 'find_ghost_by_file_or_module',
+ 'check_ghostos_workspace_exists',
+ 'parse_args_modulename_or_filename',
+ 'GhostsConf', 'GhostInfo',
+]
+
+
+def get_ghost_by_cli_argv() -> Tuple[GhostInfo, ModuleType, str, bool]:
+ filename_or_modulename, args = parse_args_modulename_or_filename()
+ return find_ghost_by_file_or_module(filename_or_modulename)
+
+
+def find_ghost_by_file_or_module(filename_or_modulename: str) -> Tuple[GhostInfo, ModuleType, str, bool]:
+ found = get_or_create_module_from_name(filename_or_modulename, "ghostos.temp.ghost")
+ # ghost info
+ ghosts_conf = GhostsConf.load_from(filename_or_modulename)
+ ghost_key = GhostsConf.file_ghost_key(filename_or_modulename)
+ if ghost_key in ghosts_conf.ghosts:
+ ghost_info = ghosts_conf.ghosts[ghost_key]
+ return ghost_info, found.module, found.filename, found.is_temp
+
+ if found.value is not None:
+ if not isinstance(found.value, Ghost):
+ raise SystemExit(f"{found.value} is not a Ghost object")
+ ghost = found.value
+ elif "__ghost__" in found.module.__dict__:
+ ghost = found.module.__dict__["__ghost__"]
+ if not isinstance(ghost, Ghost):
+ raise SystemExit(f"{filename_or_modulename} __ghost__ is not a Ghost object")
+ else:
+ ghost = new_moss_agent(found.module.__name__)
+
+ ghost_info = GhostInfo(
+ ghost=to_entity_meta(ghost),
+ )
+ return ghost_info, found.module, found.filename, found.is_temp
+
+
+def check_ghostos_workspace_exists() -> str:
+ logger = get_console_logger()
+ app_dir, ok = expect_workspace_dir()
+ if not ok:
+ logger.error("expect GhostOS workspace `%s` is not found. ", app_dir)
+ logger.debug("run `ghostos init` to create workspace")
+ sys.exit(0)
+ return app_dir
+
+
+def parse_args_modulename_or_filename() -> Tuple[str, List[str]]:
+ from sys import argv
+ # argument check
+ args = argv[1:]
+ if len(args) <= 0:
+ raise ValueError("At least one argument (python filename or modulename) is required")
+ filename_or_modulename = args[0]
+ return filename_or_modulename, args
+
+
+class Found(NamedTuple):
+ value: Optional[Any]
+ module: ModuleType
+ filename: str
+ is_temp: bool
+
+
+ACCEPTED_FILE_EXTENSIONS = (".py", ".py3")
+
+
+def get_or_create_module_from_name(
+ filename_or_modulename: str,
+ temp_modulename: str,
+) -> Found:
+ from os import path, getcwd
+
+ _, extension = path.splitext(filename_or_modulename)
+ if extension in ACCEPTED_FILE_EXTENSIONS:
+ filename = path.abspath(filename_or_modulename)
+ root_dir = getcwd()
+ if filename.startswith(root_dir):
+ relative_path = filename[len(root_dir) + 1:]
+ relative_path_basename, _ = path.splitext(relative_path)
+ temp_modulename = relative_path_basename.replace("/", ".")
+ module = create_module(temp_modulename, filename)
+ is_temp = True
+ value = None
+ else:
+ imported = import_from_path(filename_or_modulename)
+ if isinstance(imported, ModuleType):
+ module = imported
+ value = None
+ else:
+ module = inspect.getmodule(imported)
+ value = imported
+ filename = module.__file__
+ is_temp = False
+ return Found(value, module, filename, is_temp)
+
+
+class GhostInfo(BaseModel):
+ ghost: EntityMeta = Field(description="ghost meta")
+ context: Optional[EntityMeta] = Field(None)
+
+
+class GhostsConf(BaseModel):
+ ghosts: Dict[str, GhostInfo] = Field(
+ default_factory=dict,
+ description="ghost info dict, from filename to ghost info",
+ )
+
+ @classmethod
+ def load(cls, filename: str) -> Self:
+ from os.path import exists
+ if exists(filename):
+ with open(filename, "r") as f:
+ content = f.read()
+ data = json.loads(content)
+ return cls(**data)
+ return cls()
+
+ @classmethod
+ def load_from(cls, filename: str) -> Self:
+ ghosts_filename = cls.ghosts_filename(filename)
+ return cls.load(ghosts_filename)
+
+ @classmethod
+ def file_ghost_key(cls, filename: str) -> str:
+ from os.path import basename, splitext
+ file_basename = basename(filename)
+ key, _ = splitext(file_basename)
+ return key
+
+ @classmethod
+ def ghosts_filename(cls, filename: str) -> str:
+ from os.path import dirname, join
+ dir_ = dirname(filename)
+ ghosts_filename = join(dir_, ".ghosts.yml")
+ return ghosts_filename
+
+ def save(self, filename: str) -> None:
+ ghosts_filename = self.ghosts_filename(filename)
+ content = self.model_dump_json(indent=2)
+ with open(ghosts_filename, "w") as f:
+ f.write(content)
diff --git a/ghostos/scripts/init.py b/ghostos/scripts/copy_workspace.py
similarity index 67%
rename from ghostos/scripts/init.py
rename to ghostos/scripts/copy_workspace.py
index 1d5fe034..ee6c243b 100644
--- a/ghostos/scripts/init.py
+++ b/ghostos/scripts/copy_workspace.py
@@ -4,10 +4,26 @@
import shutil
import sys
-demo_dir = join(dirname(dirname(__file__)), 'demo')
+
+def copy_workspace(workspace_dir: str, target_dir: str):
+ # the codes below are generated by gpt-4o
+ # Copy files from demo_dir to target_dir
+ if not os.path.exists(target_dir):
+ os.makedirs(target_dir)
+ for item in os.listdir(workspace_dir):
+ s = os.path.join(workspace_dir, item)
+ d = os.path.join(target_dir, item)
+ if "__pycache__" in s:
+ continue
+ if os.path.isdir(s):
+ shutil.copytree(s, d, False, None)
+ else:
+ shutil.copy2(s, d)
def main():
+ from ghostos.bootstrap import get_bootstrap_config
+ conf = get_bootstrap_config()
parser = argparse.ArgumentParser(
description="initialize GhostOS skeleton files to target directory",
)
@@ -19,32 +35,9 @@ def main():
)
parsed = parser.parse_args(sys.argv[1:])
target_dir = abspath(parsed.target)
-
- # the codes below are generated by gpt-4o
- # Copy files from demo_dir to target_dir
- if not os.path.exists(target_dir):
- os.makedirs(target_dir)
- for item in os.listdir(demo_dir):
- s = os.path.join(demo_dir, item)
- d = os.path.join(target_dir, item)
- if "__pycache__" in s:
- continue
- if os.path.isdir(s):
- shutil.copytree(s, d, False, None)
- else:
- shutil.copy2(s, d)
+ copy_workspace(conf.workspace_dir, target_dir)
print(f"Copied skeleton files to {target_dir}")
-# if __name__ == '__main__':
-# from ghostos.prototypes.console import new_console_app
-# from ghostos.thoughts import new_file_editor_thought
-#
-# app = new_console_app(demo_dir, 0)
-# app.run_thought(
-# new_file_editor_thought(filepath=__file__),
-# instruction="help me to complete the main func."
-# )
-
if __name__ == '__main__':
main()
diff --git a/ghostos/scripts/demo.py b/ghostos/scripts/demo.py
deleted file mode 100644
index 8d357ecd..00000000
--- a/ghostos/scripts/demo.py
+++ /dev/null
@@ -1,44 +0,0 @@
-import argparse
-import sys
-from ghostos.prototypes.console import demo_console_app
-
-
-def main() -> None:
- parser = argparse.ArgumentParser(
- description="run ghostos demo in console",
- )
- parser.add_argument(
- "--ghost-id", '-g',
- help="ghost_id in demo/configs/ghosts.yml",
- type=str,
- default="baseline",
- )
- parser.add_argument(
- "--debug", "-d",
- help="debug mode",
- action="store_true",
- default=False,
- )
- parser.add_argument(
- "--username", '-u',
- help="username",
- type=str,
- default="BrightRed",
- )
- parser.add_argument(
- "--session-id", '-s',
- help="session id",
- type=str,
- default=None,
- )
- parsed = parser.parse_args(sys.argv[1:])
- demo_console_app.run_console(
- ghost_id=parsed.ghost_id,
- debug=parsed.debug,
- username=parsed.username,
- session_id=parsed.session_id,
- )
-
-
-if __name__ == "__main__":
- main()
diff --git a/ghostos/scripts/logconf.py b/ghostos/scripts/logconf.py
deleted file mode 100644
index 37259041..00000000
--- a/ghostos/scripts/logconf.py
+++ /dev/null
@@ -1,13 +0,0 @@
-import os
-import yaml
-from logging.config import dictConfig
-
-
-def prepare_logger():
- demo_dir = os.path.abspath(os.path.dirname(__file__) + "/../../demo")
- conf_path = os.path.join(demo_dir, "ghostos/configs/logging.yaml")
- conf_path = os.path.abspath(conf_path)
- with open(conf_path) as f:
- content = f.read()
- data = yaml.safe_load(content)
- dictConfig(data)
diff --git a/ghostos/scripts/swe_test.py b/ghostos/scripts/swe_test.py
deleted file mode 100644
index 58865b16..00000000
--- a/ghostos/scripts/swe_test.py
+++ /dev/null
@@ -1,141 +0,0 @@
-import argparse
-import sys
-import os
-import yaml
-from typing import List, Dict
-
-from ghostos.core.session import MsgThread
-from ghostos.scripts.logconf import prepare_logger
-from ghostos.core.llms import Chat
-from ghostos.core.messages import Message
-from ghostos.core.moss import test_container
-from ghostos.core.aifunc import DefaultAIFuncManagerImpl, AIFunc, DefaultAIFuncDriverImpl, AIFuncManager
-from ghostos.framework.logger import NamedLoggerProvider
-from ghostos.framework.storage import FileStorageProvider
-from ghostos.framework.llms import ConfigBasedLLMsProvider
-from ghostos.framework.threads import StorageThreadsProvider
-from ghostos.container import Container
-from ghostos.contracts.modules import Modules
-from ghostos.framework.configs import ConfigsByStorageProvider
-from ghostos.helpers import import_from_path, yaml_pretty_dump
-from rich.console import Console
-from rich.panel import Panel
-from rich.markdown import Markdown
-from rich.prompt import Prompt
-
-
-console = Console()
-
-prepare_logger()
-
-
-def prepare_container(root_dir: str) -> Container:
- container = test_container()
- container.register(FileStorageProvider(root_dir))
- container.register(NamedLoggerProvider(logger_name="debug"))
- container.register(StorageThreadsProvider(threads_dir='runtime/threads'))
- container.register(ConfigsByStorageProvider("ghostos/configs"))
- container.register(ConfigBasedLLMsProvider("llms/llms_conf.yaml"))
- return container
-
-
-def main() -> None:
- parser = argparse.ArgumentParser(
- description="run swe-evaluation aifunc test cases, show results",
- )
- parser.add_argument(
- "--case", '-c',
- help="ghostos aifunc test case name in demo/ghostos/tests/aifunc_tests.yml",
- type=str,
- default="",
- )
- parser.add_argument(
- "--import_path", '-i',
- help="the import path of the AIFunc instance, such as foo.bar:baz",
- type=str,
- default="evaluation.swe_bench_lite.debug_localization:example",
- )
- parser.add_argument(
- "--llm_api", '-l',
- help="the llm api name",
- type=str,
- default="",
- )
- parser.add_argument(
- "--auto", '-a',
- help="auto run the test or stop at each generations",
- action="store_true",
- default=False,
- )
-
- parsed = parser.parse_args(sys.argv[1:])
- llm_api = parsed.llm_api
- demo_dir = os.path.abspath(os.path.dirname(__file__) + "/../../demo")
- container = prepare_container(demo_dir)
-
- class TestDriverImpl(DefaultAIFuncDriverImpl):
- console = console
-
- def on_message(self, message: Message) -> None:
- self.console.print(
- Panel(
- Markdown(message.get_content()),
- title=f"generated message ({self.name()})",
- )
- )
- if not parsed.auto:
- value = Prompt.ask("Continue?", choices=["y", "n"], default="y")
- if value != "y":
- exit(0)
-
- def on_chat(self, chat: Chat) -> None:
- for message in chat.get_messages():
- self.console.print(Panel(
- Markdown(message.get_content()),
- title=f"chat_info ({self.name()})",
- ))
- if not parsed.auto:
- value = Prompt.ask("Continue?", choices=["y", "n"], default="y")
- if value != "y":
- exit(0)
-
- def on_system_messages(self, messages: List[Message]) -> None:
- pass
-
- def on_save(self, manager: AIFuncManager, thread: MsgThread) -> None:
- current = thread.current
- if current:
- for message in current.messages():
- self.console.print(
- Panel(
- Markdown(message.get_content()),
- title="thread new round message",
- )
- )
-
- manager_ = DefaultAIFuncManagerImpl(
- container=container,
- llm_api_name=llm_api,
- default_driver=TestDriverImpl,
- )
- modules = container.force_fetch(Modules)
- aifunc = import_from_path(parsed.import_path, modules.import_module)
- if not isinstance(aifunc, AIFunc):
- raise AttributeError(f'aifunc must be an instance of {AIFunc}, {aifunc} given')
-
- driver = manager_.get_driver(aifunc)
- # print initialized thread.
- thread_ = driver.initialize()
- thread_content = yaml_pretty_dump(thread_.model_dump(exclude_defaults=True))
- console.print(Panel(
- Markdown(f"```markdown\n{thread_content}\n```"),
- title="initialized thread",
- ))
-
- result = manager_.execute(aifunc)
- console.print(result)
- manager_.destroy()
-
-
-if __name__ == "__main__":
- main()
diff --git a/ghostos/streamlit.py b/ghostos/streamlit.py
new file mode 100644
index 00000000..4e519d26
--- /dev/null
+++ b/ghostos/streamlit.py
@@ -0,0 +1,62 @@
+from typing import Protocol, Self, TypeVar, Optional, NamedTuple, Generic
+from abc import ABC, abstractmethod
+
+__all__ = [
+ 'StreamlitObject', 'StreamlitRenderable',
+ 'is_streamlit_renderable', 'render_streamlit_object',
+ 'StreamlitRenderer', 'GroupRenderer',
+ 'Rendered',
+]
+
+T = TypeVar('T')
+
+
+class Rendered(Generic[T], NamedTuple):
+ value: T
+ changed: bool
+
+
+class StreamlitRenderable(Protocol):
+
+ @abstractmethod
+ def __streamlit_render__(self) -> Optional[Rendered[Self]]:
+ pass
+
+
+class StreamlitObject(ABC):
+ @abstractmethod
+ def __streamlit_render__(self) -> Optional[Rendered[Self]]:
+ pass
+
+
+class StreamlitRenderer(ABC):
+
+ @abstractmethod
+ def render(self, value: T) -> Optional[Rendered[T]]:
+ pass
+
+
+class GroupRenderer(StreamlitRenderer):
+ def __init__(self, *renderers: StreamlitRenderer):
+ self.renderers = list(renderers)
+
+ def render(self, value: T) -> Optional[Rendered[T]]:
+ for renderer in self.renderers:
+ result = renderer.render(value)
+ if result is None:
+ continue
+ return result
+ return None
+
+
+def is_streamlit_renderable(obj):
+ return isinstance(obj, StreamlitObject) or hasattr(obj, "__streamlit_render__")
+
+
+def render_streamlit_object(obj: T) -> Optional[Rendered[T]]:
+ if is_streamlit_renderable(obj):
+ fn = getattr(obj, "__streamlit_render__", None)
+ if fn is not None:
+ r = fn()
+ return r
+ return None
diff --git a/ghostos/thoughts/basic.py b/ghostos/thoughts/basic.py
index ff83daa8..002e9027 100644
--- a/ghostos/thoughts/basic.py
+++ b/ghostos/thoughts/basic.py
@@ -1,11 +1,11 @@
from typing import Optional, Generic, Iterable
from abc import ABC, abstractmethod
-from ghostos.core.session import Event, thread_to_chat
+from ghostos.core.runtime import Event, thread_to_prompt
from ghostos.core.ghosts import Ghost, Operator, Action
-from ghostos.core.llms import LLMApi, ChatPreparer, Chat, prepare_chat
+from ghostos.core.llms import LLMApi, PromptPipe, Prompt, run_prompt_pipeline
from ghostos.core.messages import Role
from ghostos.core.ghosts.thoughts import T, BasicThoughtDriver
-from ghostos.framework.chatpreparers import OtherAgentOrTaskPreparer
+from ghostos.framework.chatpreparers import OtherAgentOrTaskPipe
class LLMThoughtDriver(Generic[T], BasicThoughtDriver[T], ABC):
@@ -21,14 +21,14 @@ def get_llmapi(self, g: Ghost) -> LLMApi:
"""
pass
- def chat_preparers(self, g: Ghost, e: Event) -> Iterable[ChatPreparer]:
+ def chat_preparers(self, g: Ghost, e: Event) -> Iterable[PromptPipe]:
"""
return chat preparers that filter chat messages by many rules.
"""
assistant_name = g.identifier().name
- yield OtherAgentOrTaskPreparer(
+ yield OtherAgentOrTaskPipe(
assistant_name=assistant_name,
- task_id=g.session().task().task_id,
+ task_id=g.session().get_task.task_id,
)
@abstractmethod
@@ -45,15 +45,15 @@ def instruction(self, g: Ghost, e: Event) -> str:
"""
pass
- def initialize_chat(self, g: Ghost, e: Event) -> Chat:
+ def initialize_chat(self, g: Ghost, e: Event) -> Prompt:
session = g.session()
- thread = session.thread()
+ thread = session.get_thread()
system_prompt = g.system_prompt()
thought_instruction = self.instruction(g, e)
content = "\n\n".join([system_prompt, thought_instruction])
# system prompt from thought
system_messages = [Role.SYSTEM.new(content=content.strip())]
- chat = thread_to_chat(e.id, system_messages, thread)
+ chat = thread_to_prompt(e.event_id, system_messages, thread)
return chat
def think(self, g: Ghost, e: Event) -> Optional[Operator]:
@@ -66,32 +66,32 @@ def think(self, g: Ghost, e: Event) -> Optional[Operator]:
# prepare chat, filter messages.
preparers = self.chat_preparers(g, e)
- chat = prepare_chat(chat, preparers)
+ chat = run_prompt_pipeline(chat, preparers)
# prepare actions
actions = list(self.actions(g, e))
- action_map = {action.identifier().name: action for action in actions}
+ action_map = {action.__identifier__().name: action for action in actions}
# prepare chat by actions
for action in actions:
- chat = action.prepare_chat(chat)
+ chat = action.update_prompt(chat)
# prepare llm api
llm_api = self.get_llmapi(g)
# run llms
- logger.info("start llm thinking") # todo: logger
+ logger.debug("start llm thinking") # todo: logger
# prepare messenger
messenger = session.messenger(functional_tokens=chat.functional_tokens)
llm_api.deliver_chat_completion(chat, messenger)
messages, callers = messenger.flush()
# set chat system prompt to thread
- session.thread().system_prompt = chat.system_prompt()
+ session.get_thread().system_prompt = chat.system_prompt()
# callback actions
for caller in callers:
if caller.name in action_map:
- logger.info(f"llm response caller `{caller.name}` match action")
+ logger.debug(f"llm response caller `{caller.name}` match action")
action = action_map[caller.name]
op = action.act(container, session, caller)
if op is not None:
diff --git a/ghostos/thoughts/chat.py b/ghostos/thoughts/chat.py
index fae1c13a..8fcba84c 100644
--- a/ghostos/thoughts/chat.py
+++ b/ghostos/thoughts/chat.py
@@ -6,7 +6,7 @@
from pydantic import Field
from ghostos.core.llms import LLMApi
-from ghostos.core.session import Event
+from ghostos.core.runtime import Event
__all__ = ["ChatThought", "ChatThoughtDriver"]
@@ -29,4 +29,4 @@ def actions(self, g: Ghost, e: Event) -> Iterable[Action]:
return []
def instruction(self, g: Ghost, e: Event) -> str:
- return self.thought.instruction
+ return self.thought.show_instruction
diff --git a/ghostos/thoughts/directory_editor_thought.py b/ghostos/thoughts/directory_editor_thought.py
index e925186d..ab83bb6c 100644
--- a/ghostos/thoughts/directory_editor_thought.py
+++ b/ghostos/thoughts/directory_editor_thought.py
@@ -35,7 +35,7 @@ class Moss(Parent):
"""
-#
+#
# the codes between the moss xml marks are not visible to LLM
from ghostos.libraries.file_editor import DirectoryEditorImpl
@@ -43,7 +43,7 @@ class Moss(Parent):
# using TYPE_CHECKING to avoid reflect invalid importing to prompt.
if TYPE_CHECKING:
from ghostos.core.ghosts import Ghost
- from ghostos.core.session import Event, Session, MsgThread
+ from ghostos.core.runtime import Event, Session, GoThreadInfo
from ghostos.core.llms import LLMApi
from ghostos.core.moss import MossCompiler
@@ -128,7 +128,7 @@ def __magic_moss_thought_instruction__(thought: DirectoryEditorThought, g: "Ghos
return instruction
-def __magic_moss_thought_thread__(thought: DirectoryEditorThought, session: "Session", thread: "MsgThread") -> "MsgThread":
+def __magic_moss_thought_thread__(thought: DirectoryEditorThought, session: "Session", thread: "GoThreadInfo") -> "GoThreadInfo":
"""
optional magic function that prepare the thread info, such as modify thread.save_file
"""
@@ -137,4 +137,4 @@ def __magic_moss_thought_thread__(thought: DirectoryEditorThought, session: "Ses
thread.save_file = join(thought.directory, ".directory_editor_thought.thread.yml")
return thread
-#
+#
diff --git a/ghostos/thoughts/file_editor_thought.py b/ghostos/thoughts/file_editor_thought.py
index ab365c6b..6caaebec 100644
--- a/ghostos/thoughts/file_editor_thought.py
+++ b/ghostos/thoughts/file_editor_thought.py
@@ -1,7 +1,7 @@
from ghostos.core.ghosts import ModelThought, Ghost
from ghostos.core.llms import LLMApi
from ghostos.core.moss import PyContext, MossCompiler
-from ghostos.core.session import Event, Session, MsgThread
+from ghostos.core.runtime import Event, Session, GoThreadInfo
from ghostos.thoughts.moss_thought import BasicMossThoughtDriver, LLMThoughtDriver
from ghostos.thoughts import file_editor_moss
from ghostos.libraries.file_editor import FileEditorImpl, FileEditor
@@ -68,7 +68,7 @@ def is_moss_code_delivery(self) -> bool:
return self.thought.debug
def new_task_id(self, g: Ghost) -> str:
- process_id = g.session().process().process_id
+ process_id = g.session().update_prompt().process_id
task_id = f"process_{process_id}_task_{self.thought.filepath}"
# task_id in a same process will always be the same
return md5(task_id)
@@ -81,7 +81,7 @@ def prepare_moss_compiler(self, g: Ghost, compiler: MossCompiler) -> MossCompile
compiler.register(provide(FileEditor)(lambda c: self.file_editor()))
return compiler
- def prepare_thread(self, session: Session, thread: MsgThread) -> MsgThread:
+ def prepare_thread(self, session: Session, thread: GoThreadInfo) -> GoThreadInfo:
if self.thought.debug:
filepath = self.thought.filepath
saving_path = filepath + ".thread.yml"
diff --git a/ghostos/thoughts/magic_moss_thought.py b/ghostos/thoughts/magic_moss_thought.py
index 9cb1ff66..32b0b129 100644
--- a/ghostos/thoughts/magic_moss_thought.py
+++ b/ghostos/thoughts/magic_moss_thought.py
@@ -7,7 +7,7 @@
from ghostos.core.moss import PyContext, MossCompiler
from ghostos.core.ghosts import Ghost, Action
from ghostos.core.llms import LLMApi
-from ghostos.core.session import Event, Session, MsgThread
+from ghostos.core.runtime import Event, Session, GoThreadInfo
from ghostos.container import Provider
import inspect
from pydantic import Field
@@ -27,7 +27,7 @@ class MagicMossThought(ModelThought, ABC):
debug: bool = Field(default=False, description="if the debug mode is on")
-#
+#
def __magic_moss_thought_instruction__(thought: MagicMossThought, g: "Ghost", e: "Event") -> str:
"""
@@ -79,7 +79,7 @@ def __magic_moss_thought_compiling__(thought: MagicMossThought, g: "Ghost", comp
return compiler
-def __magic_moss_thought_thread__(thought: MagicMossThought, session: Session, thread: MsgThread) -> MsgThread:
+def __magic_moss_thought_thread__(thought: MagicMossThought, session: Session, thread: GoThreadInfo) -> GoThreadInfo:
"""
optional magic function that prepare the thread info, such as modify thread.save_file
:param thought:
@@ -100,7 +100,7 @@ def __on_inputs__(driver: MagicMossThoughtDriver, g: Ghost, e: Event) -> Optiona
pass
-#
+#
class MagicMossThoughtDriver(LLMThoughtDriver[MagicMossThought], BasicMossThoughtDriver):
@@ -135,7 +135,7 @@ def prepare_moss_compiler(self, g: Ghost, compiler: MossCompiler) -> MossCompile
fn = __magic_moss_thought_compiling__
return fn(self.thought, g, compiler)
- def prepare_thread(self, session: Session, thread: MsgThread) -> MsgThread:
+ def prepare_thread(self, session: Session, thread: GoThreadInfo) -> GoThreadInfo:
thread = super().prepare_thread(session, thread)
fn = self.get_magic_func_of_the_module(__magic_moss_thought_thread__.__name__)
if fn is None:
diff --git a/ghostos/thoughts/moss_thought.py b/ghostos/thoughts/moss_thought.py
index 04bdac93..27d68f8f 100644
--- a/ghostos/thoughts/moss_thought.py
+++ b/ghostos/thoughts/moss_thought.py
@@ -3,7 +3,7 @@
from ghostos.core.ghosts import Ghost, Action, ModelThought, Operator
from ghostos.core.llms import LLMApi
-from ghostos.core.session import Event, MsgThread
+from ghostos.core.runtime import Event, GoThreadInfo
from ghostos.core.moss import MossCompiler, MossRuntime, PyContext
from ghostos.thoughts.basic import LLMThoughtDriver
from ghostos.framework.actions import MossAction
@@ -47,13 +47,13 @@ def get_moss_runtime(self, g: Ghost) -> MossRuntime:
if self.moss_runtime is not None:
return self.moss_runtime
- thread = g.session().thread()
+ thread = g.session().get_thread()
compiler = g.moss()
# init default pycontext
default_pycontext = self.init_pycontext()
compiler = compiler.join_context(default_pycontext)
# bind msg thread
- compiler.bind(MsgThread, thread)
+ compiler.bind(GoThreadInfo, thread)
# join thread
pycontext = thread.get_pycontext()
compiler = compiler.join_context(pycontext)
@@ -89,11 +89,11 @@ def actions(self, g: Ghost, e: Event) -> Iterable[Action]:
class MossThoughtDriver(BasicMossThoughtDriver, LLMThoughtDriver[MossThought]):
def instruction(self, g: Ghost, e: Event) -> str:
- return self.thought.instruction
+ return self.thought.show_instruction
def on_created(self, g: Ghost, e: Event) -> Optional[Operator]:
session = g.session()
- thread = session.thread()
+ thread = session.get_thread()
pycontext = self.init_pycontext()
thread.update_pycontext(pycontext)
return super().on_created(g, e)
diff --git a/ghostos/thoughts/pymodule_editor.py b/ghostos/thoughts/pymodule_editor.py
index 12792a50..7dbfc5a1 100644
--- a/ghostos/thoughts/pymodule_editor.py
+++ b/ghostos/thoughts/pymodule_editor.py
@@ -3,7 +3,7 @@
from ghostos.core.ghosts import ModelThought, Ghost
from ghostos.core.llms import LLMApi
from ghostos.core.moss import PyContext, MossCompiler
-from ghostos.core.session import Event, Session, MsgThread
+from ghostos.core.runtime import Event, Session, GoThreadInfo
from ghostos.thoughts.basic import LLMThoughtDriver
from ghostos.thoughts.moss_thought import BasicMossThoughtDriver
from ghostos.thoughts import pymodule_editor_moss
@@ -71,12 +71,12 @@ def get_llmapi(self, g: Ghost) -> LLMApi:
return g.llms().get_api(self.thought.llm_api_name)
def new_task_id(self, g: Ghost) -> str:
- process_id = g.session().process().process_id
+ process_id = g.session().update_prompt().process_id
task_id = f"process_{process_id}_task_{self.thought.target_module}"
# task_id in a same process will always be the same
return md5(task_id)
- def prepare_thread(self, session: Session, thread: MsgThread) -> MsgThread:
+ def prepare_thread(self, session: Session, thread: GoThreadInfo) -> GoThreadInfo:
"""
save the thread where I'm convenient to see it
:param session:
@@ -103,7 +103,7 @@ def instruction(self, g: Ghost, e: Event) -> str:
)
if self.thought.referencing:
referencing = "\n\n# referencing\n\nThere are some references for you:"
- for import_path, prompt in self.thought.referencing.items():
+ for import_path, prompt in self.thought.referencing.conversation_item_states():
target = import_from_path(import_path)
source = inspect.getsource(target)
referencing += f"""
diff --git a/poetry.lock b/poetry.lock
deleted file mode 100644
index b377f9d5..00000000
--- a/poetry.lock
+++ /dev/null
@@ -1,3653 +0,0 @@
-# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand.
-
-[[package]]
-name = "aiohappyeyeballs"
-version = "2.4.0"
-description = "Happy Eyeballs for asyncio"
-optional = false
-python-versions = ">=3.8"
-files = [
- {file = "aiohappyeyeballs-2.4.0-py3-none-any.whl", hash = "sha256:7ce92076e249169a13c2f49320d1967425eaf1f407522d707d59cac7628d62bd"},
- {file = "aiohappyeyeballs-2.4.0.tar.gz", hash = "sha256:55a1714f084e63d49639800f95716da97a1f173d46a16dfcfda0016abb93b6b2"},
-]
-
-[[package]]
-name = "aiohttp"
-version = "3.10.5"
-description = "Async http client/server framework (asyncio)"
-optional = false
-python-versions = ">=3.8"
-files = [
- {file = "aiohttp-3.10.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:18a01eba2574fb9edd5f6e5fb25f66e6ce061da5dab5db75e13fe1558142e0a3"},
- {file = "aiohttp-3.10.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:94fac7c6e77ccb1ca91e9eb4cb0ac0270b9fb9b289738654120ba8cebb1189c6"},
- {file = "aiohttp-3.10.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2f1f1c75c395991ce9c94d3e4aa96e5c59c8356a15b1c9231e783865e2772699"},
- {file = "aiohttp-3.10.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4f7acae3cf1a2a2361ec4c8e787eaaa86a94171d2417aae53c0cca6ca3118ff6"},
- {file = "aiohttp-3.10.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:94c4381ffba9cc508b37d2e536b418d5ea9cfdc2848b9a7fea6aebad4ec6aac1"},
- {file = "aiohttp-3.10.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c31ad0c0c507894e3eaa843415841995bf8de4d6b2d24c6e33099f4bc9fc0d4f"},
- {file = "aiohttp-3.10.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0912b8a8fadeb32ff67a3ed44249448c20148397c1ed905d5dac185b4ca547bb"},
- {file = "aiohttp-3.10.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0d93400c18596b7dc4794d48a63fb361b01a0d8eb39f28800dc900c8fbdaca91"},
- {file = "aiohttp-3.10.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d00f3c5e0d764a5c9aa5a62d99728c56d455310bcc288a79cab10157b3af426f"},
- {file = "aiohttp-3.10.5-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:d742c36ed44f2798c8d3f4bc511f479b9ceef2b93f348671184139e7d708042c"},
- {file = "aiohttp-3.10.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:814375093edae5f1cb31e3407997cf3eacefb9010f96df10d64829362ae2df69"},
- {file = "aiohttp-3.10.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:8224f98be68a84b19f48e0bdc14224b5a71339aff3a27df69989fa47d01296f3"},
- {file = "aiohttp-3.10.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d9a487ef090aea982d748b1b0d74fe7c3950b109df967630a20584f9a99c0683"},
- {file = "aiohttp-3.10.5-cp310-cp310-win32.whl", hash = "sha256:d9ef084e3dc690ad50137cc05831c52b6ca428096e6deb3c43e95827f531d5ef"},
- {file = "aiohttp-3.10.5-cp310-cp310-win_amd64.whl", hash = "sha256:66bf9234e08fe561dccd62083bf67400bdbf1c67ba9efdc3dac03650e97c6088"},
- {file = "aiohttp-3.10.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:8c6a4e5e40156d72a40241a25cc226051c0a8d816610097a8e8f517aeacd59a2"},
- {file = "aiohttp-3.10.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2c634a3207a5445be65536d38c13791904fda0748b9eabf908d3fe86a52941cf"},
- {file = "aiohttp-3.10.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4aff049b5e629ef9b3e9e617fa6e2dfeda1bf87e01bcfecaf3949af9e210105e"},
- {file = "aiohttp-3.10.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1942244f00baaacaa8155eca94dbd9e8cc7017deb69b75ef67c78e89fdad3c77"},
- {file = "aiohttp-3.10.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e04a1f2a65ad2f93aa20f9ff9f1b672bf912413e5547f60749fa2ef8a644e061"},
- {file = "aiohttp-3.10.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7f2bfc0032a00405d4af2ba27f3c429e851d04fad1e5ceee4080a1c570476697"},
- {file = "aiohttp-3.10.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:424ae21498790e12eb759040bbb504e5e280cab64693d14775c54269fd1d2bb7"},
- {file = "aiohttp-3.10.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:975218eee0e6d24eb336d0328c768ebc5d617609affaca5dbbd6dd1984f16ed0"},
- {file = "aiohttp-3.10.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4120d7fefa1e2d8fb6f650b11489710091788de554e2b6f8347c7a20ceb003f5"},
- {file = "aiohttp-3.10.5-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:b90078989ef3fc45cf9221d3859acd1108af7560c52397ff4ace8ad7052a132e"},
- {file = "aiohttp-3.10.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:ba5a8b74c2a8af7d862399cdedce1533642fa727def0b8c3e3e02fcb52dca1b1"},
- {file = "aiohttp-3.10.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:02594361128f780eecc2a29939d9dfc870e17b45178a867bf61a11b2a4367277"},
- {file = "aiohttp-3.10.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:8fb4fc029e135859f533025bc82047334e24b0d489e75513144f25408ecaf058"},
- {file = "aiohttp-3.10.5-cp311-cp311-win32.whl", hash = "sha256:e1ca1ef5ba129718a8fc827b0867f6aa4e893c56eb00003b7367f8a733a9b072"},
- {file = "aiohttp-3.10.5-cp311-cp311-win_amd64.whl", hash = "sha256:349ef8a73a7c5665cca65c88ab24abe75447e28aa3bc4c93ea5093474dfdf0ff"},
- {file = "aiohttp-3.10.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:305be5ff2081fa1d283a76113b8df7a14c10d75602a38d9f012935df20731487"},
- {file = "aiohttp-3.10.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3a1c32a19ee6bbde02f1cb189e13a71b321256cc1d431196a9f824050b160d5a"},
- {file = "aiohttp-3.10.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:61645818edd40cc6f455b851277a21bf420ce347baa0b86eaa41d51ef58ba23d"},
- {file = "aiohttp-3.10.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c225286f2b13bab5987425558baa5cbdb2bc925b2998038fa028245ef421e75"},
- {file = "aiohttp-3.10.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8ba01ebc6175e1e6b7275c907a3a36be48a2d487549b656aa90c8a910d9f3178"},
- {file = "aiohttp-3.10.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8eaf44ccbc4e35762683078b72bf293f476561d8b68ec8a64f98cf32811c323e"},
- {file = "aiohttp-3.10.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1c43eb1ab7cbf411b8e387dc169acb31f0ca0d8c09ba63f9eac67829585b44f"},
- {file = "aiohttp-3.10.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:de7a5299827253023c55ea549444e058c0eb496931fa05d693b95140a947cb73"},
- {file = "aiohttp-3.10.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4790f0e15f00058f7599dab2b206d3049d7ac464dc2e5eae0e93fa18aee9e7bf"},
- {file = "aiohttp-3.10.5-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:44b324a6b8376a23e6ba25d368726ee3bc281e6ab306db80b5819999c737d820"},
- {file = "aiohttp-3.10.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:0d277cfb304118079e7044aad0b76685d30ecb86f83a0711fc5fb257ffe832ca"},
- {file = "aiohttp-3.10.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:54d9ddea424cd19d3ff6128601a4a4d23d54a421f9b4c0fff740505813739a91"},
- {file = "aiohttp-3.10.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:4f1c9866ccf48a6df2b06823e6ae80573529f2af3a0992ec4fe75b1a510df8a6"},
- {file = "aiohttp-3.10.5-cp312-cp312-win32.whl", hash = "sha256:dc4826823121783dccc0871e3f405417ac116055bf184ac04c36f98b75aacd12"},
- {file = "aiohttp-3.10.5-cp312-cp312-win_amd64.whl", hash = "sha256:22c0a23a3b3138a6bf76fc553789cb1a703836da86b0f306b6f0dc1617398abc"},
- {file = "aiohttp-3.10.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:7f6b639c36734eaa80a6c152a238242bedcee9b953f23bb887e9102976343092"},
- {file = "aiohttp-3.10.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f29930bc2921cef955ba39a3ff87d2c4398a0394ae217f41cb02d5c26c8b1b77"},
- {file = "aiohttp-3.10.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f489a2c9e6455d87eabf907ac0b7d230a9786be43fbe884ad184ddf9e9c1e385"},
- {file = "aiohttp-3.10.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:123dd5b16b75b2962d0fff566effb7a065e33cd4538c1692fb31c3bda2bfb972"},
- {file = "aiohttp-3.10.5-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b98e698dc34966e5976e10bbca6d26d6724e6bdea853c7c10162a3235aba6e16"},
- {file = "aiohttp-3.10.5-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c3b9162bab7e42f21243effc822652dc5bb5e8ff42a4eb62fe7782bcbcdfacf6"},
- {file = "aiohttp-3.10.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1923a5c44061bffd5eebeef58cecf68096e35003907d8201a4d0d6f6e387ccaa"},
- {file = "aiohttp-3.10.5-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d55f011da0a843c3d3df2c2cf4e537b8070a419f891c930245f05d329c4b0689"},
- {file = "aiohttp-3.10.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:afe16a84498441d05e9189a15900640a2d2b5e76cf4efe8cbb088ab4f112ee57"},
- {file = "aiohttp-3.10.5-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:f8112fb501b1e0567a1251a2fd0747baae60a4ab325a871e975b7bb67e59221f"},
- {file = "aiohttp-3.10.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:1e72589da4c90337837fdfe2026ae1952c0f4a6e793adbbfbdd40efed7c63599"},
- {file = "aiohttp-3.10.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:4d46c7b4173415d8e583045fbc4daa48b40e31b19ce595b8d92cf639396c15d5"},
- {file = "aiohttp-3.10.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:33e6bc4bab477c772a541f76cd91e11ccb6d2efa2b8d7d7883591dfb523e5987"},
- {file = "aiohttp-3.10.5-cp313-cp313-win32.whl", hash = "sha256:c58c6837a2c2a7cf3133983e64173aec11f9c2cd8e87ec2fdc16ce727bcf1a04"},
- {file = "aiohttp-3.10.5-cp313-cp313-win_amd64.whl", hash = "sha256:38172a70005252b6893088c0f5e8a47d173df7cc2b2bd88650957eb84fcf5022"},
- {file = "aiohttp-3.10.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:f6f18898ace4bcd2d41a122916475344a87f1dfdec626ecde9ee802a711bc569"},
- {file = "aiohttp-3.10.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:5ede29d91a40ba22ac1b922ef510aab871652f6c88ef60b9dcdf773c6d32ad7a"},
- {file = "aiohttp-3.10.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:673f988370f5954df96cc31fd99c7312a3af0a97f09e407399f61583f30da9bc"},
- {file = "aiohttp-3.10.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:58718e181c56a3c02d25b09d4115eb02aafe1a732ce5714ab70326d9776457c3"},
- {file = "aiohttp-3.10.5-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4b38b1570242fbab8d86a84128fb5b5234a2f70c2e32f3070143a6d94bc854cf"},
- {file = "aiohttp-3.10.5-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:074d1bff0163e107e97bd48cad9f928fa5a3eb4b9d33366137ffce08a63e37fe"},
- {file = "aiohttp-3.10.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd31f176429cecbc1ba499d4aba31aaccfea488f418d60376b911269d3b883c5"},
- {file = "aiohttp-3.10.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7384d0b87d4635ec38db9263e6a3f1eb609e2e06087f0aa7f63b76833737b471"},
- {file = "aiohttp-3.10.5-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:8989f46f3d7ef79585e98fa991e6ded55d2f48ae56d2c9fa5e491a6e4effb589"},
- {file = "aiohttp-3.10.5-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:c83f7a107abb89a227d6c454c613e7606c12a42b9a4ca9c5d7dad25d47c776ae"},
- {file = "aiohttp-3.10.5-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:cde98f323d6bf161041e7627a5fd763f9fd829bcfcd089804a5fdce7bb6e1b7d"},
- {file = "aiohttp-3.10.5-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:676f94c5480d8eefd97c0c7e3953315e4d8c2b71f3b49539beb2aa676c58272f"},
- {file = "aiohttp-3.10.5-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:2d21ac12dc943c68135ff858c3a989f2194a709e6e10b4c8977d7fcd67dfd511"},
- {file = "aiohttp-3.10.5-cp38-cp38-win32.whl", hash = "sha256:17e997105bd1a260850272bfb50e2a328e029c941c2708170d9d978d5a30ad9a"},
- {file = "aiohttp-3.10.5-cp38-cp38-win_amd64.whl", hash = "sha256:1c19de68896747a2aa6257ae4cf6ef59d73917a36a35ee9d0a6f48cff0f94db8"},
- {file = "aiohttp-3.10.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7e2fe37ac654032db1f3499fe56e77190282534810e2a8e833141a021faaab0e"},
- {file = "aiohttp-3.10.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f5bf3ead3cb66ab990ee2561373b009db5bc0e857549b6c9ba84b20bc462e172"},
- {file = "aiohttp-3.10.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1b2c16a919d936ca87a3c5f0e43af12a89a3ce7ccbce59a2d6784caba945b68b"},
- {file = "aiohttp-3.10.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad146dae5977c4dd435eb31373b3fe9b0b1bf26858c6fc452bf6af394067e10b"},
- {file = "aiohttp-3.10.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8c5c6fa16412b35999320f5c9690c0f554392dc222c04e559217e0f9ae244b92"},
- {file = "aiohttp-3.10.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:95c4dc6f61d610bc0ee1edc6f29d993f10febfe5b76bb470b486d90bbece6b22"},
- {file = "aiohttp-3.10.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:da452c2c322e9ce0cfef392e469a26d63d42860f829026a63374fde6b5c5876f"},
- {file = "aiohttp-3.10.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:898715cf566ec2869d5cb4d5fb4be408964704c46c96b4be267442d265390f32"},
- {file = "aiohttp-3.10.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:391cc3a9c1527e424c6865e087897e766a917f15dddb360174a70467572ac6ce"},
- {file = "aiohttp-3.10.5-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:380f926b51b92d02a34119d072f178d80bbda334d1a7e10fa22d467a66e494db"},
- {file = "aiohttp-3.10.5-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:ce91db90dbf37bb6fa0997f26574107e1b9d5ff939315247b7e615baa8ec313b"},
- {file = "aiohttp-3.10.5-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:9093a81e18c45227eebe4c16124ebf3e0d893830c6aca7cc310bfca8fe59d857"},
- {file = "aiohttp-3.10.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:ee40b40aa753d844162dcc80d0fe256b87cba48ca0054f64e68000453caead11"},
- {file = "aiohttp-3.10.5-cp39-cp39-win32.whl", hash = "sha256:03f2645adbe17f274444953bdea69f8327e9d278d961d85657cb0d06864814c1"},
- {file = "aiohttp-3.10.5-cp39-cp39-win_amd64.whl", hash = "sha256:d17920f18e6ee090bdd3d0bfffd769d9f2cb4c8ffde3eb203777a3895c128862"},
- {file = "aiohttp-3.10.5.tar.gz", hash = "sha256:f071854b47d39591ce9a17981c46790acb30518e2f83dfca8db2dfa091178691"},
-]
-
-[package.dependencies]
-aiohappyeyeballs = ">=2.3.0"
-aiosignal = ">=1.1.2"
-async-timeout = {version = ">=4.0,<5.0", markers = "python_version < \"3.11\""}
-attrs = ">=17.3.0"
-frozenlist = ">=1.1.1"
-multidict = ">=4.5,<7.0"
-yarl = ">=1.0,<2.0"
-
-[package.extras]
-speedups = ["Brotli", "aiodns (>=3.2.0)", "brotlicffi"]
-
-[[package]]
-name = "aiosignal"
-version = "1.3.1"
-description = "aiosignal: a list of registered asynchronous callbacks"
-optional = false
-python-versions = ">=3.7"
-files = [
- {file = "aiosignal-1.3.1-py3-none-any.whl", hash = "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17"},
- {file = "aiosignal-1.3.1.tar.gz", hash = "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc"},
-]
-
-[package.dependencies]
-frozenlist = ">=1.1.0"
-
-[[package]]
-name = "annotated-types"
-version = "0.7.0"
-description = "Reusable constraint types to use with typing.Annotated"
-optional = false
-python-versions = ">=3.8"
-files = [
- {file = "annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53"},
- {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"},
-]
-
-[[package]]
-name = "anthropic"
-version = "0.31.2"
-description = "The official Python library for the anthropic API"
-optional = false
-python-versions = ">=3.7"
-files = [
- {file = "anthropic-0.31.2-py3-none-any.whl", hash = "sha256:28d176b98c72615bfae30f0a9eee6297cc33bf52535d38156fc2805556e2f09b"},
- {file = "anthropic-0.31.2.tar.gz", hash = "sha256:0134b73df8d1f142fc68675fbadb75e920054e9e3437b99df63f10f0fc6ac26f"},
-]
-
-[package.dependencies]
-anyio = ">=3.5.0,<5"
-distro = ">=1.7.0,<2"
-httpx = ">=0.23.0,<1"
-jiter = ">=0.4.0,<1"
-pydantic = ">=1.9.0,<3"
-sniffio = "*"
-tokenizers = ">=0.13.0"
-typing-extensions = ">=4.7,<5"
-
-[package.extras]
-bedrock = ["boto3 (>=1.28.57)", "botocore (>=1.31.57)"]
-vertex = ["google-auth (>=2,<3)"]
-
-[[package]]
-name = "anyio"
-version = "4.5.0"
-description = "High level compatibility layer for multiple asynchronous event loop implementations"
-optional = false
-python-versions = ">=3.8"
-files = [
- {file = "anyio-4.5.0-py3-none-any.whl", hash = "sha256:fdeb095b7cc5a5563175eedd926ec4ae55413bb4be5770c424af0ba46ccb4a78"},
- {file = "anyio-4.5.0.tar.gz", hash = "sha256:c5a275fe5ca0afd788001f58fca1e69e29ce706d746e317d660e21f70c530ef9"},
-]
-
-[package.dependencies]
-exceptiongroup = {version = ">=1.0.2", markers = "python_version < \"3.11\""}
-idna = ">=2.8"
-sniffio = ">=1.1"
-typing-extensions = {version = ">=4.1", markers = "python_version < \"3.11\""}
-
-[package.extras]
-doc = ["Sphinx (>=7.4,<8.0)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"]
-test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (>=0.21.0b1)"]
-trio = ["trio (>=0.26.1)"]
-
-[[package]]
-name = "arxiv"
-version = "2.1.3"
-description = "Python wrapper for the arXiv API: https://arxiv.org/help/api/"
-optional = false
-python-versions = ">=3.7"
-files = [
- {file = "arxiv-2.1.3-py3-none-any.whl", hash = "sha256:6f43673ab770a9e848d7d4fc1894824df55edeac3c3572ea280c9ba2e3c0f39f"},
- {file = "arxiv-2.1.3.tar.gz", hash = "sha256:32365221994d2cf05657c1fadf63a26efc8ccdec18590281ee03515bfef8bc4e"},
-]
-
-[package.dependencies]
-feedparser = ">=6.0.10,<6.1.0"
-requests = ">=2.32.0,<2.33.0"
-
-[[package]]
-name = "async-timeout"
-version = "4.0.3"
-description = "Timeout context manager for asyncio programs"
-optional = false
-python-versions = ">=3.7"
-files = [
- {file = "async-timeout-4.0.3.tar.gz", hash = "sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f"},
- {file = "async_timeout-4.0.3-py3-none-any.whl", hash = "sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028"},
-]
-
-[[package]]
-name = "attrs"
-version = "24.2.0"
-description = "Classes Without Boilerplate"
-optional = false
-python-versions = ">=3.7"
-files = [
- {file = "attrs-24.2.0-py3-none-any.whl", hash = "sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2"},
- {file = "attrs-24.2.0.tar.gz", hash = "sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346"},
-]
-
-[package.extras]
-benchmark = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-codspeed", "pytest-mypy-plugins", "pytest-xdist[psutil]"]
-cov = ["cloudpickle", "coverage[toml] (>=5.3)", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"]
-dev = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"]
-docs = ["cogapp", "furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier (<24.7)"]
-tests = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"]
-tests-mypy = ["mypy (>=1.11.1)", "pytest-mypy-plugins"]
-
-[[package]]
-name = "certifi"
-version = "2024.8.30"
-description = "Python package for providing Mozilla's CA Bundle."
-optional = false
-python-versions = ">=3.6"
-files = [
- {file = "certifi-2024.8.30-py3-none-any.whl", hash = "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8"},
- {file = "certifi-2024.8.30.tar.gz", hash = "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9"},
-]
-
-[[package]]
-name = "charset-normalizer"
-version = "3.3.2"
-description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet."
-optional = false
-python-versions = ">=3.7.0"
-files = [
- {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"},
- {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"},
- {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"},
- {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"},
- {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"},
- {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"},
- {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"},
- {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"},
- {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"},
- {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"},
- {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"},
- {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"},
- {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"},
- {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"},
- {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"},
- {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"},
- {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"},
- {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"},
- {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"},
- {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"},
- {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"},
- {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"},
- {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"},
- {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"},
- {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"},
- {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"},
- {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"},
- {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"},
- {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"},
- {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"},
- {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"},
- {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"},
- {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"},
- {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"},
- {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"},
- {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"},
- {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"},
- {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"},
- {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"},
- {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"},
- {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"},
- {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"},
- {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"},
- {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"},
- {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"},
- {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"},
- {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"},
- {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"},
- {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"},
- {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"},
- {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"},
- {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"},
- {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"},
- {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"},
- {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"},
- {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"},
- {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"},
- {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"},
- {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"},
- {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"},
- {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"},
- {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"},
- {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"},
- {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"},
- {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"},
- {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"},
- {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"},
- {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"},
- {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"},
- {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"},
- {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"},
- {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"},
- {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"},
- {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"},
- {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"},
- {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"},
- {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"},
- {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"},
- {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"},
- {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"},
- {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"},
- {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"},
- {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"},
- {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"},
- {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"},
- {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"},
- {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"},
- {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"},
- {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"},
- {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"},
-]
-
-[[package]]
-name = "click"
-version = "8.1.7"
-description = "Composable command line interface toolkit"
-optional = false
-python-versions = ">=3.7"
-files = [
- {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"},
- {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"},
-]
-
-[package.dependencies]
-colorama = {version = "*", markers = "platform_system == \"Windows\""}
-
-[[package]]
-name = "colorama"
-version = "0.4.6"
-description = "Cross-platform colored terminal text."
-optional = false
-python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7"
-files = [
- {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"},
- {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"},
-]
-
-[[package]]
-name = "dataclasses-json"
-version = "0.6.7"
-description = "Easily serialize dataclasses to and from JSON."
-optional = false
-python-versions = "<4.0,>=3.7"
-files = [
- {file = "dataclasses_json-0.6.7-py3-none-any.whl", hash = "sha256:0dbf33f26c8d5305befd61b39d2b3414e8a407bedc2834dea9b8d642666fb40a"},
- {file = "dataclasses_json-0.6.7.tar.gz", hash = "sha256:b6b3e528266ea45b9535223bc53ca645f5208833c29229e847b3f26a1cc55fc0"},
-]
-
-[package.dependencies]
-marshmallow = ">=3.18.0,<4.0.0"
-typing-inspect = ">=0.4.0,<1"
-
-[[package]]
-name = "datasets"
-version = "2.21.0"
-description = "HuggingFace community-driven open-source library of datasets"
-optional = false
-python-versions = ">=3.8.0"
-files = [
- {file = "datasets-2.21.0-py3-none-any.whl", hash = "sha256:25e4e097110ce28824b746a107727ada94024cba11db8bc588d468414692b65a"},
- {file = "datasets-2.21.0.tar.gz", hash = "sha256:998f85a8460f1bd982e5bd058f8a0808eef424249e3df1e8cdd594ccd0dc8ba2"},
-]
-
-[package.dependencies]
-aiohttp = "*"
-dill = ">=0.3.0,<0.3.9"
-filelock = "*"
-fsspec = {version = ">=2023.1.0,<=2024.6.1", extras = ["http"]}
-huggingface-hub = ">=0.21.2"
-multiprocess = "*"
-numpy = ">=1.17"
-packaging = "*"
-pandas = "*"
-pyarrow = ">=15.0.0"
-pyyaml = ">=5.1"
-requests = ">=2.32.2"
-tqdm = ">=4.66.3"
-xxhash = "*"
-
-[package.extras]
-apache-beam = ["apache-beam (>=2.26.0)"]
-audio = ["librosa", "soundfile (>=0.12.1)", "soxr (>=0.4.0)"]
-benchmarks = ["tensorflow (==2.12.0)", "torch (==2.0.1)", "transformers (==4.30.1)"]
-dev = ["Pillow (>=9.4.0)", "absl-py", "decorator", "elasticsearch (<8.0.0)", "faiss-cpu (>=1.8.0.post1)", "jax (>=0.3.14)", "jaxlib (>=0.3.14)", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "moto[server]", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "ruff (>=0.3.0)", "s3fs", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "soxr (>=0.4.0)", "sqlalchemy", "tensorflow (>=2.16.0)", "tensorflow (>=2.6.0)", "tensorflow (>=2.6.0)", "tiktoken", "torch", "torch (>=2.0.0)", "transformers", "transformers (>=4.42.0)", "typing-extensions (>=4.6.1)", "zstandard"]
-docs = ["s3fs", "tensorflow (>=2.6.0)", "torch", "transformers"]
-jax = ["jax (>=0.3.14)", "jaxlib (>=0.3.14)"]
-metrics-tests = ["Werkzeug (>=1.0.1)", "accelerate", "bert-score (>=0.3.6)", "jiwer", "langdetect", "mauve-text", "nltk (<3.8.2)", "requests-file (>=1.5.1)", "rouge-score", "sacrebleu", "sacremoses", "scikit-learn", "scipy", "sentencepiece", "seqeval", "six (>=1.15.0,<1.16.0)", "spacy (>=3.0.0)", "texttable (>=1.6.3)", "tldextract", "tldextract (>=3.1.0)", "toml (>=0.10.1)", "typer (<0.5.0)"]
-quality = ["ruff (>=0.3.0)"]
-s3 = ["s3fs"]
-tensorflow = ["tensorflow (>=2.6.0)"]
-tensorflow-gpu = ["tensorflow (>=2.6.0)"]
-tests = ["Pillow (>=9.4.0)", "absl-py", "decorator", "elasticsearch (<8.0.0)", "faiss-cpu (>=1.8.0.post1)", "jax (>=0.3.14)", "jaxlib (>=0.3.14)", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "moto[server]", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "soxr (>=0.4.0)", "sqlalchemy", "tensorflow (>=2.16.0)", "tensorflow (>=2.6.0)", "tiktoken", "torch (>=2.0.0)", "transformers (>=4.42.0)", "typing-extensions (>=4.6.1)", "zstandard"]
-tests-numpy2 = ["Pillow (>=9.4.0)", "absl-py", "decorator", "elasticsearch (<8.0.0)", "jax (>=0.3.14)", "jaxlib (>=0.3.14)", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "moto[server]", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "soxr (>=0.4.0)", "sqlalchemy", "tiktoken", "torch (>=2.0.0)", "typing-extensions (>=4.6.1)", "zstandard"]
-torch = ["torch"]
-vision = ["Pillow (>=9.4.0)"]
-
-[[package]]
-name = "deprecated"
-version = "1.2.14"
-description = "Python @deprecated decorator to deprecate old python classes, functions or methods."
-optional = false
-python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
-files = [
- {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"},
- {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"},
-]
-
-[package.dependencies]
-wrapt = ">=1.10,<2"
-
-[package.extras]
-dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"]
-
-[[package]]
-name = "dill"
-version = "0.3.8"
-description = "serialize all of Python"
-optional = false
-python-versions = ">=3.8"
-files = [
- {file = "dill-0.3.8-py3-none-any.whl", hash = "sha256:c36ca9ffb54365bdd2f8eb3eff7d2a21237f8452b57ace88b1ac615b7e815bd7"},
- {file = "dill-0.3.8.tar.gz", hash = "sha256:3ebe3c479ad625c4553aca177444d89b486b1d84982eeacded644afc0cf797ca"},
-]
-
-[package.extras]
-graph = ["objgraph (>=1.7.2)"]
-profile = ["gprof2dot (>=2022.7.29)"]
-
-[[package]]
-name = "dirtyjson"
-version = "1.0.8"
-description = "JSON decoder for Python that can extract data from the muck"
-optional = false
-python-versions = "*"
-files = [
- {file = "dirtyjson-1.0.8-py3-none-any.whl", hash = "sha256:125e27248435a58acace26d5c2c4c11a1c0de0a9c5124c5a94ba78e517d74f53"},
- {file = "dirtyjson-1.0.8.tar.gz", hash = "sha256:90ca4a18f3ff30ce849d100dcf4a003953c79d3a2348ef056f1d9c22231a25fd"},
-]
-
-[[package]]
-name = "distro"
-version = "1.9.0"
-description = "Distro - an OS platform information API"
-optional = false
-python-versions = ">=3.6"
-files = [
- {file = "distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2"},
- {file = "distro-1.9.0.tar.gz", hash = "sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed"},
-]
-
-[[package]]
-name = "exceptiongroup"
-version = "1.2.2"
-description = "Backport of PEP 654 (exception groups)"
-optional = false
-python-versions = ">=3.7"
-files = [
- {file = "exceptiongroup-1.2.2-py3-none-any.whl", hash = "sha256:3111b9d131c238bec2f8f516e123e14ba243563fb135d3fe885990585aa7795b"},
- {file = "exceptiongroup-1.2.2.tar.gz", hash = "sha256:47c2edf7c6738fafb49fd34290706d1a1a2f4d1c6df275526b62cbb4aa5393cc"},
-]
-
-[package.extras]
-test = ["pytest (>=6)"]
-
-[[package]]
-name = "feedparser"
-version = "6.0.11"
-description = "Universal feed parser, handles RSS 0.9x, RSS 1.0, RSS 2.0, CDF, Atom 0.3, and Atom 1.0 feeds"
-optional = false
-python-versions = ">=3.6"
-files = [
- {file = "feedparser-6.0.11-py3-none-any.whl", hash = "sha256:0be7ee7b395572b19ebeb1d6aafb0028dee11169f1c934e0ed67d54992f4ad45"},
- {file = "feedparser-6.0.11.tar.gz", hash = "sha256:c9d0407b64c6f2a065d0ebb292c2b35c01050cc0dc33757461aaabdc4c4184d5"},
-]
-
-[package.dependencies]
-sgmllib3k = "*"
-
-[[package]]
-name = "filelock"
-version = "3.16.1"
-description = "A platform independent file lock."
-optional = false
-python-versions = ">=3.8"
-files = [
- {file = "filelock-3.16.1-py3-none-any.whl", hash = "sha256:2082e5703d51fbf98ea75855d9d5527e33d8ff23099bec374a134febee6946b0"},
- {file = "filelock-3.16.1.tar.gz", hash = "sha256:c249fbfcd5db47e5e2d6d62198e565475ee65e4831e2561c8e313fa7eb961435"},
-]
-
-[package.extras]
-docs = ["furo (>=2024.8.6)", "sphinx (>=8.0.2)", "sphinx-autodoc-typehints (>=2.4.1)"]
-testing = ["covdefaults (>=2.3)", "coverage (>=7.6.1)", "diff-cover (>=9.2)", "pytest (>=8.3.3)", "pytest-asyncio (>=0.24)", "pytest-cov (>=5)", "pytest-mock (>=3.14)", "pytest-timeout (>=2.3.1)", "virtualenv (>=20.26.4)"]
-typing = ["typing-extensions (>=4.12.2)"]
-
-[[package]]
-name = "frozenlist"
-version = "1.4.1"
-description = "A list-like structure which implements collections.abc.MutableSequence"
-optional = false
-python-versions = ">=3.8"
-files = [
- {file = "frozenlist-1.4.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:f9aa1878d1083b276b0196f2dfbe00c9b7e752475ed3b682025ff20c1c1f51ac"},
- {file = "frozenlist-1.4.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:29acab3f66f0f24674b7dc4736477bcd4bc3ad4b896f5f45379a67bce8b96868"},
- {file = "frozenlist-1.4.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:74fb4bee6880b529a0c6560885fce4dc95936920f9f20f53d99a213f7bf66776"},
- {file = "frozenlist-1.4.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:590344787a90ae57d62511dd7c736ed56b428f04cd8c161fcc5e7232c130c69a"},
- {file = "frozenlist-1.4.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:068b63f23b17df8569b7fdca5517edef76171cf3897eb68beb01341131fbd2ad"},
- {file = "frozenlist-1.4.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5c849d495bf5154cd8da18a9eb15db127d4dba2968d88831aff6f0331ea9bd4c"},
- {file = "frozenlist-1.4.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9750cc7fe1ae3b1611bb8cfc3f9ec11d532244235d75901fb6b8e42ce9229dfe"},
- {file = "frozenlist-1.4.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a9b2de4cf0cdd5bd2dee4c4f63a653c61d2408055ab77b151c1957f221cabf2a"},
- {file = "frozenlist-1.4.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:0633c8d5337cb5c77acbccc6357ac49a1770b8c487e5b3505c57b949b4b82e98"},
- {file = "frozenlist-1.4.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:27657df69e8801be6c3638054e202a135c7f299267f1a55ed3a598934f6c0d75"},
- {file = "frozenlist-1.4.1-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:f9a3ea26252bd92f570600098783d1371354d89d5f6b7dfd87359d669f2109b5"},
- {file = "frozenlist-1.4.1-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:4f57dab5fe3407b6c0c1cc907ac98e8a189f9e418f3b6e54d65a718aaafe3950"},
- {file = "frozenlist-1.4.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:e02a0e11cf6597299b9f3bbd3f93d79217cb90cfd1411aec33848b13f5c656cc"},
- {file = "frozenlist-1.4.1-cp310-cp310-win32.whl", hash = "sha256:a828c57f00f729620a442881cc60e57cfcec6842ba38e1b19fd3e47ac0ff8dc1"},
- {file = "frozenlist-1.4.1-cp310-cp310-win_amd64.whl", hash = "sha256:f56e2333dda1fe0f909e7cc59f021eba0d2307bc6f012a1ccf2beca6ba362439"},
- {file = "frozenlist-1.4.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:a0cb6f11204443f27a1628b0e460f37fb30f624be6051d490fa7d7e26d4af3d0"},
- {file = "frozenlist-1.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b46c8ae3a8f1f41a0d2ef350c0b6e65822d80772fe46b653ab6b6274f61d4a49"},
- {file = "frozenlist-1.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fde5bd59ab5357e3853313127f4d3565fc7dad314a74d7b5d43c22c6a5ed2ced"},
- {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:722e1124aec435320ae01ee3ac7bec11a5d47f25d0ed6328f2273d287bc3abb0"},
- {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2471c201b70d58a0f0c1f91261542a03d9a5e088ed3dc6c160d614c01649c106"},
- {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c757a9dd70d72b076d6f68efdbb9bc943665ae954dad2801b874c8c69e185068"},
- {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f146e0911cb2f1da549fc58fc7bcd2b836a44b79ef871980d605ec392ff6b0d2"},
- {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f9c515e7914626b2a2e1e311794b4c35720a0be87af52b79ff8e1429fc25f19"},
- {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:c302220494f5c1ebeb0912ea782bcd5e2f8308037b3c7553fad0e48ebad6ad82"},
- {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:442acde1e068288a4ba7acfe05f5f343e19fac87bfc96d89eb886b0363e977ec"},
- {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:1b280e6507ea8a4fa0c0a7150b4e526a8d113989e28eaaef946cc77ffd7efc0a"},
- {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:fe1a06da377e3a1062ae5fe0926e12b84eceb8a50b350ddca72dc85015873f74"},
- {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:db9e724bebd621d9beca794f2a4ff1d26eed5965b004a97f1f1685a173b869c2"},
- {file = "frozenlist-1.4.1-cp311-cp311-win32.whl", hash = "sha256:e774d53b1a477a67838a904131c4b0eef6b3d8a651f8b138b04f748fccfefe17"},
- {file = "frozenlist-1.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:fb3c2db03683b5767dedb5769b8a40ebb47d6f7f45b1b3e3b4b51ec8ad9d9825"},
- {file = "frozenlist-1.4.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:1979bc0aeb89b33b588c51c54ab0161791149f2461ea7c7c946d95d5f93b56ae"},
- {file = "frozenlist-1.4.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:cc7b01b3754ea68a62bd77ce6020afaffb44a590c2289089289363472d13aedb"},
- {file = "frozenlist-1.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c9c92be9fd329ac801cc420e08452b70e7aeab94ea4233a4804f0915c14eba9b"},
- {file = "frozenlist-1.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5c3894db91f5a489fc8fa6a9991820f368f0b3cbdb9cd8849547ccfab3392d86"},
- {file = "frozenlist-1.4.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ba60bb19387e13597fb059f32cd4d59445d7b18b69a745b8f8e5db0346f33480"},
- {file = "frozenlist-1.4.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8aefbba5f69d42246543407ed2461db31006b0f76c4e32dfd6f42215a2c41d09"},
- {file = "frozenlist-1.4.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:780d3a35680ced9ce682fbcf4cb9c2bad3136eeff760ab33707b71db84664e3a"},
- {file = "frozenlist-1.4.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9acbb16f06fe7f52f441bb6f413ebae6c37baa6ef9edd49cdd567216da8600cd"},
- {file = "frozenlist-1.4.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:23b701e65c7b36e4bf15546a89279bd4d8675faabc287d06bbcfac7d3c33e1e6"},
- {file = "frozenlist-1.4.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:3e0153a805a98f5ada7e09826255ba99fb4f7524bb81bf6b47fb702666484ae1"},
- {file = "frozenlist-1.4.1-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:dd9b1baec094d91bf36ec729445f7769d0d0cf6b64d04d86e45baf89e2b9059b"},
- {file = "frozenlist-1.4.1-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:1a4471094e146b6790f61b98616ab8e44f72661879cc63fa1049d13ef711e71e"},
- {file = "frozenlist-1.4.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:5667ed53d68d91920defdf4035d1cdaa3c3121dc0b113255124bcfada1cfa1b8"},
- {file = "frozenlist-1.4.1-cp312-cp312-win32.whl", hash = "sha256:beee944ae828747fd7cb216a70f120767fc9f4f00bacae8543c14a6831673f89"},
- {file = "frozenlist-1.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:64536573d0a2cb6e625cf309984e2d873979709f2cf22839bf2d61790b448ad5"},
- {file = "frozenlist-1.4.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:20b51fa3f588ff2fe658663db52a41a4f7aa6c04f6201449c6c7c476bd255c0d"},
- {file = "frozenlist-1.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:410478a0c562d1a5bcc2f7ea448359fcb050ed48b3c6f6f4f18c313a9bdb1826"},
- {file = "frozenlist-1.4.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c6321c9efe29975232da3bd0af0ad216800a47e93d763ce64f291917a381b8eb"},
- {file = "frozenlist-1.4.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:48f6a4533887e189dae092f1cf981f2e3885175f7a0f33c91fb5b7b682b6bab6"},
- {file = "frozenlist-1.4.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6eb73fa5426ea69ee0e012fb59cdc76a15b1283d6e32e4f8dc4482ec67d1194d"},
- {file = "frozenlist-1.4.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fbeb989b5cc29e8daf7f976b421c220f1b8c731cbf22b9130d8815418ea45887"},
- {file = "frozenlist-1.4.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:32453c1de775c889eb4e22f1197fe3bdfe457d16476ea407472b9442e6295f7a"},
- {file = "frozenlist-1.4.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:693945278a31f2086d9bf3df0fe8254bbeaef1fe71e1351c3bd730aa7d31c41b"},
- {file = "frozenlist-1.4.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:1d0ce09d36d53bbbe566fe296965b23b961764c0bcf3ce2fa45f463745c04701"},
- {file = "frozenlist-1.4.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:3a670dc61eb0d0eb7080890c13de3066790f9049b47b0de04007090807c776b0"},
- {file = "frozenlist-1.4.1-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:dca69045298ce5c11fd539682cff879cc1e664c245d1c64da929813e54241d11"},
- {file = "frozenlist-1.4.1-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:a06339f38e9ed3a64e4c4e43aec7f59084033647f908e4259d279a52d3757d09"},
- {file = "frozenlist-1.4.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b7f2f9f912dca3934c1baec2e4585a674ef16fe00218d833856408c48d5beee7"},
- {file = "frozenlist-1.4.1-cp38-cp38-win32.whl", hash = "sha256:e7004be74cbb7d9f34553a5ce5fb08be14fb33bc86f332fb71cbe5216362a497"},
- {file = "frozenlist-1.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:5a7d70357e7cee13f470c7883a063aae5fe209a493c57d86eb7f5a6f910fae09"},
- {file = "frozenlist-1.4.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:bfa4a17e17ce9abf47a74ae02f32d014c5e9404b6d9ac7f729e01562bbee601e"},
- {file = "frozenlist-1.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b7e3ed87d4138356775346e6845cccbe66cd9e207f3cd11d2f0b9fd13681359d"},
- {file = "frozenlist-1.4.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c99169d4ff810155ca50b4da3b075cbde79752443117d89429595c2e8e37fed8"},
- {file = "frozenlist-1.4.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:edb678da49d9f72c9f6c609fbe41a5dfb9a9282f9e6a2253d5a91e0fc382d7c0"},
- {file = "frozenlist-1.4.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6db4667b187a6742b33afbbaf05a7bc551ffcf1ced0000a571aedbb4aa42fc7b"},
- {file = "frozenlist-1.4.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:55fdc093b5a3cb41d420884cdaf37a1e74c3c37a31f46e66286d9145d2063bd0"},
- {file = "frozenlist-1.4.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:82e8211d69a4f4bc360ea22cd6555f8e61a1bd211d1d5d39d3d228b48c83a897"},
- {file = "frozenlist-1.4.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89aa2c2eeb20957be2d950b85974b30a01a762f3308cd02bb15e1ad632e22dc7"},
- {file = "frozenlist-1.4.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9d3e0c25a2350080e9319724dede4f31f43a6c9779be48021a7f4ebde8b2d742"},
- {file = "frozenlist-1.4.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7268252af60904bf52c26173cbadc3a071cece75f873705419c8681f24d3edea"},
- {file = "frozenlist-1.4.1-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:0c250a29735d4f15321007fb02865f0e6b6a41a6b88f1f523ca1596ab5f50bd5"},
- {file = "frozenlist-1.4.1-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:96ec70beabbd3b10e8bfe52616a13561e58fe84c0101dd031dc78f250d5128b9"},
- {file = "frozenlist-1.4.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:23b2d7679b73fe0e5a4560b672a39f98dfc6f60df63823b0a9970525325b95f6"},
- {file = "frozenlist-1.4.1-cp39-cp39-win32.whl", hash = "sha256:a7496bfe1da7fb1a4e1cc23bb67c58fab69311cc7d32b5a99c2007b4b2a0e932"},
- {file = "frozenlist-1.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:e6a20a581f9ce92d389a8c7d7c3dd47c81fd5d6e655c8dddf341e14aa48659d0"},
- {file = "frozenlist-1.4.1-py3-none-any.whl", hash = "sha256:04ced3e6a46b4cfffe20f9ae482818e34eba9b5fb0ce4056e4cc9b6e212d09b7"},
- {file = "frozenlist-1.4.1.tar.gz", hash = "sha256:c037a86e8513059a2613aaba4d817bb90b9d9b6b69aace3ce9c877e8c8ed402b"},
-]
-
-[[package]]
-name = "fsspec"
-version = "2024.6.1"
-description = "File-system specification"
-optional = false
-python-versions = ">=3.8"
-files = [
- {file = "fsspec-2024.6.1-py3-none-any.whl", hash = "sha256:3cb443f8bcd2efb31295a5b9fdb02aee81d8452c80d28f97a6d0959e6cee101e"},
- {file = "fsspec-2024.6.1.tar.gz", hash = "sha256:fad7d7e209dd4c1208e3bbfda706620e0da5142bebbd9c384afb95b07e798e49"},
-]
-
-[package.dependencies]
-aiohttp = {version = "<4.0.0a0 || >4.0.0a0,<4.0.0a1 || >4.0.0a1", optional = true, markers = "extra == \"http\""}
-
-[package.extras]
-abfs = ["adlfs"]
-adl = ["adlfs"]
-arrow = ["pyarrow (>=1)"]
-dask = ["dask", "distributed"]
-dev = ["pre-commit", "ruff"]
-doc = ["numpydoc", "sphinx", "sphinx-design", "sphinx-rtd-theme", "yarl"]
-dropbox = ["dropbox", "dropboxdrivefs", "requests"]
-full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "dask", "distributed", "dropbox", "dropboxdrivefs", "fusepy", "gcsfs", "libarchive-c", "ocifs", "panel", "paramiko", "pyarrow (>=1)", "pygit2", "requests", "s3fs", "smbprotocol", "tqdm"]
-fuse = ["fusepy"]
-gcs = ["gcsfs"]
-git = ["pygit2"]
-github = ["requests"]
-gs = ["gcsfs"]
-gui = ["panel"]
-hdfs = ["pyarrow (>=1)"]
-http = ["aiohttp (!=4.0.0a0,!=4.0.0a1)"]
-libarchive = ["libarchive-c"]
-oci = ["ocifs"]
-s3 = ["s3fs"]
-sftp = ["paramiko"]
-smb = ["smbprotocol"]
-ssh = ["paramiko"]
-test = ["aiohttp (!=4.0.0a0,!=4.0.0a1)", "numpy", "pytest", "pytest-asyncio (!=0.22.0)", "pytest-benchmark", "pytest-cov", "pytest-mock", "pytest-recording", "pytest-rerunfailures", "requests"]
-test-downstream = ["aiobotocore (>=2.5.4,<3.0.0)", "dask-expr", "dask[dataframe,test]", "moto[server] (>4,<5)", "pytest-timeout", "xarray"]
-test-full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "cloudpickle", "dask", "distributed", "dropbox", "dropboxdrivefs", "fastparquet", "fusepy", "gcsfs", "jinja2", "kerchunk", "libarchive-c", "lz4", "notebook", "numpy", "ocifs", "pandas", "panel", "paramiko", "pyarrow", "pyarrow (>=1)", "pyftpdlib", "pygit2", "pytest", "pytest-asyncio (!=0.22.0)", "pytest-benchmark", "pytest-cov", "pytest-mock", "pytest-recording", "pytest-rerunfailures", "python-snappy", "requests", "smbprotocol", "tqdm", "urllib3", "zarr", "zstandard"]
-tqdm = ["tqdm"]
-
-[[package]]
-name = "greenlet"
-version = "3.1.1"
-description = "Lightweight in-process concurrent programming"
-optional = false
-python-versions = ">=3.7"
-files = [
- {file = "greenlet-3.1.1-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:0bbae94a29c9e5c7e4a2b7f0aae5c17e8e90acbfd3bf6270eeba60c39fce3563"},
- {file = "greenlet-3.1.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0fde093fb93f35ca72a556cf72c92ea3ebfda3d79fc35bb19fbe685853869a83"},
- {file = "greenlet-3.1.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:36b89d13c49216cadb828db8dfa6ce86bbbc476a82d3a6c397f0efae0525bdd0"},
- {file = "greenlet-3.1.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:94b6150a85e1b33b40b1464a3f9988dcc5251d6ed06842abff82e42632fac120"},
- {file = "greenlet-3.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:93147c513fac16385d1036b7e5b102c7fbbdb163d556b791f0f11eada7ba65dc"},
- {file = "greenlet-3.1.1-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:da7a9bff22ce038e19bf62c4dd1ec8391062878710ded0a845bcf47cc0200617"},
- {file = "greenlet-3.1.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b2795058c23988728eec1f36a4e5e4ebad22f8320c85f3587b539b9ac84128d7"},
- {file = "greenlet-3.1.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ed10eac5830befbdd0c32f83e8aa6288361597550ba669b04c48f0f9a2c843c6"},
- {file = "greenlet-3.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:77c386de38a60d1dfb8e55b8c1101d68c79dfdd25c7095d51fec2dd800892b80"},
- {file = "greenlet-3.1.1-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:e4d333e558953648ca09d64f13e6d8f0523fa705f51cae3f03b5983489958c70"},
- {file = "greenlet-3.1.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:09fc016b73c94e98e29af67ab7b9a879c307c6731a2c9da0db5a7d9b7edd1159"},
- {file = "greenlet-3.1.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d5e975ca70269d66d17dd995dafc06f1b06e8cb1ec1e9ed54c1d1e4a7c4cf26e"},
- {file = "greenlet-3.1.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3b2813dc3de8c1ee3f924e4d4227999285fd335d1bcc0d2be6dc3f1f6a318ec1"},
- {file = "greenlet-3.1.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e347b3bfcf985a05e8c0b7d462ba6f15b1ee1c909e2dcad795e49e91b152c383"},
- {file = "greenlet-3.1.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9e8f8c9cb53cdac7ba9793c276acd90168f416b9ce36799b9b885790f8ad6c0a"},
- {file = "greenlet-3.1.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:62ee94988d6b4722ce0028644418d93a52429e977d742ca2ccbe1c4f4a792511"},
- {file = "greenlet-3.1.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1776fd7f989fc6b8d8c8cb8da1f6b82c5814957264d1f6cf818d475ec2bf6395"},
- {file = "greenlet-3.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:48ca08c771c268a768087b408658e216133aecd835c0ded47ce955381105ba39"},
- {file = "greenlet-3.1.1-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:4afe7ea89de619adc868e087b4d2359282058479d7cfb94970adf4b55284574d"},
- {file = "greenlet-3.1.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f406b22b7c9a9b4f8aa9d2ab13d6ae0ac3e85c9a809bd590ad53fed2bf70dc79"},
- {file = "greenlet-3.1.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c3a701fe5a9695b238503ce5bbe8218e03c3bcccf7e204e455e7462d770268aa"},
- {file = "greenlet-3.1.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2846930c65b47d70b9d178e89c7e1a69c95c1f68ea5aa0a58646b7a96df12441"},
- {file = "greenlet-3.1.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:99cfaa2110534e2cf3ba31a7abcac9d328d1d9f1b95beede58294a60348fba36"},
- {file = "greenlet-3.1.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1443279c19fca463fc33e65ef2a935a5b09bb90f978beab37729e1c3c6c25fe9"},
- {file = "greenlet-3.1.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b7cede291382a78f7bb5f04a529cb18e068dd29e0fb27376074b6d0317bf4dd0"},
- {file = "greenlet-3.1.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:23f20bb60ae298d7d8656c6ec6db134bca379ecefadb0b19ce6f19d1f232a942"},
- {file = "greenlet-3.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:7124e16b4c55d417577c2077be379514321916d5790fa287c9ed6f23bd2ffd01"},
- {file = "greenlet-3.1.1-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:05175c27cb459dcfc05d026c4232f9de8913ed006d42713cb8a5137bd49375f1"},
- {file = "greenlet-3.1.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:935e943ec47c4afab8965954bf49bfa639c05d4ccf9ef6e924188f762145c0ff"},
- {file = "greenlet-3.1.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:667a9706c970cb552ede35aee17339a18e8f2a87a51fba2ed39ceeeb1004798a"},
- {file = "greenlet-3.1.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b8a678974d1f3aa55f6cc34dc480169d58f2e6d8958895d68845fa4ab566509e"},
- {file = "greenlet-3.1.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:efc0f674aa41b92da8c49e0346318c6075d734994c3c4e4430b1c3f853e498e4"},
- {file = "greenlet-3.1.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0153404a4bb921f0ff1abeb5ce8a5131da56b953eda6e14b88dc6bbc04d2049e"},
- {file = "greenlet-3.1.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:275f72decf9932639c1c6dd1013a1bc266438eb32710016a1c742df5da6e60a1"},
- {file = "greenlet-3.1.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:c4aab7f6381f38a4b42f269057aee279ab0fc7bf2e929e3d4abfae97b682a12c"},
- {file = "greenlet-3.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:b42703b1cf69f2aa1df7d1030b9d77d3e584a70755674d60e710f0af570f3761"},
- {file = "greenlet-3.1.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f1695e76146579f8c06c1509c7ce4dfe0706f49c6831a817ac04eebb2fd02011"},
- {file = "greenlet-3.1.1-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7876452af029456b3f3549b696bb36a06db7c90747740c5302f74a9e9fa14b13"},
- {file = "greenlet-3.1.1-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4ead44c85f8ab905852d3de8d86f6f8baf77109f9da589cb4fa142bd3b57b475"},
- {file = "greenlet-3.1.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8320f64b777d00dd7ccdade271eaf0cad6636343293a25074cc5566160e4de7b"},
- {file = "greenlet-3.1.1-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6510bf84a6b643dabba74d3049ead221257603a253d0a9873f55f6a59a65f822"},
- {file = "greenlet-3.1.1-cp313-cp313t-musllinux_1_1_aarch64.whl", hash = "sha256:04b013dc07c96f83134b1e99888e7a79979f1a247e2a9f59697fa14b5862ed01"},
- {file = "greenlet-3.1.1-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:411f015496fec93c1c8cd4e5238da364e1da7a124bcb293f085bf2860c32c6f6"},
- {file = "greenlet-3.1.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:47da355d8687fd65240c364c90a31569a133b7b60de111c255ef5b606f2ae291"},
- {file = "greenlet-3.1.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:98884ecf2ffb7d7fe6bd517e8eb99d31ff7855a840fa6d0d63cd07c037f6a981"},
- {file = "greenlet-3.1.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f1d4aeb8891338e60d1ab6127af1fe45def5259def8094b9c7e34690c8858803"},
- {file = "greenlet-3.1.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db32b5348615a04b82240cc67983cb315309e88d444a288934ee6ceaebcad6cc"},
- {file = "greenlet-3.1.1-cp37-cp37m-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dcc62f31eae24de7f8dce72134c8651c58000d3b1868e01392baea7c32c247de"},
- {file = "greenlet-3.1.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:1d3755bcb2e02de341c55b4fca7a745a24a9e7212ac953f6b3a48d117d7257aa"},
- {file = "greenlet-3.1.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:b8da394b34370874b4572676f36acabac172602abf054cbc4ac910219f3340af"},
- {file = "greenlet-3.1.1-cp37-cp37m-win32.whl", hash = "sha256:a0dfc6c143b519113354e780a50381508139b07d2177cb6ad6a08278ec655798"},
- {file = "greenlet-3.1.1-cp37-cp37m-win_amd64.whl", hash = "sha256:54558ea205654b50c438029505def3834e80f0869a70fb15b871c29b4575ddef"},
- {file = "greenlet-3.1.1-cp38-cp38-macosx_11_0_universal2.whl", hash = "sha256:346bed03fe47414091be4ad44786d1bd8bef0c3fcad6ed3dee074a032ab408a9"},
- {file = "greenlet-3.1.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dfc59d69fc48664bc693842bd57acfdd490acafda1ab52c7836e3fc75c90a111"},
- {file = "greenlet-3.1.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d21e10da6ec19b457b82636209cbe2331ff4306b54d06fa04b7c138ba18c8a81"},
- {file = "greenlet-3.1.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:37b9de5a96111fc15418819ab4c4432e4f3c2ede61e660b1e33971eba26ef9ba"},
- {file = "greenlet-3.1.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6ef9ea3f137e5711f0dbe5f9263e8c009b7069d8a1acea822bd5e9dae0ae49c8"},
- {file = "greenlet-3.1.1-cp38-cp38-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:85f3ff71e2e60bd4b4932a043fbbe0f499e263c628390b285cb599154a3b03b1"},
- {file = "greenlet-3.1.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:95ffcf719966dd7c453f908e208e14cde192e09fde6c7186c8f1896ef778d8cd"},
- {file = "greenlet-3.1.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:03a088b9de532cbfe2ba2034b2b85e82df37874681e8c470d6fb2f8c04d7e4b7"},
- {file = "greenlet-3.1.1-cp38-cp38-win32.whl", hash = "sha256:8b8b36671f10ba80e159378df9c4f15c14098c4fd73a36b9ad715f057272fbef"},
- {file = "greenlet-3.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:7017b2be767b9d43cc31416aba48aab0d2309ee31b4dbf10a1d38fb7972bdf9d"},
- {file = "greenlet-3.1.1-cp39-cp39-macosx_11_0_universal2.whl", hash = "sha256:396979749bd95f018296af156201d6211240e7a23090f50a8d5d18c370084dc3"},
- {file = "greenlet-3.1.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca9d0ff5ad43e785350894d97e13633a66e2b50000e8a183a50a88d834752d42"},
- {file = "greenlet-3.1.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f6ff3b14f2df4c41660a7dec01045a045653998784bf8cfcb5a525bdffffbc8f"},
- {file = "greenlet-3.1.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:94ebba31df2aa506d7b14866fed00ac141a867e63143fe5bca82a8e503b36437"},
- {file = "greenlet-3.1.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:73aaad12ac0ff500f62cebed98d8789198ea0e6f233421059fa68a5aa7220145"},
- {file = "greenlet-3.1.1-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:63e4844797b975b9af3a3fb8f7866ff08775f5426925e1e0bbcfe7932059a12c"},
- {file = "greenlet-3.1.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:7939aa3ca7d2a1593596e7ac6d59391ff30281ef280d8632fa03d81f7c5f955e"},
- {file = "greenlet-3.1.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d0028e725ee18175c6e422797c407874da24381ce0690d6b9396c204c7f7276e"},
- {file = "greenlet-3.1.1-cp39-cp39-win32.whl", hash = "sha256:5e06afd14cbaf9e00899fae69b24a32f2196c19de08fcb9f4779dd4f004e5e7c"},
- {file = "greenlet-3.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:3319aa75e0e0639bc15ff54ca327e8dc7a6fe404003496e3c6925cd3142e0e22"},
- {file = "greenlet-3.1.1.tar.gz", hash = "sha256:4ce3ac6cdb6adf7946475d7ef31777c26d94bccc377e070a7986bd2d5c515467"},
-]
-
-[package.extras]
-docs = ["Sphinx", "furo"]
-test = ["objgraph", "psutil"]
-
-[[package]]
-name = "grep-ast"
-version = "0.3.3"
-description = "A tool to grep through the AST of a source file"
-optional = false
-python-versions = "*"
-files = [
- {file = "grep_ast-0.3.3-py3-none-any.whl", hash = "sha256:515cb889bffefefa26c4ab1377b9a75b3fc678aa5fa02bf9aa4f8f20999a83ad"},
- {file = "grep_ast-0.3.3.tar.gz", hash = "sha256:42b8887d57301dc55634368f8d549e9c49c913dafb4d19c9b54c3ddb604fccf4"},
-]
-
-[package.dependencies]
-pathspec = "*"
-tree-sitter-languages = ">=1.8.0"
-
-[[package]]
-name = "h11"
-version = "0.14.0"
-description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1"
-optional = false
-python-versions = ">=3.7"
-files = [
- {file = "h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761"},
- {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"},
-]
-
-[[package]]
-name = "hide-py"
-version = "0.3.0"
-description = "Hide: Headless IDE for coding agents"
-optional = false
-python-versions = "<4.0,>=3.10"
-files = [
- {file = "hide_py-0.3.0-py3-none-any.whl", hash = "sha256:5b3f68e206d721c83a0d027b733fabb51ad406783883f71fccbb7d682a63353d"},
- {file = "hide_py-0.3.0.tar.gz", hash = "sha256:3509f88d05e479580ba87300a096265bd4cf183d40b3fa39233eb5f95d1f3179"},
-]
-
-[package.dependencies]
-langchain = ">=0.1.16,<0.2.0"
-langchain-openai = ">=0.1.3,<0.2.0"
-langchainhub = ">=0.1.15,<0.2.0"
-pydantic = ">=2.7.3,<3.0.0"
-pyjson5 = ">=1.6.6,<2.0.0"
-requests = ">=2.31.0,<3.0.0"
-
-[[package]]
-name = "httpcore"
-version = "1.0.5"
-description = "A minimal low-level HTTP client."
-optional = false
-python-versions = ">=3.8"
-files = [
- {file = "httpcore-1.0.5-py3-none-any.whl", hash = "sha256:421f18bac248b25d310f3cacd198d55b8e6125c107797b609ff9b7a6ba7991b5"},
- {file = "httpcore-1.0.5.tar.gz", hash = "sha256:34a38e2f9291467ee3b44e89dd52615370e152954ba21721378a87b2960f7a61"},
-]
-
-[package.dependencies]
-certifi = "*"
-h11 = ">=0.13,<0.15"
-
-[package.extras]
-asyncio = ["anyio (>=4.0,<5.0)"]
-http2 = ["h2 (>=3,<5)"]
-socks = ["socksio (==1.*)"]
-trio = ["trio (>=0.22.0,<0.26.0)"]
-
-[[package]]
-name = "httpx"
-version = "0.27.2"
-description = "The next generation HTTP client."
-optional = false
-python-versions = ">=3.8"
-files = [
- {file = "httpx-0.27.2-py3-none-any.whl", hash = "sha256:7bb2708e112d8fdd7829cd4243970f0c223274051cb35ee80c03301ee29a3df0"},
- {file = "httpx-0.27.2.tar.gz", hash = "sha256:f7c2be1d2f3c3c3160d441802406b206c2b76f5947b11115e6df10c6c65e66c2"},
-]
-
-[package.dependencies]
-anyio = "*"
-certifi = "*"
-httpcore = "==1.*"
-idna = "*"
-sniffio = "*"
-
-[package.extras]
-brotli = ["brotli", "brotlicffi"]
-cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"]
-http2 = ["h2 (>=3,<5)"]
-socks = ["socksio (==1.*)"]
-zstd = ["zstandard (>=0.18.0)"]
-
-[[package]]
-name = "httpx-socks"
-version = "0.9.1"
-description = "Proxy (HTTP, SOCKS) transports for httpx"
-optional = false
-python-versions = "*"
-files = [
- {file = "httpx-socks-0.9.1.tar.gz", hash = "sha256:80ab86bad96fdcbb44b59940f2d3218577a7f09a6d4fdeb2ebaf9ccdff4748a9"},
- {file = "httpx_socks-0.9.1-py3-none-any.whl", hash = "sha256:d01dabfdf4da2a8d6c82986ddcfdbb5799a32a21eda0a0639934caf9411bf4a5"},
-]
-
-[package.dependencies]
-httpcore = ">=0.17.3,<2.0"
-httpx = ">=0.21.0,<0.28.0"
-python-socks = ">=2.0.0"
-
-[package.extras]
-asyncio = ["async-timeout (>=3.0.1)"]
-trio = ["trio (>=0.16.0)"]
-
-[[package]]
-name = "huggingface-hub"
-version = "0.25.0"
-description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub"
-optional = false
-python-versions = ">=3.8.0"
-files = [
- {file = "huggingface_hub-0.25.0-py3-none-any.whl", hash = "sha256:e2f357b35d72d5012cfd127108c4e14abcd61ba4ebc90a5a374dc2456cb34e12"},
- {file = "huggingface_hub-0.25.0.tar.gz", hash = "sha256:fb5fbe6c12fcd99d187ec7db95db9110fb1a20505f23040a5449a717c1a0db4d"},
-]
-
-[package.dependencies]
-filelock = "*"
-fsspec = ">=2023.5.0"
-packaging = ">=20.9"
-pyyaml = ">=5.1"
-requests = "*"
-tqdm = ">=4.42.1"
-typing-extensions = ">=3.7.4.3"
-
-[package.extras]
-all = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "fastapi", "gradio", "jedi", "minijinja (>=1.0)", "mypy (==1.5.1)", "numpy", "pytest (>=8.1.1,<8.2.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures", "pytest-vcr", "pytest-xdist", "ruff (>=0.5.0)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"]
-cli = ["InquirerPy (==0.3.4)"]
-dev = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "fastapi", "gradio", "jedi", "minijinja (>=1.0)", "mypy (==1.5.1)", "numpy", "pytest (>=8.1.1,<8.2.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures", "pytest-vcr", "pytest-xdist", "ruff (>=0.5.0)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"]
-fastai = ["fastai (>=2.4)", "fastcore (>=1.3.27)", "toml"]
-hf-transfer = ["hf-transfer (>=0.1.4)"]
-inference = ["aiohttp", "minijinja (>=1.0)"]
-quality = ["mypy (==1.5.1)", "ruff (>=0.5.0)"]
-tensorflow = ["graphviz", "pydot", "tensorflow"]
-tensorflow-testing = ["keras (<3.0)", "tensorflow"]
-testing = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "fastapi", "gradio", "jedi", "minijinja (>=1.0)", "numpy", "pytest (>=8.1.1,<8.2.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures", "pytest-vcr", "pytest-xdist", "soundfile", "urllib3 (<2.0)"]
-torch = ["safetensors[torch]", "torch"]
-typing = ["types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)"]
-
-[[package]]
-name = "idna"
-version = "3.10"
-description = "Internationalized Domain Names in Applications (IDNA)"
-optional = false
-python-versions = ">=3.6"
-files = [
- {file = "idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3"},
- {file = "idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9"},
-]
-
-[package.extras]
-all = ["flake8 (>=7.1.1)", "mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"]
-
-[[package]]
-name = "importlib-metadata"
-version = "8.5.0"
-description = "Read metadata from Python packages"
-optional = false
-python-versions = ">=3.8"
-files = [
- {file = "importlib_metadata-8.5.0-py3-none-any.whl", hash = "sha256:45e54197d28b7a7f1559e60b95e7c567032b602131fbd588f1497f47880aa68b"},
- {file = "importlib_metadata-8.5.0.tar.gz", hash = "sha256:71522656f0abace1d072b9e5481a48f07c138e00f079c38c8f883823f9c26bd7"},
-]
-
-[package.dependencies]
-zipp = ">=3.20"
-
-[package.extras]
-check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)"]
-cover = ["pytest-cov"]
-doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"]
-enabler = ["pytest-enabler (>=2.2)"]
-perf = ["ipython"]
-test = ["flufl.flake8", "importlib-resources (>=1.3)", "jaraco.test (>=5.4)", "packaging", "pyfakefs", "pytest (>=6,!=8.1.*)", "pytest-perf (>=0.9.2)"]
-type = ["pytest-mypy"]
-
-[[package]]
-name = "iniconfig"
-version = "2.0.0"
-description = "brain-dead simple config-ini parsing"
-optional = false
-python-versions = ">=3.7"
-files = [
- {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"},
- {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"},
-]
-
-[[package]]
-name = "jinja2"
-version = "3.1.4"
-description = "A very fast and expressive template engine."
-optional = false
-python-versions = ">=3.7"
-files = [
- {file = "jinja2-3.1.4-py3-none-any.whl", hash = "sha256:bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d"},
- {file = "jinja2-3.1.4.tar.gz", hash = "sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369"},
-]
-
-[package.dependencies]
-MarkupSafe = ">=2.0"
-
-[package.extras]
-i18n = ["Babel (>=2.7)"]
-
-[[package]]
-name = "jiter"
-version = "0.5.0"
-description = "Fast iterable JSON parser."
-optional = false
-python-versions = ">=3.8"
-files = [
- {file = "jiter-0.5.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:b599f4e89b3def9a94091e6ee52e1d7ad7bc33e238ebb9c4c63f211d74822c3f"},
- {file = "jiter-0.5.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2a063f71c4b06225543dddadbe09d203dc0c95ba352d8b85f1221173480a71d5"},
- {file = "jiter-0.5.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:acc0d5b8b3dd12e91dd184b87273f864b363dfabc90ef29a1092d269f18c7e28"},
- {file = "jiter-0.5.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c22541f0b672f4d741382a97c65609332a783501551445ab2df137ada01e019e"},
- {file = "jiter-0.5.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:63314832e302cc10d8dfbda0333a384bf4bcfce80d65fe99b0f3c0da8945a91a"},
- {file = "jiter-0.5.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a25fbd8a5a58061e433d6fae6d5298777c0814a8bcefa1e5ecfff20c594bd749"},
- {file = "jiter-0.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:503b2c27d87dfff5ab717a8200fbbcf4714516c9d85558048b1fc14d2de7d8dc"},
- {file = "jiter-0.5.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6d1f3d27cce923713933a844872d213d244e09b53ec99b7a7fdf73d543529d6d"},
- {file = "jiter-0.5.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:c95980207b3998f2c3b3098f357994d3fd7661121f30669ca7cb945f09510a87"},
- {file = "jiter-0.5.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:afa66939d834b0ce063f57d9895e8036ffc41c4bd90e4a99631e5f261d9b518e"},
- {file = "jiter-0.5.0-cp310-none-win32.whl", hash = "sha256:f16ca8f10e62f25fd81d5310e852df6649af17824146ca74647a018424ddeccf"},
- {file = "jiter-0.5.0-cp310-none-win_amd64.whl", hash = "sha256:b2950e4798e82dd9176935ef6a55cf6a448b5c71515a556da3f6b811a7844f1e"},
- {file = "jiter-0.5.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:d4c8e1ed0ef31ad29cae5ea16b9e41529eb50a7fba70600008e9f8de6376d553"},
- {file = "jiter-0.5.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c6f16e21276074a12d8421692515b3fd6d2ea9c94fd0734c39a12960a20e85f3"},
- {file = "jiter-0.5.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5280e68e7740c8c128d3ae5ab63335ce6d1fb6603d3b809637b11713487af9e6"},
- {file = "jiter-0.5.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:583c57fc30cc1fec360e66323aadd7fc3edeec01289bfafc35d3b9dcb29495e4"},
- {file = "jiter-0.5.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:26351cc14507bdf466b5f99aba3df3143a59da75799bf64a53a3ad3155ecded9"},
- {file = "jiter-0.5.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4829df14d656b3fb87e50ae8b48253a8851c707da9f30d45aacab2aa2ba2d614"},
- {file = "jiter-0.5.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42a4bdcf7307b86cb863b2fb9bb55029b422d8f86276a50487982d99eed7c6e"},
- {file = "jiter-0.5.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:04d461ad0aebf696f8da13c99bc1b3e06f66ecf6cfd56254cc402f6385231c06"},
- {file = "jiter-0.5.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e6375923c5f19888c9226582a124b77b622f8fd0018b843c45eeb19d9701c403"},
- {file = "jiter-0.5.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:2cec323a853c24fd0472517113768c92ae0be8f8c384ef4441d3632da8baa646"},
- {file = "jiter-0.5.0-cp311-none-win32.whl", hash = "sha256:aa1db0967130b5cab63dfe4d6ff547c88b2a394c3410db64744d491df7f069bb"},
- {file = "jiter-0.5.0-cp311-none-win_amd64.whl", hash = "sha256:aa9d2b85b2ed7dc7697597dcfaac66e63c1b3028652f751c81c65a9f220899ae"},
- {file = "jiter-0.5.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:9f664e7351604f91dcdd557603c57fc0d551bc65cc0a732fdacbf73ad335049a"},
- {file = "jiter-0.5.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:044f2f1148b5248ad2c8c3afb43430dccf676c5a5834d2f5089a4e6c5bbd64df"},
- {file = "jiter-0.5.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:702e3520384c88b6e270c55c772d4bd6d7b150608dcc94dea87ceba1b6391248"},
- {file = "jiter-0.5.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:528d742dcde73fad9d63e8242c036ab4a84389a56e04efd854062b660f559544"},
- {file = "jiter-0.5.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8cf80e5fe6ab582c82f0c3331df27a7e1565e2dcf06265afd5173d809cdbf9ba"},
- {file = "jiter-0.5.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:44dfc9ddfb9b51a5626568ef4e55ada462b7328996294fe4d36de02fce42721f"},
- {file = "jiter-0.5.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c451f7922992751a936b96c5f5b9bb9312243d9b754c34b33d0cb72c84669f4e"},
- {file = "jiter-0.5.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:308fce789a2f093dca1ff91ac391f11a9f99c35369117ad5a5c6c4903e1b3e3a"},
- {file = "jiter-0.5.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7f5ad4a7c6b0d90776fdefa294f662e8a86871e601309643de30bf94bb93a64e"},
- {file = "jiter-0.5.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:ea189db75f8eca08807d02ae27929e890c7d47599ce3d0a6a5d41f2419ecf338"},
- {file = "jiter-0.5.0-cp312-none-win32.whl", hash = "sha256:e3bbe3910c724b877846186c25fe3c802e105a2c1fc2b57d6688b9f8772026e4"},
- {file = "jiter-0.5.0-cp312-none-win_amd64.whl", hash = "sha256:a586832f70c3f1481732919215f36d41c59ca080fa27a65cf23d9490e75b2ef5"},
- {file = "jiter-0.5.0-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:f04bc2fc50dc77be9d10f73fcc4e39346402ffe21726ff41028f36e179b587e6"},
- {file = "jiter-0.5.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:6f433a4169ad22fcb550b11179bb2b4fd405de9b982601914ef448390b2954f3"},
- {file = "jiter-0.5.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad4a6398c85d3a20067e6c69890ca01f68659da94d74c800298581724e426c7e"},
- {file = "jiter-0.5.0-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6baa88334e7af3f4d7a5c66c3a63808e5efbc3698a1c57626541ddd22f8e4fbf"},
- {file = "jiter-0.5.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ece0a115c05efca597c6d938f88c9357c843f8c245dbbb53361a1c01afd7148"},
- {file = "jiter-0.5.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:335942557162ad372cc367ffaf93217117401bf930483b4b3ebdb1223dbddfa7"},
- {file = "jiter-0.5.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:649b0ee97a6e6da174bffcb3c8c051a5935d7d4f2f52ea1583b5b3e7822fbf14"},
- {file = "jiter-0.5.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f4be354c5de82157886ca7f5925dbda369b77344b4b4adf2723079715f823989"},
- {file = "jiter-0.5.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5206144578831a6de278a38896864ded4ed96af66e1e63ec5dd7f4a1fce38a3a"},
- {file = "jiter-0.5.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8120c60f8121ac3d6f072b97ef0e71770cc72b3c23084c72c4189428b1b1d3b6"},
- {file = "jiter-0.5.0-cp38-none-win32.whl", hash = "sha256:6f1223f88b6d76b519cb033a4d3687ca157c272ec5d6015c322fc5b3074d8a5e"},
- {file = "jiter-0.5.0-cp38-none-win_amd64.whl", hash = "sha256:c59614b225d9f434ea8fc0d0bec51ef5fa8c83679afedc0433905994fb36d631"},
- {file = "jiter-0.5.0-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:0af3838cfb7e6afee3f00dc66fa24695199e20ba87df26e942820345b0afc566"},
- {file = "jiter-0.5.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:550b11d669600dbc342364fd4adbe987f14d0bbedaf06feb1b983383dcc4b961"},
- {file = "jiter-0.5.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:489875bf1a0ffb3cb38a727b01e6673f0f2e395b2aad3c9387f94187cb214bbf"},
- {file = "jiter-0.5.0-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b250ca2594f5599ca82ba7e68785a669b352156260c5362ea1b4e04a0f3e2389"},
- {file = "jiter-0.5.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8ea18e01f785c6667ca15407cd6dabbe029d77474d53595a189bdc813347218e"},
- {file = "jiter-0.5.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:462a52be85b53cd9bffd94e2d788a09984274fe6cebb893d6287e1c296d50653"},
- {file = "jiter-0.5.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:92cc68b48d50fa472c79c93965e19bd48f40f207cb557a8346daa020d6ba973b"},
- {file = "jiter-0.5.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1c834133e59a8521bc87ebcad773608c6fa6ab5c7a022df24a45030826cf10bc"},
- {file = "jiter-0.5.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ab3a71ff31cf2d45cb216dc37af522d335211f3a972d2fe14ea99073de6cb104"},
- {file = "jiter-0.5.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:cccd3af9c48ac500c95e1bcbc498020c87e1781ff0345dd371462d67b76643eb"},
- {file = "jiter-0.5.0-cp39-none-win32.whl", hash = "sha256:368084d8d5c4fc40ff7c3cc513c4f73e02c85f6009217922d0823a48ee7adf61"},
- {file = "jiter-0.5.0-cp39-none-win_amd64.whl", hash = "sha256:ce03f7b4129eb72f1687fa11300fbf677b02990618428934662406d2a76742a1"},
- {file = "jiter-0.5.0.tar.gz", hash = "sha256:1d916ba875bcab5c5f7d927df998c4cb694d27dceddf3392e58beaf10563368a"},
-]
-
-[[package]]
-name = "joblib"
-version = "1.4.2"
-description = "Lightweight pipelining with Python functions"
-optional = false
-python-versions = ">=3.8"
-files = [
- {file = "joblib-1.4.2-py3-none-any.whl", hash = "sha256:06d478d5674cbc267e7496a410ee875abd68e4340feff4490bcb7afb88060ae6"},
- {file = "joblib-1.4.2.tar.gz", hash = "sha256:2382c5816b2636fbd20a09e0f4e9dad4736765fdfb7dca582943b9c1366b3f0e"},
-]
-
-[[package]]
-name = "jsonpatch"
-version = "1.33"
-description = "Apply JSON-Patches (RFC 6902)"
-optional = false
-python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*"
-files = [
- {file = "jsonpatch-1.33-py2.py3-none-any.whl", hash = "sha256:0ae28c0cd062bbd8b8ecc26d7d164fbbea9652a1a3693f3b956c1eae5145dade"},
- {file = "jsonpatch-1.33.tar.gz", hash = "sha256:9fcd4009c41e6d12348b4a0ff2563ba56a2923a7dfee731d004e212e1ee5030c"},
-]
-
-[package.dependencies]
-jsonpointer = ">=1.9"
-
-[[package]]
-name = "jsonpointer"
-version = "3.0.0"
-description = "Identify specific nodes in a JSON document (RFC 6901)"
-optional = false
-python-versions = ">=3.7"
-files = [
- {file = "jsonpointer-3.0.0-py2.py3-none-any.whl", hash = "sha256:13e088adc14fca8b6aa8177c044e12701e6ad4b28ff10e65f2267a90109c9942"},
- {file = "jsonpointer-3.0.0.tar.gz", hash = "sha256:2b2d729f2091522d61c3b31f82e11870f60b68f43fbc705cb76bf4b832af59ef"},
-]
-
-[[package]]
-name = "jsonschema"
-version = "4.23.0"
-description = "An implementation of JSON Schema validation for Python"
-optional = false
-python-versions = ">=3.8"
-files = [
- {file = "jsonschema-4.23.0-py3-none-any.whl", hash = "sha256:fbadb6f8b144a8f8cf9f0b89ba94501d143e50411a1278633f56a7acf7fd5566"},
- {file = "jsonschema-4.23.0.tar.gz", hash = "sha256:d71497fef26351a33265337fa77ffeb82423f3ea21283cd9467bb03999266bc4"},
-]
-
-[package.dependencies]
-attrs = ">=22.2.0"
-jsonschema-specifications = ">=2023.03.6"
-referencing = ">=0.28.4"
-rpds-py = ">=0.7.1"
-
-[package.extras]
-format = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3987", "uri-template", "webcolors (>=1.11)"]
-format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "uri-template", "webcolors (>=24.6.0)"]
-
-[[package]]
-name = "jsonschema-specifications"
-version = "2023.12.1"
-description = "The JSON Schema meta-schemas and vocabularies, exposed as a Registry"
-optional = false
-python-versions = ">=3.8"
-files = [
- {file = "jsonschema_specifications-2023.12.1-py3-none-any.whl", hash = "sha256:87e4fdf3a94858b8a2ba2778d9ba57d8a9cafca7c7489c46ba0d30a8bc6a9c3c"},
- {file = "jsonschema_specifications-2023.12.1.tar.gz", hash = "sha256:48a76787b3e70f5ed53f1160d2b81f586e4ca6d1548c5de7085d1682674764cc"},
-]
-
-[package.dependencies]
-referencing = ">=0.31.0"
-
-[[package]]
-name = "langchain"
-version = "0.1.20"
-description = "Building applications with LLMs through composability"
-optional = false
-python-versions = "<4.0,>=3.8.1"
-files = [
- {file = "langchain-0.1.20-py3-none-any.whl", hash = "sha256:09991999fbd6c3421a12db3c7d1f52d55601fc41d9b2a3ef51aab2e0e9c38da9"},
- {file = "langchain-0.1.20.tar.gz", hash = "sha256:f35c95eed8c8375e02dce95a34f2fd4856a4c98269d6dc34547a23dba5beab7e"},
-]
-
-[package.dependencies]
-aiohttp = ">=3.8.3,<4.0.0"
-async-timeout = {version = ">=4.0.0,<5.0.0", markers = "python_version < \"3.11\""}
-dataclasses-json = ">=0.5.7,<0.7"
-langchain-community = ">=0.0.38,<0.1"
-langchain-core = ">=0.1.52,<0.2.0"
-langchain-text-splitters = ">=0.0.1,<0.1"
-langsmith = ">=0.1.17,<0.2.0"
-numpy = ">=1,<2"
-pydantic = ">=1,<3"
-PyYAML = ">=5.3"
-requests = ">=2,<3"
-SQLAlchemy = ">=1.4,<3"
-tenacity = ">=8.1.0,<9.0.0"
-
-[package.extras]
-azure = ["azure-ai-formrecognizer (>=3.2.1,<4.0.0)", "azure-ai-textanalytics (>=5.3.0,<6.0.0)", "azure-cognitiveservices-speech (>=1.28.0,<2.0.0)", "azure-core (>=1.26.4,<2.0.0)", "azure-cosmos (>=4.4.0b1,<5.0.0)", "azure-identity (>=1.12.0,<2.0.0)", "azure-search-documents (==11.4.0b8)", "openai (<2)"]
-clarifai = ["clarifai (>=9.1.0)"]
-cli = ["typer (>=0.9.0,<0.10.0)"]
-cohere = ["cohere (>=4,<6)"]
-docarray = ["docarray[hnswlib] (>=0.32.0,<0.33.0)"]
-embeddings = ["sentence-transformers (>=2,<3)"]
-extended-testing = ["aiosqlite (>=0.19.0,<0.20.0)", "aleph-alpha-client (>=2.15.0,<3.0.0)", "anthropic (>=0.3.11,<0.4.0)", "arxiv (>=1.4,<2.0)", "assemblyai (>=0.17.0,<0.18.0)", "atlassian-python-api (>=3.36.0,<4.0.0)", "beautifulsoup4 (>=4,<5)", "bibtexparser (>=1.4.0,<2.0.0)", "cassio (>=0.1.0,<0.2.0)", "chardet (>=5.1.0,<6.0.0)", "cohere (>=4,<6)", "couchbase (>=4.1.9,<5.0.0)", "dashvector (>=1.0.1,<2.0.0)", "databricks-vectorsearch (>=0.21,<0.22)", "datasets (>=2.15.0,<3.0.0)", "dgml-utils (>=0.3.0,<0.4.0)", "esprima (>=4.0.1,<5.0.0)", "faiss-cpu (>=1,<2)", "feedparser (>=6.0.10,<7.0.0)", "fireworks-ai (>=0.9.0,<0.10.0)", "geopandas (>=0.13.1,<0.14.0)", "gitpython (>=3.1.32,<4.0.0)", "google-cloud-documentai (>=2.20.1,<3.0.0)", "gql (>=3.4.1,<4.0.0)", "hologres-vector (>=0.0.6,<0.0.7)", "html2text (>=2020.1.16,<2021.0.0)", "javelin-sdk (>=0.1.8,<0.2.0)", "jinja2 (>=3,<4)", "jq (>=1.4.1,<2.0.0)", "jsonschema (>1)", "langchain-openai (>=0.0.2,<0.1)", "lxml (>=4.9.3,<6.0)", "markdownify (>=0.11.6,<0.12.0)", "motor (>=3.3.1,<4.0.0)", "msal (>=1.25.0,<2.0.0)", "mwparserfromhell (>=0.6.4,<0.7.0)", "mwxml (>=0.3.3,<0.4.0)", "newspaper3k (>=0.2.8,<0.3.0)", "numexpr (>=2.8.6,<3.0.0)", "openai (<2)", "openapi-pydantic (>=0.3.2,<0.4.0)", "pandas (>=2.0.1,<3.0.0)", "pdfminer-six (>=20221105,<20221106)", "pgvector (>=0.1.6,<0.2.0)", "praw (>=7.7.1,<8.0.0)", "psychicapi (>=0.8.0,<0.9.0)", "py-trello (>=0.19.0,<0.20.0)", "pymupdf (>=1.22.3,<2.0.0)", "pypdf (>=3.4.0,<4.0.0)", "pypdfium2 (>=4.10.0,<5.0.0)", "pyspark (>=3.4.0,<4.0.0)", "rank-bm25 (>=0.2.2,<0.3.0)", "rapidfuzz (>=3.1.1,<4.0.0)", "rapidocr-onnxruntime (>=1.3.2,<2.0.0)", "rdflib (==7.0.0)", "requests-toolbelt (>=1.0.0,<2.0.0)", "rspace_client (>=2.5.0,<3.0.0)", "scikit-learn (>=1.2.2,<2.0.0)", "sqlite-vss (>=0.1.2,<0.2.0)", "streamlit (>=1.18.0,<2.0.0)", "sympy (>=1.12,<2.0)", "telethon (>=1.28.5,<2.0.0)", "timescale-vector (>=0.0.1,<0.0.2)", "tqdm (>=4.48.0)", "upstash-redis (>=0.15.0,<0.16.0)", "xata (>=1.0.0a7,<2.0.0)", "xmltodict (>=0.13.0,<0.14.0)"]
-javascript = ["esprima (>=4.0.1,<5.0.0)"]
-llms = ["clarifai (>=9.1.0)", "cohere (>=4,<6)", "huggingface_hub (>=0,<1)", "manifest-ml (>=0.0.1,<0.0.2)", "nlpcloud (>=1,<2)", "openai (<2)", "openlm (>=0.0.5,<0.0.6)", "torch (>=1,<3)", "transformers (>=4,<5)"]
-openai = ["openai (<2)", "tiktoken (>=0.3.2,<0.6.0)"]
-qdrant = ["qdrant-client (>=1.3.1,<2.0.0)"]
-text-helpers = ["chardet (>=5.1.0,<6.0.0)"]
-
-[[package]]
-name = "langchain-community"
-version = "0.0.38"
-description = "Community contributed LangChain integrations."
-optional = false
-python-versions = "<4.0,>=3.8.1"
-files = [
- {file = "langchain_community-0.0.38-py3-none-any.whl", hash = "sha256:ecb48660a70a08c90229be46b0cc5f6bc9f38f2833ee44c57dfab9bf3a2c121a"},
- {file = "langchain_community-0.0.38.tar.gz", hash = "sha256:127fc4b75bc67b62fe827c66c02e715a730fef8fe69bd2023d466bab06b5810d"},
-]
-
-[package.dependencies]
-aiohttp = ">=3.8.3,<4.0.0"
-dataclasses-json = ">=0.5.7,<0.7"
-langchain-core = ">=0.1.52,<0.2.0"
-langsmith = ">=0.1.0,<0.2.0"
-numpy = ">=1,<2"
-PyYAML = ">=5.3"
-requests = ">=2,<3"
-SQLAlchemy = ">=1.4,<3"
-tenacity = ">=8.1.0,<9.0.0"
-
-[package.extras]
-cli = ["typer (>=0.9.0,<0.10.0)"]
-extended-testing = ["aiosqlite (>=0.19.0,<0.20.0)", "aleph-alpha-client (>=2.15.0,<3.0.0)", "anthropic (>=0.3.11,<0.4.0)", "arxiv (>=1.4,<2.0)", "assemblyai (>=0.17.0,<0.18.0)", "atlassian-python-api (>=3.36.0,<4.0.0)", "azure-ai-documentintelligence (>=1.0.0b1,<2.0.0)", "azure-identity (>=1.15.0,<2.0.0)", "azure-search-documents (==11.4.0)", "beautifulsoup4 (>=4,<5)", "bibtexparser (>=1.4.0,<2.0.0)", "cassio (>=0.1.6,<0.2.0)", "chardet (>=5.1.0,<6.0.0)", "cloudpickle (>=2.0.0)", "cohere (>=4,<5)", "databricks-vectorsearch (>=0.21,<0.22)", "datasets (>=2.15.0,<3.0.0)", "dgml-utils (>=0.3.0,<0.4.0)", "elasticsearch (>=8.12.0,<9.0.0)", "esprima (>=4.0.1,<5.0.0)", "faiss-cpu (>=1,<2)", "feedparser (>=6.0.10,<7.0.0)", "fireworks-ai (>=0.9.0,<0.10.0)", "friendli-client (>=1.2.4,<2.0.0)", "geopandas (>=0.13.1,<0.14.0)", "gitpython (>=3.1.32,<4.0.0)", "google-cloud-documentai (>=2.20.1,<3.0.0)", "gql (>=3.4.1,<4.0.0)", "gradientai (>=1.4.0,<2.0.0)", "hdbcli (>=2.19.21,<3.0.0)", "hologres-vector (>=0.0.6,<0.0.7)", "html2text (>=2020.1.16,<2021.0.0)", "httpx (>=0.24.1,<0.25.0)", "httpx-sse (>=0.4.0,<0.5.0)", "javelin-sdk (>=0.1.8,<0.2.0)", "jinja2 (>=3,<4)", "jq (>=1.4.1,<2.0.0)", "jsonschema (>1)", "lxml (>=4.9.3,<6.0)", "markdownify (>=0.11.6,<0.12.0)", "motor (>=3.3.1,<4.0.0)", "msal (>=1.25.0,<2.0.0)", "mwparserfromhell (>=0.6.4,<0.7.0)", "mwxml (>=0.3.3,<0.4.0)", "newspaper3k (>=0.2.8,<0.3.0)", "numexpr (>=2.8.6,<3.0.0)", "nvidia-riva-client (>=2.14.0,<3.0.0)", "oci (>=2.119.1,<3.0.0)", "openai (<2)", "openapi-pydantic (>=0.3.2,<0.4.0)", "oracle-ads (>=2.9.1,<3.0.0)", "oracledb (>=2.2.0,<3.0.0)", "pandas (>=2.0.1,<3.0.0)", "pdfminer-six (>=20221105,<20221106)", "pgvector (>=0.1.6,<0.2.0)", "praw (>=7.7.1,<8.0.0)", "premai (>=0.3.25,<0.4.0)", "psychicapi (>=0.8.0,<0.9.0)", "py-trello (>=0.19.0,<0.20.0)", "pyjwt (>=2.8.0,<3.0.0)", "pymupdf (>=1.22.3,<2.0.0)", "pypdf (>=3.4.0,<4.0.0)", "pypdfium2 (>=4.10.0,<5.0.0)", "pyspark (>=3.4.0,<4.0.0)", "rank-bm25 (>=0.2.2,<0.3.0)", "rapidfuzz (>=3.1.1,<4.0.0)", "rapidocr-onnxruntime (>=1.3.2,<2.0.0)", "rdflib (==7.0.0)", "requests-toolbelt (>=1.0.0,<2.0.0)", "rspace_client (>=2.5.0,<3.0.0)", "scikit-learn (>=1.2.2,<2.0.0)", "sqlite-vss (>=0.1.2,<0.2.0)", "streamlit (>=1.18.0,<2.0.0)", "sympy (>=1.12,<2.0)", "telethon (>=1.28.5,<2.0.0)", "tidb-vector (>=0.0.3,<1.0.0)", "timescale-vector (>=0.0.1,<0.0.2)", "tqdm (>=4.48.0)", "tree-sitter (>=0.20.2,<0.21.0)", "tree-sitter-languages (>=1.8.0,<2.0.0)", "upstash-redis (>=0.15.0,<0.16.0)", "vdms (>=0.0.20,<0.0.21)", "xata (>=1.0.0a7,<2.0.0)", "xmltodict (>=0.13.0,<0.14.0)"]
-
-[[package]]
-name = "langchain-core"
-version = "0.1.52"
-description = "Building applications with LLMs through composability"
-optional = false
-python-versions = "<4.0,>=3.8.1"
-files = [
- {file = "langchain_core-0.1.52-py3-none-any.whl", hash = "sha256:62566749c92e8a1181c255c788548dc16dbc319d896cd6b9c95dc17af9b2a6db"},
- {file = "langchain_core-0.1.52.tar.gz", hash = "sha256:084c3fc452f5a6966c28ab3ec5dbc8b8d26fc3f63378073928f4e29d90b6393f"},
-]
-
-[package.dependencies]
-jsonpatch = ">=1.33,<2.0"
-langsmith = ">=0.1.0,<0.2.0"
-packaging = ">=23.2,<24.0"
-pydantic = ">=1,<3"
-PyYAML = ">=5.3"
-tenacity = ">=8.1.0,<9.0.0"
-
-[package.extras]
-extended-testing = ["jinja2 (>=3,<4)"]
-
-[[package]]
-name = "langchain-openai"
-version = "0.1.7"
-description = "An integration package connecting OpenAI and LangChain"
-optional = false
-python-versions = "<4.0,>=3.8.1"
-files = [
- {file = "langchain_openai-0.1.7-py3-none-any.whl", hash = "sha256:39c3cb22bb739900ae8294d4d9939a6138c0ca7ad11198e57038eb14c08d04ec"},
- {file = "langchain_openai-0.1.7.tar.gz", hash = "sha256:fd7e1c33ba8e2cab4b2154f3a2fd4a0d9cc6518b41cf49bb87255f9f732a4896"},
-]
-
-[package.dependencies]
-langchain-core = ">=0.1.46,<0.3"
-openai = ">=1.24.0,<2.0.0"
-tiktoken = ">=0.7,<1"
-
-[[package]]
-name = "langchain-text-splitters"
-version = "0.0.2"
-description = "LangChain text splitting utilities"
-optional = false
-python-versions = "<4.0,>=3.8.1"
-files = [
- {file = "langchain_text_splitters-0.0.2-py3-none-any.whl", hash = "sha256:13887f32705862c1e1454213cb7834a63aae57c26fcd80346703a1d09c46168d"},
- {file = "langchain_text_splitters-0.0.2.tar.gz", hash = "sha256:ac8927dc0ba08eba702f6961c9ed7df7cead8de19a9f7101ab2b5ea34201b3c1"},
-]
-
-[package.dependencies]
-langchain-core = ">=0.1.28,<0.3"
-
-[package.extras]
-extended-testing = ["beautifulsoup4 (>=4.12.3,<5.0.0)", "lxml (>=4.9.3,<6.0)"]
-
-[[package]]
-name = "langchainhub"
-version = "0.1.21"
-description = "The LangChain Hub API client"
-optional = false
-python-versions = "<4.0,>=3.8.1"
-files = [
- {file = "langchainhub-0.1.21-py3-none-any.whl", hash = "sha256:1cc002dc31e0d132a776afd044361e2b698743df5202618cf2bad399246b895f"},
- {file = "langchainhub-0.1.21.tar.gz", hash = "sha256:723383b3964a47dbaea6ad5d0ef728accefbc9d2c07480e800bdec43510a8c10"},
-]
-
-[package.dependencies]
-packaging = ">=23.2,<25"
-requests = ">=2,<3"
-types-requests = ">=2.31.0.2,<3.0.0.0"
-
-[[package]]
-name = "langsmith"
-version = "0.1.125"
-description = "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform."
-optional = false
-python-versions = "<4.0,>=3.8.1"
-files = [
- {file = "langsmith-0.1.125-py3-none-any.whl", hash = "sha256:74ce8eb2663e1ed20bfcfc88d41e0712879306956c9938d1cdbab7d60458bdca"},
- {file = "langsmith-0.1.125.tar.gz", hash = "sha256:2c0eb0c3cbf22cff55bf519b8e889041f9a591bcf97af5152c8e130333c5940e"},
-]
-
-[package.dependencies]
-httpx = ">=0.23.0,<1"
-orjson = ">=3.9.14,<4.0.0"
-pydantic = [
- {version = ">=1,<3", markers = "python_full_version < \"3.12.4\""},
- {version = ">=2.7.4,<3.0.0", markers = "python_full_version >= \"3.12.4\""},
-]
-requests = ">=2,<3"
-
-[[package]]
-name = "litellm"
-version = "1.46.8"
-description = "Library to easily interface with LLM API providers"
-optional = false
-python-versions = "!=2.7.*,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,!=3.7.*,>=3.8"
-files = [
- {file = "litellm-1.46.8-py3-none-any.whl", hash = "sha256:112acc854d67ced573dc5d60bbf8b493dea1e61244013685dace8c2d912aa1b3"},
- {file = "litellm-1.46.8.tar.gz", hash = "sha256:443c67d33e1a264641b80bf170cad1ba42d6fa9816f86df5eaaaf10c1e21b551"},
-]
-
-[package.dependencies]
-aiohttp = "*"
-click = "*"
-importlib-metadata = ">=6.8.0"
-jinja2 = ">=3.1.2,<4.0.0"
-jsonschema = ">=4.22.0,<5.0.0"
-openai = ">=1.45.0"
-pydantic = ">=2.0.0,<3.0.0"
-python-dotenv = ">=0.2.0"
-requests = ">=2.31.0,<3.0.0"
-tiktoken = ">=0.7.0"
-tokenizers = "*"
-
-[package.extras]
-extra-proxy = ["azure-identity (>=1.15.0,<2.0.0)", "azure-keyvault-secrets (>=4.8.0,<5.0.0)", "google-cloud-kms (>=2.21.3,<3.0.0)", "prisma (==0.11.0)", "resend (>=0.8.0,<0.9.0)"]
-proxy = ["PyJWT (>=2.8.0,<3.0.0)", "apscheduler (>=3.10.4,<4.0.0)", "backoff", "cryptography (>=42.0.5,<43.0.0)", "fastapi (>=0.111.0,<0.112.0)", "fastapi-sso (>=0.10.0,<0.11.0)", "gunicorn (>=22.0.0,<23.0.0)", "orjson (>=3.9.7,<4.0.0)", "pynacl (>=1.5.0,<2.0.0)", "python-multipart (>=0.0.9,<0.0.10)", "pyyaml (>=6.0.1,<7.0.0)", "rq", "uvicorn (>=0.22.0,<0.23.0)"]
-
-[[package]]
-name = "llama-index-core"
-version = "0.11.10"
-description = "Interface between LLMs and your data"
-optional = false
-python-versions = "<4.0,>=3.8.1"
-files = [
- {file = "llama_index_core-0.11.10-py3-none-any.whl", hash = "sha256:2dddd7cb4ccee89fdbbddd62e5fe3c7ae7fc431130e0a0a7155daee052874191"},
- {file = "llama_index_core-0.11.10.tar.gz", hash = "sha256:9929b11cfb24a3581620466660ab11a6360fde8c2441caa3660e0127df65c1b9"},
-]
-
-[package.dependencies]
-aiohttp = ">=3.8.6,<4.0.0"
-dataclasses-json = "*"
-deprecated = ">=1.2.9.3"
-dirtyjson = ">=1.0.8,<2.0.0"
-fsspec = ">=2023.5.0"
-httpx = "*"
-nest-asyncio = ">=1.5.8,<2.0.0"
-networkx = ">=3.0"
-nltk = ">3.8.1"
-numpy = "<2.0.0"
-pillow = ">=9.0.0"
-pydantic = ">=2.7.0,<3.0.0"
-PyYAML = ">=6.0.1"
-requests = ">=2.31.0"
-SQLAlchemy = {version = ">=1.4.49", extras = ["asyncio"]}
-tenacity = ">=8.2.0,<8.4.0 || >8.4.0,<9.0.0"
-tiktoken = ">=0.3.3"
-tqdm = ">=4.66.1,<5.0.0"
-typing-extensions = ">=4.5.0"
-typing-inspect = ">=0.8.0"
-wrapt = "*"
-
-[[package]]
-name = "llama-index-llms-openai"
-version = "0.2.9"
-description = "llama-index llms openai integration"
-optional = false
-python-versions = "<4.0,>=3.8.1"
-files = [
- {file = "llama_index_llms_openai-0.2.9-py3-none-any.whl", hash = "sha256:5f36e8cbca2c3c657380c711bd3974fe7e2344d3b6a8dde6c263e56868d01e27"},
- {file = "llama_index_llms_openai-0.2.9.tar.gz", hash = "sha256:56376f39e3a40253b5c4fb90d0fb6af093f21bb2935925615f0c28a28d028187"},
-]
-
-[package.dependencies]
-llama-index-core = ">=0.11.7,<0.12.0"
-openai = ">=1.40.0,<2.0.0"
-
-[[package]]
-name = "markdown-it-py"
-version = "3.0.0"
-description = "Python port of markdown-it. Markdown parsing, done right!"
-optional = false
-python-versions = ">=3.8"
-files = [
- {file = "markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb"},
- {file = "markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1"},
-]
-
-[package.dependencies]
-mdurl = ">=0.1,<1.0"
-
-[package.extras]
-benchmarking = ["psutil", "pytest", "pytest-benchmark"]
-code-style = ["pre-commit (>=3.0,<4.0)"]
-compare = ["commonmark (>=0.9,<1.0)", "markdown (>=3.4,<4.0)", "mistletoe (>=1.0,<2.0)", "mistune (>=2.0,<3.0)", "panflute (>=2.3,<3.0)"]
-linkify = ["linkify-it-py (>=1,<3)"]
-plugins = ["mdit-py-plugins"]
-profiling = ["gprof2dot"]
-rtd = ["jupyter_sphinx", "mdit-py-plugins", "myst-parser", "pyyaml", "sphinx", "sphinx-copybutton", "sphinx-design", "sphinx_book_theme"]
-testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"]
-
-[[package]]
-name = "markupsafe"
-version = "2.1.5"
-description = "Safely add untrusted strings to HTML/XML markup."
-optional = false
-python-versions = ">=3.7"
-files = [
- {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"},
- {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"},
- {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"},
- {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"},
- {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"},
- {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"},
- {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"},
- {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"},
- {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"},
- {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"},
- {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"},
- {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"},
- {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"},
- {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"},
- {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"},
- {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"},
- {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"},
- {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"},
- {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"},
- {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"},
- {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"},
- {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"},
- {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"},
- {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"},
- {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"},
- {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"},
- {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"},
- {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"},
- {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"},
- {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"},
- {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"},
- {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"},
- {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"},
- {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"},
- {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"},
- {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"},
- {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"},
- {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"},
- {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"},
- {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"},
- {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"},
- {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"},
- {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"},
- {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"},
- {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"},
- {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"},
- {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"},
- {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"},
- {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"},
- {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"},
- {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"},
- {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"},
- {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"},
- {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"},
- {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"},
- {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"},
- {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"},
- {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"},
- {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"},
- {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"},
-]
-
-[[package]]
-name = "marshmallow"
-version = "3.22.0"
-description = "A lightweight library for converting complex datatypes to and from native Python datatypes."
-optional = false
-python-versions = ">=3.8"
-files = [
- {file = "marshmallow-3.22.0-py3-none-any.whl", hash = "sha256:71a2dce49ef901c3f97ed296ae5051135fd3febd2bf43afe0ae9a82143a494d9"},
- {file = "marshmallow-3.22.0.tar.gz", hash = "sha256:4972f529104a220bb8637d595aa4c9762afbe7f7a77d82dc58c1615d70c5823e"},
-]
-
-[package.dependencies]
-packaging = ">=17.0"
-
-[package.extras]
-dev = ["marshmallow[tests]", "pre-commit (>=3.5,<4.0)", "tox"]
-docs = ["alabaster (==1.0.0)", "autodocsumm (==0.2.13)", "sphinx (==8.0.2)", "sphinx-issues (==4.1.0)", "sphinx-version-warning (==1.1.2)"]
-tests = ["pytest", "pytz", "simplejson"]
-
-[[package]]
-name = "mdurl"
-version = "0.1.2"
-description = "Markdown URL utilities"
-optional = false
-python-versions = ">=3.7"
-files = [
- {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"},
- {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"},
-]
-
-[[package]]
-name = "mpmath"
-version = "1.3.0"
-description = "Python library for arbitrary-precision floating-point arithmetic"
-optional = false
-python-versions = "*"
-files = [
- {file = "mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c"},
- {file = "mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f"},
-]
-
-[package.extras]
-develop = ["codecov", "pycodestyle", "pytest (>=4.6)", "pytest-cov", "wheel"]
-docs = ["sphinx"]
-gmpy = ["gmpy2 (>=2.1.0a4)"]
-tests = ["pytest (>=4.6)"]
-
-[[package]]
-name = "multidict"
-version = "6.1.0"
-description = "multidict implementation"
-optional = false
-python-versions = ">=3.8"
-files = [
- {file = "multidict-6.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3380252550e372e8511d49481bd836264c009adb826b23fefcc5dd3c69692f60"},
- {file = "multidict-6.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:99f826cbf970077383d7de805c0681799491cb939c25450b9b5b3ced03ca99f1"},
- {file = "multidict-6.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a114d03b938376557927ab23f1e950827c3b893ccb94b62fd95d430fd0e5cf53"},
- {file = "multidict-6.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b1c416351ee6271b2f49b56ad7f308072f6f44b37118d69c2cad94f3fa8a40d5"},
- {file = "multidict-6.1.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6b5d83030255983181005e6cfbac1617ce9746b219bc2aad52201ad121226581"},
- {file = "multidict-6.1.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3e97b5e938051226dc025ec80980c285b053ffb1e25a3db2a3aa3bc046bf7f56"},
- {file = "multidict-6.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d618649d4e70ac6efcbba75be98b26ef5078faad23592f9b51ca492953012429"},
- {file = "multidict-6.1.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:10524ebd769727ac77ef2278390fb0068d83f3acb7773792a5080f2b0abf7748"},
- {file = "multidict-6.1.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:ff3827aef427c89a25cc96ded1759271a93603aba9fb977a6d264648ebf989db"},
- {file = "multidict-6.1.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:06809f4f0f7ab7ea2cabf9caca7d79c22c0758b58a71f9d32943ae13c7ace056"},
- {file = "multidict-6.1.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:f179dee3b863ab1c59580ff60f9d99f632f34ccb38bf67a33ec6b3ecadd0fd76"},
- {file = "multidict-6.1.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:aaed8b0562be4a0876ee3b6946f6869b7bcdb571a5d1496683505944e268b160"},
- {file = "multidict-6.1.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3c8b88a2ccf5493b6c8da9076fb151ba106960a2df90c2633f342f120751a9e7"},
- {file = "multidict-6.1.0-cp310-cp310-win32.whl", hash = "sha256:4a9cb68166a34117d6646c0023c7b759bf197bee5ad4272f420a0141d7eb03a0"},
- {file = "multidict-6.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:20b9b5fbe0b88d0bdef2012ef7dee867f874b72528cf1d08f1d59b0e3850129d"},
- {file = "multidict-6.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:3efe2c2cb5763f2f1b275ad2bf7a287d3f7ebbef35648a9726e3b69284a4f3d6"},
- {file = "multidict-6.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c7053d3b0353a8b9de430a4f4b4268ac9a4fb3481af37dfe49825bf45ca24156"},
- {file = "multidict-6.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:27e5fc84ccef8dfaabb09d82b7d179c7cf1a3fbc8a966f8274fcb4ab2eb4cadb"},
- {file = "multidict-6.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0e2b90b43e696f25c62656389d32236e049568b39320e2735d51f08fd362761b"},
- {file = "multidict-6.1.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d83a047959d38a7ff552ff94be767b7fd79b831ad1cd9920662db05fec24fe72"},
- {file = "multidict-6.1.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d1a9dd711d0877a1ece3d2e4fea11a8e75741ca21954c919406b44e7cf971304"},
- {file = "multidict-6.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec2abea24d98246b94913b76a125e855eb5c434f7c46546046372fe60f666351"},
- {file = "multidict-6.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4867cafcbc6585e4b678876c489b9273b13e9fff9f6d6d66add5e15d11d926cb"},
- {file = "multidict-6.1.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:5b48204e8d955c47c55b72779802b219a39acc3ee3d0116d5080c388970b76e3"},
- {file = "multidict-6.1.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:d8fff389528cad1618fb4b26b95550327495462cd745d879a8c7c2115248e399"},
- {file = "multidict-6.1.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a7a9541cd308eed5e30318430a9c74d2132e9a8cb46b901326272d780bf2d423"},
- {file = "multidict-6.1.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:da1758c76f50c39a2efd5e9859ce7d776317eb1dd34317c8152ac9251fc574a3"},
- {file = "multidict-6.1.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c943a53e9186688b45b323602298ab727d8865d8c9ee0b17f8d62d14b56f0753"},
- {file = "multidict-6.1.0-cp311-cp311-win32.whl", hash = "sha256:90f8717cb649eea3504091e640a1b8568faad18bd4b9fcd692853a04475a4b80"},
- {file = "multidict-6.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:82176036e65644a6cc5bd619f65f6f19781e8ec2e5330f51aa9ada7504cc1926"},
- {file = "multidict-6.1.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:b04772ed465fa3cc947db808fa306d79b43e896beb677a56fb2347ca1a49c1fa"},
- {file = "multidict-6.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:6180c0ae073bddeb5a97a38c03f30c233e0a4d39cd86166251617d1bbd0af436"},
- {file = "multidict-6.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:071120490b47aa997cca00666923a83f02c7fbb44f71cf7f136df753f7fa8761"},
- {file = "multidict-6.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50b3a2710631848991d0bf7de077502e8994c804bb805aeb2925a981de58ec2e"},
- {file = "multidict-6.1.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b58c621844d55e71c1b7f7c498ce5aa6985d743a1a59034c57a905b3f153c1ef"},
- {file = "multidict-6.1.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:55b6d90641869892caa9ca42ff913f7ff1c5ece06474fbd32fb2cf6834726c95"},
- {file = "multidict-6.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b820514bfc0b98a30e3d85462084779900347e4d49267f747ff54060cc33925"},
- {file = "multidict-6.1.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:10a9b09aba0c5b48c53761b7c720aaaf7cf236d5fe394cd399c7ba662d5f9966"},
- {file = "multidict-6.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1e16bf3e5fc9f44632affb159d30a437bfe286ce9e02754759be5536b169b305"},
- {file = "multidict-6.1.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:76f364861c3bfc98cbbcbd402d83454ed9e01a5224bb3a28bf70002a230f73e2"},
- {file = "multidict-6.1.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:820c661588bd01a0aa62a1283f20d2be4281b086f80dad9e955e690c75fb54a2"},
- {file = "multidict-6.1.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:0e5f362e895bc5b9e67fe6e4ded2492d8124bdf817827f33c5b46c2fe3ffaca6"},
- {file = "multidict-6.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3ec660d19bbc671e3a6443325f07263be452c453ac9e512f5eb935e7d4ac28b3"},
- {file = "multidict-6.1.0-cp312-cp312-win32.whl", hash = "sha256:58130ecf8f7b8112cdb841486404f1282b9c86ccb30d3519faf301b2e5659133"},
- {file = "multidict-6.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:188215fc0aafb8e03341995e7c4797860181562380f81ed0a87ff455b70bf1f1"},
- {file = "multidict-6.1.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:d569388c381b24671589335a3be6e1d45546c2988c2ebe30fdcada8457a31008"},
- {file = "multidict-6.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:052e10d2d37810b99cc170b785945421141bf7bb7d2f8799d431e7db229c385f"},
- {file = "multidict-6.1.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f90c822a402cb865e396a504f9fc8173ef34212a342d92e362ca498cad308e28"},
- {file = "multidict-6.1.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b225d95519a5bf73860323e633a664b0d85ad3d5bede6d30d95b35d4dfe8805b"},
- {file = "multidict-6.1.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:23bfd518810af7de1116313ebd9092cb9aa629beb12f6ed631ad53356ed6b86c"},
- {file = "multidict-6.1.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5c09fcfdccdd0b57867577b719c69e347a436b86cd83747f179dbf0cc0d4c1f3"},
- {file = "multidict-6.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf6bea52ec97e95560af5ae576bdac3aa3aae0b6758c6efa115236d9e07dae44"},
- {file = "multidict-6.1.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:57feec87371dbb3520da6192213c7d6fc892d5589a93db548331954de8248fd2"},
- {file = "multidict-6.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:0c3f390dc53279cbc8ba976e5f8035eab997829066756d811616b652b00a23a3"},
- {file = "multidict-6.1.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:59bfeae4b25ec05b34f1956eaa1cb38032282cd4dfabc5056d0a1ec4d696d3aa"},
- {file = "multidict-6.1.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b2f59caeaf7632cc633b5cf6fc449372b83bbdf0da4ae04d5be36118e46cc0aa"},
- {file = "multidict-6.1.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:37bb93b2178e02b7b618893990941900fd25b6b9ac0fa49931a40aecdf083fe4"},
- {file = "multidict-6.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4e9f48f58c2c523d5a06faea47866cd35b32655c46b443f163d08c6d0ddb17d6"},
- {file = "multidict-6.1.0-cp313-cp313-win32.whl", hash = "sha256:3a37ffb35399029b45c6cc33640a92bef403c9fd388acce75cdc88f58bd19a81"},
- {file = "multidict-6.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:e9aa71e15d9d9beaad2c6b9319edcdc0a49a43ef5c0a4c8265ca9ee7d6c67774"},
- {file = "multidict-6.1.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:db7457bac39421addd0c8449933ac32d8042aae84a14911a757ae6ca3eef1392"},
- {file = "multidict-6.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d094ddec350a2fb899fec68d8353c78233debde9b7d8b4beeafa70825f1c281a"},
- {file = "multidict-6.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:5845c1fd4866bb5dd3125d89b90e57ed3138241540897de748cdf19de8a2fca2"},
- {file = "multidict-6.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9079dfc6a70abe341f521f78405b8949f96db48da98aeb43f9907f342f627cdc"},
- {file = "multidict-6.1.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3914f5aaa0f36d5d60e8ece6a308ee1c9784cd75ec8151062614657a114c4478"},
- {file = "multidict-6.1.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c08be4f460903e5a9d0f76818db3250f12e9c344e79314d1d570fc69d7f4eae4"},
- {file = "multidict-6.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d093be959277cb7dee84b801eb1af388b6ad3ca6a6b6bf1ed7585895789d027d"},
- {file = "multidict-6.1.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3702ea6872c5a2a4eeefa6ffd36b042e9773f05b1f37ae3ef7264b1163c2dcf6"},
- {file = "multidict-6.1.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:2090f6a85cafc5b2db085124d752757c9d251548cedabe9bd31afe6363e0aff2"},
- {file = "multidict-6.1.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:f67f217af4b1ff66c68a87318012de788dd95fcfeb24cc889011f4e1c7454dfd"},
- {file = "multidict-6.1.0-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:189f652a87e876098bbc67b4da1049afb5f5dfbaa310dd67c594b01c10388db6"},
- {file = "multidict-6.1.0-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:6bb5992037f7a9eff7991ebe4273ea7f51f1c1c511e6a2ce511d0e7bdb754492"},
- {file = "multidict-6.1.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:ac10f4c2b9e770c4e393876e35a7046879d195cd123b4f116d299d442b335bcd"},
- {file = "multidict-6.1.0-cp38-cp38-win32.whl", hash = "sha256:e27bbb6d14416713a8bd7aaa1313c0fc8d44ee48d74497a0ff4c3a1b6ccb5167"},
- {file = "multidict-6.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:22f3105d4fb15c8f57ff3959a58fcab6ce36814486500cd7485651230ad4d4ef"},
- {file = "multidict-6.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:4e18b656c5e844539d506a0a06432274d7bd52a7487e6828c63a63d69185626c"},
- {file = "multidict-6.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a185f876e69897a6f3325c3f19f26a297fa058c5e456bfcff8015e9a27e83ae1"},
- {file = "multidict-6.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ab7c4ceb38d91570a650dba194e1ca87c2b543488fe9309b4212694174fd539c"},
- {file = "multidict-6.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e617fb6b0b6953fffd762669610c1c4ffd05632c138d61ac7e14ad187870669c"},
- {file = "multidict-6.1.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:16e5f4bf4e603eb1fdd5d8180f1a25f30056f22e55ce51fb3d6ad4ab29f7d96f"},
- {file = "multidict-6.1.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f4c035da3f544b1882bac24115f3e2e8760f10a0107614fc9839fd232200b875"},
- {file = "multidict-6.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:957cf8e4b6e123a9eea554fa7ebc85674674b713551de587eb318a2df3e00255"},
- {file = "multidict-6.1.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:483a6aea59cb89904e1ceabd2b47368b5600fb7de78a6e4a2c2987b2d256cf30"},
- {file = "multidict-6.1.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:87701f25a2352e5bf7454caa64757642734da9f6b11384c1f9d1a8e699758057"},
- {file = "multidict-6.1.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:682b987361e5fd7a139ed565e30d81fd81e9629acc7d925a205366877d8c8657"},
- {file = "multidict-6.1.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:ce2186a7df133a9c895dea3331ddc5ddad42cdd0d1ea2f0a51e5d161e4762f28"},
- {file = "multidict-6.1.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:9f636b730f7e8cb19feb87094949ba54ee5357440b9658b2a32a5ce4bce53972"},
- {file = "multidict-6.1.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:73eae06aa53af2ea5270cc066dcaf02cc60d2994bbb2c4ef5764949257d10f43"},
- {file = "multidict-6.1.0-cp39-cp39-win32.whl", hash = "sha256:1ca0083e80e791cffc6efce7660ad24af66c8d4079d2a750b29001b53ff59ada"},
- {file = "multidict-6.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:aa466da5b15ccea564bdab9c89175c762bc12825f4659c11227f515cee76fa4a"},
- {file = "multidict-6.1.0-py3-none-any.whl", hash = "sha256:48e171e52d1c4d33888e529b999e5900356b9ae588c2f09a52dcefb158b27506"},
- {file = "multidict-6.1.0.tar.gz", hash = "sha256:22ae2ebf9b0c69d206c003e2f6a914ea33f0a932d4aa16f236afc049d9958f4a"},
-]
-
-[package.dependencies]
-typing-extensions = {version = ">=4.1.0", markers = "python_version < \"3.11\""}
-
-[[package]]
-name = "multiprocess"
-version = "0.70.16"
-description = "better multiprocessing and multithreading in Python"
-optional = false
-python-versions = ">=3.8"
-files = [
- {file = "multiprocess-0.70.16-pp310-pypy310_pp73-macosx_10_13_x86_64.whl", hash = "sha256:476887be10e2f59ff183c006af746cb6f1fd0eadcfd4ef49e605cbe2659920ee"},
- {file = "multiprocess-0.70.16-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:d951bed82c8f73929ac82c61f01a7b5ce8f3e5ef40f5b52553b4f547ce2b08ec"},
- {file = "multiprocess-0.70.16-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:37b55f71c07e2d741374998c043b9520b626a8dddc8b3129222ca4f1a06ef67a"},
- {file = "multiprocess-0.70.16-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:ba8c31889abf4511c7308a8c52bb4a30b9d590e7f58523302ba00237702ca054"},
- {file = "multiprocess-0.70.16-pp39-pypy39_pp73-macosx_10_13_x86_64.whl", hash = "sha256:0dfd078c306e08d46d7a8d06fb120313d87aa43af60d66da43ffff40b44d2f41"},
- {file = "multiprocess-0.70.16-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e7b9d0f307cd9bd50851afaac0dba2cb6c44449efff697df7c7645f7d3f2be3a"},
- {file = "multiprocess-0.70.16-py310-none-any.whl", hash = "sha256:c4a9944c67bd49f823687463660a2d6daae94c289adff97e0f9d696ba6371d02"},
- {file = "multiprocess-0.70.16-py311-none-any.whl", hash = "sha256:af4cabb0dac72abfb1e794fa7855c325fd2b55a10a44628a3c1ad3311c04127a"},
- {file = "multiprocess-0.70.16-py312-none-any.whl", hash = "sha256:fc0544c531920dde3b00c29863377f87e1632601092ea2daca74e4beb40faa2e"},
- {file = "multiprocess-0.70.16-py38-none-any.whl", hash = "sha256:a71d82033454891091a226dfc319d0cfa8019a4e888ef9ca910372a446de4435"},
- {file = "multiprocess-0.70.16-py39-none-any.whl", hash = "sha256:a0bafd3ae1b732eac64be2e72038231c1ba97724b60b09400d68f229fcc2fbf3"},
- {file = "multiprocess-0.70.16.tar.gz", hash = "sha256:161af703d4652a0e1410be6abccecde4a7ddffd19341be0a7011b94aeb171ac1"},
-]
-
-[package.dependencies]
-dill = ">=0.3.8"
-
-[[package]]
-name = "mypy-extensions"
-version = "1.0.0"
-description = "Type system extensions for programs checked with the mypy type checker."
-optional = false
-python-versions = ">=3.5"
-files = [
- {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"},
- {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"},
-]
-
-[[package]]
-name = "nest-asyncio"
-version = "1.6.0"
-description = "Patch asyncio to allow nested event loops"
-optional = false
-python-versions = ">=3.5"
-files = [
- {file = "nest_asyncio-1.6.0-py3-none-any.whl", hash = "sha256:87af6efd6b5e897c81050477ef65c62e2b2f35d51703cae01aff2905b1852e1c"},
- {file = "nest_asyncio-1.6.0.tar.gz", hash = "sha256:6f172d5449aca15afd6c646851f4e31e02c598d553a667e38cafa997cfec55fe"},
-]
-
-[[package]]
-name = "networkx"
-version = "3.3"
-description = "Python package for creating and manipulating graphs and networks"
-optional = false
-python-versions = ">=3.10"
-files = [
- {file = "networkx-3.3-py3-none-any.whl", hash = "sha256:28575580c6ebdaf4505b22c6256a2b9de86b316dc63ba9e93abde3d78dfdbcf2"},
- {file = "networkx-3.3.tar.gz", hash = "sha256:0c127d8b2f4865f59ae9cb8aafcd60b5c70f3241ebd66f7defad7c4ab90126c9"},
-]
-
-[package.extras]
-default = ["matplotlib (>=3.6)", "numpy (>=1.23)", "pandas (>=1.4)", "scipy (>=1.9,!=1.11.0,!=1.11.1)"]
-developer = ["changelist (==0.5)", "mypy (>=1.1)", "pre-commit (>=3.2)", "rtoml"]
-doc = ["myst-nb (>=1.0)", "numpydoc (>=1.7)", "pillow (>=9.4)", "pydata-sphinx-theme (>=0.14)", "sphinx (>=7)", "sphinx-gallery (>=0.14)", "texext (>=0.6.7)"]
-extra = ["lxml (>=4.6)", "pydot (>=2.0)", "pygraphviz (>=1.12)", "sympy (>=1.10)"]
-test = ["pytest (>=7.2)", "pytest-cov (>=4.0)"]
-
-[[package]]
-name = "nltk"
-version = "3.9.1"
-description = "Natural Language Toolkit"
-optional = false
-python-versions = ">=3.8"
-files = [
- {file = "nltk-3.9.1-py3-none-any.whl", hash = "sha256:4fa26829c5b00715afe3061398a8989dc643b92ce7dd93fb4585a70930d168a1"},
- {file = "nltk-3.9.1.tar.gz", hash = "sha256:87d127bd3de4bd89a4f81265e5fa59cb1b199b27440175370f7417d2bc7ae868"},
-]
-
-[package.dependencies]
-click = "*"
-joblib = "*"
-regex = ">=2021.8.3"
-tqdm = "*"
-
-[package.extras]
-all = ["matplotlib", "numpy", "pyparsing", "python-crfsuite", "requests", "scikit-learn", "scipy", "twython"]
-corenlp = ["requests"]
-machine-learning = ["numpy", "python-crfsuite", "scikit-learn", "scipy"]
-plot = ["matplotlib"]
-tgrep = ["pyparsing"]
-twitter = ["twython"]
-
-[[package]]
-name = "numpy"
-version = "1.26.4"
-description = "Fundamental package for array computing in Python"
-optional = false
-python-versions = ">=3.9"
-files = [
- {file = "numpy-1.26.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0"},
- {file = "numpy-1.26.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a"},
- {file = "numpy-1.26.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d209d8969599b27ad20994c8e41936ee0964e6da07478d6c35016bc386b66ad4"},
- {file = "numpy-1.26.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f"},
- {file = "numpy-1.26.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:62b8e4b1e28009ef2846b4c7852046736bab361f7aeadeb6a5b89ebec3c7055a"},
- {file = "numpy-1.26.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a4abb4f9001ad2858e7ac189089c42178fcce737e4169dc61321660f1a96c7d2"},
- {file = "numpy-1.26.4-cp310-cp310-win32.whl", hash = "sha256:bfe25acf8b437eb2a8b2d49d443800a5f18508cd811fea3181723922a8a82b07"},
- {file = "numpy-1.26.4-cp310-cp310-win_amd64.whl", hash = "sha256:b97fe8060236edf3662adfc2c633f56a08ae30560c56310562cb4f95500022d5"},
- {file = "numpy-1.26.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71"},
- {file = "numpy-1.26.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef"},
- {file = "numpy-1.26.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e"},
- {file = "numpy-1.26.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5"},
- {file = "numpy-1.26.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a"},
- {file = "numpy-1.26.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a"},
- {file = "numpy-1.26.4-cp311-cp311-win32.whl", hash = "sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20"},
- {file = "numpy-1.26.4-cp311-cp311-win_amd64.whl", hash = "sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2"},
- {file = "numpy-1.26.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218"},
- {file = "numpy-1.26.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b"},
- {file = "numpy-1.26.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b"},
- {file = "numpy-1.26.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed"},
- {file = "numpy-1.26.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a"},
- {file = "numpy-1.26.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0"},
- {file = "numpy-1.26.4-cp312-cp312-win32.whl", hash = "sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110"},
- {file = "numpy-1.26.4-cp312-cp312-win_amd64.whl", hash = "sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818"},
- {file = "numpy-1.26.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7349ab0fa0c429c82442a27a9673fc802ffdb7c7775fad780226cb234965e53c"},
- {file = "numpy-1.26.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:52b8b60467cd7dd1e9ed082188b4e6bb35aa5cdd01777621a1658910745b90be"},
- {file = "numpy-1.26.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d5241e0a80d808d70546c697135da2c613f30e28251ff8307eb72ba696945764"},
- {file = "numpy-1.26.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3"},
- {file = "numpy-1.26.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:679b0076f67ecc0138fd2ede3a8fd196dddc2ad3254069bcb9faf9a79b1cebcd"},
- {file = "numpy-1.26.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:47711010ad8555514b434df65f7d7b076bb8261df1ca9bb78f53d3b2db02e95c"},
- {file = "numpy-1.26.4-cp39-cp39-win32.whl", hash = "sha256:a354325ee03388678242a4d7ebcd08b5c727033fcff3b2f536aea978e15ee9e6"},
- {file = "numpy-1.26.4-cp39-cp39-win_amd64.whl", hash = "sha256:3373d5d70a5fe74a2c1bb6d2cfd9609ecf686d47a2d7b1d37a8f3b6bf6003aea"},
- {file = "numpy-1.26.4-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:afedb719a9dcfc7eaf2287b839d8198e06dcd4cb5d276a3df279231138e83d30"},
- {file = "numpy-1.26.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95a7476c59002f2f6c590b9b7b998306fba6a5aa646b1e22ddfeaf8f78c3a29c"},
- {file = "numpy-1.26.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7e50d0a0cc3189f9cb0aeb3a6a6af18c16f59f004b866cd2be1c14b36134a4a0"},
- {file = "numpy-1.26.4.tar.gz", hash = "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010"},
-]
-
-[[package]]
-name = "openai"
-version = "1.47.0"
-description = "The official Python library for the openai API"
-optional = false
-python-versions = ">=3.7.1"
-files = [
- {file = "openai-1.47.0-py3-none-any.whl", hash = "sha256:9ccc8737dfa791f7bd903db4758c176b8544a8cd89d3a3d2add3cea02a34c3a0"},
- {file = "openai-1.47.0.tar.gz", hash = "sha256:6e14d6f77c8cf546646afcd87a2ef752505b3710d2564a2e433e17307dfa86a0"},
-]
-
-[package.dependencies]
-anyio = ">=3.5.0,<5"
-distro = ">=1.7.0,<2"
-httpx = ">=0.23.0,<1"
-jiter = ">=0.4.0,<1"
-pydantic = ">=1.9.0,<3"
-sniffio = "*"
-tqdm = ">4"
-typing-extensions = ">=4.11,<5"
-
-[package.extras]
-datalib = ["numpy (>=1)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1.0.11)"]
-
-[[package]]
-name = "orjson"
-version = "3.10.7"
-description = "Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy"
-optional = false
-python-versions = ">=3.8"
-files = [
- {file = "orjson-3.10.7-cp310-cp310-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:74f4544f5a6405b90da8ea724d15ac9c36da4d72a738c64685003337401f5c12"},
- {file = "orjson-3.10.7-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:34a566f22c28222b08875b18b0dfbf8a947e69df21a9ed5c51a6bf91cfb944ac"},
- {file = "orjson-3.10.7-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bf6ba8ebc8ef5792e2337fb0419f8009729335bb400ece005606336b7fd7bab7"},
- {file = "orjson-3.10.7-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ac7cf6222b29fbda9e3a472b41e6a5538b48f2c8f99261eecd60aafbdb60690c"},
- {file = "orjson-3.10.7-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:de817e2f5fc75a9e7dd350c4b0f54617b280e26d1631811a43e7e968fa71e3e9"},
- {file = "orjson-3.10.7-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:348bdd16b32556cf8d7257b17cf2bdb7ab7976af4af41ebe79f9796c218f7e91"},
- {file = "orjson-3.10.7-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:479fd0844ddc3ca77e0fd99644c7fe2de8e8be1efcd57705b5c92e5186e8a250"},
- {file = "orjson-3.10.7-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:fdf5197a21dd660cf19dfd2a3ce79574588f8f5e2dbf21bda9ee2d2b46924d84"},
- {file = "orjson-3.10.7-cp310-none-win32.whl", hash = "sha256:d374d36726746c81a49f3ff8daa2898dccab6596864ebe43d50733275c629175"},
- {file = "orjson-3.10.7-cp310-none-win_amd64.whl", hash = "sha256:cb61938aec8b0ffb6eef484d480188a1777e67b05d58e41b435c74b9d84e0b9c"},
- {file = "orjson-3.10.7-cp311-cp311-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:7db8539039698ddfb9a524b4dd19508256107568cdad24f3682d5773e60504a2"},
- {file = "orjson-3.10.7-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:480f455222cb7a1dea35c57a67578848537d2602b46c464472c995297117fa09"},
- {file = "orjson-3.10.7-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8a9c9b168b3a19e37fe2778c0003359f07822c90fdff8f98d9d2a91b3144d8e0"},
- {file = "orjson-3.10.7-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8de062de550f63185e4c1c54151bdddfc5625e37daf0aa1e75d2a1293e3b7d9a"},
- {file = "orjson-3.10.7-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6b0dd04483499d1de9c8f6203f8975caf17a6000b9c0c54630cef02e44ee624e"},
- {file = "orjson-3.10.7-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b58d3795dafa334fc8fd46f7c5dc013e6ad06fd5b9a4cc98cb1456e7d3558bd6"},
- {file = "orjson-3.10.7-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:33cfb96c24034a878d83d1a9415799a73dc77480e6c40417e5dda0710d559ee6"},
- {file = "orjson-3.10.7-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e724cebe1fadc2b23c6f7415bad5ee6239e00a69f30ee423f319c6af70e2a5c0"},
- {file = "orjson-3.10.7-cp311-none-win32.whl", hash = "sha256:82763b46053727a7168d29c772ed5c870fdae2f61aa8a25994c7984a19b1021f"},
- {file = "orjson-3.10.7-cp311-none-win_amd64.whl", hash = "sha256:eb8d384a24778abf29afb8e41d68fdd9a156cf6e5390c04cc07bbc24b89e98b5"},
- {file = "orjson-3.10.7-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:44a96f2d4c3af51bfac6bc4ef7b182aa33f2f054fd7f34cc0ee9a320d051d41f"},
- {file = "orjson-3.10.7-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76ac14cd57df0572453543f8f2575e2d01ae9e790c21f57627803f5e79b0d3c3"},
- {file = "orjson-3.10.7-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bdbb61dcc365dd9be94e8f7df91975edc9364d6a78c8f7adb69c1cdff318ec93"},
- {file = "orjson-3.10.7-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b48b3db6bb6e0a08fa8c83b47bc169623f801e5cc4f24442ab2b6617da3b5313"},
- {file = "orjson-3.10.7-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:23820a1563a1d386414fef15c249040042b8e5d07b40ab3fe3efbfbbcbcb8864"},
- {file = "orjson-3.10.7-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0c6a008e91d10a2564edbb6ee5069a9e66df3fbe11c9a005cb411f441fd2c09"},
- {file = "orjson-3.10.7-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d352ee8ac1926d6193f602cbe36b1643bbd1bbcb25e3c1a657a4390f3000c9a5"},
- {file = "orjson-3.10.7-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d2d9f990623f15c0ae7ac608103c33dfe1486d2ed974ac3f40b693bad1a22a7b"},
- {file = "orjson-3.10.7-cp312-none-win32.whl", hash = "sha256:7c4c17f8157bd520cdb7195f75ddbd31671997cbe10aee559c2d613592e7d7eb"},
- {file = "orjson-3.10.7-cp312-none-win_amd64.whl", hash = "sha256:1d9c0e733e02ada3ed6098a10a8ee0052dd55774de3d9110d29868d24b17faa1"},
- {file = "orjson-3.10.7-cp313-cp313-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:77d325ed866876c0fa6492598ec01fe30e803272a6e8b10e992288b009cbe149"},
- {file = "orjson-3.10.7-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ea2c232deedcb605e853ae1db2cc94f7390ac776743b699b50b071b02bea6fe"},
- {file = "orjson-3.10.7-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3dcfbede6737fdbef3ce9c37af3fb6142e8e1ebc10336daa05872bfb1d87839c"},
- {file = "orjson-3.10.7-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:11748c135f281203f4ee695b7f80bb1358a82a63905f9f0b794769483ea854ad"},
- {file = "orjson-3.10.7-cp313-none-win32.whl", hash = "sha256:a7e19150d215c7a13f39eb787d84db274298d3f83d85463e61d277bbd7f401d2"},
- {file = "orjson-3.10.7-cp313-none-win_amd64.whl", hash = "sha256:eef44224729e9525d5261cc8d28d6b11cafc90e6bd0be2157bde69a52ec83024"},
- {file = "orjson-3.10.7-cp38-cp38-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:6ea2b2258eff652c82652d5e0f02bd5e0463a6a52abb78e49ac288827aaa1469"},
- {file = "orjson-3.10.7-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:430ee4d85841e1483d487e7b81401785a5dfd69db5de01314538f31f8fbf7ee1"},
- {file = "orjson-3.10.7-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4b6146e439af4c2472c56f8540d799a67a81226e11992008cb47e1267a9b3225"},
- {file = "orjson-3.10.7-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:084e537806b458911137f76097e53ce7bf5806dda33ddf6aaa66a028f8d43a23"},
- {file = "orjson-3.10.7-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4829cf2195838e3f93b70fd3b4292156fc5e097aac3739859ac0dcc722b27ac0"},
- {file = "orjson-3.10.7-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1193b2416cbad1a769f868b1749535d5da47626ac29445803dae7cc64b3f5c98"},
- {file = "orjson-3.10.7-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:4e6c3da13e5a57e4b3dca2de059f243ebec705857522f188f0180ae88badd354"},
- {file = "orjson-3.10.7-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:c31008598424dfbe52ce8c5b47e0752dca918a4fdc4a2a32004efd9fab41d866"},
- {file = "orjson-3.10.7-cp38-none-win32.whl", hash = "sha256:7122a99831f9e7fe977dc45784d3b2edc821c172d545e6420c375e5a935f5a1c"},
- {file = "orjson-3.10.7-cp38-none-win_amd64.whl", hash = "sha256:a763bc0e58504cc803739e7df040685816145a6f3c8a589787084b54ebc9f16e"},
- {file = "orjson-3.10.7-cp39-cp39-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:e76be12658a6fa376fcd331b1ea4e58f5a06fd0220653450f0d415b8fd0fbe20"},
- {file = "orjson-3.10.7-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ed350d6978d28b92939bfeb1a0570c523f6170efc3f0a0ef1f1df287cd4f4960"},
- {file = "orjson-3.10.7-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:144888c76f8520e39bfa121b31fd637e18d4cc2f115727865fdf9fa325b10412"},
- {file = "orjson-3.10.7-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:09b2d92fd95ad2402188cf51573acde57eb269eddabaa60f69ea0d733e789fe9"},
- {file = "orjson-3.10.7-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5b24a579123fa884f3a3caadaed7b75eb5715ee2b17ab5c66ac97d29b18fe57f"},
- {file = "orjson-3.10.7-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e72591bcfe7512353bd609875ab38050efe3d55e18934e2f18950c108334b4ff"},
- {file = "orjson-3.10.7-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:f4db56635b58cd1a200b0a23744ff44206ee6aa428185e2b6c4a65b3197abdcd"},
- {file = "orjson-3.10.7-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:0fa5886854673222618638c6df7718ea7fe2f3f2384c452c9ccedc70b4a510a5"},
- {file = "orjson-3.10.7-cp39-none-win32.whl", hash = "sha256:8272527d08450ab16eb405f47e0f4ef0e5ff5981c3d82afe0efd25dcbef2bcd2"},
- {file = "orjson-3.10.7-cp39-none-win_amd64.whl", hash = "sha256:974683d4618c0c7dbf4f69c95a979734bf183d0658611760017f6e70a145af58"},
- {file = "orjson-3.10.7.tar.gz", hash = "sha256:75ef0640403f945f3a1f9f6400686560dbfb0fb5b16589ad62cd477043c4eee3"},
-]
-
-[[package]]
-name = "packaging"
-version = "23.2"
-description = "Core utilities for Python packages"
-optional = false
-python-versions = ">=3.7"
-files = [
- {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"},
- {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"},
-]
-
-[[package]]
-name = "pandas"
-version = "2.2.3"
-description = "Powerful data structures for data analysis, time series, and statistics"
-optional = false
-python-versions = ">=3.9"
-files = [
- {file = "pandas-2.2.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1948ddde24197a0f7add2bdc4ca83bf2b1ef84a1bc8ccffd95eda17fd836ecb5"},
- {file = "pandas-2.2.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:381175499d3802cde0eabbaf6324cce0c4f5d52ca6f8c377c29ad442f50f6348"},
- {file = "pandas-2.2.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:86976a1c5b25ae3f8ccae3a5306e443569ee3c3faf444dfd0f41cda24667ad57"},
- {file = "pandas-2.2.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:37e0aced3e8f539eccf2e099f65cdb9c8aa85109b0be6e93e2baff94264bdc6f"},
- {file = "pandas-2.2.3-cp310-cp310-win_amd64.whl", hash = "sha256:56534ce0746a58afaf7942ba4863e0ef81c9c50d3f0ae93e9497d6a41a057645"},
- {file = "pandas-2.2.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:66108071e1b935240e74525006034333f98bcdb87ea116de573a6a0dccb6c039"},
- {file = "pandas-2.2.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7c2875855b0ff77b2a64a0365e24455d9990730d6431b9e0ee18ad8acee13dbd"},
- {file = "pandas-2.2.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c124333816c3a9b03fbeef3a9f230ba9a737e9e5bb4060aa2107a86cc0a497fc"},
- {file = "pandas-2.2.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:29401dbfa9ad77319367d36940cd8a0b3a11aba16063e39632d98b0e931ddf32"},
- {file = "pandas-2.2.3-cp311-cp311-win_amd64.whl", hash = "sha256:3fc6873a41186404dad67245896a6e440baacc92f5b716ccd1bc9ed2995ab2c5"},
- {file = "pandas-2.2.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b1d432e8d08679a40e2a6d8b2f9770a5c21793a6f9f47fdd52c5ce1948a5a8a9"},
- {file = "pandas-2.2.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a5a1595fe639f5988ba6a8e5bc9649af3baf26df3998a0abe56c02609392e0a4"},
- {file = "pandas-2.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fffb8ae78d8af97f849404f21411c95062db1496aeb3e56f146f0355c9989319"},
- {file = "pandas-2.2.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:062309c1b9ea12a50e8ce661145c6aab431b1e99530d3cd60640e255778bd43a"},
- {file = "pandas-2.2.3-cp312-cp312-win_amd64.whl", hash = "sha256:59ef3764d0fe818125a5097d2ae867ca3fa64df032331b7e0917cf5d7bf66b13"},
- {file = "pandas-2.2.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f00d1345d84d8c86a63e476bb4955e46458b304b9575dcf71102b5c705320015"},
- {file = "pandas-2.2.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3508d914817e153ad359d7e069d752cdd736a247c322d932eb89e6bc84217f28"},
- {file = "pandas-2.2.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3a255b2c19987fbbe62a9dfd6cff7ff2aa9ccab3fc75218fd4b7530f01efa24"},
- {file = "pandas-2.2.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6374c452ff3ec675a8f46fd9ab25c4ad0ba590b71cf0656f8b6daa5202bca3fb"},
- {file = "pandas-2.2.3-cp313-cp313-win_amd64.whl", hash = "sha256:61c5ad4043f791b61dd4752191d9f07f0ae412515d59ba8f005832a532f8736d"},
- {file = "pandas-2.2.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3b71f27954685ee685317063bf13c7709a7ba74fc996b84fc6821c59b0f06468"},
- {file = "pandas-2.2.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:38cf8125c40dae9d5acc10fa66af8ea6fdf760b2714ee482ca691fc66e6fcb18"},
- {file = "pandas-2.2.3-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1db71525a1538b30142094edb9adc10be3f3e176748cd7acc2240c2f2e5aa3a4"},
- {file = "pandas-2.2.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:ad5b65698ab28ed8d7f18790a0dc58005c7629f227be9ecc1072aa74c0c1d43a"},
- {file = "pandas-2.2.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bc6b93f9b966093cb0fd62ff1a7e4c09e6d546ad7c1de191767baffc57628f39"},
- {file = "pandas-2.2.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5dbca4c1acd72e8eeef4753eeca07de9b1db4f398669d5994086f788a5d7cc30"},
- {file = "pandas-2.2.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:99df71520d25fade9db7c1076ac94eb994f4d2673ef2aa2e86ee039b6746d20c"},
- {file = "pandas-2.2.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:7eee9e7cea6adf3e3d24e304ac6b8300646e2a5d1cd3a3c2abed9101b0846761"},
- {file = "pandas-2.2.3-cp39-cp39-win_amd64.whl", hash = "sha256:4850ba03528b6dd51d6c5d273c46f183f39a9baf3f0143e566b89450965b105e"},
- {file = "pandas-2.2.3.tar.gz", hash = "sha256:4f18ba62b61d7e192368b84517265a99b4d7ee8912f8708660fb4a366cc82667"},
-]
-
-[package.dependencies]
-numpy = [
- {version = ">=1.22.4", markers = "python_version < \"3.11\""},
- {version = ">=1.23.2", markers = "python_version == \"3.11\""},
- {version = ">=1.26.0", markers = "python_version >= \"3.12\""},
-]
-python-dateutil = ">=2.8.2"
-pytz = ">=2020.1"
-tzdata = ">=2022.7"
-
-[package.extras]
-all = ["PyQt5 (>=5.15.9)", "SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)", "beautifulsoup4 (>=4.11.2)", "bottleneck (>=1.3.6)", "dataframe-api-compat (>=0.1.7)", "fastparquet (>=2022.12.0)", "fsspec (>=2022.11.0)", "gcsfs (>=2022.11.0)", "html5lib (>=1.1)", "hypothesis (>=6.46.1)", "jinja2 (>=3.1.2)", "lxml (>=4.9.2)", "matplotlib (>=3.6.3)", "numba (>=0.56.4)", "numexpr (>=2.8.4)", "odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "pandas-gbq (>=0.19.0)", "psycopg2 (>=2.9.6)", "pyarrow (>=10.0.1)", "pymysql (>=1.0.2)", "pyreadstat (>=1.2.0)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "qtpy (>=2.3.0)", "s3fs (>=2022.11.0)", "scipy (>=1.10.0)", "tables (>=3.8.0)", "tabulate (>=0.9.0)", "xarray (>=2022.12.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)", "zstandard (>=0.19.0)"]
-aws = ["s3fs (>=2022.11.0)"]
-clipboard = ["PyQt5 (>=5.15.9)", "qtpy (>=2.3.0)"]
-compression = ["zstandard (>=0.19.0)"]
-computation = ["scipy (>=1.10.0)", "xarray (>=2022.12.0)"]
-consortium-standard = ["dataframe-api-compat (>=0.1.7)"]
-excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)"]
-feather = ["pyarrow (>=10.0.1)"]
-fss = ["fsspec (>=2022.11.0)"]
-gcp = ["gcsfs (>=2022.11.0)", "pandas-gbq (>=0.19.0)"]
-hdf5 = ["tables (>=3.8.0)"]
-html = ["beautifulsoup4 (>=4.11.2)", "html5lib (>=1.1)", "lxml (>=4.9.2)"]
-mysql = ["SQLAlchemy (>=2.0.0)", "pymysql (>=1.0.2)"]
-output-formatting = ["jinja2 (>=3.1.2)", "tabulate (>=0.9.0)"]
-parquet = ["pyarrow (>=10.0.1)"]
-performance = ["bottleneck (>=1.3.6)", "numba (>=0.56.4)", "numexpr (>=2.8.4)"]
-plot = ["matplotlib (>=3.6.3)"]
-postgresql = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "psycopg2 (>=2.9.6)"]
-pyarrow = ["pyarrow (>=10.0.1)"]
-spss = ["pyreadstat (>=1.2.0)"]
-sql-other = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)"]
-test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)"]
-xml = ["lxml (>=4.9.2)"]
-
-[[package]]
-name = "pathspec"
-version = "0.12.1"
-description = "Utility library for gitignore style pattern matching of file paths."
-optional = false
-python-versions = ">=3.8"
-files = [
- {file = "pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08"},
- {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"},
-]
-
-[[package]]
-name = "pillow"
-version = "10.4.0"
-description = "Python Imaging Library (Fork)"
-optional = false
-python-versions = ">=3.8"
-files = [
- {file = "pillow-10.4.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:4d9667937cfa347525b319ae34375c37b9ee6b525440f3ef48542fcf66f2731e"},
- {file = "pillow-10.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:543f3dc61c18dafb755773efc89aae60d06b6596a63914107f75459cf984164d"},
- {file = "pillow-10.4.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7928ecbf1ece13956b95d9cbcfc77137652b02763ba384d9ab508099a2eca856"},
- {file = "pillow-10.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4d49b85c4348ea0b31ea63bc75a9f3857869174e2bf17e7aba02945cd218e6f"},
- {file = "pillow-10.4.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:6c762a5b0997f5659a5ef2266abc1d8851ad7749ad9a6a5506eb23d314e4f46b"},
- {file = "pillow-10.4.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:a985e028fc183bf12a77a8bbf36318db4238a3ded7fa9df1b9a133f1cb79f8fc"},
- {file = "pillow-10.4.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:812f7342b0eee081eaec84d91423d1b4650bb9828eb53d8511bcef8ce5aecf1e"},
- {file = "pillow-10.4.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ac1452d2fbe4978c2eec89fb5a23b8387aba707ac72810d9490118817d9c0b46"},
- {file = "pillow-10.4.0-cp310-cp310-win32.whl", hash = "sha256:bcd5e41a859bf2e84fdc42f4edb7d9aba0a13d29a2abadccafad99de3feff984"},
- {file = "pillow-10.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:ecd85a8d3e79cd7158dec1c9e5808e821feea088e2f69a974db5edf84dc53141"},
- {file = "pillow-10.4.0-cp310-cp310-win_arm64.whl", hash = "sha256:ff337c552345e95702c5fde3158acb0625111017d0e5f24bf3acdb9cc16b90d1"},
- {file = "pillow-10.4.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:0a9ec697746f268507404647e531e92889890a087e03681a3606d9b920fbee3c"},
- {file = "pillow-10.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:dfe91cb65544a1321e631e696759491ae04a2ea11d36715eca01ce07284738be"},
- {file = "pillow-10.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5dc6761a6efc781e6a1544206f22c80c3af4c8cf461206d46a1e6006e4429ff3"},
- {file = "pillow-10.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5e84b6cc6a4a3d76c153a6b19270b3526a5a8ed6b09501d3af891daa2a9de7d6"},
- {file = "pillow-10.4.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:bbc527b519bd3aa9d7f429d152fea69f9ad37c95f0b02aebddff592688998abe"},
- {file = "pillow-10.4.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:76a911dfe51a36041f2e756b00f96ed84677cdeb75d25c767f296c1c1eda1319"},
- {file = "pillow-10.4.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:59291fb29317122398786c2d44427bbd1a6d7ff54017075b22be9d21aa59bd8d"},
- {file = "pillow-10.4.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:416d3a5d0e8cfe4f27f574362435bc9bae57f679a7158e0096ad2beb427b8696"},
- {file = "pillow-10.4.0-cp311-cp311-win32.whl", hash = "sha256:7086cc1d5eebb91ad24ded9f58bec6c688e9f0ed7eb3dbbf1e4800280a896496"},
- {file = "pillow-10.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:cbed61494057c0f83b83eb3a310f0bf774b09513307c434d4366ed64f4128a91"},
- {file = "pillow-10.4.0-cp311-cp311-win_arm64.whl", hash = "sha256:f5f0c3e969c8f12dd2bb7e0b15d5c468b51e5017e01e2e867335c81903046a22"},
- {file = "pillow-10.4.0-cp312-cp312-macosx_10_10_x86_64.whl", hash = "sha256:673655af3eadf4df6b5457033f086e90299fdd7a47983a13827acf7459c15d94"},
- {file = "pillow-10.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:866b6942a92f56300012f5fbac71f2d610312ee65e22f1aa2609e491284e5597"},
- {file = "pillow-10.4.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29dbdc4207642ea6aad70fbde1a9338753d33fb23ed6956e706936706f52dd80"},
- {file = "pillow-10.4.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf2342ac639c4cf38799a44950bbc2dfcb685f052b9e262f446482afaf4bffca"},
- {file = "pillow-10.4.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:f5b92f4d70791b4a67157321c4e8225d60b119c5cc9aee8ecf153aace4aad4ef"},
- {file = "pillow-10.4.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:86dcb5a1eb778d8b25659d5e4341269e8590ad6b4e8b44d9f4b07f8d136c414a"},
- {file = "pillow-10.4.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:780c072c2e11c9b2c7ca37f9a2ee8ba66f44367ac3e5c7832afcfe5104fd6d1b"},
- {file = "pillow-10.4.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:37fb69d905be665f68f28a8bba3c6d3223c8efe1edf14cc4cfa06c241f8c81d9"},
- {file = "pillow-10.4.0-cp312-cp312-win32.whl", hash = "sha256:7dfecdbad5c301d7b5bde160150b4db4c659cee2b69589705b6f8a0c509d9f42"},
- {file = "pillow-10.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:1d846aea995ad352d4bdcc847535bd56e0fd88d36829d2c90be880ef1ee4668a"},
- {file = "pillow-10.4.0-cp312-cp312-win_arm64.whl", hash = "sha256:e553cad5179a66ba15bb18b353a19020e73a7921296a7979c4a2b7f6a5cd57f9"},
- {file = "pillow-10.4.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8bc1a764ed8c957a2e9cacf97c8b2b053b70307cf2996aafd70e91a082e70df3"},
- {file = "pillow-10.4.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:6209bb41dc692ddfee4942517c19ee81b86c864b626dbfca272ec0f7cff5d9fb"},
- {file = "pillow-10.4.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bee197b30783295d2eb680b311af15a20a8b24024a19c3a26431ff83eb8d1f70"},
- {file = "pillow-10.4.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ef61f5dd14c300786318482456481463b9d6b91ebe5ef12f405afbba77ed0be"},
- {file = "pillow-10.4.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:297e388da6e248c98bc4a02e018966af0c5f92dfacf5a5ca22fa01cb3179bca0"},
- {file = "pillow-10.4.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:e4db64794ccdf6cb83a59d73405f63adbe2a1887012e308828596100a0b2f6cc"},
- {file = "pillow-10.4.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:bd2880a07482090a3bcb01f4265f1936a903d70bc740bfcb1fd4e8a2ffe5cf5a"},
- {file = "pillow-10.4.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4b35b21b819ac1dbd1233317adeecd63495f6babf21b7b2512d244ff6c6ce309"},
- {file = "pillow-10.4.0-cp313-cp313-win32.whl", hash = "sha256:551d3fd6e9dc15e4c1eb6fc4ba2b39c0c7933fa113b220057a34f4bb3268a060"},
- {file = "pillow-10.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:030abdbe43ee02e0de642aee345efa443740aa4d828bfe8e2eb11922ea6a21ea"},
- {file = "pillow-10.4.0-cp313-cp313-win_arm64.whl", hash = "sha256:5b001114dd152cfd6b23befeb28d7aee43553e2402c9f159807bf55f33af8a8d"},
- {file = "pillow-10.4.0-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:8d4d5063501b6dd4024b8ac2f04962d661222d120381272deea52e3fc52d3736"},
- {file = "pillow-10.4.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7c1ee6f42250df403c5f103cbd2768a28fe1a0ea1f0f03fe151c8741e1469c8b"},
- {file = "pillow-10.4.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b15e02e9bb4c21e39876698abf233c8c579127986f8207200bc8a8f6bb27acf2"},
- {file = "pillow-10.4.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a8d4bade9952ea9a77d0c3e49cbd8b2890a399422258a77f357b9cc9be8d680"},
- {file = "pillow-10.4.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:43efea75eb06b95d1631cb784aa40156177bf9dd5b4b03ff38979e048258bc6b"},
- {file = "pillow-10.4.0-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:950be4d8ba92aca4b2bb0741285a46bfae3ca699ef913ec8416c1b78eadd64cd"},
- {file = "pillow-10.4.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:d7480af14364494365e89d6fddc510a13e5a2c3584cb19ef65415ca57252fb84"},
- {file = "pillow-10.4.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:73664fe514b34c8f02452ffb73b7a92c6774e39a647087f83d67f010eb9a0cf0"},
- {file = "pillow-10.4.0-cp38-cp38-win32.whl", hash = "sha256:e88d5e6ad0d026fba7bdab8c3f225a69f063f116462c49892b0149e21b6c0a0e"},
- {file = "pillow-10.4.0-cp38-cp38-win_amd64.whl", hash = "sha256:5161eef006d335e46895297f642341111945e2c1c899eb406882a6c61a4357ab"},
- {file = "pillow-10.4.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:0ae24a547e8b711ccaaf99c9ae3cd975470e1a30caa80a6aaee9a2f19c05701d"},
- {file = "pillow-10.4.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:298478fe4f77a4408895605f3482b6cc6222c018b2ce565c2b6b9c354ac3229b"},
- {file = "pillow-10.4.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:134ace6dc392116566980ee7436477d844520a26a4b1bd4053f6f47d096997fd"},
- {file = "pillow-10.4.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:930044bb7679ab003b14023138b50181899da3f25de50e9dbee23b61b4de2126"},
- {file = "pillow-10.4.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:c76e5786951e72ed3686e122d14c5d7012f16c8303a674d18cdcd6d89557fc5b"},
- {file = "pillow-10.4.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:b2724fdb354a868ddf9a880cb84d102da914e99119211ef7ecbdc613b8c96b3c"},
- {file = "pillow-10.4.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:dbc6ae66518ab3c5847659e9988c3b60dc94ffb48ef9168656e0019a93dbf8a1"},
- {file = "pillow-10.4.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:06b2f7898047ae93fad74467ec3d28fe84f7831370e3c258afa533f81ef7f3df"},
- {file = "pillow-10.4.0-cp39-cp39-win32.whl", hash = "sha256:7970285ab628a3779aecc35823296a7869f889b8329c16ad5a71e4901a3dc4ef"},
- {file = "pillow-10.4.0-cp39-cp39-win_amd64.whl", hash = "sha256:961a7293b2457b405967af9c77dcaa43cc1a8cd50d23c532e62d48ab6cdd56f5"},
- {file = "pillow-10.4.0-cp39-cp39-win_arm64.whl", hash = "sha256:32cda9e3d601a52baccb2856b8ea1fc213c90b340c542dcef77140dfa3278a9e"},
- {file = "pillow-10.4.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:5b4815f2e65b30f5fbae9dfffa8636d992d49705723fe86a3661806e069352d4"},
- {file = "pillow-10.4.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:8f0aef4ef59694b12cadee839e2ba6afeab89c0f39a3adc02ed51d109117b8da"},
- {file = "pillow-10.4.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9f4727572e2918acaa9077c919cbbeb73bd2b3ebcfe033b72f858fc9fbef0026"},
- {file = "pillow-10.4.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ff25afb18123cea58a591ea0244b92eb1e61a1fd497bf6d6384f09bc3262ec3e"},
- {file = "pillow-10.4.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:dc3e2db6ba09ffd7d02ae9141cfa0ae23393ee7687248d46a7507b75d610f4f5"},
- {file = "pillow-10.4.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:02a2be69f9c9b8c1e97cf2713e789d4e398c751ecfd9967c18d0ce304efbf885"},
- {file = "pillow-10.4.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:0755ffd4a0c6f267cccbae2e9903d95477ca2f77c4fcf3a3a09570001856c8a5"},
- {file = "pillow-10.4.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:a02364621fe369e06200d4a16558e056fe2805d3468350df3aef21e00d26214b"},
- {file = "pillow-10.4.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:1b5dea9831a90e9d0721ec417a80d4cbd7022093ac38a568db2dd78363b00908"},
- {file = "pillow-10.4.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9b885f89040bb8c4a1573566bbb2f44f5c505ef6e74cec7ab9068c900047f04b"},
- {file = "pillow-10.4.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87dd88ded2e6d74d31e1e0a99a726a6765cda32d00ba72dc37f0651f306daaa8"},
- {file = "pillow-10.4.0-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:2db98790afc70118bd0255c2eeb465e9767ecf1f3c25f9a1abb8ffc8cfd1fe0a"},
- {file = "pillow-10.4.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:f7baece4ce06bade126fb84b8af1c33439a76d8a6fd818970215e0560ca28c27"},
- {file = "pillow-10.4.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:cfdd747216947628af7b259d274771d84db2268ca062dd5faf373639d00113a3"},
- {file = "pillow-10.4.0.tar.gz", hash = "sha256:166c1cd4d24309b30d61f79f4a9114b7b2313d7450912277855ff5dfd7cd4a06"},
-]
-
-[package.extras]
-docs = ["furo", "olefile", "sphinx (>=7.3)", "sphinx-copybutton", "sphinx-inline-tabs", "sphinxext-opengraph"]
-fpx = ["olefile"]
-mic = ["olefile"]
-tests = ["check-manifest", "coverage", "defusedxml", "markdown2", "olefile", "packaging", "pyroma", "pytest", "pytest-cov", "pytest-timeout"]
-typing = ["typing-extensions"]
-xmp = ["defusedxml"]
-
-[[package]]
-name = "pluggy"
-version = "1.5.0"
-description = "plugin and hook calling mechanisms for python"
-optional = false
-python-versions = ">=3.8"
-files = [
- {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"},
- {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"},
-]
-
-[package.extras]
-dev = ["pre-commit", "tox"]
-testing = ["pytest", "pytest-benchmark"]
-
-[[package]]
-name = "prompt-toolkit"
-version = "3.0.47"
-description = "Library for building powerful interactive command lines in Python"
-optional = false
-python-versions = ">=3.7.0"
-files = [
- {file = "prompt_toolkit-3.0.47-py3-none-any.whl", hash = "sha256:0d7bfa67001d5e39d02c224b663abc33687405033a8c422d0d675a5a13361d10"},
- {file = "prompt_toolkit-3.0.47.tar.gz", hash = "sha256:1e1b29cb58080b1e69f207c893a1a7bf16d127a5c30c9d17a25a5d77792e5360"},
-]
-
-[package.dependencies]
-wcwidth = "*"
-
-[[package]]
-name = "pyarrow"
-version = "17.0.0"
-description = "Python library for Apache Arrow"
-optional = false
-python-versions = ">=3.8"
-files = [
- {file = "pyarrow-17.0.0-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:a5c8b238d47e48812ee577ee20c9a2779e6a5904f1708ae240f53ecbee7c9f07"},
- {file = "pyarrow-17.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:db023dc4c6cae1015de9e198d41250688383c3f9af8f565370ab2b4cb5f62655"},
- {file = "pyarrow-17.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da1e060b3876faa11cee287839f9cc7cdc00649f475714b8680a05fd9071d545"},
- {file = "pyarrow-17.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75c06d4624c0ad6674364bb46ef38c3132768139ddec1c56582dbac54f2663e2"},
- {file = "pyarrow-17.0.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:fa3c246cc58cb5a4a5cb407a18f193354ea47dd0648194e6265bd24177982fe8"},
- {file = "pyarrow-17.0.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:f7ae2de664e0b158d1607699a16a488de3d008ba99b3a7aa5de1cbc13574d047"},
- {file = "pyarrow-17.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:5984f416552eea15fd9cee03da53542bf4cddaef5afecefb9aa8d1010c335087"},
- {file = "pyarrow-17.0.0-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:1c8856e2ef09eb87ecf937104aacfa0708f22dfeb039c363ec99735190ffb977"},
- {file = "pyarrow-17.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2e19f569567efcbbd42084e87f948778eb371d308e137a0f97afe19bb860ccb3"},
- {file = "pyarrow-17.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b244dc8e08a23b3e352899a006a26ae7b4d0da7bb636872fa8f5884e70acf15"},
- {file = "pyarrow-17.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0b72e87fe3e1db343995562f7fff8aee354b55ee83d13afba65400c178ab2597"},
- {file = "pyarrow-17.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:dc5c31c37409dfbc5d014047817cb4ccd8c1ea25d19576acf1a001fe07f5b420"},
- {file = "pyarrow-17.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:e3343cb1e88bc2ea605986d4b94948716edc7a8d14afd4e2c097232f729758b4"},
- {file = "pyarrow-17.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:a27532c38f3de9eb3e90ecab63dfda948a8ca859a66e3a47f5f42d1e403c4d03"},
- {file = "pyarrow-17.0.0-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:9b8a823cea605221e61f34859dcc03207e52e409ccf6354634143e23af7c8d22"},
- {file = "pyarrow-17.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f1e70de6cb5790a50b01d2b686d54aaf73da01266850b05e3af2a1bc89e16053"},
- {file = "pyarrow-17.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0071ce35788c6f9077ff9ecba4858108eebe2ea5a3f7cf2cf55ebc1dbc6ee24a"},
- {file = "pyarrow-17.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:757074882f844411fcca735e39aae74248a1531367a7c80799b4266390ae51cc"},
- {file = "pyarrow-17.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:9ba11c4f16976e89146781a83833df7f82077cdab7dc6232c897789343f7891a"},
- {file = "pyarrow-17.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:b0c6ac301093b42d34410b187bba560b17c0330f64907bfa4f7f7f2444b0cf9b"},
- {file = "pyarrow-17.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:392bc9feabc647338e6c89267635e111d71edad5fcffba204425a7c8d13610d7"},
- {file = "pyarrow-17.0.0-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:af5ff82a04b2171415f1410cff7ebb79861afc5dae50be73ce06d6e870615204"},
- {file = "pyarrow-17.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:edca18eaca89cd6382dfbcff3dd2d87633433043650c07375d095cd3517561d8"},
- {file = "pyarrow-17.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7c7916bff914ac5d4a8fe25b7a25e432ff921e72f6f2b7547d1e325c1ad9d155"},
- {file = "pyarrow-17.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f553ca691b9e94b202ff741bdd40f6ccb70cdd5fbf65c187af132f1317de6145"},
- {file = "pyarrow-17.0.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:0cdb0e627c86c373205a2f94a510ac4376fdc523f8bb36beab2e7f204416163c"},
- {file = "pyarrow-17.0.0-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:d7d192305d9d8bc9082d10f361fc70a73590a4c65cf31c3e6926cd72b76bc35c"},
- {file = "pyarrow-17.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:02dae06ce212d8b3244dd3e7d12d9c4d3046945a5933d28026598e9dbbda1fca"},
- {file = "pyarrow-17.0.0-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:13d7a460b412f31e4c0efa1148e1d29bdf18ad1411eb6757d38f8fbdcc8645fb"},
- {file = "pyarrow-17.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9b564a51fbccfab5a04a80453e5ac6c9954a9c5ef2890d1bcf63741909c3f8df"},
- {file = "pyarrow-17.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:32503827abbc5aadedfa235f5ece8c4f8f8b0a3cf01066bc8d29de7539532687"},
- {file = "pyarrow-17.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a155acc7f154b9ffcc85497509bcd0d43efb80d6f733b0dc3bb14e281f131c8b"},
- {file = "pyarrow-17.0.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:dec8d129254d0188a49f8a1fc99e0560dc1b85f60af729f47de4046015f9b0a5"},
- {file = "pyarrow-17.0.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:a48ddf5c3c6a6c505904545c25a4ae13646ae1f8ba703c4df4a1bfe4f4006bda"},
- {file = "pyarrow-17.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:42bf93249a083aca230ba7e2786c5f673507fa97bbd9725a1e2754715151a204"},
- {file = "pyarrow-17.0.0.tar.gz", hash = "sha256:4beca9521ed2c0921c1023e68d097d0299b62c362639ea315572a58f3f50fd28"},
-]
-
-[package.dependencies]
-numpy = ">=1.16.6"
-
-[package.extras]
-test = ["cffi", "hypothesis", "pandas", "pytest", "pytz"]
-
-[[package]]
-name = "pydantic"
-version = "2.9.2"
-description = "Data validation using Python type hints"
-optional = false
-python-versions = ">=3.8"
-files = [
- {file = "pydantic-2.9.2-py3-none-any.whl", hash = "sha256:f048cec7b26778210e28a0459867920654d48e5e62db0958433636cde4254f12"},
- {file = "pydantic-2.9.2.tar.gz", hash = "sha256:d155cef71265d1e9807ed1c32b4c8deec042a44a50a4188b25ac67ecd81a9c0f"},
-]
-
-[package.dependencies]
-annotated-types = ">=0.6.0"
-pydantic-core = "2.23.4"
-typing-extensions = {version = ">=4.6.1", markers = "python_version < \"3.13\""}
-
-[package.extras]
-email = ["email-validator (>=2.0.0)"]
-timezone = ["tzdata"]
-
-[[package]]
-name = "pydantic-core"
-version = "2.23.4"
-description = "Core functionality for Pydantic validation and serialization"
-optional = false
-python-versions = ">=3.8"
-files = [
- {file = "pydantic_core-2.23.4-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:b10bd51f823d891193d4717448fab065733958bdb6a6b351967bd349d48d5c9b"},
- {file = "pydantic_core-2.23.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4fc714bdbfb534f94034efaa6eadd74e5b93c8fa6315565a222f7b6f42ca1166"},
- {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63e46b3169866bd62849936de036f901a9356e36376079b05efa83caeaa02ceb"},
- {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ed1a53de42fbe34853ba90513cea21673481cd81ed1be739f7f2efb931b24916"},
- {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cfdd16ab5e59fc31b5e906d1a3f666571abc367598e3e02c83403acabc092e07"},
- {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:255a8ef062cbf6674450e668482456abac99a5583bbafb73f9ad469540a3a232"},
- {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4a7cd62e831afe623fbb7aabbb4fe583212115b3ef38a9f6b71869ba644624a2"},
- {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f09e2ff1f17c2b51f2bc76d1cc33da96298f0a036a137f5440ab3ec5360b624f"},
- {file = "pydantic_core-2.23.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e38e63e6f3d1cec5a27e0afe90a085af8b6806ee208b33030e65b6516353f1a3"},
- {file = "pydantic_core-2.23.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0dbd8dbed2085ed23b5c04afa29d8fd2771674223135dc9bc937f3c09284d071"},
- {file = "pydantic_core-2.23.4-cp310-none-win32.whl", hash = "sha256:6531b7ca5f951d663c339002e91aaebda765ec7d61b7d1e3991051906ddde119"},
- {file = "pydantic_core-2.23.4-cp310-none-win_amd64.whl", hash = "sha256:7c9129eb40958b3d4500fa2467e6a83356b3b61bfff1b414c7361d9220f9ae8f"},
- {file = "pydantic_core-2.23.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:77733e3892bb0a7fa797826361ce8a9184d25c8dffaec60b7ffe928153680ba8"},
- {file = "pydantic_core-2.23.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1b84d168f6c48fabd1f2027a3d1bdfe62f92cade1fb273a5d68e621da0e44e6d"},
- {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:df49e7a0861a8c36d089c1ed57d308623d60416dab2647a4a17fe050ba85de0e"},
- {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ff02b6d461a6de369f07ec15e465a88895f3223eb75073ffea56b84d9331f607"},
- {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:996a38a83508c54c78a5f41456b0103c30508fed9abcad0a59b876d7398f25fd"},
- {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d97683ddee4723ae8c95d1eddac7c192e8c552da0c73a925a89fa8649bf13eea"},
- {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:216f9b2d7713eb98cb83c80b9c794de1f6b7e3145eef40400c62e86cee5f4e1e"},
- {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6f783e0ec4803c787bcea93e13e9932edab72068f68ecffdf86a99fd5918878b"},
- {file = "pydantic_core-2.23.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d0776dea117cf5272382634bd2a5c1b6eb16767c223c6a5317cd3e2a757c61a0"},
- {file = "pydantic_core-2.23.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d5f7a395a8cf1621939692dba2a6b6a830efa6b3cee787d82c7de1ad2930de64"},
- {file = "pydantic_core-2.23.4-cp311-none-win32.whl", hash = "sha256:74b9127ffea03643e998e0c5ad9bd3811d3dac8c676e47db17b0ee7c3c3bf35f"},
- {file = "pydantic_core-2.23.4-cp311-none-win_amd64.whl", hash = "sha256:98d134c954828488b153d88ba1f34e14259284f256180ce659e8d83e9c05eaa3"},
- {file = "pydantic_core-2.23.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:f3e0da4ebaef65158d4dfd7d3678aad692f7666877df0002b8a522cdf088f231"},
- {file = "pydantic_core-2.23.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f69a8e0b033b747bb3e36a44e7732f0c99f7edd5cea723d45bc0d6e95377ffee"},
- {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:723314c1d51722ab28bfcd5240d858512ffd3116449c557a1336cbe3919beb87"},
- {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bb2802e667b7051a1bebbfe93684841cc9351004e2badbd6411bf357ab8d5ac8"},
- {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d18ca8148bebe1b0a382a27a8ee60350091a6ddaf475fa05ef50dc35b5df6327"},
- {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:33e3d65a85a2a4a0dc3b092b938a4062b1a05f3a9abde65ea93b233bca0e03f2"},
- {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:128585782e5bfa515c590ccee4b727fb76925dd04a98864182b22e89a4e6ed36"},
- {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:68665f4c17edcceecc112dfed5dbe6f92261fb9d6054b47d01bf6371a6196126"},
- {file = "pydantic_core-2.23.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:20152074317d9bed6b7a95ade3b7d6054845d70584216160860425f4fbd5ee9e"},
- {file = "pydantic_core-2.23.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:9261d3ce84fa1d38ed649c3638feefeae23d32ba9182963e465d58d62203bd24"},
- {file = "pydantic_core-2.23.4-cp312-none-win32.whl", hash = "sha256:4ba762ed58e8d68657fc1281e9bb72e1c3e79cc5d464be146e260c541ec12d84"},
- {file = "pydantic_core-2.23.4-cp312-none-win_amd64.whl", hash = "sha256:97df63000f4fea395b2824da80e169731088656d1818a11b95f3b173747b6cd9"},
- {file = "pydantic_core-2.23.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:7530e201d10d7d14abce4fb54cfe5b94a0aefc87da539d0346a484ead376c3cc"},
- {file = "pydantic_core-2.23.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:df933278128ea1cd77772673c73954e53a1c95a4fdf41eef97c2b779271bd0bd"},
- {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0cb3da3fd1b6a5d0279a01877713dbda118a2a4fc6f0d821a57da2e464793f05"},
- {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:42c6dcb030aefb668a2b7009c85b27f90e51e6a3b4d5c9bc4c57631292015b0d"},
- {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:696dd8d674d6ce621ab9d45b205df149399e4bb9aa34102c970b721554828510"},
- {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2971bb5ffe72cc0f555c13e19b23c85b654dd2a8f7ab493c262071377bfce9f6"},
- {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8394d940e5d400d04cad4f75c0598665cbb81aecefaca82ca85bd28264af7f9b"},
- {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:0dff76e0602ca7d4cdaacc1ac4c005e0ce0dcfe095d5b5259163a80d3a10d327"},
- {file = "pydantic_core-2.23.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:7d32706badfe136888bdea71c0def994644e09fff0bfe47441deaed8e96fdbc6"},
- {file = "pydantic_core-2.23.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:ed541d70698978a20eb63d8c5d72f2cc6d7079d9d90f6b50bad07826f1320f5f"},
- {file = "pydantic_core-2.23.4-cp313-none-win32.whl", hash = "sha256:3d5639516376dce1940ea36edf408c554475369f5da2abd45d44621cb616f769"},
- {file = "pydantic_core-2.23.4-cp313-none-win_amd64.whl", hash = "sha256:5a1504ad17ba4210df3a045132a7baeeba5a200e930f57512ee02909fc5c4cb5"},
- {file = "pydantic_core-2.23.4-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:d4488a93b071c04dc20f5cecc3631fc78b9789dd72483ba15d423b5b3689b555"},
- {file = "pydantic_core-2.23.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:81965a16b675b35e1d09dd14df53f190f9129c0202356ed44ab2728b1c905658"},
- {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ffa2ebd4c8530079140dd2d7f794a9d9a73cbb8e9d59ffe24c63436efa8f271"},
- {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:61817945f2fe7d166e75fbfb28004034b48e44878177fc54d81688e7b85a3665"},
- {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:29d2c342c4bc01b88402d60189f3df065fb0dda3654744d5a165a5288a657368"},
- {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5e11661ce0fd30a6790e8bcdf263b9ec5988e95e63cf901972107efc49218b13"},
- {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d18368b137c6295db49ce7218b1a9ba15c5bc254c96d7c9f9e924a9bc7825ad"},
- {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ec4e55f79b1c4ffb2eecd8a0cfba9955a2588497d96851f4c8f99aa4a1d39b12"},
- {file = "pydantic_core-2.23.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:374a5e5049eda9e0a44c696c7ade3ff355f06b1fe0bb945ea3cac2bc336478a2"},
- {file = "pydantic_core-2.23.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5c364564d17da23db1106787675fc7af45f2f7b58b4173bfdd105564e132e6fb"},
- {file = "pydantic_core-2.23.4-cp38-none-win32.whl", hash = "sha256:d7a80d21d613eec45e3d41eb22f8f94ddc758a6c4720842dc74c0581f54993d6"},
- {file = "pydantic_core-2.23.4-cp38-none-win_amd64.whl", hash = "sha256:5f5ff8d839f4566a474a969508fe1c5e59c31c80d9e140566f9a37bba7b8d556"},
- {file = "pydantic_core-2.23.4-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:a4fa4fc04dff799089689f4fd502ce7d59de529fc2f40a2c8836886c03e0175a"},
- {file = "pydantic_core-2.23.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0a7df63886be5e270da67e0966cf4afbae86069501d35c8c1b3b6c168f42cb36"},
- {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dcedcd19a557e182628afa1d553c3895a9f825b936415d0dbd3cd0bbcfd29b4b"},
- {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5f54b118ce5de9ac21c363d9b3caa6c800341e8c47a508787e5868c6b79c9323"},
- {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:86d2f57d3e1379a9525c5ab067b27dbb8a0642fb5d454e17a9ac434f9ce523e3"},
- {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:de6d1d1b9e5101508cb37ab0d972357cac5235f5c6533d1071964c47139257df"},
- {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1278e0d324f6908e872730c9102b0112477a7f7cf88b308e4fc36ce1bdb6d58c"},
- {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9a6b5099eeec78827553827f4c6b8615978bb4b6a88e5d9b93eddf8bb6790f55"},
- {file = "pydantic_core-2.23.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:e55541f756f9b3ee346b840103f32779c695a19826a4c442b7954550a0972040"},
- {file = "pydantic_core-2.23.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a5c7ba8ffb6d6f8f2ab08743be203654bb1aaa8c9dcb09f82ddd34eadb695605"},
- {file = "pydantic_core-2.23.4-cp39-none-win32.whl", hash = "sha256:37b0fe330e4a58d3c58b24d91d1eb102aeec675a3db4c292ec3928ecd892a9a6"},
- {file = "pydantic_core-2.23.4-cp39-none-win_amd64.whl", hash = "sha256:1498bec4c05c9c787bde9125cfdcc63a41004ff167f495063191b863399b1a29"},
- {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:f455ee30a9d61d3e1a15abd5068827773d6e4dc513e795f380cdd59932c782d5"},
- {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:1e90d2e3bd2c3863d48525d297cd143fe541be8bbf6f579504b9712cb6b643ec"},
- {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e203fdf807ac7e12ab59ca2bfcabb38c7cf0b33c41efeb00f8e5da1d86af480"},
- {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e08277a400de01bc72436a0ccd02bdf596631411f592ad985dcee21445bd0068"},
- {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f220b0eea5965dec25480b6333c788fb72ce5f9129e8759ef876a1d805d00801"},
- {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:d06b0c8da4f16d1d1e352134427cb194a0a6e19ad5db9161bf32b2113409e728"},
- {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:ba1a0996f6c2773bd83e63f18914c1de3c9dd26d55f4ac302a7efe93fb8e7433"},
- {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:9a5bce9d23aac8f0cf0836ecfc033896aa8443b501c58d0602dbfd5bd5b37753"},
- {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:78ddaaa81421a29574a682b3179d4cf9e6d405a09b99d93ddcf7e5239c742e21"},
- {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:883a91b5dd7d26492ff2f04f40fbb652de40fcc0afe07e8129e8ae779c2110eb"},
- {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88ad334a15b32a791ea935af224b9de1bf99bcd62fabf745d5f3442199d86d59"},
- {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:233710f069d251feb12a56da21e14cca67994eab08362207785cf8c598e74577"},
- {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:19442362866a753485ba5e4be408964644dd6a09123d9416c54cd49171f50744"},
- {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:624e278a7d29b6445e4e813af92af37820fafb6dcc55c012c834f9e26f9aaaef"},
- {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f5ef8f42bec47f21d07668a043f077d507e5bf4e668d5c6dfe6aaba89de1a5b8"},
- {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:aea443fffa9fbe3af1a9ba721a87f926fe548d32cab71d188a6ede77d0ff244e"},
- {file = "pydantic_core-2.23.4.tar.gz", hash = "sha256:2584f7cf844ac4d970fba483a717dbe10c1c1c96a969bf65d61ffe94df1b2863"},
-]
-
-[package.dependencies]
-typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0"
-
-[[package]]
-name = "pygments"
-version = "2.18.0"
-description = "Pygments is a syntax highlighting package written in Python."
-optional = false
-python-versions = ">=3.8"
-files = [
- {file = "pygments-2.18.0-py3-none-any.whl", hash = "sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a"},
- {file = "pygments-2.18.0.tar.gz", hash = "sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199"},
-]
-
-[package.extras]
-windows-terminal = ["colorama (>=0.4.6)"]
-
-[[package]]
-name = "pyjson5"
-version = "1.6.6"
-description = "JSON5 serializer and parser for Python 3 written in Cython."
-optional = false
-python-versions = "~=3.5"
-files = [
- {file = "pyjson5-1.6.6-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:567437862f410a5912eee4cf13dd01a8c28ce9c9bf95590b9b9a4cb20e9daaed"},
- {file = "pyjson5-1.6.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d1649e043e1277aae474e72f8fa3431cbf83605059e733043e718f77f59aef29"},
- {file = "pyjson5-1.6.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:420d9f970d678d8d1fab8aefdd483747c75e64681766e771494910b8030f41fa"},
- {file = "pyjson5-1.6.6-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:59166de551e7321dbd3aa552ef647d652873701faadb021eae20f55ba6705829"},
- {file = "pyjson5-1.6.6-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:314537a41af768367ab19a50d697a6ea85cb1c39bac4a2b93680cab930002e31"},
- {file = "pyjson5-1.6.6-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ed4c50a117e8ff0733e2c2b761adf183150ee9bf9294f06c01983d76e89f5f2c"},
- {file = "pyjson5-1.6.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b258ba83c2d888568a098cab8f7df1ceffded589386eb5d70960ab83cffa9d"},
- {file = "pyjson5-1.6.6-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:211a9e260cf7d122e4d8881ae8724067cf7dfa48317760adb59b96bcf8e0a407"},
- {file = "pyjson5-1.6.6-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e4b0a14f9226aa425bec207b9ea5ec149b6b9ff63d40399001b9ad4a1f9920df"},
- {file = "pyjson5-1.6.6-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:a873c3a0d38f4f901e2d721ea3875ecf8d704a6d6c642cf31f5e17d37b60ca38"},
- {file = "pyjson5-1.6.6-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:64381957edf773f4c67cc32c6e44fc791034efde5d654626171829d1219e4247"},
- {file = "pyjson5-1.6.6-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:c5ee52cc58080472bd5668a2e114b779a56e200cdae52870e61a72c95c0760d4"},
- {file = "pyjson5-1.6.6-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:e6f791cb35ec48a5822a888225b54b36e53e10ae7ec193f85d26c188702861a9"},
- {file = "pyjson5-1.6.6-cp310-cp310-win32.whl", hash = "sha256:80a96a264b7eb7fcbeaf646f44e18c22f98d1f72cc9e9ca284f785612df457d8"},
- {file = "pyjson5-1.6.6-cp310-cp310-win_amd64.whl", hash = "sha256:a79e914cba2a45725c2b10801bbdc7975e2b0926c4406f3cbd67d3976c2ded9c"},
- {file = "pyjson5-1.6.6-cp310-cp310-win_arm64.whl", hash = "sha256:ab2d7b402b8f27a866431dc1b476c4f9f0ccbb0811425c846985f05f5de9236b"},
- {file = "pyjson5-1.6.6-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:127f145467b61ef8a9a1d01c5c33a433d14dbca70c70d0b0800f4a19581303ff"},
- {file = "pyjson5-1.6.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0d828b98667ebca9fbd172f221746ecba105081b77f5ac6bbbf055f2faa38e14"},
- {file = "pyjson5-1.6.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7d1fe850aa1763e8f9945a7120f032f5f707d0372d52c8e0fecda2a79640820e"},
- {file = "pyjson5-1.6.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a8e4974cd32697f3a2c6738b597eaf66f2d23eb34dcdd79b8b6bb92549b56395"},
- {file = "pyjson5-1.6.6-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:011957e1e7143c82ee5701dd39292d3864f6989247d1423462b1ef405390435e"},
- {file = "pyjson5-1.6.6-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:122a980366db9c56b35bead60debf832d614ebe47a65d34bc292b2211e42e547"},
- {file = "pyjson5-1.6.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0abc60630bb1601ae9005e251d30cef363cb4d6cef2d2fb7a160329569170e38"},
- {file = "pyjson5-1.6.6-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7f9f8072be8d2cd3ba40346efeaaaba089a6e9be04ef23a3d690aaa897c53f71"},
- {file = "pyjson5-1.6.6-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:990bf3aa3df840dfb98e3663d645d6c897d00ccf1e6cc2a594a547e8c239cc52"},
- {file = "pyjson5-1.6.6-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:bff1c85756a80fba14285bcaef576419b4b159543fdb8c20714331c1d873e9e1"},
- {file = "pyjson5-1.6.6-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:0570cd200f5b52938769705c4f3092f1fcbb850f62278d30db8b8c6ae5ef0617"},
- {file = "pyjson5-1.6.6-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:6e358035e6fd975933f00a01c2ed3cc1e344f9a06b6edc445ad4b3bca7509de4"},
- {file = "pyjson5-1.6.6-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:86ac9e2b5dc086a979a115661f7886208352a45f3276453db914adbda54adbb7"},
- {file = "pyjson5-1.6.6-cp311-cp311-win32.whl", hash = "sha256:415805f111faa16b050827beda8d763d4391becc24f86438848c7cb23ce63c55"},
- {file = "pyjson5-1.6.6-cp311-cp311-win_amd64.whl", hash = "sha256:9293b3768e27a69ef764daa10168c199887ffbe820ef86ea9c2ff155bdd27fba"},
- {file = "pyjson5-1.6.6-cp311-cp311-win_arm64.whl", hash = "sha256:3145da3885e063b360bd9cc0bff6c9b0c83e3a2e822f83b4f7f40b26b41e1598"},
- {file = "pyjson5-1.6.6-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:b2f96647371dcab50060e5d6908f152ad33c10e534f9695f81e3f59733748f0f"},
- {file = "pyjson5-1.6.6-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c0836a8181e4e857a91c1ab55adfee2dc4e60fff5e67f9e45f885a69586702f8"},
- {file = "pyjson5-1.6.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4f1a9eebe9d793e5b6ed00691744255400f57666004f454b09e2e651657e15fe"},
- {file = "pyjson5-1.6.6-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:02d60cd0f98d39347416d9172f38cd9601dfcf9976536688deb82c387ac22db1"},
- {file = "pyjson5-1.6.6-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7366857ff307ef1bae3f1096651a2b2f86ef87b8ff4ea7264c6d7db9176adaab"},
- {file = "pyjson5-1.6.6-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cdf1f8140eccab24bd917a4111cc2783b1b430e8b9a6443b2ec4dce7267cfa4e"},
- {file = "pyjson5-1.6.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8b738a46b7a4c6910a8cd34c3fec752c26afb5156939241b080311df1ace484"},
- {file = "pyjson5-1.6.6-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cf1f437b4258661d0cf884675487220fd035f67e50a648732e02b22b1e417d60"},
- {file = "pyjson5-1.6.6-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:5940c9c2feb9dbcda0cb87d2fc7edf330de231469c882f1e584d96814b6efd0d"},
- {file = "pyjson5-1.6.6-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:270600deefe532138d00cec1e4029c716fc86eaa52dabfb12494dca1cb30e611"},
- {file = "pyjson5-1.6.6-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:bdcc8073544acbb625dfcd64af8d6fdddefa4dd8f6898eb0bea1d50bfc7db071"},
- {file = "pyjson5-1.6.6-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:e62998daf49ccfa106459724ab3e57a87afc6b4e17e3422db633bf837918ee01"},
- {file = "pyjson5-1.6.6-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1e3cf693ac82c7ee1da149e172bbe0cf59d11b06c31331f254daf8825d566033"},
- {file = "pyjson5-1.6.6-cp312-cp312-win32.whl", hash = "sha256:cd44211d3430fc32abad56a991fe5279243cbdae303a1c00ce418e0562f315cb"},
- {file = "pyjson5-1.6.6-cp312-cp312-win_amd64.whl", hash = "sha256:50edc8c1f8c903f6e5df8547ae4195e3456ba2cdb45fbbad14022937a27ffa7c"},
- {file = "pyjson5-1.6.6-cp312-cp312-win_arm64.whl", hash = "sha256:14cad404a8dff0ea57370b98350b8fedb7611ee8023090b8a991673b4542abf2"},
- {file = "pyjson5-1.6.6-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:8eddf775c2846e24f5f17c0ef2dc20f7de628148d03394a46ac1fd866a59ab78"},
- {file = "pyjson5-1.6.6-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:34500d5373dff030ab5183a69b3eb545a3a89385a3949142ea61ca95e653cbf8"},
- {file = "pyjson5-1.6.6-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d6cb856116850100d46df398cd68fc05d61ae5525eb0164b1aa9310e14c957d6"},
- {file = "pyjson5-1.6.6-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:410c15018434271408bb9410ac7db5195beccc33d8d1fbc8fb4a39858427e0df"},
- {file = "pyjson5-1.6.6-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fb6a4bfdfb47904f1a4fdc135ab000e36bf79a0bc70aa9ec98275493ae19fd55"},
- {file = "pyjson5-1.6.6-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b21596c5d805c5e8fae5ba2c80317f2a6441c62ea91d1bd6515c43c23911a16a"},
- {file = "pyjson5-1.6.6-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:4724d6bc1e369b29fb9fab7b10530cc11ccba60dc403cef3a83c213bc88d541c"},
- {file = "pyjson5-1.6.6-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:0d8063c83603e4eda99dc9396113521f0d277b027ccb3182e8e47ea284db8a70"},
- {file = "pyjson5-1.6.6-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:e23d76671b1ea35d31144ea94f911d7096808e63834ee412105e9c281d50802a"},
- {file = "pyjson5-1.6.6-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:1c2ae4f322d2c859abd45efa893d06737b12575ca32fb068d2ab1ff7e1dacf7c"},
- {file = "pyjson5-1.6.6-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:0c069dfbc4939ce5d8b6d72b7755bb287988aab819435349e6f52da4e62fac9c"},
- {file = "pyjson5-1.6.6-cp36-cp36m-win32.whl", hash = "sha256:7fe091a563a1fe8f1c495d96d26fd40f9d19b9c3674dbf89dd3c3df8bf46cfe5"},
- {file = "pyjson5-1.6.6-cp36-cp36m-win_amd64.whl", hash = "sha256:3d2829447d9d6a51a5976a3eb0a2a0069122ab741378c60367883e60160cb6de"},
- {file = "pyjson5-1.6.6-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:87063a381b2db5f5af794ba6a3ec16c9a9f8fc5e974e41c299f761e7af138ec3"},
- {file = "pyjson5-1.6.6-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94b2d2a26ecdd9ecef704a5a0c86a272e85d1df6bb30f44fb7f825d6975f1fb3"},
- {file = "pyjson5-1.6.6-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:55df02651c98dc8ad7fa30fd4fc5a4789a83ed0498312154c35ee46bdd027ccd"},
- {file = "pyjson5-1.6.6-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:985f51077649fd0c84fcc7057d48b862c843f30f77c9f5c27102363c653c9e5e"},
- {file = "pyjson5-1.6.6-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c8af6410aa6525783ceef0008f4f86480a4f04c4b921dd4295a2f2ba62f682d"},
- {file = "pyjson5-1.6.6-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:caebf75a83015299e45006819caac156ac2ef1e057158c6c1453c0ebf73de1d9"},
- {file = "pyjson5-1.6.6-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:f069e4c6336b19afdc594d04c81eb900af5d834a6cd8b395015389974b044972"},
- {file = "pyjson5-1.6.6-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:94e20d28d2cfba1b2be59c63ad8ae221e709a84158dc7f138d3b7258319f59b2"},
- {file = "pyjson5-1.6.6-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:8b869fead9daa2ef5972f30abd76fdb8acfe3d6f73c0e3783fe13e1775d4aa05"},
- {file = "pyjson5-1.6.6-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:a093e24dd14a97f06d1596158d413b3b556cdf2b0d7a1af45b546d1d90224de7"},
- {file = "pyjson5-1.6.6-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:493fa3a81ce879e7ae47d1aa6a2b465a207087857b693404e523f069f531d01d"},
- {file = "pyjson5-1.6.6-cp37-cp37m-win32.whl", hash = "sha256:149f6ca0e346fce26ccb154e6633d3dbe10f197aae2b306cf3e3f09a57b6e4f7"},
- {file = "pyjson5-1.6.6-cp37-cp37m-win_amd64.whl", hash = "sha256:7dffc9dcbdf09663f6aefcf03a48cfb029437ee60c0d4372e2218f30929d3035"},
- {file = "pyjson5-1.6.6-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5c7643308f995d746af8210b9a418c287885a0f8c0104b5e5160c190e56fbd0c"},
- {file = "pyjson5-1.6.6-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:144ea7654b64226cd24c6cc856827180b2e04ddc4404f9513ba41c6f284aa4c7"},
- {file = "pyjson5-1.6.6-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:f3fb709498a53e0f9fb8d18612ae88b11bd472cce80431e312f1a6ad65bce673"},
- {file = "pyjson5-1.6.6-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:de97c9e246bce7231dab34248a66218b48be5af5d5ae69c78421a095b0e0ab26"},
- {file = "pyjson5-1.6.6-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:57231901f4c96cc5bc630daef3fc3eadc95b8babe019b9f26b4915296a755bb5"},
- {file = "pyjson5-1.6.6-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:759ef75276235d9491ecf31c4c7ba057cdcd45707c04d2bdd0c0a18bd9f36f71"},
- {file = "pyjson5-1.6.6-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cb1fec1b2a03a219c9fb3cccb255140bc11daa86ce844ffe7354db7b2bc9223f"},
- {file = "pyjson5-1.6.6-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f3a120f1ac4dfe9d7fdfadd3cd2e0f02260c2e1c1418767946fa6ce6a786dcd0"},
- {file = "pyjson5-1.6.6-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:e6dedeb9aa7a4d6703c1b5ffd3ec97f4ee6549f4f6550f644f43a74f43fcc89b"},
- {file = "pyjson5-1.6.6-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:7d6e8f92e2f0b84e6ede805aa68bbee408f639c29f64fd14960a242bb518dcc6"},
- {file = "pyjson5-1.6.6-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:d9661f68bcf033a80da5292a87ab1abcbd6462ec308f83cc9a96d5df4255a500"},
- {file = "pyjson5-1.6.6-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:db57d37d1c2cc07e50dc8a37c1fd4903a38feb1b236e8f9094249d621deb39e5"},
- {file = "pyjson5-1.6.6-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:1d0bd69aa1b82410593341eb43c546c567bee5acb151666857c9df98e2cdfc09"},
- {file = "pyjson5-1.6.6-cp38-cp38-win32.whl", hash = "sha256:6b19a7025546406ca91184125aadc89a961189a9f5d4a76c0534a04f814c8882"},
- {file = "pyjson5-1.6.6-cp38-cp38-win_amd64.whl", hash = "sha256:e94bb47d5fbacf7859f16117ab572b063705fdc6d3caf0afd92e02bbe1a0adfb"},
- {file = "pyjson5-1.6.6-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:d539ed7039ca0677a9eee156bd7551e158fd4c8e67b500fba4e9b42c2178dbde"},
- {file = "pyjson5-1.6.6-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c8c19dd552551b256ec70beed516d47953cbf153cc7b04ec7189b9000211f372"},
- {file = "pyjson5-1.6.6-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:36bdfa9334d00be616a63fd15295e9885d0e91dfa75cda5b6a8d1a2b406d6252"},
- {file = "pyjson5-1.6.6-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f6453b85fd4839660beff29165cdee0e0c486d32d8e7be002daffbf40c64c445"},
- {file = "pyjson5-1.6.6-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:62cd10320a1e80b1ce386ccaed9073cffd729e0e5b7a8793f2083291770b0db3"},
- {file = "pyjson5-1.6.6-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2add15588939d8b1b7c59fd3f5480cce76231e0b014a2edebf3443ba92789d38"},
- {file = "pyjson5-1.6.6-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d8eb8edf6607a6648316f8a5a76bbd1bcb6626990dd9bd6c4b1dee91ec73650e"},
- {file = "pyjson5-1.6.6-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2cec7163effe96abe40ff098ffd2b16f5b322628bdac34c7aa5d316588524c42"},
- {file = "pyjson5-1.6.6-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:a29645584914f7b1840e916777d402264ea3cbe3e04ff47ea00a799de17552d6"},
- {file = "pyjson5-1.6.6-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:259f1797cd25e13396b6cb08dc0625c5de26a4214a8279e5304d6c91e5c8eb50"},
- {file = "pyjson5-1.6.6-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:48d02c97e34e64eefca4794dc3717373637aec4bd97b6e094cbed926b1893097"},
- {file = "pyjson5-1.6.6-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:ce1c193b0b35c40aa8e2097c88cb92674fa4b37016724cd8e2dc2a12784fad4f"},
- {file = "pyjson5-1.6.6-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:df2429ec7da1495122b63df437c04d3d5af8cf23654848d20637fa41c4aee1b5"},
- {file = "pyjson5-1.6.6-cp39-cp39-win32.whl", hash = "sha256:f46d276c64b787c895323c82acb0f0151c4e0b275cf1ef001e2c7633a29f3068"},
- {file = "pyjson5-1.6.6-cp39-cp39-win_amd64.whl", hash = "sha256:521e7691fe38bd56bc4764e7c9c54b4783910f602904d8ca1a6571a9e82a3f82"},
- {file = "pyjson5-1.6.6-cp39-cp39-win_arm64.whl", hash = "sha256:76154b0e0245b9dbb97be34055d5be38cb6a55508785594b53e0f4d57b0267eb"},
- {file = "pyjson5-1.6.6-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:8afb7a6f3e020b9a2996bb62f98067ebf793e86599f349910a7129fbfaebdc79"},
- {file = "pyjson5-1.6.6-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d9facfbdf1952d3c96016ce17159a3ce97b296454bc900e94c5ea1f0ae85f00c"},
- {file = "pyjson5-1.6.6-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9371ff9d5c6a92676935171eafa9cc6a541702352f747f77000b343d3101b0c0"},
- {file = "pyjson5-1.6.6-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f48861b7aaafee14c2c1769e28070cae1aeb011e54cdc75a7dae8ed732115c72"},
- {file = "pyjson5-1.6.6-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:cbdcd7cb1745ce82b744fdadcfce167217c4c815e76e05897f806f99397fa092"},
- {file = "pyjson5-1.6.6-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:beaa44d409e16e31341713890baa4e612909250247201af49ddc57815c52a2e7"},
- {file = "pyjson5-1.6.6-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4fedb2ab5cfc821f9b9ed44dc1eae5d22a39a040950bda77c8b26e04df327629"},
- {file = "pyjson5-1.6.6-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:868619dc4d271ea9ec5f4041901238167526be369e3598867ca0d9170827692e"},
- {file = "pyjson5-1.6.6-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1ea61368eb930a39441a7e9dd34ecb9af026a7b35bd03570910558abcd296215"},
- {file = "pyjson5-1.6.6-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:0b9718821bfc36c61dd2ae8456fafbe3e9eb46df13cb2ac1ade38c5163ff9c92"},
- {file = "pyjson5-1.6.6-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:d373e93e3384670b71fa42f4453cc1f979e15b34a21c0b746c5a6d14b6ebbb12"},
- {file = "pyjson5-1.6.6-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6e2c27e3dd5caef56910c6da6a0c97cfc015a1dbdc6c2f2bd63a3ad86a16b0bd"},
- {file = "pyjson5-1.6.6-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aae2ad989da263216f556f3c5a3ea3eaf8c45894c9fea641c483576adb27494f"},
- {file = "pyjson5-1.6.6-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d29b5cefef1d2a2044fb800d7d7af2705b7acafac1bfd2630840a9a080d1c70d"},
- {file = "pyjson5-1.6.6-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:ae55e95b6c59e9ad86662b6be213a6ea3f1e0c7f3d5627d88ca3dbe293a0a23a"},
- {file = "pyjson5-1.6.6-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:e5f192cf3570c0588511afac7f0aa155f66dbf0e61ae39c8db63f67e0a3c9788"},
- {file = "pyjson5-1.6.6-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:14671bbb72df85b895570f092e1b0aa34437a8df3e609e70a00459943aa1b75c"},
- {file = "pyjson5-1.6.6-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f541197328ac3433863e2e67e4d34fccf1590bb88cc7f2c3fc2a81b8cde2681"},
- {file = "pyjson5-1.6.6-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d404ec8a8d8301d803ccf037b3f0fb5fc8051aaa8a9a737cd6d4c490911d316a"},
- {file = "pyjson5-1.6.6-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:a3b0e45a2f1a84e78d8ecd70aecce0f84b360af37b18b2123646c3b310ea10a7"},
- {file = "pyjson5-1.6.6.tar.gz", hash = "sha256:20df8c5dbbe0d653f403da88b7520be44fc8d74697bbdd1ead688458b5691a02"},
-]
-
-[[package]]
-name = "pytest"
-version = "8.3.3"
-description = "pytest: simple powerful testing with Python"
-optional = false
-python-versions = ">=3.8"
-files = [
- {file = "pytest-8.3.3-py3-none-any.whl", hash = "sha256:a6853c7375b2663155079443d2e45de913a911a11d669df02a50814944db57b2"},
- {file = "pytest-8.3.3.tar.gz", hash = "sha256:70b98107bd648308a7952b06e6ca9a50bc660be218d53c257cc1fc94fda10181"},
-]
-
-[package.dependencies]
-colorama = {version = "*", markers = "sys_platform == \"win32\""}
-exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""}
-iniconfig = "*"
-packaging = "*"
-pluggy = ">=1.5,<2"
-tomli = {version = ">=1", markers = "python_version < \"3.11\""}
-
-[package.extras]
-dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"]
-
-[[package]]
-name = "python-dateutil"
-version = "2.9.0.post0"
-description = "Extensions to the standard Python datetime module"
-optional = false
-python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7"
-files = [
- {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"},
- {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"},
-]
-
-[package.dependencies]
-six = ">=1.5"
-
-[[package]]
-name = "python-dotenv"
-version = "1.0.1"
-description = "Read key-value pairs from a .env file and set them as environment variables"
-optional = false
-python-versions = ">=3.8"
-files = [
- {file = "python-dotenv-1.0.1.tar.gz", hash = "sha256:e324ee90a023d808f1959c46bcbc04446a10ced277783dc6ee09987c37ec10ca"},
- {file = "python_dotenv-1.0.1-py3-none-any.whl", hash = "sha256:f7b63ef50f1b690dddf550d03497b66d609393b40b564ed0d674909a68ebf16a"},
-]
-
-[package.extras]
-cli = ["click (>=5.0)"]
-
-[[package]]
-name = "python-socks"
-version = "2.5.1"
-description = "Core proxy (SOCKS4, SOCKS5, HTTP tunneling) functionality for Python"
-optional = false
-python-versions = "*"
-files = [
- {file = "python_socks-2.5.1-py3-none-any.whl", hash = "sha256:00e9a0c3a208e14429d42c820ddbe0755e17596f639fd558f3e8d925fb34bcec"},
- {file = "python_socks-2.5.1.tar.gz", hash = "sha256:7ed6559864d28858fbb7a85c6d96bb280e95af814d1d5d6dc50f92e35bfa340e"},
-]
-
-[package.extras]
-anyio = ["anyio (>=3.3.4,<5.0.0)"]
-asyncio = ["async-timeout (>=3.0.1)"]
-curio = ["curio (>=1.4)"]
-trio = ["trio (>=0.16.0)"]
-
-[[package]]
-name = "pytz"
-version = "2024.2"
-description = "World timezone definitions, modern and historical"
-optional = false
-python-versions = "*"
-files = [
- {file = "pytz-2024.2-py2.py3-none-any.whl", hash = "sha256:31c7c1817eb7fae7ca4b8c7ee50c72f93aa2dd863de768e1ef4245d426aa0725"},
- {file = "pytz-2024.2.tar.gz", hash = "sha256:2aa355083c50a0f93fa581709deac0c9ad65cca8a9e9beac660adcbd493c798a"},
-]
-
-[[package]]
-name = "pyyaml"
-version = "6.0.2"
-description = "YAML parser and emitter for Python"
-optional = false
-python-versions = ">=3.8"
-files = [
- {file = "PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086"},
- {file = "PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf"},
- {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237"},
- {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b"},
- {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed"},
- {file = "PyYAML-6.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180"},
- {file = "PyYAML-6.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68"},
- {file = "PyYAML-6.0.2-cp310-cp310-win32.whl", hash = "sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99"},
- {file = "PyYAML-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e"},
- {file = "PyYAML-6.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774"},
- {file = "PyYAML-6.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee"},
- {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c"},
- {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317"},
- {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85"},
- {file = "PyYAML-6.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4"},
- {file = "PyYAML-6.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e"},
- {file = "PyYAML-6.0.2-cp311-cp311-win32.whl", hash = "sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5"},
- {file = "PyYAML-6.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44"},
- {file = "PyYAML-6.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab"},
- {file = "PyYAML-6.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725"},
- {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5"},
- {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425"},
- {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476"},
- {file = "PyYAML-6.0.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48"},
- {file = "PyYAML-6.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b"},
- {file = "PyYAML-6.0.2-cp312-cp312-win32.whl", hash = "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4"},
- {file = "PyYAML-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8"},
- {file = "PyYAML-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba"},
- {file = "PyYAML-6.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1"},
- {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133"},
- {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484"},
- {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5"},
- {file = "PyYAML-6.0.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc"},
- {file = "PyYAML-6.0.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652"},
- {file = "PyYAML-6.0.2-cp313-cp313-win32.whl", hash = "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183"},
- {file = "PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563"},
- {file = "PyYAML-6.0.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:24471b829b3bf607e04e88d79542a9d48bb037c2267d7927a874e6c205ca7e9a"},
- {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7fded462629cfa4b685c5416b949ebad6cec74af5e2d42905d41e257e0869f5"},
- {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d84a1718ee396f54f3a086ea0a66d8e552b2ab2017ef8b420e92edbc841c352d"},
- {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9056c1ecd25795207ad294bcf39f2db3d845767be0ea6e6a34d856f006006083"},
- {file = "PyYAML-6.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:82d09873e40955485746739bcb8b4586983670466c23382c19cffecbf1fd8706"},
- {file = "PyYAML-6.0.2-cp38-cp38-win32.whl", hash = "sha256:43fa96a3ca0d6b1812e01ced1044a003533c47f6ee8aca31724f78e93ccc089a"},
- {file = "PyYAML-6.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:01179a4a8559ab5de078078f37e5c1a30d76bb88519906844fd7bdea1b7729ff"},
- {file = "PyYAML-6.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:688ba32a1cffef67fd2e9398a2efebaea461578b0923624778664cc1c914db5d"},
- {file = "PyYAML-6.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a8786accb172bd8afb8be14490a16625cbc387036876ab6ba70912730faf8e1f"},
- {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8e03406cac8513435335dbab54c0d385e4a49e4945d2909a581c83647ca0290"},
- {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12"},
- {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b1fdb9dc17f5a7677423d508ab4f243a726dea51fa5e70992e59a7411c89d19"},
- {file = "PyYAML-6.0.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0b69e4ce7a131fe56b7e4d770c67429700908fc0752af059838b1cfb41960e4e"},
- {file = "PyYAML-6.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a9f8c2e67970f13b16084e04f134610fd1d374bf477b17ec1599185cf611d725"},
- {file = "PyYAML-6.0.2-cp39-cp39-win32.whl", hash = "sha256:6395c297d42274772abc367baaa79683958044e5d3835486c16da75d2a694631"},
- {file = "PyYAML-6.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:39693e1f8320ae4f43943590b49779ffb98acb81f788220ea932a6b6c51004d8"},
- {file = "pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e"},
-]
-
-[[package]]
-name = "referencing"
-version = "0.35.1"
-description = "JSON Referencing + Python"
-optional = false
-python-versions = ">=3.8"
-files = [
- {file = "referencing-0.35.1-py3-none-any.whl", hash = "sha256:eda6d3234d62814d1c64e305c1331c9a3a6132da475ab6382eaa997b21ee75de"},
- {file = "referencing-0.35.1.tar.gz", hash = "sha256:25b42124a6c8b632a425174f24087783efb348a6f1e0008e63cd4466fedf703c"},
-]
-
-[package.dependencies]
-attrs = ">=22.2.0"
-rpds-py = ">=0.7.0"
-
-[[package]]
-name = "regex"
-version = "2024.9.11"
-description = "Alternative regular expression module, to replace re."
-optional = false
-python-versions = ">=3.8"
-files = [
- {file = "regex-2024.9.11-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:1494fa8725c285a81d01dc8c06b55287a1ee5e0e382d8413adc0a9197aac6408"},
- {file = "regex-2024.9.11-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0e12c481ad92d129c78f13a2a3662317e46ee7ef96c94fd332e1c29131875b7d"},
- {file = "regex-2024.9.11-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:16e13a7929791ac1216afde26f712802e3df7bf0360b32e4914dca3ab8baeea5"},
- {file = "regex-2024.9.11-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:46989629904bad940bbec2106528140a218b4a36bb3042d8406980be1941429c"},
- {file = "regex-2024.9.11-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a906ed5e47a0ce5f04b2c981af1c9acf9e8696066900bf03b9d7879a6f679fc8"},
- {file = "regex-2024.9.11-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e9a091b0550b3b0207784a7d6d0f1a00d1d1c8a11699c1a4d93db3fbefc3ad35"},
- {file = "regex-2024.9.11-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5ddcd9a179c0a6fa8add279a4444015acddcd7f232a49071ae57fa6e278f1f71"},
- {file = "regex-2024.9.11-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6b41e1adc61fa347662b09398e31ad446afadff932a24807d3ceb955ed865cc8"},
- {file = "regex-2024.9.11-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ced479f601cd2f8ca1fd7b23925a7e0ad512a56d6e9476f79b8f381d9d37090a"},
- {file = "regex-2024.9.11-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:635a1d96665f84b292e401c3d62775851aedc31d4f8784117b3c68c4fcd4118d"},
- {file = "regex-2024.9.11-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:c0256beda696edcf7d97ef16b2a33a8e5a875affd6fa6567b54f7c577b30a137"},
- {file = "regex-2024.9.11-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:3ce4f1185db3fbde8ed8aa223fc9620f276c58de8b0d4f8cc86fd1360829edb6"},
- {file = "regex-2024.9.11-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:09d77559e80dcc9d24570da3745ab859a9cf91953062e4ab126ba9d5993688ca"},
- {file = "regex-2024.9.11-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:7a22ccefd4db3f12b526eccb129390942fe874a3a9fdbdd24cf55773a1faab1a"},
- {file = "regex-2024.9.11-cp310-cp310-win32.whl", hash = "sha256:f745ec09bc1b0bd15cfc73df6fa4f726dcc26bb16c23a03f9e3367d357eeedd0"},
- {file = "regex-2024.9.11-cp310-cp310-win_amd64.whl", hash = "sha256:01c2acb51f8a7d6494c8c5eafe3d8e06d76563d8a8a4643b37e9b2dd8a2ff623"},
- {file = "regex-2024.9.11-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2cce2449e5927a0bf084d346da6cd5eb016b2beca10d0013ab50e3c226ffc0df"},
- {file = "regex-2024.9.11-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3b37fa423beefa44919e009745ccbf353d8c981516e807995b2bd11c2c77d268"},
- {file = "regex-2024.9.11-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:64ce2799bd75039b480cc0360907c4fb2f50022f030bf9e7a8705b636e408fad"},
- {file = "regex-2024.9.11-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a4cc92bb6db56ab0c1cbd17294e14f5e9224f0cc6521167ef388332604e92679"},
- {file = "regex-2024.9.11-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d05ac6fa06959c4172eccd99a222e1fbf17b5670c4d596cb1e5cde99600674c4"},
- {file = "regex-2024.9.11-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:040562757795eeea356394a7fb13076ad4f99d3c62ab0f8bdfb21f99a1f85664"},
- {file = "regex-2024.9.11-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6113c008a7780792efc80f9dfe10ba0cd043cbf8dc9a76ef757850f51b4edc50"},
- {file = "regex-2024.9.11-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8e5fb5f77c8745a60105403a774fe2c1759b71d3e7b4ca237a5e67ad066c7199"},
- {file = "regex-2024.9.11-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:54d9ff35d4515debf14bc27f1e3b38bfc453eff3220f5bce159642fa762fe5d4"},
- {file = "regex-2024.9.11-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:df5cbb1fbc74a8305b6065d4ade43b993be03dbe0f8b30032cced0d7740994bd"},
- {file = "regex-2024.9.11-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:7fb89ee5d106e4a7a51bce305ac4efb981536301895f7bdcf93ec92ae0d91c7f"},
- {file = "regex-2024.9.11-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:a738b937d512b30bf75995c0159c0ddf9eec0775c9d72ac0202076c72f24aa96"},
- {file = "regex-2024.9.11-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e28f9faeb14b6f23ac55bfbbfd3643f5c7c18ede093977f1df249f73fd22c7b1"},
- {file = "regex-2024.9.11-cp311-cp311-win32.whl", hash = "sha256:18e707ce6c92d7282dfce370cd205098384b8ee21544e7cb29b8aab955b66fa9"},
- {file = "regex-2024.9.11-cp311-cp311-win_amd64.whl", hash = "sha256:313ea15e5ff2a8cbbad96ccef6be638393041b0a7863183c2d31e0c6116688cf"},
- {file = "regex-2024.9.11-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:b0d0a6c64fcc4ef9c69bd5b3b3626cc3776520a1637d8abaa62b9edc147a58f7"},
- {file = "regex-2024.9.11-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:49b0e06786ea663f933f3710a51e9385ce0cba0ea56b67107fd841a55d56a231"},
- {file = "regex-2024.9.11-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5b513b6997a0b2f10e4fd3a1313568e373926e8c252bd76c960f96fd039cd28d"},
- {file = "regex-2024.9.11-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ee439691d8c23e76f9802c42a95cfeebf9d47cf4ffd06f18489122dbb0a7ad64"},
- {file = "regex-2024.9.11-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a8f877c89719d759e52783f7fe6e1c67121076b87b40542966c02de5503ace42"},
- {file = "regex-2024.9.11-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:23b30c62d0f16827f2ae9f2bb87619bc4fba2044911e2e6c2eb1af0161cdb766"},
- {file = "regex-2024.9.11-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85ab7824093d8f10d44330fe1e6493f756f252d145323dd17ab6b48733ff6c0a"},
- {file = "regex-2024.9.11-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8dee5b4810a89447151999428fe096977346cf2f29f4d5e29609d2e19e0199c9"},
- {file = "regex-2024.9.11-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:98eeee2f2e63edae2181c886d7911ce502e1292794f4c5ee71e60e23e8d26b5d"},
- {file = "regex-2024.9.11-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:57fdd2e0b2694ce6fc2e5ccf189789c3e2962916fb38779d3e3521ff8fe7a822"},
- {file = "regex-2024.9.11-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:d552c78411f60b1fdaafd117a1fca2f02e562e309223b9d44b7de8be451ec5e0"},
- {file = "regex-2024.9.11-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:a0b2b80321c2ed3fcf0385ec9e51a12253c50f146fddb2abbb10f033fe3d049a"},
- {file = "regex-2024.9.11-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:18406efb2f5a0e57e3a5881cd9354c1512d3bb4f5c45d96d110a66114d84d23a"},
- {file = "regex-2024.9.11-cp312-cp312-win32.whl", hash = "sha256:e464b467f1588e2c42d26814231edecbcfe77f5ac414d92cbf4e7b55b2c2a776"},
- {file = "regex-2024.9.11-cp312-cp312-win_amd64.whl", hash = "sha256:9e8719792ca63c6b8340380352c24dcb8cd7ec49dae36e963742a275dfae6009"},
- {file = "regex-2024.9.11-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:c157bb447303070f256e084668b702073db99bbb61d44f85d811025fcf38f784"},
- {file = "regex-2024.9.11-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4db21ece84dfeefc5d8a3863f101995de646c6cb0536952c321a2650aa202c36"},
- {file = "regex-2024.9.11-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:220e92a30b426daf23bb67a7962900ed4613589bab80382be09b48896d211e92"},
- {file = "regex-2024.9.11-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eb1ae19e64c14c7ec1995f40bd932448713d3c73509e82d8cd7744dc00e29e86"},
- {file = "regex-2024.9.11-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f47cd43a5bfa48f86925fe26fbdd0a488ff15b62468abb5d2a1e092a4fb10e85"},
- {file = "regex-2024.9.11-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9d4a76b96f398697fe01117093613166e6aa8195d63f1b4ec3f21ab637632963"},
- {file = "regex-2024.9.11-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ea51dcc0835eea2ea31d66456210a4e01a076d820e9039b04ae8d17ac11dee6"},
- {file = "regex-2024.9.11-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b7aaa315101c6567a9a45d2839322c51c8d6e81f67683d529512f5bcfb99c802"},
- {file = "regex-2024.9.11-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:c57d08ad67aba97af57a7263c2d9006d5c404d721c5f7542f077f109ec2a4a29"},
- {file = "regex-2024.9.11-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:f8404bf61298bb6f8224bb9176c1424548ee1181130818fcd2cbffddc768bed8"},
- {file = "regex-2024.9.11-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:dd4490a33eb909ef5078ab20f5f000087afa2a4daa27b4c072ccb3cb3050ad84"},
- {file = "regex-2024.9.11-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:eee9130eaad130649fd73e5cd92f60e55708952260ede70da64de420cdcad554"},
- {file = "regex-2024.9.11-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6a2644a93da36c784e546de579ec1806bfd2763ef47babc1b03d765fe560c9f8"},
- {file = "regex-2024.9.11-cp313-cp313-win32.whl", hash = "sha256:e997fd30430c57138adc06bba4c7c2968fb13d101e57dd5bb9355bf8ce3fa7e8"},
- {file = "regex-2024.9.11-cp313-cp313-win_amd64.whl", hash = "sha256:042c55879cfeb21a8adacc84ea347721d3d83a159da6acdf1116859e2427c43f"},
- {file = "regex-2024.9.11-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:35f4a6f96aa6cb3f2f7247027b07b15a374f0d5b912c0001418d1d55024d5cb4"},
- {file = "regex-2024.9.11-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:55b96e7ce3a69a8449a66984c268062fbaa0d8ae437b285428e12797baefce7e"},
- {file = "regex-2024.9.11-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:cb130fccd1a37ed894824b8c046321540263013da72745d755f2d35114b81a60"},
- {file = "regex-2024.9.11-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:323c1f04be6b2968944d730e5c2091c8c89767903ecaa135203eec4565ed2b2b"},
- {file = "regex-2024.9.11-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:be1c8ed48c4c4065ecb19d882a0ce1afe0745dfad8ce48c49586b90a55f02366"},
- {file = "regex-2024.9.11-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b5b029322e6e7b94fff16cd120ab35a253236a5f99a79fb04fda7ae71ca20ae8"},
- {file = "regex-2024.9.11-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f6fff13ef6b5f29221d6904aa816c34701462956aa72a77f1f151a8ec4f56aeb"},
- {file = "regex-2024.9.11-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:587d4af3979376652010e400accc30404e6c16b7df574048ab1f581af82065e4"},
- {file = "regex-2024.9.11-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:079400a8269544b955ffa9e31f186f01d96829110a3bf79dc338e9910f794fca"},
- {file = "regex-2024.9.11-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:f9268774428ec173654985ce55fc6caf4c6d11ade0f6f914d48ef4719eb05ebb"},
- {file = "regex-2024.9.11-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:23f9985c8784e544d53fc2930fc1ac1a7319f5d5332d228437acc9f418f2f168"},
- {file = "regex-2024.9.11-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:ae2941333154baff9838e88aa71c1d84f4438189ecc6021a12c7573728b5838e"},
- {file = "regex-2024.9.11-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:e93f1c331ca8e86fe877a48ad64e77882c0c4da0097f2212873a69bbfea95d0c"},
- {file = "regex-2024.9.11-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:846bc79ee753acf93aef4184c040d709940c9d001029ceb7b7a52747b80ed2dd"},
- {file = "regex-2024.9.11-cp38-cp38-win32.whl", hash = "sha256:c94bb0a9f1db10a1d16c00880bdebd5f9faf267273b8f5bd1878126e0fbde771"},
- {file = "regex-2024.9.11-cp38-cp38-win_amd64.whl", hash = "sha256:2b08fce89fbd45664d3df6ad93e554b6c16933ffa9d55cb7e01182baaf971508"},
- {file = "regex-2024.9.11-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:07f45f287469039ffc2c53caf6803cd506eb5f5f637f1d4acb37a738f71dd066"},
- {file = "regex-2024.9.11-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4838e24ee015101d9f901988001038f7f0d90dc0c3b115541a1365fb439add62"},
- {file = "regex-2024.9.11-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:6edd623bae6a737f10ce853ea076f56f507fd7726bee96a41ee3d68d347e4d16"},
- {file = "regex-2024.9.11-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c69ada171c2d0e97a4b5aa78fbb835e0ffbb6b13fc5da968c09811346564f0d3"},
- {file = "regex-2024.9.11-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:02087ea0a03b4af1ed6ebab2c54d7118127fee8d71b26398e8e4b05b78963199"},
- {file = "regex-2024.9.11-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:69dee6a020693d12a3cf892aba4808fe168d2a4cef368eb9bf74f5398bfd4ee8"},
- {file = "regex-2024.9.11-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:297f54910247508e6e5cae669f2bc308985c60540a4edd1c77203ef19bfa63ca"},
- {file = "regex-2024.9.11-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ecea58b43a67b1b79805f1a0255730edaf5191ecef84dbc4cc85eb30bc8b63b9"},
- {file = "regex-2024.9.11-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:eab4bb380f15e189d1313195b062a6aa908f5bd687a0ceccd47c8211e9cf0d4a"},
- {file = "regex-2024.9.11-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:0cbff728659ce4bbf4c30b2a1be040faafaa9eca6ecde40aaff86f7889f4ab39"},
- {file = "regex-2024.9.11-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:54c4a097b8bc5bb0dfc83ae498061d53ad7b5762e00f4adaa23bee22b012e6ba"},
- {file = "regex-2024.9.11-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:73d6d2f64f4d894c96626a75578b0bf7d9e56dcda8c3d037a2118fdfe9b1c664"},
- {file = "regex-2024.9.11-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:e53b5fbab5d675aec9f0c501274c467c0f9a5d23696cfc94247e1fb56501ed89"},
- {file = "regex-2024.9.11-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:0ffbcf9221e04502fc35e54d1ce9567541979c3fdfb93d2c554f0ca583a19b35"},
- {file = "regex-2024.9.11-cp39-cp39-win32.whl", hash = "sha256:e4c22e1ac1f1ec1e09f72e6c44d8f2244173db7eb9629cc3a346a8d7ccc31142"},
- {file = "regex-2024.9.11-cp39-cp39-win_amd64.whl", hash = "sha256:faa3c142464efec496967359ca99696c896c591c56c53506bac1ad465f66e919"},
- {file = "regex-2024.9.11.tar.gz", hash = "sha256:6c188c307e8433bcb63dc1915022deb553b4203a70722fc542c363bf120a01fd"},
-]
-
-[[package]]
-name = "requests"
-version = "2.32.3"
-description = "Python HTTP for Humans."
-optional = false
-python-versions = ">=3.8"
-files = [
- {file = "requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"},
- {file = "requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760"},
-]
-
-[package.dependencies]
-certifi = ">=2017.4.17"
-charset-normalizer = ">=2,<4"
-idna = ">=2.5,<4"
-urllib3 = ">=1.21.1,<3"
-
-[package.extras]
-socks = ["PySocks (>=1.5.6,!=1.5.7)"]
-use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"]
-
-[[package]]
-name = "restrictedpython"
-version = "7.2"
-description = "RestrictedPython is a defined subset of the Python language which allows to provide a program input into a trusted environment."
-optional = false
-python-versions = "<3.13,>=3.7"
-files = [
- {file = "RestrictedPython-7.2-py3-none-any.whl", hash = "sha256:139cb41da6e57521745a566d05825f7a09e6a884f7fa922568cff0a70b84ce6b"},
- {file = "RestrictedPython-7.2.tar.gz", hash = "sha256:4d1d30f709a6621ca7c4236f08b67b732a651c8099145f137078c7dd4bed3d21"},
-]
-
-[package.extras]
-docs = ["Sphinx", "sphinx-rtd-theme"]
-test = ["pytest", "pytest-mock"]
-
-[[package]]
-name = "rich"
-version = "13.8.1"
-description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal"
-optional = false
-python-versions = ">=3.7.0"
-files = [
- {file = "rich-13.8.1-py3-none-any.whl", hash = "sha256:1760a3c0848469b97b558fc61c85233e3dafb69c7a071b4d60c38099d3cd4c06"},
- {file = "rich-13.8.1.tar.gz", hash = "sha256:8260cda28e3db6bf04d2d1ef4dbc03ba80a824c88b0e7668a0f23126a424844a"},
-]
-
-[package.dependencies]
-markdown-it-py = ">=2.2.0"
-pygments = ">=2.13.0,<3.0.0"
-
-[package.extras]
-jupyter = ["ipywidgets (>=7.5.1,<9)"]
-
-[[package]]
-name = "rpds-py"
-version = "0.20.0"
-description = "Python bindings to Rust's persistent data structures (rpds)"
-optional = false
-python-versions = ">=3.8"
-files = [
- {file = "rpds_py-0.20.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:3ad0fda1635f8439cde85c700f964b23ed5fc2d28016b32b9ee5fe30da5c84e2"},
- {file = "rpds_py-0.20.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9bb4a0d90fdb03437c109a17eade42dfbf6190408f29b2744114d11586611d6f"},
- {file = "rpds_py-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c6377e647bbfd0a0b159fe557f2c6c602c159fc752fa316572f012fc0bf67150"},
- {file = "rpds_py-0.20.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:eb851b7df9dda52dc1415ebee12362047ce771fc36914586b2e9fcbd7d293b3e"},
- {file = "rpds_py-0.20.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1e0f80b739e5a8f54837be5d5c924483996b603d5502bfff79bf33da06164ee2"},
- {file = "rpds_py-0.20.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5a8c94dad2e45324fc74dce25e1645d4d14df9a4e54a30fa0ae8bad9a63928e3"},
- {file = "rpds_py-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8e604fe73ba048c06085beaf51147eaec7df856824bfe7b98657cf436623daf"},
- {file = "rpds_py-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:df3de6b7726b52966edf29663e57306b23ef775faf0ac01a3e9f4012a24a4140"},
- {file = "rpds_py-0.20.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:cf258ede5bc22a45c8e726b29835b9303c285ab46fc7c3a4cc770736b5304c9f"},
- {file = "rpds_py-0.20.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:55fea87029cded5df854ca7e192ec7bdb7ecd1d9a3f63d5c4eb09148acf4a7ce"},
- {file = "rpds_py-0.20.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ae94bd0b2f02c28e199e9bc51485d0c5601f58780636185660f86bf80c89af94"},
- {file = "rpds_py-0.20.0-cp310-none-win32.whl", hash = "sha256:28527c685f237c05445efec62426d285e47a58fb05ba0090a4340b73ecda6dee"},
- {file = "rpds_py-0.20.0-cp310-none-win_amd64.whl", hash = "sha256:238a2d5b1cad28cdc6ed15faf93a998336eb041c4e440dd7f902528b8891b399"},
- {file = "rpds_py-0.20.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:ac2f4f7a98934c2ed6505aead07b979e6f999389f16b714448fb39bbaa86a489"},
- {file = "rpds_py-0.20.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:220002c1b846db9afd83371d08d239fdc865e8f8c5795bbaec20916a76db3318"},
- {file = "rpds_py-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8d7919548df3f25374a1f5d01fbcd38dacab338ef5f33e044744b5c36729c8db"},
- {file = "rpds_py-0.20.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:758406267907b3781beee0f0edfe4a179fbd97c0be2e9b1154d7f0a1279cf8e5"},
- {file = "rpds_py-0.20.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3d61339e9f84a3f0767b1995adfb171a0d00a1185192718a17af6e124728e0f5"},
- {file = "rpds_py-0.20.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1259c7b3705ac0a0bd38197565a5d603218591d3f6cee6e614e380b6ba61c6f6"},
- {file = "rpds_py-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5c1dc0f53856b9cc9a0ccca0a7cc61d3d20a7088201c0937f3f4048c1718a209"},
- {file = "rpds_py-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7e60cb630f674a31f0368ed32b2a6b4331b8350d67de53c0359992444b116dd3"},
- {file = "rpds_py-0.20.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:dbe982f38565bb50cb7fb061ebf762c2f254ca3d8c20d4006878766e84266272"},
- {file = "rpds_py-0.20.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:514b3293b64187172bc77c8fb0cdae26981618021053b30d8371c3a902d4d5ad"},
- {file = "rpds_py-0.20.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d0a26ffe9d4dd35e4dfdd1e71f46401cff0181c75ac174711ccff0459135fa58"},
- {file = "rpds_py-0.20.0-cp311-none-win32.whl", hash = "sha256:89c19a494bf3ad08c1da49445cc5d13d8fefc265f48ee7e7556839acdacf69d0"},
- {file = "rpds_py-0.20.0-cp311-none-win_amd64.whl", hash = "sha256:c638144ce971df84650d3ed0096e2ae7af8e62ecbbb7b201c8935c370df00a2c"},
- {file = "rpds_py-0.20.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a84ab91cbe7aab97f7446652d0ed37d35b68a465aeef8fc41932a9d7eee2c1a6"},
- {file = "rpds_py-0.20.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:56e27147a5a4c2c21633ff8475d185734c0e4befd1c989b5b95a5d0db699b21b"},
- {file = "rpds_py-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2580b0c34583b85efec8c5c5ec9edf2dfe817330cc882ee972ae650e7b5ef739"},
- {file = "rpds_py-0.20.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b80d4a7900cf6b66bb9cee5c352b2d708e29e5a37fe9bf784fa97fc11504bf6c"},
- {file = "rpds_py-0.20.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:50eccbf054e62a7b2209b28dc7a22d6254860209d6753e6b78cfaeb0075d7bee"},
- {file = "rpds_py-0.20.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:49a8063ea4296b3a7e81a5dfb8f7b2d73f0b1c20c2af401fb0cdf22e14711a96"},
- {file = "rpds_py-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ea438162a9fcbee3ecf36c23e6c68237479f89f962f82dae83dc15feeceb37e4"},
- {file = "rpds_py-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:18d7585c463087bddcfa74c2ba267339f14f2515158ac4db30b1f9cbdb62c8ef"},
- {file = "rpds_py-0.20.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d4c7d1a051eeb39f5c9547e82ea27cbcc28338482242e3e0b7768033cb083821"},
- {file = "rpds_py-0.20.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:e4df1e3b3bec320790f699890d41c59d250f6beda159ea3c44c3f5bac1976940"},
- {file = "rpds_py-0.20.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2cf126d33a91ee6eedc7f3197b53e87a2acdac63602c0f03a02dd69e4b138174"},
- {file = "rpds_py-0.20.0-cp312-none-win32.whl", hash = "sha256:8bc7690f7caee50b04a79bf017a8d020c1f48c2a1077ffe172abec59870f1139"},
- {file = "rpds_py-0.20.0-cp312-none-win_amd64.whl", hash = "sha256:0e13e6952ef264c40587d510ad676a988df19adea20444c2b295e536457bc585"},
- {file = "rpds_py-0.20.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:aa9a0521aeca7d4941499a73ad7d4f8ffa3d1affc50b9ea11d992cd7eff18a29"},
- {file = "rpds_py-0.20.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4a1f1d51eccb7e6c32ae89243cb352389228ea62f89cd80823ea7dd1b98e0b91"},
- {file = "rpds_py-0.20.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8a86a9b96070674fc88b6f9f71a97d2c1d3e5165574615d1f9168ecba4cecb24"},
- {file = "rpds_py-0.20.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6c8ef2ebf76df43f5750b46851ed1cdf8f109d7787ca40035fe19fbdc1acc5a7"},
- {file = "rpds_py-0.20.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b74b25f024b421d5859d156750ea9a65651793d51b76a2e9238c05c9d5f203a9"},
- {file = "rpds_py-0.20.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:57eb94a8c16ab08fef6404301c38318e2c5a32216bf5de453e2714c964c125c8"},
- {file = "rpds_py-0.20.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e1940dae14e715e2e02dfd5b0f64a52e8374a517a1e531ad9412319dc3ac7879"},
- {file = "rpds_py-0.20.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d20277fd62e1b992a50c43f13fbe13277a31f8c9f70d59759c88f644d66c619f"},
- {file = "rpds_py-0.20.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:06db23d43f26478303e954c34c75182356ca9aa7797d22c5345b16871ab9c45c"},
- {file = "rpds_py-0.20.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:b2a5db5397d82fa847e4c624b0c98fe59d2d9b7cf0ce6de09e4d2e80f8f5b3f2"},
- {file = "rpds_py-0.20.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5a35df9f5548fd79cb2f52d27182108c3e6641a4feb0f39067911bf2adaa3e57"},
- {file = "rpds_py-0.20.0-cp313-none-win32.whl", hash = "sha256:fd2d84f40633bc475ef2d5490b9c19543fbf18596dcb1b291e3a12ea5d722f7a"},
- {file = "rpds_py-0.20.0-cp313-none-win_amd64.whl", hash = "sha256:9bc2d153989e3216b0559251b0c260cfd168ec78b1fac33dd485750a228db5a2"},
- {file = "rpds_py-0.20.0-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:f2fbf7db2012d4876fb0d66b5b9ba6591197b0f165db8d99371d976546472a24"},
- {file = "rpds_py-0.20.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:1e5f3cd7397c8f86c8cc72d5a791071431c108edd79872cdd96e00abd8497d29"},
- {file = "rpds_py-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce9845054c13696f7af7f2b353e6b4f676dab1b4b215d7fe5e05c6f8bb06f965"},
- {file = "rpds_py-0.20.0-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c3e130fd0ec56cb76eb49ef52faead8ff09d13f4527e9b0c400307ff72b408e1"},
- {file = "rpds_py-0.20.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4b16aa0107ecb512b568244ef461f27697164d9a68d8b35090e9b0c1c8b27752"},
- {file = "rpds_py-0.20.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:aa7f429242aae2947246587d2964fad750b79e8c233a2367f71b554e9447949c"},
- {file = "rpds_py-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:af0fc424a5842a11e28956e69395fbbeab2c97c42253169d87e90aac2886d751"},
- {file = "rpds_py-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b8c00a3b1e70c1d3891f0db1b05292747f0dbcfb49c43f9244d04c70fbc40eb8"},
- {file = "rpds_py-0.20.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:40ce74fc86ee4645d0a225498d091d8bc61f39b709ebef8204cb8b5a464d3c0e"},
- {file = "rpds_py-0.20.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:4fe84294c7019456e56d93e8ababdad5a329cd25975be749c3f5f558abb48253"},
- {file = "rpds_py-0.20.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:338ca4539aad4ce70a656e5187a3a31c5204f261aef9f6ab50e50bcdffaf050a"},
- {file = "rpds_py-0.20.0-cp38-none-win32.whl", hash = "sha256:54b43a2b07db18314669092bb2de584524d1ef414588780261e31e85846c26a5"},
- {file = "rpds_py-0.20.0-cp38-none-win_amd64.whl", hash = "sha256:a1862d2d7ce1674cffa6d186d53ca95c6e17ed2b06b3f4c476173565c862d232"},
- {file = "rpds_py-0.20.0-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:3fde368e9140312b6e8b6c09fb9f8c8c2f00999d1823403ae90cc00480221b22"},
- {file = "rpds_py-0.20.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9824fb430c9cf9af743cf7aaf6707bf14323fb51ee74425c380f4c846ea70789"},
- {file = "rpds_py-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:11ef6ce74616342888b69878d45e9f779b95d4bd48b382a229fe624a409b72c5"},
- {file = "rpds_py-0.20.0-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c52d3f2f82b763a24ef52f5d24358553e8403ce05f893b5347098014f2d9eff2"},
- {file = "rpds_py-0.20.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9d35cef91e59ebbeaa45214861874bc6f19eb35de96db73e467a8358d701a96c"},
- {file = "rpds_py-0.20.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d72278a30111e5b5525c1dd96120d9e958464316f55adb030433ea905866f4de"},
- {file = "rpds_py-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b4c29cbbba378759ac5786730d1c3cb4ec6f8ababf5c42a9ce303dc4b3d08cda"},
- {file = "rpds_py-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6632f2d04f15d1bd6fe0eedd3b86d9061b836ddca4c03d5cf5c7e9e6b7c14580"},
- {file = "rpds_py-0.20.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:d0b67d87bb45ed1cd020e8fbf2307d449b68abc45402fe1a4ac9e46c3c8b192b"},
- {file = "rpds_py-0.20.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:ec31a99ca63bf3cd7f1a5ac9fe95c5e2d060d3c768a09bc1d16e235840861420"},
- {file = "rpds_py-0.20.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:22e6c9976e38f4d8c4a63bd8a8edac5307dffd3ee7e6026d97f3cc3a2dc02a0b"},
- {file = "rpds_py-0.20.0-cp39-none-win32.whl", hash = "sha256:569b3ea770c2717b730b61998b6c54996adee3cef69fc28d444f3e7920313cf7"},
- {file = "rpds_py-0.20.0-cp39-none-win_amd64.whl", hash = "sha256:e6900ecdd50ce0facf703f7a00df12374b74bbc8ad9fe0f6559947fb20f82364"},
- {file = "rpds_py-0.20.0-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:617c7357272c67696fd052811e352ac54ed1d9b49ab370261a80d3b6ce385045"},
- {file = "rpds_py-0.20.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:9426133526f69fcaba6e42146b4e12d6bc6c839b8b555097020e2b78ce908dcc"},
- {file = "rpds_py-0.20.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:deb62214c42a261cb3eb04d474f7155279c1a8a8c30ac89b7dcb1721d92c3c02"},
- {file = "rpds_py-0.20.0-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fcaeb7b57f1a1e071ebd748984359fef83ecb026325b9d4ca847c95bc7311c92"},
- {file = "rpds_py-0.20.0-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d454b8749b4bd70dd0a79f428731ee263fa6995f83ccb8bada706e8d1d3ff89d"},
- {file = "rpds_py-0.20.0-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d807dc2051abe041b6649681dce568f8e10668e3c1c6543ebae58f2d7e617855"},
- {file = "rpds_py-0.20.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c3c20f0ddeb6e29126d45f89206b8291352b8c5b44384e78a6499d68b52ae511"},
- {file = "rpds_py-0.20.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b7f19250ceef892adf27f0399b9e5afad019288e9be756d6919cb58892129f51"},
- {file = "rpds_py-0.20.0-pp310-pypy310_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:4f1ed4749a08379555cebf4650453f14452eaa9c43d0a95c49db50c18b7da075"},
- {file = "rpds_py-0.20.0-pp310-pypy310_pp73-musllinux_1_2_i686.whl", hash = "sha256:dcedf0b42bcb4cfff4101d7771a10532415a6106062f005ab97d1d0ab5681c60"},
- {file = "rpds_py-0.20.0-pp310-pypy310_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:39ed0d010457a78f54090fafb5d108501b5aa5604cc22408fc1c0c77eac14344"},
- {file = "rpds_py-0.20.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:bb273176be34a746bdac0b0d7e4e2c467323d13640b736c4c477881a3220a989"},
- {file = "rpds_py-0.20.0-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:f918a1a130a6dfe1d7fe0f105064141342e7dd1611f2e6a21cd2f5c8cb1cfb3e"},
- {file = "rpds_py-0.20.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:f60012a73aa396be721558caa3a6fd49b3dd0033d1675c6d59c4502e870fcf0c"},
- {file = "rpds_py-0.20.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3d2b1ad682a3dfda2a4e8ad8572f3100f95fad98cb99faf37ff0ddfe9cbf9d03"},
- {file = "rpds_py-0.20.0-pp39-pypy39_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:614fdafe9f5f19c63ea02817fa4861c606a59a604a77c8cdef5aa01d28b97921"},
- {file = "rpds_py-0.20.0-pp39-pypy39_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fa518bcd7600c584bf42e6617ee8132869e877db2f76bcdc281ec6a4113a53ab"},
- {file = "rpds_py-0.20.0-pp39-pypy39_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f0475242f447cc6cb8a9dd486d68b2ef7fbee84427124c232bff5f63b1fe11e5"},
- {file = "rpds_py-0.20.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f90a4cd061914a60bd51c68bcb4357086991bd0bb93d8aa66a6da7701370708f"},
- {file = "rpds_py-0.20.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:def7400461c3a3f26e49078302e1c1b38f6752342c77e3cf72ce91ca69fb1bc1"},
- {file = "rpds_py-0.20.0-pp39-pypy39_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:65794e4048ee837494aea3c21a28ad5fc080994dfba5b036cf84de37f7ad5074"},
- {file = "rpds_py-0.20.0-pp39-pypy39_pp73-musllinux_1_2_i686.whl", hash = "sha256:faefcc78f53a88f3076b7f8be0a8f8d35133a3ecf7f3770895c25f8813460f08"},
- {file = "rpds_py-0.20.0-pp39-pypy39_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:5b4f105deeffa28bbcdff6c49b34e74903139afa690e35d2d9e3c2c2fba18cec"},
- {file = "rpds_py-0.20.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:fdfc3a892927458d98f3d55428ae46b921d1f7543b89382fdb483f5640daaec8"},
- {file = "rpds_py-0.20.0.tar.gz", hash = "sha256:d72a210824facfdaf8768cf2d7ca25a042c30320b3020de2fa04640920d4e121"},
-]
-
-[[package]]
-name = "sgmllib3k"
-version = "1.0.0"
-description = "Py3k port of sgmllib."
-optional = false
-python-versions = "*"
-files = [
- {file = "sgmllib3k-1.0.0.tar.gz", hash = "sha256:7868fb1c8bfa764c1ac563d3cf369c381d1325d36124933a726f29fcdaa812e9"},
-]
-
-[[package]]
-name = "six"
-version = "1.16.0"
-description = "Python 2 and 3 compatibility utilities"
-optional = false
-python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*"
-files = [
- {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"},
- {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"},
-]
-
-[[package]]
-name = "sniffio"
-version = "1.3.1"
-description = "Sniff out which async library your code is running under"
-optional = false
-python-versions = ">=3.7"
-files = [
- {file = "sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2"},
- {file = "sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc"},
-]
-
-[[package]]
-name = "sqlalchemy"
-version = "2.0.35"
-description = "Database Abstraction Library"
-optional = false
-python-versions = ">=3.7"
-files = [
- {file = "SQLAlchemy-2.0.35-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:67219632be22f14750f0d1c70e62f204ba69d28f62fd6432ba05ab295853de9b"},
- {file = "SQLAlchemy-2.0.35-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4668bd8faf7e5b71c0319407b608f278f279668f358857dbfd10ef1954ac9f90"},
- {file = "SQLAlchemy-2.0.35-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb8bea573863762bbf45d1e13f87c2d2fd32cee2dbd50d050f83f87429c9e1ea"},
- {file = "SQLAlchemy-2.0.35-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f552023710d4b93d8fb29a91fadf97de89c5926c6bd758897875435f2a939f33"},
- {file = "SQLAlchemy-2.0.35-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:016b2e665f778f13d3c438651dd4de244214b527a275e0acf1d44c05bc6026a9"},
- {file = "SQLAlchemy-2.0.35-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:7befc148de64b6060937231cbff8d01ccf0bfd75aa26383ffdf8d82b12ec04ff"},
- {file = "SQLAlchemy-2.0.35-cp310-cp310-win32.whl", hash = "sha256:22b83aed390e3099584b839b93f80a0f4a95ee7f48270c97c90acd40ee646f0b"},
- {file = "SQLAlchemy-2.0.35-cp310-cp310-win_amd64.whl", hash = "sha256:a29762cd3d116585278ffb2e5b8cc311fb095ea278b96feef28d0b423154858e"},
- {file = "SQLAlchemy-2.0.35-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:e21f66748ab725ade40fa7af8ec8b5019c68ab00b929f6643e1b1af461eddb60"},
- {file = "SQLAlchemy-2.0.35-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8a6219108a15fc6d24de499d0d515c7235c617b2540d97116b663dade1a54d62"},
- {file = "SQLAlchemy-2.0.35-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:042622a5306c23b972192283f4e22372da3b8ddf5f7aac1cc5d9c9b222ab3ff6"},
- {file = "SQLAlchemy-2.0.35-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:627dee0c280eea91aed87b20a1f849e9ae2fe719d52cbf847c0e0ea34464b3f7"},
- {file = "SQLAlchemy-2.0.35-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4fdcd72a789c1c31ed242fd8c1bcd9ea186a98ee8e5408a50e610edfef980d71"},
- {file = "SQLAlchemy-2.0.35-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:89b64cd8898a3a6f642db4eb7b26d1b28a497d4022eccd7717ca066823e9fb01"},
- {file = "SQLAlchemy-2.0.35-cp311-cp311-win32.whl", hash = "sha256:6a93c5a0dfe8d34951e8a6f499a9479ffb9258123551fa007fc708ae2ac2bc5e"},
- {file = "SQLAlchemy-2.0.35-cp311-cp311-win_amd64.whl", hash = "sha256:c68fe3fcde03920c46697585620135b4ecfdfc1ed23e75cc2c2ae9f8502c10b8"},
- {file = "SQLAlchemy-2.0.35-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:eb60b026d8ad0c97917cb81d3662d0b39b8ff1335e3fabb24984c6acd0c900a2"},
- {file = "SQLAlchemy-2.0.35-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6921ee01caf375363be5e9ae70d08ce7ca9d7e0e8983183080211a062d299468"},
- {file = "SQLAlchemy-2.0.35-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8cdf1a0dbe5ced887a9b127da4ffd7354e9c1a3b9bb330dce84df6b70ccb3a8d"},
- {file = "SQLAlchemy-2.0.35-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:93a71c8601e823236ac0e5d087e4f397874a421017b3318fd92c0b14acf2b6db"},
- {file = "SQLAlchemy-2.0.35-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e04b622bb8a88f10e439084486f2f6349bf4d50605ac3e445869c7ea5cf0fa8c"},
- {file = "SQLAlchemy-2.0.35-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1b56961e2d31389aaadf4906d453859f35302b4eb818d34a26fab72596076bb8"},
- {file = "SQLAlchemy-2.0.35-cp312-cp312-win32.whl", hash = "sha256:0f9f3f9a3763b9c4deb8c5d09c4cc52ffe49f9876af41cc1b2ad0138878453cf"},
- {file = "SQLAlchemy-2.0.35-cp312-cp312-win_amd64.whl", hash = "sha256:25b0f63e7fcc2a6290cb5f7f5b4fc4047843504983a28856ce9b35d8f7de03cc"},
- {file = "SQLAlchemy-2.0.35-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:f021d334f2ca692523aaf7bbf7592ceff70c8594fad853416a81d66b35e3abf9"},
- {file = "SQLAlchemy-2.0.35-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:05c3f58cf91683102f2f0265c0db3bd3892e9eedabe059720492dbaa4f922da1"},
- {file = "SQLAlchemy-2.0.35-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:032d979ce77a6c2432653322ba4cbeabf5a6837f704d16fa38b5a05d8e21fa00"},
- {file = "SQLAlchemy-2.0.35-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:2e795c2f7d7249b75bb5f479b432a51b59041580d20599d4e112b5f2046437a3"},
- {file = "SQLAlchemy-2.0.35-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:cc32b2990fc34380ec2f6195f33a76b6cdaa9eecf09f0c9404b74fc120aef36f"},
- {file = "SQLAlchemy-2.0.35-cp37-cp37m-win32.whl", hash = "sha256:9509c4123491d0e63fb5e16199e09f8e262066e58903e84615c301dde8fa2e87"},
- {file = "SQLAlchemy-2.0.35-cp37-cp37m-win_amd64.whl", hash = "sha256:3655af10ebcc0f1e4e06c5900bb33e080d6a1fa4228f502121f28a3b1753cde5"},
- {file = "SQLAlchemy-2.0.35-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:4c31943b61ed8fdd63dfd12ccc919f2bf95eefca133767db6fbbd15da62078ec"},
- {file = "SQLAlchemy-2.0.35-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:a62dd5d7cc8626a3634208df458c5fe4f21200d96a74d122c83bc2015b333bc1"},
- {file = "SQLAlchemy-2.0.35-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0630774b0977804fba4b6bbea6852ab56c14965a2b0c7fc7282c5f7d90a1ae72"},
- {file = "SQLAlchemy-2.0.35-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d625eddf7efeba2abfd9c014a22c0f6b3796e0ffb48f5d5ab106568ef01ff5a"},
- {file = "SQLAlchemy-2.0.35-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:ada603db10bb865bbe591939de854faf2c60f43c9b763e90f653224138f910d9"},
- {file = "SQLAlchemy-2.0.35-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:c41411e192f8d3ea39ea70e0fae48762cd11a2244e03751a98bd3c0ca9a4e936"},
- {file = "SQLAlchemy-2.0.35-cp38-cp38-win32.whl", hash = "sha256:d299797d75cd747e7797b1b41817111406b8b10a4f88b6e8fe5b5e59598b43b0"},
- {file = "SQLAlchemy-2.0.35-cp38-cp38-win_amd64.whl", hash = "sha256:0375a141e1c0878103eb3d719eb6d5aa444b490c96f3fedab8471c7f6ffe70ee"},
- {file = "SQLAlchemy-2.0.35-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ccae5de2a0140d8be6838c331604f91d6fafd0735dbdcee1ac78fc8fbaba76b4"},
- {file = "SQLAlchemy-2.0.35-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2a275a806f73e849e1c309ac11108ea1a14cd7058577aba962cd7190e27c9e3c"},
- {file = "SQLAlchemy-2.0.35-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:732e026240cdd1c1b2e3ac515c7a23820430ed94292ce33806a95869c46bd139"},
- {file = "SQLAlchemy-2.0.35-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:890da8cd1941fa3dab28c5bac3b9da8502e7e366f895b3b8e500896f12f94d11"},
- {file = "SQLAlchemy-2.0.35-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:c0d8326269dbf944b9201911b0d9f3dc524d64779a07518199a58384c3d37a44"},
- {file = "SQLAlchemy-2.0.35-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:b76d63495b0508ab9fc23f8152bac63205d2a704cd009a2b0722f4c8e0cba8e0"},
- {file = "SQLAlchemy-2.0.35-cp39-cp39-win32.whl", hash = "sha256:69683e02e8a9de37f17985905a5eca18ad651bf592314b4d3d799029797d0eb3"},
- {file = "SQLAlchemy-2.0.35-cp39-cp39-win_amd64.whl", hash = "sha256:aee110e4ef3c528f3abbc3c2018c121e708938adeeff9006428dd7c8555e9b3f"},
- {file = "SQLAlchemy-2.0.35-py3-none-any.whl", hash = "sha256:2ab3f0336c0387662ce6221ad30ab3a5e6499aab01b9790879b6578fd9b8faa1"},
- {file = "sqlalchemy-2.0.35.tar.gz", hash = "sha256:e11d7ea4d24f0a262bccf9a7cd6284c976c5369dac21db237cff59586045ab9f"},
-]
-
-[package.dependencies]
-greenlet = {version = "!=0.4.17", optional = true, markers = "python_version < \"3.13\" and (platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\") or extra == \"asyncio\""}
-typing-extensions = ">=4.6.0"
-
-[package.extras]
-aiomysql = ["aiomysql (>=0.2.0)", "greenlet (!=0.4.17)"]
-aioodbc = ["aioodbc", "greenlet (!=0.4.17)"]
-aiosqlite = ["aiosqlite", "greenlet (!=0.4.17)", "typing_extensions (!=3.10.0.1)"]
-asyncio = ["greenlet (!=0.4.17)"]
-asyncmy = ["asyncmy (>=0.2.3,!=0.2.4,!=0.2.6)", "greenlet (!=0.4.17)"]
-mariadb-connector = ["mariadb (>=1.0.1,!=1.1.2,!=1.1.5)"]
-mssql = ["pyodbc"]
-mssql-pymssql = ["pymssql"]
-mssql-pyodbc = ["pyodbc"]
-mypy = ["mypy (>=0.910)"]
-mysql = ["mysqlclient (>=1.4.0)"]
-mysql-connector = ["mysql-connector-python"]
-oracle = ["cx_oracle (>=8)"]
-oracle-oracledb = ["oracledb (>=1.0.1)"]
-postgresql = ["psycopg2 (>=2.7)"]
-postgresql-asyncpg = ["asyncpg", "greenlet (!=0.4.17)"]
-postgresql-pg8000 = ["pg8000 (>=1.29.1)"]
-postgresql-psycopg = ["psycopg (>=3.0.7)"]
-postgresql-psycopg2binary = ["psycopg2-binary"]
-postgresql-psycopg2cffi = ["psycopg2cffi"]
-postgresql-psycopgbinary = ["psycopg[binary] (>=3.0.7)"]
-pymysql = ["pymysql"]
-sqlcipher = ["sqlcipher3_binary"]
-
-[[package]]
-name = "sympy"
-version = "1.13.3"
-description = "Computer algebra system (CAS) in Python"
-optional = false
-python-versions = ">=3.8"
-files = [
- {file = "sympy-1.13.3-py3-none-any.whl", hash = "sha256:54612cf55a62755ee71824ce692986f23c88ffa77207b30c1368eda4a7060f73"},
- {file = "sympy-1.13.3.tar.gz", hash = "sha256:b27fd2c6530e0ab39e275fc9b683895367e51d5da91baa8d3d64db2565fec4d9"},
-]
-
-[package.dependencies]
-mpmath = ">=1.1.0,<1.4"
-
-[package.extras]
-dev = ["hypothesis (>=6.70.0)", "pytest (>=7.1.0)"]
-
-[[package]]
-name = "tenacity"
-version = "8.5.0"
-description = "Retry code until it succeeds"
-optional = false
-python-versions = ">=3.8"
-files = [
- {file = "tenacity-8.5.0-py3-none-any.whl", hash = "sha256:b594c2a5945830c267ce6b79a166228323ed52718f30302c1359836112346687"},
- {file = "tenacity-8.5.0.tar.gz", hash = "sha256:8bc6c0c8a09b31e6cad13c47afbed1a567518250a9a171418582ed8d9c20ca78"},
-]
-
-[package.extras]
-doc = ["reno", "sphinx"]
-test = ["pytest", "tornado (>=4.5)", "typeguard"]
-
-[[package]]
-name = "tiktoken"
-version = "0.7.0"
-description = "tiktoken is a fast BPE tokeniser for use with OpenAI's models"
-optional = false
-python-versions = ">=3.8"
-files = [
- {file = "tiktoken-0.7.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:485f3cc6aba7c6b6ce388ba634fbba656d9ee27f766216f45146beb4ac18b25f"},
- {file = "tiktoken-0.7.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e54be9a2cd2f6d6ffa3517b064983fb695c9a9d8aa7d574d1ef3c3f931a99225"},
- {file = "tiktoken-0.7.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:79383a6e2c654c6040e5f8506f3750db9ddd71b550c724e673203b4f6b4b4590"},
- {file = "tiktoken-0.7.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d4511c52caacf3c4981d1ae2df85908bd31853f33d30b345c8b6830763f769c"},
- {file = "tiktoken-0.7.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:13c94efacdd3de9aff824a788353aa5749c0faee1fbe3816df365ea450b82311"},
- {file = "tiktoken-0.7.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8e58c7eb29d2ab35a7a8929cbeea60216a4ccdf42efa8974d8e176d50c9a3df5"},
- {file = "tiktoken-0.7.0-cp310-cp310-win_amd64.whl", hash = "sha256:21a20c3bd1dd3e55b91c1331bf25f4af522c525e771691adbc9a69336fa7f702"},
- {file = "tiktoken-0.7.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:10c7674f81e6e350fcbed7c09a65bca9356eaab27fb2dac65a1e440f2bcfe30f"},
- {file = "tiktoken-0.7.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:084cec29713bc9d4189a937f8a35dbdfa785bd1235a34c1124fe2323821ee93f"},
- {file = "tiktoken-0.7.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:811229fde1652fedcca7c6dfe76724d0908775b353556d8a71ed74d866f73f7b"},
- {file = "tiktoken-0.7.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:86b6e7dc2e7ad1b3757e8a24597415bafcfb454cebf9a33a01f2e6ba2e663992"},
- {file = "tiktoken-0.7.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1063c5748be36344c7e18c7913c53e2cca116764c2080177e57d62c7ad4576d1"},
- {file = "tiktoken-0.7.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:20295d21419bfcca092644f7e2f2138ff947a6eb8cfc732c09cc7d76988d4a89"},
- {file = "tiktoken-0.7.0-cp311-cp311-win_amd64.whl", hash = "sha256:959d993749b083acc57a317cbc643fb85c014d055b2119b739487288f4e5d1cb"},
- {file = "tiktoken-0.7.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:71c55d066388c55a9c00f61d2c456a6086673ab7dec22dd739c23f77195b1908"},
- {file = "tiktoken-0.7.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:09ed925bccaa8043e34c519fbb2f99110bd07c6fd67714793c21ac298e449410"},
- {file = "tiktoken-0.7.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:03c6c40ff1db0f48a7b4d2dafeae73a5607aacb472fa11f125e7baf9dce73704"},
- {file = "tiktoken-0.7.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d20b5c6af30e621b4aca094ee61777a44118f52d886dbe4f02b70dfe05c15350"},
- {file = "tiktoken-0.7.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d427614c3e074004efa2f2411e16c826f9df427d3c70a54725cae860f09e4bf4"},
- {file = "tiktoken-0.7.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8c46d7af7b8c6987fac9b9f61041b452afe92eb087d29c9ce54951280f899a97"},
- {file = "tiktoken-0.7.0-cp312-cp312-win_amd64.whl", hash = "sha256:0bc603c30b9e371e7c4c7935aba02af5994a909fc3c0fe66e7004070858d3f8f"},
- {file = "tiktoken-0.7.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2398fecd38c921bcd68418675a6d155fad5f5e14c2e92fcf5fe566fa5485a858"},
- {file = "tiktoken-0.7.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:8f5f6afb52fb8a7ea1c811e435e4188f2bef81b5e0f7a8635cc79b0eef0193d6"},
- {file = "tiktoken-0.7.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:861f9ee616766d736be4147abac500732b505bf7013cfaf019b85892637f235e"},
- {file = "tiktoken-0.7.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:54031f95c6939f6b78122c0aa03a93273a96365103793a22e1793ee86da31685"},
- {file = "tiktoken-0.7.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:fffdcb319b614cf14f04d02a52e26b1d1ae14a570f90e9b55461a72672f7b13d"},
- {file = "tiktoken-0.7.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:c72baaeaefa03ff9ba9688624143c858d1f6b755bb85d456d59e529e17234769"},
- {file = "tiktoken-0.7.0-cp38-cp38-win_amd64.whl", hash = "sha256:131b8aeb043a8f112aad9f46011dced25d62629091e51d9dc1adbf4a1cc6aa98"},
- {file = "tiktoken-0.7.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:cabc6dc77460df44ec5b879e68692c63551ae4fae7460dd4ff17181df75f1db7"},
- {file = "tiktoken-0.7.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8d57f29171255f74c0aeacd0651e29aa47dff6f070cb9f35ebc14c82278f3b25"},
- {file = "tiktoken-0.7.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2ee92776fdbb3efa02a83f968c19d4997a55c8e9ce7be821ceee04a1d1ee149c"},
- {file = "tiktoken-0.7.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e215292e99cb41fbc96988ef62ea63bb0ce1e15f2c147a61acc319f8b4cbe5bf"},
- {file = "tiktoken-0.7.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:8a81bac94769cab437dd3ab0b8a4bc4e0f9cf6835bcaa88de71f39af1791727a"},
- {file = "tiktoken-0.7.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:d6d73ea93e91d5ca771256dfc9d1d29f5a554b83821a1dc0891987636e0ae226"},
- {file = "tiktoken-0.7.0-cp39-cp39-win_amd64.whl", hash = "sha256:2bcb28ddf79ffa424f171dfeef9a4daff61a94c631ca6813f43967cb263b83b9"},
- {file = "tiktoken-0.7.0.tar.gz", hash = "sha256:1077266e949c24e0291f6c350433c6f0971365ece2b173a23bc3b9f9defef6b6"},
-]
-
-[package.dependencies]
-regex = ">=2022.1.18"
-requests = ">=2.26.0"
-
-[package.extras]
-blobfile = ["blobfile (>=2)"]
-
-[[package]]
-name = "tokenizers"
-version = "0.20.0"
-description = ""
-optional = false
-python-versions = ">=3.7"
-files = [
- {file = "tokenizers-0.20.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:6cff5c5e37c41bc5faa519d6f3df0679e4b37da54ea1f42121719c5e2b4905c0"},
- {file = "tokenizers-0.20.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:62a56bf75c27443432456f4ca5ca055befa95e25be8a28141cc495cac8ae4d6d"},
- {file = "tokenizers-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68cc7de6a63f09c4a86909c2597b995aa66e19df852a23aea894929c74369929"},
- {file = "tokenizers-0.20.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:053c37ecee482cc958fdee53af3c6534286a86f5d35aac476f7c246830e53ae5"},
- {file = "tokenizers-0.20.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3d7074aaabc151a6363fa03db5493fc95b423b2a1874456783989e96d541c7b6"},
- {file = "tokenizers-0.20.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a11435780f2acd89e8fefe5e81cecf01776f6edb9b3ac95bcb76baee76b30b90"},
- {file = "tokenizers-0.20.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9a81cd2712973b007d84268d45fc3f6f90a79c31dfe7f1925e6732f8d2959987"},
- {file = "tokenizers-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d7dfd796ab9d909f76fb93080e1c7c8309f196ecb316eb130718cd5e34231c69"},
- {file = "tokenizers-0.20.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:8029ad2aa8cb00605c9374566034c1cc1b15130713e0eb5afcef6cface8255c9"},
- {file = "tokenizers-0.20.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ca4d54260ebe97d59dfa9a30baa20d0c4dd9137d99a8801700055c561145c24e"},
- {file = "tokenizers-0.20.0-cp310-none-win32.whl", hash = "sha256:95ee16b57cec11b86a7940174ec5197d506439b0f415ab3859f254b1dffe9df0"},
- {file = "tokenizers-0.20.0-cp310-none-win_amd64.whl", hash = "sha256:0a61a11e93eeadbf02aea082ffc75241c4198e0608bbbac4f65a9026851dcf37"},
- {file = "tokenizers-0.20.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:6636b798b3c4d6c9b1af1a918bd07c867808e5a21c64324e95318a237e6366c3"},
- {file = "tokenizers-0.20.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5ec603e42eaf499ffd58b9258162add948717cf21372458132f14e13a6bc7172"},
- {file = "tokenizers-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cce124264903a8ea6f8f48e1cc7669e5ef638c18bd4ab0a88769d5f92debdf7f"},
- {file = "tokenizers-0.20.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:07bbeba0231cf8de07aa6b9e33e9779ff103d47042eeeb859a8c432e3292fb98"},
- {file = "tokenizers-0.20.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:06c0ca8397b35d38b83a44a9c6929790c1692957d88541df061cb34d82ebbf08"},
- {file = "tokenizers-0.20.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ca6557ac3b83d912dfbb1f70ab56bd4b0594043916688e906ede09f42e192401"},
- {file = "tokenizers-0.20.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2a5ad94c9e80ac6098328bee2e3264dbced4c6faa34429994d473f795ec58ef4"},
- {file = "tokenizers-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0b5c7f906ee6bec30a9dc20268a8b80f3b9584de1c9f051671cb057dc6ce28f6"},
- {file = "tokenizers-0.20.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:31e087e9ee1b8f075b002bfee257e858dc695f955b43903e1bb4aa9f170e37fe"},
- {file = "tokenizers-0.20.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c3124fb6f3346cb3d8d775375d3b429bf4dcfc24f739822702009d20a4297990"},
- {file = "tokenizers-0.20.0-cp311-none-win32.whl", hash = "sha256:a4bb8b40ba9eefa621fdcabf04a74aa6038ae3be0c614c6458bd91a4697a452f"},
- {file = "tokenizers-0.20.0-cp311-none-win_amd64.whl", hash = "sha256:2b709d371f1fe60a28ef0c5c67815952d455ca7f34dbe7197eaaed3cc54b658e"},
- {file = "tokenizers-0.20.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:15c81a17d0d66f4987c6ca16f4bea7ec253b8c7ed1bb00fdc5d038b1bb56e714"},
- {file = "tokenizers-0.20.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6a531cdf1fb6dc41c984c785a3b299cb0586de0b35683842a3afbb1e5207f910"},
- {file = "tokenizers-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:06caabeb4587f8404e0cd9d40f458e9cba3e815c8155a38e579a74ff3e2a4301"},
- {file = "tokenizers-0.20.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8768f964f23f5b9f50546c0369c75ab3262de926983888bbe8b98be05392a79c"},
- {file = "tokenizers-0.20.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:626403860152c816f97b649fd279bd622c3d417678c93b4b1a8909b6380b69a8"},
- {file = "tokenizers-0.20.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9c1b88fa9e5ff062326f4bf82681da5a96fca7104d921a6bd7b1e6fcf224af26"},
- {file = "tokenizers-0.20.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3d7e559436a07dc547f22ce1101f26d8b2fad387e28ec8e7e1e3b11695d681d8"},
- {file = "tokenizers-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e48afb75e50449848964e4a67b0da01261dd3aa8df8daecf10db8fd7f5b076eb"},
- {file = "tokenizers-0.20.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:baf5d0e1ff44710a95eefc196dd87666ffc609fd447c5e5b68272a7c3d342a1d"},
- {file = "tokenizers-0.20.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:e5e56df0e8ed23ba60ae3848c3f069a0710c4b197218fe4f89e27eba38510768"},
- {file = "tokenizers-0.20.0-cp312-none-win32.whl", hash = "sha256:ec53e5ecc142a82432f9c6c677dbbe5a2bfee92b8abf409a9ecb0d425ee0ce75"},
- {file = "tokenizers-0.20.0-cp312-none-win_amd64.whl", hash = "sha256:f18661ece72e39c0dfaa174d6223248a15b457dbd4b0fc07809b8e6d3ca1a234"},
- {file = "tokenizers-0.20.0-cp37-cp37m-macosx_10_12_x86_64.whl", hash = "sha256:f7065b1084d8d1a03dc89d9aad69bcbc8415d4bc123c367063eb32958cd85054"},
- {file = "tokenizers-0.20.0-cp37-cp37m-macosx_11_0_arm64.whl", hash = "sha256:e5d4069e4714e3f7ba0a4d3d44f9d84a432cd4e4aa85c3d7dd1f51440f12e4a1"},
- {file = "tokenizers-0.20.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:799b808529e54b7e1a36350bda2aeb470e8390e484d3e98c10395cee61d4e3c6"},
- {file = "tokenizers-0.20.0-cp37-cp37m-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7f9baa027cc8a281ad5f7725a93c204d7a46986f88edbe8ef7357f40a23fb9c7"},
- {file = "tokenizers-0.20.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:010ec7f3f7a96adc4c2a34a3ada41fa14b4b936b5628b4ff7b33791258646c6b"},
- {file = "tokenizers-0.20.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:98d88f06155335b14fd78e32ee28ca5b2eb30fced4614e06eb14ae5f7fba24ed"},
- {file = "tokenizers-0.20.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e13eb000ef540c2280758d1b9cfa5fe424b0424ae4458f440e6340a4f18b2638"},
- {file = "tokenizers-0.20.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fab3cf066ff426f7e6d70435dc28a9ff01b2747be83810e397cba106f39430b0"},
- {file = "tokenizers-0.20.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:39fa3761b30a89368f322e5daf4130dce8495b79ad831f370449cdacfb0c0d37"},
- {file = "tokenizers-0.20.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:c8da0fba4d179ddf2607821575998df3c294aa59aa8df5a6646dc64bc7352bce"},
- {file = "tokenizers-0.20.0-cp37-none-win32.whl", hash = "sha256:fada996d6da8cf213f6e3c91c12297ad4f6cdf7a85c2fadcd05ec32fa6846fcd"},
- {file = "tokenizers-0.20.0-cp37-none-win_amd64.whl", hash = "sha256:7d29aad702279e0760c265fcae832e89349078e3418dd329732d4503259fd6bd"},
- {file = "tokenizers-0.20.0-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:099c68207f3ef0227ecb6f80ab98ea74de559f7b124adc7b17778af0250ee90a"},
- {file = "tokenizers-0.20.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:68012d8a8cddb2eab3880870d7e2086cb359c7f7a2b03f5795044f5abff4e850"},
- {file = "tokenizers-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9253bdd209c6aee168deca7d0e780581bf303e0058f268f9bb06859379de19b6"},
- {file = "tokenizers-0.20.0-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8f868600ddbcb0545905ed075eb7218a0756bf6c09dae7528ea2f8436ebd2c93"},
- {file = "tokenizers-0.20.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9a9643d9c8c5f99b6aba43fd10034f77cc6c22c31f496d2f0ee183047d948fa0"},
- {file = "tokenizers-0.20.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c375c6a889aeab44734028bc65cc070acf93ccb0f9368be42b67a98e1063d3f6"},
- {file = "tokenizers-0.20.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e359f852328e254f070bbd09a19a568421d23388f04aad9f2fb7da7704c7228d"},
- {file = "tokenizers-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d98b01a309d4387f3b1c1dd68a8b8136af50376cf146c1b7e8d8ead217a5be4b"},
- {file = "tokenizers-0.20.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:459f7537119554c2899067dec1ac74a00d02beef6558f4ee2e99513bf6d568af"},
- {file = "tokenizers-0.20.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:392b87ec89452628c045c9f2a88bc2a827f4c79e7d84bc3b72752b74c2581f70"},
- {file = "tokenizers-0.20.0-cp38-none-win32.whl", hash = "sha256:55a393f893d2ed4dd95a1553c2e42d4d4086878266f437b03590d3f81984c4fe"},
- {file = "tokenizers-0.20.0-cp38-none-win_amd64.whl", hash = "sha256:30ffe33c5c2f2aab8e9a3340d0110dd9f7ace7eec7362e20a697802306bd8068"},
- {file = "tokenizers-0.20.0-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:aa2d4a6fed2a7e3f860c7fc9d48764bb30f2649d83915d66150d6340e06742b8"},
- {file = "tokenizers-0.20.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b5ef0f814084a897e9071fc4a868595f018c5c92889197bdc4bf19018769b148"},
- {file = "tokenizers-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc1e1b791e8c3bf4c4f265f180dadaff1c957bf27129e16fdd5e5d43c2d3762c"},
- {file = "tokenizers-0.20.0-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2b69e55e481459c07885263743a0d3c18d52db19bae8226a19bcca4aaa213fff"},
- {file = "tokenizers-0.20.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4806b4d82e27a2512bc23057b2986bc8b85824914286975b84d8105ff40d03d9"},
- {file = "tokenizers-0.20.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9859e9ef13adf5a473ccab39d31bff9c550606ae3c784bf772b40f615742a24f"},
- {file = "tokenizers-0.20.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ef703efedf4c20488a8eb17637b55973745b27997ff87bad88ed499b397d1144"},
- {file = "tokenizers-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6eec0061bab94b1841ab87d10831fdf1b48ebaed60e6d66d66dbe1d873f92bf5"},
- {file = "tokenizers-0.20.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:980f3d0d7e73f845b69087f29a63c11c7eb924c4ad6b358da60f3db4cf24bdb4"},
- {file = "tokenizers-0.20.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7c157550a2f3851b29d7fdc9dc059fcf81ff0c0fc49a1e5173a89d533ed043fa"},
- {file = "tokenizers-0.20.0-cp39-none-win32.whl", hash = "sha256:8a3d2f4d08608ec4f9895ec25b4b36a97f05812543190a5f2c3cd19e8f041e5a"},
- {file = "tokenizers-0.20.0-cp39-none-win_amd64.whl", hash = "sha256:d90188d12afd0c75e537f9a1d92f9c7375650188ee4f48fdc76f9e38afbd2251"},
- {file = "tokenizers-0.20.0-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:d68e15f1815357b059ec266062340c343ea7f98f7f330602df81ffa3474b6122"},
- {file = "tokenizers-0.20.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:23f9ecec637b9bc80da5f703808d29ed5329e56b5aa8d791d1088014f48afadc"},
- {file = "tokenizers-0.20.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f830b318ee599e3d0665b3e325f85bc75ee2d2ca6285f52e439dc22b64691580"},
- {file = "tokenizers-0.20.0-pp310-pypy310_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3dc750def789cb1de1b5a37657919545e1d9ffa667658b3fa9cb7862407a1b8"},
- {file = "tokenizers-0.20.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e26e6c755ae884c2ea6135cd215bdd0fccafe4ee62405014b8c3cd19954e3ab9"},
- {file = "tokenizers-0.20.0-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:a1158c7174f427182e08baa2a8ded2940f2b4a3e94969a85cc9cfd16004cbcea"},
- {file = "tokenizers-0.20.0-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:6324826287a3fc198898d3dcf758fe4a8479e42d6039f4c59e2cedd3cf92f64e"},
- {file = "tokenizers-0.20.0-pp37-pypy37_pp73-macosx_10_12_x86_64.whl", hash = "sha256:7d8653149405bb0c16feaf9cfee327fdb6aaef9dc2998349fec686f35e81c4e2"},
- {file = "tokenizers-0.20.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b8a2dc1e402a155e97309287ca085c80eb1b7fab8ae91527d3b729181639fa51"},
- {file = "tokenizers-0.20.0-pp37-pypy37_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:07bef67b20aa6e5f7868c42c7c5eae4d24f856274a464ae62e47a0f2cccec3da"},
- {file = "tokenizers-0.20.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:da06e397182ff53789c506c7833220c192952c57e1581a53f503d8d953e2d67e"},
- {file = "tokenizers-0.20.0-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:302f7e11a14814028b7fc88c45a41f1bbe9b5b35fd76d6869558d1d1809baa43"},
- {file = "tokenizers-0.20.0-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:055ec46e807b875589dfbe3d9259f9a6ee43394fb553b03b3d1e9541662dbf25"},
- {file = "tokenizers-0.20.0-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:e3144b8acebfa6ae062e8f45f7ed52e4b50fb6c62f93afc8871b525ab9fdcab3"},
- {file = "tokenizers-0.20.0-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:b52aa3fd14b2a07588c00a19f66511cff5cca8f7266ca3edcdd17f3512ad159f"},
- {file = "tokenizers-0.20.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2b8cf52779ffc5d4d63a0170fbeb512372bad0dd014ce92bbb9149756c831124"},
- {file = "tokenizers-0.20.0-pp38-pypy38_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:983a45dd11a876124378dae71d6d9761822199b68a4c73f32873d8cdaf326a5b"},
- {file = "tokenizers-0.20.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:df6b819c9a19831ebec581e71a7686a54ab45d90faf3842269a10c11d746de0c"},
- {file = "tokenizers-0.20.0-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:e738cfd80795fcafcef89c5731c84b05638a4ab3f412f97d5ed7765466576eb1"},
- {file = "tokenizers-0.20.0-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:c8842c7be2fadb9c9edcee233b1b7fe7ade406c99b0973f07439985c1c1d0683"},
- {file = "tokenizers-0.20.0-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:e47a82355511c373a4a430c4909dc1e518e00031207b1fec536c49127388886b"},
- {file = "tokenizers-0.20.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:9afbf359004551179a5db19424180c81276682773cff2c5d002f6eaaffe17230"},
- {file = "tokenizers-0.20.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a07eaa8799a92e6af6f472c21a75bf71575de2af3c0284120b7a09297c0de2f3"},
- {file = "tokenizers-0.20.0-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0994b2e5fc53a301071806bc4303e4bc3bdc3f490e92a21338146a36746b0872"},
- {file = "tokenizers-0.20.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0b6466e0355b603d10e3cc3d282d350b646341b601e50969464a54939f9848d0"},
- {file = "tokenizers-0.20.0-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:1e86594c2a433cb1ea09cfbe596454448c566e57ee8905bd557e489d93e89986"},
- {file = "tokenizers-0.20.0-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:3e14cdef1efa96ecead6ea64a891828432c3ebba128bdc0596e3059fea104ef3"},
- {file = "tokenizers-0.20.0.tar.gz", hash = "sha256:39d7acc43f564c274085cafcd1dae9d36f332456de1a31970296a6b8da4eac8d"},
-]
-
-[package.dependencies]
-huggingface-hub = ">=0.16.4,<1.0"
-
-[package.extras]
-dev = ["tokenizers[testing]"]
-docs = ["setuptools-rust", "sphinx", "sphinx-rtd-theme"]
-testing = ["black (==22.3)", "datasets", "numpy", "pytest", "requests", "ruff"]
-
-[[package]]
-name = "tomli"
-version = "2.0.1"
-description = "A lil' TOML parser"
-optional = false
-python-versions = ">=3.7"
-files = [
- {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"},
- {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"},
-]
-
-[[package]]
-name = "tqdm"
-version = "4.66.5"
-description = "Fast, Extensible Progress Meter"
-optional = false
-python-versions = ">=3.7"
-files = [
- {file = "tqdm-4.66.5-py3-none-any.whl", hash = "sha256:90279a3770753eafc9194a0364852159802111925aa30eb3f9d85b0e805ac7cd"},
- {file = "tqdm-4.66.5.tar.gz", hash = "sha256:e1020aef2e5096702d8a025ac7d16b1577279c9d63f8375b63083e9a5f0fcbad"},
-]
-
-[package.dependencies]
-colorama = {version = "*", markers = "platform_system == \"Windows\""}
-
-[package.extras]
-dev = ["pytest (>=6)", "pytest-cov", "pytest-timeout", "pytest-xdist"]
-notebook = ["ipywidgets (>=6)"]
-slack = ["slack-sdk"]
-telegram = ["requests"]
-
-[[package]]
-name = "tree-sitter"
-version = "0.21.3"
-description = "Python bindings for the Tree-Sitter parsing library"
-optional = false
-python-versions = ">=3.8"
-files = [
- {file = "tree-sitter-0.21.3.tar.gz", hash = "sha256:b5de3028921522365aa864d95b3c41926e0ba6a85ee5bd000e10dc49b0766988"},
- {file = "tree_sitter-0.21.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:351f302b6615230c9dac9829f0ba20a94362cd658206ca9a7b2d58d73373dfb0"},
- {file = "tree_sitter-0.21.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:766e79ae1e61271e7fdfecf35b6401ad9b47fc07a0965ad78e7f97fddfdf47a6"},
- {file = "tree_sitter-0.21.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2c4d3d4d4b44857e87de55302af7f2d051c912c466ef20e8f18158e64df3542a"},
- {file = "tree_sitter-0.21.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:84eedb06615461b9e2847be7c47b9c5f2195d7d66d31b33c0a227eff4e0a0199"},
- {file = "tree_sitter-0.21.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:9d33ea425df8c3d6436926fe2991429d59c335431bf4e3c71e77c17eb508be5a"},
- {file = "tree_sitter-0.21.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fae1ee0ff6d85e2fd5cd8ceb9fe4af4012220ee1e4cbe813305a316caf7a6f63"},
- {file = "tree_sitter-0.21.3-cp310-cp310-win_amd64.whl", hash = "sha256:bb41be86a987391f9970571aebe005ccd10222f39c25efd15826583c761a37e5"},
- {file = "tree_sitter-0.21.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:54b22c3c2aab3e3639a4b255d9df8455da2921d050c4829b6a5663b057f10db5"},
- {file = "tree_sitter-0.21.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ab6e88c1e2d5e84ff0f9e5cd83f21b8e5074ad292a2cf19df3ba31d94fbcecd4"},
- {file = "tree_sitter-0.21.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc3fd34ed4cd5db445bc448361b5da46a2a781c648328dc5879d768f16a46771"},
- {file = "tree_sitter-0.21.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fabc7182f6083269ce3cfcad202fe01516aa80df64573b390af6cd853e8444a1"},
- {file = "tree_sitter-0.21.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:4f874c3f7d2a2faf5c91982dc7d88ff2a8f183a21fe475c29bee3009773b0558"},
- {file = "tree_sitter-0.21.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:ee61ee3b7a4eedf9d8f1635c68ba4a6fa8c46929601fc48a907c6cfef0cfbcb2"},
- {file = "tree_sitter-0.21.3-cp311-cp311-win_amd64.whl", hash = "sha256:0b7256c723642de1c05fbb776b27742204a2382e337af22f4d9e279d77df7aa2"},
- {file = "tree_sitter-0.21.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:669b3e5a52cb1e37d60c7b16cc2221c76520445bb4f12dd17fd7220217f5abf3"},
- {file = "tree_sitter-0.21.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2aa2a5099a9f667730ff26d57533cc893d766667f4d8a9877e76a9e74f48f0d3"},
- {file = "tree_sitter-0.21.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6a3e06ae2a517cf6f1abb682974f76fa760298e6d5a3ecf2cf140c70f898adf0"},
- {file = "tree_sitter-0.21.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:af992dfe08b4fefcfcdb40548d0d26d5d2e0a0f2d833487372f3728cd0772b48"},
- {file = "tree_sitter-0.21.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c7cbab1dd9765138505c4a55e2aa857575bac4f1f8a8b0457744a4fefa1288e6"},
- {file = "tree_sitter-0.21.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:e1e66aeb457d1529370fcb0997ae5584c6879e0e662f1b11b2f295ea57e22f54"},
- {file = "tree_sitter-0.21.3-cp312-cp312-win_amd64.whl", hash = "sha256:013c750252dc3bd0e069d82e9658de35ed50eecf31c6586d0de7f942546824c5"},
- {file = "tree_sitter-0.21.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:4986a8cb4acebd168474ec2e5db440e59c7888819b3449a43ce8b17ed0331b07"},
- {file = "tree_sitter-0.21.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:6e217fee2e7be7dbce4496caa3d1c466977d7e81277b677f954d3c90e3272ec2"},
- {file = "tree_sitter-0.21.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f32a88afff4f2bc0f20632b0a2aa35fa9ae7d518f083409eca253518e0950929"},
- {file = "tree_sitter-0.21.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3652ac9e47cdddf213c5d5d6854194469097e62f7181c0a9aa8435449a163a9"},
- {file = "tree_sitter-0.21.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:60b4df3298ff467bc01e2c0f6c2fb43aca088038202304bf8e41edd9fa348f45"},
- {file = "tree_sitter-0.21.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:00e4d0c99dff595398ef5e88a1b1ddd53adb13233fb677c1fd8e497fb2361629"},
- {file = "tree_sitter-0.21.3-cp38-cp38-win_amd64.whl", hash = "sha256:50c91353a26946e4dd6779837ecaf8aa123aafa2d3209f261ab5280daf0962f5"},
- {file = "tree_sitter-0.21.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b17b8648b296ccc21a88d72ca054b809ee82d4b14483e419474e7216240ea278"},
- {file = "tree_sitter-0.21.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f2f057fd01d3a95cbce6794c6e9f6db3d376cb3bb14e5b0528d77f0ec21d6478"},
- {file = "tree_sitter-0.21.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:839759de30230ffd60687edbb119b31521d5ac016749358e5285816798bb804a"},
- {file = "tree_sitter-0.21.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5df40aa29cb7e323898194246df7a03b9676955a0ac1f6bce06bc4903a70b5f7"},
- {file = "tree_sitter-0.21.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:1d9be27dde007b569fa78ff9af5fe40d2532c998add9997a9729e348bb78fa59"},
- {file = "tree_sitter-0.21.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c4ac87735e6f98fe085244c7c020f0177d13d4c117db72ba041faa980d25d69d"},
- {file = "tree_sitter-0.21.3-cp39-cp39-win_amd64.whl", hash = "sha256:fbbd137f7d9a5309fb4cb82e2c3250ba101b0dd08a8abdce815661e6cf2cbc19"},
-]
-
-[[package]]
-name = "tree-sitter-languages"
-version = "1.10.2"
-description = "Binary Python wheels for all tree sitter languages."
-optional = false
-python-versions = "*"
-files = [
- {file = "tree_sitter_languages-1.10.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5580348f0b20233b1d5431fa178ccd3d07423ca4a3275df02a44608fd72344b9"},
- {file = "tree_sitter_languages-1.10.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:103c7466644486b1e9e03850df46fc6aa12f13ca636c74f173270276220ac80b"},
- {file = "tree_sitter_languages-1.10.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d13db84511c6f1a7dc40383b66deafa74dabd8b877e3d65ab253f3719eccafd6"},
- {file = "tree_sitter_languages-1.10.2-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:57adfa32be7e465b54aa72f915f6c78a2b66b227df4f656b5d4fbd1ca7a92b3f"},
- {file = "tree_sitter_languages-1.10.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1c6385e033e460ceb8f33f3f940335f422ef2b763700a04f0089391a68b56153"},
- {file = "tree_sitter_languages-1.10.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:dfa3f38cc5381c5aba01dd7494f59b8a9050e82ff6e06e1233e3a0cbae297e3c"},
- {file = "tree_sitter_languages-1.10.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:9f195155acf47f8bc5de7cee46ecd07b2f5697f007ba89435b51ef4c0b953ea5"},
- {file = "tree_sitter_languages-1.10.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2de330e2ac6d7426ca025a3ec0f10d5640c3682c1d0c7702e812dcfb44b58120"},
- {file = "tree_sitter_languages-1.10.2-cp310-cp310-win32.whl", hash = "sha256:c9731cf745f135d9770eeba9bb4e2ff4dabc107b5ae9b8211e919f6b9100ea6d"},
- {file = "tree_sitter_languages-1.10.2-cp310-cp310-win_amd64.whl", hash = "sha256:6dd75851c41d0c3c4987a9b7692d90fa8848706c23115669d8224ffd6571e357"},
- {file = "tree_sitter_languages-1.10.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7eb7d7542b2091c875fe52719209631fca36f8c10fa66970d2c576ae6a1b8289"},
- {file = "tree_sitter_languages-1.10.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6b41bcb00974b1c8a1800c7f1bb476a1d15a0463e760ee24872f2d53b08ee424"},
- {file = "tree_sitter_languages-1.10.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6f370cd7845c6c81df05680d5bd96db8a99d32b56f4728c5d05978911130a853"},
- {file = "tree_sitter_languages-1.10.2-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a1dc195c88ef4c72607e112a809a69190e096a2e5ebc6201548b3e05fdd169ad"},
- {file = "tree_sitter_languages-1.10.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ae34ac314a7170be24998a0f994c1ac80761d8d4bd126af27ee53a023d3b849"},
- {file = "tree_sitter_languages-1.10.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:01b5742d5f5bd675489486b582bd482215880b26dde042c067f8265a6e925d9c"},
- {file = "tree_sitter_languages-1.10.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:ab1cbc46244d34fd16f21edaa20231b2a57f09f092a06ee3d469f3117e6eb954"},
- {file = "tree_sitter_languages-1.10.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0b1149e7467a4e92b8a70e6005fe762f880f493cf811fc003554b29f04f5e7c8"},
- {file = "tree_sitter_languages-1.10.2-cp311-cp311-win32.whl", hash = "sha256:049276343962f4696390ee555acc2c1a65873270c66a6cbe5cb0bca83bcdf3c6"},
- {file = "tree_sitter_languages-1.10.2-cp311-cp311-win_amd64.whl", hash = "sha256:7f3fdd468a577f04db3b63454d939e26e360229b53c80361920aa1ebf2cd7491"},
- {file = "tree_sitter_languages-1.10.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c0f4c8b2734c45859edc7fcaaeaab97a074114111b5ba51ab4ec7ed52104763c"},
- {file = "tree_sitter_languages-1.10.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:eecd3c1244ac3425b7a82ba9125b4ddb45d953bbe61de114c0334fd89b7fe782"},
- {file = "tree_sitter_languages-1.10.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:15db3c8510bc39a80147ee7421bf4782c15c09581c1dc2237ea89cefbd95b846"},
- {file = "tree_sitter_languages-1.10.2-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:92c6487a6feea683154d3e06e6db68c30e0ae749a7ce4ce90b9e4e46b78c85c7"},
- {file = "tree_sitter_languages-1.10.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d2f1cd1d1bdd65332f9c2b67d49dcf148cf1ded752851d159ac3e5ee4f4d260"},
- {file = "tree_sitter_languages-1.10.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:976c8039165b8e12f17a01ddee9f4e23ec6e352b165ad29b44d2bf04e2fbe77e"},
- {file = "tree_sitter_languages-1.10.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:dafbbdf16bf668a580902e1620f4baa1913e79438abcce721a50647564c687b9"},
- {file = "tree_sitter_languages-1.10.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1aeabd3d60d6d276b73cd8f3739d595b1299d123cc079a317f1a5b3c5461e2ca"},
- {file = "tree_sitter_languages-1.10.2-cp312-cp312-win32.whl", hash = "sha256:fab8ee641914098e8933b87ea3d657bea4dd00723c1ee7038b847b12eeeef4f5"},
- {file = "tree_sitter_languages-1.10.2-cp312-cp312-win_amd64.whl", hash = "sha256:5e606430d736367e5787fa5a7a0c5a1ec9b85eded0b3596bbc0d83532a40810b"},
- {file = "tree_sitter_languages-1.10.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:838d5b48a7ed7a17658721952c77fda4570d2a069f933502653b17e15a9c39c9"},
- {file = "tree_sitter_languages-1.10.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:987b3c71b1d278c2889e018ee77b8ee05c384e2e3334dec798f8b611c4ab2d1e"},
- {file = "tree_sitter_languages-1.10.2-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:faa00abcb2c819027df58472da055d22fa7dfcb77c77413d8500c32ebe24d38b"},
- {file = "tree_sitter_languages-1.10.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e102fbbf02322d9201a86a814e79a9734ac80679fdb9682144479044f401a73"},
- {file = "tree_sitter_languages-1.10.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:8f0b87cf1a7b03174ba18dfd81582be82bfed26803aebfe222bd20e444aba003"},
- {file = "tree_sitter_languages-1.10.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c0f1b9af9cb67f0b942b020da9fdd000aad5e92f2383ae0ba7a330b318d31912"},
- {file = "tree_sitter_languages-1.10.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:5a4076c921f7a4d31e643843de7dfe040b65b63a238a5aa8d31d93aabe6572aa"},
- {file = "tree_sitter_languages-1.10.2-cp37-cp37m-win32.whl", hash = "sha256:fa6391a3a5d83d32db80815161237b67d70576f090ce5f38339206e917a6f8bd"},
- {file = "tree_sitter_languages-1.10.2-cp37-cp37m-win_amd64.whl", hash = "sha256:55649d3f254585a064121513627cf9788c1cfdadbc5f097f33d5ba750685a4c0"},
- {file = "tree_sitter_languages-1.10.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6f85d1edaa2d22d80d4ea5b6d12b95cf3644017b6c227d0d42854439e02e8893"},
- {file = "tree_sitter_languages-1.10.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d78feed4a764ef3141cb54bf00fe94d514d8b6e26e09423e23b4c616fcb7938c"},
- {file = "tree_sitter_languages-1.10.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da1aca27531f9dd5308637d76643372856f0f65d0d28677d1bcf4211e8ed1ad0"},
- {file = "tree_sitter_languages-1.10.2-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1031ea440dafb72237437d754eff8940153a3b051e3d18932ac25e75ce060a15"},
- {file = "tree_sitter_languages-1.10.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:99d3249beaef2c9fe558ecc9a97853c260433a849dcc68266d9770d196c2e102"},
- {file = "tree_sitter_languages-1.10.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:59a4450f262a55148fb7e68681522f0c2a2f6b7d89666312a2b32708d8f416e1"},
- {file = "tree_sitter_languages-1.10.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ce74eab0e430370d5e15a96b6c6205f93405c177a8b2e71e1526643b2fb9bab1"},
- {file = "tree_sitter_languages-1.10.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:9b4dd2b6b3d24c85dffe33d6c343448869eaf4f41c19ddba662eb5d65d8808f4"},
- {file = "tree_sitter_languages-1.10.2-cp38-cp38-win32.whl", hash = "sha256:92d734fb968fe3927a7596d9f0459f81a8fa7b07e16569476b28e27d0d753348"},
- {file = "tree_sitter_languages-1.10.2-cp38-cp38-win_amd64.whl", hash = "sha256:46a13f7d38f2eeb75f7cf127d1201346093748c270d686131f0cbc50e42870a1"},
- {file = "tree_sitter_languages-1.10.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f8c6a936ae99fdd8857e91f86c11c2f5e507ff30631d141d98132bb7ab2c8638"},
- {file = "tree_sitter_languages-1.10.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c283a61423f49cdfa7b5a5dfbb39221e3bd126fca33479cd80749d4d7a6b7349"},
- {file = "tree_sitter_languages-1.10.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76e60be6bdcff923386a54a5edcb6ff33fc38ab0118636a762024fa2bc98de55"},
- {file = "tree_sitter_languages-1.10.2-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c00069f9575bd831eabcce2cdfab158dde1ed151e7e5614c2d985ff7d78a7de1"},
- {file = "tree_sitter_languages-1.10.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:475ff53203d8a43ccb19bb322fa2fb200d764001cc037793f1fadd714bb343da"},
- {file = "tree_sitter_languages-1.10.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:26fe7c9c412e4141dea87ea4b3592fd12e385465b5bdab106b0d5125754d4f60"},
- {file = "tree_sitter_languages-1.10.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:8fed27319957458340f24fe14daad467cd45021da034eef583519f83113a8c5e"},
- {file = "tree_sitter_languages-1.10.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:3657a491a7f96cc75a3568ddd062d25f3be82b6a942c68801a7b226ff7130181"},
- {file = "tree_sitter_languages-1.10.2-cp39-cp39-win32.whl", hash = "sha256:33f7d584d01a7a3c893072f34cfc64ec031f3cfe57eebc32da2f8ac046e101a7"},
- {file = "tree_sitter_languages-1.10.2-cp39-cp39-win_amd64.whl", hash = "sha256:1b944af3ee729fa70fc8ae82224a9ff597cdb63addea084e0ea2fa2b0ec39bb7"},
-]
-
-[package.dependencies]
-tree-sitter = "*"
-
-[[package]]
-name = "types-requests"
-version = "2.32.0.20240914"
-description = "Typing stubs for requests"
-optional = false
-python-versions = ">=3.8"
-files = [
- {file = "types-requests-2.32.0.20240914.tar.gz", hash = "sha256:2850e178db3919d9bf809e434eef65ba49d0e7e33ac92d588f4a5e295fffd405"},
- {file = "types_requests-2.32.0.20240914-py3-none-any.whl", hash = "sha256:59c2f673eb55f32a99b2894faf6020e1a9f4a402ad0f192bfee0b64469054310"},
-]
-
-[package.dependencies]
-urllib3 = ">=2"
-
-[[package]]
-name = "typing-extensions"
-version = "4.12.2"
-description = "Backported and Experimental Type Hints for Python 3.8+"
-optional = false
-python-versions = ">=3.8"
-files = [
- {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"},
- {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"},
-]
-
-[[package]]
-name = "typing-inspect"
-version = "0.9.0"
-description = "Runtime inspection utilities for typing module."
-optional = false
-python-versions = "*"
-files = [
- {file = "typing_inspect-0.9.0-py3-none-any.whl", hash = "sha256:9ee6fc59062311ef8547596ab6b955e1b8aa46242d854bfc78f4f6b0eff35f9f"},
- {file = "typing_inspect-0.9.0.tar.gz", hash = "sha256:b23fc42ff6f6ef6954e4852c1fb512cdd18dbea03134f91f856a95ccc9461f78"},
-]
-
-[package.dependencies]
-mypy-extensions = ">=0.3.0"
-typing-extensions = ">=3.7.4"
-
-[[package]]
-name = "tzdata"
-version = "2024.1"
-description = "Provider of IANA time zone data"
-optional = false
-python-versions = ">=2"
-files = [
- {file = "tzdata-2024.1-py2.py3-none-any.whl", hash = "sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252"},
- {file = "tzdata-2024.1.tar.gz", hash = "sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd"},
-]
-
-[[package]]
-name = "urllib3"
-version = "2.2.3"
-description = "HTTP library with thread-safe connection pooling, file post, and more."
-optional = false
-python-versions = ">=3.8"
-files = [
- {file = "urllib3-2.2.3-py3-none-any.whl", hash = "sha256:ca899ca043dcb1bafa3e262d73aa25c465bfb49e0bd9dd5d59f1d0acba2f8fac"},
- {file = "urllib3-2.2.3.tar.gz", hash = "sha256:e7d814a81dad81e6caf2ec9fdedb284ecc9c73076b62654547cc64ccdcae26e9"},
-]
-
-[package.extras]
-brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"]
-h2 = ["h2 (>=4,<5)"]
-socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"]
-zstd = ["zstandard (>=0.18.0)"]
-
-[[package]]
-name = "wcwidth"
-version = "0.2.13"
-description = "Measures the displayed width of unicode strings in a terminal"
-optional = false
-python-versions = "*"
-files = [
- {file = "wcwidth-0.2.13-py2.py3-none-any.whl", hash = "sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859"},
- {file = "wcwidth-0.2.13.tar.gz", hash = "sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5"},
-]
-
-[[package]]
-name = "wrapt"
-version = "1.16.0"
-description = "Module for decorators, wrappers and monkey patching."
-optional = false
-python-versions = ">=3.6"
-files = [
- {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"},
- {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"},
- {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"},
- {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"},
- {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"},
- {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"},
- {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"},
- {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"},
- {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"},
- {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"},
- {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"},
- {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"},
- {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"},
- {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"},
- {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"},
- {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"},
- {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"},
- {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"},
- {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"},
- {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"},
- {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"},
- {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"},
- {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"},
- {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"},
- {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"},
- {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"},
- {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"},
- {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"},
- {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"},
- {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"},
- {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"},
- {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"},
- {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"},
- {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"},
- {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"},
- {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"},
- {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"},
- {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"},
- {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"},
- {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"},
- {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"},
- {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"},
- {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"},
- {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"},
- {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"},
- {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"},
- {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"},
- {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"},
- {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"},
- {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"},
- {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"},
- {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"},
- {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"},
- {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"},
- {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"},
- {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"},
- {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"},
- {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"},
- {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"},
- {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"},
- {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"},
- {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"},
- {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"},
- {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"},
- {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"},
- {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"},
- {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"},
- {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"},
- {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"},
- {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"},
-]
-
-[[package]]
-name = "xxhash"
-version = "3.5.0"
-description = "Python binding for xxHash"
-optional = false
-python-versions = ">=3.7"
-files = [
- {file = "xxhash-3.5.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ece616532c499ee9afbb83078b1b952beffef121d989841f7f4b3dc5ac0fd212"},
- {file = "xxhash-3.5.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3171f693dbc2cef6477054a665dc255d996646b4023fe56cb4db80e26f4cc520"},
- {file = "xxhash-3.5.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7c5d3e570ef46adaf93fc81b44aca6002b5a4d8ca11bd0580c07eac537f36680"},
- {file = "xxhash-3.5.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7cb29a034301e2982df8b1fe6328a84f4b676106a13e9135a0d7e0c3e9f806da"},
- {file = "xxhash-3.5.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5d0d307d27099bb0cbeea7260eb39ed4fdb99c5542e21e94bb6fd29e49c57a23"},
- {file = "xxhash-3.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0342aafd421795d740e514bc9858ebddfc705a75a8c5046ac56d85fe97bf196"},
- {file = "xxhash-3.5.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3dbbd9892c5ebffeca1ed620cf0ade13eb55a0d8c84e0751a6653adc6ac40d0c"},
- {file = "xxhash-3.5.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4cc2d67fdb4d057730c75a64c5923abfa17775ae234a71b0200346bfb0a7f482"},
- {file = "xxhash-3.5.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:ec28adb204b759306a3d64358a5e5c07d7b1dd0ccbce04aa76cb9377b7b70296"},
- {file = "xxhash-3.5.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:1328f6d8cca2b86acb14104e381225a3d7b42c92c4b86ceae814e5c400dbb415"},
- {file = "xxhash-3.5.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:8d47ebd9f5d9607fd039c1fbf4994e3b071ea23eff42f4ecef246ab2b7334198"},
- {file = "xxhash-3.5.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b96d559e0fcddd3343c510a0fe2b127fbff16bf346dd76280b82292567523442"},
- {file = "xxhash-3.5.0-cp310-cp310-win32.whl", hash = "sha256:61c722ed8d49ac9bc26c7071eeaa1f6ff24053d553146d5df031802deffd03da"},
- {file = "xxhash-3.5.0-cp310-cp310-win_amd64.whl", hash = "sha256:9bed5144c6923cc902cd14bb8963f2d5e034def4486ab0bbe1f58f03f042f9a9"},
- {file = "xxhash-3.5.0-cp310-cp310-win_arm64.whl", hash = "sha256:893074d651cf25c1cc14e3bea4fceefd67f2921b1bb8e40fcfeba56820de80c6"},
- {file = "xxhash-3.5.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:02c2e816896dc6f85922ced60097bcf6f008dedfc5073dcba32f9c8dd786f3c1"},
- {file = "xxhash-3.5.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6027dcd885e21581e46d3c7f682cfb2b870942feeed58a21c29583512c3f09f8"},
- {file = "xxhash-3.5.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1308fa542bbdbf2fa85e9e66b1077eea3a88bef38ee8a06270b4298a7a62a166"},
- {file = "xxhash-3.5.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c28b2fdcee797e1c1961cd3bcd3d545cab22ad202c846235197935e1df2f8ef7"},
- {file = "xxhash-3.5.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:924361811732ddad75ff23e90efd9ccfda4f664132feecb90895bade6a1b4623"},
- {file = "xxhash-3.5.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89997aa1c4b6a5b1e5b588979d1da048a3c6f15e55c11d117a56b75c84531f5a"},
- {file = "xxhash-3.5.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:685c4f4e8c59837de103344eb1c8a3851f670309eb5c361f746805c5471b8c88"},
- {file = "xxhash-3.5.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:dbd2ecfbfee70bc1a4acb7461fa6af7748ec2ab08ac0fa298f281c51518f982c"},
- {file = "xxhash-3.5.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:25b5a51dc3dfb20a10833c8eee25903fd2e14059e9afcd329c9da20609a307b2"},
- {file = "xxhash-3.5.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a8fb786fb754ef6ff8c120cb96629fb518f8eb5a61a16aac3a979a9dbd40a084"},
- {file = "xxhash-3.5.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:a905ad00ad1e1c34fe4e9d7c1d949ab09c6fa90c919860c1534ff479f40fd12d"},
- {file = "xxhash-3.5.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:963be41bcd49f53af6d795f65c0da9b4cc518c0dd9c47145c98f61cb464f4839"},
- {file = "xxhash-3.5.0-cp311-cp311-win32.whl", hash = "sha256:109b436096d0a2dd039c355fa3414160ec4d843dfecc64a14077332a00aeb7da"},
- {file = "xxhash-3.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:b702f806693201ad6c0a05ddbbe4c8f359626d0b3305f766077d51388a6bac58"},
- {file = "xxhash-3.5.0-cp311-cp311-win_arm64.whl", hash = "sha256:c4dcb4120d0cc3cc448624147dba64e9021b278c63e34a38789b688fd0da9bf3"},
- {file = "xxhash-3.5.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:14470ace8bd3b5d51318782cd94e6f94431974f16cb3b8dc15d52f3b69df8e00"},
- {file = "xxhash-3.5.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:59aa1203de1cb96dbeab595ded0ad0c0056bb2245ae11fac11c0ceea861382b9"},
- {file = "xxhash-3.5.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:08424f6648526076e28fae6ea2806c0a7d504b9ef05ae61d196d571e5c879c84"},
- {file = "xxhash-3.5.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:61a1ff00674879725b194695e17f23d3248998b843eb5e933007ca743310f793"},
- {file = "xxhash-3.5.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f2f2c61bee5844d41c3eb015ac652a0229e901074951ae48581d58bfb2ba01be"},
- {file = "xxhash-3.5.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d32a592cac88d18cc09a89172e1c32d7f2a6e516c3dfde1b9adb90ab5df54a6"},
- {file = "xxhash-3.5.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:70dabf941dede727cca579e8c205e61121afc9b28516752fd65724be1355cc90"},
- {file = "xxhash-3.5.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e5d0ddaca65ecca9c10dcf01730165fd858533d0be84c75c327487c37a906a27"},
- {file = "xxhash-3.5.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:3e5b5e16c5a480fe5f59f56c30abdeba09ffd75da8d13f6b9b6fd224d0b4d0a2"},
- {file = "xxhash-3.5.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:149b7914451eb154b3dfaa721315117ea1dac2cc55a01bfbd4df7c68c5dd683d"},
- {file = "xxhash-3.5.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:eade977f5c96c677035ff39c56ac74d851b1cca7d607ab3d8f23c6b859379cab"},
- {file = "xxhash-3.5.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fa9f547bd98f5553d03160967866a71056a60960be00356a15ecc44efb40ba8e"},
- {file = "xxhash-3.5.0-cp312-cp312-win32.whl", hash = "sha256:f7b58d1fd3551b8c80a971199543379be1cee3d0d409e1f6d8b01c1a2eebf1f8"},
- {file = "xxhash-3.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:fa0cafd3a2af231b4e113fba24a65d7922af91aeb23774a8b78228e6cd785e3e"},
- {file = "xxhash-3.5.0-cp312-cp312-win_arm64.whl", hash = "sha256:586886c7e89cb9828bcd8a5686b12e161368e0064d040e225e72607b43858ba2"},
- {file = "xxhash-3.5.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:37889a0d13b0b7d739cfc128b1c902f04e32de17b33d74b637ad42f1c55101f6"},
- {file = "xxhash-3.5.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:97a662338797c660178e682f3bc180277b9569a59abfb5925e8620fba00b9fc5"},
- {file = "xxhash-3.5.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7f85e0108d51092bdda90672476c7d909c04ada6923c14ff9d913c4f7dc8a3bc"},
- {file = "xxhash-3.5.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cd2fd827b0ba763ac919440042302315c564fdb797294d86e8cdd4578e3bc7f3"},
- {file = "xxhash-3.5.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:82085c2abec437abebf457c1d12fccb30cc8b3774a0814872511f0f0562c768c"},
- {file = "xxhash-3.5.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:07fda5de378626e502b42b311b049848c2ef38784d0d67b6f30bb5008642f8eb"},
- {file = "xxhash-3.5.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c279f0d2b34ef15f922b77966640ade58b4ccdfef1c4d94b20f2a364617a493f"},
- {file = "xxhash-3.5.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:89e66ceed67b213dec5a773e2f7a9e8c58f64daeb38c7859d8815d2c89f39ad7"},
- {file = "xxhash-3.5.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:bcd51708a633410737111e998ceb3b45d3dbc98c0931f743d9bb0a209033a326"},
- {file = "xxhash-3.5.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3ff2c0a34eae7df88c868be53a8dd56fbdf592109e21d4bfa092a27b0bf4a7bf"},
- {file = "xxhash-3.5.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:4e28503dccc7d32e0b9817aa0cbfc1f45f563b2c995b7a66c4c8a0d232e840c7"},
- {file = "xxhash-3.5.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a6c50017518329ed65a9e4829154626f008916d36295b6a3ba336e2458824c8c"},
- {file = "xxhash-3.5.0-cp313-cp313-win32.whl", hash = "sha256:53a068fe70301ec30d868ece566ac90d873e3bb059cf83c32e76012c889b8637"},
- {file = "xxhash-3.5.0-cp313-cp313-win_amd64.whl", hash = "sha256:80babcc30e7a1a484eab952d76a4f4673ff601f54d5142c26826502740e70b43"},
- {file = "xxhash-3.5.0-cp313-cp313-win_arm64.whl", hash = "sha256:4811336f1ce11cac89dcbd18f3a25c527c16311709a89313c3acaf771def2d4b"},
- {file = "xxhash-3.5.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:6e5f70f6dca1d3b09bccb7daf4e087075ff776e3da9ac870f86ca316736bb4aa"},
- {file = "xxhash-3.5.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e76e83efc7b443052dd1e585a76201e40b3411fe3da7af4fe434ec51b2f163b"},
- {file = "xxhash-3.5.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:33eac61d0796ca0591f94548dcfe37bb193671e0c9bcf065789b5792f2eda644"},
- {file = "xxhash-3.5.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0ec70a89be933ea49222fafc3999987d7899fc676f688dd12252509434636622"},
- {file = "xxhash-3.5.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd86b8e7f703ec6ff4f351cfdb9f428955859537125904aa8c963604f2e9d3e7"},
- {file = "xxhash-3.5.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0adfbd36003d9f86c8c97110039f7539b379f28656a04097e7434d3eaf9aa131"},
- {file = "xxhash-3.5.0-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:63107013578c8a730419adc05608756c3fa640bdc6abe806c3123a49fb829f43"},
- {file = "xxhash-3.5.0-cp37-cp37m-musllinux_1_2_i686.whl", hash = "sha256:683b94dbd1ca67557850b86423318a2e323511648f9f3f7b1840408a02b9a48c"},
- {file = "xxhash-3.5.0-cp37-cp37m-musllinux_1_2_ppc64le.whl", hash = "sha256:5d2a01dcce81789cf4b12d478b5464632204f4c834dc2d064902ee27d2d1f0ee"},
- {file = "xxhash-3.5.0-cp37-cp37m-musllinux_1_2_s390x.whl", hash = "sha256:a9d360a792cbcce2fe7b66b8d51274ec297c53cbc423401480e53b26161a290d"},
- {file = "xxhash-3.5.0-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:f0b48edbebea1b7421a9c687c304f7b44d0677c46498a046079d445454504737"},
- {file = "xxhash-3.5.0-cp37-cp37m-win32.whl", hash = "sha256:7ccb800c9418e438b44b060a32adeb8393764da7441eb52aa2aa195448935306"},
- {file = "xxhash-3.5.0-cp37-cp37m-win_amd64.whl", hash = "sha256:c3bc7bf8cb8806f8d1c9bf149c18708cb1c406520097d6b0a73977460ea03602"},
- {file = "xxhash-3.5.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:74752ecaa544657d88b1d1c94ae68031e364a4d47005a90288f3bab3da3c970f"},
- {file = "xxhash-3.5.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:dee1316133c9b463aa81aca676bc506d3f80d8f65aeb0bba2b78d0b30c51d7bd"},
- {file = "xxhash-3.5.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:602d339548d35a8579c6b013339fb34aee2df9b4e105f985443d2860e4d7ffaa"},
- {file = "xxhash-3.5.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:695735deeddfb35da1677dbc16a083445360e37ff46d8ac5c6fcd64917ff9ade"},
- {file = "xxhash-3.5.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1030a39ba01b0c519b1a82f80e8802630d16ab95dc3f2b2386a0b5c8ed5cbb10"},
- {file = "xxhash-3.5.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a5bc08f33c4966f4eb6590d6ff3ceae76151ad744576b5fc6c4ba8edd459fdec"},
- {file = "xxhash-3.5.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:160e0c19ee500482ddfb5d5570a0415f565d8ae2b3fd69c5dcfce8a58107b1c3"},
- {file = "xxhash-3.5.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:f1abffa122452481a61c3551ab3c89d72238e279e517705b8b03847b1d93d738"},
- {file = "xxhash-3.5.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:d5e9db7ef3ecbfc0b4733579cea45713a76852b002cf605420b12ef3ef1ec148"},
- {file = "xxhash-3.5.0-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:23241ff6423378a731d84864bf923a41649dc67b144debd1077f02e6249a0d54"},
- {file = "xxhash-3.5.0-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:82b833d5563fefd6fceafb1aed2f3f3ebe19f84760fdd289f8b926731c2e6e91"},
- {file = "xxhash-3.5.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:0a80ad0ffd78bef9509eee27b4a29e56f5414b87fb01a888353e3d5bda7038bd"},
- {file = "xxhash-3.5.0-cp38-cp38-win32.whl", hash = "sha256:50ac2184ffb1b999e11e27c7e3e70cc1139047e7ebc1aa95ed12f4269abe98d4"},
- {file = "xxhash-3.5.0-cp38-cp38-win_amd64.whl", hash = "sha256:392f52ebbb932db566973693de48f15ce787cabd15cf6334e855ed22ea0be5b3"},
- {file = "xxhash-3.5.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bfc8cdd7f33d57f0468b0614ae634cc38ab9202c6957a60e31d285a71ebe0301"},
- {file = "xxhash-3.5.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e0c48b6300cd0b0106bf49169c3e0536408dfbeb1ccb53180068a18b03c662ab"},
- {file = "xxhash-3.5.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fe1a92cfbaa0a1253e339ccec42dbe6db262615e52df591b68726ab10338003f"},
- {file = "xxhash-3.5.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:33513d6cc3ed3b559134fb307aae9bdd94d7e7c02907b37896a6c45ff9ce51bd"},
- {file = "xxhash-3.5.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:eefc37f6138f522e771ac6db71a6d4838ec7933939676f3753eafd7d3f4c40bc"},
- {file = "xxhash-3.5.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a606c8070ada8aa2a88e181773fa1ef17ba65ce5dd168b9d08038e2a61b33754"},
- {file = "xxhash-3.5.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:42eca420c8fa072cc1dd62597635d140e78e384a79bb4944f825fbef8bfeeef6"},
- {file = "xxhash-3.5.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:604253b2143e13218ff1ef0b59ce67f18b8bd1c4205d2ffda22b09b426386898"},
- {file = "xxhash-3.5.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:6e93a5ad22f434d7876665444a97e713a8f60b5b1a3521e8df11b98309bff833"},
- {file = "xxhash-3.5.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:7a46e1d6d2817ba8024de44c4fd79913a90e5f7265434cef97026215b7d30df6"},
- {file = "xxhash-3.5.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:30eb2efe6503c379b7ab99c81ba4a779748e3830241f032ab46bd182bf5873af"},
- {file = "xxhash-3.5.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:c8aa771ff2c13dd9cda8166d685d7333d389fae30a4d2bb39d63ab5775de8606"},
- {file = "xxhash-3.5.0-cp39-cp39-win32.whl", hash = "sha256:5ed9ebc46f24cf91034544b26b131241b699edbfc99ec5e7f8f3d02d6eb7fba4"},
- {file = "xxhash-3.5.0-cp39-cp39-win_amd64.whl", hash = "sha256:220f3f896c6b8d0316f63f16c077d52c412619e475f9372333474ee15133a558"},
- {file = "xxhash-3.5.0-cp39-cp39-win_arm64.whl", hash = "sha256:a7b1d8315d9b5e9f89eb2933b73afae6ec9597a258d52190944437158b49d38e"},
- {file = "xxhash-3.5.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:2014c5b3ff15e64feecb6b713af12093f75b7926049e26a580e94dcad3c73d8c"},
- {file = "xxhash-3.5.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fab81ef75003eda96239a23eda4e4543cedc22e34c373edcaf744e721a163986"},
- {file = "xxhash-3.5.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4e2febf914ace002132aa09169cc572e0d8959d0f305f93d5828c4836f9bc5a6"},
- {file = "xxhash-3.5.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5d3a10609c51da2a1c0ea0293fc3968ca0a18bd73838455b5bca3069d7f8e32b"},
- {file = "xxhash-3.5.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:5a74f23335b9689b66eb6dbe2a931a88fcd7a4c2cc4b1cb0edba8ce381c7a1da"},
- {file = "xxhash-3.5.0-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:2b4154c00eb22e4d543f472cfca430e7962a0f1d0f3778334f2e08a7ba59363c"},
- {file = "xxhash-3.5.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d30bbc1644f726b825b3278764240f449d75f1a8bdda892e641d4a688b1494ae"},
- {file = "xxhash-3.5.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6fa0b72f2423e2aa53077e54a61c28e181d23effeaafd73fcb9c494e60930c8e"},
- {file = "xxhash-3.5.0-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:13de2b76c1835399b2e419a296d5b38dc4855385d9e96916299170085ef72f57"},
- {file = "xxhash-3.5.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:0691bfcc4f9c656bcb96cc5db94b4d75980b9d5589f2e59de790091028580837"},
- {file = "xxhash-3.5.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:297595fe6138d4da2c8ce9e72a04d73e58725bb60f3a19048bc96ab2ff31c692"},
- {file = "xxhash-3.5.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cc1276d369452040cbb943300dc8abeedab14245ea44056a2943183822513a18"},
- {file = "xxhash-3.5.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2061188a1ba352fc699c82bff722f4baacb4b4b8b2f0c745d2001e56d0dfb514"},
- {file = "xxhash-3.5.0-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:38c384c434021e4f62b8d9ba0bc9467e14d394893077e2c66d826243025e1f81"},
- {file = "xxhash-3.5.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:e6a4dd644d72ab316b580a1c120b375890e4c52ec392d4aef3c63361ec4d77d1"},
- {file = "xxhash-3.5.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:531af8845aaadcadf951b7e0c1345c6b9c68a990eeb74ff9acd8501a0ad6a1c9"},
- {file = "xxhash-3.5.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ce379bcaa9fcc00f19affa7773084dd09f5b59947b3fb47a1ceb0179f91aaa1"},
- {file = "xxhash-3.5.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd1b2281d01723f076df3c8188f43f2472248a6b63118b036e641243656b1b0f"},
- {file = "xxhash-3.5.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9c770750cc80e8694492244bca7251385188bc5597b6a39d98a9f30e8da984e0"},
- {file = "xxhash-3.5.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:b150b8467852e1bd844387459aa6fbe11d7f38b56e901f9f3b3e6aba0d660240"},
- {file = "xxhash-3.5.0.tar.gz", hash = "sha256:84f2caddf951c9cbf8dc2e22a89d4ccf5d86391ac6418fe81e3c67d0cf60b45f"},
-]
-
-[[package]]
-name = "yarl"
-version = "1.11.1"
-description = "Yet another URL library"
-optional = false
-python-versions = ">=3.8"
-files = [
- {file = "yarl-1.11.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:400cd42185f92de559d29eeb529e71d80dfbd2f45c36844914a4a34297ca6f00"},
- {file = "yarl-1.11.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8258c86f47e080a258993eed877d579c71da7bda26af86ce6c2d2d072c11320d"},
- {file = "yarl-1.11.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2164cd9725092761fed26f299e3f276bb4b537ca58e6ff6b252eae9631b5c96e"},
- {file = "yarl-1.11.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08ea567c16f140af8ddc7cb58e27e9138a1386e3e6e53982abaa6f2377b38cc"},
- {file = "yarl-1.11.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:768ecc550096b028754ea28bf90fde071c379c62c43afa574edc6f33ee5daaec"},
- {file = "yarl-1.11.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2909fa3a7d249ef64eeb2faa04b7957e34fefb6ec9966506312349ed8a7e77bf"},
- {file = "yarl-1.11.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:01a8697ec24f17c349c4f655763c4db70eebc56a5f82995e5e26e837c6eb0e49"},
- {file = "yarl-1.11.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e286580b6511aac7c3268a78cdb861ec739d3e5a2a53b4809faef6b49778eaff"},
- {file = "yarl-1.11.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4179522dc0305c3fc9782549175c8e8849252fefeb077c92a73889ccbcd508ad"},
- {file = "yarl-1.11.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:27fcb271a41b746bd0e2a92182df507e1c204759f460ff784ca614e12dd85145"},
- {file = "yarl-1.11.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:f61db3b7e870914dbd9434b560075e0366771eecbe6d2b5561f5bc7485f39efd"},
- {file = "yarl-1.11.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:c92261eb2ad367629dc437536463dc934030c9e7caca861cc51990fe6c565f26"},
- {file = "yarl-1.11.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d95b52fbef190ca87d8c42f49e314eace4fc52070f3dfa5f87a6594b0c1c6e46"},
- {file = "yarl-1.11.1-cp310-cp310-win32.whl", hash = "sha256:489fa8bde4f1244ad6c5f6d11bb33e09cf0d1d0367edb197619c3e3fc06f3d91"},
- {file = "yarl-1.11.1-cp310-cp310-win_amd64.whl", hash = "sha256:476e20c433b356e16e9a141449f25161e6b69984fb4cdbd7cd4bd54c17844998"},
- {file = "yarl-1.11.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:946eedc12895873891aaceb39bceb484b4977f70373e0122da483f6c38faaa68"},
- {file = "yarl-1.11.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:21a7c12321436b066c11ec19c7e3cb9aec18884fe0d5b25d03d756a9e654edfe"},
- {file = "yarl-1.11.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c35f493b867912f6fda721a59cc7c4766d382040bdf1ddaeeaa7fa4d072f4675"},
- {file = "yarl-1.11.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25861303e0be76b60fddc1250ec5986c42f0a5c0c50ff57cc30b1be199c00e63"},
- {file = "yarl-1.11.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e4b53f73077e839b3f89c992223f15b1d2ab314bdbdf502afdc7bb18e95eae27"},
- {file = "yarl-1.11.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:327c724b01b8641a1bf1ab3b232fb638706e50f76c0b5bf16051ab65c868fac5"},
- {file = "yarl-1.11.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4307d9a3417eea87715c9736d050c83e8c1904e9b7aada6ce61b46361b733d92"},
- {file = "yarl-1.11.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:48a28bed68ab8fb7e380775f0029a079f08a17799cb3387a65d14ace16c12e2b"},
- {file = "yarl-1.11.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:067b961853c8e62725ff2893226fef3d0da060656a9827f3f520fb1d19b2b68a"},
- {file = "yarl-1.11.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:8215f6f21394d1f46e222abeb06316e77ef328d628f593502d8fc2a9117bde83"},
- {file = "yarl-1.11.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:498442e3af2a860a663baa14fbf23fb04b0dd758039c0e7c8f91cb9279799bff"},
- {file = "yarl-1.11.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:69721b8effdb588cb055cc22f7c5105ca6fdaa5aeb3ea09021d517882c4a904c"},
- {file = "yarl-1.11.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:1e969fa4c1e0b1a391f3fcbcb9ec31e84440253325b534519be0d28f4b6b533e"},
- {file = "yarl-1.11.1-cp311-cp311-win32.whl", hash = "sha256:7d51324a04fc4b0e097ff8a153e9276c2593106a811704025bbc1d6916f45ca6"},
- {file = "yarl-1.11.1-cp311-cp311-win_amd64.whl", hash = "sha256:15061ce6584ece023457fb8b7a7a69ec40bf7114d781a8c4f5dcd68e28b5c53b"},
- {file = "yarl-1.11.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:a4264515f9117be204935cd230fb2a052dd3792789cc94c101c535d349b3dab0"},
- {file = "yarl-1.11.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:f41fa79114a1d2eddb5eea7b912d6160508f57440bd302ce96eaa384914cd265"},
- {file = "yarl-1.11.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:02da8759b47d964f9173c8675710720b468aa1c1693be0c9c64abb9d8d9a4867"},
- {file = "yarl-1.11.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9361628f28f48dcf8b2f528420d4d68102f593f9c2e592bfc842f5fb337e44fd"},
- {file = "yarl-1.11.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b91044952da03b6f95fdba398d7993dd983b64d3c31c358a4c89e3c19b6f7aef"},
- {file = "yarl-1.11.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:74db2ef03b442276d25951749a803ddb6e270d02dda1d1c556f6ae595a0d76a8"},
- {file = "yarl-1.11.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e975a2211952a8a083d1b9d9ba26472981ae338e720b419eb50535de3c02870"},
- {file = "yarl-1.11.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8aef97ba1dd2138112890ef848e17d8526fe80b21f743b4ee65947ea184f07a2"},
- {file = "yarl-1.11.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a7915ea49b0c113641dc4d9338efa9bd66b6a9a485ffe75b9907e8573ca94b84"},
- {file = "yarl-1.11.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:504cf0d4c5e4579a51261d6091267f9fd997ef58558c4ffa7a3e1460bd2336fa"},
- {file = "yarl-1.11.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:3de5292f9f0ee285e6bd168b2a77b2a00d74cbcfa420ed078456d3023d2f6dff"},
- {file = "yarl-1.11.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:a34e1e30f1774fa35d37202bbeae62423e9a79d78d0874e5556a593479fdf239"},
- {file = "yarl-1.11.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:66b63c504d2ca43bf7221a1f72fbe981ff56ecb39004c70a94485d13e37ebf45"},
- {file = "yarl-1.11.1-cp312-cp312-win32.whl", hash = "sha256:a28b70c9e2213de425d9cba5ab2e7f7a1c8ca23a99c4b5159bf77b9c31251447"},
- {file = "yarl-1.11.1-cp312-cp312-win_amd64.whl", hash = "sha256:17b5a386d0d36fb828e2fb3ef08c8829c1ebf977eef88e5367d1c8c94b454639"},
- {file = "yarl-1.11.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:1fa2e7a406fbd45b61b4433e3aa254a2c3e14c4b3186f6e952d08a730807fa0c"},
- {file = "yarl-1.11.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:750f656832d7d3cb0c76be137ee79405cc17e792f31e0a01eee390e383b2936e"},
- {file = "yarl-1.11.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0b8486f322d8f6a38539136a22c55f94d269addb24db5cb6f61adc61eabc9d93"},
- {file = "yarl-1.11.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3fce4da3703ee6048ad4138fe74619c50874afe98b1ad87b2698ef95bf92c96d"},
- {file = "yarl-1.11.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8ed653638ef669e0efc6fe2acb792275cb419bf9cb5c5049399f3556995f23c7"},
- {file = "yarl-1.11.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:18ac56c9dd70941ecad42b5a906820824ca72ff84ad6fa18db33c2537ae2e089"},
- {file = "yarl-1.11.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:688654f8507464745ab563b041d1fb7dab5d9912ca6b06e61d1c4708366832f5"},
- {file = "yarl-1.11.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4973eac1e2ff63cf187073cd4e1f1148dcd119314ab79b88e1b3fad74a18c9d5"},
- {file = "yarl-1.11.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:964a428132227edff96d6f3cf261573cb0f1a60c9a764ce28cda9525f18f7786"},
- {file = "yarl-1.11.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:6d23754b9939cbab02c63434776df1170e43b09c6a517585c7ce2b3d449b7318"},
- {file = "yarl-1.11.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:c2dc4250fe94d8cd864d66018f8344d4af50e3758e9d725e94fecfa27588ff82"},
- {file = "yarl-1.11.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:09696438cb43ea6f9492ef237761b043f9179f455f405279e609f2bc9100212a"},
- {file = "yarl-1.11.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:999bfee0a5b7385a0af5ffb606393509cfde70ecca4f01c36985be6d33e336da"},
- {file = "yarl-1.11.1-cp313-cp313-win32.whl", hash = "sha256:ce928c9c6409c79e10f39604a7e214b3cb69552952fbda8d836c052832e6a979"},
- {file = "yarl-1.11.1-cp313-cp313-win_amd64.whl", hash = "sha256:501c503eed2bb306638ccb60c174f856cc3246c861829ff40eaa80e2f0330367"},
- {file = "yarl-1.11.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:dae7bd0daeb33aa3e79e72877d3d51052e8b19c9025ecf0374f542ea8ec120e4"},
- {file = "yarl-1.11.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:3ff6b1617aa39279fe18a76c8d165469c48b159931d9b48239065767ee455b2b"},
- {file = "yarl-1.11.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3257978c870728a52dcce8c2902bf01f6c53b65094b457bf87b2644ee6238ddc"},
- {file = "yarl-1.11.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0f351fa31234699d6084ff98283cb1e852270fe9e250a3b3bf7804eb493bd937"},
- {file = "yarl-1.11.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8aef1b64da41d18026632d99a06b3fefe1d08e85dd81d849fa7c96301ed22f1b"},
- {file = "yarl-1.11.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7175a87ab8f7fbde37160a15e58e138ba3b2b0e05492d7351314a250d61b1591"},
- {file = "yarl-1.11.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba444bdd4caa2a94456ef67a2f383710928820dd0117aae6650a4d17029fa25e"},
- {file = "yarl-1.11.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0ea9682124fc062e3d931c6911934a678cb28453f957ddccf51f568c2f2b5e05"},
- {file = "yarl-1.11.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:8418c053aeb236b20b0ab8fa6bacfc2feaaf7d4683dd96528610989c99723d5f"},
- {file = "yarl-1.11.1-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:61a5f2c14d0a1adfdd82258f756b23a550c13ba4c86c84106be4c111a3a4e413"},
- {file = "yarl-1.11.1-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:f3a6d90cab0bdf07df8f176eae3a07127daafcf7457b997b2bf46776da2c7eb7"},
- {file = "yarl-1.11.1-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:077da604852be488c9a05a524068cdae1e972b7dc02438161c32420fb4ec5e14"},
- {file = "yarl-1.11.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:15439f3c5c72686b6c3ff235279630d08936ace67d0fe5c8d5bbc3ef06f5a420"},
- {file = "yarl-1.11.1-cp38-cp38-win32.whl", hash = "sha256:238a21849dd7554cb4d25a14ffbfa0ef380bb7ba201f45b144a14454a72ffa5a"},
- {file = "yarl-1.11.1-cp38-cp38-win_amd64.whl", hash = "sha256:67459cf8cf31da0e2cbdb4b040507e535d25cfbb1604ca76396a3a66b8ba37a6"},
- {file = "yarl-1.11.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:884eab2ce97cbaf89f264372eae58388862c33c4f551c15680dd80f53c89a269"},
- {file = "yarl-1.11.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8a336eaa7ee7e87cdece3cedb395c9657d227bfceb6781295cf56abcd3386a26"},
- {file = "yarl-1.11.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:87f020d010ba80a247c4abc335fc13421037800ca20b42af5ae40e5fd75e7909"},
- {file = "yarl-1.11.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:637c7ddb585a62d4469f843dac221f23eec3cbad31693b23abbc2c366ad41ff4"},
- {file = "yarl-1.11.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:48dfd117ab93f0129084577a07287376cc69c08138694396f305636e229caa1a"},
- {file = "yarl-1.11.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:75e0ae31fb5ccab6eda09ba1494e87eb226dcbd2372dae96b87800e1dcc98804"},
- {file = "yarl-1.11.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f46f81501160c28d0c0b7333b4f7be8983dbbc161983b6fb814024d1b4952f79"},
- {file = "yarl-1.11.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:04293941646647b3bfb1719d1d11ff1028e9c30199509a844da3c0f5919dc520"},
- {file = "yarl-1.11.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:250e888fa62d73e721f3041e3a9abf427788a1934b426b45e1b92f62c1f68366"},
- {file = "yarl-1.11.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:e8f63904df26d1a66aabc141bfd258bf738b9bc7bc6bdef22713b4f5ef789a4c"},
- {file = "yarl-1.11.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:aac44097d838dda26526cffb63bdd8737a2dbdf5f2c68efb72ad83aec6673c7e"},
- {file = "yarl-1.11.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:267b24f891e74eccbdff42241c5fb4f974de2d6271dcc7d7e0c9ae1079a560d9"},
- {file = "yarl-1.11.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:6907daa4b9d7a688063ed098c472f96e8181733c525e03e866fb5db480a424df"},
- {file = "yarl-1.11.1-cp39-cp39-win32.whl", hash = "sha256:14438dfc5015661f75f85bc5adad0743678eefee266ff0c9a8e32969d5d69f74"},
- {file = "yarl-1.11.1-cp39-cp39-win_amd64.whl", hash = "sha256:94d0caaa912bfcdc702a4204cd5e2bb01eb917fc4f5ea2315aa23962549561b0"},
- {file = "yarl-1.11.1-py3-none-any.whl", hash = "sha256:72bf26f66456baa0584eff63e44545c9f0eaed9b73cb6601b647c91f14c11f38"},
- {file = "yarl-1.11.1.tar.gz", hash = "sha256:1bb2d9e212fb7449b8fb73bc461b51eaa17cc8430b4a87d87be7b25052d92f53"},
-]
-
-[package.dependencies]
-idna = ">=2.0"
-multidict = ">=4.0"
-
-[[package]]
-name = "zipp"
-version = "3.20.2"
-description = "Backport of pathlib-compatible object wrapper for zip files"
-optional = false
-python-versions = ">=3.8"
-files = [
- {file = "zipp-3.20.2-py3-none-any.whl", hash = "sha256:a817ac80d6cf4b23bf7f2828b7cabf326f15a001bea8b1f9b49631780ba28350"},
- {file = "zipp-3.20.2.tar.gz", hash = "sha256:bc9eb26f4506fda01b81bcde0ca78103b6e62f991b381fec825435c836edbc29"},
-]
-
-[package.extras]
-check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)"]
-cover = ["pytest-cov"]
-doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"]
-enabler = ["pytest-enabler (>=2.2)"]
-test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-ignore-flaky"]
-type = ["pytest-mypy"]
-
-[metadata]
-lock-version = "2.0"
-python-versions = "^3.10, <3.13"
-content-hash = "40be9244276197c5d21b27cf0a64f67bee2aff3c2ca1b76d5fb51a9a6fe08b6c"
diff --git a/pyproject.toml b/pyproject.toml
index ba1e9aa7..bd3ba186 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,39 +1,62 @@
[tool.poetry]
name = "ghostos"
-version = "0.0.1"
-description = "An agent framework offering agents and meta-agents a Python code interface to operate everything."
+version = "0.0.1-dev5"
+description = "A framework offers an operating system simulator with a Python Code Interface for AI Agents"
authors = ["zhuming ", "Nile Zhou "]
license = "MIT"
readme = "README.md"
+[tool.poetry.urls]
+homepage = "https://github.com/ghost-in-moss/GhostOS"
+repository = "https://github.com/ghost-in-moss/GhostOS"
+documentation = "https://github.com/ghost-in-moss/GhostOS/docs"
+issues = "https://github.com/ghost-in-moss/GhostOS/issues"
+
[tool.poetry.dependencies]
-python = "^3.10, <3.13"
+python = ">=3.9,<3.14,!=3.9.7"
pydantic = "^2.7.0"
-pytest = "^8.1.1"
openai = "^1.19.0"
pyyaml = "^6.0.1"
rich = "^13.7.1"
httpx-socks = "^0.9.1"
-restrictedpython = "^7.1"
datasets = "^2.20.0"
anthropic = "^0.31.2"
sympy = "^1.13.1"
tree-sitter = "0.21.3"
tree-sitter-languages = "^1.10.2"
-networkx = "^3.3"
-grep-ast = "^0.3.3"
litellm = "^1.43.18"
-hide-py = "^0.3.0"
prompt-toolkit = "^3.0.47"
arxiv = "^2.1.3"
llama-index-core = "^0.11.9"
llama-index-llms-openai = "^0.2.7"
+streamlit = "^1.39.0"
+pydantic-settings = "^2.5.2"
+streamlit-antd-components = "^0.3.2"
+streamlit-react-jsonschema = "^0.1.3"
+python-dotenv = "^1.0.1"
+babel = "^2.16.0"
+websockets = "^13.1"
+pysocks = "^1.7.1"
+requests = { extras = ["socks"], version = "^2.32.3" }
+streamlit-paste-button = "^0.1.2"
+pyaudio = { version = "^0.2.14", optional = true }
+spherov2 = { version = "^0.12.1", optional = true }
+bleak = [
+ { version = "^0.22.3", python = ">=3.10,<3.14", optional = true }
+]
[tool.poetry.scripts]
-init = "ghostos.scripts.init:main"
-demo = "ghostos.demo.scripts.demo:main"
-llm_test = "ghostos.demo.scripts.llm_test:main"
-clear_runtime = "ghostos.demo.scripts.clear_runtime:main"
+ghostos = "ghostos.scripts.cli:main"
+
+[tool.poetry.group.dev.dependencies]
+pytest = "^8.1.1"
+mypy = "^1.13.0"
+
+
+[tool.poetry.extras]
+realtime = ['pyaudio', 'websockets']
+sphero = ["spherov2", "bleak", "pyaudio"]
+
[build-system]
requires = ["poetry-core"]
diff --git a/tests/contracts/test_configs.py b/tests/contracts/test_configs.py
new file mode 100644
index 00000000..cb8e86e3
--- /dev/null
+++ b/tests/contracts/test_configs.py
@@ -0,0 +1,27 @@
+from typing import List
+from ghostos.contracts.configs import Config, YamlConfig
+from ghostos.framework.configs import MemoryConfigs
+
+
+class FooConf(YamlConfig):
+ relative_path = "hello.yml"
+ foo: str = "abc"
+ bar: float = 1.1
+
+
+def test_config_marshal():
+ cases: List[Config] = [
+ FooConf(),
+ ]
+
+ configs = MemoryConfigs()
+
+ for c in cases:
+ marshaled = c.marshal()
+ un_marshaled = c.unmarshal(marshaled)
+ marshaled2 = un_marshaled.marshal()
+ assert marshaled == marshaled2, c
+
+ configs.save(c)
+ got = configs.get(type(c))
+ assert got.marshal() == marshaled
diff --git a/tests/contracts/test_modules.py b/tests/contracts/test_modules.py
new file mode 100644
index 00000000..99042135
--- /dev/null
+++ b/tests/contracts/test_modules.py
@@ -0,0 +1,11 @@
+from ghostos.contracts.modules import DefaultModules
+
+
+def test_default_modules_iter():
+ m = DefaultModules()
+ from ghostos import contracts
+ result = list(m.iter_modules(contracts))
+
+ assert "ghostos.contracts" not in result
+ result2 = list(m.iter_modules("ghostos.contracts"))
+ assert result == result2
diff --git a/tests/contracts/test_pool.py b/tests/contracts/test_pool.py
new file mode 100644
index 00000000..3a7d5b6d
--- /dev/null
+++ b/tests/contracts/test_pool.py
@@ -0,0 +1,18 @@
+from ghostos.contracts.pool import DefaultPool
+
+
+def test_default_pool():
+ class Foo:
+ count = 0
+
+ def go(self):
+ self.count += 1
+
+ pool = DefaultPool(4)
+ foo = Foo()
+ pool.submit(foo.go)
+ pool.submit(foo.go)
+ pool.submit(foo.go)
+ pool.submit(foo.go)
+ pool.shutdown()
+ assert foo.count == 4
diff --git a/tests/core/aifuncs/test_aifunc_repository.py b/tests/core/aifuncs/test_aifunc_repository.py
new file mode 100644
index 00000000..6b9ea65b
--- /dev/null
+++ b/tests/core/aifuncs/test_aifunc_repository.py
@@ -0,0 +1,22 @@
+from ghostos.core.aifunc import AIFuncRepoByConfigsProvider, AIFuncRepository, AIFuncsConf
+from ghostos.framework.configs import Configs, MemoryConfigs
+from ghostos.contracts.modules import Modules, DefaultModules
+from ghostos.container import Container
+from ghostos.demo import aifuncs_demo
+
+
+def test_aifunc_repository():
+ container = Container()
+ container.set(Modules, DefaultModules())
+ container.set(Configs, MemoryConfigs({
+ AIFuncsConf.conf_path(): "{}",
+
+ }))
+ container.register(AIFuncRepoByConfigsProvider())
+ container.bootstrap()
+
+ repo = container.force_fetch(AIFuncRepository)
+ result = repo.scan(str(aifuncs_demo.__name__), recursive=True, save=False)
+ assert len(result) > 1
+
+
diff --git a/tests/core/aifuncs/test_exec_frame.py b/tests/core/aifuncs/test_exec_frame.py
new file mode 100644
index 00000000..0206b8a9
--- /dev/null
+++ b/tests/core/aifuncs/test_exec_frame.py
@@ -0,0 +1,39 @@
+from ghostos.core.aifunc import ExecFrame, ExecStep, AIFunc, AIFuncResult
+from threading import Thread
+
+
+class Tool(AIFunc):
+ foo: str = "foo"
+
+
+class ToolResult(AIFuncResult):
+ err: str = ""
+
+
+def test_exec_frame():
+ def next_step(f: ExecFrame, depth: int):
+ if depth > 3:
+ return
+ for i in range(3):
+ st = f.new_step()
+ threads = []
+ for k in range(3):
+ sub_frame = st.new_frame(Tool())
+ th = Thread(target=next_step, args=(sub_frame, depth + 1))
+ th.start()
+ threads.append(th)
+ for th in threads:
+ th.join()
+
+ t = Tool()
+ fr = ExecFrame.from_func(t)
+ next_step(fr, 0)
+
+ assert len(fr.steps) == 3
+ for step in fr.steps:
+ assert step.depth == 0
+ assert len(step.frames) == 3
+ assert fr.depth == 0
+ assert fr.steps[0].frames[0].steps[0].frames[0].depth == 2
+
+
diff --git a/tests/core/ghosts/test_thoughts.py b/tests/core/ghosts/test_thoughts.py
deleted file mode 100644
index cb938118..00000000
--- a/tests/core/ghosts/test_thoughts.py
+++ /dev/null
@@ -1,18 +0,0 @@
-from ghostos.core.moss.prompts import get_prompt
-from ghostos.core.ghosts.thoughts import Thought, ModelThought
-
-
-def test_get_prompt_from_thought():
- prompt = get_prompt(Thought)
- assert prompt == Thought.__class_prompt__()
-
-
-def test_get_prompt_from_thought_with_no_thought():
- prompt = get_prompt(ModelThought)
- assert prompt == ModelThought.__class_prompt__()
-
- class TestThought(ModelThought):
- foo: int = 123
-
- prompt = get_prompt(TestThought)
- assert "class TestThought" in prompt
diff --git a/tests/core/messages/test_arr_stream_receiver.py b/tests/core/messages/test_arr_stream_receiver.py
new file mode 100644
index 00000000..5c5c59ac
--- /dev/null
+++ b/tests/core/messages/test_arr_stream_receiver.py
@@ -0,0 +1,283 @@
+from typing import Iterable
+from ghostos.core.messages.transport import new_basic_connection, Stream, ReceiverBuffer
+from ghostos.core.messages.pipeline import SequencePipe
+from ghostos.core.messages.message import Message, MessageType
+from threading import Thread
+import time
+
+
+def iter_content(content: str, gap: float) -> Iterable[Message]:
+ for c in content:
+ item = Message.new_chunk(content=c)
+ yield item
+ if gap > 0:
+ time.sleep(gap)
+
+
+def test_new_connection_baseline():
+ stream, retriever = new_basic_connection(timeout=5, idle=0.2, complete_only=False)
+ assert stream.alive()
+ assert not retriever.closed()
+ content = "hello world, ha ha ha ha"
+
+ def send_data(s: Stream, c: str):
+ with s:
+ s.send(iter_content(c, 0.02))
+
+ t = Thread(target=send_data, args=(stream, content))
+ t.start()
+ last = None
+ first = None
+ with retriever:
+ count = 0
+ for item in retriever.recv():
+ if not first:
+ first = item
+ count += 1
+ last = item
+ assert count == len(content) + 1
+ assert first is not None
+ assert first.is_head()
+ assert last.is_complete()
+ t.join()
+
+
+def test_new_connection_complete_only():
+ stream, retriever = new_basic_connection(timeout=5, idle=0.2, complete_only=True)
+ content = "hello world"
+
+ def send_data(s: Stream, c: str):
+ with s:
+ s.send(iter_content(c, 0.02))
+
+ t = Thread(target=send_data, args=(stream, content))
+ t.start()
+ with retriever:
+ messages = list(retriever.recv())
+ assert len(messages) == 1
+ assert messages[0].is_complete()
+ assert messages[0].content == content
+ t.join()
+
+
+def test_new_connection_timeout():
+ stream, retriever = new_basic_connection(timeout=0.2, idle=0.2, complete_only=False)
+ content = "hello world"
+
+ def send_data(s: Stream, c: str):
+ error = None
+ try:
+ with s:
+ s.send(iter_content(c, 1))
+ except RuntimeError as e:
+ error = e
+ finally:
+ assert error is not None
+
+ t = Thread(target=send_data, args=(stream, content))
+ t.start()
+ with retriever:
+ messages = list(retriever.recv())
+
+ assert retriever.closed()
+ assert retriever.error() is not None
+ assert not stream.alive()
+ assert messages[-1] is retriever.error()
+ t.join()
+
+
+def test_new_connection_sync():
+ stream, retriever = new_basic_connection(timeout=5, idle=0.2, complete_only=False)
+ content = "hello world"
+
+ def send_data(s: Stream, c: str):
+ with s:
+ s.send(iter_content(c, 0.02))
+
+ send_data(stream, content)
+ with retriever:
+ messages = list(retriever.recv())
+ assert len(messages) == len(content) + 1
+ assert messages[len(content)].is_complete()
+ assert messages[len(content)].content == content
+ assert messages[3].get_seq() == "chunk"
+
+
+def test_new_connection_wait():
+ stream, retriever = new_basic_connection(timeout=5, idle=0.2, complete_only=False)
+ content = "hello world"
+
+ def send_data(s: Stream, c: str):
+ with s:
+ s.send(iter_content(c, 0.02))
+
+ t = Thread(target=send_data, args=(stream, content))
+ t.start()
+ with retriever:
+ retriever.wait()
+ t.join()
+
+
+def test_new_connection_recv_with_sequence():
+ stream, retriever = new_basic_connection(timeout=0, idle=0.1, complete_only=False)
+ content = "hello world"
+
+ def send_data(s: Stream, c: str):
+ with s:
+ messages = SequencePipe().across(iter_content(c, 0.02))
+ s.send(messages)
+
+ send_data(stream, content)
+
+ got = retriever.recv()
+ assert len(list(got)) == len(content) + 1
+
+
+def test_new_connection_wait_with_sequence():
+ stream, retriever = new_basic_connection(timeout=5, idle=0.2, complete_only=False)
+ content = "hello world"
+
+ def send_data(s: Stream, c: str):
+ with s:
+ messages = SequencePipe().across(iter_content(c, 0.02))
+ messages = list(messages)
+ s.send(messages)
+
+ send_data(stream, content)
+
+ got = retriever.wait()
+ assert len(got) == 1
+
+
+def test_new_connection_with_pool():
+ from ghostos.contracts.pool import DefaultPool
+ pool = DefaultPool(10)
+ stream, retriever = new_basic_connection(timeout=5, idle=0.2, complete_only=False)
+ content = "hello world"
+
+ def send_data(s: Stream, c: str):
+ with s:
+ s.send(iter_content(c, 0.02))
+ s.send(iter_content(c, 0.02))
+
+ pool.submit(send_data, stream, content)
+
+ with retriever:
+ messages = retriever.wait()
+ assert len(messages) == 2
+ assert retriever.error() is None
+ pool.shutdown(wait=True)
+
+
+def test_array_receiver_buffer_baseline():
+ stream, retriever = new_basic_connection(timeout=5, idle=0.2, complete_only=False)
+ content = "hello world"
+
+ def send_data(s: Stream, c: str):
+ messages = SequencePipe().across(iter_content(c, 0))
+ messages = list(messages)
+ s.send(messages)
+
+ with stream:
+ send_data(stream, content)
+ send_data(stream, content)
+
+ buffer = ReceiverBuffer.new(retriever.recv())
+ assert buffer is not None
+ assert buffer.head().content == "h"
+ for chunk in buffer.chunks():
+ assert chunk.content in content
+ assert len(chunk.content) == 1
+ assert not chunk.is_complete()
+
+ assert buffer.tail().content == content
+ assert buffer.tail().is_complete()
+ buffer = buffer.next()
+ assert buffer is not None
+ assert buffer.head().content == "h"
+ assert buffer.tail().content == content
+
+ buffer = buffer.next()
+ assert buffer is None
+
+
+def test_array_receiver_buffer_async():
+ from ghostos.contracts.pool import DefaultPool
+ pool = DefaultPool(10)
+ stream, retriever = new_basic_connection(timeout=5, idle=0.2, complete_only=False)
+ content = "hello world"
+
+ def send_data(s: Stream, c: str):
+ with s:
+ s.send(iter_content(c, 0.02))
+ s.send(iter_content(c, 0.02))
+
+ pool.submit(send_data, stream, content)
+
+ with retriever:
+ buffer = ReceiverBuffer.new(retriever.recv())
+ assert buffer.tail().content == content
+ buffer = buffer.next()
+ assert buffer.tail().content == content
+ buffer = buffer.next()
+ assert buffer is None
+ pool.shutdown(wait=True)
+
+
+def test_array_receiver_with_error():
+ stream, retriever = new_basic_connection(timeout=5, idle=0.2, complete_only=False)
+ content = "hello world"
+
+ def send_data(s: Stream, c: str):
+ with s:
+ s.send(iter_content(c, 0.02))
+ s.send([MessageType.ERROR.new(content="error")])
+
+ send_data(stream, content)
+ with retriever:
+ messages = retriever.wait()
+ assert len(messages) == 2
+ assert messages[1].is_complete()
+ assert messages[1].type == MessageType.ERROR
+
+
+def test_array_receiver_bad_case_1():
+ item = Message(
+ msg_id='25c6d3d9-9bb1-45e1-ac7e-585380975ea1',
+ call_id='call_SyYPOCVP60bvyLIMP3gemVYy',
+ index=None,
+ type='function_call',
+ stage='',
+ role='assistant',
+ name='moss',
+ content='',
+ memory=None,
+ attrs=None,
+ payloads={'task_info': {'task_id': '8d98d7772baa6776c7a169ef2028c06a', 'task_name': 'SpheroGPT',
+ 'process_id': '7167a681-cc2e-43aa-aab8-1781f9308e3f',
+ 'shell_id': 'ghostos_streamlit_app', 'thread_id': '8d98d7772baa6776c7a169ef2028c06a'}},
+ callers=[],
+ seq='chunk',
+ created=1732633767.653,
+ )
+ item2 = Message(
+ **{
+ "msg_id": "",
+ "call_id": None,
+ "index": None,
+ "type": "function_call",
+ "stage": "",
+ "role": "assistant",
+ "name": "SpheroGPT",
+ "content": "os",
+ "memory": None,
+ "attrs": None,
+ "payloads": {},
+ "callers": [],
+ "seq": "chunk",
+ "created": 0.0,
+ })
+
+ patched = item.patch(item2)
+ assert patched is not None
+ assert patched.name == "moss"
diff --git a/tests/core/messages/test_message_parser.py b/tests/core/messages/test_message_parser.py
index 7c0e4eec..3a51ad8a 100644
--- a/tests/core/messages/test_message_parser.py
+++ b/tests/core/messages/test_message_parser.py
@@ -1,8 +1,34 @@
-from ghostos.core.messages import MessageKindParser
+from ghostos.core.messages import MessageKindParser, VariableMessage, FunctionCaller
+from ghostos.framework.variables import test_variables
+from pydantic import BaseModel
def test_message_parser():
- parser = MessageKindParser()
+ parser = MessageKindParser(test_variables)
messages = list(parser.parse(['Hello World']))
assert len(messages) == 1
assert messages[0].content == 'Hello World'
+
+
+def test_message_parser_with_message_class():
+ parser = MessageKindParser(test_variables)
+ caller = FunctionCaller(name="hello", arguments="world")
+ item = caller.new_output("output")
+ messages = list(parser.parse([item]))
+ assert messages[0].content == "output"
+
+
+class Foo(BaseModel):
+ foo: str = "hello"
+
+
+def test_variable_message():
+ parser = MessageKindParser(test_variables)
+ messages = list(parser.parse([Foo()]))
+ assert len(messages) > 0
+
+ message = messages[0]
+ var = VariableMessage.from_message(message)
+ assert var is not None
+ value = test_variables.load(var.attrs.vid)
+ assert value.foo == "hello"
diff --git a/tests/core/messages/test_messages.py b/tests/core/messages/test_messages.py
index d8e30557..8139ed3e 100644
--- a/tests/core/messages/test_messages.py
+++ b/tests/core/messages/test_messages.py
@@ -1,6 +1,6 @@
from ghostos.core.messages import (
Role,
- Message,
+ Message, MessageType,
)
@@ -23,7 +23,7 @@ def test_message_basic_merge():
msg = Message.new_head(role="assistant")
for c in string:
- msg = msg.patch(msg.new_pack(content=c, role="assistant"))
+ msg = msg.patch(msg.new_chunk(content=c, role="assistant"))
assert msg.content == "hello world"
@@ -31,10 +31,10 @@ def test_message_with_full_type():
msg = Message.new_head()
content = "hello world"
for c in content:
- msg = msg.patch(msg.new_pack(content=c))
+ msg = msg.patch(msg.new_chunk(content=c))
last = msg.model_copy(update=dict(content="good"))
- last.pack = False
+ last.seq = "complete"
buffed = msg.patch(last)
assert buffed is not None and buffed.content == "good"
@@ -46,7 +46,7 @@ def test_head_is_not_empty():
def test_head_pack_patch():
msg = Message.new_head(content="a")
- patch = msg.patch(Message.new_pack(content="b"))
+ patch = msg.patch(Message.new_chunk(content="b"))
assert patch is not None
assert patch.content == "ab"
@@ -54,7 +54,7 @@ def test_head_pack_patch():
def test_tail_patch():
msg = Message.new_head(content="")
for c in "hello":
- pack = Message.new_pack(content=c)
+ pack = Message.new_chunk(content=c)
patch = msg.patch(pack)
assert patch is not None
tail = Message.new_tail(content=" world")
@@ -69,16 +69,44 @@ def test_tail_patch():
def test_patch_default_type_message():
msg = Message.new_head(typ_="kind")
- patch = msg.patch(Message.new_pack(content="c", typ_=""))
+ patch = msg.patch(Message.new_chunk(content="c", typ_=""))
assert patch is not None
- patch = msg.patch(Message.new_pack(content="c", typ_="kind"))
+ patch = msg.patch(Message.new_chunk(content="c", typ_="kind"))
assert patch is not None
- pack = Message.new_pack(content="c", typ_="foo")
+ pack = Message.new_chunk(content="c", typ_="foo")
assert pack.type == "foo"
patch = msg.patch(pack)
assert patch is None
-
-
+def test_function_call_message():
+ head = Message.new_head(
+ typ_=MessageType.FUNCTION_CALL,
+ call_id="abc",
+ name="abc",
+ )
+ patched = head.patch(
+ Message.new_chunk(
+ typ_=MessageType.FUNCTION_CALL,
+ content="hello world"
+ )
+ )
+ assert patched is not None
+ assert patched.call_id == "abc"
+ assert patched.name == "abc"
+ assert patched.content == "hello world"
+
+
+def test_message_path_bad_case():
+ item1 = Message(msg_id='d5ff6a6a-2b05-4819-864d-82afdf9ac5fc', call_id=None,
+ from_id='chatcmpl-AXs0YM2VxVZbo50C1lIOC0qlWumtN', index=None, type='function_call', stage='',
+ role='assistant', name=None, content='{"', memory=None, attrs=None, payloads={}, callers=[],
+ seq='chunk',
+ created=0.0)
+ item2 = Message(msg_id='d5ff6a6a-2b05-4819-864d-82afdf9ac5fc', call_id='call_DCaC3PJy336sZ9ryhxijgFlq',
+ from_id='chatcmpl-AXs0YM2VxVZbo50C1lIOC0qlWumtN', index=None, type='function_call', stage='',
+ role='assistant', name='moss', content='{"', memory=None, attrs=None, payloads={}, callers=[],
+ seq='chunk', created=1732636557.282)
+ patched = item1.patch(item2)
+ assert patched is not None
diff --git a/tests/core/messages/test_openai_parser.py b/tests/core/messages/test_openai_parser.py
new file mode 100644
index 00000000..88c8ff6a
--- /dev/null
+++ b/tests/core/messages/test_openai_parser.py
@@ -0,0 +1,65 @@
+from ghostos.core.messages.openai import DefaultOpenAIMessageParser
+from openai.types.chat.chat_completion_chunk import (
+ ChatCompletionChunk, Choice, ChoiceDelta,
+ ChoiceDeltaToolCall, ChoiceDeltaToolCallFunction,
+)
+from ghostos.core.messages.message import MessageType
+from ghostos.core.messages.pipeline import SequencePipe, pipeline
+from ghostos.core.messages.transport import new_basic_connection
+
+
+def test_openai_parser_bad_case_1():
+ items = [
+ ChatCompletionChunk(id='chatcmpl-AXs0YM2VxVZbo50C1lIOC0qlWumtN', choices=[
+ Choice(delta=ChoiceDelta(content='。', fuction_call=None, refusal=None, role=None, tool_calls=None),
+ finish_reason=None, index=0, logprobs=None)], created=1732635794, model='gpt-4o-2024-08-06',
+ object='chat.completion.chunk', service_tier=None, system_fingerprint='fp_831e067d82',
+ usage=None),
+ ChatCompletionChunk(id='chatcmpl-AXs0YM2VxVZbo50C1lIOC0qlWumtN', choices=[Choice(
+ delta=ChoiceDelta(content=None, function_call=None, refusal=None, role=None, tool_calls=[
+ ChoiceDeltaToolCall(index=0, id='call_DCaC3PJy336sZ9ryhxijgFlq',
+ function=ChoiceDeltaToolCallFunction(arguments='', name='moss'), type='function')]),
+ finish_reason=None, index=0, logprobs=None)], created=1732635794, model='gpt-4o-2024-08-06',
+ object='chat.completion.chunk', service_tier=None, system_fingerprint='fp_831e067d82',
+ usage=None),
+ ChatCompletionChunk(
+ id='chatcmpl-AXs0YM2VxVZbo50C1lIOC0qlWumtN',
+ choices=[Choice(
+ delta=ChoiceDelta(
+ content=None, function_call=None, refusal=None, role=None,
+ tool_calls=[
+ ChoiceDeltaToolCall(
+ index=0, id=None,
+ function=ChoiceDeltaToolCallFunction(
+ arguments='{"',
+ name=None,
+ ),
+ type=None)
+ ]),
+ finish_reason=None,
+ index=0,
+ logprobs=None
+ ), ],
+ created=1732635794,
+ model='gpt-4o-2024-08-06',
+ object='chat.completion.chunk',
+ service_tier=None,
+ system_fingerprint='fp_831e067d82',
+ usage=None,
+ )
+ ]
+ parser = DefaultOpenAIMessageParser(None, None)
+ pipes = [SequencePipe(), SequencePipe(), SequencePipe()]
+ messages = parser.from_chat_completion_chunks(items)
+ messages = list(pipeline(pipes, messages))
+ assert len(messages) == len(items) + 2
+
+ stream, receiver = new_basic_connection()
+ with stream:
+ stream.send(messages)
+ with receiver:
+ got = receiver.wait()
+ assert len(got) == 2
+ assert got[0].get_unique_id() != got[1].get_unique_id()
+ assert got[0].type == ""
+ assert got[1].type == MessageType.FUNCTION_CALL
diff --git a/tests/core/messages/test_pipeline.py b/tests/core/messages/test_pipeline.py
new file mode 100644
index 00000000..9ddf0a7f
--- /dev/null
+++ b/tests/core/messages/test_pipeline.py
@@ -0,0 +1,44 @@
+from typing import Iterable
+from ghostos.core.messages import Message
+from ghostos.core.messages.pipeline import SequencePipe, pipeline
+
+
+def test_multi_sequence_pipes():
+ content = "hello world"
+
+ def iter_content(c: str) -> Iterable[Message]:
+ for char in c:
+ yield Message.new_chunk(content=char)
+
+ messages = iter_content(content)
+ parsed = pipeline([SequencePipe(), SequencePipe(), SequencePipe()], messages)
+ got = list(parsed)
+ assert len(got) == len(content) + 1
+ assert got[0].is_head()
+ assert got[0].created > 0
+ assert got[0].content == "h"
+ assert got[-1].is_complete()
+ assert got[-2].is_chunk()
+
+
+def test_multi_sequence_pipe_with_tail():
+ content = "hello world"
+
+ def iter_content(c: str) -> Iterable[Message]:
+ for char in c:
+ yield Message.new_chunk(content=char)
+
+ messages = iter_content(content)
+ messages = SequencePipe().across(messages)
+ messages = list(messages)
+ assert len(messages) == len(content) + 1
+ messages = SequencePipe().across(messages)
+ messages = list(messages)
+ assert len(messages) == len(content) + 1
+
+
+def test_sequence_pipe_with_tail():
+ item = Message.new_tail(content="hello")
+ messages = SequencePipe().across([item])
+ messages = list(messages)
+ assert len(messages) == 1
diff --git a/tests/core/moss/examples/test_baseline.py b/tests/core/moss/examples/test_baseline.py
index 9166c338..4cad8fcd 100644
--- a/tests/core/moss/examples/test_baseline.py
+++ b/tests/core/moss/examples/test_baseline.py
@@ -1,12 +1,15 @@
-from ghostos.core.moss import test_container
-from ghostos.core.moss.abc import MossCompiler, Moss, MOSS_TYPE_NAME
+import time
+
+from ghostos.core.moss import moss_container
+from ghostos.core.moss.abcd import MossCompiler, Moss, MOSS_TYPE_NAME, MossRuntime
from ghostos.core.moss.pycontext import PyContext
from ghostos.core.moss.examples import baseline
from ghostos.contracts.modules import ImportWrapper
+from ghostos.container import Container
def test_baseline_exec():
- container = test_container()
+ container = moss_container()
compiler = container.force_fetch(MossCompiler)
assert compiler is not None
@@ -41,11 +44,15 @@ def test_baseline_exec():
moss = runtime.moss()
assert isinstance(moss, Moss)
- assert isinstance(moss, moss_type)
+ # assert isinstance(moss, moss_type)
prompter = runtime.prompter()
assert prompter is not None
- prompt = prompter.dump_context_prompt()
+ prompt = prompter.dump_module_prompt()
+
+ injection_prompt = prompter.moss_injections_prompt()
+ print("++++", injection_prompt)
+ assert "tester" in injection_prompt
# plus 方法存在.
assert 'def plus' in prompt
@@ -71,13 +78,11 @@ def test_baseline_exec():
life = result.pycontext.properties["life"]
assert life is not None
# 生命周期被执行.
- value = life.value
+ value = result.pycontext.get_prop("life")
assert isinstance(value, list)
- assert "__moss_compile__" in ["__moss_compile__"], "in array test"
+ assert "__moss_compile__" in value, "in array test"
assert '__moss_compile__' in value, "__moss_compile__ not found"
- assert "__moss_attr_prompts__" in value, "__moss_attr_prompts__ not found"
- assert "__moss_prompt__" in value, "__moss_prompt__ not found"
- assert "__moss_exec__" in value, "__moss_exec__ not found"
+ assert 123 == result.pycontext.get_prop("bar")
moss = runtime.moss()
# 验证用 injections 注入.
@@ -85,15 +90,18 @@ def test_baseline_exec():
# 验证依赖注入.
foo = getattr(moss, 'foo')
Foo = runtime.module().__dict__['Foo']
+ assert Foo is baseline.Foo
+ moss.fetch(Foo)
assert foo is not None and isinstance(foo, Foo)
assert foo.foo() == "hello"
# 最后成功销毁.
- runtime.destroy()
+ runtime.close()
+ container.shutdown()
def test_baseline_in_test_mode():
- container = test_container()
+ container = moss_container()
compiler = container.force_fetch(MossCompiler)
assert compiler is not None
@@ -102,86 +110,66 @@ def test_baseline_in_test_mode():
assert compiler.pycontext().module == baseline.__name__
# 获取目标代码.
- runtime = compiler.compile("__test__")
- assert runtime is not None
-
- module = runtime.module()
- # 名字相同.
- assert module.__name__ != baseline.__name__
- assert module.__name__ == "__test__"
- hack_import = module.__dict__.get('__import__', None)
- # 这时就不是原来的 module 了.
- assert hack_import is not None
- assert isinstance(hack_import, ImportWrapper)
-
- # 先测试 ctx
- # with runtime.runtime_ctx():
- # print("hello")
- # buffed = runtime.dump_std_output()
- # assert buffed.startswith("hello")
-
- exists_moss_type = module.__dict__.get(MOSS_TYPE_NAME)
- moss_type = runtime.moss_type()
- # 使用了默认的 MOSS
- assert issubclass(moss_type, Moss)
- assert moss_type is exists_moss_type
-
- moss = runtime.moss()
- assert isinstance(moss, Moss)
- assert isinstance(moss, moss_type)
-
- prompter = runtime.prompter()
- assert prompter is not None
- prompt = prompter.dump_context_prompt()
-
- # 独立编译的模块和之前一样.
- # plus 方法存在.
- assert 'def plus' in prompt
- # 在 moss 标记内的不展示.
- assert "__test__" not in prompt
- # 虽然import 了 inspect 的两个方法, 但一个的 prompt 被重置了.
- assert "def getmembers(" in prompt
- assert "def getsource(" not in prompt
- # 添加的意义不明的注释也应该存在了.
- assert "# hello world" in prompt
-
- # assert moss
- moss = runtime.moss()
- assert getattr(moss, "bar") is 123
-
- # 运行 main 方法.
- result = runtime.execute(target="main", local_args=["moss"])
- # main 方法的运行结果.
- assert result.returns == 4
-
- # 动态加载的 attr.
- assert "life" in result.pycontext.properties, f"life is not found in dumped pycontext {result.pycontext}"
- life = result.pycontext.properties["life"]
- assert life is not None
- # 生命周期被执行.
- value = life.value
- assert isinstance(value, list)
- assert "__moss_compile__" in value, "__moss_compile__ not found"
-
- moss = runtime.moss()
- # 验证用 injections 注入.
- assert getattr(moss, 'bar') == 123
- # 验证依赖注入.
- foo = getattr(moss, 'foo')
- Foo = runtime.module().__dict__['Foo']
- assert foo is not None and isinstance(foo, Foo)
- assert foo.foo() == "hello"
-
- # 最后成功销毁.
- runtime.destroy()
+ with compiler:
+ runtime = compiler.compile("__test__")
+ assert runtime is not None
+
+ module = runtime.module()
+ # 名字相同.
+ assert module.__name__ != baseline.__name__
+ assert module.__name__ == "__test__"
+ with runtime:
+ moss = runtime.moss()
+ moss.hello = "world"
+ result = runtime.execute(target="test_main", local_args=["moss"])
+ assert result.returns == 3
+ assert result.pycontext.get_prop("hello") == "world"
+ container.shutdown()
def test_baseline_with_pycontext_code():
- container = test_container()
+ container = moss_container()
compiler = container.force_fetch(MossCompiler)
assert compiler is not None
-
# join context
line = "print('hello')"
compiler.join_context(PyContext(module=baseline.__name__, code=line))
assert line in compiler.pycontext_code()
+ container.shutdown()
+
+
+def test_moss_gc():
+ from threading import Thread
+ from gc import collect
+ from ghostos.core.moss.impl import MossStub, MossTempModuleType
+ container = moss_container()
+ assert Container.instance_count < 10
+ moss_stub_count = MossStub.instance_count
+ assert moss_stub_count < 10
+
+ def run(c: Container):
+ compiler = c.force_fetch(MossCompiler)
+ compiler = compiler.join_context(PyContext(module=baseline.__name__))
+ runtime = compiler.compile("__test__")
+ assert runtime.moss() is not None
+ assert MossRuntime.instance_count > 0
+ assert MossStub.instance_count > 0
+ assert MossTempModuleType.__instance_count__ > 0
+ with runtime:
+ runtime.execute(target="test_main", local_args=["moss"])
+
+ threads = []
+ for i in range(10):
+ t = Thread(target=run, args=(container,))
+ t.start()
+ threads.append(t)
+ for t in threads:
+ t.join()
+
+ # assert gc success
+ collect()
+ time.sleep(0.05)
+ assert MossTempModuleType.__instance_count__ < 10
+ assert MossRuntime.instance_count == 0
+ assert MossStub.instance_count <= moss_stub_count
+ assert Container.instance_count < 10
diff --git a/tests/core/moss/test_decorators.py b/tests/core/moss/test_decorators.py
index 3a787046..b57fe389 100644
--- a/tests/core/moss/test_decorators.py
+++ b/tests/core/moss/test_decorators.py
@@ -28,9 +28,11 @@ class Case(NamedTuple):
Case(cls_source_code()(Foo), strip_source_indent(inspect.getsource(Foo))),
Case(Foo, strip_source_indent(inspect.getsource(Foo))),
]
+ idx = 0
for case in cases:
prompt = get_prompt(case.value)
- assert prompt == case.expect
+ assert prompt == case.expect, f"{idx} and case is {case}"
+ idx += 1
@definition(doc="test")
@@ -68,6 +70,6 @@ class BarImpl(Bar):
bar: int = 234
bar_prompt = get_prompt(Bar)
- bar_impl_prompt = get_prompt(BarImpl)
assert "Bar:" in bar_prompt
+ bar_impl_prompt = get_prompt(BarImpl)
assert "BarImpl(Bar):" in bar_impl_prompt
diff --git a/tests/core/moss/test_prompts.py b/tests/core/moss/test_prompts.py
index 3dfdc75d..1c2fedf5 100644
--- a/tests/core/moss/test_prompts.py
+++ b/tests/core/moss/test_prompts.py
@@ -1,12 +1,11 @@
import inspect
-from types import ModuleType
from ghostos.core.moss import prompts
from ghostos.core.moss.prompts import reflect_module_locals, compile_attr_prompts
import unittest
from ghostos.core.moss.impl import MossRuntimeImpl
-from ghostos.core.moss.abc import (
+from ghostos.core.moss.abcd import (
MOSS_HIDDEN_MARK, MOSS_HIDDEN_UNMARK,
)
@@ -22,15 +21,10 @@ def test_prompts_baseline():
array.append((name, prompt))
data[name] = prompt
# 从 utils 模块里定义的.
- assert "is_typing" in data
+ assert "get_callable_definition" in data
# typing 库本身的不会出现.
assert "Optional" not in data
# 引用的抽象类应该存在.
- assert "PromptAble" in data
-
- prompt = compile_attr_prompts(ModuleType("test"), array)
- assert "class PromptAble" in prompt
-
def test_prompts_mark_judgement():
@@ -74,7 +68,7 @@ def hidden_function():
return "hidden"
{MOSS_HIDDEN_UNMARK}
print(foo())"""
- assert parser(code3, exclude_moss_mark_code=False) == expected3
+ assert parser(code3, exclude_hide_code=False) == expected3
# test_multiple_hidden_sections
code4 = f"""def foo():
diff --git a/tests/core/moss/test_pycontext.py b/tests/core/moss/test_pycontext.py
index d1d967b0..faa2b2b9 100644
--- a/tests/core/moss/test_pycontext.py
+++ b/tests/core/moss/test_pycontext.py
@@ -1,25 +1,19 @@
+import json
from typing import NamedTuple, Any, List, TypedDict, Optional
from types import ModuleType
-from ghostos.core.moss.pycontext import PyContext, Injection, Property, attr
+from ghostos.core.moss.pycontext import PyContext
from pydantic import BaseModel, Field
-def test_pycontext_imported():
- c = PyContext()
- c.inject(Injection(import_from="foo:bar"))
- assert len(c.injections) == 1
-
-
def test_pycontext_join_baseline():
left = PyContext()
- i = Injection.reflect(Injection)
- left.inject(i)
+ left.set_prop("foo", 123)
right = PyContext()
- right.inject(Injection(import_from="foo:bar"))
+ right.set_prop("bar", 234)
joined = left.join(right)
- assert len(left.injections) == 1
- assert len(right.injections) == 1
- assert len(joined.injections) == 2
+ assert len(left.properties) == 1
+ assert len(right.properties) == 1
+ assert len(joined.properties) == 2
class Foo(BaseModel):
@@ -31,26 +25,23 @@ class Bar(TypedDict, total=False):
def test_property_with_values():
- case = NamedTuple("Case", [("name", str), ("value", Any), ("desc", str)])
+ case = NamedTuple("Case", [("name", str), ("value", Any)])
cases: List[case] = [
- case("foo", 123, ""),
- case("bar", None, "none"),
- case("a", 1.0, "abc"),
- case("", False, "abc"),
- case("foo", Foo(), ""),
- case("bar", Bar(), ""),
+ case("foo", 123),
+ case("bar", None),
+ case("a", 1.0),
+ case("", False),
+ case("foo", Foo()),
+ case("bar", Bar(bar="hello")),
]
+ pycontext = PyContext()
for c in cases:
- p = Property.from_value(name=c.name, value=c.value, desc=c.desc)
- assert p.generate_value() is c.value
- assert p.name is c.name
- assert p.desc is c.desc
-
- j = p.model_dump(exclude_defaults=True)
- p = Property(**j)
- assert p.generate_value() == c.value
- assert p.name == c.name
- assert p.desc == c.desc
+ pycontext.set_prop(c.name, c.value)
+ j = pycontext.model_dump_json()
+ data = json.loads(j)
+ new_one = PyContext(**data)
+ value = new_one.get_prop(c.name)
+ assert c.value == value, j
def test_property_with_local_module():
@@ -65,31 +56,10 @@ class Foo(BaseModel):
exec(compiled, module.__dict__)
foo = module.__dict__["foo"]
assert foo.foo == 123
- p = Property.from_value(name="foo", value=foo)
- assert foo is p.generate_value(module)
- j = p.model_dump(exclude_defaults=True)
- p = Property(**j)
+ pycontext = PyContext()
+ pycontext.set_prop(name="foo", value=foo)
+ assert foo == pycontext.get_prop("foo", module)
+ j = pycontext.model_dump(exclude_defaults=True)
+ p = PyContext(**j)
# 从当前 module 里重新还原出来.
- assert p.generate_value(module) == foo
-
-
-def test_bind_property_as_attr():
- class Zoo:
- foo: Foo = attr(Foo(foo=123), desc="foo")
- bar: Optional[str] = attr(None, desc="bar")
-
- z = Zoo()
- assert Zoo.foo is z.foo
- assert Zoo.foo.foo is 123
- assert Zoo.bar is None
- z.bar = "bar"
- assert z.bar == "bar"
- # 给实例赋值时污染了类.
- assert Zoo.bar == "bar"
-
- foo_prop = Zoo.__dict__["foo"]
- assert isinstance(foo_prop, Property)
- assert foo_prop.name == "foo"
-
- assert str(foo_prop)
- assert f"{foo_prop}"
+ assert p.get_prop("foo", module) == foo
diff --git a/tests/core/test_bootstrap.py b/tests/core/test_bootstrap.py
new file mode 100644
index 00000000..8abb7b1c
--- /dev/null
+++ b/tests/core/test_bootstrap.py
@@ -0,0 +1,6 @@
+from ghostos.bootstrap import expect_workspace_dir
+
+
+def test_expect_app_dir():
+ dirname, ok = expect_workspace_dir()
+ assert isinstance(ok, bool)
diff --git a/tests/framework/eventbuses/test_mem_impl.py b/tests/framework/eventbuses/test_mem_impl.py
index bd007eb7..58783a24 100644
--- a/tests/framework/eventbuses/test_mem_impl.py
+++ b/tests/framework/eventbuses/test_mem_impl.py
@@ -1,10 +1,10 @@
from ghostos.framework.eventbuses.memimpl import MemEventBusImpl
-from ghostos.core.session.events import DefaultEventType
+from ghostos.core.runtime.events import EventTypes
def test_mem_impl_send_pop_event():
bus = MemEventBusImpl()
- e = DefaultEventType.INPUT.new("foo", [])
+ e = EventTypes.INPUT.new("foo", [])
bus.send_event(e, notify=True)
task_id = bus.pop_task_notification()
assert task_id is not None
diff --git a/tests/framework/ghostos/test_session.py b/tests/framework/ghostos/test_session.py
new file mode 100644
index 00000000..2d42301d
--- /dev/null
+++ b/tests/framework/ghostos/test_session.py
@@ -0,0 +1,47 @@
+from ghostos.framework.messengers import DefaultMessenger
+from ghostos.core.runtime.threads import GoThreadInfo
+from ghostos.core.messages import Message
+from threading import Lock, Thread
+
+
+def test_thread_sending_message_with_stage():
+ thread = GoThreadInfo.new(None)
+ lock = Lock()
+
+ def send_thread(content: str, stage: str):
+ items = []
+ for c in content:
+ msg = Message.new_chunk(content=c)
+ items.append(msg)
+ messenger = DefaultMessenger(None, stage=stage)
+ messenger.send(items)
+ messages, callers = messenger.flush()
+ with lock:
+ thread.append(*messages)
+
+ cases = [
+ ("hello world1", ""),
+ ("hello world2", "a"),
+ ("hello world3", "a"),
+ ("hello world4", "b"),
+ ("hello world5", ""),
+ ]
+
+ run = []
+ for c in cases:
+ t = Thread(target=send_thread, args=c)
+ t.start()
+ run.append(t)
+
+ for t in run:
+ t.join()
+
+ assert len(thread.last_turn().added) == 5
+ for message in thread.last_turn().added:
+ assert message.content.startswith("hello world")
+
+ prompt = thread.to_prompt([], [""])
+ assert len(prompt.added) == 2
+ prompt = thread.to_prompt([], ["a", "b"])
+ assert len(prompt.added) == 3
+
diff --git a/tests/framework/llms/test_llms.py b/tests/framework/llms/test_llms_config.py
similarity index 51%
rename from tests/framework/llms/test_llms.py
rename to tests/framework/llms/test_llms_config.py
index 157be094..c6bd739a 100644
--- a/tests/framework/llms/test_llms.py
+++ b/tests/framework/llms/test_llms_config.py
@@ -1,19 +1,46 @@
import os
+
+import yaml
+
from ghostos.container import Container
-from ghostos.core.llms import LLMsConfig, LLMs
+from ghostos.core.llms import LLMsConfig, ServiceConf, ModelConf, LLMs
from ghostos.contracts.configs import YamlConfig, Configs
from ghostos.framework.configs import ConfigsByStorageProvider
-from ghostos.framework.storage import FileStorageProvider
-from ghostos.framework.llms import ConfigBasedLLMsProvider
+from ghostos.framework.storage import MemStorage, Storage
+from ghostos.framework.logger import LoggerItf, FakeLogger
+from ghostos.framework.llms import ConfigBasedLLMsProvider, PromptStorage, PromptStorageImpl
def _prepare_container() -> Container:
- dirname = os.path.dirname
- demo_dir = dirname(__file__) + "/../../../ghostos/demo/"
- demo_dir = os.path.abspath(demo_dir)
container = Container()
- container.register(FileStorageProvider(demo_dir))
+ storage = MemStorage()
+ container.set(LoggerItf, FakeLogger())
+ container.set(Storage, storage)
container.register(ConfigsByStorageProvider('configs'))
+ container.set(PromptStorage, PromptStorageImpl(storage.sub_storage("prompts")))
+
+ data = LLMsConfig(
+ services=[
+ ServiceConf(
+ name='moonshot',
+ base_url="http://moonshot.com",
+ token="$MOONSHOT_TOKEN",
+ )
+ ],
+ default="moonshot-v1-32k",
+ models={
+ "moonshot-v1-32k": ModelConf(
+ model="moonshot-v1-32k",
+ service="moonshot"
+ ),
+ "gpt-4": dict(
+ model="moonshot-v1-32k",
+ service="moonshot"
+ )
+ }
+ )
+
+ storage.put("configs/llms_conf.yml", yaml.safe_dump(data.model_dump()).encode())
return container
@@ -41,7 +68,7 @@ def test_llms():
存在文件依赖关系.
"""
container: Container = _prepare_container()
- container.register(ConfigBasedLLMsProvider("llms_conf.yml"))
+ container.register(ConfigBasedLLMsProvider())
llms = container.force_fetch(LLMs)
api = llms.get_api("gpt-4")
diff --git a/tests/framework/llms/test_prompt_storage.py b/tests/framework/llms/test_prompt_storage.py
new file mode 100644
index 00000000..318a29b5
--- /dev/null
+++ b/tests/framework/llms/test_prompt_storage.py
@@ -0,0 +1,18 @@
+from ghostos.framework.llms import PromptStorageImpl, Prompt
+from ghostos.framework.storage import MemStorage
+from ghostos.core.messages import Message
+
+
+def test_prompt_storage_baseline():
+ storage = MemStorage()
+ prompts = PromptStorageImpl(storage)
+
+ prompt = Prompt()
+ prompt.inputs.append(Message.new_tail(content="hello world"))
+ id_ = prompt.id
+
+ prompts.save(prompt)
+ got = prompts.get(id_)
+ assert got.inputs == prompt.inputs
+ assert got.id == prompt.id
+ assert got == prompt
diff --git a/tests/framework/messages/test_buffer.py b/tests/framework/messages/test_buffer.py
index 722beee0..b9918a4a 100644
--- a/tests/framework/messages/test_buffer.py
+++ b/tests/framework/messages/test_buffer.py
@@ -1,218 +1,246 @@
-from ghostos.core.messages import (
- Message
-)
-from ghostos.core.llms import FunctionalToken
-from ghostos.framework.messages import DefaultBuffer
-
-
-def test_default_buffer_baseline():
- buffer = DefaultBuffer()
- buffer2 = DefaultBuffer()
-
- content1 = "hello"
- content2 = "world"
-
- msg1 = Message.new_head()
- sent = buffer.buff(msg1)
- i = 0
- for item in sent:
- buffer2.buff(item)
- i += 1
- # 空首包也发送, 对齐 moonshot 协议.
- assert i == 1
-
- for c in content1:
- pack = Message.new_pack(content=c)
- sent = buffer.buff(pack)
- for item in sent:
- buffer2.buff(item)
-
- buffed = buffer.flush()
- assert len(buffed.messages) == 1
- assert buffed.messages[0].content == content1
- assert buffed.messages[0].memory is None
-
- new_head = Message.new_head()
- buffer2.buff(new_head)
-
- for c in content2:
- pack = Message.new_pack(content=c)
- buffer2.buff(pack)
-
- buffed = buffer2.flush()
- print(buffed)
- assert len(buffed.messages) == 2
-
-
-def test_functional_token_baseline():
- buffer = DefaultBuffer(
- functional_tokens=[
- FunctionalToken(token=":moss>", name="moss", description="desc", deliver=False)
- ]
- )
-
- content = """
-hello
-:moss>
-world
-"""
-
- for c in content:
- msg = Message.new_pack(content=c)
- buffer.buff(msg)
-
- flushed = buffer.flush()
- assert len(flushed.messages) == 1
- assert len(flushed.callers) == 1
- assert flushed.callers[0].name == "moss"
- assert flushed.callers[0].arguments == "\nworld\n"
- assert flushed.messages[0].content == "\nhello\n"
-
-
-def test_buffer_sent():
- buffer = DefaultBuffer()
- content = "hello world"
- count = 0
- count_has_message_id = 0
-
- for c in content:
- msg = Message.new_pack(content=c)
- sent = buffer.buff(msg)
- for i in sent:
- assert not i.is_empty()
- if i.msg_id:
- count_has_message_id += 1
- count += 1
- assert count == len(content)
- assert count_has_message_id == 1
- assert len(buffer.flush().messages) == 1
-
-
-def test_buffer_sent_one_tail():
- buffer = DefaultBuffer()
- content = "hello world"
- tails = 0
- for c in content:
- msg = Message.new_pack(content=c)
- sent = buffer.buff(msg)
- for i in sent:
- if not i.pack:
- tails += 1
- buffed = buffer.flush()
- for i in buffed.unsent:
- if not i.pack:
- tails += 1
- assert tails == 1
-
-
-def test_buffer_with_moss_token():
- data = '''{
-"msg_id": "e28c37c8-4292-4c5e-8c22-25b85fd65af3",
-"created": 1722267720.0,
-"pack": false,
-"content": ""
-}'''
- import json
- j = json.loads(data)
- message = Message(**j)
- assert message.content is not None
-
- buffer = DefaultBuffer(
- functional_tokens=[FunctionalToken(token=">moss:", name="moss", description="desc", deliver=False)]
- )
-
- content = "好的，我会帮你播放这首歌。\n\n>moss:\ndef main(os: MOSS) -> Operator:\n # Search for the song \"七里香\" by 周杰伦\n song_list = os.player.search(\"\", \"周杰伦\", \"七里香\")\n \n # Check if the song is found\n if \"七里香\" in song_list:\n # Play the song\n playing = os.player.play(\"七里香\")\n \n # Check if the song is playing\n if playing:\n return\n os.mindflow.finish(\"正在播放周杰伦的《七里香》。\")\n else:\n return os.mindflow.fail(\"无法播放周杰伦的《七里香》。\")\n else:\n return os.mindflow.fail(\"未找到周杰伦的《七里香》。\")"
- for c in content:
- p = Message.new_pack(content=c)
- buffer.buff(p)
- buffed = buffer.flush()
- assert len(buffed.messages) == 1
- assert len(buffed.callers) == 1
-
-
-def test_buffer_with_sep_content():
- functional_tokens = [FunctionalToken(
- token=">moss:",
- name="moss",
- description="desc",
- deliver=False,
- )]
-
- buffer = DefaultBuffer(functional_tokens=functional_tokens)
-
- contents = ["he", "llo >mo", "ss: w", "orld"]
- content = "".join(contents)
- for c in contents:
- msg = Message.new_pack(content=c)
- buffer.buff(msg)
- flushed = buffer.flush()
- assert len(flushed.messages) == 1
- assert len(list(flushed.callers)) > 0
- message = flushed.messages[0]
- assert message.content == "hello "
- assert message.memory == content
- caller = flushed.callers[0]
- assert caller.name == "moss"
- assert caller.arguments == " world"
-
- unsent = list(flushed.unsent)
- assert len(unsent) == 1
- assert unsent[0].content == "hello "
- assert unsent[0].memory == content
- assert len(unsent[0].callers) == 1
-
-
-def test_buffer_with_tail_item():
- buffer = DefaultBuffer()
- header = Message.new_head(content="")
- buffer.buff(header)
- content = "hello"
- for c in content:
- msg = Message.new_pack(content=c)
- buffer.buff(msg)
- tail = Message.new_tail(content="hello world", msg_id=header.msg_id)
- buffer.buff(tail)
- flushed = buffer.flush()
- assert len(flushed.messages) == 1
- assert flushed.messages[0].content == "hello world"
-
-
-def test_buffer_header_with_payload():
- buffer = DefaultBuffer()
- header = Message.new_head(content="")
- header.payloads["foo"] = {}
- buffer.buff(header)
- content = "hello"
- buffer.buff(Message.new_pack(content=""))
- for c in content:
- msg = Message.new_pack(content=c)
- buffer.buff(msg)
- flushed = buffer.flush()
- assert len(flushed.messages) == 1
- assert flushed.messages[0].content == "hello"
-
-
-def test_buffer_with_xml_functional_token():
- functional_tokens = [FunctionalToken(
- token="",
- end_token=" ",
- name="moss",
- description="desc",
- deliver=False,
- )]
-
- buffer = DefaultBuffer(functional_tokens=functional_tokens)
- contents = ["he", "llo w", "orld']
- content = "".join(contents)
- for c in contents:
- msg = Message.new_pack(content=c)
- buffer.buff(msg)
- flushed = buffer.flush()
- assert len(flushed.messages) == 1
- assert len(list(flushed.callers)) > 0
- message = flushed.messages[0]
- assert message.content == "hello "
- assert message.memory == content
- caller = flushed.callers[0]
- assert caller.name == "moss"
- assert caller.arguments == "world"
+# deprecated
+# from ghostos.core.messages import (
+# Message
+# )
+# from ghostos.core.llms import FunctionalToken
+# from ghostos.framework.messages import DefaultBuffer
+#
+#
+# def test_default_buffer_baseline():
+# buffer = DefaultBuffer()
+# buffer2 = DefaultBuffer()
+#
+# content1 = "hello"
+# content2 = "world"
+#
+# msg1 = Message.new_head()
+# sent = buffer.add(msg1)
+# i = 0
+# for item in sent:
+# buffer2.add(item)
+# i += 1
+# # 空首包也发送, 对齐 moonshot 协议.
+# assert i == 1
+#
+# for c in content1:
+# pack = Message.new_chunk(content=c)
+# sent = buffer.add(pack)
+# for item in sent:
+# buffer2.add(item)
+#
+# buffed = buffer.flush()
+# assert len(buffed.messages) == 1
+# assert buffed.messages[0].content == content1
+# assert buffed.messages[0].memory is None
+#
+# new_head = Message.new_head()
+# buffer2.add(new_head)
+#
+# for c in content2:
+# pack = Message.new_chunk(content=c)
+# buffer2.add(pack)
+#
+# buffed = buffer2.flush()
+# print(buffed)
+# assert len(buffed.messages) == 2
+#
+#
+# def test_functional_token_baseline():
+# buffer = DefaultBuffer(
+# functional_tokens=[
+# FunctionalToken(token=":moss>", name="moss", description="desc", deliver=False)
+# ]
+# )
+#
+# content = """
+# hello
+# :moss>
+# world
+# """
+#
+# for c in content:
+# msg = Message.new_chunk(content=c)
+# buffer.add(msg)
+#
+# flushed = buffer.flush()
+# assert len(flushed.messages) == 1
+# assert len(flushed.callers) == 1
+# assert flushed.callers[0].name == "moss"
+# assert flushed.callers[0].arguments == "\nworld\n"
+# assert flushed.messages[0].content == "\nhello\n"
+#
+#
+# def test_buffer_sent():
+# buffer = DefaultBuffer()
+# content = "hello world"
+# count = 0
+# count_has_message_id = 0
+#
+# for c in content:
+# msg = Message.new_chunk(content=c)
+# sent = buffer.add(msg)
+# for i in sent:
+# assert not i.is_empty()
+# if i.msg_id:
+# count_has_message_id += 1
+# count += 1
+# assert count == len(content)
+# assert count_has_message_id == count
+# assert len(buffer.flush().messages) == 1
+#
+#
+# def test_buffer_sent_one_tail():
+# buffer = DefaultBuffer()
+# content = "hello world"
+# tails = 0
+# for c in content:
+# msg = Message.new_chunk(content=c)
+# sent = buffer.add(msg)
+# for i in sent:
+# if not i.chunk:
+# tails += 1
+# buffed = buffer.flush()
+# for i in buffed.unsent:
+# if not i.chunk:
+# tails += 1
+# assert tails == 1
+#
+#
+# def test_buffer_with_moss_token():
+# data = '''{
+# "msg_id": "e28c37c8-4292-4c5e-8c22-25b85fd65af3",
+# "created": 1722267720.0,
+# "pack": false,
+# "content": ""
+# }'''
+# import json
+# j = json.loads(data)
+# message = Message(**j)
+# assert message.content is not None
+#
+# buffer = DefaultBuffer(
+# functional_tokens=[FunctionalToken(token=">moss:", name="moss", description="desc", deliver=False)]
+# )
+#
+# content = "好的，我会帮你播放这首歌。\n\n>moss:\ndef main(os: MOSS) -> Operator:\n # Search for the song \"七里香\" by 周杰伦\n song_list = os.player.search(\"\", \"周杰伦\", \"七里香\")\n \n # Check if the song is found\n if \"七里香\" in song_list:\n # Play the song\n playing = os.player.play(\"七里香\")\n \n # Check if the song is playing\n if playing:\n return\n os.mindflow.finish(\"正在播放周杰伦的《七里香》。\")\n else:\n return os.mindflow.fail(\"无法播放周杰伦的《七里香》。\")\n else:\n return os.mindflow.fail(\"未找到周杰伦的《七里香》。\")"
+# for c in content:
+# p = Message.new_chunk(content=c)
+# buffer.add(p)
+# buffed = buffer.flush()
+# assert len(buffed.messages) == 1
+# assert len(buffed.callers) == 1
+#
+#
+# def test_buffer_with_sep_content():
+# functional_tokens = [FunctionalToken(
+# token=">moss:",
+# name="moss",
+# description="desc",
+# deliver=False,
+# )]
+#
+# buffer = DefaultBuffer(functional_tokens=functional_tokens)
+#
+# contents = ["he", "llo >mo", "ss: w", "orld"]
+# content = "".join(contents)
+# for c in contents:
+# msg = Message.new_chunk(content=c)
+# buffer.add(msg)
+# flushed = buffer.flush()
+# assert len(flushed.messages) == 1
+# assert len(list(flushed.callers)) > 0
+# message = flushed.messages[0]
+# assert message.content == "hello "
+# assert message.memory == content
+# caller = flushed.callers[0]
+# assert caller.name == "moss"
+# assert caller.arguments == " world"
+#
+# unsent = list(flushed.unsent)
+# assert len(unsent) == 1
+# assert unsent[0].content == "hello "
+# assert unsent[0].memory == content
+# assert len(unsent[0].callers) == 1
+#
+#
+# def test_buffer_with_tail_item():
+# buffer = DefaultBuffer()
+# header = Message.new_head(content="")
+# buffer.add(header)
+# content = "hello"
+# for c in content:
+# msg = Message.new_chunk(content=c)
+# buffer.add(msg)
+# tail = Message.new_tail(content="hello world", msg_id=header.msg_id)
+# buffer.add(tail)
+# flushed = buffer.flush()
+# assert len(flushed.messages) == 1
+# assert flushed.messages[0].content == "hello world"
+#
+#
+# def test_buffer_header_with_payload():
+# buffer = DefaultBuffer()
+# header = Message.new_head(content="")
+# header.payloads["foo"] = {}
+# buffer.add(header)
+# content = "hello"
+# buffer.add(Message.new_chunk(content=""))
+# for c in content:
+# msg = Message.new_chunk(content=c)
+# buffer.add(msg)
+# flushed = buffer.flush()
+# assert len(flushed.messages) == 1
+# assert flushed.messages[0].content == "hello"
+#
+#
+# def test_buffer_with_xml_functional_token():
+# functional_tokens = [FunctionalToken(
+# token="",
+# end_token=" ",
+# name="moss",
+# description="desc",
+# deliver=False,
+# )]
+#
+# buffer = DefaultBuffer(functional_tokens=functional_tokens)
+# contents = ["he", "llo w", "orld']
+# content = "".join(contents)
+# for c in contents:
+# msg = Message.new_chunk(content=c)
+# buffer.add(msg)
+# flushed = buffer.flush()
+# assert len(flushed.messages) == 1
+# assert len(list(flushed.callers)) > 0
+# message = flushed.messages[0]
+# assert message.content == "hello "
+# assert message.memory == content
+# caller = flushed.callers[0]
+# assert caller.name == "moss"
+# assert caller.arguments == "world"
+#
+#
+# def test_buffer_with_visible_functional_token():
+# functional_tokens = [FunctionalToken(
+# token="",
+# end_token=" ",
+# name="moss",
+# description="desc",
+# visible=True,
+# deliver=False,
+# )]
+#
+# buffer = DefaultBuffer(functional_tokens=functional_tokens)
+# contents = ["he", "llo w", "orld']
+# content = "".join(contents)
+# for c in contents:
+# msg = Message.new_chunk(content=c)
+# buffer.add(msg)
+# flushed = buffer.flush()
+# assert len(flushed.messages) == 1
+# assert len(list(flushed.callers)) > 0
+# message = flushed.messages[0]
+# assert message.content == content
+# assert message.memory is None
+# caller = flushed.callers[0]
+# assert caller.name == "moss"
+# assert caller.arguments == "world"
diff --git a/tests/framework/messenger/test_messenger.py b/tests/framework/messenger/test_messenger.py
index c7ed7263..726464b8 100644
--- a/tests/framework/messenger/test_messenger.py
+++ b/tests/framework/messenger/test_messenger.py
@@ -1,66 +1,57 @@
from ghostos.framework.messengers import DefaultMessenger
-from ghostos.core.session.threads import MsgThread
-from ghostos.core.messages import Message
-from ghostos.core.llms import FunctionalToken
+from ghostos.core.messages import Message, new_basic_connection, MessageType, ReceiverBuffer
def test_default_messenger_baseline():
- thread = MsgThread()
- messenger = DefaultMessenger(thread=thread)
+ messenger = DefaultMessenger(None)
content = "hello world"
+ items = []
for c in content:
- msg = Message.new_pack(content=c)
- success = messenger.deliver(msg)
- assert success
- messenger.flush()
- assert len(thread.current.generates) == 1
- assert thread.current.generates[0].content == content
-
-
-def test_messenger_with_moss_xml_token():
- functional_tokens = [FunctionalToken(
- token=">moss:",
- name="moss",
- description="desc",
- deliver=False,
- )]
-
- thread = MsgThread()
- messenger = DefaultMessenger(thread=thread, functional_tokens=functional_tokens)
-
- contents = ["he", "llo >mo", "ss: w", "orld"]
- content = "".join(contents)
- for c in contents:
- msg = Message.new_pack(content=c)
- messenger.deliver(msg)
- flushed = messenger.flush()
- assert len(list(flushed.callers)) > 0
- message = flushed.messages[0]
- assert message.content != content
- assert message.memory == content
- caller = flushed.callers[0]
- assert caller.name == "moss"
- assert caller.arguments == " world"
-
- assert len(thread.current.generates) == 1
- assert len(thread.current.generates[0].callers) == 1
-
-
-def test_messenger_with_single_message():
- functional_tokens = [FunctionalToken(
- token="",
- end_token=" ",
- name="moss",
- description="desc",
- deliver=False,
- )]
-
- thread = MsgThread()
- messenger = DefaultMessenger(thread=thread, functional_tokens=functional_tokens)
-
- content = "def main():\n pass "
- messenger.say(content)
- flushed = messenger.flush()
- assert flushed.messages[0].content == ""
- assert flushed.messages[0].memory == content
- assert len(flushed.callers) == 1
+ msg = Message.new_chunk(content=c)
+ items.append(msg)
+ messenger.send(items)
+ messages, callers = messenger.flush()
+ assert len(messages) == 1
+ assert len(callers) == 0
+
+
+def test_messenger_with_upstream():
+ stream, receiver = new_basic_connection()
+ messenger = DefaultMessenger(stream)
+ items = []
+ content = "hello world"
+ for c in content:
+ msg = Message.new_chunk(content=c)
+ items.append(msg)
+ with stream:
+ messenger.send(items)
+ flushed, _ = messenger.flush()
+ messages = receiver.wait()
+ assert len(flushed) == 1
+ assert len(messages) == 1
+
+
+def test_messenger_with_function_call():
+ stream, receiver = new_basic_connection()
+ messenger = DefaultMessenger(stream)
+ items = []
+ content = "hello world"
+ for c in content:
+ msg = Message.new_chunk(content=c)
+ items.append(msg)
+ for c in content:
+ msg = Message.new_chunk(content=c, typ_=MessageType.FUNCTION_CALL, call_id="123", name="good")
+ items.append(msg)
+ with stream:
+ messenger.send(items)
+ flushed, callers = messenger.flush()
+ assert len(flushed) == 2
+ assert len(callers) == 1
+ with receiver:
+ buffer = ReceiverBuffer.new(receiver.recv())
+ assert MessageType.is_text(buffer.head())
+ assert len(list(buffer.chunks())) == len(content)
+ buffer = buffer.next()
+ assert MessageType.FUNCTION_CALL.match(buffer.head())
+ assert len(list(buffer.chunks())) == len(content)
+ assert buffer.next() is None
diff --git a/tests/framework/openai_realtime/test_events.py b/tests/framework/openai_realtime/test_events.py
new file mode 100644
index 00000000..70b04f99
--- /dev/null
+++ b/tests/framework/openai_realtime/test_events.py
@@ -0,0 +1,8 @@
+from ghostos.framework.openai_realtime.event_data_objects import SessionObject
+from ghostos.framework.openai_realtime.event_from_client import SessionUpdate
+
+
+def test_session_update_event():
+ session = SessionObject()
+ ce = SessionUpdate(session=session)
+ assert ce.session == session
diff --git a/tests/framework/openai_realtime/test_objects.py b/tests/framework/openai_realtime/test_objects.py
new file mode 100644
index 00000000..0461dbf1
--- /dev/null
+++ b/tests/framework/openai_realtime/test_objects.py
@@ -0,0 +1,22 @@
+from ghostos.framework.openai_realtime.configs import OpenAIRealtimeAppConf
+from ghostos.framework.openai_realtime.event_data_objects import SessionObject, MessageItem
+from ghostos.core.messages.message_classes import FunctionCallMessage
+
+
+def test_configs_session():
+ conf = OpenAIRealtimeAppConf()
+ assert isinstance(conf.session, SessionObject)
+
+
+def test_message_item():
+ item = MessageItem(id='item_Agblchy3WGzjwTH8jOMfn', type='function_call', status='completed', role=None,
+ content=None,
+ call_id='call_6Bp2ZuJoFu1NI7fI', name='moss',
+ arguments='{"code":"def run(moss: Moss):\\n moss.body.new_move(True).spin(360, 1)\\n"}',
+ output=None)
+ head = item.to_message_head()
+ assert head.name == "moss"
+ complete = item.to_complete_message()
+ assert complete.name == "moss"
+ call = FunctionCallMessage.from_message(complete)
+ assert call.caller.name == "moss"
diff --git a/tests/framework/openai_realtime/test_realtime_configs.py b/tests/framework/openai_realtime/test_realtime_configs.py
new file mode 100644
index 00000000..d8e44f3d
--- /dev/null
+++ b/tests/framework/openai_realtime/test_realtime_configs.py
@@ -0,0 +1,9 @@
+from ghostos.framework.openai_realtime.configs import OpenAIRealtimeAppConf
+
+
+def test_vad_session():
+ conf = OpenAIRealtimeAppConf()
+ session = conf.get_session_obj(vad_mode=True)
+ assert session.turn_detection is not None
+ session = conf.get_session_obj(vad_mode=False)
+ assert session.turn_detection is None
diff --git a/tests/framework/tasks/test_storage_impl.py b/tests/framework/tasks/test_storage_impl.py
index e8bd7715..40312082 100644
--- a/tests/framework/tasks/test_storage_impl.py
+++ b/tests/framework/tasks/test_storage_impl.py
@@ -1,37 +1,58 @@
from ghostos.framework.storage import MemStorage
-from ghostos.framework.tasks.storage_tasks import StorageTasksImpl
+from ghostos.framework.tasks.storage_tasks import StorageGoTasksImpl
from ghostos.framework.logger import FakeLogger
-from ghostos.core.session import Task
+from ghostos.core.runtime import GoTaskStruct, TaskBrief
from ghostos.entity import EntityMeta
+import time
def test_storage_tasks_impl():
storage = MemStorage()
- tasks = StorageTasksImpl(storage, FakeLogger())
- task = Task.new(
+ tasks = StorageGoTasksImpl(storage, FakeLogger())
+ task = GoTaskStruct.new(
task_id="task_id",
- session_id="session_id",
+ shell_id="shell_id",
process_id="process_id",
+ depth=0,
name="name",
description="description",
- meta=EntityMeta(type="type", data={}),
+ meta=EntityMeta(type="type", content=""),
)
- t = tasks.get_task(task.task_id, False)
+ t = tasks.get_task(task.task_id)
assert t is None
tasks.save_task(task)
- t = tasks.get_task(task.task_id, False)
+ t = tasks.get_task(task.task_id)
assert t is not None
- assert t.lock is None
- locked = tasks.get_task(task.task_id, True)
- assert locked.lock is not None
- locked2 = tasks.get_task(task.task_id, True)
- assert locked2 is None
- tasks.unlock_task(locked.task_id, locked.lock)
+ with tasks.lock_task(task.task_id):
+ locker = tasks.lock_task(task.task_id)
+ new_turn = task.new_turn()
+ tasks.save_task(new_turn)
+ assert locker.acquire() is False
- locked2 = tasks.get_task(task.task_id, True)
- assert locked2.lock is not None
+ locker = tasks.lock_task(task.task_id)
+ assert locker.acquire() is True
+ locker.release()
- new_lock = tasks.refresh_task_lock(locked2.task_id, locked2.lock)
- assert new_lock is not locked2.lock
+ new_got = tasks.get_task(task.task_id)
+ assert new_got != task
+ assert new_got == new_turn
+
+ assert TaskBrief.from_task(task) == TaskBrief.from_task(new_got)
+
+
+def test_storage_tasks_impl_lock():
+ storage = MemStorage()
+ tasks = StorageGoTasksImpl(storage, FakeLogger())
+ locker = tasks.lock_task("task_id", overdue=0.1)
+ assert not locker.acquired()
+ for i in range(5):
+ time.sleep(0.05)
+ assert locker.acquire()
+ assert locker.acquired()
+ assert locker.release()
+ assert not locker.acquired()
+ with locker:
+ assert locker.acquired()
+ assert not locker.acquired()
diff --git a/tests/framework/threads/test_storage_threads.py b/tests/framework/threads/test_storage_threads.py
new file mode 100644
index 00000000..b812d544
--- /dev/null
+++ b/tests/framework/threads/test_storage_threads.py
@@ -0,0 +1,35 @@
+from ghostos.framework.threads import MsgThreadRepoByStorageProvider, GoThreads, GoThreadInfo
+from ghostos.framework.storage import MemStorage, Storage
+from ghostos.framework.logger import FakeLogger, LoggerItf
+from ghostos.core.messages import Message
+from ghostos.core.moss import PyContext
+from ghostos.container import Container
+
+
+def _prepare_container() -> Container:
+ container = Container()
+ container.set(Storage, MemStorage())
+ container.set(LoggerItf, FakeLogger())
+ container.register(MsgThreadRepoByStorageProvider())
+ return container
+
+
+def test_threads_baseline():
+ thread = GoThreadInfo()
+ pycontext = PyContext(module=PyContext.__module__)
+ thread.new_turn(None, pycontext=pycontext)
+ thread.append(Message.new_tail(content="hello world"))
+
+ tid = thread.id
+ container = _prepare_container()
+ threads = container.force_fetch(GoThreads)
+ threads.save_thread(thread)
+
+ got = threads.get_thread(tid, create=False)
+ assert got is not None
+ assert got == thread
+
+ fork = threads.fork_thread(got)
+ assert fork.id != got.id
+ assert fork.root_id == got.id
+ assert fork.parent_id == got.id
diff --git a/tests/framework/variables/test_variables.py b/tests/framework/variables/test_variables.py
new file mode 100644
index 00000000..a3233930
--- /dev/null
+++ b/tests/framework/variables/test_variables.py
@@ -0,0 +1,28 @@
+from ghostos.framework.variables.variables_impl import VariablesImpl
+from ghostos.framework.storage import MemStorage
+from pydantic import BaseModel
+
+
+class Foo(BaseModel):
+ a: int = 123
+ b: str = 'test'
+
+
+def test_variables_impl_baseline():
+ variables = VariablesImpl(MemStorage())
+ v = variables.save(9527, "random int")
+ assert v.desc == "random int"
+ got = variables.load(v.vid, int, True)
+ assert got == 9527
+
+ cases = [
+ (9527, "random int", int, True),
+ ("hello world", "", str, False),
+ (Foo(), "", Foo, True),
+ (Foo(), "", None, False),
+ ]
+ for case in cases:
+ value, desc, expect, force = case
+ v = variables.save(value, desc)
+ got = variables.load(v.vid, expect, force)
+ assert got == value, f"{value} != {got}"
diff --git a/tests/helpers/test_modules.py b/tests/helpers/test_modules_helper.py
similarity index 100%
rename from tests/helpers/test_modules.py
rename to tests/helpers/test_modules_helper.py
diff --git a/tests/helpers/test_timeleft.py b/tests/helpers/test_timeleft.py
new file mode 100644
index 00000000..9ec97c69
--- /dev/null
+++ b/tests/helpers/test_timeleft.py
@@ -0,0 +1,8 @@
+from ghostos.helpers import Timeleft
+
+
+def test_timeleft_with_zero():
+ left = Timeleft(0)
+ assert left.alive()
+ assert left.alive()
+ assert left.left() == 0
diff --git a/tests/helpers/test_tree_sitter.py b/tests/helpers/test_tree_sitter.py
index 33376c38..7a5321f8 100644
--- a/tests/helpers/test_tree_sitter.py
+++ b/tests/helpers/test_tree_sitter.py
@@ -1 +1,39 @@
-from ghostos.helpers.tree_sitter import PyNode, PyModuleNode
+from ghostos.helpers.tree_sitter import code_syntax_check
+
+
+def test_lint_code_success():
+ code = """
+import inspect
+
+def main():
+ print("hello world")
+
+source = inspect.getsource(main)
+print(source)
+"""
+ error = code_syntax_check(code.strip())
+ assert error is None
+
+
+def test_lint_code_without_quote():
+ code = """
+def main():
+ print("hello world)
+
+source = inspect.getsource(main)
+print(source)
+"""
+ error = code_syntax_check(code.strip())
+ assert error is not None
+
+
+def test_lint_code_many_errors():
+ code = """
+def main():
+ print("hello world)
+
+source = inspect.getsource(main
+print(source)
+"""
+ error = code_syntax_check(code.strip())
+ assert error and "hello world)" in error
diff --git a/tests/python/test_asyncio.py b/tests/python/test_asyncio.py
new file mode 100644
index 00000000..686dbbc9
--- /dev/null
+++ b/tests/python/test_asyncio.py
@@ -0,0 +1,57 @@
+import asyncio
+import concurrent
+
+
+def test_loop_run_until_complete():
+ async def foo():
+ return 123
+
+ loop = asyncio.new_event_loop()
+ loop.run_until_complete(foo())
+
+
+def test_gather():
+ async def bar():
+ await asyncio.sleep(0.1)
+ # print("bar")
+ return 123
+
+ async def baz():
+ # print("baz")
+ return 123
+
+ async def foo():
+ await asyncio.gather(bar(), baz())
+
+ lp = asyncio.new_event_loop()
+ lp.run_until_complete(foo())
+
+
+def test_producer_and_consumer():
+ class Main:
+ stop = False
+
+ got = []
+
+ async def _producer(self, q: asyncio.Queue):
+ count = 0
+ while not self.stop:
+ q.put_nowait(count)
+ count += 1
+ await asyncio.sleep(0.1)
+
+ async def _consumer(self, q: asyncio.Queue):
+ while not self.stop:
+ v = await q.get()
+ self.got.append(v)
+ self.stop = v > 2
+
+ async def run(self):
+ async with asyncio.TaskGroup() as tg:
+ q = asyncio.Queue()
+ tg.create_task(self._producer(q))
+ tg.create_task(self._consumer(q))
+
+ main = Main()
+ asyncio.run(main.run())
+ assert main.got == [0, 1, 2, 3]
diff --git a/tests/python/test_bytes.py b/tests/python/test_bytes.py
new file mode 100644
index 00000000..02bbdefb
--- /dev/null
+++ b/tests/python/test_bytes.py
@@ -0,0 +1,8 @@
+from io import BytesIO
+
+
+def test_bytes():
+ b = BytesIO()
+ b.write(b'hello')
+ got = b.getvalue()
+ assert len(got) == 5
diff --git a/tests/python/test_class.py b/tests/python/test_class.py
index b089480d..a5bbe6e2 100644
--- a/tests/python/test_class.py
+++ b/tests/python/test_class.py
@@ -1,4 +1,4 @@
-from typing import Dict, SupportsAbs as _SupportsAbs
+from typing import Dict, SupportsAbs as _SupportsAbs, List
import inspect
@@ -201,3 +201,97 @@ class Foo1(Foo):
foo: int = 11
assert Foo1 is not Foo
+
+
+def test_generic_class_is_same():
+ from typing import Generic, TypeVar
+ T = TypeVar("T")
+
+ class Foo(Generic[T]):
+ def __init__(self, val: T):
+ self.val = val
+
+ assert Foo[int] is Foo[int]
+ obj1 = Foo[int](1)
+ obj2 = Foo[int](2)
+ assert type(obj1) is type(obj2)
+
+
+def test_protocol_and_abc():
+ from abc import ABC
+ from typing import Protocol
+
+ class _Foo(Protocol):
+ foo = 1
+
+ class _Bar(_Foo, ABC):
+ pass
+
+ class _Baz(_Bar):
+ pass
+
+ b = _Baz()
+ assert b.foo == 1
+
+
+def test_attr_of_class():
+ class Foo:
+ foo = 1
+ bar: int
+ baz: int = 3
+
+ assert Foo.foo == 1
+ assert Foo().foo == 1
+ assert not hasattr(Foo, "bar")
+ assert not hasattr(Foo(), "bar")
+ from typing import get_type_hints
+ props = get_type_hints(Foo)
+ assert "bar" in props
+ assert "baz" in props
+ assert hasattr(Foo, "baz")
+ assert hasattr(Foo(), "baz")
+
+
+def test_class_var_list():
+ class Foo:
+ foo: List[str] = []
+
+ def __init__(self, val: List[str]):
+ self.foo = val
+
+ f = Foo(["a", "b"])
+ assert f.foo == ["a", "b"]
+ assert Foo.foo == []
+ f2 = Foo([])
+ assert f.foo == ["a", "b"]
+ assert f2.foo == []
+
+ class Bar:
+ bar: List[str] = []
+
+ def __init__(self, val: List[str]):
+ self.bar.extend(val)
+
+ b = Bar(["a", "b"])
+ assert b.bar == ["a", "b"]
+ # be updated
+ assert Bar.bar == ["a", "b"]
+
+
+def test_class_eval():
+ class Foo:
+ def __init__(self, code: str):
+ self.code = code
+ self.foo = 1
+
+ def run(self):
+ code = "\n".join([line.lstrip() for line in self.code.splitlines()])
+ exec(code)
+
+ f = Foo(
+ "print(self)\n"
+ "print(self.foo)\n"
+ "self.foo = 2\n"
+ )
+ f.run()
+ assert f.foo == 2
diff --git a/tests/python/test_collection.py b/tests/python/test_collection.py
new file mode 100644
index 00000000..8f3376d6
--- /dev/null
+++ b/tests/python/test_collection.py
@@ -0,0 +1,30 @@
+from collections import deque
+
+
+def test_deque():
+ d = deque([1, 2, 3, 4, 5])
+ assert 5 == len(d)
+ assert 1 == d.popleft()
+ assert 5 == d.pop()
+ assert d.count(5) == 0
+ assert d.count(2) == 1
+ d.pop()
+ d.pop()
+ d.pop()
+
+ e = None
+ assert len(d) == 0
+ try:
+ d.popleft()
+ except IndexError as err:
+ e = err
+ assert e is not None
+
+
+def test_yield_from_deque():
+ d = deque([1, 2, 3, 4, 5])
+
+ def foo(dq: deque):
+ yield from dq
+
+ assert list(foo(d)) == [1, 2, 3, 4, 5]
diff --git a/tests/python/test_context.py b/tests/python/test_context.py
new file mode 100644
index 00000000..4cb78828
--- /dev/null
+++ b/tests/python/test_context.py
@@ -0,0 +1,20 @@
+def test_with_statement():
+ class Foo:
+ error = None
+
+ def __enter__(self):
+ return self
+
+ def run(self):
+ raise RuntimeError("failed")
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ if exc_val:
+ self.error = exc_val
+ return True
+
+ with Foo() as foo:
+ foo.run()
+
+ assert foo.error is not None
+ assert isinstance(foo.error, RuntimeError)
diff --git a/tests/python/test_dict.py b/tests/python/test_dict.py
index 7d2c41a7..e0b77fd3 100644
--- a/tests/python/test_dict.py
+++ b/tests/python/test_dict.py
@@ -47,3 +47,19 @@ class Bar(TypedDict, total=True):
bar2 = Bar(a="world")
assert "a" in bar2
+
+
+def test_dict_sort():
+ a = {3: 3, 4: 4, 1: 1, 2: 2, }
+ values = sorted(a.keys())
+ assert values == [1, 2, 3, 4]
+
+
+def test_get_dict_by_str_type():
+ class Key(str):
+
+ def get(self, data_: dict):
+ return data_.get(str(self), None)
+
+ data = {"a": 1, "b": 2}
+ assert Key("a").get(data) == 1
diff --git a/tests/python/test_func.py b/tests/python/test_func.py
index bffd322c..4b47fb4e 100644
--- a/tests/python/test_func.py
+++ b/tests/python/test_func.py
@@ -1,3 +1,19 @@
def test_func_set_attr():
setattr(test_func_set_attr, "__test__", "test")
assert test_func_set_attr.__test__ == "test"
+
+
+def test_func_args():
+ def foo(bar: int, baz: str, *args, **kwargs) -> bool:
+ pass
+
+ assert foo.__annotations__['bar'] is int
+ assert foo.__annotations__['return'] is bool
+
+
+def test_func_iterable_args():
+ def foo(*args: int) -> int:
+ return len(list(args))
+
+ value = foo(1, 2, 3, *[4, 5, 6], 7, 8, *[9])
+ assert value == 9
diff --git a/tests/python/test_inspect.py b/tests/python/test_inspect.py
index ebc9c926..dff46aa4 100644
--- a/tests/python/test_inspect.py
+++ b/tests/python/test_inspect.py
@@ -149,3 +149,21 @@ class Child(Parent):
source = inspect.getsource(Child)
assert "foo" not in source
+
+
+class SomeClass:
+ foo: int = 123
+
+ __add_info: str = ""
+
+
+class SubClass(SomeClass):
+ bar: int = 456
+
+
+SubClass.__add_info = "test"
+
+
+def test_getsource_without_added_code():
+ code = inspect.getsource(SubClass)
+ assert "__add_info" not in code
diff --git a/tests/python/test_pkg.py b/tests/python/test_pkg.py
new file mode 100644
index 00000000..e7e24a89
--- /dev/null
+++ b/tests/python/test_pkg.py
@@ -0,0 +1,7 @@
+import pkgutil
+
+
+def test_iter_modules():
+ from ghostos.core import moss
+ values = pkgutil.iter_modules(moss.__path__, prefix=moss.__name__ + '.')
+ assert len(list(values)) > 1
diff --git a/tests/python/test_module.py b/tests/python/test_py_module.py
similarity index 100%
rename from tests/python/test_module.py
rename to tests/python/test_py_module.py
diff --git a/tests/python/test_tree_sitter.py b/tests/python/test_py_tree_sitter.py
similarity index 100%
rename from tests/python/test_tree_sitter.py
rename to tests/python/test_py_tree_sitter.py
diff --git a/tests/python/test_pydantic.py b/tests/python/test_pydantic.py
index 5dc0e8d8..068d4799 100644
--- a/tests/python/test_pydantic.py
+++ b/tests/python/test_pydantic.py
@@ -1,7 +1,10 @@
+import time
+
from pydantic import BaseModel, Field
from pydantic.errors import PydanticSchemaGenerationError
-from typing import TypedDict, Required, Iterable, List, Optional
+from typing import TypedDict, Required, Iterable, List, Optional, ClassVar, Type
from typing_extensions import Literal
+from datetime import datetime
def test_pydantic_new_typed_dict() -> None:
@@ -138,3 +141,113 @@ class Baz(Bar):
bar_data = bar.model_dump(exclude_defaults=True)
assert len(bar_data) == 0
assert not hasattr(bar, 'c')
+
+
+def test_bytes_in_model():
+ class Foo(BaseModel):
+ foo: bytes
+
+ f = Foo(foo="test".encode())
+ assert f.foo.decode() == "test"
+
+
+def test_multi_type_attr():
+ class Foo(BaseModel):
+ foo: int = 0
+
+ class Bar(BaseModel):
+ bar: str = ""
+
+ class Baz(BaseModel):
+ baz: List[BaseModel]
+
+ b = Baz(baz=[Foo(), Bar()])
+ data = b.model_dump(serialize_as_any=True)
+ assert data == {"baz": [{"foo": 0}, {"bar": ""}]}
+
+ unmarshalled = Baz(**data)
+ assert not isinstance(unmarshalled.baz[0], Foo)
+
+
+def test_model_with_subclass():
+ class Foo(BaseModel):
+ class Bar(BaseModel):
+ bar: str = "hello"
+
+ bar: Bar = Field(default_factory=Bar)
+
+ f = Foo()
+ assert f.bar.bar == "hello"
+
+
+def test_model_with_none_model_object():
+ class Foo:
+ foo = 123
+
+ err = None
+ try:
+
+ class Bar(BaseModel):
+ foo: Foo
+ except PydanticSchemaGenerationError as e:
+ err = e
+ assert err is not None
+
+
+def test_datetime_model():
+ class Foo(BaseModel):
+ time: datetime = Field(default_factory=datetime.now)
+
+ f = Foo()
+ assert f.time.timestamp() > 0
+
+
+def test_model_with_subclass_define():
+ class Foo(BaseModel):
+ foo: int = 123
+ BarType: ClassVar[Optional[Type]] = None
+
+ class Foo2(Foo):
+ class BarType(BaseModel):
+ bar: int = 123
+
+ bar: BarType = Field(default_factory=BarType)
+
+ foo2 = Foo2()
+ assert foo2.bar.bar == 123
+
+
+def test_model_with_datetime():
+ class Foo(BaseModel):
+ now: datetime = Field(default_factory=datetime.now)
+
+ foo = Foo(now=int(time.time()))
+ assert foo.now.timestamp() > 0
+
+
+def test_print_model():
+ class Foo(BaseModel):
+ foo: str = "hello"
+
+ f = Foo()
+ assert "(" not in str(f)
+ assert "(" in repr(f)
+
+
+def test_enum_with_none():
+ class Foo(BaseModel):
+ foo: Optional[str] = Field(None, enum={"hello", "world"})
+
+ f = Foo()
+ assert f.foo is None
+
+
+def test_foo_bar():
+ class Bar(BaseModel):
+ bar: int = 123
+
+ class Foo(BaseModel):
+ bar: Bar = Field(default_factory=Bar)
+
+ f = Foo()
+ assert f.bar.bar == 123
diff --git a/tests/python/test_queue.py b/tests/python/test_queue.py
new file mode 100644
index 00000000..4202140f
--- /dev/null
+++ b/tests/python/test_queue.py
@@ -0,0 +1,8 @@
+from queue import Queue
+
+
+def test_queue():
+ q = Queue()
+ q.put(None)
+ value = q.get(block=True, timeout=5)
+ assert value is None
diff --git a/tests/python/test_restrictedpython.py b/tests/python/test_restrictedpython.py
deleted file mode 100644
index 59d3213d..00000000
--- a/tests/python/test_restrictedpython.py
+++ /dev/null
@@ -1,16 +0,0 @@
-from RestrictedPython import compile_restricted, safe_globals
-
-
-def test_restrictedpython_import():
- code = """
-import sympy
-import inspect
-source = inspect.getsource(sympy)
-"""
- compiled = compile_restricted(code, filename="", mode='exec')
- e = None
- try:
- exec(compiled, safe_globals)
- except ImportError as err:
- e = err
- assert e is not None
diff --git a/tests/python/test_set.py b/tests/python/test_set.py
new file mode 100644
index 00000000..deb40b10
--- /dev/null
+++ b/tests/python/test_set.py
@@ -0,0 +1,13 @@
+def test_set_len():
+ s = {1, 2, 3}
+ assert len(s) == 3
+
+
+def test_set_order():
+ for i in range(10):
+ a = [1, 2, 3]
+ b = set(a)
+ c = []
+ for k in b:
+ c.append(k)
+ assert a == c
diff --git a/tests/python/test_slice.py b/tests/python/test_slice.py
index e9cf117f..14bf4727 100644
--- a/tests/python/test_slice.py
+++ b/tests/python/test_slice.py
@@ -13,6 +13,13 @@ def test_slice_negative_index():
assert arr[:-1] == [0]
+def test_slice_pop_0():
+ arr = [0, 1]
+ arr.pop(0)
+ arr.pop(0)
+ assert arr == []
+
+
def test_thread_safe_append():
from threading import Thread
@@ -42,3 +49,29 @@ def test_array_insert_more_than_pointed():
a = [1, 2, 3, 4]
a[1:3] = [5, 6, 7, 8]
assert a == [1, 5, 6, 7, 8, 4]
+
+
+def test_sort_dicts():
+ cases = [
+ {'a': 1, 'b': 2, 'c': 3, 'd': 4},
+ {'a': 2, 'b': 2, 'c': 3, 'd': 4},
+ {'a': 3, 'b': 2, 'c': 3, 'd': 4},
+ {'a': 4, 'b': 2, 'c': 3, 'd': 4},
+ ]
+ values = sorted(cases, key=lambda x: x['a'], reverse=True)
+ actual = [c['a'] for c in values]
+ assert actual == [4, 3, 2, 1]
+
+
+def test_arr_tail():
+ arr = [1, 2, 3, 4]
+ assert arr[-2:] == [3, 4]
+ assert arr[-20:] == [1, 2, 3, 4]
+
+
+def test_negative_slice_index():
+ arr = [1, 2, 3, 4]
+ first = arr[:2]
+ end = arr[2:]
+ assert first == [1, 2]
+ assert end == [3, 4]
diff --git a/tests/python/test_threads.py b/tests/python/test_threads.py
new file mode 100644
index 00000000..2d2dcc62
--- /dev/null
+++ b/tests/python/test_threads.py
@@ -0,0 +1,27 @@
+from threading import Thread, Event
+import time
+
+
+class TestCommand:
+
+ def __init__(self, content: str, duration: float):
+ self.content = content
+ self.duration = duration
+
+ def run(self):
+ start = time.time()
+ now = time.time()
+ while now - start < self.duration:
+ print(self.content)
+ time.sleep(1)
+ now = time.time()
+
+
+def test_stop_able_threads():
+ from multiprocessing import Process
+
+ t = TestCommand('hello', 2)
+ p = Process(target=t.run, args=())
+ p.start()
+ p.terminate()
+ p.join()
diff --git a/tests/python/test_typing.py b/tests/python/test_typing.py
index ef37c3af..7dc7d81c 100644
--- a/tests/python/test_typing.py
+++ b/tests/python/test_typing.py
@@ -1,4 +1,4 @@
-from typing import Union, TypedDict, Optional
+from typing import Union, TypedDict, Optional, Literal
import inspect
@@ -69,3 +69,11 @@ def loo(self):
assert typehints['car'] is str
assert 'good' not in typehints
assert 'loo' not in typehints
+
+
+def test_literal_int():
+ def foo(v: Literal[1, 2, 3]) -> int:
+ return v
+
+ a = foo(3)
+ assert a == 3
diff --git a/tests/python/test_yield.py b/tests/python/test_yield.py
new file mode 100644
index 00000000..e10ddf0c
--- /dev/null
+++ b/tests/python/test_yield.py
@@ -0,0 +1,92 @@
+from typing import Iterable
+
+
+def test_yield_is_blocking():
+ tests = []
+
+ def foo(values: Iterable[int]) -> Iterable[int]:
+ for value in values:
+ yield value
+ tests.append("foo")
+
+ def bar(values: Iterable[int]) -> Iterable[int]:
+ for value in values:
+ yield value
+ tests.append("bar")
+
+ list(bar(foo([1, 2, 3])))
+ # yield still block
+ # and if upstream not iter any, the downstream function is not running.
+ assert tests == ["foo", "bar"]
+
+
+def test_yield_none():
+ def foo():
+ yield 1
+ yield 2
+ yield None
+ print("foo")
+
+ v = list(foo())
+ assert v == [1, 2, None]
+
+ def bar():
+ try:
+ yield 1
+ yield 2
+ return None
+ finally:
+ # print("bar")
+ pass
+
+ v = list(bar())
+ assert v == [1, 2]
+
+
+def test_yield_with_finally():
+ values = []
+
+ def foo():
+ try:
+ values.append(1)
+ yield 1
+ values.append(2)
+ yield 2
+ finally:
+ values.append(3)
+
+ foo()
+ # finally is not called as well.
+ assert values == []
+
+
+# iterable can not define __awaits__
+# def test_yield_is_blocking_with_none():
+# tests = []
+#
+# async def foo(values: Iterable[int]) -> Iterable[int]:
+# for value in values:
+# yield value
+# tests.append("foo")
+#
+# async def bar(values: Iterable[int]) -> Iterable[int]:
+# for value in values:
+# yield value
+# yield None
+# tests.append("bar")
+#
+# async def main():
+# list(await bar(await foo([1, 2, 3])))
+#
+# import asyncio
+# asyncio.run(main())
+# assert tests == ["foo", "bar"]
+
+
+def test_yield_after_yield_from():
+ def foo():
+ yield from []
+ yield 1
+
+ values = list(foo())
+ assert values == [1]
diff --git a/tests/test_abc.py b/tests/test_abc.py
deleted file mode 100644
index 86dea525..00000000
--- a/tests/test_abc.py
+++ /dev/null
@@ -1,7 +0,0 @@
-from ghostos.abc import PromptAble, PromptAbleClass
-import inspect
-
-
-def test_is_abstract():
- assert inspect.isabstract(PromptAble)
- assert inspect.isabstract(PromptAbleClass)
diff --git a/tests/test_container.py b/tests/test_container.py
index bf74cbf7..2e6592c6 100644
--- a/tests/test_container.py
+++ b/tests/test_container.py
@@ -1,9 +1,9 @@
from __future__ import annotations
from abc import ABCMeta, abstractmethod
-from typing import Type, Dict
+from typing import Type, Dict, get_args, get_origin, ClassVar
-from ghostos.container import Container, Provider
+from ghostos.container import Container, Provider, provide
def test_container_baseline():
@@ -96,3 +96,95 @@ class Foo:
container.set(Foo, Foo())
container.bootstrap()
assert container.force_fetch(Foo).foo == 1
+
+
+def test_provider_generic_types():
+ class SomeProvider(Provider[int]):
+
+ def singleton(self) -> bool:
+ return True
+
+ def factory(self, con: Container) -> int:
+ return 3
+
+ # baseline
+ args = get_args(Provider[int])
+ assert args[0] is int
+ assert get_origin(Provider[int]) is Provider
+
+ p = SomeProvider()
+ con = Container()
+ assert p.singleton()
+ assert p.factory(con) == 3
+ assert p.contract() is int
+
+
+def test_provide_with_lambda():
+ container = Container()
+ container.register(provide(int)(lambda c: 10))
+ container.register(provide(str)(lambda c: "hello"))
+
+ assert container.force_fetch(int) == 10
+ assert container.force_fetch(str) == "hello"
+
+
+def test_provide_in_loop():
+ container = Container()
+ for a, fn in {int: lambda c: 10, str: lambda c: "hello"}.items():
+ container.register(provide(a)(fn))
+
+ assert container.force_fetch(int) == 10
+ assert container.force_fetch(str) == "hello"
+
+
+def test_container_set_str():
+ container = Container()
+ container.set("foo", "bar")
+ assert container.get("foo") == "bar"
+
+
+def test_container_inherit():
+ class Foo:
+ def __init__(self, foo: int):
+ self.foo = foo
+
+ class Bar:
+ def __init__(self, foo: Foo):
+ self.foo = foo
+
+ bar: str = "hello"
+
+ container = Container()
+ container.register(provide(Bar, singleton=False)(lambda c: Bar(c.force_fetch(Foo))))
+ sub_container = Container(container)
+ # sub container register Foo that Bar needed
+ sub_container.register(provide(Foo, singleton=False)(lambda c: Foo(2)))
+ bar = sub_container.force_fetch(Bar)
+ assert bar.bar == "hello"
+ assert bar.foo.foo == 2
+
+
+def test_bloodline():
+ container = Container()
+ assert container.bloodline is not None
+ sub = Container(parent=container, name="hello")
+ assert len(sub.bloodline) == 2
+
+
+def test_container_shutdown():
+ class Foo:
+ instance_count: ClassVar[int] = 0
+
+ def __init__(self):
+ Foo.instance_count += 1
+
+ def shutdown(self):
+ Foo.instance_count -= 1
+
+ container = Container()
+ f = Foo()
+ container.set(Foo, f)
+ container.add_shutdown(f.shutdown)
+ assert Foo.instance_count == 1
+ container.shutdown()
+ assert Foo.instance_count == 0
diff --git a/tests/test_entity.py b/tests/test_entity.py
new file mode 100644
index 00000000..d286bcb4
--- /dev/null
+++ b/tests/test_entity.py
@@ -0,0 +1,38 @@
+from ghostos.entity import to_entity_meta, from_entity_meta
+from pydantic import BaseModel
+
+
+class Foo:
+ foo = 1
+
+ def __eq__(self, other):
+ return self.foo == other.foo
+
+
+class Baz(BaseModel):
+ baz: str = "hello"
+
+
+class Bar(BaseModel):
+ baz: Baz
+
+
+def test_entities():
+ cases = [
+ 1,
+ 0.5,
+ None,
+ False,
+ True,
+ "hello world",
+ [1, 2, 3, 4.5, "hello world"],
+ {"a": 1, "b": 2},
+ {1:"a", "b": 2},
+ Foo(),
+ Baz(),
+ ]
+
+ for c in cases:
+ meta = to_entity_meta(c)
+ value = from_entity_meta(meta)
+ assert value == c, f"{c}: {value}"
diff --git a/tests/test_prompter.py b/tests/test_prompter.py
new file mode 100644
index 00000000..6dbdbddf
--- /dev/null
+++ b/tests/test_prompter.py
@@ -0,0 +1,53 @@
+from ghostos.prompter import (
+ TextPrmt, PromptAbleClass, PromptAbleObj, ModelPrompter,
+ InspectPrmt,
+)
+from ghostos.container import Container
+import inspect
+
+
+def test_is_abstract():
+ assert inspect.isabstract(PromptAbleObj)
+ assert inspect.isabstract(PromptAbleClass)
+
+
+def test_group_prompters():
+ prompter = TextPrmt(
+ title="1"
+ ).with_children(
+ TextPrmt(title="1.1"),
+ TextPrmt(title="1.2").with_children(
+ TextPrmt(title="1.2.1"),
+ TextPrmt(title="1.2.2", content="hello world"),
+ )
+ )
+
+ c = Container()
+ p = prompter.get_prompt(container=c)
+ assert "# 1\n" in p
+ assert "\n### 1.2.2\n" in p
+ # test buffer is ok
+ assert p == prompter.get_prompt(c)
+
+
+def test_inspect_prompters():
+ prmt = InspectPrmt()
+ prmt.inspect_source(InspectPrmt)
+ prmt.inspect_source(test_group_prompters)
+ c = Container()
+ prompt = prmt.get_prompt(c)
+ assert f":{test_group_prompters.__name__}" in prompt
+
+
+def test_model_prompters():
+ class TestPrompter(ModelPrompter):
+ line: str = "TestPrompter"
+
+ def self_prompt(self, container: Container) -> str:
+ return self.line
+
+ def get_title(self) -> str:
+ return ""
+
+ t = TestPrompter()
+ assert "TestPrompter" in t.get_prompt(Container())
diff --git a/tests/test_streamlit_render.py b/tests/test_streamlit_render.py
new file mode 100644
index 00000000..d525c59c
--- /dev/null
+++ b/tests/test_streamlit_render.py
@@ -0,0 +1,21 @@
+from typing import Optional, Self
+
+from ghostos.streamlit import (
+ is_streamlit_renderable,
+ StreamlitObject,
+ render_streamlit_object,
+ Rendered
+)
+
+
+def test_render_streamlit_object():
+ class Foo(StreamlitObject):
+
+ def __streamlit_render__(self) -> Optional[Rendered[Self]]:
+ return Rendered(value=self, changed=False)
+
+ foo = Foo()
+ assert is_streamlit_renderable(foo)
+ r = render_streamlit_object(foo)
+ assert not r.changed
+ assert r.value is foo
- 