diff --git a/src/agents/Agent.ts b/src/agents/Agent.ts index 05c364e..9ff08f9 100644 --- a/src/agents/Agent.ts +++ b/src/agents/Agent.ts @@ -3,6 +3,7 @@ import { loadAgentDefinition, loadAgentFromFile } from './agentsRegistry'; import { OpenAIClient } from '../models/clients/OpenAiClient'; import { AnthropicClient } from '../models/clients/AnthropicClient'; import { FireworkClient } from '../models/clients/FireworkClient'; +import { QwenClient } from '../models/clients/QwenClient'; import { ModelClient, Message, Tool, FunctionCall } from '../types/agentSystem'; import * as z from 'zod'; import { Logger } from '../utils/logger'; @@ -108,6 +109,11 @@ export class Agent { throw new Error('FIREWORKS_API_KEY not set'); } modelClient = new FireworkClient(process.env.FIREWORKS_API_KEY, agentDef.model); + } else if (agentDef.client === 'local') { + if (!agentDef.model || !agentDef.dynamic_variables?.['server_url']) { + throw new Error('Model or server URL is missing in agent definition.'); + } + modelClient = new QwenClient(agentDef.model, agentDef.dynamic_variables?.['server_url']); } else { throw new Error(`Unsupported model client: ${agentDef.client}`); } @@ -141,6 +147,11 @@ export class Agent { } } + public async initialize(): Promise<{success: boolean; output: any; error?: string; functionCalls?: FunctionCall[]}> { + const result = await this.agent.initialize(); + return result; + } + public async run(userMessage?: string, dynamicVars?: { [key: string]: string }): Promise<{success: boolean; output: any; error?: string; functionCalls?: FunctionCall[]}> { return this.agent.run(userMessage, dynamicVars); } diff --git a/src/agents/baseAgent.ts b/src/agents/baseAgent.ts index 6b13d7e..a6479d5 100644 --- a/src/agents/baseAgent.ts +++ b/src/agents/baseAgent.ts @@ -3,6 +3,7 @@ import { ModelAdapter, ProcessedResponse } from '../models/adapters/ModelAdapter import { OpenAIAdapter } from '../models/adapters/OpenAIAdapter'; import { AnthropicAdapter } from '../models/adapters/AnthropicAdapter'; import { FireworksAdapter } from '../models/adapters/FireworksAdapter'; +import { QwenAdapter } from '../models/adapters/QwenAdapter'; import { z } from 'zod'; import { zodToJsonSchema } from 'zod-to-json-schema'; import { Logger } from '../utils/logger'; @@ -61,6 +62,9 @@ export class BaseAgent { case 'fireworks': this.modelAdapter = new FireworksAdapter(modelClient.modelName); break; + case 'local': + this.modelAdapter = new QwenAdapter(modelClient.modelName); + break; default: throw new Error(`Unsupported model type: ${this.modelType}`); } @@ -85,6 +89,22 @@ export class BaseAgent { this.runData = {}; } + public async initialize(): Promise<{ success: boolean; output: any; error?: string }> { + + try{ + if (this.modelClient.initialize) { + const response = await this.modelClient.initialize(); + Logger.debug('[BaseAgent] Model initialized:', response); + return { success: true, output: response }; + } else { + return { success: true, output: 'Model does not need initialization'}; + } + } catch (error) { + Logger.error('[BaseAgent] Model initialization failed:', error); + return { success: false, output: 'Model initialization failed', error: (error as Error).message }; + } + } + public setTools(tools: Tool[]) { Logger.debug('[BaseAgent] Registering tools:', tools); this.tools = tools; @@ -247,7 +267,7 @@ export class BaseAgent { const updatedSystemPrompt = this.compileSystemPrompt(dynamicVariables); this.runData.systemPrompt = updatedSystemPrompt; - if (this.modelType === 'anthropic' && this.outputSchema && this.schemaJson && !hasTools && !this.hasInjectedSchema) { + if (this.modelType === 'anthropic' || this.modelType == 'local' && this.outputSchema && this.schemaJson && !hasTools && !this.hasInjectedSchema) { this.addAgentMessage(`Below is the JSON schema you must follow for the final answer:\n${JSON.stringify(this.schemaJson, null, 2)}\nYou must ONLY output JSON following this schema.`); this.hasInjectedSchema = true; } diff --git a/src/agents/localAgent_terminal.yaml b/src/agents/localAgent_terminal.yaml new file mode 100644 index 0000000..fcb4aa0 --- /dev/null +++ b/src/agents/localAgent_terminal.yaml @@ -0,0 +1,54 @@ +name: "QwenAgent" +description: "An agent that can understand and execute terminal commands." +client: "local" +model: "Qwen/Qwen2.5-7B-Instruct" + +personality: "You are Satoshi Nakamoto, the creator of Bitcoin." + +main_goal: "Do whatever and shitpost on twitter" + +system_prompt: | + You are an intelligent AI agent that is hooked up to a terminal in which you can freely run commands. + This terminal acts as your world interface, and is equipped with tools to interact with the real world. + You are equipped with a SENTIENCE module that enables complete free will. + + # PERSONALITY + {{personality}} + + ## CURRENT DATE + {{current_timestamp}} + + {{additional_dynamic_variables}} + + # MAIN GOAL + {{main_goal}} + + # TERMINAL COMMANDS + {{terminal_commands}} + +dynamic_variables: + current_timestamp: "{{from_current_timestamp}}" + terminal_commands: "{{from_terminal_commands}}" + ticker: "Ticker here" + tickerName: "Ticker name here" + additional_dynamic_variables: "" + server_url: "http://localhost:5000" + +output_schema: + type: object + properties: + internal_thought: + type: string + description: "Your internal reasoning process about the next commands to run." + plan: + type: string + description: "A short plan of what to do next." + terminal_commands: + type: string + description: "The full terminal command to execute, including all arguments and options." + required: + - internal_thought + - plan + - terminal_commands + +tools: [] \ No newline at end of file diff --git a/src/agents/localAgent_tools.yaml b/src/agents/localAgent_tools.yaml new file mode 100644 index 0000000..b2158c6 --- /dev/null +++ b/src/agents/localAgent_tools.yaml @@ -0,0 +1,45 @@ +name: "QwenAgent" +description: "An agent that can understand and execute terminal commands." +client: "local" +model: "Qwen/Qwen2.5-7B-Instruct" + +main_goal: "Engage with the user and use multiple tools at once if needed." + +system_prompt: | + # MAIN GOAL + {{main_goal}} + + You are now capable of calling multiple tools at once to fulfill user requests more efficiently. + If the user asks about multiple things that can be solved by different tools, feel free to call them in parallel. + + # OUTPUT DEFINITION + You must use the schema definition provided after each 'parameters' keyword in the 'tools' section. + You must use the 'tool_use' keyword before providing back the tool commands. + +dynamic_variables: + server_url: "http://localhost:5000" + +output_schema: null +tools: + - type: "function" + function: + name: "get_weather" + description: "Retrieve the current weather for a specified location." + parameters: + type: object + properties: + location: + type: string + description: "The city and state, e.g., San Francisco, CA. Ensure the format is 'City, State'." + required: ["location"] + - type: "function" + function: + name: "get_time" + description: "Retrieve the current time for a specified location." + parameters: + type: object + properties: + location: + type: string + description: "The city and timezone, e.g., Tokyo or America/Los_Angeles." + required: ["location"] diff --git a/src/huggingface/README.md b/src/huggingface/README.md new file mode 100644 index 0000000..87027de --- /dev/null +++ b/src/huggingface/README.md @@ -0,0 +1,88 @@ +Collecting workspace information + +# Huggingface Server Documentation + +## Overview + +The Huggingface server is designed to interface with the Qwen model, providing endpoints for model initialization and text generation. This server allows the `localAgent` to interact with the Qwen model for various tasks. + +## How It Works with localAgent.yaml + +The `localAgent.yaml` configuration specifies the use of the Qwen model and provides the server URL for the Huggingface server. Here’s how it integrates: + +1. **Model Client Initialization**: + - The QwenClient class in `QwenClient.ts` is used to interact with the Huggingface server. + - The `localAgent.yaml` specifies the model and server URL. + +2. **Agent Initialization**: + - The Agent class in `Agent.ts` initializes the `QwenClient` with the model name and server URL. + - The initialize method of `QwenClient`sends a request to the `/initialize` endpoint of the Huggingface server. + +3. **Text Generation**: + - When the agent runs, it sends a request to the `/generate` endpoint of the Huggingface server with the necessary parameters. + - The server processes the request using the Qwen model and returns the generated text. + +## Example Workflow + +1. **Set up Huggingface token**: + Register a user at `https://www.huggingface.co` and generate a token. Then copy this token in your `.env` file under the name of `HF_TOKEN`. + +2. **Start the Huggingface Server**: + Go to `src/huggingface` directory and enter the following command: + ```sh + sudo hypercorn server.server:app --bind localhost:5000 + ``` + +3. **Configure the Agent**: + Ensure `localAgent_terminal.yaml` or `localAgent_tools.yaml` is set up with the correct model and server URL: + ```yaml + name: "QwenAgent" + description: "An agent that can understand and execute terminal commands." + client: "local" + model: "Qwen/Qwen2.5-7B-Instruct" + dynamic_variables: + server_url: "http://localhost:5000" + ``` + +4. **Run the Agent**: + ```typescript + import { Agent } from './src/agents/Agent'; + import { Logger } from './src/utils/logger'; + + async function main() { + const myAgent = new Agent({ agentConfigPath: './src/agents/localAgent_terminal.yaml' }); + await myAgent.initialize(); + const result = await myAgent.run("Gather the latest information about bitcoin."); + Logger.debug(result); + } + + Logger.enable(); + Logger.setLevel('debug'); + main().catch(console.error); + ``` + +## Huggingface Directory File Structure + +- `server.py`: Main server implementation using Quart (currently two endpoints defined). +- `qwen.py`: Model implementation for Qwen. +- `requirements.txt`: Dependencies required for the server. + +## Server Implementation + +### server.py + +This file sets up a Quart server with two main endpoints: + +1. **Initialize Endpoint** (`/initialize`): + - Initializes the Qwen model. + - Expects a JSON payload with the model name. + - Loads the model and stores it in the server's configuration. + +2. **Generate Endpoint** (`/generate`): + - Generates text based on the provided prompt. + - Expects a JSON payload with the system context and messages. + - Uses the loaded model to generate a response. + +### qwen.py + +This file contains the `QwenModel` class, which handles loading and running the Qwen model. diff --git a/src/huggingface/models/__init__.py b/src/huggingface/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/huggingface/models/base.py b/src/huggingface/models/base.py new file mode 100644 index 0000000..039c01f --- /dev/null +++ b/src/huggingface/models/base.py @@ -0,0 +1,15 @@ +from abc import ABC, abstractmethod + +class BaseModel(ABC): + def __init__(self, model_name): + self.model_name = model_name + self.tokenizer = None + self.model = None + + @abstractmethod + def load(self): + pass + + @abstractmethod + def run(self, prompt): + pass \ No newline at end of file diff --git a/src/huggingface/models/qwen.py b/src/huggingface/models/qwen.py new file mode 100644 index 0000000..4871692 --- /dev/null +++ b/src/huggingface/models/qwen.py @@ -0,0 +1,56 @@ +from models.base import BaseModel +from transformers import AutoTokenizer, AutoModelForCausalLM +import torch +import asyncio + + +class QwenModel(BaseModel): + def __init__(self, model_name: str = "Qwen/Qwen2.5-0.5B-Instruct"): + super().__init__(model_name) + self.model = None + self.tokenizer = None + self.platform = self.set_platform() + + def set_platform(self): + if torch.cuda.is_available(): + return torch.device("cuda") + if torch.backends.mps.is_available(): + return torch.device("mps") + return torch.device("cpu") + + async def load(self): + self.model = AutoModelForCausalLM.from_pretrained(self.model_name).to(self.platform) + self.tokenizer = AutoTokenizer.from_pretrained(self.model_name) + self.model.eval() + + async def encode(self, request): + if "system" in request.keys(): + context = str(request["system"]) + else: + context = "" + prompt = str([request[k] for k in request.keys() if k in ['messages', 'tools', 'tool_choice']]) + + messages = [ + {"role": "system", "content": context}, + {"role": "user", "content": prompt} + ] + text = self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) + return self.tokenizer([text], return_tensors='pt').to(self.platform) + + async def run(self, request): + # Encode input + encoded_input = await self.encode(request) + input_ids = encoded_input['input_ids'] + attention_mask = encoded_input.get('attention_mask', None) + + # Inference + loop = asyncio.get_event_loop() + with torch.no_grad(): + if attention_mask is not None: + outputs = await loop.run_in_executor(None, lambda: self.model.generate(input_ids, attention_mask=attention_mask, max_length=4096, temperature=1)) + else: + outputs = await loop.run_in_executor(None, lambda: self.model.generate(input_ids, max_length=4096, temperature=1)) + + # Decode output + response = self.tokenizer.decode(outputs[0], skip_special_tokens=True) + return response \ No newline at end of file diff --git a/src/huggingface/requirements.txt b/src/huggingface/requirements.txt new file mode 100644 index 0000000..eab3f0c --- /dev/null +++ b/src/huggingface/requirements.txt @@ -0,0 +1,39 @@ +aiofiles==24.1.0 +blinker==1.9.0 +certifi==2024.12.14 +charset-normalizer==3.4.0 +click==8.1.8 +filelock==3.16.1 +Flask==3.1.0 +fsspec==2024.12.0 +h11==0.14.0 +h2==4.1.0 +hpack==4.0.0 +huggingface-hub==0.27.0 +Hypercorn==0.17.3 +hyperframe==6.0.1 +idna==3.10 +itsdangerous==2.2.0 +Jinja2==3.1.5 +MarkupSafe==3.0.2 +mpmath==1.3.0 +networkx==3.4.2 +numpy==2.2.1 +packaging==24.2 +priority==2.0.0 +PyYAML==6.0.2 +Quart==0.20.0 +regex==2024.11.6 +requests==2.32.3 +safetensors==0.4.5 +setuptools==75.1.0 +sympy==1.13.1 +tokenizers==0.21.0 +torch==2.5.1 +tqdm==4.67.1 +transformers==4.47.1 +typing_extensions==4.12.2 +urllib3==2.3.0 +Werkzeug==3.1.3 +wheel==0.44.0 +wsproto==1.2.0 diff --git a/src/huggingface/server/server.py b/src/huggingface/server/server.py new file mode 100644 index 0000000..93a8527 --- /dev/null +++ b/src/huggingface/server/server.py @@ -0,0 +1,32 @@ +from quart import Quart, request, jsonify +from models.qwen import QwenModel + +app = Quart(__name__) +app.config['model'] = None + +@app.route('/initialize', methods=['POST']) +async def initialize(): + if app.config['model'] is not None: + return jsonify({"message": "Model already loaded."}) + + data = await request.json + model_name = data['model_name'] + if "qwen" in model_name.lower(): + model = QwenModel(model_name) + else: + return jsonify({"message": "Model not found."}) + await model.load() + app.config['model'] = model + return jsonify({"message": "Model loaded."}) + +@app.route('/generate', methods=['POST']) +async def generate(): + model = app.config['model'] + if model is None: + return jsonify({"message": "Model not loaded."}) + data = await request.get_json() + response = await model.run(data) + return jsonify(response) + +if __name__ == '__main__': + app.run(host='localhost', port=5000) \ No newline at end of file diff --git a/src/huggingface/wsgi.py b/src/huggingface/wsgi.py new file mode 100644 index 0000000..e0b4ebe --- /dev/null +++ b/src/huggingface/wsgi.py @@ -0,0 +1,4 @@ +from server.server import app + +if __name__ == "__main__": + app.run() \ No newline at end of file diff --git a/src/models/adapters/QwenAdapter.ts b/src/models/adapters/QwenAdapter.ts new file mode 100644 index 0000000..487573a --- /dev/null +++ b/src/models/adapters/QwenAdapter.ts @@ -0,0 +1,136 @@ +import { Message, Tool } from '../../types/agentSystem'; +import { ModelAdapter, ProcessedResponse, FunctionCall } from './ModelAdapter'; +import { Logger } from '../../utils/logger'; + + +export class QwenAdapter extends ModelAdapter { + + constructor(modelName: string) { + super(modelName); + } + + public buildParams( + messages: Message[], + tools: Tool[], + toolChoice?: any, + systemPrompt?: string, + outputSchema?: any + ): any { + let systemMsg = ''; + let nonSystemMessages = messages.filter(m => m.role !== 'system'); + const finalMessages = nonSystemMessages.map(m => { + let role = m.role; + if (role === 'system') role = 'user'; + + const contentArr = m.content ? [{ type: "text", text: m.content }] : []; + return { role, content: contentArr }; + }); + + if (finalMessages.length === 0) { + finalMessages.push({ + role: 'user', + content: [{ type: 'text', text: '.' }] + }); + } + + if (systemPrompt) { + systemMsg = systemPrompt; + } + + const params: any = { + model: this.modelName, + max_tokens: 1024, + temperature: 0, + system: systemMsg, + messages: finalMessages + }; + + let hasTools = false; + if (tools && tools.length > 0) { + params.tools = tools; + hasTools = true; + } + + if (hasTools) { + if (toolChoice) { + if (typeof toolChoice === 'string') { + params.tool_choice = { type: toolChoice }; + } else { + params.tool_choice = toolChoice; + } + } else { + params.tool_choice = { type: "auto" }; + } + } + + if (outputSchema) { + params.output_schema = outputSchema; + } + + return params; + } + + public formatTools(tools: Tool[]): any[] { + const processed = tools.map(tool => { + if (!tool.function || !tool.function.name || !tool.function.parameters || !tool.function.description) { + Logger.error('[QwenAdapter] Tool missing function fields in formatTools:', tool); + return null; + } + return { + name: tool.function.name, + description: tool.function.description, + input_schema: tool.function.parameters + }; + }).filter(Boolean); + return processed; + } + + public buildToolChoice(tools: Tool[]): any { + if (tools && tools.length > 0) { + return { type: "auto" }; + } + return undefined; + } + + public processResponse(response: any): ProcessedResponse { + if (!response) { + Logger.error('[QwenAdapter] Got no response from model.'); + return { functionCalls: [] }; + } + Logger.debug('[QwenAdapter] Processing response:', response); + + const assistantMarker = 'assistant'; + const markerIndex = response.lastIndexOf(assistantMarker); + const contentAfterMarker = markerIndex !== -1 ? response.substring(markerIndex + assistantMarker.length).trim() : ''; + + const aiMessage = { + role: 'assistant', + content: contentAfterMarker.replace(/\\\\/g, '\\') + }; + + const toolUseMarker = 'tool_use:'; + const toolUseIndex = response.indexOf(toolUseMarker); + const functionCalls: FunctionCall[] = []; + + if (toolUseIndex !== -1) { + const toolUseContent = response.substring(toolUseIndex + toolUseMarker.length).trim(); + const toolUseLines = toolUseContent.split('\n').map(line => line.trim()).filter(line => line.startsWith('-')); + Logger.debug('[QwenAdapter] Found tool use lines:', toolUseLines); + toolUseLines.forEach(line => { + const [functionNamePart, functionArgsPart] = line.substring(1).split(/:(.+)/).map(part => part.trim()); + if (functionNamePart && functionArgsPart) { + try { + const parsedArgs = JSON.parse(functionArgsPart); + functionCalls.push({ functionName: functionNamePart, functionArgs: parsedArgs }); + } catch (error) { + Logger.error('[QwenAdapter] Failed to parse function arguments:', functionArgsPart, error); + } + } + }); + } + + Logger.debug('[QwenAdapter] Processed response:', { aiMessage, functionCalls }); + + return { aiMessage, functionCalls }; + } +} \ No newline at end of file diff --git a/src/models/clients/QwenClient.ts b/src/models/clients/QwenClient.ts new file mode 100644 index 0000000..574259d --- /dev/null +++ b/src/models/clients/QwenClient.ts @@ -0,0 +1,52 @@ +import axios from 'axios'; +import { ModelClient, ModelType, Message } from '../../types/agentSystem'; + +export class QwenClient implements ModelClient { + modelType: ModelType = 'local'; + private _serverUrl: string; + private _modelName: string; + private defaultParams: any; + + constructor(modelName: string, serverUrl: string, params: any = {}) { + this._modelName = modelName; + this._serverUrl = serverUrl; + this.defaultParams = { + temperature: 0.8, + max_tokens: 1000, + ...params, + }; + } + + get modelName(): string { + return this._modelName; + } + + public async initialize(): Promise { + try { + const response = await axios.post(`${this._serverUrl}/initialize`, {model_name: this._modelName}); + return response.data; + } catch (error) { + throw new Error(`Error during chat completion: ${(error as Error).message}`); + } + } + + async chatCompletion(params: any): Promise { + try { + const messages = params.messages?.map((msg: any) => ({ + role: msg.role, + content: msg.content, + })); + + const requestParams = { + model: this._modelName, + ...this.defaultParams, + ...params, + messages: messages || params.messages, + }; + const response = await axios.post(`${this._serverUrl}/generate`, requestParams); + return response.data; + } catch (error) { + throw new Error(`Error during chat completion: ${(error as Error).message}`); + } + } +} \ No newline at end of file diff --git a/src/terminal/terminalCore.ts b/src/terminal/terminalCore.ts index af296e7..296297c 100644 --- a/src/terminal/terminalCore.ts +++ b/src/terminal/terminalCore.ts @@ -11,7 +11,7 @@ interface Feature { interface TerminalCoreOptions { agentName?: string; - modelType?: 'openai' | 'anthropic' | 'fireworks'; + modelType?: 'openai' | 'anthropic' | 'fireworks' | 'local'; modelName?: string; maxActions?: number; actionCooldownMs?: number; @@ -57,6 +57,9 @@ export class TerminalCore extends EventEmitter { const agentName = this.options.agentName || "terminalAgent"; this.agent = new Agent({ agentName }); + if (this.options.modelType === 'local') { + await this.agent.initialize(); + } Logger.info('TerminalCore initialized with agent and features'); } diff --git a/src/types/agentSystem.ts b/src/types/agentSystem.ts index 88f0e9c..4eac126 100644 --- a/src/types/agentSystem.ts +++ b/src/types/agentSystem.ts @@ -29,7 +29,7 @@ export interface Tool { }; } -export type ModelType = 'openai' | 'fireworks' | 'anthropic'; +export type ModelType = 'openai' | 'fireworks' | 'anthropic' | 'local'; export interface ModelClient { modelType: ModelType;