diff --git a/CLAUDE.md b/CLAUDE.md index 0f941b098..ab0033fb5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,7 +4,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ## Project Overview -The Proprietary Trading Network (PTN) is a Bittensor subnet (netuid 8 mainnet, 116 testnet) developed by Taoshi. It operates as a competitive trading signal network where miners submit trading strategies and validators evaluate their performance using sophisticated metrics. +The Vanta Network (formerly Proprietary Trading Network/PTN) is a Bittensor subnet (netuid 8 mainnet, 116 testnet) developed by Taoshi. It operates as a competitive trading signal network where miners submit trading strategies and validators evaluate their performance using sophisticated metrics and risk-adjusted scoring. ## Development Commands @@ -18,17 +18,22 @@ python3 -m pip install -e . ### Running Components ```bash -# Validator (production with PM2) +# Validator (production with PM2) - uses "vanta" process name ./run.sh --netuid 8 --wallet.name --wallet.hotkey # Miner python neurons/miner.py --netuid 8 --wallet.name --wallet.hotkey -# Validator (development) +# Validator (development mode) python neurons/validator.py --netuid 8 --wallet.name --wallet.hotkey # Signal reception server for miners ./run_receive_signals_server.sh + +# Utility scripts in runnable/ +python runnable/check_validator_weights.py +python runnable/daily_portfolio_returns.py +python runnable/local_debt_ledger.py ``` ### Testing @@ -53,86 +58,271 @@ npm run preview # Preview production build ## Architecture Overview ### Core Network Components -- **`neurons/`** - Main network participants (miner.py, validator.py) -- **`vali_objects/`** - Validator logic, configurations, performance tracking -- **`miner_objects/`** - Miner tools including React dashboard and order placement -- **`shared_objects/`** - Common utilities for time management, validation, utilities +- **`neurons/`** - Main network participants + - `validator.py` - Validator orchestration and network management + - `miner.py` - Miner signal generation and submission + - `validator_base.py` - Base validator functionality + - `backtest_manager.py` - Backtesting utilities +- **`vali_objects/`** - Validator logic and services + - `challenge_period/` - Challenge period management for new miners + - `plagiarism/` - Plagiarism detection and scoring + - `position_management/` - Position tracking and management + - `price_fetcher/` - Real-time price data services + - `scoring/` - Performance metrics calculation + - `statistics/` - Miner performance statistics + - `utils/` - Utility services (elimination, asset selection, MDD checking, limit orders) + - `vali_dataclasses/` - Data structures for positions, orders, ledgers +- **`shared_objects/`** - Common infrastructure + - `rpc/` - RPC architecture (server_orchestrator, rpc_server_base, rpc_client_base) + - `locks/` - Position locking mechanisms + - `metagraph/` - Metagraph management and caching + - Utilities: cache_controller, slack_notifier, error_utils +- **`miner_objects/`** - Miner tooling + - `miner_dashboard/` - React/TypeScript dashboard for monitoring + - `prop_net_order_placer.py` - Order placement utilities + - `position_inspector.py` - Position analysis tools - **`template/`** - Bittensor protocol definitions and base classes ### Data Infrastructure -- **`data_generator/`** - Financial data services (Polygon, Tiingo, Binance, Bybit, Kraken) -- **`vanta_api/`** - API management for real-time data and communication -- **`mining/`** - Signal processing pipeline (received/processed/failed signals) -- **`validation/`** - Validator state (eliminations, plagiarism, performance ledgers) +- **`data_generator/`** - Financial market data services (Polygon, Tiingo, Binance, Bybit, Kraken) +- **`vanta_api/`** - Vanta Network API layer + - `rest_server.py` - REST API server for signal submission and queries + - `websocket_server.py` / `websocket_client.py` - Real-time WebSocket communication + - `api_manager.py` - API key and authentication management + - `nonce_manager.py` - Request nonce handling +- **`mining/`** - Signal processing pipeline + - `received_signals/` - Incoming miner signals + - `processed_signals/` - Validated and processed signals +- **`validation/`** - Validator state persistence + - `miners/` - Per-miner performance and position data + - `plagiarism/` - Plagiarism detection cache + - `tmp/` - Temporary processing files +- **`runnable/`** - Utility scripts and analysis tools + - Portfolio analytics, debt ledger management, elimination analysis + - Checkpoint validation and migration scripts +- **`tests/`** - Test suites + - `vali_tests/` - Comprehensive validator unit and integration tests + - `validation/` - Validation-specific test scenarios + - `shared_objects/` - Shared infrastructure tests + +### RPC Architecture +The system uses a distributed RPC architecture for inter-process communication: +- **Server Orchestrator**: Manages lifecycle of all RPC servers +- **18+ RPC Services**: Position management, elimination, plagiarism, price fetching, ledgers, etc. +- **Port Range**: 50000-50022 (centrally managed in vali_config.py) +- **Connection Modes**: LOCAL (direct/testing) and RPC (network/production) ### Key Configuration Files -- **`vali_objects/vali_config.py`** - Main validator configuration including supported trade pairs -- **`requirements.txt`** - Python dependencies (Bittensor 9.7.0, financial APIs, ML libraries) -- **`meta/meta.json`** - Version management (subnet_version: 6.3.0) +- **`vali_objects/vali_config.py`** - Main validator configuration + - RPC service definitions and ports + - Trade pair definitions (crypto, forex, equities, indices) + - Scoring weights and risk parameters + - Challenge period and elimination thresholds +- **`miner_config.py`** - Miner configuration +- **`requirements.txt`** - Python dependencies (Bittensor 9.9.0, Pydantic 2.10.3, financial APIs) +- **`meta/meta.json`** - Version management (subnet_version: 8.8.8) +- **`setup.py`** - Package setup (taoshi-prop-net) ## Trading System Architecture ### Signal Flow -1. Miners submit LONG/SHORT/FLAT signals for forex and crypto pairs -2. Validators receive signals via API endpoints -3. Real-time price validation using multiple data sources -4. Position tracking with leverage limits and slippage modeling -5. Performance calculation using 5 risk-adjusted metrics (20% each) +1. Miners submit LONG/SHORT/FLAT signals via Vanta API (REST/WebSocket) +2. Validators receive and validate signals through `vanta_api/rest_server.py` +3. Real-time price validation using multiple data sources (Polygon, Tiingo, Binance, Bybit, Kraken) +4. Position tracking via RPC services with leverage limits and slippage modeling +5. Performance calculation using debt-based scoring system + +### Supported Assets +- **Crypto**: BTC/USD, ETH/USD, SOL/USD, XRP/USD, DOGE/USD, ADA/USD (6 pairs) +- **Forex**: 32 major currency pairs (EUR/USD, GBP/USD, USD/JPY, etc.) + - Grouped into G1-G5 subcategories by liquidity/volume +- **Equities**: 7 major stocks (NVDA, AAPL, TSLA, AMZN, MSFT, GOOG, META) - currently blocked +- **Indices**: 6 global indices (SPX, DJI, NDX, VIX, FTSE, GDAXI) - currently blocked +- **Commodities**: XAU/USD, XAG/USD - currently blocked ### Performance Evaluation -- **Metrics**: Calmar, Sharpe, Omega, Sortino ratios + total return -- **Risk Management**: 10% max drawdown elimination threshold -- **Fees**: Carry fees (10.95%/3% annually) and slippage costs -- **Scoring**: Weighted average with recent performance emphasis +- **Current Scoring**: Debt-based system tracking emissions, performance, and penalties + - PnL weight: 100% (other metrics set to 0 in current config) + - Weighted average with decay rate (0.075) for recent performance emphasis + - 120-day target ledger window +- **Legacy Metrics** (configurable): Calmar, Sharpe, Omega, Sortino ratios + returns +- **Risk Management**: + - 10% max drawdown elimination threshold (MAX_TOTAL_DRAWDOWN = 0.9) + - 5% daily drawdown limit (MAX_DAILY_DRAWDOWN = 0.95) + - Risk-adjusted performance penalties based on Sharpe, Sortino, Calmar, Omega ratios +- **Fees**: + - Carry fees: 10.95% annually (crypto), 3% annually (forex) + - Spread fees: 0.1% × leverage (crypto only) + - Slippage costs: Higher for high leverage and low liquidity assets +- **Leverage Limits**: + - Crypto: 0.01 to 0.5x + - Forex: 0.1 to 5x + - Equities: 0.1 to 3x + - Indices: 0.1 to 5x + - Portfolio cap: 10x across all positions ### Elimination Mechanisms -- **Plagiarism**: Order similarity analysis for copy detection -- **Drawdown**: Automatic elimination at 10% max drawdown -- **Probation**: 30-day period for miners below 25th rank +- **Plagiarism**: Cross-correlation analysis detecting order similarity + - 75% similarity threshold, 10-day lookback window + - Time-lag analysis for follower detection + - 2-week review period before elimination +- **Max Drawdown**: Automatic elimination at 10% MDD + - Continuous monitoring via `mdd_checker/` service + - 60-second refresh interval +- **Challenge Period**: New miners enter 61-90 day challenge period + - Must reach 75th percentile to enter main competition + - Minimal weights during challenge period +- **Probation**: Miners below rank 25 in asset class + - 60-day probation period + - Must outscore 15th-ranked miner to avoid elimination ## Development Patterns ### File Naming Conventions - Use snake_case for Python files -- Prefix test files with `test_` -- Configuration files use descriptive names (vali_config.py, miner_config.py) +- RPC servers: `*_server.py` (e.g., `position_manager_server.py`) +- RPC clients: `*_client.py` (e.g., `elimination_client.py`) +- Test files: `test_*.py` prefix +- Configuration files: descriptive names (vali_config.py, miner_config.py) ### Code Organization -- Validators handle all position tracking and performance calculation -- Miners focus on signal generation and submission -- Shared objects contain common utilities (time, validation, crypto) -- Real-time data flows through dedicated API layer +- **RPC Architecture**: Services communicate via RPC for modularity and fault isolation + - `shared_objects/rpc/rpc_server_base.py` - Base RPC server class + - `shared_objects/rpc/rpc_client_base.py` - Base RPC client class + - `shared_objects/rpc/server_orchestrator.py` - Manages server lifecycle + - Connection modes: LOCAL (testing) vs RPC (production) +- **Validators**: Orchestrate multiple RPC services for position tracking, scoring, elimination +- **Miners**: Signal generation and submission via Vanta API +- **Shared Objects**: Common utilities (locks, metagraph, cache, error handling) +- **Data Flow**: Real-time → Vanta API → RPC services → Performance ledgers + +### RPC Service Pattern +```python +# Server implementation inherits from RPCServerBase +from shared_objects.rpc.rpc_server_base import RPCServerBase + +class MyServer(RPCServerBase): + def __init__(self, config, connection_mode=RPCConnectionMode.RPC): + super().__init__( + service_name="MyServer", + port=config.RPC_MY_PORT, + config=config, + connection_mode=connection_mode + ) + + def my_rpc_method(self, arg1, arg2): + # RPC-exposed method + return result + +# Client usage +from vali_objects.vali_config import RPCConnectionMode + +# Production (RPC mode) +client = MyClient(connection_mode=RPCConnectionMode.RPC) + +# Testing (LOCAL mode - bypass RPC) +client = MyClient(connection_mode=RPCConnectionMode.LOCAL) +client.set_direct_server(server_instance) +``` ### External Dependencies -- **Bittensor 9.7.0** for blockchain integration -- **Financial APIs**: Polygon ($248/month), Tiingo ($50/month) -- **ML Stack**: scikit-learn, pandas, scipy for analysis -- **Web**: Flask for APIs, React/TypeScript/Vite for dashboard +- **Bittensor 9.9.0** - Blockchain and subnet integration +- **Pydantic 2.10.3** - Data validation and serialization +- **Financial APIs**: + - Polygon API Client 1.15.3 ($248/month) + - Tiingo 0.15.6 ($50/month) +- **ML Stack**: scikit-learn 1.5.0, scipy 1.13.0, pandas 2.2.2 +- **Web Services**: + - Flask 3.0.3 + Waitress 2.1.2 for REST API + - WebSockets for real-time communication +- **Data Visualization**: matplotlib 3.9.0 +- **Cloud Services**: Google Cloud Storage 2.17.0, Secret Manager 2.21.1 +- **Taoshi SDKs**: + - collateral_sdk@1.0.6 - Collateral management + - vanta-cli@2.0.0 - Vanta network CLI tools ## Production Deployment ### PM2 Process Management The `run.sh` script provides production deployment with: -- Automatic version checking and updates from GitHub -- Process monitoring and restart capabilities -- Version comparison and rollback safety +- Process name: `vanta` (migrated from legacy `ptn` name) +- Automatic version checking every 30 minutes (at :07 and :37) +- Git pull and automatic updates from GitHub (taoshidev/vanta-network) +- Exponential backoff retry logic (1s → 60s max) for version checks +- Process monitoring with auto-restart on failure +- Minimum uptime: 5 minutes, Max restarts: 5 + +### Version Management +- Current version: 8.8.8 (in `meta/meta.json`) +- Version checking against GitHub API +- Automatic pip install and package updates +- Safe rollback: git pull only if version is newer ### State Management -- **Backups**: Automatic timestamped validator state backups -- **Persistence**: Position data, performance ledgers, elimination tracking -- **Recovery**: Validator state regeneration capabilities +- **Backups**: Automatic timestamped validator state backups via `vali_bkp_utils.py` + - Compressed checkpoint files (`validator_checkpoint.json.gz`) + - Migration scripts in `runnable/` for state transformations +- **Persistence**: + - Position data per miner in `validation/miners/` + - Performance ledgers (RPC service) + - Debt ledgers (RPC service) + - Elimination tracking (RPC service) + - Plagiarism scores in `validation/plagiarism/` +- **Recovery**: State regeneration via `restore_validator_from_backup.py` + +### RPC Service Management +- Server orchestrator manages 18+ RPC services +- Health monitoring and automatic restarts +- Exponential backoff for failed connections +- Port conflict detection and resolution +- Graceful shutdown coordination ## Testing Strategy -Test files located in `tests/vali_tests/` cover: -- Position management and tracking -- Plagiarism detection algorithms -- Market hours and pricing validation -- Risk profiling and metrics calculation -- Challenge period integration -- Elimination manager functionality +### Test Organization +- **`tests/vali_tests/`** - Comprehensive validator test suite (60+ test files) + - Position management and tracking (`test_positions*.py`) + - Plagiarism detection (`test_plagiarism*.py`) + - Elimination logic (`test_elimination*.py`) + - Challenge period (`test_challengeperiod*.py`) + - Debt and performance ledgers (`test_debt*.py`, `test_ledger*.py`) + - Limit orders (`test_limit_order*.py`) + - Auto-sync and checkpointing (`test_auto_sync*.py`) + - Risk profiling and metrics (`test_risk*.py`, `test_metrics*.py`) + - Asset selection and segmentation +- **`tests/validation/`** - Validation scenario tests +- **`tests/shared_objects/`** - Infrastructure tests + +### Test Execution +```bash +# Run entire test suite +python tests/run_vali_testing_suite.py + +# Run specific test file +python tests/run_vali_testing_suite.py test_positions.py + +# Run with pytest directly +python -m pytest tests/vali_tests/test_elimination_manager.py -v +``` + +### Test Patterns +- Use `RPCConnectionMode.LOCAL` for fast unit tests (bypass RPC) +- Use `RPCConnectionMode.RPC` for integration tests (full RPC behavior) +- Mock utilities in `tests/vali_tests/mock_utils.py` +- Base objects in `tests/vali_tests/base_objects/` +- Fixtures defined in `conftest.py` ## Requirements -- Python 3.10+ (required) -- Hardware: 2-4 vCPU, 8-16 GB RAM -- Network registration: 2.5 TAO on mainnet \ No newline at end of file +- **Python**: 3.10+ (required), supports 3.10, 3.11, 3.12 +- **Hardware**: + - CPU: 2-4 vCPU minimum + - RAM: 8-16 GB recommended + - Storage: Sufficient for checkpoints and position data +- **Network**: + - Registration: 2.5 TAO on mainnet + - Stable internet connection for API access + - Open ports: 50000-50022 (RPC services), 48888 (REST API), 8765 (WebSocket) +- **Software**: + - PM2 for process management + - jq for JSON parsing (required by run.sh) + - Git for version management \ No newline at end of file diff --git a/entitiy_management/README.txt b/entitiy_management/README.txt new file mode 100644 index 000000000..3640a55bb --- /dev/null +++ b/entitiy_management/README.txt @@ -0,0 +1,55 @@ +propose a solution for a new feature "Entity miners" + + + One miner hotkey VANTA_ENTITY_HOTKEY will correspond to an entity. + + We will track entities with an EntityManager which persists data to disk, offers getters and setters via a client, + and has a server class that delegates to a manager instance (just like challenge_period flow). + + Each entity i,e VANTA_ENTITY_HOTKEY can have subaccounts (monotonically increasing id). + Subaccounts get their own synthetic hotkey which is f"{VANTA_ENTITY_HOTKEY}_{subaccount_id}" + If a subaccount gets eliminated, that id can never be assigned again. An entity can only have MAX_SUBACCOUNTS_PER_ENTITY + subaccounts at once. The limit is 500. Thus instead of tracking eliminated subaccount ids, + we can simply maintain the active subaccount ids as well as the next id to be assigned + + + We must support rest api requests of entity data using an EntityManagerClient in rest server. + + 1. `POST register_subaccount` → returns {success, subaccount_id, subaccount_uuid} + 1. Verifies entity collateral and slot allowance + 2. `GET subaccount_status/{subaccouunt_id}` → active/eliminated/unknown + +This is the approach we want to utilize for the subaccount registration process: +VantaRestServer endpoint exposed which does the collateral operations (placeholder for now) + and then returns the newly-registered subaccount id to the caller. + The validator then send a synapse message to all other validators so they are synced with the new subaccount id. + Refer for the flow in broadcast_asset_selection_to_validators to see how we should do this. + + +EntityManager (RPCServerBase) will have its own daemon that periodically assess elimination criteria for entitiy miners. +Put a placeholder in this logic for now. + + +Most endpoints in VantaRestServer will support subaccounts directly since the passed in hotkey can be synthetic and +our existing code will be able to work with synthetic hotkeys as long as we adjust the metagraph logic to detect +synthetic hotkeys (have an underscore) and then making the appropriate call to the EntityManagerClient to see if +that subaccount is still active. and if the VANTA_ENTITY_HOTKEY hotkey is still in the raw metagraph. Our has_hotkey method +with this update should allow to work smoothly but let me know if there are other key parts of our system that +need to be updated to support synthetic hotkeys. + + +1. The entity hotkey (VANTA_ENTITY_HOTKEY) cannot place orders itself. Only its subaccounts can. This will need +to be enforced in validator.py. + +2. Account sizes for synthetic hotkeys is set to a fixed value using a ContractClient after a blackbox function +transfers collateral from VANTA_ENTITY_HOTKEY. Leave placeholder functions for this. This account size init is done during +the subaccount registration flow. + +3. debt based scoring will read debt ledgers for all miners including subaccounts. It needs to agrgeagte the debt +ledgers for all subaccounts into a single debt ledger representing the sum of all subaccount performance. +The key for this debt ledger will simply be the entity hotkey (VANTA_ENTITY_HOTKEY). + +4. Sub-accounts challenge period is an instantaneous pass if they get 3% returns against 6% drawdown within 90 days. Just like how in mdd checker, we can get returns and drawdown in different intervals, we will implement this in our +EntityManager daemon. A PerfLedgerClient is thus needed. + +- Each entity miner can host up to **500 active sub-accounts** diff --git a/entitiy_management/README_steps.txt b/entitiy_management/README_steps.txt new file mode 100644 index 000000000..9e460175e --- /dev/null +++ b/entitiy_management/README_steps.txt @@ -0,0 +1,404 @@ +# Implementation Prompt: Entity Miners Feature for Vanta Network + +## Feature Overview +Implement an "Entity Miners" system that allows one entity hotkey (VANTA_ENTITY_HOTKEY) to manage multiple subaccounts, each with synthetic hotkeys. This enables entities to operate multiple trading strategies under a single parent entity with collateral verification and elimination tracking. + +### Phase 1 Specifications +- Each entity miner can host up to **500 active sub-accounts** during Phase 1 +- Order rate limits are enforced **per sub-account** (per synthetic hotkey) +- Since synthetic hotkeys are passed into the system, existing per-hotkey rate limiting automatically applies +- **No changes needed to rate limiting logic** - it already works per-hotkey +- This allows entities to submit orders much faster by distributing across multiple sub-accounts + +### Sub-account Challenge Period +- Sub-accounts enter a **90-day challenge period** upon creation +- **Instantaneous pass criteria**: 3% returns against 6% drawdown within 90 days +- Challenge period assessment runs in EntityManager daemon (similar to MDD checker) +- Uses PerfLedgerClient to get returns and drawdown at different intervals +- Once passed, sub-account exits challenge period and operates normally +- Failed sub-accounts are eliminated after 90 days if criteria not met + +## Context Files to Review +Before starting, review these files to understand existing patterns: +1. `vali_objects/challenge_period/` - Reference for Manager/Server/Client RPC pattern +2. `vali_objects/utils/elimination_manager.py` - Elimination logic patterns +3. `vali_objects/utils/mdd_checker/` - MDD checker pattern (returns and drawdown at intervals) +4. `vali_objects/vali_dataclasses/perf_ledger.py` - Performance ledger client usage +5. `shared_objects/rpc/rpc_server_base.py` - Base RPC server class +6. `shared_objects/rpc/rpc_client_base.py` - Base RPC client class +7. `neurons/validator.py` - Look for `broadcast_asset_selection_to_validators` method +8. `vali_objects/vali_config.py` - Port definitions and configuration +9. `vanta_api/rest_server.py` - REST API patterns +10. `shared_objects/metagraph_manager.py` - Metagraph logic and `has_hotkey` method + +## Architecture Requirements + +### 1. Data Model +Create entity data structures with: +- Entity hotkey (VANTA_ENTITY_HOTKEY) +- Subaccount list with monotonically increasing IDs +- Synthetic hotkey format: `{VANTA_ENTITY_HOTKEY}_{subaccount_id}` +- Subaccount status: active/eliminated/unknown +- Challenge period tracking per subaccount: + - challenge_period_active: bool (default True for new subaccounts) + - challenge_period_passed: bool (default False) + - challenge_period_start_ms: timestamp + - challenge_period_end_ms: timestamp (90 days from start) +- Collateral tracking per entity +- Slot allowance tracking +- Registration timestamps +- UUID generation for subaccounts + +### 2. EntityManager (entitiy_management/entity_manager.py) +Implement core business logic following the challenge_period pattern: +- Persistent disk storage (similar to challenge_period state management) +- Entity registration and tracking +- Subaccount creation with monotonic ID generation + - Track active subaccount IDs (max 500 active at once) + - Track next_subaccount_id (monotonically increasing, never reused) + - Eliminated IDs implicitly: IDs in [0, next_id) that aren't in active set + - Initialize challenge period fields on creation +- Subaccount status management (active/eliminated) +- Challenge period assessment (daemon): + - Uses PerfLedgerClient to get returns and drawdown + - Checks 3% returns against 6% drawdown threshold + - Operates similar to MDD checker with interval checks + - Marks challenge_period_passed=True on success + - Eliminates subaccount after 90 days if not passed +- Collateral verification methods (placeholder implementation) +- Slot allowance checking +- Elimination criteria assessment (placeholder for periodic daemon) +- Thread-safe operations with proper locking +- Getters: get_entity_data, get_subaccount_status, get_all_entities, is_synthetic_hotkey, is_registered_entity +- Setters: register_entity, create_subaccount, eliminate_subaccount, update_collateral, mark_challenge_period_passed +- Dependencies: PerfLedgerClient (for challenge period assessment) +- PLACEHOLDER: Collateral transfer during subaccount creation (blackbox function) +- PLACEHOLDER: Account size initialization via ContractClient during subaccount creation + +### 3. EntityServer (entitiy_management/entity_server.py) +RPC server inheriting from RPCServerBase: +- Service name: "EntityServer" +- Port: Add RPC_ENTITY_SERVER_PORT to vali_config.py (e.g., 50023) +- Connection modes: LOCAL and RPC support +- Delegate all operations to EntityManager instance +- RPC-exposed methods matching manager's public API +- Background daemon thread for periodic assessment: + - Challenge period evaluation (every 5 minutes or configurable interval) + - Checks returns/drawdown via PerfLedgerClient + - Marks passed subaccounts or eliminates expired ones + - Elimination assessment (placeholder logic) +- Health check endpoint +- Graceful shutdown handling + +### 4. EntityClient (entitiy_management/entity_client.py) +RPC client inheriting from RPCClientBase: +- Connect to EntityServer +- Proxy methods for all manager operations +- Support both LOCAL and RPC connection modes +- Error handling and retry logic +- Methods: register_entity, create_subaccount, get_subaccount_status, get_entity_data, is_synthetic_hotkey, eliminate_subaccount + +### 5. REST API Integration (vanta_api/rest_server.py) +Add new endpoints to VantaRestServer: + +**POST /register_subaccount** +- Input: entity_hotkey, signature/auth +- Collateral verification (placeholder call) +- Slot allowance check via EntityClient +- Create subaccount via EntityClient +- Broadcast new subaccount to all validators (synapse message) +- Return: {success: bool, subaccount_id: int, subaccount_uuid: str, synthetic_hotkey: str} + +**GET /subaccount_status/{subaccount_id}** +- Query EntityClient for status +- Return: {status: "active"|"eliminated"|"unknown", synthetic_hotkey: str} + +**GET /entity_data/{entity_hotkey}** +- Query EntityClient for full entity data +- Return: {entity_hotkey, subaccounts: [...], collateral, active_count} + +### 6. Validator Order Placement Restrictions (neurons/validator.py) +Add validation to prevent entity hotkeys from placing orders: +- **RULE**: Only synthetic hotkeys (subaccounts) can place orders +- **RULE**: Entity hotkeys (VANTA_ENTITY_HOTKEY) cannot place orders directly +- Add validation in signal processing: + - Check if hotkey is a registered entity (not synthetic) + - If entity hotkey detected, reject order with error message + - Only allow orders from synthetic hotkeys +- Implementation location: Signal validation in validator.py +- Use EntityClient.is_registered_entity() and is_synthetic_hotkey() methods + +### 7. Validator Syncing (neurons/validator.py or similar) +Implement broadcast mechanism following broadcast_asset_selection_to_validators pattern: +- Create new synapse type for subaccount registration +- `broadcast_subaccount_registration(entity_hotkey, subaccount_id, subaccount_uuid)` method +- Send to all validators in metagraph +- Handle responses and log failures +- Ensure idempotent registration (handle duplicates gracefully) + +### 8. Metagraph Integration (shared_objects/metagraph_manager.py) +Update metagraph logic to support synthetic hotkeys: + +**Update `has_hotkey` method:** +- Detect synthetic hotkeys (contains underscore) +- Parse entity hotkey and subaccount_id from synthetic format +- Verify entity hotkey exists in raw metagraph +- Query EntityClient to check if subaccount is active +- Return True only if both entity exists and subaccount is active + +**Consider updates to:** +- UID resolution for synthetic hotkeys (may need synthetic UID mapping) +- Hotkey validation methods +- Any caching mechanisms that assume hotkeys don't have underscores + +### 9. Debt Ledger Aggregation for Entity Scoring +Implement aggregation layer for entity debt-based scoring: + +**Requirement**: Aggregate all subaccount debt ledgers into a single entity debt ledger +- **Key**: Entity hotkey (VANTA_ENTITY_HOTKEY) +- **Value**: Sum of all active subaccount performance metrics +- **Update trigger**: Whenever any subaccount performance changes + +**Implementation**: +- Add method: `aggregate_entity_debt_ledger(entity_hotkey) -> DebtLedger` + - Query EntityClient for all active subaccounts + - Read individual debt ledgers for each synthetic hotkey + - Sum performance metrics (PnL, returns, etc.) + - Store aggregated ledger under entity hotkey +- Location: Debt ledger scoring system (vali_objects/vali_dataclasses/debt_ledger.py or scoring/) +- Scoring reads entity-level aggregated ledger for weight calculation +- Individual subaccounts maintain separate ledgers for tracking + +**Aggregation logic**: +```python +def aggregate_entity_debt_ledger(entity_hotkey: str) -> DebtLedger: + entity_data = entity_client.get_entity_data(entity_hotkey) + active_subaccounts = entity_data.get_active_subaccounts() + + aggregated_ledger = DebtLedger(hotkey=entity_hotkey) + + for subaccount in active_subaccounts: + synthetic_hotkey = subaccount.synthetic_hotkey + subaccount_ledger = debt_ledger_client.get_ledger(synthetic_hotkey) + aggregated_ledger.add_ledger(subaccount_ledger) # Sum metrics + + return aggregated_ledger +``` + +### 10. Subaccount Registration Enhancements +Add collateral and account size initialization to registration flow: + +**During subaccount creation** (entity_manager.py or REST endpoint): +1. **PLACEHOLDER**: Transfer collateral from entity to subaccount + - `blackbox_transfer_collateral(from_hotkey=entity_hotkey, to_hotkey=synthetic_hotkey, amount=AMOUNT)` + - This is a placeholder for future collateral SDK integration + +2. **PLACEHOLDER**: Initialize account size via ContractClient + - `contract_client.set_account_size(hotkey=synthetic_hotkey, account_size=FIXED_SIZE)` + - Fixed account size for all subaccounts (e.g., 10000 USD) + - This happens after collateral transfer succeeds + +**Updated create_subaccount flow**: +```python +# Create subaccount metadata +subaccount_info = SubaccountInfo(...) + +# PLACEHOLDER: Transfer collateral +# success = blackbox_transfer_collateral(entity_hotkey, synthetic_hotkey, amount) +# if not success: +# return False, None, "Collateral transfer failed" + +# PLACEHOLDER: Set account size +# contract_client.set_account_size(synthetic_hotkey, FIXED_SUBACCOUNT_SIZE) + +return True, subaccount_info, "Success" +``` + +### 11. Configuration (vali_objects/vali_config.py) +Add configuration parameters: +- RPC_ENTITY_SERVER_PORT = 50023 +- ENTITY_ELIMINATION_CHECK_INTERVAL = 300 # 5 minutes (for challenge period + elimination checks) +- ENTITY_MAX_SUBACCOUNTS = 500 # Maximum active subaccounts per entity +- ENTITY_DATA_DIR = "validation/entities/" # Persistence directory +- FIXED_SUBACCOUNT_SIZE = 10000.0 # Fixed account size for subaccounts (USD) +- SUBACCOUNT_COLLATERAL_AMOUNT = 1000.0 # Placeholder collateral amount +- SUBACCOUNT_CHALLENGE_PERIOD_DAYS = 90 # Challenge period duration +- SUBACCOUNT_CHALLENGE_RETURNS_THRESHOLD = 0.03 # 3% returns required +- SUBACCOUNT_CHALLENGE_DRAWDOWN_THRESHOLD = 0.06 # 6% max drawdown allowed + +## Implementation Steps + +### Phase 1: Core Infrastructure (entitiy_management/) +1. Implement EntityManager class with data structures and persistence + - Add challenge period fields to SubaccountInfo + - Initialize challenge period on subaccount creation +2. Implement EntityServer with RPC capabilities + - Configure daemon interval for challenge period checks (5 minutes) +3. Implement EntityClient for RPC communication +4. Add PerfLedgerClient dependency to EntityManager +5. Implement challenge period assessment logic in daemon: + - Check returns and drawdown via PerfLedgerClient + - Mark challenge_period_passed=True on success (3% returns, 6% drawdown) + - Eliminate subaccounts after 90 days if not passed +6. Add comprehensive unit tests for manager logic +7. Add integration tests for RPC communication (LOCAL and RPC modes) +8. Add challenge period tests (pass, fail, edge cases) + +### Phase 2: Validator Order Placement Restrictions +1. Add is_registered_entity() method to EntityManager +2. Expose is_registered_entity_rpc() in EntityServer +3. Add is_registered_entity() to EntityClient +4. Update validator.py signal validation to check entity hotkeys +5. Reject orders from entity hotkeys (non-synthetic) +6. Test order rejection for entity hotkeys +7. Test order acceptance for synthetic hotkeys + +Note: Rate limiting works automatically per synthetic hotkey since existing logic is per-hotkey. No changes needed. + +### Phase 3: Metagraph Integration +1. Update has_hotkey method in metagraph_manager.py +2. Add synthetic hotkey detection utility methods +3. Test synthetic hotkey validation end-to-end +4. Ensure existing position management works with synthetic hotkeys + +### Phase 4: REST API & Validator Syncing +1. Add REST endpoints to VantaRestServer +2. Add placeholder collateral transfer in subaccount creation +3. Add placeholder account size initialization via ContractClient +4. Implement synapse message type for subaccount registration +5. Implement broadcast mechanism in validator +6. Add API authentication/authorization for entity endpoints +7. Test REST API with synthetic hotkeys + +### Phase 5: Debt Ledger Aggregation +1. Review debt ledger system architecture +2. Implement aggregate_entity_debt_ledger() method +3. Add DebtLedger.add_ledger() method for summing metrics +4. Update scoring logic to use aggregated entity ledgers +5. Test aggregation with multiple active subaccounts +6. Test that eliminated subaccounts are excluded from aggregation +7. Verify entity-level weights use aggregated performance + +### Phase 6: System Integration +1. Integrate EntityServer into server orchestrator startup +2. Test full flow: register entity → create subaccount → submit orders → track positions +3. Verify order rejection for entity hotkeys +4. Verify order acceptance for synthetic hotkeys +5. Test debt ledger aggregation for scoring +6. Verify elimination logic (placeholder) runs periodically +7. Test validator syncing across multiple validator instances +8. Load testing with multiple entities and subaccounts + +### Phase 7: Placeholder Implementation Notes +1. Collateral verification: Add TODO comment with interface specification +2. Collateral transfer: Add placeholder blackbox_transfer_collateral() +3. Account size: Add placeholder ContractClient.set_account_size() +4. Elimination criteria: Add placeholder that logs but doesn't eliminate +5. Real collateral integration: Document expected collateral SDK integration points + +## Testing Strategy + +### Unit Tests (tests/vali_tests/test_entity_*.py) +- test_entity_manager.py: Test all manager operations, persistence, thread safety +- test_entity_server_client.py: Test RPC communication in both modes +- test_synthetic_hotkeys.py: Test hotkey parsing and validation + +### Integration Tests +- test_entity_rest_api.py: Test REST endpoints with EntityClient +- test_entity_metagraph.py: Test metagraph with synthetic hotkeys +- test_entity_positions.py: Test position tracking with synthetic hotkeys +- test_entity_elimination.py: Test elimination flow for subaccounts + +### Key Test Cases +1. Entity registration with collateral check +2. Subaccount creation with monotonic IDs +3. Monotonic ID behavior: Verify next_subaccount_id never reuses eliminated IDs +4. Active subaccount limit: Verify max 500 active subaccounts per entity +5. Synthetic hotkey generation and parsing +6. Subaccount elimination and status updates +7. Challenge period initialization on subaccount creation +8. Challenge period pass: 3% returns against 6% drawdown +9. Challenge period failure: Elimination after 90 days +10. Challenge period assessment via daemon (PerfLedgerClient integration) +11. Entity hotkey order rejection (cannot place orders) +12. Synthetic hotkey order acceptance (can place orders) +13. Verify rate limiting works independently per synthetic hotkey (existing logic) +14. Debt ledger aggregation for entity scoring +15. Aggregation excludes eliminated subaccounts +16. Collateral transfer placeholder integration +17. Account size initialization placeholder integration +18. Metagraph has_hotkey with synthetic hotkeys +19. REST API authentication and authorization +20. Validator broadcast and sync +21. Position submission and tracking with synthetic hotkeys +22. Concurrent subaccount operations (thread safety) +23. Persistence and recovery from disk +24. Challenge period state persistence across restarts + +## Code Style & Patterns +- Follow existing RPC architecture patterns (see challenge_period/) +- Use Pydantic for data validation and serialization +- Implement proper logging with context (entity_hotkey, subaccount_id) +- Use snake_case for file and method names +- Add docstrings for all public methods +- Handle errors gracefully with descriptive messages +- Use type hints throughout +- Follow validator state persistence patterns + +## Success Criteria +✓ Entity can register and create multiple subaccounts +✓ Synthetic hotkeys work seamlessly with existing position management +✓ REST API endpoints functional and secured +✓ Validator syncing maintains consistency across network +✓ Metagraph correctly validates synthetic hotkeys +✓ Elimination logic framework in place (with placeholders) +✓ Comprehensive test coverage (>80%) +✓ Documentation complete with usage examples + +## Key Edge Cases to Handle +- Duplicate subaccount registration attempts +- Entity hotkey that looks synthetic (contains underscore) +- Entity hotkey attempting to place orders (should be rejected) +- Subaccount operations after entity is eliminated +- Subaccount operations after parent entity is removed from metagraph +- Race conditions in subaccount ID generation +- Validator sync failures and retry logic +- Disk persistence failures and recovery +- Collateral verification failures +- Collateral transfer failures during subaccount creation +- Account size initialization failures +- Maximum subaccount limit enforcement (500 active) +- Challenge period edge cases: + - Subaccount reaching exactly 3% returns at exactly 6% drawdown + - PerfLedgerClient unavailable during daemon check + - Challenge period expiry during validator restart + - Multiple subaccounts passing challenge period simultaneously + - Challenge period status persistence across crashes + - Subaccount elimination while in active challenge period +- Debt ledger aggregation when some subaccounts have no ledger data +- Debt ledger aggregation performance with 500 active subaccounts +- next_subaccount_id overflow (unlikely but handle gracefully) + +## Additional Considerations +- Migration path for existing miners to entities (if needed) +- Monitoring and alerting for entity/subaccount operations +- Rate limiting for subaccount creation +- Audit logging for all entity operations +- Performance impact of synthetic hotkey validation +- Backward compatibility with non-entity miners + +--- + +## Quick Start Command for Implementation +Once you start implementing, use this approach: +1. Read all context files listed above to understand patterns +2. Start with entity_manager.py (core logic, no RPC dependencies) +3. Add entity_server.py and entity_client.py (RPC layer) +4. Update vali_config.py with new configuration +5. Update metagraph logic for synthetic hotkeys +6. Add REST API endpoints +7. Implement validator broadcast mechanism +8. Write comprehensive tests for each component +9. Integration test the full flow + +Use LOCAL connection mode for fast unit testing, RPC mode for integration testing. diff --git a/entitiy_management/entity_client.py b/entitiy_management/entity_client.py new file mode 100644 index 000000000..c38a8098c --- /dev/null +++ b/entitiy_management/entity_client.py @@ -0,0 +1,300 @@ +# developer: jbonilla +# Copyright � 2024 Taoshi Inc +""" +EntityClient - Lightweight RPC client for entity miner management. + +This client connects to the EntityServer via RPC. +Can be created in ANY process - just needs the server to be running. + +Usage: + from entitiy_management.entity_client import EntityClient + + # Connect to server (uses ValiConfig.RPC_ENTITY_PORT by default) + client = EntityClient() + + # Register an entity + success, message = client.register_entity("my_entity_hotkey") + + # Create a subaccount + success, subaccount_info, message = client.create_subaccount("my_entity_hotkey") + + # Check if hotkey is synthetic + if client.is_synthetic_hotkey(hotkey): + entity_hotkey, subaccount_id = client.parse_synthetic_hotkey(hotkey) +""" +from typing import Optional, Tuple, Dict + +import template.protocol +from shared_objects.rpc.rpc_client_base import RPCClientBase +from vali_objects.vali_config import ValiConfig, RPCConnectionMode + + +class EntityClient(RPCClientBase): + """ + Lightweight RPC client for EntityServer. + + Can be created in ANY process. No server ownership. + Port is obtained from ValiConfig.RPC_ENTITY_PORT. + + In LOCAL mode (connection_mode=RPCConnectionMode.LOCAL), the client won't connect via RPC. + Instead, use set_direct_server() to provide a direct EntityServer instance. + """ + + def __init__( + self, + port: int = None, + connection_mode: RPCConnectionMode = RPCConnectionMode.RPC, + running_unit_tests: bool = False, + connect_immediately: bool = False + ): + """ + Initialize entity client. + + Args: + port: Port number of the entity server (default: ValiConfig.RPC_ENTITY_PORT) + connection_mode: RPCConnectionMode.LOCAL for tests (use set_direct_server()), RPCConnectionMode.RPC for production + running_unit_tests: Whether running in test mode + connect_immediately: Whether to connect immediately (default: False for lazy connection) + """ + self._direct_server = None + self.running_unit_tests = running_unit_tests + + # In LOCAL mode, don't connect via RPC - tests will set direct server + super().__init__( + service_name=ValiConfig.RPC_ENTITY_SERVICE_NAME, + port=port or ValiConfig.RPC_ENTITY_PORT, + max_retries=5, + retry_delay_s=1.0, + connect_immediately=connect_immediately, + connection_mode=connection_mode + ) + + # ==================== Entity Registration Methods ==================== + + def register_entity( + self, + entity_hotkey: str, + collateral_amount: float = 0.0, + max_subaccounts: int = None + ) -> Tuple[bool, str]: + """ + Register a new entity. + + Args: + entity_hotkey: The VANTA_ENTITY_HOTKEY + collateral_amount: Collateral amount (placeholder) + max_subaccounts: Maximum allowed subaccounts + + Returns: + (success: bool, message: str) + """ + return self._server.register_entity_rpc(entity_hotkey, collateral_amount, max_subaccounts) + + def create_subaccount(self, entity_hotkey: str) -> Tuple[bool, Optional[dict], str]: + """ + Create a new subaccount for an entity. + + Args: + entity_hotkey: The VANTA_ENTITY_HOTKEY + + Returns: + (success: bool, subaccount_info_dict: Optional[dict], message: str) + """ + return self._server.create_subaccount_rpc(entity_hotkey) + + def eliminate_subaccount( + self, + entity_hotkey: str, + subaccount_id: int, + reason: str = "unknown" + ) -> Tuple[bool, str]: + """ + Eliminate a subaccount. + + Args: + entity_hotkey: The VANTA_ENTITY_HOTKEY + subaccount_id: The subaccount ID to eliminate + reason: Elimination reason + + Returns: + (success: bool, message: str) + """ + return self._server.eliminate_subaccount_rpc(entity_hotkey, subaccount_id, reason) + + def update_collateral(self, entity_hotkey: str, collateral_amount: float) -> Tuple[bool, str]: + """ + Update collateral for an entity. + + Args: + entity_hotkey: The VANTA_ENTITY_HOTKEY + collateral_amount: New collateral amount + + Returns: + (success: bool, message: str) + """ + return self._server.update_collateral_rpc(entity_hotkey, collateral_amount) + + # ==================== Query Methods ==================== + + def get_subaccount_status(self, synthetic_hotkey: str) -> Tuple[bool, Optional[str], str]: + """ + Get the status of a subaccount by synthetic hotkey. + + Args: + synthetic_hotkey: The synthetic hotkey ({entity_hotkey}_{subaccount_id}) + + Returns: + (found: bool, status: Optional[str], synthetic_hotkey: str) + """ + return self._server.get_subaccount_status_rpc(synthetic_hotkey) + + def get_entity_data(self, entity_hotkey: str) -> Optional[dict]: + """ + Get full entity data. + + Args: + entity_hotkey: The VANTA_ENTITY_HOTKEY + + Returns: + Entity data as dict or None + """ + return self._server.get_entity_data_rpc(entity_hotkey) + + def get_all_entities(self) -> Dict[str, dict]: + """ + Get all entities. + + Returns: + Dict mapping entity_hotkey -> entity_data_dict + """ + return self._server.get_all_entities_rpc() + + def is_synthetic_hotkey(self, hotkey: str) -> bool: + """ + Check if a hotkey is synthetic (contains underscore with integer suffix). + + Args: + hotkey: The hotkey to check + + Returns: + True if synthetic, False otherwise + """ + return self._server.is_synthetic_hotkey_rpc(hotkey) + + def parse_synthetic_hotkey(self, synthetic_hotkey: str) -> Tuple[Optional[str], Optional[int]]: + """ + Parse a synthetic hotkey into entity_hotkey and subaccount_id. + + Args: + synthetic_hotkey: The synthetic hotkey ({entity_hotkey}_{subaccount_id}) + + Returns: + (entity_hotkey, subaccount_id) or (None, None) if invalid + """ + return self._server.parse_synthetic_hotkey_rpc(synthetic_hotkey) + + # ==================== Validator Broadcast Methods ==================== + + def broadcast_subaccount_registration( + self, + entity_hotkey: str, + subaccount_id: int, + subaccount_uuid: str, + synthetic_hotkey: str + ) -> None: + """ + Broadcast subaccount registration to other validators. + + Args: + entity_hotkey: The VANTA_ENTITY_HOTKEY + subaccount_id: The subaccount ID + subaccount_uuid: The subaccount UUID + synthetic_hotkey: The synthetic hotkey + """ + return self._server.broadcast_subaccount_registration_rpc( + entity_hotkey, subaccount_id, subaccount_uuid, synthetic_hotkey + ) + + def receive_subaccount_registration_update(self, subaccount_data: dict) -> bool: + """ + Process incoming subaccount registration from another validator. + + Args: + subaccount_data: Dict containing entity_hotkey, subaccount_id, subaccount_uuid, synthetic_hotkey + + Returns: + bool: True if successful, False otherwise + """ + # This delegates to EntityManager.receive_subaccount_registration via RPC + # Returns True if successful, False otherwise + success = self._server.receive_subaccount_registration_rpc( + template.protocol.SubaccountRegistration(subaccount_data=subaccount_data) + ).successfully_processed + return success + + def receive_subaccount_registration( + self, + synapse: 'template.protocol.SubaccountRegistration' + ) -> 'template.protocol.SubaccountRegistration': + """ + Receive subaccount registration synapse (for axon attachment). + + This delegates to the server's RPC handler. Used by validator_base.py for axon attachment. + + Args: + synapse: SubaccountRegistration synapse from another validator + + Returns: + Updated synapse with success/error status + """ + return self._server.receive_subaccount_registration_rpc(synapse) + + # ==================== Health Check Methods ==================== + + def health_check(self) -> dict: + """ + Get health status from server. + + Returns: + dict: Health status with 'status', 'service', 'timestamp_ms' and service-specific info + """ + return self._server.health_check_rpc() + + # ==================== Testing/Admin Methods ==================== + + def clear_all_entities(self) -> None: + """Clear all entity data (for testing only).""" + self._server.clear_all_entities_rpc() + + def to_checkpoint_dict(self) -> dict: + """Get entity data as a checkpoint dict for serialization.""" + return self._server.to_checkpoint_dict_rpc() + + # ==================== Daemon Control Methods ==================== + + def start_daemon(self) -> bool: + """ + Start the daemon thread remotely via RPC. + + Returns: + bool: True if daemon was started, False if already running + """ + return self._server.start_daemon_rpc() + + def stop_daemon(self) -> bool: + """ + Stop the daemon thread remotely via RPC. + + Returns: + bool: True if daemon was stopped, False if not running + """ + return self._server.stop_daemon_rpc() + + def is_daemon_running(self) -> bool: + """ + Check if daemon is running via RPC. + + Returns: + bool: True if daemon is running, False otherwise + """ + return self._server.is_daemon_running_rpc() diff --git a/entitiy_management/entity_manager.py b/entitiy_management/entity_manager.py new file mode 100644 index 000000000..168c863cc --- /dev/null +++ b/entitiy_management/entity_manager.py @@ -0,0 +1,771 @@ +# developer: jbonilla +# Copyright � 2024 Taoshi Inc +""" +EntityManager - Core business logic for entity miner management. + +This manager handles all business logic for entity operations including: +- Entity registration and tracking +- Subaccount creation with monotonic IDs +- Subaccount status management (active/eliminated) +- Collateral verification (placeholder) +- Slot allowance checking +- Thread-safe operations with proper locking + +Pattern follows ChallengePeriodManager: +- Manager holds all business logic +- Server wraps this and exposes via RPC +- Local dicts (NOT IPC) for performance +- Disk persistence via JSON +""" +import uuid +import time +import threading +import asyncio +import bittensor as bt +from typing import Dict, Optional, Tuple, List +from collections import defaultdict +from pydantic import BaseModel, Field + +import template.protocol +from vali_objects.utils.vali_bkp_utils import ValiBkpUtils +from vali_objects.utils.vali_utils import ValiUtils +from vali_objects.vali_config import ValiConfig, RPCConnectionMode +from shared_objects.cache_controller import CacheController +from time_util.time_util import TimeUtil + + +class SubaccountInfo(BaseModel): + """Data structure for a single subaccount.""" + subaccount_id: int = Field(description="Monotonically increasing ID") + subaccount_uuid: str = Field(description="Unique UUID for this subaccount") + synthetic_hotkey: str = Field(description="Synthetic hotkey: {entity_hotkey}_{subaccount_id}") + status: str = Field(default="active", description="Status: active, eliminated, or unknown") + created_at_ms: int = Field(description="Timestamp when subaccount was created") + eliminated_at_ms: Optional[int] = Field(default=None, description="Timestamp when subaccount was eliminated") + + # Challenge period fields + challenge_period_active: bool = Field(default=True, description="Whether subaccount is in challenge period") + challenge_period_passed: bool = Field(default=False, description="Whether subaccount has passed challenge period") + challenge_period_start_ms: int = Field(description="Challenge period start timestamp") + challenge_period_end_ms: int = Field(description="Challenge period end timestamp (90 days from start)") + + +class EntityData(BaseModel): + """Data structure for an entity.""" + entity_hotkey: str = Field(description="The VANTA_ENTITY_HOTKEY") + subaccounts: Dict[int, SubaccountInfo] = Field(default_factory=dict, description="Map subaccount_id -> SubaccountInfo") + next_subaccount_id: int = Field(default=0, description="Next subaccount ID to assign (monotonic)") + collateral_amount: float = Field(default=0.0, description="Collateral amount (placeholder)") + max_subaccounts: int = Field(default=10, description="Maximum allowed subaccounts") + registered_at_ms: int = Field(description="Timestamp when entity was registered") + + class Config: + arbitrary_types_allowed = True + + def get_active_subaccounts(self) -> List[SubaccountInfo]: + """Get all active subaccounts.""" + return [sa for sa in self.subaccounts.values() if sa.status == "active"] + + def get_eliminated_subaccounts(self) -> List[SubaccountInfo]: + """Get all eliminated subaccounts.""" + return [sa for sa in self.subaccounts.values() if sa.status == "eliminated"] + + def get_synthetic_hotkey(self, subaccount_id: int) -> Optional[str]: + """Get synthetic hotkey for a subaccount ID.""" + sa = self.subaccounts.get(subaccount_id) + return sa.synthetic_hotkey if sa else None + + +class EntityManager(CacheController): + """ + Entity Manager - Contains all business logic for entity miner management. + + This manager is wrapped by EntityServer which exposes methods via RPC. + All heavy logic resides here - server delegates to this manager. + + Pattern: + - Server holds a `self._manager` instance + - Server delegates all RPC methods to manager methods + - Manager creates its own clients internally (forward compatibility) + - Local dicts (NOT IPC) for fast access + - Thread-safe operations with locks + """ + + def __init__( + self, + *, + is_backtesting=False, + running_unit_tests: bool = False, + connection_mode: RPCConnectionMode = RPCConnectionMode.RPC, + config=None + ): + """ + Initialize EntityManager. + + Args: + is_backtesting: Whether running in backtesting mode + running_unit_tests: Whether running in test mode + connection_mode: RPCConnectionMode.LOCAL for tests, RPCConnectionMode.RPC for production + config: Validator config (for netuid, wallet) - optional, used for broadcasting + """ + super().__init__(running_unit_tests=running_unit_tests, is_backtesting=is_backtesting, connection_mode=connection_mode) + + self.running_unit_tests = running_unit_tests + self.connection_mode = connection_mode + + # Local dicts (NOT IPC managerized) - much faster! + self.entities: Dict[str, EntityData] = {} + + # Local lock (NOT shared across processes) - RPC methods are auto-serialized + self.entities_lock = threading.Lock() + + # Initialize wallet and metagraph for broadcasting (optional) + if not running_unit_tests and config is not None: + self.is_testnet = config.netuid == 116 + self._wallet = bt.wallet(config=config) + bt.logging.info("[ENTITY_MANAGER] Wallet initialized for broadcasting") + else: + self.is_testnet = False + self._wallet = None + + # Create own MetagraphClient for broadcasting + from shared_objects.rpc.metagraph_server import MetagraphClient + self._metagraph_client = MetagraphClient(connection_mode=connection_mode) + + # Create own PerfLedgerClient for challenge period assessment + from vali_objects.vali_dataclasses.perf_ledger_server import PerfLedgerClient + self._perf_ledger_client = PerfLedgerClient(connection_mode=connection_mode) + + self.ENTITY_FILE = ValiBkpUtils.get_entity_file_location(running_unit_tests=running_unit_tests) + + # Load initial entities from disk + if not self.is_backtesting: + disk_data = ValiUtils.get_vali_json_file_dict(self.ENTITY_FILE) + self.entities = self.parse_checkpoint_dict(disk_data) + + if not self.is_backtesting and len(self.entities) == 0: + self._write_entities_from_memory_to_disk() + + bt.logging.info("[ENTITY_MANAGER] EntityManager initialized with local dicts (no IPC)") + + @property + def wallet(self): + """Get wallet for broadcasting.""" + return self._wallet + + @property + def metagraph(self): + """Get metagraph client for broadcasting.""" + return self._metagraph_client + + @property + def perf_ledger(self): + """Get perf ledger client for challenge period assessment.""" + return self._perf_ledger_client + + # ==================== Core Business Logic ==================== + + def register_entity( + self, + entity_hotkey: str, + collateral_amount: float = 0.0, + max_subaccounts: int = None + ) -> Tuple[bool, str]: + """ + Register a new entity. + + Args: + entity_hotkey: The VANTA_ENTITY_HOTKEY + collateral_amount: Collateral amount (placeholder) + max_subaccounts: Maximum allowed subaccounts (default from ValiConfig) + + Returns: + (success: bool, message: str) + """ + if max_subaccounts is None: + max_subaccounts = ValiConfig.ENTITY_MAX_SUBACCOUNTS + + with self.entities_lock: + if entity_hotkey in self.entities: + return False, f"Entity {entity_hotkey} already registered" + + # TODO: Add collateral verification here + # collateral_verified = self._verify_collateral(entity_hotkey, collateral_amount) + # if not collateral_verified: + # return False, "Insufficient collateral" + + entity_data = EntityData( + entity_hotkey=entity_hotkey, + subaccounts={}, + next_subaccount_id=0, + collateral_amount=collateral_amount, + max_subaccounts=max_subaccounts, + registered_at_ms=TimeUtil.now_in_millis() + ) + + self.entities[entity_hotkey] = entity_data + self._write_entities_from_memory_to_disk() + + bt.logging.info(f"[ENTITY_MANAGER] Registered entity {entity_hotkey} with max_subaccounts={max_subaccounts}") + return True, f"Entity {entity_hotkey} registered successfully" + + def create_subaccount(self, entity_hotkey: str) -> Tuple[bool, Optional[SubaccountInfo], str]: + """ + Create a new subaccount for an entity. + + Args: + entity_hotkey: The VANTA_ENTITY_HOTKEY + + Returns: + (success: bool, subaccount_info: Optional[SubaccountInfo], message: str) + """ + with self.entities_lock: + entity_data = self.entities.get(entity_hotkey) + if not entity_data: + return False, None, f"Entity {entity_hotkey} not registered" + + # Check slot allowance + active_count = len(entity_data.get_active_subaccounts()) + if active_count >= entity_data.max_subaccounts: + return False, None, f"Entity {entity_hotkey} has reached maximum subaccounts ({entity_data.max_subaccounts})" + + # Generate monotonic ID + subaccount_id = entity_data.next_subaccount_id + entity_data.next_subaccount_id += 1 + + # Generate UUID and synthetic hotkey + subaccount_uuid = str(uuid.uuid4()) + synthetic_hotkey = f"{entity_hotkey}_{subaccount_id}" + + # Initialize challenge period timestamps + now_ms = TimeUtil.now_in_millis() + challenge_period_days = ValiConfig.SUBACCOUNT_CHALLENGE_PERIOD_DAYS + challenge_period_end_ms = now_ms + (challenge_period_days * 24 * 60 * 60 * 1000) + + # Create subaccount info with challenge period + subaccount_info = SubaccountInfo( + subaccount_id=subaccount_id, + subaccount_uuid=subaccount_uuid, + synthetic_hotkey=synthetic_hotkey, + status="active", + created_at_ms=now_ms, + challenge_period_active=True, + challenge_period_passed=False, + challenge_period_start_ms=now_ms, + challenge_period_end_ms=challenge_period_end_ms + ) + + # TODO: Transfer collateral from entity to subaccount + # This should use the collateral SDK to transfer collateral from entity_hotkey to synthetic_hotkey + # collateral_transfer_amount = calculate_subaccount_collateral(entity_data.collateral_amount, entity_data.max_subaccounts) + # collateral_sdk.transfer_collateral(from_hotkey=entity_hotkey, to_hotkey=synthetic_hotkey, amount=collateral_transfer_amount) + bt.logging.info(f"[ENTITY_MANAGER] TODO: Transfer collateral from {entity_hotkey} to {synthetic_hotkey}") + + # TODO: Set account size for the subaccount using ContractClient + # This should set a fixed account size for the synthetic hotkey + # from vali_objects.utils.vali_utils import ValiUtils + # contract_client = ValiUtils.get_contract_client() + # FIXED_ACCOUNT_SIZE = 1000.0 # Define this constant in ValiConfig + # contract_client.set_account_size(synthetic_hotkey, FIXED_ACCOUNT_SIZE) + bt.logging.info(f"[ENTITY_MANAGER] TODO: Set account size for {synthetic_hotkey} using ContractClient.set_account_size()") + + entity_data.subaccounts[subaccount_id] = subaccount_info + self._write_entities_from_memory_to_disk() + + bt.logging.info( + f"[ENTITY_MANAGER] Created subaccount {subaccount_id} for entity {entity_hotkey}: {synthetic_hotkey}" + ) + return True, subaccount_info, f"Subaccount {subaccount_id} created successfully" + + def eliminate_subaccount( + self, + entity_hotkey: str, + subaccount_id: int, + reason: str = "unknown" + ) -> Tuple[bool, str]: + """ + Eliminate a subaccount. + + Args: + entity_hotkey: The VANTA_ENTITY_HOTKEY + subaccount_id: The subaccount ID to eliminate + reason: Elimination reason + + Returns: + (success: bool, message: str) + """ + with self.entities_lock: + entity_data = self.entities.get(entity_hotkey) + if not entity_data: + return False, f"Entity {entity_hotkey} not found" + + subaccount = entity_data.subaccounts.get(subaccount_id) + if not subaccount: + return False, f"Subaccount {subaccount_id} not found for entity {entity_hotkey}" + + if subaccount.status == "eliminated": + return False, f"Subaccount {subaccount_id} already eliminated" + + subaccount.status = "eliminated" + subaccount.eliminated_at_ms = TimeUtil.now_in_millis() + self._write_entities_from_memory_to_disk() + + bt.logging.info( + f"[ENTITY_MANAGER] Eliminated subaccount {subaccount_id} for entity {entity_hotkey}. Reason: {reason}" + ) + return True, f"Subaccount {subaccount_id} eliminated successfully" + + def get_subaccount_status(self, synthetic_hotkey: str) -> Tuple[bool, Optional[str], str]: + """ + Get the status of a subaccount by synthetic hotkey. + + Args: + synthetic_hotkey: The synthetic hotkey ({entity_hotkey}_{subaccount_id}) + + Returns: + (found: bool, status: Optional[str], synthetic_hotkey: str) + """ + if not self.is_synthetic_hotkey(synthetic_hotkey): + return False, None, synthetic_hotkey + + entity_hotkey, subaccount_id = self.parse_synthetic_hotkey(synthetic_hotkey) + + with self.entities_lock: + entity_data = self.entities.get(entity_hotkey) + if not entity_data: + return False, "unknown", synthetic_hotkey + + subaccount = entity_data.subaccounts.get(subaccount_id) + if not subaccount: + return False, "unknown", synthetic_hotkey + + return True, subaccount.status, synthetic_hotkey + + def get_entity_data(self, entity_hotkey: str) -> Optional[EntityData]: + """ + Get full entity data. + + Args: + entity_hotkey: The VANTA_ENTITY_HOTKEY + + Returns: + EntityData or None + """ + with self.entities_lock: + return self.entities.get(entity_hotkey) + + def get_all_entities(self) -> Dict[str, EntityData]: + """Get all entities.""" + with self.entities_lock: + return dict(self.entities) + + def is_synthetic_hotkey(self, hotkey: str) -> bool: + """ + Check if a hotkey is synthetic (contains underscore). + + Args: + hotkey: The hotkey to check + + Returns: + True if synthetic, False otherwise + """ + # Edge case: What if an entity hotkey itself contains an underscore? + # We handle this by checking if the part after the last underscore is a valid integer + if "_" not in hotkey: + return False + + # Try to parse as synthetic hotkey + parts = hotkey.rsplit("_", 1) + if len(parts) != 2: + return False + + try: + int(parts[1]) # Check if last part is a valid integer + return True + except ValueError: + return False + + def parse_synthetic_hotkey(self, synthetic_hotkey: str) -> Tuple[Optional[str], Optional[int]]: + """ + Parse a synthetic hotkey into entity_hotkey and subaccount_id. + + Args: + synthetic_hotkey: The synthetic hotkey ({entity_hotkey}_{subaccount_id}) + + Returns: + (entity_hotkey, subaccount_id) or (None, None) if invalid + """ + if not self.is_synthetic_hotkey(synthetic_hotkey): + return None, None + + parts = synthetic_hotkey.rsplit("_", 1) + entity_hotkey = parts[0] + try: + subaccount_id = int(parts[1]) + return entity_hotkey, subaccount_id + except ValueError: + return None, None + + def update_collateral(self, entity_hotkey: str, collateral_amount: float) -> Tuple[bool, str]: + """ + Update collateral for an entity (placeholder). + + Args: + entity_hotkey: The VANTA_ENTITY_HOTKEY + collateral_amount: New collateral amount + + Returns: + (success: bool, message: str) + """ + with self.entities_lock: + entity_data = self.entities.get(entity_hotkey) + if not entity_data: + return False, f"Entity {entity_hotkey} not found" + + # TODO: Verify collateral with collateral SDK + entity_data.collateral_amount = collateral_amount + self._write_entities_from_memory_to_disk() + + bt.logging.info(f"[ENTITY_MANAGER] Updated collateral for {entity_hotkey}: {collateral_amount}") + return True, f"Collateral updated successfully" + + # ==================== Challenge Period & Elimination Assessment ==================== + + def assess_challenge_periods(self) -> int: + """ + Assess challenge periods for all active subaccounts. + + Runs periodically (every 5 minutes) to check: + - If subaccount has passed challenge period criteria (3% returns AND ≤6% drawdown) + - If challenge period has expired without passing + + Returns: + int: Number of subaccounts assessed + """ + assessed_count = 0 + now_ms = TimeUtil.now_in_millis() + + with self.entities_lock: + for entity_hotkey, entity_data in self.entities.items(): + for subaccount_id, subaccount in entity_data.subaccounts.items(): + # Skip if not active or not in challenge period + if subaccount.status != "active" or not subaccount.challenge_period_active: + continue + + assessed_count += 1 + synthetic_hotkey = subaccount.synthetic_hotkey + + # Check if challenge period has expired + if now_ms > subaccount.challenge_period_end_ms: + # Period expired without passing - eliminate subaccount + bt.logging.info( + f"[ENTITY_MANAGER] Challenge period expired for {synthetic_hotkey} " + f"({ValiConfig.SUBACCOUNT_CHALLENGE_PERIOD_DAYS} days). Eliminating." + ) + subaccount.status = "eliminated" + subaccount.eliminated_at_ms = now_ms + subaccount.challenge_period_active = False + continue + + # Challenge period still active - check if criteria met + try: + # Get performance metrics from PerfLedgerClient + returns = self.perf_ledger.get_returns_rpc(synthetic_hotkey) + drawdown = self.perf_ledger.get_drawdown_rpc(synthetic_hotkey) + + # Check if both criteria are met: + # 1. Returns >= 3% + # 2. Drawdown <= 6% (drawdown is represented as a positive value) + if returns is not None and drawdown is not None: + returns_threshold = ValiConfig.SUBACCOUNT_CHALLENGE_RETURNS_THRESHOLD + drawdown_threshold = ValiConfig.SUBACCOUNT_CHALLENGE_DRAWDOWN_THRESHOLD + + if returns >= returns_threshold and drawdown <= drawdown_threshold: + # Challenge period passed! + bt.logging.info( + f"[ENTITY_MANAGER] Challenge period passed for {synthetic_hotkey}: " + f"returns={returns:.4f}, drawdown={drawdown:.4f}" + ) + subaccount.challenge_period_active = False + subaccount.challenge_period_passed = True + else: + bt.logging.debug( + f"[ENTITY_MANAGER] {synthetic_hotkey} still in challenge period: " + f"returns={returns:.4f} (need {returns_threshold}), " + f"drawdown={drawdown:.4f} (max {drawdown_threshold})" + ) + else: + # No metrics yet - subaccount hasn't started trading + bt.logging.debug( + f"[ENTITY_MANAGER] {synthetic_hotkey} has no performance metrics yet" + ) + + except Exception as e: + bt.logging.warning( + f"[ENTITY_MANAGER] Error checking challenge period for {synthetic_hotkey}: {e}" + ) + + # Persist any changes to disk + if assessed_count > 0: + self._write_entities_from_memory_to_disk() + + bt.logging.info(f"[ENTITY_MANAGER] Challenge period assessment complete: {assessed_count} subaccounts assessed") + return assessed_count + + # ==================== Persistence ==================== + + def _write_entities_from_memory_to_disk(self): + """Write entity data from memory to disk.""" + if self.is_backtesting: + return + + entity_data = self.to_checkpoint_dict() + ValiBkpUtils.write_file(self.ENTITY_FILE, entity_data) + + def to_checkpoint_dict(self) -> dict: + """Get entity data as a checkpoint dict for serialization.""" + with self.entities_lock: + checkpoint = {} + for entity_hotkey, entity_data in self.entities.items(): + checkpoint[entity_hotkey] = entity_data.model_dump() + return checkpoint + + @staticmethod + def parse_checkpoint_dict(json_dict: dict) -> Dict[str, EntityData]: + """Parse checkpoint dict from disk.""" + entities = {} + for entity_hotkey, entity_dict in json_dict.items(): + # Convert subaccount dicts back to SubaccountInfo objects + subaccounts_dict = {} + for sub_id_str, sub_dict in entity_dict.get("subaccounts", {}).items(): + subaccounts_dict[int(sub_id_str)] = SubaccountInfo(**sub_dict) + + entity_dict["subaccounts"] = subaccounts_dict + entities[entity_hotkey] = EntityData(**entity_dict) + + return entities + + # ==================== Validator Broadcast Methods ==================== + + def broadcast_subaccount_registration( + self, + entity_hotkey: str, + subaccount_id: int, + subaccount_uuid: str, + synthetic_hotkey: str + ): + """ + Broadcast SubaccountRegistration synapse to other validators. + Runs in a separate thread to avoid blocking the main process. + + Args: + entity_hotkey: The VANTA_ENTITY_HOTKEY + subaccount_id: The subaccount ID + subaccount_uuid: The subaccount UUID + synthetic_hotkey: The synthetic hotkey + """ + def run_broadcast(): + try: + asyncio.run(self._async_broadcast_subaccount_registration( + entity_hotkey, subaccount_id, subaccount_uuid, synthetic_hotkey + )) + except Exception as e: + bt.logging.error( + f"[ENTITY_MANAGER] Failed to broadcast subaccount registration for {synthetic_hotkey}: {e}" + ) + + thread = threading.Thread(target=run_broadcast, daemon=True) + thread.start() + + async def _async_broadcast_subaccount_registration( + self, + entity_hotkey: str, + subaccount_id: int, + subaccount_uuid: str, + synthetic_hotkey: str + ): + """ + Asynchronously broadcast SubaccountRegistration synapse to other validators. + + Args: + entity_hotkey: The VANTA_ENTITY_HOTKEY + subaccount_id: The subaccount ID + subaccount_uuid: The subaccount UUID + synthetic_hotkey: The synthetic hotkey + """ + try: + if not self.wallet: + bt.logging.debug("[ENTITY_MANAGER] No wallet configured, skipping broadcast") + return + + if not self.metagraph: + bt.logging.debug("[ENTITY_MANAGER] No metagraph configured, skipping broadcast") + return + + # Get other validators to broadcast to + if self.is_testnet: + validator_axons = [ + n.axon_info for n in self.metagraph.get_neurons() + if n.axon_info.ip != ValiConfig.AXON_NO_IP + and n.axon_info.hotkey != self.wallet.hotkey.ss58_address + ] + else: + validator_axons = [ + n.axon_info for n in self.metagraph.get_neurons() + if n.stake > bt.Balance(ValiConfig.STAKE_MIN) + and n.axon_info.ip != ValiConfig.AXON_NO_IP + and n.axon_info.hotkey != self.wallet.hotkey.ss58_address + ] + + if not validator_axons: + bt.logging.debug("[ENTITY_MANAGER] No other validators to broadcast SubaccountRegistration to") + return + + # Create SubaccountRegistration synapse with the data + subaccount_data = { + "entity_hotkey": entity_hotkey, + "subaccount_id": subaccount_id, + "subaccount_uuid": subaccount_uuid, + "synthetic_hotkey": synthetic_hotkey + } + + subaccount_synapse = template.protocol.SubaccountRegistration( + subaccount_data=subaccount_data + ) + + bt.logging.info( + f"[ENTITY_MANAGER] Broadcasting SubaccountRegistration for {synthetic_hotkey} " + f"to {len(validator_axons)} validators" + ) + + # Send to other validators using dendrite + async with bt.dendrite(wallet=self.wallet) as dendrite: + responses = await dendrite.aquery(validator_axons, subaccount_synapse) + + # Log results + success_count = 0 + for response in responses: + if response.successfully_processed: + success_count += 1 + elif response.error_message: + bt.logging.warning( + f"[ENTITY_MANAGER] Failed to send SubaccountRegistration to " + f"{response.axon.hotkey}: {response.error_message}" + ) + + bt.logging.info( + f"[ENTITY_MANAGER] SubaccountRegistration broadcast completed: " + f"{success_count}/{len(responses)} validators updated" + ) + + except Exception as e: + bt.logging.error(f"[ENTITY_MANAGER] Error in async broadcast subaccount registration: {e}") + import traceback + bt.logging.error(traceback.format_exc()) + + def receive_subaccount_registration(self, subaccount_data: dict) -> bool: + """ + Process an incoming subaccount registration from another validator. + Ensures idempotent registration (handles duplicates gracefully). + + Args: + subaccount_data: Dictionary containing entity_hotkey, subaccount_id, subaccount_uuid, synthetic_hotkey + + Returns: + bool: True if successful, False otherwise + """ + try: + with self.entities_lock: + # Extract data from the synapse + entity_hotkey = subaccount_data.get("entity_hotkey") + subaccount_id = subaccount_data.get("subaccount_id") + subaccount_uuid = subaccount_data.get("subaccount_uuid") + synthetic_hotkey = subaccount_data.get("synthetic_hotkey") + + bt.logging.info( + f"[ENTITY_MANAGER] Processing subaccount registration for {synthetic_hotkey}" + ) + + if not all([entity_hotkey, subaccount_id is not None, subaccount_uuid, synthetic_hotkey]): + bt.logging.warning( + f"[ENTITY_MANAGER] Invalid subaccount registration data received: {subaccount_data}" + ) + return False + + # Get or create entity data + entity_data = self.entities.get(entity_hotkey) + if not entity_data: + # Auto-create entity if doesn't exist (from broadcast) + entity_data = EntityData( + entity_hotkey=entity_hotkey, + subaccounts={}, + next_subaccount_id=subaccount_id + 1, # Ensure monotonic ID continues + registered_at_ms=TimeUtil.now_in_millis() + ) + self.entities[entity_hotkey] = entity_data + bt.logging.info(f"[ENTITY_MANAGER] Auto-created entity {entity_hotkey} from broadcast") + + # Check if subaccount already exists (idempotent) + if subaccount_id in entity_data.subaccounts: + existing_sub = entity_data.subaccounts[subaccount_id] + if existing_sub.subaccount_uuid == subaccount_uuid: + bt.logging.debug( + f"[ENTITY_MANAGER] Subaccount {synthetic_hotkey} already exists (idempotent)" + ) + return True + else: + bt.logging.warning( + f"[ENTITY_MANAGER] Subaccount ID conflict for {entity_hotkey}:{subaccount_id}" + ) + return False + + # Create new subaccount info with challenge period + now_ms = TimeUtil.now_in_millis() + challenge_period_days = ValiConfig.SUBACCOUNT_CHALLENGE_PERIOD_DAYS + challenge_period_end_ms = now_ms + (challenge_period_days * 24 * 60 * 60 * 1000) + + subaccount_info = SubaccountInfo( + subaccount_id=subaccount_id, + subaccount_uuid=subaccount_uuid, + synthetic_hotkey=synthetic_hotkey, + status="active", + created_at_ms=now_ms, + challenge_period_active=True, + challenge_period_passed=False, + challenge_period_start_ms=now_ms, + challenge_period_end_ms=challenge_period_end_ms + ) + + # Add to entity + entity_data.subaccounts[subaccount_id] = subaccount_info + + # Update next_subaccount_id if needed + if subaccount_id >= entity_data.next_subaccount_id: + entity_data.next_subaccount_id = subaccount_id + 1 + + # Save to disk + self._write_entities_from_memory_to_disk() + + bt.logging.info( + f"[ENTITY_MANAGER] Registered subaccount {synthetic_hotkey} via broadcast" + ) + return True + + except Exception as e: + bt.logging.error(f"[ENTITY_MANAGER] Error processing subaccount registration: {e}") + import traceback + bt.logging.error(traceback.format_exc()) + return False + + # ==================== Testing/Admin Methods ==================== + + def clear_all_entities(self): + """Clear all entity data (for testing).""" + if not self.running_unit_tests: + raise Exception("Clearing entities is only allowed during unit tests.") + + with self.entities_lock: + self.entities.clear() + self._write_entities_from_memory_to_disk() + + bt.logging.info("[ENTITY_MANAGER] Cleared all entity data") diff --git a/entitiy_management/entity_server.py b/entitiy_management/entity_server.py new file mode 100644 index 000000000..4fa0dabcd --- /dev/null +++ b/entitiy_management/entity_server.py @@ -0,0 +1,324 @@ +# developer: jbonilla +# Copyright � 2024 Taoshi Inc +""" +EntityServer - RPC server for entity miner management. + +This server runs in its own process and exposes entity management via RPC. +Clients connect using EntityClient. + +Follows the same pattern as ChallengePeriodServer. +""" +import bittensor as bt +from typing import Optional, Tuple, Dict, List + +from entitiy_management.entity_manager import EntityManager, SubaccountInfo, EntityData +from vali_objects.vali_config import ValiConfig, RPCConnectionMode +from shared_objects.rpc.rpc_server_base import RPCServerBase + + +class EntityServer(RPCServerBase): + """ + RPC server for entity miner management. + + Wraps EntityManager and exposes its methods via RPC. + All public methods ending in _rpc are exposed via RPC to EntityClient. + + This follows the same pattern as ChallengePeriodServer and EliminationServer. + """ + service_name = ValiConfig.RPC_ENTITY_SERVICE_NAME + service_port = ValiConfig.RPC_ENTITY_PORT + + def __init__( + self, + *, + is_backtesting=False, + slack_notifier=None, + start_server=True, + start_daemon=False, + running_unit_tests: bool = False, + connection_mode: RPCConnectionMode = RPCConnectionMode.RPC + ): + """ + Initialize EntityServer IN-PROCESS (never spawns). + + Args: + is_backtesting: Whether running in backtesting mode + slack_notifier: Slack notifier for alerts + start_server: Whether to start RPC server immediately + start_daemon: Whether to start daemon immediately + running_unit_tests: Whether running in test mode + connection_mode: RPCConnectionMode.LOCAL for tests, RPCConnectionMode.RPC for production + """ + self.running_unit_tests = running_unit_tests + + # Always create in-process - constructor NEVER spawns + bt.logging.info("[ENTITY_SERVER] Creating EntityServer in-process") + + # Create the actual EntityManager FIRST, before RPCServerBase.__init__ + # This ensures _manager exists before RPC server starts accepting calls (if start_server=True) + # CRITICAL: Prevents race condition where RPC calls fail with AttributeError during initialization + self._manager = EntityManager( + is_backtesting=is_backtesting, + running_unit_tests=running_unit_tests, + connection_mode=connection_mode + ) + + bt.logging.info("[ENTITY_SERVER] EntityManager initialized") + + # Initialize RPCServerBase (may start RPC server immediately if start_server=True) + # At this point, self._manager exists, so RPC calls won't fail + # daemon_interval_s: 5 minutes (challenge period + elimination assessment) + # hang_timeout_s: Dynamically set to 2x interval to prevent false alarms during normal sleep + daemon_interval_s = ValiConfig.ENTITY_ELIMINATION_CHECK_INTERVAL # 300s (5 minutes) + hang_timeout_s = daemon_interval_s * 2.0 # 600s (10 minutes, 2x interval) + + RPCServerBase.__init__( + self, + service_name=ValiConfig.RPC_ENTITY_SERVICE_NAME, + port=ValiConfig.RPC_ENTITY_PORT, + slack_notifier=slack_notifier, + start_server=start_server, + start_daemon=False, # We'll start daemon after full initialization + daemon_interval_s=daemon_interval_s, + hang_timeout_s=hang_timeout_s, + connection_mode=connection_mode + ) + + # Start daemon if requested (deferred until all initialization complete) + if start_daemon: + self.start_daemon() + + # ==================== RPCServerBase Abstract Methods ==================== + + def run_daemon_iteration(self) -> None: + """ + Single iteration of daemon work. Called by RPCServerBase daemon loop. + + Runs every 5 minutes to assess challenge periods for all active subaccounts: + - Check if subaccounts have passed challenge period criteria (3% returns AND ≤6% drawdown) + - Eliminate subaccounts that have expired challenge period without passing + """ + # Run challenge period assessment + assessed_count = self._manager.assess_challenge_periods() + bt.logging.debug(f"[ENTITY_SERVER] Challenge period assessment completed: {assessed_count} subaccounts assessed") + + # ==================== RPC Methods (exposed to client) ==================== + + def get_health_check_details(self) -> dict: + """Add service-specific health check details.""" + all_entities = self._manager.get_all_entities() + total_subaccounts = sum(len(entity.subaccounts) for entity in all_entities.values()) + active_subaccounts = sum(len(entity.get_active_subaccounts()) for entity in all_entities.values()) + + return { + "total_entities": len(all_entities), + "total_subaccounts": total_subaccounts, + "active_subaccounts": active_subaccounts + } + + # ==================== Entity Registration RPC Methods ==================== + + def register_entity_rpc( + self, + entity_hotkey: str, + collateral_amount: float = 0.0, + max_subaccounts: int = None + ) -> Tuple[bool, str]: + """ + Register a new entity. + + Args: + entity_hotkey: The VANTA_ENTITY_HOTKEY + collateral_amount: Collateral amount (placeholder) + max_subaccounts: Maximum allowed subaccounts + + Returns: + (success: bool, message: str) + """ + return self._manager.register_entity(entity_hotkey, collateral_amount, max_subaccounts) + + def create_subaccount_rpc(self, entity_hotkey: str) -> Tuple[bool, Optional[dict], str]: + """ + Create a new subaccount for an entity. + + Args: + entity_hotkey: The VANTA_ENTITY_HOTKEY + + Returns: + (success: bool, subaccount_info_dict: Optional[dict], message: str) + """ + success, subaccount_info, message = self._manager.create_subaccount(entity_hotkey) + + # Convert SubaccountInfo to dict for RPC serialization + subaccount_dict = subaccount_info.model_dump() if subaccount_info else None + + return success, subaccount_dict, message + + def eliminate_subaccount_rpc( + self, + entity_hotkey: str, + subaccount_id: int, + reason: str = "unknown" + ) -> Tuple[bool, str]: + """ + Eliminate a subaccount. + + Args: + entity_hotkey: The VANTA_ENTITY_HOTKEY + subaccount_id: The subaccount ID to eliminate + reason: Elimination reason + + Returns: + (success: bool, message: str) + """ + return self._manager.eliminate_subaccount(entity_hotkey, subaccount_id, reason) + + def update_collateral_rpc(self, entity_hotkey: str, collateral_amount: float) -> Tuple[bool, str]: + """ + Update collateral for an entity. + + Args: + entity_hotkey: The VANTA_ENTITY_HOTKEY + collateral_amount: New collateral amount + + Returns: + (success: bool, message: str) + """ + return self._manager.update_collateral(entity_hotkey, collateral_amount) + + # ==================== Query RPC Methods ==================== + + def get_subaccount_status_rpc(self, synthetic_hotkey: str) -> Tuple[bool, Optional[str], str]: + """ + Get the status of a subaccount by synthetic hotkey. + + Args: + synthetic_hotkey: The synthetic hotkey ({entity_hotkey}_{subaccount_id}) + + Returns: + (found: bool, status: Optional[str], synthetic_hotkey: str) + """ + return self._manager.get_subaccount_status(synthetic_hotkey) + + def get_entity_data_rpc(self, entity_hotkey: str) -> Optional[dict]: + """ + Get full entity data. + + Args: + entity_hotkey: The VANTA_ENTITY_HOTKEY + + Returns: + Entity data as dict or None + """ + entity_data = self._manager.get_entity_data(entity_hotkey) + return entity_data.model_dump() if entity_data else None + + def get_all_entities_rpc(self) -> Dict[str, dict]: + """ + Get all entities. + + Returns: + Dict mapping entity_hotkey -> entity_data_dict + """ + all_entities = self._manager.get_all_entities() + return {hotkey: entity.model_dump() for hotkey, entity in all_entities.items()} + + def is_synthetic_hotkey_rpc(self, hotkey: str) -> bool: + """ + Check if a hotkey is synthetic (contains underscore with integer suffix). + + Args: + hotkey: The hotkey to check + + Returns: + True if synthetic, False otherwise + """ + return self._manager.is_synthetic_hotkey(hotkey) + + def parse_synthetic_hotkey_rpc(self, synthetic_hotkey: str) -> Tuple[Optional[str], Optional[int]]: + """ + Parse a synthetic hotkey into entity_hotkey and subaccount_id. + + Args: + synthetic_hotkey: The synthetic hotkey ({entity_hotkey}_{subaccount_id}) + + Returns: + (entity_hotkey, subaccount_id) or (None, None) if invalid + """ + return self._manager.parse_synthetic_hotkey(synthetic_hotkey) + + # ==================== Validator Broadcast RPC Methods ==================== + + def broadcast_subaccount_registration_rpc( + self, + entity_hotkey: str, + subaccount_id: int, + subaccount_uuid: str, + synthetic_hotkey: str + ) -> None: + """ + Broadcast subaccount registration to other validators. + + Args: + entity_hotkey: The VANTA_ENTITY_HOTKEY + subaccount_id: The subaccount ID + subaccount_uuid: The subaccount UUID + synthetic_hotkey: The synthetic hotkey + """ + self._manager.broadcast_subaccount_registration( + entity_hotkey, subaccount_id, subaccount_uuid, synthetic_hotkey + ) + + def receive_subaccount_registration_rpc( + self, + synapse: template.protocol.SubaccountRegistration + ) -> template.protocol.SubaccountRegistration: + """ + Receive subaccount registration synapse (RPC method for axon handler). + + This is called by the validator's axon when receiving a SubaccountRegistration synapse. + + Args: + synapse: SubaccountRegistration synapse from another validator + + Returns: + Updated synapse with success/error status + """ + try: + sender_hotkey = synapse.dendrite.hotkey + bt.logging.info( + f"[ENTITY_SERVER] Received SubaccountRegistration synapse from validator hotkey [{sender_hotkey}]" + ) + success = self._manager.receive_subaccount_registration(synapse.subaccount_data) + + if success: + synapse.successfully_processed = True + synapse.error_message = "" + bt.logging.info( + f"[ENTITY_SERVER] Successfully processed SubaccountRegistration synapse from {sender_hotkey}" + ) + else: + synapse.successfully_processed = False + synapse.error_message = "Failed to process subaccount registration" + bt.logging.warning( + f"[ENTITY_SERVER] Failed to process SubaccountRegistration synapse from {sender_hotkey}" + ) + + except Exception as e: + synapse.successfully_processed = False + synapse.error_message = f"Error processing subaccount registration: {e}" + bt.logging.error(f"[ENTITY_SERVER] Error processing SubaccountRegistration synapse: {e}") + import traceback + bt.logging.error(traceback.format_exc()) + + return synapse + + # ==================== Testing/Admin RPC Methods ==================== + + def clear_all_entities_rpc(self) -> None: + """Clear all entity data (for testing only).""" + self._manager.clear_all_entities() + + def to_checkpoint_dict_rpc(self) -> dict: + """Get entity data as a checkpoint dict for serialization.""" + return self._manager.to_checkpoint_dict() diff --git a/neurons/validator.py b/neurons/validator.py index 7a6117417..2b0988bb2 100644 --- a/neurons/validator.py +++ b/neurons/validator.py @@ -172,6 +172,7 @@ def __init__(self): self.asset_selection_client = orchestrator.get_client('asset_selection') self.perf_ledger_client = orchestrator.get_client('perf_ledger') self.debt_ledger_client = orchestrator.get_client('debt_ledger') + self.entity_client = orchestrator.get_client('entity') # Create MetagraphUpdater with simple parameters (no PTNManager) # This will run in a thread in the main process @@ -478,6 +479,32 @@ def should_fail_early(self, synapse: template.protocol.SendSignal | template.pro synapse.error_message = msg return True + # Entity hotkey validation: Don't allow orders from entity hotkeys (non-synthetic) + # Only synthetic hotkeys (subaccounts) can place orders + entity_check_start = time.perf_counter() + if self.entity_client.is_synthetic_hotkey(sender_hotkey): + # This is a synthetic hotkey - verify it's active + found, status, _ = self.entity_client.get_subaccount_status(sender_hotkey) + if not found or status != 'active': + msg = (f"Synthetic hotkey {sender_hotkey} is not active or not found. " + f"Please ensure your subaccount is properly registered.") + bt.logging.warning(msg) + synapse.successfully_processed = False + synapse.error_message = msg + return True + else: + # Not a synthetic hotkey - check if it's an entity hotkey + entity_data = self.entity_client.get_entity_data(sender_hotkey) + if entity_data: + msg = (f"Entity hotkey {sender_hotkey} cannot place orders directly. " + f"Please use a subaccount (synthetic hotkey) to place orders.") + bt.logging.warning(msg) + synapse.successfully_processed = False + synapse.error_message = msg + return True + entity_check_ms = (time.perf_counter() - entity_check_start) * 1000 + bt.logging.info(f"[FAIL_EARLY_DEBUG] entity_hotkey_validation took {entity_check_ms:.2f}ms") + order_uuid = synapse.miner_order_uuid tp = Order.parse_trade_pair_from_signal(signal) if order_uuid and self.uuid_tracker.exists(order_uuid): diff --git a/neurons/validator_base.py b/neurons/validator_base.py index b86562443..7372c4113 100644 --- a/neurons/validator_base.py +++ b/neurons/validator_base.py @@ -24,6 +24,10 @@ def __init__(self, wallet, config, metagraph, p2p_syncer, asset_selection_client from vali_objects.contract.contract_server import ContractClient self._contract_client = ContractClient(running_unit_tests=False) + # Create own EntityClient (forward compatibility - no parameter passing) + from entitiy_management.entity_client import EntityClient + self._entity_client = EntityClient(running_unit_tests=False) + self.wire_axon() # Each hotkey gets a unique identity (UID) in the network for differentiation. @@ -35,6 +39,11 @@ def contract_manager(self): """Get contract client (forward compatibility - created internally).""" return self._contract_client + @property + def entity_client(self): + """Get entity client (forward compatibility - created internally).""" + return self._entity_client + def receive_signal(self, synapse: template.protocol.SendSignal) -> template.protocol.SendSignal: """ Abstract method - must be implemented by child class. @@ -158,6 +167,9 @@ def cr_blacklist_fn(synapse: template.protocol.CollateralRecord) -> Tuple[bool, def as_blacklist_fn(synapse: template.protocol.AssetSelection) -> Tuple[bool, str]: return self.blacklist_fn(synapse, self.metagraph_server) + def sr_blacklist_fn(synapse: template.protocol.SubaccountRegistration) -> Tuple[bool, str]: + return self.blacklist_fn(synapse, self.metagraph_server) + self.axon.attach( forward_fn=self.receive_signal, blacklist_fn=rs_blacklist_fn @@ -178,6 +190,10 @@ def as_blacklist_fn(synapse: template.protocol.AssetSelection) -> Tuple[bool, st forward_fn=self.asset_selection_client.receive_asset_selection, blacklist_fn=as_blacklist_fn ) + self.axon.attach( + forward_fn=self.entity_client.receive_subaccount_registration, + blacklist_fn=sr_blacklist_fn + ) # Serve passes the axon information to the network + netuid we are hosting on. # This will auto-update if the axon port of external ip have changed. diff --git a/shared_objects/rpc/metagraph_server.py b/shared_objects/rpc/metagraph_server.py index 99972bfaa..d52305ffd 100644 --- a/shared_objects/rpc/metagraph_server.py +++ b/shared_objects/rpc/metagraph_server.py @@ -22,11 +22,12 @@ Thread-safe: All RPC methods are atomic (lock-free via atomic tuple assignment). """ import bittensor as bt -from typing import Set, List +from typing import Set, List, Optional, Tuple from shared_objects.rpc.rpc_server_base import RPCServerBase from shared_objects.rpc.rpc_client_base import RPCClientBase from vali_objects.vali_config import ValiConfig, RPCConnectionMode +from entitiy_management.entity_client import EntityClient class MetagraphServer(RPCServerBase): @@ -91,6 +92,14 @@ def __init__( # Cached hotkeys_set for O(1) has_hotkey() lookups self._hotkeys_set: Set[str] = set() + # Entity client for synthetic hotkey validation + # Using lazy connection (connect_immediately=False) to avoid circular dependency + self._entity_client = EntityClient( + connection_mode=connection_mode, + running_unit_tests=running_unit_tests, + connect_immediately=False + ) + # Initialize RPCServerBase (NO daemon for MetagraphServer - it's just a data store) super().__init__( service_name=ValiConfig.RPC_METAGRAPH_SERVICE_NAME, @@ -130,16 +139,57 @@ def has_hotkey_rpc(self, hotkey: str) -> bool: Fast O(1) hotkey existence check using cached set. Lock-free - set membership check is atomic in Python. + Supports synthetic hotkeys (format: {entity_hotkey}_{subaccount_id}): + - If hotkey is not in metagraph, checks if it's a synthetic hotkey + - Validates entity hotkey exists in metagraph + - Validates subaccount is active via EntityClient + Args: - hotkey: The hotkey to check + hotkey: The hotkey to check (can be regular or synthetic) Returns: - bool: True if hotkey exists or is DEVELOPMENT, False otherwise + bool: True if hotkey exists or is DEVELOPMENT or is valid synthetic hotkey, False otherwise """ + # Check for DEVELOPMENT_HOTKEY if hotkey == self.DEVELOPMENT_HOTKEY: return True - # Lock-free! Python's 'in' operator is atomic for reads - return hotkey in self._hotkeys_set + + # Check if regular hotkey exists in metagraph (lock-free, atomic) + if hotkey in self._hotkeys_set: + return True + + # Not in metagraph - check if it's a synthetic hotkey + # Synthetic hotkey format: {entity_hotkey}_{subaccount_id} + # Use EntityClient to validate (it has the is_synthetic_hotkey logic) + try: + if self._entity_client.is_synthetic_hotkey(hotkey): + # Parse synthetic hotkey + entity_hotkey, subaccount_id = self._entity_client.parse_synthetic_hotkey(hotkey) + + if entity_hotkey is None or subaccount_id is None: + # Invalid synthetic hotkey format + return False + + # Verify entity hotkey exists in metagraph + if entity_hotkey not in self._hotkeys_set: + bt.logging.debug(f"[METAGRAPH] Synthetic hotkey '{hotkey}' rejected: entity '{entity_hotkey}' not in metagraph") + return False + + # Verify subaccount is active via EntityClient + found, status, _ = self._entity_client.get_subaccount_status(hotkey) + if found and status == "active": + bt.logging.trace(f"[METAGRAPH] Synthetic hotkey '{hotkey}' validated: entity in metagraph, subaccount active") + return True + else: + bt.logging.debug(f"[METAGRAPH] Synthetic hotkey '{hotkey}' rejected: subaccount not found or not active (status={status})") + return False + except Exception as e: + # If EntityClient fails (server not running, etc.), treat as not found + bt.logging.warning(f"[METAGRAPH] Failed to validate synthetic hotkey '{hotkey}': {e}") + return False + + # Not in metagraph and not a valid synthetic hotkey + return False def get_hotkeys_rpc(self) -> list: """Get list of all hotkeys (lock-free read)""" @@ -399,6 +449,15 @@ def __init__( connection_mode: RPCConnectionMode.LOCAL for tests, RPCConnectionMode.RPC for production """ self.running_unit_tests = running_unit_tests + + # Entity client for synthetic hotkey support (if needed locally) + # Using lazy connection to avoid circular dependency + self._entity_client = EntityClient( + connection_mode=connection_mode, + running_unit_tests=running_unit_tests, + connect_immediately=False + ) + super().__init__( service_name=ValiConfig.RPC_METAGRAPH_SERVICE_NAME, port=ValiConfig.RPC_METAGRAPH_PORT, diff --git a/shared_objects/rpc/server_orchestrator.py b/shared_objects/rpc/server_orchestrator.py index 6a77b954d..7528cb300 100644 --- a/shared_objects/rpc/server_orchestrator.py +++ b/shared_objects/rpc/server_orchestrator.py @@ -279,6 +279,14 @@ class ServerOrchestrator: required_in_validator=False, # Must be started manually AFTER MetagraphUpdater (depends on WeightSetterServer) spawn_kwargs={'start_daemon': False} # Daemon started later ), + 'entity': ServerConfig( + server_class=None, + client_class=None, + required_in_testing=True, + required_in_miner=False, # Miners don't need entity management + required_in_validator=True, # Validators need entity management for subaccount tracking + spawn_kwargs={'start_daemon': False} # Daemon started later via orchestrator + ), } @classmethod @@ -358,6 +366,8 @@ def _load_classes(self): from vali_objects.utils.weight_calculator_server import WeightCalculatorServer # WeightCalculatorClient doesn't exist yet - server manages its own clients internally # from vali_objects.utils.weight_calculator_client import WeightCalculatorClient + from entitiy_management.entity_server import EntityServer + from entitiy_management.entity_client import EntityClient # Update registry with classes self.SERVERS['common_data'].server_class = CommonDataServer @@ -414,6 +424,9 @@ def _load_classes(self): self.SERVERS['weight_calculator'].server_class = WeightCalculatorServer self.SERVERS['weight_calculator'].client_class = None # No client - server manages its own clients + self.SERVERS['entity'].server_class = EntityServer + self.SERVERS['entity'].client_class = EntityClient + self._classes_loaded = True def _register_cleanup_handlers(self): @@ -576,6 +589,7 @@ def _get_start_order(self, server_names: list) -> list: order = [ 'common_data', 'metagraph', + 'entity', # Depends on metagraph (for synthetic hotkey validation) 'position_lock', 'perf_ledger', 'live_price_fetcher', @@ -879,6 +893,11 @@ def clear_contract(): contract_client.re_init_account_sizes() # Reload from disk safe_clear('contract', clear_contract) + # Clear entity data (entities and subaccounts) + entity_client = get_client_safe('entity') + if entity_client: + safe_clear('entity', lambda: entity_client.clear_all_entities()) + bt.logging.debug("All test data cleared") def is_running(self) -> bool: @@ -976,7 +995,8 @@ def tearDown(self): 'plagiarism_detector', 'mdd_checker', 'core_outputs', - 'miner_statistics' + 'miner_statistics', + 'entity' ] for server_name in daemon_servers: @@ -1070,6 +1090,7 @@ def start_validator_servers( # Start daemons for servers that deferred initialization if start_daemons: daemon_servers = [ + 'entity', 'position_manager', 'elimination', 'challenge_period', diff --git a/template/protocol.py b/template/protocol.py index cd6553ae2..8148ea87c 100644 --- a/template/protocol.py +++ b/template/protocol.py @@ -64,3 +64,10 @@ class AssetSelection(bt.Synapse): error_message: str = Field("", title="Error Message", frozen=False, max_length=4096) computed_body_hash: str = Field("", title="Computed Body Hash", frozen=False) AssetSelection.required_hash_fields = ["asset_selection"] + +class SubaccountRegistration(bt.Synapse): + subaccount_data: typing.Dict = Field(default_factory=dict, title="Subaccount Registration Data", frozen=False, max_length=4096) + successfully_processed: bool = Field(False, title="Successfully Processed", frozen=False) + error_message: str = Field("", title="Error Message", frozen=False, max_length=4096) + computed_body_hash: str = Field("", title="Computed Body Hash", frozen=False) +SubaccountRegistration.required_hash_fields = ["subaccount_data"] diff --git a/tests/vali_tests/test_entity_management.py b/tests/vali_tests/test_entity_management.py new file mode 100644 index 000000000..718a2e75a --- /dev/null +++ b/tests/vali_tests/test_entity_management.py @@ -0,0 +1,535 @@ +# developer: jbonilla +# Copyright © 2024 Taoshi Inc +""" +Entity Management unit tests using the new client/server architecture. + +This test file validates the core entity management functionality including: +- Entity registration +- Subaccount creation and tracking +- Synthetic hotkey validation +- Subaccount elimination +- Metagraph integration +""" +import unittest + +from shared_objects.rpc.server_orchestrator import ServerOrchestrator, ServerMode +from tests.vali_tests.base_objects.test_base import TestBase +from vali_objects.utils.vali_utils import ValiUtils +from time_util.time_util import TimeUtil + + +class TestEntityManagement(TestBase): + """ + Entity Management unit tests using ServerOrchestrator. + + Servers start once (via singleton orchestrator) and are shared across: + - All test methods in this class + - All test classes that use ServerOrchestrator + + This eliminates redundant server spawning and dramatically reduces test startup time. + Per-test isolation is achieved by clearing data state (not restarting servers). + """ + + # Class-level references (set in setUpClass via ServerOrchestrator) + orchestrator = None + entity_client = None + metagraph_client = None + + @classmethod + def setUpClass(cls): + """One-time setup: Start all servers using ServerOrchestrator (shared across all test classes).""" + # Get the singleton orchestrator and start all required servers + cls.orchestrator = ServerOrchestrator.get_instance() + + # Start all servers in TESTING mode (idempotent - safe if already started by another test class) + secrets = ValiUtils.get_secrets(running_unit_tests=True) + cls.orchestrator.start_all_servers( + mode=ServerMode.TESTING, + secrets=secrets + ) + + # Get clients from orchestrator (servers guaranteed ready, no connection delays) + cls.entity_client = cls.orchestrator.get_client('entity') + cls.metagraph_client = cls.orchestrator.get_client('metagraph') + + @classmethod + def tearDownClass(cls): + """ + One-time teardown: No action needed. + + Note: Servers and clients are managed by ServerOrchestrator singleton and shared + across all test classes. They will be shut down automatically at process exit. + """ + pass + + def setUp(self): + """Per-test setup: Reset data state (fast - no server restarts).""" + # Clear all data for test isolation (both memory and disk) + self.orchestrator.clear_all_test_data() + + # Set up test entities + self.ENTITY_HOTKEY_1 = "entity_hotkey_1" + self.ENTITY_HOTKEY_2 = "entity_hotkey_2" + self.ENTITY_HOTKEY_3 = "entity_hotkey_3" + + # Initialize metagraph with test entities + self.metagraph_client.set_hotkeys([ + self.ENTITY_HOTKEY_1, + self.ENTITY_HOTKEY_2, + self.ENTITY_HOTKEY_3 + ]) + + def tearDown(self): + """Per-test teardown: Clear data for next test.""" + self.orchestrator.clear_all_test_data() + + # ==================== Entity Registration Tests ==================== + + def test_register_entity_success(self): + """Test successful entity registration.""" + success, message = self.entity_client.register_entity( + entity_hotkey=self.ENTITY_HOTKEY_1, + collateral_amount=1000.0, + max_subaccounts=5 + ) + + self.assertTrue(success, f"Entity registration failed: {message}") + + # Verify entity exists + entity_data = self.entity_client.get_entity_data(self.ENTITY_HOTKEY_1) + self.assertIsNotNone(entity_data) + self.assertEqual(entity_data['entity_hotkey'], self.ENTITY_HOTKEY_1) + self.assertEqual(entity_data['collateral_amount'], 1000.0) + self.assertEqual(entity_data['max_subaccounts'], 5) + self.assertEqual(len(entity_data['subaccounts']), 0) + + def test_register_entity_duplicate(self): + """Test that registering the same entity twice fails.""" + # Register first time + success, _ = self.entity_client.register_entity( + entity_hotkey=self.ENTITY_HOTKEY_1, + collateral_amount=1000.0 + ) + self.assertTrue(success) + + # Try to register again + success, message = self.entity_client.register_entity( + entity_hotkey=self.ENTITY_HOTKEY_1, + collateral_amount=2000.0 + ) + self.assertFalse(success) + self.assertIn("already registered", message.lower()) + + def test_register_entity_default_values(self): + """Test entity registration with default values.""" + success, _ = self.entity_client.register_entity( + entity_hotkey=self.ENTITY_HOTKEY_1 + ) + self.assertTrue(success) + + entity_data = self.entity_client.get_entity_data(self.ENTITY_HOTKEY_1) + self.assertEqual(entity_data['collateral_amount'], 0.0) + self.assertEqual(entity_data['max_subaccounts'], 10) # ValiConfig default + + # ==================== Subaccount Creation Tests ==================== + + def test_create_subaccount_success(self): + """Test successful subaccount creation.""" + # Register entity first + self.entity_client.register_entity(entity_hotkey=self.ENTITY_HOTKEY_1) + + # Create subaccount + success, subaccount_info, message = self.entity_client.create_subaccount( + entity_hotkey=self.ENTITY_HOTKEY_1 + ) + + self.assertTrue(success, f"Subaccount creation failed: {message}") + self.assertIsNotNone(subaccount_info) + self.assertEqual(subaccount_info['subaccount_id'], 0) + self.assertEqual(subaccount_info['status'], 'active') + + # Verify synthetic hotkey format + synthetic_hotkey = subaccount_info['synthetic_hotkey'] + self.assertEqual(synthetic_hotkey, f"{self.ENTITY_HOTKEY_1}_0") + + def test_create_multiple_subaccounts(self): + """Test creating multiple subaccounts for an entity.""" + # Register entity + self.entity_client.register_entity(entity_hotkey=self.ENTITY_HOTKEY_1) + + # Create 3 subaccounts + subaccount_ids = [] + for i in range(3): + success, subaccount_info, _ = self.entity_client.create_subaccount( + entity_hotkey=self.ENTITY_HOTKEY_1 + ) + self.assertTrue(success) + subaccount_ids.append(subaccount_info['subaccount_id']) + + # Verify sequential IDs (0, 1, 2) + self.assertEqual(subaccount_ids, [0, 1, 2]) + + # Verify entity data + entity_data = self.entity_client.get_entity_data(self.ENTITY_HOTKEY_1) + self.assertEqual(len(entity_data['subaccounts']), 3) + + def test_create_subaccount_max_limit(self): + """Test that subaccount creation fails when max limit is reached.""" + # Register entity with max_subaccounts=2 + self.entity_client.register_entity( + entity_hotkey=self.ENTITY_HOTKEY_1, + max_subaccounts=2 + ) + + # Create 2 subaccounts (should succeed) + for i in range(2): + success, _, _ = self.entity_client.create_subaccount(self.ENTITY_HOTKEY_1) + self.assertTrue(success) + + # Try to create 3rd subaccount (should fail) + success, subaccount_info, message = self.entity_client.create_subaccount( + self.ENTITY_HOTKEY_1 + ) + self.assertFalse(success) + self.assertIsNone(subaccount_info) + self.assertIn("maximum", message.lower()) + + def test_create_subaccount_unregistered_entity(self): + """Test that subaccount creation fails for unregistered entity.""" + success, subaccount_info, message = self.entity_client.create_subaccount( + entity_hotkey="unregistered_entity" + ) + + self.assertFalse(success) + self.assertIsNone(subaccount_info) + self.assertIn("not registered", message.lower()) + + # ==================== Synthetic Hotkey Tests ==================== + + def test_is_synthetic_hotkey_valid(self): + """Test synthetic hotkey detection.""" + # Valid synthetic hotkeys + self.assertTrue(self.entity_client.is_synthetic_hotkey("entity_123")) + self.assertTrue(self.entity_client.is_synthetic_hotkey("my_entity_0")) + self.assertTrue(self.entity_client.is_synthetic_hotkey("foo_bar_99")) + + # Invalid synthetic hotkeys (no underscore + integer) + self.assertFalse(self.entity_client.is_synthetic_hotkey("regular_hotkey")) + self.assertFalse(self.entity_client.is_synthetic_hotkey("no_number_")) + self.assertFalse(self.entity_client.is_synthetic_hotkey("just_text")) + + def test_parse_synthetic_hotkey_valid(self): + """Test parsing valid synthetic hotkeys.""" + entity_hotkey, subaccount_id = self.entity_client.parse_synthetic_hotkey( + "my_entity_5" + ) + self.assertEqual(entity_hotkey, "my_entity") + self.assertEqual(subaccount_id, 5) + + # Test with entity hotkey containing underscores + entity_hotkey, subaccount_id = self.entity_client.parse_synthetic_hotkey( + "entity_with_underscores_123" + ) + self.assertEqual(entity_hotkey, "entity_with_underscores") + self.assertEqual(subaccount_id, 123) + + def test_parse_synthetic_hotkey_invalid(self): + """Test parsing invalid synthetic hotkeys.""" + entity_hotkey, subaccount_id = self.entity_client.parse_synthetic_hotkey( + "invalid_hotkey" + ) + self.assertIsNone(entity_hotkey) + self.assertIsNone(subaccount_id) + + # ==================== Subaccount Status Tests ==================== + + def test_get_subaccount_status_active(self): + """Test getting status of an active subaccount.""" + # Register entity and create subaccount + self.entity_client.register_entity(entity_hotkey=self.ENTITY_HOTKEY_1) + _, subaccount_info, _ = self.entity_client.create_subaccount(self.ENTITY_HOTKEY_1) + synthetic_hotkey = subaccount_info['synthetic_hotkey'] + + # Get status + found, status, returned_hotkey = self.entity_client.get_subaccount_status( + synthetic_hotkey + ) + + self.assertTrue(found) + self.assertEqual(status, 'active') + self.assertEqual(returned_hotkey, synthetic_hotkey) + + def test_get_subaccount_status_eliminated(self): + """Test getting status of an eliminated subaccount.""" + # Register entity and create subaccount + self.entity_client.register_entity(entity_hotkey=self.ENTITY_HOTKEY_1) + _, subaccount_info, _ = self.entity_client.create_subaccount(self.ENTITY_HOTKEY_1) + synthetic_hotkey = subaccount_info['synthetic_hotkey'] + + # Eliminate subaccount + self.entity_client.eliminate_subaccount( + entity_hotkey=self.ENTITY_HOTKEY_1, + subaccount_id=0, + reason="test_elimination" + ) + + # Get status + found, status, returned_hotkey = self.entity_client.get_subaccount_status( + synthetic_hotkey + ) + + self.assertTrue(found) + self.assertEqual(status, 'eliminated') + self.assertEqual(returned_hotkey, synthetic_hotkey) + + def test_get_subaccount_status_not_found(self): + """Test getting status of non-existent subaccount.""" + found, status, returned_hotkey = self.entity_client.get_subaccount_status( + "nonexistent_entity_0" + ) + + self.assertFalse(found) + self.assertIsNone(status) + + # ==================== Subaccount Elimination Tests ==================== + + def test_eliminate_subaccount_success(self): + """Test successful subaccount elimination.""" + # Register entity and create subaccount + self.entity_client.register_entity(entity_hotkey=self.ENTITY_HOTKEY_1) + self.entity_client.create_subaccount(self.ENTITY_HOTKEY_1) + + # Eliminate subaccount + success, message = self.entity_client.eliminate_subaccount( + entity_hotkey=self.ENTITY_HOTKEY_1, + subaccount_id=0, + reason="test_elimination" + ) + + self.assertTrue(success, f"Subaccount elimination failed: {message}") + + # Verify status changed to eliminated + found, status, _ = self.entity_client.get_subaccount_status( + f"{self.ENTITY_HOTKEY_1}_0" + ) + self.assertTrue(found) + self.assertEqual(status, 'eliminated') + + def test_eliminate_subaccount_nonexistent(self): + """Test eliminating a non-existent subaccount.""" + # Register entity without creating subaccounts + self.entity_client.register_entity(entity_hotkey=self.ENTITY_HOTKEY_1) + + # Try to eliminate non-existent subaccount + success, message = self.entity_client.eliminate_subaccount( + entity_hotkey=self.ENTITY_HOTKEY_1, + subaccount_id=999, + reason="test" + ) + + self.assertFalse(success) + self.assertIn("not found", message.lower()) + + def test_eliminate_already_eliminated_subaccount(self): + """Test eliminating an already eliminated subaccount.""" + # Register entity and create subaccount + self.entity_client.register_entity(entity_hotkey=self.ENTITY_HOTKEY_1) + self.entity_client.create_subaccount(self.ENTITY_HOTKEY_1) + + # Eliminate subaccount first time + success, _ = self.entity_client.eliminate_subaccount( + entity_hotkey=self.ENTITY_HOTKEY_1, + subaccount_id=0, + reason="first_elimination" + ) + self.assertTrue(success) + + # Try to eliminate again + success, message = self.entity_client.eliminate_subaccount( + entity_hotkey=self.ENTITY_HOTKEY_1, + subaccount_id=0, + reason="second_elimination" + ) + + # Should still succeed (idempotent) + self.assertTrue(success) + + # ==================== Metagraph Integration Tests ==================== + + def test_metagraph_has_hotkey_entity(self): + """Test that regular entity hotkeys are recognized by metagraph.""" + # Entity hotkey should be in metagraph (set in setUp) + self.assertTrue(self.metagraph_client.has_hotkey(self.ENTITY_HOTKEY_1)) + + def test_metagraph_has_hotkey_synthetic_active(self): + """Test that active synthetic hotkeys are recognized by metagraph.""" + # Register entity and create subaccount + self.entity_client.register_entity(entity_hotkey=self.ENTITY_HOTKEY_1) + _, subaccount_info, _ = self.entity_client.create_subaccount(self.ENTITY_HOTKEY_1) + synthetic_hotkey = subaccount_info['synthetic_hotkey'] + + # Synthetic hotkey should be recognized (entity in metagraph + subaccount active) + self.assertTrue(self.metagraph_client.has_hotkey(synthetic_hotkey)) + + def test_metagraph_has_hotkey_synthetic_eliminated(self): + """Test that eliminated synthetic hotkeys are NOT recognized by metagraph.""" + # Register entity and create subaccount + self.entity_client.register_entity(entity_hotkey=self.ENTITY_HOTKEY_1) + _, subaccount_info, _ = self.entity_client.create_subaccount(self.ENTITY_HOTKEY_1) + synthetic_hotkey = subaccount_info['synthetic_hotkey'] + + # Eliminate subaccount + self.entity_client.eliminate_subaccount( + entity_hotkey=self.ENTITY_HOTKEY_1, + subaccount_id=0, + reason="test" + ) + + # Synthetic hotkey should NOT be recognized (eliminated) + self.assertFalse(self.metagraph_client.has_hotkey(synthetic_hotkey)) + + def test_metagraph_has_hotkey_synthetic_entity_not_in_metagraph(self): + """Test that synthetic hotkeys fail if entity not in metagraph.""" + # Register entity that's NOT in metagraph + unregistered_entity = "entity_not_in_metagraph" + self.entity_client.register_entity(entity_hotkey=unregistered_entity) + _, subaccount_info, _ = self.entity_client.create_subaccount(unregistered_entity) + synthetic_hotkey = subaccount_info['synthetic_hotkey'] + + # Synthetic hotkey should NOT be recognized (entity not in metagraph) + self.assertFalse(self.metagraph_client.has_hotkey(synthetic_hotkey)) + + # ==================== Query Tests ==================== + + def test_get_all_entities(self): + """Test getting all entities.""" + # Register multiple entities + self.entity_client.register_entity(entity_hotkey=self.ENTITY_HOTKEY_1) + self.entity_client.register_entity(entity_hotkey=self.ENTITY_HOTKEY_2) + self.entity_client.register_entity(entity_hotkey=self.ENTITY_HOTKEY_3) + + # Get all entities + all_entities = self.entity_client.get_all_entities() + + self.assertEqual(len(all_entities), 3) + self.assertIn(self.ENTITY_HOTKEY_1, all_entities) + self.assertIn(self.ENTITY_HOTKEY_2, all_entities) + self.assertIn(self.ENTITY_HOTKEY_3, all_entities) + + def test_get_entity_data_nonexistent(self): + """Test getting data for non-existent entity.""" + entity_data = self.entity_client.get_entity_data("nonexistent_entity") + self.assertIsNone(entity_data) + + def test_update_collateral(self): + """Test updating collateral for an entity.""" + # Register entity with initial collateral + self.entity_client.register_entity( + entity_hotkey=self.ENTITY_HOTKEY_1, + collateral_amount=1000.0 + ) + + # Update collateral + success, message = self.entity_client.update_collateral( + entity_hotkey=self.ENTITY_HOTKEY_1, + collateral_amount=2000.0 + ) + + self.assertTrue(success, f"Collateral update failed: {message}") + + # Verify updated collateral + entity_data = self.entity_client.get_entity_data(self.ENTITY_HOTKEY_1) + self.assertEqual(entity_data['collateral_amount'], 2000.0) + + # ==================== Validator Order Placement Logic Tests ==================== + # These tests verify the behavior expected by validator.py's should_fail_early() + # method for entity hotkey validation (lines 482-506 in neurons/validator.py). + + def test_validator_entity_hotkey_detection(self): + """ + Test that entity hotkeys can be detected for order rejection. + + Validator logic: + - Entity hotkeys (non-synthetic) should be rejected + - Only synthetic hotkeys can place orders + """ + # Register an entity + self.entity_client.register_entity(entity_hotkey=self.ENTITY_HOTKEY_1) + + # Verify entity hotkey is NOT synthetic (should be rejected for orders) + is_synthetic = self.entity_client.is_synthetic_hotkey(self.ENTITY_HOTKEY_1) + self.assertFalse(is_synthetic, "Entity hotkey should not be synthetic") + + # Verify entity data exists (allows validator to detect and reject) + entity_data = self.entity_client.get_entity_data(self.ENTITY_HOTKEY_1) + self.assertIsNotNone(entity_data, "Entity data should exist for rejection check") + + def test_validator_synthetic_hotkey_active_acceptance(self): + """ + Test that active synthetic hotkeys are accepted for orders. + + Validator logic: + - Synthetic hotkeys with status='active' should be accepted + """ + # Register entity and create active subaccount + self.entity_client.register_entity(entity_hotkey=self.ENTITY_HOTKEY_1) + _, subaccount_info, _ = self.entity_client.create_subaccount(self.ENTITY_HOTKEY_1) + synthetic_hotkey = subaccount_info['synthetic_hotkey'] + + # Verify hotkey is synthetic + is_synthetic = self.entity_client.is_synthetic_hotkey(synthetic_hotkey) + self.assertTrue(is_synthetic, "Subaccount hotkey should be synthetic") + + # Verify status is active (should be accepted for orders) + found, status, _ = self.entity_client.get_subaccount_status(synthetic_hotkey) + self.assertTrue(found) + self.assertEqual(status, 'active', "Active subaccount should be accepted for orders") + + def test_validator_synthetic_hotkey_eliminated_rejection(self): + """ + Test that eliminated synthetic hotkeys are rejected for orders. + + Validator logic: + - Synthetic hotkeys with status='eliminated' should be rejected + """ + # Register entity and create subaccount + self.entity_client.register_entity(entity_hotkey=self.ENTITY_HOTKEY_1) + _, subaccount_info, _ = self.entity_client.create_subaccount(self.ENTITY_HOTKEY_1) + synthetic_hotkey = subaccount_info['synthetic_hotkey'] + + # Eliminate the subaccount + self.entity_client.eliminate_subaccount( + entity_hotkey=self.ENTITY_HOTKEY_1, + subaccount_id=0, + reason="test_elimination" + ) + + # Verify hotkey is synthetic + is_synthetic = self.entity_client.is_synthetic_hotkey(synthetic_hotkey) + self.assertTrue(is_synthetic, "Subaccount hotkey should be synthetic") + + # Verify status is eliminated (should be rejected for orders) + found, status, _ = self.entity_client.get_subaccount_status(synthetic_hotkey) + self.assertTrue(found) + self.assertEqual(status, 'eliminated', "Eliminated subaccount should be rejected for orders") + + def test_validator_non_entity_regular_hotkey_acceptance(self): + """ + Test that regular miner hotkeys (non-entity, non-synthetic) are accepted. + + Validator logic: + - Regular hotkeys that are neither entity nor synthetic should pass through + """ + regular_hotkey = "regular_miner_hotkey" + + # Verify it's not synthetic + is_synthetic = self.entity_client.is_synthetic_hotkey(regular_hotkey) + self.assertFalse(is_synthetic, "Regular hotkey should not be synthetic") + + # Verify it's not an entity + entity_data = self.entity_client.get_entity_data(regular_hotkey) + self.assertIsNone(entity_data, "Regular hotkey should not be an entity") + + +if __name__ == '__main__': + unittest.main() diff --git a/vali_objects/utils/vali_bkp_utils.py b/vali_objects/utils/vali_bkp_utils.py index bba06132f..1698b0ac0 100644 --- a/vali_objects/utils/vali_bkp_utils.py +++ b/vali_objects/utils/vali_bkp_utils.py @@ -211,6 +211,11 @@ def get_miner_account_sizes_file_location(running_unit_tests=False) -> str: suffix = "/tests" if running_unit_tests else "" return ValiConfig.BASE_DIR + f"{suffix}/validation/miner_account_sizes.json" + @staticmethod + def get_entity_file_location(running_unit_tests=False) -> str: + suffix = "/tests" if running_unit_tests else "" + return ValiConfig.BASE_DIR + f"{suffix}/validation/entities.json" + @staticmethod def get_secrets_dir(): return ValiConfig.BASE_DIR + "/secrets.json" diff --git a/vali_objects/vali_config.py b/vali_objects/vali_config.py index ee1d4041d..467252470 100644 --- a/vali_objects/vali_config.py +++ b/vali_objects/vali_config.py @@ -225,6 +225,9 @@ class ValiConfig: RPC_REST_SERVER_PORT = 50022 RPC_REST_SERVER_SERVICE_NAME = "VantaRestServer" + RPC_ENTITY_PORT = 50023 + RPC_ENTITY_SERVICE_NAME = "EntityServer" + # Public API Configuration (well-known network endpoints) REST_API_HOST = "127.0.0.1" REST_API_PORT = 48888 @@ -422,6 +425,18 @@ def get_rpc_authkey(service_name: str, port: int) -> bytes: ELIMINATION_CACHE_REFRESH_INTERVAL_S = 5 # Elimination cache refresh interval in seconds ELIMINATION_FILE_DELETION_DELAY_MS = 2 * 24 * 60 * 60 * 1000 # 2 days + # Entity Miners Configuration + ENTITY_ELIMINATION_CHECK_INTERVAL = 300 # 5 minutes (in seconds) - for challenge period + elimination checks + ENTITY_MAX_SUBACCOUNTS = 500 # Default maximum subaccounts per entity (Phase 1) + ENTITY_DATA_DIR = "validation/entities/" # Entity data persistence directory + FIXED_SUBACCOUNT_SIZE = 10000.0 # Fixed account size for subaccounts (USD) - placeholder + SUBACCOUNT_COLLATERAL_AMOUNT = 1000.0 # Placeholder collateral amount per subaccount + + # Challenge Period Configuration + SUBACCOUNT_CHALLENGE_PERIOD_DAYS = 90 # Challenge period duration (90 days) + SUBACCOUNT_CHALLENGE_RETURNS_THRESHOLD = 0.03 # 3% returns required to pass challenge period + SUBACCOUNT_CHALLENGE_DRAWDOWN_THRESHOLD = 0.06 # 6% max drawdown allowed during challenge period + # Distributional statistics SOFTMAX_TEMPERATURE = 0.15 diff --git a/vali_objects/vali_dataclasses/ledger/debt/debt_ledger_manager.py b/vali_objects/vali_dataclasses/ledger/debt/debt_ledger_manager.py index 3c8ca2360..5b47635ef 100644 --- a/vali_objects/vali_dataclasses/ledger/debt/debt_ledger_manager.py +++ b/vali_objects/vali_dataclasses/ledger/debt/debt_ledger_manager.py @@ -63,6 +63,14 @@ def __init__(self, slack_webhook_url=None, running_unit_tests=False, from vali_objects.contract.contract_server import ContractClient self._contract_client = ContractClient(running_unit_tests=running_unit_tests) + # Create EntityClient for entity miner aggregation + from entitiy_management.entity_client import EntityClient + self._entity_client = EntityClient( + connection_mode=connection_mode, + running_unit_tests=running_unit_tests, + connect_immediately=False + ) + # IMPORTANT: PenaltyLedgerManager runs WITHOUT its own daemon process (run_daemon=False) # because DebtLedgerServer itself is already a daemon process, and daemon processes # cannot spawn child processes. The DebtLedgerServer daemon thread calls @@ -695,3 +703,208 @@ def build_debt_ledgers(self, verbose: bool = False, delta_update: bool = True): f"{len(self.debt_ledgers)} hotkeys tracked " f"(target_cp_duration_ms: {target_cp_duration_ms}ms)" ) + + # Aggregate entity debt ledgers after build completes + bt.logging.info("Aggregating entity debt ledgers...") + self.aggregate_entity_debt_ledgers(target_cp_duration_ms, verbose=verbose) + + def aggregate_entity_debt_ledgers(self, target_cp_duration_ms: int, verbose: bool = False): + """ + Aggregate debt ledgers from all active subaccounts under their entity hotkeys. + + This method should be called after build_debt_ledgers() completes to ensure + all subaccount ledgers are up-to-date before aggregation. + + For each entity: + - Get all active subaccounts + - Aggregate their debt ledgers timestamp by timestamp + - Store aggregated ledger under entity_hotkey + + Aggregation rules: + - Sum: emissions, PnL, fees, open_ms, n_updates, balances + - Weighted average: portfolio_return (weighted by max_portfolio_value) + - Worst case: max_drawdown (take minimum), penalties (take minimum) + - Max: max_portfolio_value (sum across subaccounts) + + Args: + target_cp_duration_ms: Target checkpoint duration in milliseconds + verbose: Enable detailed logging + """ + try: + # Get all registered entities + all_entities = self._entity_client.get_all_entities() + + if not all_entities: + bt.logging.info("No entities registered - skipping entity aggregation") + return + + bt.logging.info(f"Aggregating debt ledgers for {len(all_entities)} entities") + + entity_count = 0 + for entity_hotkey, entity_data in all_entities.items(): + # Get active subaccounts for this entity + active_subaccounts = [sa for sa in entity_data.get('subaccounts', {}).values() + if sa.get('status') == 'active'] + + if not active_subaccounts: + if verbose: + bt.logging.info(f"Entity {entity_hotkey} has no active subaccounts - skipping") + continue + + # Get debt ledgers for all active subaccounts + subaccount_ledgers = [] + for subaccount in active_subaccounts: + synthetic_hotkey = subaccount.get('synthetic_hotkey') + if not synthetic_hotkey: + continue + + ledger = self.debt_ledgers.get(synthetic_hotkey) + if ledger and ledger.checkpoints: + subaccount_ledgers.append((synthetic_hotkey, ledger)) + + if not subaccount_ledgers: + if verbose: + bt.logging.info( + f"Entity {entity_hotkey} has {len(active_subaccounts)} active subaccounts " + f"but no debt ledgers found - skipping" + ) + continue + + # Collect all unique timestamps across all subaccount ledgers + all_timestamps = set() + for _, ledger in subaccount_ledgers: + for checkpoint in ledger.checkpoints: + all_timestamps.add(checkpoint.timestamp_ms) + + if not all_timestamps: + continue + + # Sort timestamps chronologically + sorted_timestamps = sorted(all_timestamps) + + # Create aggregated checkpoints for each timestamp + aggregated_checkpoints = [] + for timestamp_ms in sorted_timestamps: + # Collect checkpoints from all subaccounts at this timestamp + checkpoints_at_time = [] + for synthetic_hotkey, ledger in subaccount_ledgers: + checkpoint = ledger.get_checkpoint_at_time(timestamp_ms, target_cp_duration_ms) + if checkpoint: + checkpoints_at_time.append(checkpoint) + + if not checkpoints_at_time: + continue + + # Aggregate fields across all subaccounts at this timestamp + # Sum additive fields + agg_chunk_emissions_alpha = sum(cp.chunk_emissions_alpha for cp in checkpoints_at_time) + agg_chunk_emissions_tao = sum(cp.chunk_emissions_tao for cp in checkpoints_at_time) + agg_chunk_emissions_usd = sum(cp.chunk_emissions_usd for cp in checkpoints_at_time) + agg_tao_balance = sum(cp.tao_balance_snapshot for cp in checkpoints_at_time) + agg_alpha_balance = sum(cp.alpha_balance_snapshot for cp in checkpoints_at_time) + agg_realized_pnl = sum(cp.realized_pnl for cp in checkpoints_at_time) + agg_unrealized_pnl = sum(cp.unrealized_pnl for cp in checkpoints_at_time) + agg_spread_fee = sum(cp.spread_fee_loss for cp in checkpoints_at_time) + agg_carry_fee = sum(cp.carry_fee_loss for cp in checkpoints_at_time) + agg_max_portfolio_value = sum(cp.max_portfolio_value for cp in checkpoints_at_time) + agg_open_ms = sum(cp.open_ms for cp in checkpoints_at_time) + agg_n_updates = sum(cp.n_updates for cp in checkpoints_at_time) + + # Weighted average for portfolio_return (weighted by max_portfolio_value) + total_weight = sum(cp.max_portfolio_value for cp in checkpoints_at_time) + if total_weight > 0: + agg_portfolio_return = sum( + cp.portfolio_return * cp.max_portfolio_value + for cp in checkpoints_at_time + ) / total_weight + else: + # If no weight, use simple average + agg_portfolio_return = sum(cp.portfolio_return for cp in checkpoints_at_time) / len(checkpoints_at_time) + + # Worst case for max_drawdown (minimum = worst drawdown) + agg_max_drawdown = min(cp.max_drawdown for cp in checkpoints_at_time) + + # Worst case for penalties (minimum = most restrictive) + agg_drawdown_penalty = min(cp.drawdown_penalty for cp in checkpoints_at_time) + agg_risk_profile_penalty = min(cp.risk_profile_penalty for cp in checkpoints_at_time) + agg_min_collateral_penalty = min(cp.min_collateral_penalty for cp in checkpoints_at_time) + agg_risk_adjusted_perf_penalty = min(cp.risk_adjusted_performance_penalty for cp in checkpoints_at_time) + agg_total_penalty = min(cp.total_penalty for cp in checkpoints_at_time) + + # Average conversion rates (simple average) + agg_alpha_to_tao_rate = sum(cp.avg_alpha_to_tao_rate for cp in checkpoints_at_time) / len(checkpoints_at_time) + agg_tao_to_usd_rate = sum(cp.avg_tao_to_usd_rate for cp in checkpoints_at_time) / len(checkpoints_at_time) + + # Take the most restrictive challenge period status + # Priority: PLAGIARISM > CHALLENGE > PROBATION > MAINCOMP > UNKNOWN + status_priority = { + 'PLAGIARISM': 0, + 'CHALLENGE': 1, + 'PROBATION': 2, + 'MAINCOMP': 3, + 'UNKNOWN': 4 + } + agg_challenge_status = min( + (cp.challenge_period_status for cp in checkpoints_at_time), + key=lambda s: status_priority.get(s, 999) + ) + + # Use accum_ms from first checkpoint (should be same for all at this timestamp) + agg_accum_ms = checkpoints_at_time[0].accum_ms + + # Create aggregated checkpoint + aggregated_checkpoint = DebtCheckpoint( + timestamp_ms=timestamp_ms, + # Emissions + chunk_emissions_alpha=agg_chunk_emissions_alpha, + chunk_emissions_tao=agg_chunk_emissions_tao, + chunk_emissions_usd=agg_chunk_emissions_usd, + avg_alpha_to_tao_rate=agg_alpha_to_tao_rate, + avg_tao_to_usd_rate=agg_tao_to_usd_rate, + tao_balance_snapshot=agg_tao_balance, + alpha_balance_snapshot=agg_alpha_balance, + # Performance + portfolio_return=agg_portfolio_return, + realized_pnl=agg_realized_pnl, + unrealized_pnl=agg_unrealized_pnl, + spread_fee_loss=agg_spread_fee, + carry_fee_loss=agg_carry_fee, + max_drawdown=agg_max_drawdown, + max_portfolio_value=agg_max_portfolio_value, + open_ms=agg_open_ms, + accum_ms=agg_accum_ms, + n_updates=agg_n_updates, + # Penalties + drawdown_penalty=agg_drawdown_penalty, + risk_profile_penalty=agg_risk_profile_penalty, + min_collateral_penalty=agg_min_collateral_penalty, + risk_adjusted_performance_penalty=agg_risk_adjusted_perf_penalty, + total_penalty=agg_total_penalty, + challenge_period_status=agg_challenge_status, + ) + + aggregated_checkpoints.append(aggregated_checkpoint) + + if not aggregated_checkpoints: + continue + + # Create aggregated debt ledger for entity + entity_ledger = DebtLedger(entity_hotkey, checkpoints=aggregated_checkpoints) + + # Store in debt_ledgers dict + self.debt_ledgers[entity_hotkey] = entity_ledger + entity_count += 1 + + if verbose: + bt.logging.info( + f"Aggregated {len(aggregated_checkpoints)} checkpoints for entity {entity_hotkey} " + f"from {len(subaccount_ledgers)} active subaccounts" + ) + + bt.logging.info( + f"Entity aggregation completed: {entity_count} entities aggregated " + f"({len(all_entities) - entity_count} skipped with no data)" + ) + + except Exception as e: + bt.logging.error(f"Error aggregating entity debt ledgers: {e}", exc_info=True) diff --git a/vanta_api/rest_server.py b/vanta_api/rest_server.py index 02a639030..5f78dd37c 100644 --- a/vanta_api/rest_server.py +++ b/vanta_api/rest_server.py @@ -32,6 +32,7 @@ from vanta_api.api_key_refresh import APIKeyMixin from vanta_api.nonce_manager import NonceManager from shared_objects.rpc.rpc_server_base import RPCServerBase +from entitiy_management.entity_client import EntityClient class APIMetricsTracker: @@ -383,6 +384,11 @@ def __init__(self, api_keys_file, shared_queue=None, refresh_interval=15, self._statistics_outputs_client = MinerStatisticsClient(connection_mode=connection_mode) print(f"[REST-INIT] Step 2f/9: StatisticsOutputsClient created ✓") + print(f"[REST-INIT] Step 2g/9: Creating EntityClient...") + # Create own EntityClient (forward compatibility - no parameter passing) + self._entity_client = EntityClient(connection_mode=connection_mode, running_unit_tests=running_unit_tests) + print(f"[REST-INIT] Step 2g/9: EntityClient created ✓") + print(f"[REST-INIT] Step 3/9: Setting REST server configuration...") # IMPORTANT: Store Flask HTTP server config separately from RPC port # Flask serves REST API on configurable host/port (self.flask_host/flask_port) @@ -1494,6 +1500,260 @@ def process_development_order(): bt.logging.error(traceback.format_exc()) return jsonify({'error': f'Internal server error: {str(e)}'}), 500 + # ============================================================================ + # ENTITY MANAGEMENT ENDPOINTS + # ============================================================================ + + @self.app.route("/entity/register", methods=["POST"]) + def register_entity(): + """ + Register a new entity. + + Example: + curl -X POST http://localhost:48888/entity/register \\ + -H "Authorization: Bearer YOUR_API_KEY" \\ + -H "Content-Type: application/json" \\ + -d '{"entity_hotkey": "5GhDr...", "collateral_amount": 1000.0, "max_subaccounts": 10}' + """ + # Check API key authentication + api_key = self._get_api_key_safe() + if not self.is_valid_api_key(api_key): + return jsonify({'error': 'Unauthorized access'}), 401 + + # Check if entity client is available + if not self._entity_client: + return jsonify({'error': 'Entity management not available'}), 503 + + try: + # Parse and validate request + if not request.is_json: + return jsonify({'error': 'Content-Type must be application/json'}), 400 + + data = request.get_json() + if not data: + return jsonify({'error': 'Invalid JSON body'}), 400 + + # Validate required fields + if 'entity_hotkey' not in data: + return jsonify({'error': 'Missing required field: entity_hotkey'}), 400 + + entity_hotkey = data['entity_hotkey'] + collateral_amount = data.get('collateral_amount', 0.0) + max_subaccounts = data.get('max_subaccounts', None) + + # Register entity via RPC + success, message = self._entity_client.register_entity( + entity_hotkey=entity_hotkey, + collateral_amount=collateral_amount, + max_subaccounts=max_subaccounts + ) + + if success: + return jsonify({ + 'status': 'success', + 'message': message, + 'entity_hotkey': entity_hotkey + }), 200 + else: + return jsonify({'error': message}), 400 + + except Exception as e: + bt.logging.error(f"Error registering entity: {e}") + return jsonify({'error': 'Internal server error registering entity'}), 500 + + @self.app.route("/entity/subaccount", methods=["POST"]) + def create_subaccount(): + """ + Create a new subaccount for an entity. + + Example: + curl -X POST http://localhost:48888/entity/subaccount \\ + -H "Authorization: Bearer YOUR_API_KEY" \\ + -H "Content-Type: application/json" \\ + -d '{"entity_hotkey": "5GhDr..."}' + """ + # Check API key authentication + api_key = self._get_api_key_safe() + if not self.is_valid_api_key(api_key): + return jsonify({'error': 'Unauthorized access'}), 401 + + # Check if entity client is available + if not self._entity_client: + return jsonify({'error': 'Entity management not available'}), 503 + + try: + # Parse and validate request + if not request.is_json: + return jsonify({'error': 'Content-Type must be application/json'}), 400 + + data = request.get_json() + if not data: + return jsonify({'error': 'Invalid JSON body'}), 400 + + # Validate required fields + if 'entity_hotkey' not in data: + return jsonify({'error': 'Missing required field: entity_hotkey'}), 400 + + entity_hotkey = data['entity_hotkey'] + + # Create subaccount via RPC + success, subaccount_info, message = self._entity_client.create_subaccount(entity_hotkey) + + if success: + # Broadcast subaccount registration to other validators + try: + self._entity_client.broadcast_subaccount_registration( + entity_hotkey=entity_hotkey, + subaccount_id=subaccount_info['subaccount_id'], + subaccount_uuid=subaccount_info['subaccount_uuid'], + synthetic_hotkey=subaccount_info['synthetic_hotkey'] + ) + bt.logging.info(f"[REST_API] Broadcasted subaccount registration for {subaccount_info['synthetic_hotkey']}") + except Exception as e: + # Don't fail the request if broadcast fails - it's a background operation + bt.logging.warning(f"[REST_API] Failed to broadcast subaccount registration: {e}") + + return jsonify({ + 'status': 'success', + 'message': message, + 'subaccount': subaccount_info + }), 200 + else: + return jsonify({'error': message}), 400 + + except Exception as e: + bt.logging.error(f"Error creating subaccount: {e}") + return jsonify({'error': 'Internal server error creating subaccount'}), 500 + + @self.app.route("/entity/", methods=["GET"]) + def get_entity(entity_hotkey): + """ + Get entity data for a specific entity. + + Example: + curl -H "Authorization: Bearer YOUR_API_KEY" http://localhost:48888/entity/5GhDr... + """ + # Check API key authentication + api_key = self._get_api_key_safe() + if not self.is_valid_api_key(api_key): + return jsonify({'error': 'Unauthorized access'}), 401 + + # Check if entity client is available + if not self._entity_client: + return jsonify({'error': 'Entity management not available'}), 503 + + try: + # Get entity data via RPC + entity_data = self._entity_client.get_entity_data(entity_hotkey) + + if entity_data: + return jsonify({ + 'status': 'success', + 'entity': entity_data + }), 200 + else: + return jsonify({'error': f'Entity {entity_hotkey} not found'}), 404 + + except Exception as e: + bt.logging.error(f"Error retrieving entity {entity_hotkey}: {e}") + return jsonify({'error': 'Internal server error retrieving entity'}), 500 + + @self.app.route("/entities", methods=["GET"]) + def get_all_entities(): + """ + Get all registered entities. + + Example: + curl -H "Authorization: Bearer YOUR_API_KEY" http://localhost:48888/entities + """ + # Check API key authentication + api_key = self._get_api_key_safe() + if not self.is_valid_api_key(api_key): + return jsonify({'error': 'Unauthorized access'}), 401 + + # Check if entity client is available + if not self._entity_client: + return jsonify({'error': 'Entity management not available'}), 503 + + try: + # Get all entities via RPC + entities = self._entity_client.get_all_entities() + + return jsonify({ + 'status': 'success', + 'entities': entities, + 'entity_count': len(entities), + 'timestamp': TimeUtil.now_in_millis() + }), 200 + + except Exception as e: + bt.logging.error(f"Error retrieving all entities: {e}") + return jsonify({'error': 'Internal server error retrieving entities'}), 500 + + @self.app.route("/entity/subaccount/eliminate", methods=["POST"]) + def eliminate_subaccount(): + """ + Eliminate a subaccount. + + Example: + curl -X POST http://localhost:48888/entity/subaccount/eliminate \\ + -H "Authorization: Bearer YOUR_API_KEY" \\ + -H "Content-Type: application/json" \\ + -d '{"entity_hotkey": "5GhDr...", "subaccount_id": 0, "reason": "manual_elimination"}' + """ + # Check API key authentication + api_key = self._get_api_key_safe() + if not self.is_valid_api_key(api_key): + return jsonify({'error': 'Unauthorized access'}), 401 + + # Check if entity client is available + if not self._entity_client: + return jsonify({'error': 'Entity management not available'}), 503 + + try: + # Parse and validate request + if not request.is_json: + return jsonify({'error': 'Content-Type must be application/json'}), 400 + + data = request.get_json() + if not data: + return jsonify({'error': 'Invalid JSON body'}), 400 + + # Validate required fields + required_fields = ['entity_hotkey', 'subaccount_id'] + for field in required_fields: + if field not in data: + return jsonify({'error': f'Missing required field: {field}'}), 400 + + entity_hotkey = data['entity_hotkey'] + subaccount_id = data['subaccount_id'] + reason = data.get('reason', 'manual_elimination') + + # Validate subaccount_id is an integer + try: + subaccount_id = int(subaccount_id) + except (ValueError, TypeError): + return jsonify({'error': 'subaccount_id must be an integer'}), 400 + + # Eliminate subaccount via RPC + success, message = self._entity_client.eliminate_subaccount( + entity_hotkey=entity_hotkey, + subaccount_id=subaccount_id, + reason=reason + ) + + if success: + return jsonify({ + 'status': 'success', + 'message': message + }), 200 + else: + return jsonify({'error': message}), 400 + + except Exception as e: + bt.logging.error(f"Error eliminating subaccount: {e}") + return jsonify({'error': 'Internal server error eliminating subaccount'}), 500 + def _verify_coldkey_owns_hotkey(self, coldkey_ss58: str, hotkey_ss58: str) -> bool: """ Verify that a coldkey owns the specified hotkey using subtensor.