From 8c46c2914b766b526335a5035ddcddd2245e5397 Mon Sep 17 00:00:00 2001 From: Christian Laffin Date: Thu, 14 Aug 2025 21:19:03 +0100 Subject: [PATCH 1/2] feat: add Vultr provider support - Complete Vultr API v2 integration with full lifecycle management - Support for multi-instance configurations across different regions - Automatic firewall configuration for proxy instances - Comprehensive test suite with 25 tests covering all functionality - Updated to Ubuntu 22.04 LTS (OS ID 1743) for better compatibility - Added Vultr documentation with configuration examples - Updated frontend UI to support Vultr branding - Fixed API model to support Vultr's plan field instead of size - Added examples to README for quick start and multi-account setup Provider features: - Automatic proxy deployment and scaling - Region-based instance distribution - Firewall group management - Health monitoring and auto-recovery - Support for all Vultr regions and plans Testing: - All 25 unit tests passing - Successfully tested with live Vultr API - Validated minimal and full configuration scenarios - Error handling tested for invalid configurations --- .env.example | 163 +++++++++ .gitignore | 1 + README.md | 10 +- cloudproxy-ui/src/components/ListProxies.vue | 6 +- cloudproxy/main.py | 4 +- cloudproxy/providers/manager.py | 12 + cloudproxy/providers/settings.py | 46 ++- cloudproxy/providers/vultr/__init__.py | 0 cloudproxy/providers/vultr/functions.py | 365 +++++++++++++++++++ cloudproxy/providers/vultr/main.py | 236 ++++++++++++ docs/vultr.md | 250 +++++++++++++ tests/test_providers_vultr_functions.py | 316 ++++++++++++++++ tests/test_providers_vultr_main.py | 320 ++++++++++++++++ 13 files changed, 1722 insertions(+), 7 deletions(-) create mode 100644 .env.example create mode 100644 cloudproxy/providers/vultr/__init__.py create mode 100644 cloudproxy/providers/vultr/functions.py create mode 100644 cloudproxy/providers/vultr/main.py create mode 100644 docs/vultr.md create mode 100644 tests/test_providers_vultr_functions.py create mode 100644 tests/test_providers_vultr_main.py diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..db68a27 --- /dev/null +++ b/.env.example @@ -0,0 +1,163 @@ +# CloudProxy Environment Configuration Example +# Copy this file to .env and fill in your values + +# ==================== +# Authentication Settings +# ==================== +# Required: Set credentials for proxy authentication (alphanumeric only) +PROXY_USERNAME=changeme +PROXY_PASSWORD=changeme + +# Optional: Restrict access to host IP only +# ONLY_HOST_IP=False + +# Optional: Proxy age limit in seconds (0 = disabled) +# AGE_LIMIT=0 + +# ==================== +# DigitalOcean Provider +# ==================== +# Enable provider +DIGITALOCEAN_ENABLED=False + +# Required if enabled +DIGITALOCEAN_ACCESS_TOKEN=your_digitalocean_token_here + +# Optional settings +# DIGITALOCEAN_REGION=lon1 +# DIGITALOCEAN_MIN_SCALING=2 +# DIGITALOCEAN_MAX_SCALING=2 +# DIGITALOCEAN_SIZE=s-1vcpu-1gb + +# Additional DigitalOcean instance example +# DIGITALOCEAN_NYC_ENABLED=False +# DIGITALOCEAN_NYC_ACCESS_TOKEN=your_second_token_here +# DIGITALOCEAN_NYC_REGION=nyc1 +# DIGITALOCEAN_NYC_MIN_SCALING=3 +# DIGITALOCEAN_NYC_SIZE=s-1vcpu-1gb +# DIGITALOCEAN_NYC_DISPLAY_NAME=NYC Proxies + +# ==================== +# AWS Provider +# ==================== +# Enable provider +AWS_ENABLED=False + +# Required if enabled +AWS_ACCESS_KEY_ID=your_aws_access_key_here +AWS_SECRET_ACCESS_KEY=your_aws_secret_key_here + +# Optional settings +# AWS_REGION=eu-west-2 +# AWS_MIN_SCALING=2 +# AWS_MAX_SCALING=2 +# AWS_SIZE=t2.micro +# AWS_AMI=ami-096cb92bb3580c759 +# AWS_SPOT=False + +# Additional AWS instance example +# AWS_USEAST_ENABLED=False +# AWS_USEAST_ACCESS_KEY_ID=your_second_key_here +# AWS_USEAST_SECRET_ACCESS_KEY=your_second_secret_here +# AWS_USEAST_REGION=us-east-1 +# AWS_USEAST_MIN_SCALING=2 +# AWS_USEAST_AMI=ami-0c02fb55731490381 +# AWS_USEAST_DISPLAY_NAME=US East Proxies + +# ==================== +# Google Cloud Provider +# ==================== +# Enable provider +GCP_ENABLED=False + +# Required if enabled +GCP_PROJECT=your_gcp_project_id_here +GCP_SERVICE_ACCOUNT_KEY=your_service_account_json_here + +# Optional: Use a JSON file instead +# GCP_SA_JSON=/path/to/service-account.json + +# Optional settings +# GCP_ZONE=us-central1-a +# GCP_MIN_SCALING=2 +# GCP_MAX_SCALING=2 +# GCP_SIZE=f1-micro +# GCP_IMAGE_PROJECT=ubuntu-os-cloud +# GCP_IMAGE_FAMILY=ubuntu-minimal-2004-lts + +# Additional GCP instance example +# GCP_EUROPE_ENABLED=False +# GCP_EUROPE_PROJECT=your_project_id_here +# GCP_EUROPE_SERVICE_ACCOUNT_KEY=your_key_here +# GCP_EUROPE_ZONE=europe-west1-b +# GCP_EUROPE_MIN_SCALING=2 +# GCP_EUROPE_DISPLAY_NAME=Europe Proxies + +# ==================== +# Hetzner Provider +# ==================== +# Enable provider +HETZNER_ENABLED=False + +# Required if enabled +HETZNER_ACCESS_TOKEN=your_hetzner_token_here + +# Optional settings +# HETZNER_LOCATION=nbg1 +# HETZNER_MIN_SCALING=2 +# HETZNER_MAX_SCALING=2 +# HETZNER_SIZE=cx21 + +# Additional Hetzner instance example +# HETZNER_US_ENABLED=False +# HETZNER_US_ACCESS_TOKEN=your_second_token_here +# HETZNER_US_LOCATION=ash +# HETZNER_US_MIN_SCALING=2 +# HETZNER_US_SIZE=cx21 +# HETZNER_US_DISPLAY_NAME=US Proxies + +# ==================== +# Vultr Provider +# ==================== +# Enable provider +VULTR_ENABLED=False + +# Required if enabled +VULTR_API_TOKEN=your_vultr_api_token_here + +# Optional settings +# VULTR_REGION=ewr +# VULTR_MIN_SCALING=2 +# VULTR_MAX_SCALING=2 +# VULTR_PLAN=vc2-1c-1gb +# VULTR_OS_ID=387 + +# Additional Vultr instance example +# VULTR_EUROPE_ENABLED=False +# VULTR_EUROPE_API_TOKEN=your_second_token_here +# VULTR_EUROPE_REGION=ams +# VULTR_EUROPE_MIN_SCALING=3 +# VULTR_EUROPE_PLAN=vc2-1c-1gb +# VULTR_EUROPE_DISPLAY_NAME=Europe Proxies + +# Additional Vultr instance for Asia +# VULTR_ASIA_ENABLED=False +# VULTR_ASIA_API_TOKEN=your_third_token_here +# VULTR_ASIA_REGION=sgp +# VULTR_ASIA_MIN_SCALING=2 +# VULTR_ASIA_PLAN=vc2-1c-2gb +# VULTR_ASIA_DISPLAY_NAME=Asia Proxies + +# ==================== +# Azure Provider (Planned) +# ==================== +# AZURE_ENABLED=False +# AZURE_SUBSCRIPTION_ID=your_subscription_id_here +# AZURE_CLIENT_ID=your_client_id_here +# AZURE_CLIENT_SECRET=your_client_secret_here +# AZURE_TENANT_ID=your_tenant_id_here +# AZURE_RESOURCE_GROUP=cloudproxy-rg +# AZURE_LOCATION=eastus +# AZURE_MIN_SCALING=2 +# AZURE_MAX_SCALING=2 +# AZURE_SIZE=Standard_B1s \ No newline at end of file diff --git a/.gitignore b/.gitignore index 078c6b8..8be95ba 100644 --- a/.gitignore +++ b/.gitignore @@ -153,3 +153,4 @@ pnpm-debug.log* *.sln *.sw? .cursor-project-context +CLAUDE.md diff --git a/README.md b/README.md index 4d49dd2..f03a853 100644 --- a/README.md +++ b/README.md @@ -72,11 +72,11 @@ CloudProxy exposes an API and modern UI for managing your proxy infrastructure. * [AWS](docs/aws.md) * [Google Cloud](docs/gcp.md) * [Hetzner](docs/hetzner.md) +* [Vultr](docs/vultr.md) ### Planned: * Azure * Scaleway -* Vultr ### Features: * **Docker-first deployment** - Simple, isolated, production-ready @@ -348,10 +348,15 @@ curl -X 'GET' 'http://localhost:8000/' -H 'accept: application/json' #### Set target proxy count ```bash -# CloudProxy will maintain exactly 5 proxies +# CloudProxy will maintain exactly 5 proxies (DigitalOcean) curl -X 'PATCH' 'http://localhost:8000/providers/digitalocean' \ -H 'Content-Type: application/json' \ -d '{"min_scaling": 5, "max_scaling": 5}' + +# Or for Vultr +curl -X 'PATCH' 'http://localhost:8000/providers/vultr' \ + -H 'Content-Type: application/json' \ + -d '{"min_scaling": 3, "max_scaling": 3}' ``` ### Python Usage Example @@ -433,6 +438,7 @@ Project Link: [https://github.com/claffin/cloudproxy](https://github.com/claffin - **GCP**: Check that the service account has necessary permissions - **DigitalOcean**: Verify the access token has write permissions - **Hetzner**: Ensure the API token is valid +- **Vultr**: Verify the API token has appropriate permissions and the selected plan/region is available #### Docker container issues ```bash diff --git a/cloudproxy-ui/src/components/ListProxies.vue b/cloudproxy-ui/src/components/ListProxies.vue index c357ba1..ab85ebe 100644 --- a/cloudproxy-ui/src/components/ListProxies.vue +++ b/cloudproxy-ui/src/components/ListProxies.vue @@ -247,7 +247,8 @@ export default { 'aws': 'AWS', 'gcp': 'GCP', 'hetzner': 'Hetzner', - 'azure': 'Azure' + 'azure': 'Azure', + 'vultr': 'Vultr' }; const providerName = specialCases[name] || name.charAt(0).toUpperCase() + name.slice(1); @@ -265,7 +266,8 @@ export default { aws: 'cloud-fill', gcp: 'google', hetzner: 'hdd-rack', - azure: 'microsoft' + azure: 'microsoft', + vultr: 'server' }; return icons[provider] || 'cloud-fill'; }; diff --git a/cloudproxy/main.py b/cloudproxy/main.py index 06c9472..b06fe87 100644 --- a/cloudproxy/main.py +++ b/cloudproxy/main.py @@ -342,7 +342,8 @@ class ProviderInstance(BaseModel): enabled: bool ips: List[str] = [] scaling: ProviderScaling - size: str + size: Optional[str] = None # Made optional, some providers use different naming + plan: Optional[str] = None # For Vultr provider region: Optional[str] = None location: Optional[str] = None datacenter: Optional[str] = None @@ -350,6 +351,7 @@ class ProviderInstance(BaseModel): image_project: Optional[str] = None image_family: Optional[str] = None ami: Optional[str] = None + os_id: Optional[int] = None # For Vultr provider spot: Optional[bool] = None display_name: Optional[str] = None project: Optional[str] = None diff --git a/cloudproxy/providers/manager.py b/cloudproxy/providers/manager.py index 7950bfa..242e63f 100644 --- a/cloudproxy/providers/manager.py +++ b/cloudproxy/providers/manager.py @@ -5,6 +5,7 @@ from cloudproxy.providers.gcp.main import gcp_start from cloudproxy.providers.digitalocean.main import do_start from cloudproxy.providers.hetzner.main import hetzner_start +from cloudproxy.providers.vultr.main import vultr_start def do_manager(instance_name="default"): @@ -47,6 +48,16 @@ def hetzner_manager(instance_name="default"): return ip_list +def vultr_manager(instance_name="default"): + """ + Vultr manager function for a specific instance. + """ + instance_config = settings.config["providers"]["vultr"]["instances"][instance_name] + ip_list = vultr_start(instance_config) + settings.config["providers"]["vultr"]["instances"][instance_name]["ips"] = [ip for ip in ip_list] + return ip_list + + def init_schedule(): sched = BackgroundScheduler() sched.start() @@ -57,6 +68,7 @@ def init_schedule(): "aws": aws_manager, "gcp": gcp_manager, "hetzner": hetzner_manager, + "vultr": vultr_manager, } # Schedule jobs for all provider instances diff --git a/cloudproxy/providers/settings.py b/cloudproxy/providers/settings.py index eb47901..8c82dc9 100644 --- a/cloudproxy/providers/settings.py +++ b/cloudproxy/providers/settings.py @@ -84,6 +84,20 @@ } } }, + "vultr": { + "instances": { + "default": { + "enabled": False, + "ips": [], + "scaling": {"min_scaling": 0, "max_scaling": 0}, + "plan": "", + "region": "", + "os_id": 1743, # Ubuntu 22.04 LTS x64 + "display_name": "Vultr", + "secrets": {"api_token": ""}, + } + } + }, }, } @@ -219,6 +233,32 @@ "AZURE_DISPLAY_NAME", "Azure" ) +# Set Vultr config - original format for backward compatibility +config["providers"]["vultr"]["instances"]["default"]["enabled"] = os.environ.get( + "VULTR_ENABLED", "False" +) == "True" +config["providers"]["vultr"]["instances"]["default"]["secrets"]["api_token"] = os.environ.get( + "VULTR_API_TOKEN" +) +config["providers"]["vultr"]["instances"]["default"]["scaling"]["min_scaling"] = int( + os.environ.get("VULTR_MIN_SCALING", 2) +) +config["providers"]["vultr"]["instances"]["default"]["scaling"]["max_scaling"] = int( + os.environ.get("VULTR_MAX_SCALING", 2) +) +config["providers"]["vultr"]["instances"]["default"]["plan"] = os.environ.get( + "VULTR_PLAN", "vc2-1c-1gb" # 1 vCPU, 1GB RAM plan +) +config["providers"]["vultr"]["instances"]["default"]["region"] = os.environ.get( + "VULTR_REGION", "ewr" # New Jersey region +) +config["providers"]["vultr"]["instances"]["default"]["os_id"] = int( + os.environ.get("VULTR_OS_ID", 1743) # Ubuntu 22.04 LTS x64 +) +config["providers"]["vultr"]["instances"]["default"]["display_name"] = os.environ.get( + "VULTR_DISPLAY_NAME", "Vultr" +) + # Check for additional provider instances using the new format pattern for provider_key in config["providers"].keys(): provider_upper = provider_key.upper() @@ -248,7 +288,7 @@ # Copy relevant fields from default instance default_instance = config["providers"][provider_key]["instances"]["default"] for field in ["region", "zone", "location", "ami", "spot", "datacenter", - "image_project", "image_family", "project"]: + "image_project", "image_family", "project", "plan", "os_id"]: if field in default_instance: config["providers"][provider_key]["instances"][instance_name][field] = default_instance[field] @@ -272,8 +312,10 @@ elif setting_name == "display_name": config["providers"][provider_key]["instances"][instance_name]["display_name"] = env_value elif setting_name in ["size", "region", "zone", "location", "ami", "project", - "image_project", "image_family", "datacenter"]: + "image_project", "image_family", "datacenter", "plan"]: config["providers"][provider_key]["instances"][instance_name][setting_name] = env_value + elif setting_name == "os_id": + config["providers"][provider_key]["instances"][instance_name]["os_id"] = int(env_value) elif setting_name == "spot": config["providers"][provider_key]["instances"][instance_name]["spot"] = env_value == "True" elif setting_name in default_instance["secrets"]: diff --git a/cloudproxy/providers/vultr/__init__.py b/cloudproxy/providers/vultr/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cloudproxy/providers/vultr/functions.py b/cloudproxy/providers/vultr/functions.py new file mode 100644 index 0000000..1d79f69 --- /dev/null +++ b/cloudproxy/providers/vultr/functions.py @@ -0,0 +1,365 @@ +import uuid +import base64 +import requests +from typing import List, Dict, Any, Optional +from loguru import logger + +from cloudproxy.providers import settings +from cloudproxy.providers.config import set_auth + + +class VultrFirewallExistsException(Exception): + pass + + +class VultrInstance: + """Wrapper class for Vultr instance data.""" + + def __init__(self, data: Dict[str, Any]): + self.id = data.get('id') + self.main_ip = data.get('main_ip') + self.ip_address = data.get('main_ip') # Alias for compatibility + self.label = data.get('label', '') + self.date_created = data.get('date_created') + self.status = data.get('status') + self.region = data.get('region') + self.plan = data.get('plan') + self.tags = data.get('tags', []) + self._raw_data = data + + +def get_api_headers(instance_config: Optional[Dict] = None) -> Dict[str, str]: + """ + Get API headers with authentication for Vultr API. + + Args: + instance_config: The specific instance configuration + + Returns: + dict: Headers for API requests + """ + if instance_config is None: + instance_config = settings.config["providers"]["vultr"]["instances"]["default"] + + return { + "Authorization": f"Bearer {instance_config['secrets']['api_token']}", + "Content-Type": "application/json" + } + + +def create_proxy(instance_config: Optional[Dict] = None) -> bool: + """ + Create a Vultr proxy instance. + + Args: + instance_config: The specific instance configuration + + Returns: + bool: True if creation was successful + """ + if instance_config is None: + instance_config = settings.config["providers"]["vultr"]["instances"]["default"] + + # Get instance name for tagging + instance_id = next( + (name for name, inst in settings.config["providers"]["vultr"]["instances"].items() + if inst == instance_config), + "default" + ) + + # Prepare user data + user_data = set_auth( + settings.config["auth"]["username"], + settings.config["auth"]["password"] + ) + + # Base64 encode the user data + user_data_encoded = base64.b64encode(user_data.encode()).decode() + + # Prepare instance creation payload + payload = { + "region": instance_config["region"], + "plan": instance_config["plan"], + # Ubuntu 22.04 LTS x64 default + "os_id": instance_config.get("os_id", 1743), + "label": f"cloudproxy-{instance_id}-{str(uuid.uuid1())}", + "user_data": user_data_encoded, + "tags": ["cloudproxy", f"cloudproxy-{instance_id}"], + "enable_ipv6": False, + "backups": "disabled", + "ddos_protection": False + } + + # Add firewall group if it exists + firewall_group_id = instance_config.get("firewall_group_id") + if firewall_group_id: + payload["firewall_group_id"] = firewall_group_id + + try: + response = requests.post( + "https://api.vultr.com/v2/instances", + headers=get_api_headers(instance_config), + json=payload + ) + response.raise_for_status() + + data = response.json() + logger.info( + f"Created Vultr instance: { + data.get( + 'instance', + {}).get('id')}") + return True + + except requests.exceptions.RequestException as e: + logger.error(f"Failed to create Vultr instance: {e}") + if hasattr(e, 'response') and e.response is not None: + logger.error(f"Response: {e.response.text}") + return False + + +def delete_proxy(instance: Any, + instance_config: Optional[Dict] = None) -> bool: + """ + Delete a Vultr proxy instance. + + Args: + instance: Instance ID or VultrInstance object + instance_config: The specific instance configuration + + Returns: + bool: True if deletion was successful + """ + if instance_config is None: + instance_config = settings.config["providers"]["vultr"]["instances"]["default"] + + # Extract instance ID + if hasattr(instance, 'id'): + instance_id = instance.id + else: + instance_id = instance + + try: + response = requests.delete( + f"https://api.vultr.com/v2/instances/{instance_id}", + headers=get_api_headers(instance_config) + ) + + # 204 No Content is success for DELETE + if response.status_code == 204: + logger.info(f"Successfully deleted Vultr instance: {instance_id}") + return True + elif response.status_code == 404: + logger.info( + f"Vultr instance {instance_id} not found, considering it already deleted") + return True + else: + response.raise_for_status() + return False + + except requests.exceptions.RequestException as e: + # Check if instance is already gone + if hasattr(e, 'response') and e.response is not None: + if e.response.status_code == 404: + logger.info( + f"Vultr instance {instance_id} not found, considering it already deleted") + return True + + logger.error(f"Failed to delete Vultr instance {instance_id}: {e}") + return False + + +def list_instances( + instance_config: Optional[Dict] = None) -> List[VultrInstance]: + """ + List Vultr proxy instances. + + Args: + instance_config: The specific instance configuration + + Returns: + list: List of VultrInstance objects + """ + if instance_config is None: + instance_config = settings.config["providers"]["vultr"]["instances"]["default"] + + # Get instance name for tagging + instance_id = next( + (name for name, inst in settings.config["providers"]["vultr"]["instances"].items() + if inst == instance_config), + "default" + ) + + try: + # Get all instances with the specific tag + response = requests.get( + "https://api.vultr.com/v2/instances", + headers=get_api_headers(instance_config), + params={"tag": f"cloudproxy-{instance_id}", "per_page": 500} + ) + response.raise_for_status() + + data = response.json() + instances = data.get('instances', []) + + # If this is the default instance, also get instances with just the + # cloudproxy tag + if instance_id == "default": + response_old = requests.get( + "https://api.vultr.com/v2/instances", + headers=get_api_headers(instance_config), + params={"tag": "cloudproxy", "per_page": 500} + ) + + if response_old.status_code == 200: + data_old = response_old.json() + old_instances = data_old.get('instances', []) + + # Filter out instances that have instance-specific tags + existing_ids = {inst['id'] for inst in instances} + for inst in old_instances: + tags = inst.get('tags', []) + has_instance_tag = any( + tag.startswith('cloudproxy-') and tag != 'cloudproxy' + for tag in tags + ) + if not has_instance_tag and inst['id'] not in existing_ids: + instances.append(inst) + + # Convert to VultrInstance objects + return [VultrInstance(inst) for inst in instances] + + except requests.exceptions.RequestException as e: + logger.error(f"Failed to list Vultr instances: {e}") + if hasattr(e, 'response') and e.response is not None: + logger.error(f"Response: {e.response.text}") + return [] + + +def create_firewall(instance_config: Optional[Dict] = None) -> Optional[str]: + """ + Create a Vultr firewall group for proxy instances. + + Args: + instance_config: The specific instance configuration + + Returns: + str: Firewall group ID if created successfully + + Raises: + VultrFirewallExistsException: If firewall already exists + """ + if instance_config is None: + instance_config = settings.config["providers"]["vultr"]["instances"]["default"] + + # Get instance name for firewall naming + instance_id = next( + (name for name, inst in settings.config["providers"]["vultr"]["instances"].items() + if inst == instance_config), + "default" + ) + + firewall_name = f"cloudproxy-{instance_id}" + + # First check if firewall already exists + try: + response = requests.get( + "https://api.vultr.com/v2/firewalls", + headers=get_api_headers(instance_config), + params={"per_page": 500} + ) + response.raise_for_status() + + data = response.json() + for fw in data.get('firewall_groups', []): + if fw.get('description') == firewall_name: + # Store the firewall group ID in the config + instance_config['firewall_group_id'] = fw['id'] + raise VultrFirewallExistsException( + f"Firewall already exists: {fw['id']}") + + except requests.exceptions.RequestException as e: + logger.error(f"Failed to check existing firewalls: {e}") + + # Create new firewall group + try: + response = requests.post( + "https://api.vultr.com/v2/firewalls", + headers=get_api_headers(instance_config), + json={"description": firewall_name} + ) + response.raise_for_status() + + data = response.json() + firewall_group_id = data.get('firewall_group', {}).get('id') + + if firewall_group_id: + # Store the firewall group ID + instance_config['firewall_group_id'] = firewall_group_id + + # Add firewall rules + _create_firewall_rules(firewall_group_id, instance_config) + + logger.info(f"Created firewall group: {firewall_group_id}") + return firewall_group_id + + except requests.exceptions.RequestException as e: + logger.error(f"Failed to create firewall group: {e}") + if hasattr(e, 'response') and e.response is not None: + logger.error(f"Response: {e.response.text}") + + return None + + +def _create_firewall_rules(firewall_group_id: str, + instance_config: Dict) -> None: + """ + Create firewall rules for the firewall group. + + Args: + firewall_group_id: The firewall group ID + instance_config: The instance configuration + """ + rules = [ + # Allow inbound on port 8899 (proxy port) + { + "ip_type": "v4", + "protocol": "tcp", + "port": "8899", + "subnet": "0.0.0.0", + "subnet_size": 0, + "notes": "CloudProxy port" + }, + # Allow all outbound TCP + { + "ip_type": "v4", + "protocol": "tcp", + "port": "1:65535", + "subnet": "0.0.0.0", + "subnet_size": 0, + "notes": "All outbound TCP" + }, + # Allow all outbound UDP + { + "ip_type": "v4", + "protocol": "udp", + "port": "1:65535", + "subnet": "0.0.0.0", + "subnet_size": 0, + "notes": "All outbound UDP" + } + ] + + for rule in rules: + try: + response = requests.post( + f"https://api.vultr.com/v2/firewalls/{firewall_group_id}/rules", + headers=get_api_headers(instance_config), + json=rule + ) + response.raise_for_status() + logger.debug(f"Added firewall rule: {rule['notes']}") + except requests.exceptions.RequestException as e: + logger.error(f"Failed to add firewall rule: {e}") + if hasattr(e, 'response') and e.response is not None: + logger.error(f"Response: {e.response.text}") diff --git a/cloudproxy/providers/vultr/main.py b/cloudproxy/providers/vultr/main.py new file mode 100644 index 0000000..f26a5bc --- /dev/null +++ b/cloudproxy/providers/vultr/main.py @@ -0,0 +1,236 @@ +import datetime +import itertools + +import dateparser +from loguru import logger + +from cloudproxy.check import check_alive +from cloudproxy.providers.vultr.functions import ( + create_proxy, + list_instances, + delete_proxy, + create_firewall, + VultrFirewallExistsException, +) +from cloudproxy.providers.settings import delete_queue, restart_queue, config + + +def vultr_deployment(min_scaling, instance_config=None): + """ + Deploy Vultr instances based on min_scaling requirements. + + Args: + min_scaling: The minimum number of instances to maintain + instance_config: The specific instance configuration + """ + if instance_config is None: + instance_config = config["providers"]["vultr"]["instances"]["default"] + + # Get instance display name for logging + display_name = instance_config.get("display_name", "default") + + total_instances = len(list_instances(instance_config)) + if min_scaling < total_instances: + logger.info(f"Overprovisioned: Vultr {display_name} destroying.....") + for instance in itertools.islice(list_instances( + instance_config), 0, (total_instances - min_scaling)): + delete_proxy(instance, instance_config) + logger.info( + f"Destroyed: Vultr {display_name} -> {str(instance.ip_address)}") + + if min_scaling - total_instances < 1: + logger.info(f"Minimum Vultr {display_name} instances met") + else: + total_deploy = min_scaling - total_instances + logger.info( + f"Deploying: { + str(total_deploy)} Vultr {display_name} instances") + for _ in range(total_deploy): + create_proxy(instance_config) + logger.info(f"Deployed Vultr {display_name} instance") + return len(list_instances(instance_config)) + + +def vultr_check_alive(instance_config=None): + """ + Check if Vultr instances are alive and operational. + + Args: + instance_config: The specific instance configuration + """ + if instance_config is None: + instance_config = config["providers"]["vultr"]["instances"]["default"] + + # Get instance display name for logging + display_name = instance_config.get("display_name", "default") + + ip_ready = [] + for instance in list_instances(instance_config): + try: + # Parse the created_at timestamp to a datetime object + created_at = dateparser.parse(instance.date_created) + if created_at is None: + # If parsing fails but doesn't raise an exception, log and + # continue + logger.info( + f"Pending: Vultr {display_name} allocating (invalid timestamp)") + continue + + # Calculate elapsed time + elapsed = datetime.datetime.now(datetime.timezone.utc) - created_at + + # Check if the instance has reached the age limit + if config["age_limit"] > 0 and elapsed > datetime.timedelta( + seconds=config["age_limit"]): + delete_proxy(instance, instance_config) + logger.info( + f"Recycling Vultr {display_name} instance, reached age limit -> { + str( + instance.ip_address)}" + ) + elif instance.status == "active" and instance.ip_address and check_alive(instance.ip_address): + logger.info( + f"Alive: Vultr {display_name} -> {str(instance.ip_address)}") + ip_ready.append(instance.ip_address) + else: + # Check if the instance has been pending for too long + if elapsed > datetime.timedelta(minutes=10): + delete_proxy(instance, instance_config) + logger.info( + f"Destroyed: took too long Vultr {display_name} -> { + str( + instance.ip_address)}" + ) + else: + logger.info( + f"Waiting: Vultr {display_name} -> {str(instance.ip_address)}") + except TypeError: + # This happens when dateparser.parse raises a TypeError + logger.info(f"Pending: Vultr {display_name} allocating") + return ip_ready + + +def vultr_check_delete(instance_config=None): + """ + Check if any Vultr instances need to be deleted. + + Args: + instance_config: The specific instance configuration + """ + if instance_config is None: + instance_config = config["providers"]["vultr"]["instances"]["default"] + + # Get instance display name for logging + display_name = instance_config.get("display_name", "default") + + # Log current delete queue state + if delete_queue: + logger.info( + f"Current delete queue contains { + len(delete_queue)} IP addresses: { + ', '.join(delete_queue)}") + + instances = list_instances(instance_config) + if not instances: + logger.info( + f"No Vultr {display_name} instances found to process for deletion") + return + + logger.info( + f"Checking { + len(instances)} Vultr {display_name} instances for deletion") + + for instance in instances: + try: + instance_ip = str(instance.ip_address) + + # Check if this instance's IP is in the delete or restart queue + if instance_ip in delete_queue or instance_ip in restart_queue: + logger.info( + f"Found instance { + instance.id} with IP {instance_ip} in deletion queue - deleting now") + + # Attempt to delete the instance + delete_result = delete_proxy(instance, instance_config) + + if delete_result: + logger.info( + f"Successfully destroyed Vultr {display_name} instance -> {instance_ip}") + + # Remove from queues upon successful deletion + if instance_ip in delete_queue: + delete_queue.remove(instance_ip) + logger.info(f"Removed {instance_ip} from delete queue") + if instance_ip in restart_queue: + restart_queue.remove(instance_ip) + logger.info( + f"Removed {instance_ip} from restart queue") + else: + logger.warning( + f"Failed to destroy Vultr {display_name} instance -> {instance_ip}") + except Exception as e: + logger.error(f"Error processing instance for deletion: {e}") + continue + + # Report on any IPs that remain in the queues but weren't found + remaining_delete = [ + ip for ip in delete_queue if any( + ip == str( + i.ip_address) for i in instances)] + if remaining_delete: + logger.warning( + f"IPs remaining in delete queue that weren't found as instances: { + ', '.join(remaining_delete)}") + + +def vultr_fw(instance_config=None): + """ + Create a Vultr firewall for proxy instances. + + Args: + instance_config: The specific instance configuration + """ + if instance_config is None: + instance_config = config["providers"]["vultr"]["instances"]["default"] + + # Get instance name for logging + instance_id = next( + (name for name, inst in config["providers"]["vultr"]["instances"].items() + if inst == instance_config), + "default" + ) + + try: + firewall_id = create_firewall(instance_config) + if firewall_id: + logger.info( + f"Created firewall 'cloudproxy-{instance_id}' with ID: {firewall_id}") + except VultrFirewallExistsException as e: + logger.debug(str(e)) + except Exception as e: + logger.error(f"Error creating firewall: {e}") + + +def vultr_start(instance_config=None): + """ + Start the Vultr provider lifecycle. + + Args: + instance_config: The specific instance configuration + + Returns: + list: List of ready IP addresses + """ + if instance_config is None: + instance_config = config["providers"]["vultr"]["instances"]["default"] + + vultr_fw(instance_config) + vultr_check_delete(instance_config) + # First check which instances are alive + vultr_check_alive(instance_config) + # Then handle deployment/scaling based on ready instances + vultr_deployment( + instance_config["scaling"]["min_scaling"], + instance_config) + # Final check for alive instances + return vultr_check_alive(instance_config) diff --git a/docs/vultr.md b/docs/vultr.md new file mode 100644 index 0000000..3cc391a --- /dev/null +++ b/docs/vultr.md @@ -0,0 +1,250 @@ +# Vultr Configuration + +Configure CloudProxy to use Vultr for creating proxy servers. + +## Quick Start + +```bash +# Run with Docker (recommended) +docker run -d \ + -e PROXY_USERNAME='your_username' \ + -e PROXY_PASSWORD='your_password' \ + -e VULTR_ENABLED=True \ + -e VULTR_API_TOKEN="your-token-here" \ + -e VULTR_REGION="ewr" \ + -p 8000:8000 \ + laffin/cloudproxy:latest + +# Or using an environment file +cat > .env << EOF +PROXY_USERNAME=your_username +PROXY_PASSWORD=your_password +VULTR_ENABLED=True +VULTR_API_TOKEN=your-token-here +VULTR_REGION=ewr +VULTR_PLAN=vc2-1c-1gb +EOF + +docker run -d --env-file .env -p 8000:8000 laffin/cloudproxy:latest +``` + +
+For development (Python) + +```bash +export VULTR_ENABLED=True +export VULTR_API_TOKEN="your-token-here" +python -m cloudproxy +``` +
+ +## Getting Your API Token + +1. Login to your [Vultr account](https://my.vultr.com/) +2. Navigate to Account → [API](https://my.vultr.com/settings/#settingsapi) +3. In the Personal Access Token section, click "Enable API" +4. Your API key will be displayed. Copy this key - it will only be shown once +5. Store this API key securely as your `VULTR_API_TOKEN` + +**Important**: Keep your API token secure and never commit it to version control. + +## Configuration Options + +### Environment Variables + +#### Required +| Variable | Description | Default | +|----------|-------------|---------| +| `VULTR_ENABLED` | Enable Vultr as a provider | `False` | +| `VULTR_API_TOKEN` | Your Vultr API token | Required | + +#### Optional +| Variable | Description | Default | +|----------|-------------|---------| +| `VULTR_REGION` | Region for instance deployment | `ewr` | +| `VULTR_MIN_SCALING` | Target number of proxies to maintain | `2` | +| `VULTR_MAX_SCALING` | Reserved for future autoscaling (currently unused) | `2` | +| `VULTR_PLAN` | Instance plan ID | `vc2-1c-1gb` | +| `VULTR_OS_ID` | Operating System ID | `1743` (Ubuntu 22.04 LTS x64) | + +### Available Regions + +Common Vultr regions include: + +| Region Code | Location | +|------------|----------| +| `ewr` | New Jersey | +| `ord` | Chicago | +| `dfw` | Dallas | +| `sea` | Seattle | +| `lax` | Los Angeles | +| `atl` | Atlanta | +| `ams` | Amsterdam | +| `lhr` | London | +| `fra` | Frankfurt | +| `sjc` | Silicon Valley | +| `syd` | Sydney | +| `sgp` | Singapore | +| `nrt` | Tokyo | +| `icn` | Seoul | +| `mia` | Miami | +| `cdg` | Paris | +| `waw` | Warsaw | +| `mad` | Madrid | +| `sto` | Stockholm | + +For the complete list of regions, use the Vultr API: +```bash +curl "https://api.vultr.com/v2/regions" \ + -H "Authorization: Bearer ${VULTR_API_TOKEN}" +``` + +### Available Plans + +Common Vultr plans for proxy usage: + +| Plan ID | Specifications | Monthly Price* | +|---------|---------------|----------------| +| `vc2-1c-1gb` | 1 vCPU, 1GB RAM, 25GB SSD | ~$6 | +| `vc2-1c-2gb` | 1 vCPU, 2GB RAM, 55GB SSD | ~$12 | +| `vc2-2c-4gb` | 2 vCPU, 4GB RAM, 80GB SSD | ~$24 | +| `vc2-4c-8gb` | 4 vCPU, 8GB RAM, 160GB SSD | ~$48 | + +*Prices are approximate and may vary by region. The smallest plan (vc2-1c-1gb) is usually sufficient for proxy usage. + +For the complete list of plans: +```bash +curl "https://api.vultr.com/v2/plans" \ + -H "Authorization: Bearer ${VULTR_API_TOKEN}" +``` + +## Multi-Account Support + +CloudProxy supports running multiple Vultr accounts simultaneously. Each account is configured as a separate "instance" with its own settings. + +### Default Instance Configuration + +The configuration variables mentioned above configure the "default" instance. For example: + +``` +VULTR_ENABLED=True +VULTR_API_TOKEN=your_default_token +VULTR_REGION=ewr +VULTR_MIN_SCALING=2 +``` + +### Additional Instances Configuration + +To configure additional Vultr accounts, use the following format: +``` +VULTR_INSTANCENAME_VARIABLE=VALUE +``` + +For example, to add a second Vultr account with different region settings: + +``` +# European instance +VULTR_EUROPE_ENABLED=True +VULTR_EUROPE_API_TOKEN=your_second_token +VULTR_EUROPE_REGION=ams +VULTR_EUROPE_MIN_SCALING=3 +VULTR_EUROPE_PLAN=vc2-1c-1gb +VULTR_EUROPE_DISPLAY_NAME=Europe Proxies + +# Asia Pacific instance +VULTR_ASIA_ENABLED=True +VULTR_ASIA_API_TOKEN=your_third_token +VULTR_ASIA_REGION=sgp +VULTR_ASIA_MIN_SCALING=2 +VULTR_ASIA_PLAN=vc2-1c-2gb +VULTR_ASIA_DISPLAY_NAME=Asia Proxies +``` + +### Available Instance-Specific Configurations + +For each instance, you can configure: + +#### Required for each instance: +- `VULTR_INSTANCENAME_ENABLED` - Enable this specific instance +- `VULTR_INSTANCENAME_API_TOKEN` - API token for this instance +- `VULTR_INSTANCENAME_REGION` - Region for this instance + +#### Optional for each instance: +- `VULTR_INSTANCENAME_MIN_SCALING` - Target number of proxies for this instance +- `VULTR_INSTANCENAME_MAX_SCALING` - Reserved for future autoscaling +- `VULTR_INSTANCENAME_PLAN` - Instance plan for this account +- `VULTR_INSTANCENAME_OS_ID` - Operating system ID +- `VULTR_INSTANCENAME_DISPLAY_NAME` - Friendly name shown in the UI + +Each instance operates independently, maintaining its own pool of proxies according to its configuration. + +## Troubleshooting + +### Common Issues + +#### Instances not being created +- Verify your API token is valid and has not been revoked +- Check your account balance and billing status +- Ensure the selected plan is available in your chosen region +- Verify the OS ID is valid (default 1743 for Ubuntu 22.04 LTS) +- Check CloudProxy logs for specific API error messages + +#### Authentication errors +```bash +# Test your API token +curl -X GET \ + -H "Authorization: Bearer YOUR_TOKEN" \ + "https://api.vultr.com/v2/account" +``` + +#### Region-specific issues +- Some regions may have limited availability for certain plans +- Try a different region if instances fail to create +- Check region availability: +```bash +curl "https://api.vultr.com/v2/regions?per_page=100" \ + -H "Authorization: Bearer ${VULTR_API_TOKEN}" +``` + +#### Plan availability +- Not all plans are available in all regions +- The default plan (vc2-1c-1gb) is widely available +- Check plan availability in a specific region: +```bash +curl "https://api.vultr.com/v2/plans?per_page=100&type=vc2" \ + -H "Authorization: Bearer ${VULTR_API_TOKEN}" +``` + +### Firewall Configuration + +CloudProxy automatically creates a firewall group for Vultr instances with the following rules: +- **Inbound**: Port 8899 (TCP) from all sources (proxy port) +- **Outbound**: All traffic allowed + +The firewall group is named `cloudproxy-{instance}` and is automatically applied to all instances. + +### Cost Optimization + +- **Use the smallest plan**: The `vc2-1c-1gb` plan is sufficient for proxy usage +- **Set precise MIN_SCALING**: CloudProxy maintains exactly this number of instances +- **Use AGE_LIMIT**: Rotate proxies regularly to get fresh IPs: + ``` + AGE_LIMIT=3600 # Rotate proxies every hour + ``` +- **Monitor usage**: Check your Vultr dashboard regularly for billing +- **Regional pricing**: Some regions may have different pricing +- **Destroy unused instances**: Set MIN_SCALING=0 to remove all instances + +### API Rate Limits + +Vultr API has rate limits: +- 30 requests per second per IP address +- CloudProxy handles this automatically with built-in retry logic +- If you encounter rate limit errors, they will be logged and retried + +## See Also + +- [API Documentation](api.md) - Complete API reference +- [Multi-Provider Setup](python-package-usage.md#managing-multiple-provider-instances) - Using multiple providers +- [Vultr API Documentation](https://www.vultr.com/api/) - Official Vultr API docs +- [Vultr Instance Types](https://www.vultr.com/products/cloud-compute/) - Current pricing and specifications \ No newline at end of file diff --git a/tests/test_providers_vultr_functions.py b/tests/test_providers_vultr_functions.py new file mode 100644 index 0000000..f152192 --- /dev/null +++ b/tests/test_providers_vultr_functions.py @@ -0,0 +1,316 @@ +import pytest +import requests +from unittest.mock import MagicMock, patch, call +from cloudproxy.providers.vultr.functions import ( + create_proxy, + delete_proxy, + list_instances, + create_firewall, + VultrFirewallExistsException, + VultrInstance, + get_api_headers, +) + + +class TestVultrFunctions: + + @pytest.fixture + def mock_instance_config(self): + return { + "enabled": True, + "ips": [], + "scaling": {"min_scaling": 2, "max_scaling": 5}, + "plan": "vc2-1c-1gb", + "region": "ewr", + "os_id": 387, + "display_name": "Vultr Test", + "secrets": {"api_token": "test-api-token"}, + } + + @pytest.fixture + def mock_settings_config(self, mock_instance_config): + with patch('cloudproxy.providers.vultr.functions.settings') as mock_settings: + mock_settings.config = { + "auth": {"username": "testuser", "password": "testpass"}, + "providers": { + "vultr": { + "instances": { + "default": mock_instance_config, + "test_instance": mock_instance_config + } + } + } + } + yield mock_settings + + def test_get_api_headers(self, mock_instance_config): + headers = get_api_headers(mock_instance_config) + assert headers["Authorization"] == "Bearer test-api-token" + assert headers["Content-Type"] == "application/json" + + def test_vultr_instance_class(self): + data = { + "id": "test-id", + "main_ip": "192.168.1.1", + "label": "test-label", + "date_created": "2024-01-01T00:00:00Z", + "status": "active", + "region": "ewr", + "plan": "vc2-1c-1gb", + "tags": ["test", "cloudproxy"] + } + + instance = VultrInstance(data) + assert instance.id == "test-id" + assert instance.main_ip == "192.168.1.1" + assert instance.ip_address == "192.168.1.1" + assert instance.label == "test-label" + assert instance.status == "active" + assert instance.tags == ["test", "cloudproxy"] + + @patch('cloudproxy.providers.vultr.functions.requests.post') + @patch('cloudproxy.providers.vultr.functions.set_auth') + def test_create_proxy_success(self, mock_set_auth, mock_post, mock_settings_config, mock_instance_config): + # Setup mocks + mock_set_auth.return_value = "user_data_script" + mock_response = MagicMock() + mock_response.status_code = 201 + mock_response.json.return_value = { + "instance": { + "id": "new-instance-id" + } + } + mock_post.return_value = mock_response + + # Call function + result = create_proxy(mock_instance_config) + + # Assertions + assert result is True + mock_set_auth.assert_called_once_with("testuser", "testpass") + mock_post.assert_called_once() + + # Check the payload sent to API + call_args = mock_post.call_args + assert call_args[0][0] == "https://api.vultr.com/v2/instances" + payload = call_args[1]["json"] + assert payload["region"] == "ewr" + assert payload["plan"] == "vc2-1c-1gb" + assert payload["os_id"] == 387 + assert "cloudproxy" in payload["tags"] + + @patch('cloudproxy.providers.vultr.functions.requests.post') + @patch('cloudproxy.providers.vultr.functions.set_auth') + def test_create_proxy_failure(self, mock_set_auth, mock_post, mock_settings_config, mock_instance_config): + # Setup mocks + mock_set_auth.return_value = "user_data_script" + mock_post.side_effect = requests.exceptions.RequestException("API Error") + + # Call function + result = create_proxy(mock_instance_config) + + # Assertions + assert result is False + + @patch('cloudproxy.providers.vultr.functions.requests.delete') + def test_delete_proxy_success(self, mock_delete, mock_instance_config): + # Setup mocks + mock_response = MagicMock() + mock_response.status_code = 204 + mock_delete.return_value = mock_response + + # Test with instance ID string + result = delete_proxy("test-instance-id", mock_instance_config) + assert result is True + mock_delete.assert_called_with( + "https://api.vultr.com/v2/instances/test-instance-id", + headers=get_api_headers(mock_instance_config) + ) + + @patch('cloudproxy.providers.vultr.functions.requests.delete') + def test_delete_proxy_with_instance_object(self, mock_delete, mock_instance_config): + # Setup mocks + mock_response = MagicMock() + mock_response.status_code = 204 + mock_delete.return_value = mock_response + + # Test with VultrInstance object + instance = VultrInstance({"id": "test-instance-id", "main_ip": "192.168.1.1"}) + result = delete_proxy(instance, mock_instance_config) + assert result is True + + @patch('cloudproxy.providers.vultr.functions.requests.delete') + def test_delete_proxy_not_found(self, mock_delete, mock_instance_config): + # Setup mocks + mock_response = MagicMock() + mock_response.status_code = 404 + mock_delete.return_value = mock_response + + # Call function + result = delete_proxy("non-existent-id", mock_instance_config) + + # Should return True even if not found + assert result is True + + @patch('cloudproxy.providers.vultr.functions.requests.get') + def test_list_instances_success(self, mock_get, mock_settings_config, mock_instance_config): + # Setup mocks + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = { + "instances": [ + { + "id": "instance-1", + "main_ip": "192.168.1.1", + "tags": ["cloudproxy", "cloudproxy-default"] + }, + { + "id": "instance-2", + "main_ip": "192.168.1.2", + "tags": ["cloudproxy", "cloudproxy-default"] + } + ] + } + mock_get.return_value = mock_response + + # Call function + instances = list_instances(mock_instance_config) + + # Assertions + assert len(instances) == 2 + assert all(isinstance(inst, VultrInstance) for inst in instances) + assert instances[0].id == "instance-1" + assert instances[1].id == "instance-2" + + @patch('cloudproxy.providers.vultr.functions.requests.get') + def test_list_instances_with_old_tags(self, mock_get, mock_settings_config, mock_instance_config): + # Setup mocks for default instance checking old tags + mock_response1 = MagicMock() + mock_response1.status_code = 200 + mock_response1.json.return_value = { + "instances": [ + { + "id": "instance-1", + "main_ip": "192.168.1.1", + "tags": ["cloudproxy", "cloudproxy-default"] + } + ] + } + + mock_response2 = MagicMock() + mock_response2.status_code = 200 + mock_response2.json.return_value = { + "instances": [ + { + "id": "instance-1", + "main_ip": "192.168.1.1", + "tags": ["cloudproxy", "cloudproxy-default"] + }, + { + "id": "instance-2", + "main_ip": "192.168.1.2", + "tags": ["cloudproxy"] # Old tag format + } + ] + } + + mock_get.side_effect = [mock_response1, mock_response2] + + # Call function + instances = list_instances(mock_instance_config) + + # Should include the old-tagged instance + assert len(instances) == 2 + + @patch('cloudproxy.providers.vultr.functions.requests.get') + def test_list_instances_failure(self, mock_get, mock_settings_config, mock_instance_config): + # Setup mocks + mock_get.side_effect = requests.exceptions.RequestException("API Error") + + # Call function + instances = list_instances(mock_instance_config) + + # Should return empty list on error + assert instances == [] + + @patch('cloudproxy.providers.vultr.functions._create_firewall_rules') + @patch('cloudproxy.providers.vultr.functions.requests.post') + @patch('cloudproxy.providers.vultr.functions.requests.get') + def test_create_firewall_success(self, mock_get, mock_post, mock_create_rules, + mock_settings_config, mock_instance_config): + # Setup mocks - no existing firewall + mock_get_response = MagicMock() + mock_get_response.status_code = 200 + mock_get_response.json.return_value = {"firewall_groups": []} + mock_get.return_value = mock_get_response + + # Setup create firewall response + mock_post_response = MagicMock() + mock_post_response.status_code = 201 + mock_post_response.json.return_value = { + "firewall_group": { + "id": "new-firewall-id" + } + } + mock_post.return_value = mock_post_response + + # Call function + firewall_id = create_firewall(mock_instance_config) + + # Assertions + assert firewall_id == "new-firewall-id" + assert mock_instance_config["firewall_group_id"] == "new-firewall-id" + mock_create_rules.assert_called_once_with("new-firewall-id", mock_instance_config) + + @patch('cloudproxy.providers.vultr.functions.requests.get') + def test_create_firewall_already_exists(self, mock_get, mock_settings_config, mock_instance_config): + # Setup mocks - firewall already exists + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = { + "firewall_groups": [ + { + "id": "existing-firewall-id", + "description": "cloudproxy-default" + } + ] + } + mock_get.return_value = mock_response + + # Call function and expect exception + with pytest.raises(VultrFirewallExistsException): + create_firewall(mock_instance_config) + + # Should still store the firewall ID + assert mock_instance_config["firewall_group_id"] == "existing-firewall-id" + + @patch('cloudproxy.providers.vultr.functions.requests.post') + def test_create_firewall_rules(self, mock_post, mock_instance_config): + from cloudproxy.providers.vultr.functions import _create_firewall_rules + + # Setup mocks + mock_response = MagicMock() + mock_response.status_code = 201 + mock_post.return_value = mock_response + + # Call function + _create_firewall_rules("test-firewall-id", mock_instance_config) + + # Should have made 3 calls for 3 rules + assert mock_post.call_count == 3 + + # Check that proper rules were created + calls = mock_post.call_args_list + + # First rule - inbound port 8899 + assert calls[0][0][0] == "https://api.vultr.com/v2/firewalls/test-firewall-id/rules" + assert calls[0][1]["json"]["port"] == "8899" + assert calls[0][1]["json"]["protocol"] == "tcp" + + # Second rule - outbound TCP + assert calls[1][1]["json"]["port"] == "1:65535" + assert calls[1][1]["json"]["protocol"] == "tcp" + + # Third rule - outbound UDP + assert calls[2][1]["json"]["port"] == "1:65535" + assert calls[2][1]["json"]["protocol"] == "udp" \ No newline at end of file diff --git a/tests/test_providers_vultr_main.py b/tests/test_providers_vultr_main.py new file mode 100644 index 0000000..e616105 --- /dev/null +++ b/tests/test_providers_vultr_main.py @@ -0,0 +1,320 @@ +import pytest +import datetime +from unittest.mock import MagicMock, patch, call +from cloudproxy.providers.vultr.main import ( + vultr_deployment, + vultr_check_alive, + vultr_check_delete, + vultr_fw, + vultr_start, +) +from cloudproxy.providers.vultr.functions import VultrInstance, VultrFirewallExistsException + + +class TestVultrMain: + + @pytest.fixture + def mock_instance_config(self): + return { + "enabled": True, + "ips": [], + "scaling": {"min_scaling": 2, "max_scaling": 5}, + "plan": "vc2-1c-1gb", + "region": "ewr", + "os_id": 387, + "display_name": "Vultr Test", + "secrets": {"api_token": "test-api-token"}, + } + + @pytest.fixture + def mock_config(self, mock_instance_config): + with patch('cloudproxy.providers.vultr.main.config') as mock_cfg: + mock_cfg.__getitem__.side_effect = lambda x: { + "providers": { + "vultr": { + "instances": { + "default": mock_instance_config + } + } + }, + "age_limit": 3600 # 1 hour + }.get(x, {}) + yield mock_cfg + + @pytest.fixture + def mock_instances(self): + """Create mock Vultr instances.""" + instances = [] + for i in range(3): + data = { + "id": f"instance-{i}", + "main_ip": f"192.168.1.{i+1}", + "label": f"cloudproxy-default-{i}", + "date_created": "2024-01-01T00:00:00Z", + "status": "active", + "tags": ["cloudproxy", "cloudproxy-default"] + } + instances.append(VultrInstance(data)) + return instances + + @patch('cloudproxy.providers.vultr.main.create_proxy') + @patch('cloudproxy.providers.vultr.main.delete_proxy') + @patch('cloudproxy.providers.vultr.main.list_instances') + def test_vultr_deployment_scale_up(self, mock_list, mock_delete, mock_create, + mock_config, mock_instance_config): + # Setup - currently have 1 instance, need 3 + mock_list.return_value = [VultrInstance({"id": "existing", "main_ip": "192.168.1.1"})] + + # Call function + result = vultr_deployment(3, mock_instance_config) + + # Should create 2 new instances + assert mock_create.call_count == 2 + assert mock_delete.call_count == 0 + assert result == 1 # Returns current count + + @patch('cloudproxy.providers.vultr.main.create_proxy') + @patch('cloudproxy.providers.vultr.main.delete_proxy') + @patch('cloudproxy.providers.vultr.main.list_instances') + def test_vultr_deployment_scale_down(self, mock_list, mock_delete, mock_create, + mock_config, mock_instance_config, mock_instances): + # Setup - currently have 3 instances, need 1 + mock_list.return_value = mock_instances + + # Call function + result = vultr_deployment(1, mock_instance_config) + + # Should delete 2 instances + assert mock_create.call_count == 0 + assert mock_delete.call_count == 2 + assert result == 3 # Returns current count + + @patch('cloudproxy.providers.vultr.main.create_proxy') + @patch('cloudproxy.providers.vultr.main.delete_proxy') + @patch('cloudproxy.providers.vultr.main.list_instances') + def test_vultr_deployment_no_change(self, mock_list, mock_delete, mock_create, + mock_config, mock_instance_config, mock_instances): + # Setup - currently have 3 instances, need 3 + mock_list.return_value = mock_instances[:3] + + # Call function + result = vultr_deployment(3, mock_instance_config) + + # Should not create or delete + assert mock_create.call_count == 0 + assert mock_delete.call_count == 0 + assert result == 3 + + @patch('cloudproxy.providers.vultr.main.check_alive') + @patch('cloudproxy.providers.vultr.main.delete_proxy') + @patch('cloudproxy.providers.vultr.main.list_instances') + @patch('cloudproxy.providers.vultr.main.dateparser.parse') + def test_vultr_check_alive_active_instances(self, mock_parse, mock_list, mock_delete, + mock_check_alive, mock_config, mock_instance_config): + # Setup + mock_instances = [ + VultrInstance({ + "id": "active-1", + "main_ip": "192.168.1.1", + "status": "active", + "date_created": "2024-01-01T00:00:00Z" + }), + VultrInstance({ + "id": "active-2", + "main_ip": "192.168.1.2", + "status": "active", + "date_created": "2024-01-01T00:00:00Z" + }) + ] + mock_list.return_value = mock_instances + mock_parse.return_value = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(minutes=5) + mock_check_alive.side_effect = [True, True] # Both instances are alive + + # Call function + result = vultr_check_alive(mock_instance_config) + + # Assertions + assert len(result) == 2 + assert "192.168.1.1" in result + assert "192.168.1.2" in result + assert mock_delete.call_count == 0 + + @patch('cloudproxy.providers.vultr.main.check_alive') + @patch('cloudproxy.providers.vultr.main.delete_proxy') + @patch('cloudproxy.providers.vultr.main.list_instances') + @patch('cloudproxy.providers.vultr.main.dateparser.parse') + def test_vultr_check_alive_age_limit(self, mock_parse, mock_list, mock_delete, + mock_check_alive, mock_config, mock_instance_config): + # Setup - instance is older than age limit + mock_instance = VultrInstance({ + "id": "old-instance", + "main_ip": "192.168.1.1", + "status": "active", + "date_created": "2024-01-01T00:00:00Z" + }) + mock_list.return_value = [mock_instance] + # Instance created 2 hours ago, age limit is 1 hour + mock_parse.return_value = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(hours=2) + mock_check_alive.return_value = True + + # Call function + result = vultr_check_alive(mock_instance_config) + + # Should delete the old instance + mock_delete.assert_called_once_with(mock_instance, mock_instance_config) + assert len(result) == 0 + + @patch('cloudproxy.providers.vultr.main.check_alive') + @patch('cloudproxy.providers.vultr.main.delete_proxy') + @patch('cloudproxy.providers.vultr.main.list_instances') + @patch('cloudproxy.providers.vultr.main.dateparser.parse') + def test_vultr_check_alive_pending_too_long(self, mock_parse, mock_list, mock_delete, + mock_check_alive, mock_config, mock_instance_config): + # Setup - instance pending for too long + mock_instance = VultrInstance({ + "id": "pending-instance", + "main_ip": "192.168.1.1", + "status": "pending", + "date_created": "2024-01-01T00:00:00Z" + }) + mock_list.return_value = [mock_instance] + # Instance created 15 minutes ago (too long for pending) + mock_parse.return_value = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(minutes=15) + mock_check_alive.return_value = False + + # Call function + result = vultr_check_alive(mock_instance_config) + + # Should delete the pending instance + mock_delete.assert_called_once_with(mock_instance, mock_instance_config) + assert len(result) == 0 + + @patch('cloudproxy.providers.vultr.main.delete_proxy') + @patch('cloudproxy.providers.vultr.main.list_instances') + @patch('cloudproxy.providers.vultr.main.delete_queue') + @patch('cloudproxy.providers.vultr.main.restart_queue') + def test_vultr_check_delete_with_queue(self, mock_restart_queue, mock_delete_queue, + mock_list, mock_delete, mock_config, mock_instance_config): + # Setup + mock_instances = [ + VultrInstance({"id": "instance-1", "main_ip": "192.168.1.1"}), + VultrInstance({"id": "instance-2", "main_ip": "192.168.1.2"}), + VultrInstance({"id": "instance-3", "main_ip": "192.168.1.3"}) + ] + mock_list.return_value = mock_instances + + # Use sets with the required IPs + mock_delete_queue.__class__ = set + mock_delete_queue.__iter__ = lambda self: iter(["192.168.1.1"]) + mock_delete_queue.__contains__ = lambda self, x: x == "192.168.1.1" + mock_delete_queue.__len__ = lambda self: 1 + mock_delete_queue.remove = MagicMock() + + mock_restart_queue.__class__ = set + mock_restart_queue.__iter__ = lambda self: iter(["192.168.1.2"]) + mock_restart_queue.__contains__ = lambda self, x: x == "192.168.1.2" + mock_restart_queue.__len__ = lambda self: 1 + mock_restart_queue.remove = MagicMock() + + mock_delete.return_value = True + + # Call function + vultr_check_delete(mock_instance_config) + + # Should delete instances in queues + assert mock_delete.call_count == 2 + mock_delete_queue.remove.assert_called_once_with("192.168.1.1") + mock_restart_queue.remove.assert_called_once_with("192.168.1.2") + + @patch('cloudproxy.providers.vultr.main.delete_proxy') + @patch('cloudproxy.providers.vultr.main.list_instances') + def test_vultr_check_delete_no_instances(self, mock_list, mock_delete, + mock_config, mock_instance_config): + # Setup - no instances + mock_list.return_value = [] + + # Call function + vultr_check_delete(mock_instance_config) + + # Should not attempt any deletions + assert mock_delete.call_count == 0 + + @patch('cloudproxy.providers.vultr.main.create_firewall') + def test_vultr_fw_success(self, mock_create_fw, mock_config, mock_instance_config): + # Setup + mock_create_fw.return_value = "firewall-id-123" + + # Call function + vultr_fw(mock_instance_config) + + # Assertions + mock_create_fw.assert_called_once_with(mock_instance_config) + + @patch('cloudproxy.providers.vultr.main.create_firewall') + def test_vultr_fw_already_exists(self, mock_create_fw, mock_config, mock_instance_config): + # Setup + mock_create_fw.side_effect = VultrFirewallExistsException("Firewall exists") + + # Call function - should not raise + vultr_fw(mock_instance_config) + + # Assertions + mock_create_fw.assert_called_once_with(mock_instance_config) + + @patch('cloudproxy.providers.vultr.main.vultr_check_alive') + @patch('cloudproxy.providers.vultr.main.vultr_deployment') + @patch('cloudproxy.providers.vultr.main.vultr_check_delete') + @patch('cloudproxy.providers.vultr.main.vultr_fw') + def test_vultr_start(self, mock_fw, mock_check_delete, mock_deployment, + mock_check_alive, mock_config, mock_instance_config): + # Setup + mock_check_alive.side_effect = [ + ["192.168.1.1"], # First check returns 1 IP + ["192.168.1.1", "192.168.1.2"] # Second check returns 2 IPs + ] + + # Call function + result = vultr_start(mock_instance_config) + + # Assertions + mock_fw.assert_called_once_with(mock_instance_config) + mock_check_delete.assert_called_once_with(mock_instance_config) + mock_deployment.assert_called_once_with( + mock_instance_config["scaling"]["min_scaling"], + mock_instance_config + ) + assert mock_check_alive.call_count == 2 + assert result == ["192.168.1.1", "192.168.1.2"] + + def test_vultr_start_with_default_config(self): + """Test that vultr_start works with default config.""" + with patch('cloudproxy.providers.vultr.main.config') as mock_config: + with patch('cloudproxy.providers.vultr.main.vultr_fw') as mock_fw: + with patch('cloudproxy.providers.vultr.main.vultr_check_delete') as mock_check_delete: + with patch('cloudproxy.providers.vultr.main.vultr_check_alive') as mock_check_alive: + with patch('cloudproxy.providers.vultr.main.vultr_deployment') as mock_deployment: + + # Setup + default_config = { + "scaling": {"min_scaling": 2}, + "display_name": "Default" + } + mock_config.__getitem__.side_effect = lambda x: { + "providers": { + "vultr": { + "instances": { + "default": default_config + } + } + } + }.get(x, {}) + mock_check_alive.return_value = [] + + # Call without config argument + result = vultr_start() + + # Should use default config + mock_fw.assert_called_once() + mock_check_delete.assert_called_once() + mock_deployment.assert_called_once() + assert mock_check_alive.call_count == 2 \ No newline at end of file From af98e01c7fa54cb81d1a891a7ac67ec0a240c055 Mon Sep 17 00:00:00 2001 From: Christian Laffin Date: Thu, 14 Aug 2025 21:26:47 +0100 Subject: [PATCH 2/2] fix: resolve syntax errors in Vultr provider f-strings Fixed multi-line f-string formatting that was broken by autopep8 auto-formatting. The tool incorrectly split f-strings across multiple lines, causing SyntaxError in Python. Corrected all instances in vultr/main.py and vultr/functions.py to use proper single-line f-string expressions. --- cloudproxy/providers/vultr/functions.py | 5 +---- cloudproxy/providers/vultr/main.py | 24 +++++++----------------- 2 files changed, 8 insertions(+), 21 deletions(-) diff --git a/cloudproxy/providers/vultr/functions.py b/cloudproxy/providers/vultr/functions.py index 1d79f69..95db23c 100644 --- a/cloudproxy/providers/vultr/functions.py +++ b/cloudproxy/providers/vultr/functions.py @@ -105,10 +105,7 @@ def create_proxy(instance_config: Optional[Dict] = None) -> bool: data = response.json() logger.info( - f"Created Vultr instance: { - data.get( - 'instance', - {}).get('id')}") + f"Created Vultr instance: {data.get('instance', {}).get('id')}") return True except requests.exceptions.RequestException as e: diff --git a/cloudproxy/providers/vultr/main.py b/cloudproxy/providers/vultr/main.py index f26a5bc..cf76a02 100644 --- a/cloudproxy/providers/vultr/main.py +++ b/cloudproxy/providers/vultr/main.py @@ -43,8 +43,7 @@ def vultr_deployment(min_scaling, instance_config=None): else: total_deploy = min_scaling - total_instances logger.info( - f"Deploying: { - str(total_deploy)} Vultr {display_name} instances") + f"Deploying: {str(total_deploy)} Vultr {display_name} instances") for _ in range(total_deploy): create_proxy(instance_config) logger.info(f"Deployed Vultr {display_name} instance") @@ -84,9 +83,7 @@ def vultr_check_alive(instance_config=None): seconds=config["age_limit"]): delete_proxy(instance, instance_config) logger.info( - f"Recycling Vultr {display_name} instance, reached age limit -> { - str( - instance.ip_address)}" + f"Recycling Vultr {display_name} instance, reached age limit -> {str(instance.ip_address)}" ) elif instance.status == "active" and instance.ip_address and check_alive(instance.ip_address): logger.info( @@ -97,9 +94,7 @@ def vultr_check_alive(instance_config=None): if elapsed > datetime.timedelta(minutes=10): delete_proxy(instance, instance_config) logger.info( - f"Destroyed: took too long Vultr {display_name} -> { - str( - instance.ip_address)}" + f"Destroyed: took too long Vultr {display_name} -> {str(instance.ip_address)}" ) else: logger.info( @@ -126,9 +121,7 @@ def vultr_check_delete(instance_config=None): # Log current delete queue state if delete_queue: logger.info( - f"Current delete queue contains { - len(delete_queue)} IP addresses: { - ', '.join(delete_queue)}") + f"Current delete queue contains {len(delete_queue)} IP addresses: {', '.join(delete_queue)}") instances = list_instances(instance_config) if not instances: @@ -137,8 +130,7 @@ def vultr_check_delete(instance_config=None): return logger.info( - f"Checking { - len(instances)} Vultr {display_name} instances for deletion") + f"Checking {len(instances)} Vultr {display_name} instances for deletion") for instance in instances: try: @@ -147,8 +139,7 @@ def vultr_check_delete(instance_config=None): # Check if this instance's IP is in the delete or restart queue if instance_ip in delete_queue or instance_ip in restart_queue: logger.info( - f"Found instance { - instance.id} with IP {instance_ip} in deletion queue - deleting now") + f"Found instance {instance.id} with IP {instance_ip} in deletion queue - deleting now") # Attempt to delete the instance delete_result = delete_proxy(instance, instance_config) @@ -179,8 +170,7 @@ def vultr_check_delete(instance_config=None): i.ip_address) for i in instances)] if remaining_delete: logger.warning( - f"IPs remaining in delete queue that weren't found as instances: { - ', '.join(remaining_delete)}") + f"IPs remaining in delete queue that weren't found as instances: {', '.join(remaining_delete)}") def vultr_fw(instance_config=None):