diff --git a/.env.example b/.env.example index db68a27..b0fec2f 100644 --- a/.env.example +++ b/.env.example @@ -14,6 +14,18 @@ PROXY_PASSWORD=changeme # Optional: Proxy age limit in seconds (0 = disabled) # AGE_LIMIT=0 +# ==================== +# Rolling Deployment Settings +# ==================== +# Optional: Enable rolling deployments for zero-downtime proxy recycling +# ROLLING_DEPLOYMENT=False + +# Optional: Minimum number of proxies to keep available during recycling +# ROLLING_MIN_AVAILABLE=3 + +# Optional: Maximum number of proxies to recycle simultaneously +# ROLLING_BATCH_SIZE=2 + # ==================== # DigitalOcean Provider # ==================== diff --git a/README.md b/README.md index f03a853..2d32e3d 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,7 @@ - [Web Interface](#web-interface) - [API Documentation](#api-documentation) - [Programmatic Usage](#programmatic-usage) +- [Rolling Deployments](#rolling-deployments) - [Multi-Account Provider Support](#multi-account-provider-support) - [API Examples](#cloudproxy-api-examples) - [Roadmap](#roadmap) @@ -85,6 +86,7 @@ CloudProxy exposes an API and modern UI for managing your proxy infrastructure. * Multi-provider support * Multiple accounts per provider * Automatic proxy rotation +* **Rolling deployments** - Zero-downtime proxy recycling * Health monitoring * Fixed proxy pool management (maintains target count) @@ -293,6 +295,47 @@ my_request = requests.get("https://api.ipify.org", proxies=proxies) For more detailed examples of using CloudProxy as a Python package, see the [Python Package Usage Guide](docs/python-package-usage.md). +## Rolling Deployments + +CloudProxy supports rolling deployments to ensure zero-downtime proxy recycling. This feature maintains a minimum number of healthy proxies during age-based recycling operations. + +### Configuration + +Enable rolling deployments with these environment variables: + +```bash +# Enable rolling deployments +ROLLING_DEPLOYMENT=True + +# Minimum proxies to keep available during recycling +ROLLING_MIN_AVAILABLE=3 + +# Maximum proxies to recycle simultaneously +ROLLING_BATCH_SIZE=2 +``` + +### How It Works + +When proxies reach their age limit: +1. The system checks if recycling would violate minimum availability +2. Proxies are recycled in batches to maintain service continuity +3. New proxies are created as old ones are removed +4. The process continues until all aged proxies are replaced + +### Monitoring + +Check rolling deployment status via the API: + +```bash +# Get overall status +curl http://localhost:8000/rolling + +# Get provider-specific status +curl http://localhost:8000/rolling/digitalocean +``` + +For detailed documentation, see the [Rolling Deployments Guide](docs/rolling-deployments.md). + ## Multi-Account Provider Support CloudProxy now supports multiple accounts per provider, allowing you to: diff --git a/cloudproxy-ui/src/App.vue b/cloudproxy-ui/src/App.vue index ec2005e..f0f4b5b 100644 --- a/cloudproxy-ui/src/App.vue +++ b/cloudproxy-ui/src/App.vue @@ -35,6 +35,7 @@
+
@@ -46,11 +47,13 @@ + + \ No newline at end of file diff --git a/cloudproxy/main.py b/cloudproxy/main.py index b06fe87..ab590be 100644 --- a/cloudproxy/main.py +++ b/cloudproxy/main.py @@ -19,6 +19,7 @@ from cloudproxy.providers import settings from cloudproxy.providers.settings import delete_queue, restart_queue +from cloudproxy.providers.rolling import rolling_manager sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) @@ -726,6 +727,162 @@ def configure_instance( config=ProviderInstance(**instance_config) ) +# Rolling Deployment Models +class RollingDeploymentConfig(BaseModel): + enabled: bool = Field(description="Whether rolling deployment is enabled") + min_available: int = Field(ge=0, description="Minimum number of proxies to keep available during recycling") + batch_size: int = Field(ge=1, description="Maximum number of proxies to recycle simultaneously") + +class RollingDeploymentStatus(BaseModel): + healthy: int = Field(description="Number of healthy proxies") + pending: int = Field(description="Number of pending proxies") + pending_recycle: int = Field(description="Number of proxies pending recycling") + recycling: int = Field(description="Number of proxies currently being recycled") + last_update: str = Field(description="Last update timestamp") + healthy_ips: List[str] = Field(description="List of healthy proxy IPs") + pending_recycle_ips: List[str] = Field(description="List of IPs pending recycling") + recycling_ips: List[str] = Field(description="List of IPs currently being recycled") + +class RollingDeploymentResponse(BaseModel): + metadata: Metadata = Field(default_factory=Metadata) + message: str + config: RollingDeploymentConfig + status: Dict[str, RollingDeploymentStatus] = Field(description="Status by provider/instance") + +@app.get("/rolling", tags=["Rolling Deployment"], response_model=RollingDeploymentResponse) +def get_rolling_deployment_status(): + """ + Get the current rolling deployment configuration and status. + + Returns: + RollingDeploymentResponse: Current rolling deployment configuration and status + """ + config = RollingDeploymentConfig( + enabled=settings.config["rolling_deployment"]["enabled"], + min_available=settings.config["rolling_deployment"]["min_available"], + batch_size=settings.config["rolling_deployment"]["batch_size"] + ) + + raw_status = rolling_manager.get_recycling_status() + status = {} + for key, data in raw_status.items(): + status[key] = RollingDeploymentStatus(**data) + + return RollingDeploymentResponse( + message="Rolling deployment status retrieved successfully", + config=config, + status=status + ) + +@app.patch("/rolling", tags=["Rolling Deployment"], response_model=RollingDeploymentResponse) +def update_rolling_deployment_config(update: RollingDeploymentConfig): + """ + Update the rolling deployment configuration. + + Args: + update: New rolling deployment configuration + + Returns: + RollingDeploymentResponse: Updated configuration and current status + """ + # Update configuration + settings.config["rolling_deployment"]["enabled"] = update.enabled + settings.config["rolling_deployment"]["min_available"] = update.min_available + settings.config["rolling_deployment"]["batch_size"] = update.batch_size + + # Get current status + raw_status = rolling_manager.get_recycling_status() + status = {} + for key, data in raw_status.items(): + status[key] = RollingDeploymentStatus(**data) + + return RollingDeploymentResponse( + message="Rolling deployment configuration updated successfully", + config=update, + status=status + ) + +@app.get("/rolling/{provider}", tags=["Rolling Deployment"], response_model=RollingDeploymentResponse) +def get_provider_rolling_status(provider: str): + """ + Get rolling deployment status for a specific provider. + + Args: + provider: The name of the provider + + Returns: + RollingDeploymentResponse: Rolling deployment status for the provider + + Raises: + HTTPException: If the provider is not found + """ + if provider not in settings.config["providers"]: + raise HTTPException( + status_code=404, + detail=f"Provider '{provider}' not found" + ) + + config = RollingDeploymentConfig( + enabled=settings.config["rolling_deployment"]["enabled"], + min_available=settings.config["rolling_deployment"]["min_available"], + batch_size=settings.config["rolling_deployment"]["batch_size"] + ) + + raw_status = rolling_manager.get_recycling_status(provider=provider) + status = {} + for key, data in raw_status.items(): + status[key] = RollingDeploymentStatus(**data) + + return RollingDeploymentResponse( + message=f"Rolling deployment status for '{provider}' retrieved successfully", + config=config, + status=status + ) + +@app.get("/rolling/{provider}/{instance}", tags=["Rolling Deployment"], response_model=RollingDeploymentResponse) +def get_instance_rolling_status(provider: str, instance: str): + """ + Get rolling deployment status for a specific provider instance. + + Args: + provider: The name of the provider + instance: The name of the instance + + Returns: + RollingDeploymentResponse: Rolling deployment status for the instance + + Raises: + HTTPException: If the provider or instance is not found + """ + if provider not in settings.config["providers"]: + raise HTTPException( + status_code=404, + detail=f"Provider '{provider}' not found" + ) + + if instance not in settings.config["providers"][provider]["instances"]: + raise HTTPException( + status_code=404, + detail=f"Provider '{provider}' instance '{instance}' not found" + ) + + config = RollingDeploymentConfig( + enabled=settings.config["rolling_deployment"]["enabled"], + min_available=settings.config["rolling_deployment"]["min_available"], + batch_size=settings.config["rolling_deployment"]["batch_size"] + ) + + raw_status = rolling_manager.get_recycling_status(provider=provider, instance=instance) + status = {} + for key, data in raw_status.items(): + status[key] = RollingDeploymentStatus(**data) + + return RollingDeploymentResponse( + message=f"Rolling deployment status for '{provider}/{instance}' retrieved successfully", + config=config, + status=status + ) + if __name__ == "__main__": main() diff --git a/cloudproxy/providers/aws/main.py b/cloudproxy/providers/aws/main.py index f122464..8c19e07 100644 --- a/cloudproxy/providers/aws/main.py +++ b/cloudproxy/providers/aws/main.py @@ -12,6 +12,7 @@ start_proxy, ) from cloudproxy.providers.settings import delete_queue, restart_queue, config +from cloudproxy.providers.rolling import rolling_manager def aws_deployment(min_scaling, instance_config=None): @@ -58,18 +59,28 @@ def aws_check_alive(instance_config=None): """ if instance_config is None: instance_config = config["providers"]["aws"]["instances"]["default"] + + # Get instance name for rolling deployment tracking + instance_name = next( + (name for name, inst in config["providers"]["aws"]["instances"].items() + if inst == instance_config), + "default" + ) ip_ready = [] + pending_ips = [] + instances_to_recycle = [] + + # First pass: identify healthy and pending instances for instance in list_instances(instance_config): try: elapsed = datetime.datetime.now( datetime.timezone.utc ) - instance["Instances"][0]["LaunchTime"] + if config["age_limit"] > 0 and elapsed > datetime.timedelta(seconds=config["age_limit"]): - delete_proxy(instance["Instances"][0]["InstanceId"], instance_config) - logger.info( - f"Recycling AWS {instance_config.get('display_name', 'default')} instance, reached age limit -> " + instance["Instances"][0]["PublicIpAddress"] - ) + # Queue for potential recycling + instances_to_recycle.append((instance, elapsed)) elif instance["Instances"][0]["State"]["Name"] == "stopped": logger.info( f"Waking up: AWS {instance_config.get('display_name', 'default')} -> Instance " + instance["Instances"][0]["InstanceId"] @@ -87,7 +98,9 @@ def aws_check_alive(instance_config=None): logger.info( f"Pending: AWS {instance_config.get('display_name', 'default')} -> " + instance["Instances"][0]["PublicIpAddress"] ) - # Must be "pending" if none of the above, check if alive or not. + if "PublicIpAddress" in instance["Instances"][0]: + pending_ips.append(instance["Instances"][0]["PublicIpAddress"]) + # Must be "running" if none of the above, check if alive or not. elif check_alive(instance["Instances"][0]["PublicIpAddress"]): logger.info( f"Alive: AWS {instance_config.get('display_name', 'default')} -> " + instance["Instances"][0]["PublicIpAddress"] @@ -104,8 +117,57 @@ def aws_check_alive(instance_config=None): logger.info( f"Waiting: AWS {instance_config.get('display_name', 'default')} -> " + instance["Instances"][0]["PublicIpAddress"] ) + if "PublicIpAddress" in instance["Instances"][0]: + pending_ips.append(instance["Instances"][0]["PublicIpAddress"]) except (TypeError, KeyError): logger.info(f"Pending: AWS {instance_config.get('display_name', 'default')} -> allocating ip") + + # Update rolling manager with current proxy health status + rolling_manager.update_proxy_health("aws", instance_name, ip_ready, pending_ips) + + # Handle rolling deployments for age-limited instances + if instances_to_recycle and config["rolling_deployment"]["enabled"]: + rolling_config = config["rolling_deployment"] + + for inst, elapsed in instances_to_recycle: + if "PublicIpAddress" in inst["Instances"][0]: + instance_ip = inst["Instances"][0]["PublicIpAddress"] + + # Check if we can recycle this instance according to rolling deployment rules + if rolling_manager.can_recycle_proxy( + provider="aws", + instance=instance_name, + proxy_ip=instance_ip, + total_healthy=len(ip_ready), + min_available=rolling_config["min_available"], + batch_size=rolling_config["batch_size"], + rolling_enabled=True, + min_scaling=instance_config["scaling"]["min_scaling"] + ): + # Mark as recycling and delete + rolling_manager.mark_proxy_recycling("aws", instance_name, instance_ip) + delete_proxy(inst["Instances"][0]["InstanceId"], instance_config) + rolling_manager.mark_proxy_recycled("aws", instance_name, instance_ip) + logger.info( + f"Rolling deployment: Recycled AWS {instance_config.get('display_name', 'default')} instance (age limit) -> {instance_ip}" + ) + else: + logger.info( + f"Rolling deployment: Deferred recycling AWS {instance_config.get('display_name', 'default')} instance -> {instance_ip}" + ) + elif instances_to_recycle and not config["rolling_deployment"]["enabled"]: + # Standard non-rolling recycling + for inst, elapsed in instances_to_recycle: + delete_proxy(inst["Instances"][0]["InstanceId"], instance_config) + if "PublicIpAddress" in inst["Instances"][0]: + logger.info( + f"Recycling AWS {instance_config.get('display_name', 'default')} instance, reached age limit -> " + inst["Instances"][0]["PublicIpAddress"] + ) + else: + logger.info( + f"Recycling AWS {instance_config.get('display_name', 'default')} instance, reached age limit -> " + inst["Instances"][0]["InstanceId"] + ) + return ip_ready diff --git a/cloudproxy/providers/config.py b/cloudproxy/providers/config.py index 2d01e8f..5005921 100644 --- a/cloudproxy/providers/config.py +++ b/cloudproxy/providers/config.py @@ -26,5 +26,8 @@ def set_auth(username, password): filedata = filedata.replace("sudo ufw allow 8899/tcp", f"sudo ufw allow from {ip_address} to any port 8899 proto tcp") # Update tinyproxy access rule filedata = filedata.replace("Allow 127.0.0.1", f"Allow 127.0.0.1\nAllow {ip_address}") + else: + # When ONLY_HOST_IP is False, allow connections from any IP + filedata = filedata.replace("Allow 127.0.0.1", "Allow 0.0.0.0/0") return filedata diff --git a/cloudproxy/providers/digitalocean/main.py b/cloudproxy/providers/digitalocean/main.py index f7ceea2..621c7b4 100644 --- a/cloudproxy/providers/digitalocean/main.py +++ b/cloudproxy/providers/digitalocean/main.py @@ -14,6 +14,7 @@ ) from cloudproxy.providers import settings from cloudproxy.providers.settings import delete_queue, restart_queue, config +from cloudproxy.providers.rolling import rolling_manager def do_deployment(min_scaling, instance_config=None): @@ -61,7 +62,18 @@ def do_check_alive(instance_config=None): # Get instance display name for logging display_name = instance_config.get("display_name", "default") + # Get instance name for rolling deployment tracking + instance_name = next( + (name for name, inst in config["providers"]["digitalocean"]["instances"].items() + if inst == instance_config), + "default" + ) + ip_ready = [] + pending_ips = [] + droplets_to_recycle = [] + + # First pass: identify healthy and pending droplets for droplet in list_droplets(instance_config): try: # Parse the created_at timestamp to a datetime object @@ -69,6 +81,7 @@ def do_check_alive(instance_config=None): if created_at is None: # If parsing fails but doesn't raise an exception, log and continue logger.info(f"Pending: DO {display_name} allocating (invalid timestamp)") + pending_ips.append(str(droplet.ip_address)) continue # Calculate elapsed time @@ -76,10 +89,7 @@ def do_check_alive(instance_config=None): # Check if the droplet has reached the age limit if config["age_limit"] > 0 and elapsed > datetime.timedelta(seconds=config["age_limit"]): - delete_proxy(droplet, instance_config) - logger.info( - f"Recycling DO {display_name} droplet, reached age limit -> {str(droplet.ip_address)}" - ) + droplets_to_recycle.append((droplet, elapsed)) elif check_alive(droplet.ip_address): logger.info(f"Alive: DO {display_name} -> {str(droplet.ip_address)}") ip_ready.append(droplet.ip_address) @@ -92,9 +102,53 @@ def do_check_alive(instance_config=None): ) else: logger.info(f"Waiting: DO {display_name} -> {str(droplet.ip_address)}") + pending_ips.append(str(droplet.ip_address)) except TypeError: # This happens when dateparser.parse raises a TypeError logger.info(f"Pending: DO {display_name} allocating") + if hasattr(droplet, 'ip_address'): + pending_ips.append(str(droplet.ip_address)) + + # Update rolling manager with current proxy health status + rolling_manager.update_proxy_health("digitalocean", instance_name, ip_ready, pending_ips) + + # Handle rolling deployments for age-limited droplets + if droplets_to_recycle and config["rolling_deployment"]["enabled"]: + rolling_config = config["rolling_deployment"] + + for droplet, elapsed in droplets_to_recycle: + droplet_ip = str(droplet.ip_address) + + # Check if we can recycle this droplet according to rolling deployment rules + if rolling_manager.can_recycle_proxy( + provider="digitalocean", + instance=instance_name, + proxy_ip=droplet_ip, + total_healthy=len(ip_ready), + min_available=rolling_config["min_available"], + batch_size=rolling_config["batch_size"], + rolling_enabled=True, + min_scaling=instance_config["scaling"]["min_scaling"] + ): + # Mark as recycling and delete + rolling_manager.mark_proxy_recycling("digitalocean", instance_name, droplet_ip) + delete_proxy(droplet, instance_config) + rolling_manager.mark_proxy_recycled("digitalocean", instance_name, droplet_ip) + logger.info( + f"Rolling deployment: Recycled DO {display_name} droplet (age limit) -> {droplet_ip}" + ) + else: + logger.info( + f"Rolling deployment: Deferred recycling DO {display_name} droplet -> {droplet_ip}" + ) + elif droplets_to_recycle and not config["rolling_deployment"]["enabled"]: + # Standard non-rolling recycling + for droplet, elapsed in droplets_to_recycle: + delete_proxy(droplet, instance_config) + logger.info( + f"Recycling DO {display_name} droplet, reached age limit -> {str(droplet.ip_address)}" + ) + return ip_ready diff --git a/cloudproxy/providers/gcp/main.py b/cloudproxy/providers/gcp/main.py index 5cc71b2..8390b6c 100644 --- a/cloudproxy/providers/gcp/main.py +++ b/cloudproxy/providers/gcp/main.py @@ -12,6 +12,7 @@ start_proxy, ) from cloudproxy.providers.settings import delete_queue, restart_queue, config +from cloudproxy.providers.rolling import rolling_manager def gcp_deployment(min_scaling, instance_config=None): """ @@ -53,8 +54,18 @@ def gcp_check_alive(instance_config=None): """ if instance_config is None: instance_config = config["providers"]["gcp"]["instances"]["default"] + + # Get instance name for rolling deployment tracking + instance_name = next( + (name for name, inst in config["providers"]["gcp"]["instances"].items() + if inst == instance_config), + "default" + ) ip_ready = [] + pending_ips = [] + instances_to_recycle = [] + for instance in list_instances(instance_config): try: elapsed = datetime.datetime.now( @@ -62,10 +73,8 @@ def gcp_check_alive(instance_config=None): ) - datetime.datetime.strptime(instance["creationTimestamp"], '%Y-%m-%dT%H:%M:%S.%f%z') if config["age_limit"] > 0 and elapsed > datetime.timedelta(seconds=config["age_limit"]): - access_configs = instance['networkInterfaces'][0]['accessConfigs'][0] - msg = f"{instance['name']} {access_configs['natIP'] if 'natIP' in access_configs else ''}" - delete_proxy(instance['name']) - logger.info("Recycling instance, reached age limit -> " + msg) + # Queue for potential recycling + instances_to_recycle.append((instance, elapsed)) elif instance['status'] == "TERMINATED": logger.info("Waking up: GCP -> Instance " + instance['name']) @@ -82,6 +91,8 @@ def gcp_check_alive(instance_config=None): access_configs = instance['networkInterfaces'][0]['accessConfigs'][0] msg = f"{instance['name']} {access_configs['natIP'] if 'natIP' in access_configs else ''}" logger.info("Provisioning: GCP -> " + msg) + if 'natIP' in access_configs: + pending_ips.append(access_configs['natIP']) # If none of the above, check if alive or not. elif check_alive(instance['networkInterfaces'][0]['accessConfigs'][0]['natIP']): @@ -98,8 +109,49 @@ def gcp_check_alive(instance_config=None): logger.info("Destroyed: took too long GCP -> " + msg) else: logger.info("Waiting: GCP -> " + msg) + if 'natIP' in access_configs: + pending_ips.append(access_configs['natIP']) except (TypeError, KeyError): logger.info("Pending: GCP -> Allocating IP") + + # Update rolling manager with current proxy health status + rolling_manager.update_proxy_health("gcp", instance_name, ip_ready, pending_ips) + + # Handle rolling deployments for age-limited instances + if instances_to_recycle and config["rolling_deployment"]["enabled"]: + rolling_config = config["rolling_deployment"] + + for inst, elapsed in instances_to_recycle: + access_configs = inst['networkInterfaces'][0]['accessConfigs'][0] + if 'natIP' in access_configs: + instance_ip = access_configs['natIP'] + + # Check if we can recycle this instance according to rolling deployment rules + if rolling_manager.can_recycle_proxy( + provider="gcp", + instance=instance_name, + proxy_ip=instance_ip, + total_healthy=len(ip_ready), + min_available=rolling_config["min_available"], + batch_size=rolling_config["batch_size"], + rolling_enabled=True, + min_scaling=instance_config["scaling"]["min_scaling"] + ): + # Mark as recycling and delete + rolling_manager.mark_proxy_recycling("gcp", instance_name, instance_ip) + delete_proxy(inst['name'], instance_config) + rolling_manager.mark_proxy_recycled("gcp", instance_name, instance_ip) + logger.info(f"Rolling deployment: Recycled GCP instance (age limit) -> {inst['name']} {instance_ip}") + else: + logger.info(f"Rolling deployment: Deferred recycling GCP instance -> {inst['name']} {instance_ip}") + elif instances_to_recycle and not config["rolling_deployment"]["enabled"]: + # Standard non-rolling recycling + for inst, elapsed in instances_to_recycle: + access_configs = inst['networkInterfaces'][0]['accessConfigs'][0] + msg = f"{inst['name']} {access_configs['natIP'] if 'natIP' in access_configs else ''}" + delete_proxy(inst['name'], instance_config) + logger.info("Recycling instance, reached age limit -> " + msg) + return ip_ready def gcp_check_delete(instance_config=None): diff --git a/cloudproxy/providers/hetzner/main.py b/cloudproxy/providers/hetzner/main.py index ec6de90..77b9a8e 100644 --- a/cloudproxy/providers/hetzner/main.py +++ b/cloudproxy/providers/hetzner/main.py @@ -8,6 +8,7 @@ from cloudproxy.providers import settings from cloudproxy.providers.hetzner.functions import list_proxies, delete_proxy, create_proxy from cloudproxy.providers.settings import config, delete_queue, restart_queue +from cloudproxy.providers.rolling import rolling_manager def hetzner_deployment(min_scaling, instance_config=None): @@ -58,16 +59,24 @@ def hetzner_check_alive(instance_config=None): # Get instance display name for logging display_name = instance_config.get("display_name", "default") + # Get instance name for rolling deployment tracking + instance_name = next( + (name for name, inst in config["providers"]["hetzner"]["instances"].items() + if inst == instance_config), + "default" + ) + ip_ready = [] + pending_ips = [] + proxies_to_recycle = [] + for proxy in list_proxies(instance_config): elapsed = datetime.datetime.now( datetime.timezone.utc ) - dateparser.parse(str(proxy.created)) if config["age_limit"] > 0 and elapsed > datetime.timedelta(seconds=config["age_limit"]): - delete_proxy(proxy, instance_config) - logger.info( - f"Recycling Hetzner {display_name} proxy, reached age limit -> {str(proxy.public_net.ipv4.ip)}" - ) + # Queue for potential recycling + proxies_to_recycle.append((proxy, elapsed)) elif check_alive(proxy.public_net.ipv4.ip): logger.info(f"Alive: Hetzner {display_name} -> {str(proxy.public_net.ipv4.ip)}") ip_ready.append(proxy.public_net.ipv4.ip) @@ -79,6 +88,42 @@ def hetzner_check_alive(instance_config=None): ) else: logger.info(f"Waiting: Hetzner {display_name} -> {str(proxy.public_net.ipv4.ip)}") + pending_ips.append(str(proxy.public_net.ipv4.ip)) + + # Update rolling manager with current proxy health status + rolling_manager.update_proxy_health("hetzner", instance_name, ip_ready, pending_ips) + + # Handle rolling deployments for age-limited proxies + if proxies_to_recycle and config["rolling_deployment"]["enabled"]: + rolling_config = config["rolling_deployment"] + + for prox, elapsed in proxies_to_recycle: + proxy_ip = str(prox.public_net.ipv4.ip) + + # Check if we can recycle this proxy according to rolling deployment rules + if rolling_manager.can_recycle_proxy( + provider="hetzner", + instance=instance_name, + proxy_ip=proxy_ip, + total_healthy=len(ip_ready), + min_available=rolling_config["min_available"], + batch_size=rolling_config["batch_size"], + rolling_enabled=True, + min_scaling=instance_config["scaling"]["min_scaling"] + ): + # Mark as recycling and delete + rolling_manager.mark_proxy_recycling("hetzner", instance_name, proxy_ip) + delete_proxy(prox, instance_config) + rolling_manager.mark_proxy_recycled("hetzner", instance_name, proxy_ip) + logger.info(f"Rolling deployment: Recycled Hetzner {display_name} proxy (age limit) -> {proxy_ip}") + else: + logger.info(f"Rolling deployment: Deferred recycling Hetzner {display_name} proxy -> {proxy_ip}") + elif proxies_to_recycle and not config["rolling_deployment"]["enabled"]: + # Standard non-rolling recycling + for prox, elapsed in proxies_to_recycle: + delete_proxy(prox, instance_config) + logger.info(f"Recycling Hetzner {display_name} proxy, reached age limit -> {str(prox.public_net.ipv4.ip)}") + return ip_ready diff --git a/cloudproxy/providers/rolling.py b/cloudproxy/providers/rolling.py new file mode 100644 index 0000000..a25fd0d --- /dev/null +++ b/cloudproxy/providers/rolling.py @@ -0,0 +1,240 @@ +""" +Rolling deployment manager for CloudProxy. + +This module handles the logic for rolling deployments, ensuring that a minimum +number of healthy proxies are always available during recycling operations. +""" + +import datetime +from typing import List, Dict, Set, Optional, Tuple +from dataclasses import dataclass, field +from enum import Enum +from loguru import logger + + +class ProxyState(Enum): + """Represents the state of a proxy in the rolling deployment process.""" + HEALTHY = "healthy" + PENDING_RECYCLE = "pending_recycle" + RECYCLING = "recycling" + PENDING = "pending" # Newly created, not yet healthy + + +@dataclass +class ProxyInfo: + """Information about a proxy for rolling deployment management.""" + ip: str + state: ProxyState + created_at: datetime.datetime + provider: str + instance: str + age_seconds: Optional[int] = None + + +@dataclass +class RollingDeploymentState: + """Tracks the state of rolling deployment for a provider instance.""" + provider: str + instance: str + healthy_proxies: Set[str] = field(default_factory=set) + pending_recycle: Set[str] = field(default_factory=set) + recycling: Set[str] = field(default_factory=set) + pending: Set[str] = field(default_factory=set) + last_update: datetime.datetime = field(default_factory=lambda: datetime.datetime.now(datetime.timezone.utc)) + + +class RollingDeploymentManager: + """Manages rolling deployments across all providers.""" + + def __init__(self): + self.states: Dict[Tuple[str, str], RollingDeploymentState] = {} + + def get_state(self, provider: str, instance: str) -> RollingDeploymentState: + """Get or create state for a provider instance.""" + key = (provider, instance) + if key not in self.states: + self.states[key] = RollingDeploymentState(provider=provider, instance=instance) + return self.states[key] + + def can_recycle_proxy( + self, + provider: str, + instance: str, + proxy_ip: str, + total_healthy: int, + min_available: int, + batch_size: int, + rolling_enabled: bool, + min_scaling: int = None + ) -> bool: + """ + Determine if a proxy can be recycled based on rolling deployment rules. + + Args: + provider: The cloud provider name + instance: The provider instance name + proxy_ip: IP address of the proxy to recycle + total_healthy: Total number of currently healthy proxies + min_available: Minimum number of proxies that must remain available + batch_size: Maximum number of proxies that can be recycled simultaneously + rolling_enabled: Whether rolling deployment is enabled + min_scaling: The minimum scaling configuration for the provider instance + + Returns: + True if the proxy can be recycled, False otherwise + """ + if not rolling_enabled: + # If rolling deployment is disabled, always allow recycling + return True + + state = self.get_state(provider, instance) + + # Validate configuration: min_available should not exceed min_scaling + if min_scaling is not None and min_available >= min_scaling: + logger.warning( + f"Rolling deployment: Configuration issue for {provider}/{instance}. " + f"min_available ({min_available}) >= min_scaling ({min_scaling}). " + f"This would prevent all recycling. Using min_scaling - 1 as effective minimum." + ) + # Use a sensible default: ensure at least one proxy can be recycled + effective_min_available = max(1, min_scaling - 1) + else: + effective_min_available = min_available + + # Check if we're already recycling too many proxies + currently_recycling = len(state.recycling) + len(state.pending_recycle) + if currently_recycling >= batch_size: + logger.info( + f"Rolling deployment: Cannot recycle {proxy_ip} for {provider}/{instance}. " + f"Already recycling {currently_recycling}/{batch_size} proxies" + ) + return False + + # Check if recycling this proxy would violate minimum availability + available_after_recycle = total_healthy - currently_recycling - 1 + if available_after_recycle < effective_min_available: + logger.info( + f"Rolling deployment: Cannot recycle {proxy_ip} for {provider}/{instance}. " + f"Would reduce available proxies below minimum ({available_after_recycle} < {effective_min_available})" + ) + return False + + # Mark proxy as pending recycle + state.pending_recycle.add(proxy_ip) + state.healthy_proxies.discard(proxy_ip) + logger.info( + f"Rolling deployment: Marked {proxy_ip} for recycling in {provider}/{instance}. " + f"Currently recycling {currently_recycling + 1}/{batch_size} proxies" + ) + return True + + def mark_proxy_recycling(self, provider: str, instance: str, proxy_ip: str): + """Mark a proxy as actively being recycled.""" + state = self.get_state(provider, instance) + state.pending_recycle.discard(proxy_ip) + state.recycling.add(proxy_ip) + state.last_update = datetime.datetime.now(datetime.timezone.utc) + + def mark_proxy_recycled(self, provider: str, instance: str, proxy_ip: str): + """Mark a proxy as successfully recycled (deleted).""" + state = self.get_state(provider, instance) + state.pending_recycle.discard(proxy_ip) + state.recycling.discard(proxy_ip) + state.healthy_proxies.discard(proxy_ip) + state.pending.discard(proxy_ip) + state.last_update = datetime.datetime.now(datetime.timezone.utc) + logger.info(f"Rolling deployment: Completed recycling {proxy_ip} in {provider}/{instance}") + + def update_proxy_health( + self, + provider: str, + instance: str, + healthy_ips: List[str], + pending_ips: List[str] = None + ): + """ + Update the health status of proxies for a provider instance. + + Args: + provider: The cloud provider name + instance: The provider instance name + healthy_ips: List of IPs that are currently healthy + pending_ips: List of IPs that are pending (newly created) + """ + state = self.get_state(provider, instance) + + # Update healthy proxies + state.healthy_proxies = set(healthy_ips) + + # Update pending proxies if provided + if pending_ips is not None: + state.pending = set(pending_ips) + + # Clean up recycling list if proxies no longer exist + existing_ips = state.healthy_proxies | state.pending + state.recycling = state.recycling & existing_ips + state.pending_recycle = state.pending_recycle & existing_ips + + state.last_update = datetime.datetime.now(datetime.timezone.utc) + + def get_recycling_status(self, provider: str = None, instance: str = None) -> Dict: + """ + Get the current rolling deployment status. + + Args: + provider: Optional provider filter + instance: Optional instance filter + + Returns: + Dictionary containing rolling deployment status + """ + status = {} + + for (prov, inst), state in self.states.items(): + if provider and prov != provider: + continue + if instance and inst != instance: + continue + + key = f"{prov}/{inst}" + status[key] = { + "healthy": len(state.healthy_proxies), + "pending": len(state.pending), + "pending_recycle": len(state.pending_recycle), + "recycling": len(state.recycling), + "last_update": state.last_update.isoformat(), + "healthy_ips": list(state.healthy_proxies), + "pending_recycle_ips": list(state.pending_recycle), + "recycling_ips": list(state.recycling), + } + + return status + + def should_create_replacement( + self, + provider: str, + instance: str, + min_scaling: int + ) -> bool: + """ + Determine if we should create replacement proxies proactively. + + Args: + provider: The cloud provider name + instance: The provider instance name + min_scaling: Minimum number of proxies to maintain + + Returns: + True if replacements should be created + """ + state = self.get_state(provider, instance) + + # Total expected proxies after recycling completes + total_after_recycle = len(state.healthy_proxies) + len(state.pending) + + # We should create replacements if we'll be below min_scaling + return total_after_recycle < min_scaling + + +# Global instance +rolling_manager = RollingDeploymentManager() \ No newline at end of file diff --git a/cloudproxy/providers/settings.py b/cloudproxy/providers/settings.py index 8c82dc9..1d1a815 100644 --- a/cloudproxy/providers/settings.py +++ b/cloudproxy/providers/settings.py @@ -6,6 +6,11 @@ "no_auth": False, "only_host_ip": False, "age_limit": 0, + "rolling_deployment": { + "enabled": False, + "min_available": 3, + "batch_size": 2, + }, "providers": { "digitalocean": { "instances": { @@ -113,6 +118,11 @@ config["no_auth"] = config["auth"]["username"] == "changeme" and config["auth"]["password"] == "changeme" config["only_host_ip"] = os.environ.get("ONLY_HOST_IP", False) +# Set rolling deployment configuration +config["rolling_deployment"]["enabled"] = os.environ.get("ROLLING_DEPLOYMENT", "False") == "True" +config["rolling_deployment"]["min_available"] = int(os.environ.get("ROLLING_MIN_AVAILABLE", 3)) +config["rolling_deployment"]["batch_size"] = int(os.environ.get("ROLLING_BATCH_SIZE", 2)) + # Set DigitalOcean config - original format for backward compatibility config["providers"]["digitalocean"]["instances"]["default"]["enabled"] = os.environ.get( "DIGITALOCEAN_ENABLED", "False" diff --git a/cloudproxy/providers/vultr/main.py b/cloudproxy/providers/vultr/main.py index cf76a02..eca4de9 100644 --- a/cloudproxy/providers/vultr/main.py +++ b/cloudproxy/providers/vultr/main.py @@ -13,6 +13,7 @@ VultrFirewallExistsException, ) from cloudproxy.providers.settings import delete_queue, restart_queue, config +from cloudproxy.providers.rolling import rolling_manager def vultr_deployment(min_scaling, instance_config=None): @@ -62,8 +63,18 @@ def vultr_check_alive(instance_config=None): # Get instance display name for logging display_name = instance_config.get("display_name", "default") + + # Get instance name for rolling deployment tracking + instance_name = next( + (name for name, inst in config["providers"]["vultr"]["instances"].items() + if inst == instance_config), + "default" + ) ip_ready = [] + pending_ips = [] + instances_to_recycle = [] + for instance in list_instances(instance_config): try: # Parse the created_at timestamp to a datetime object @@ -81,10 +92,8 @@ def vultr_check_alive(instance_config=None): # Check if the instance has reached the age limit if config["age_limit"] > 0 and elapsed > datetime.timedelta( seconds=config["age_limit"]): - delete_proxy(instance, instance_config) - logger.info( - f"Recycling Vultr {display_name} instance, reached age limit -> {str(instance.ip_address)}" - ) + # Queue for potential recycling + instances_to_recycle.append((instance, elapsed)) elif instance.status == "active" and instance.ip_address and check_alive(instance.ip_address): logger.info( f"Alive: Vultr {display_name} -> {str(instance.ip_address)}") @@ -99,9 +108,55 @@ def vultr_check_alive(instance_config=None): else: logger.info( f"Waiting: Vultr {display_name} -> {str(instance.ip_address)}") + if instance.ip_address: + pending_ips.append(instance.ip_address) except TypeError: # This happens when dateparser.parse raises a TypeError logger.info(f"Pending: Vultr {display_name} allocating") + if hasattr(instance, 'ip_address') and instance.ip_address: + pending_ips.append(instance.ip_address) + + # Update rolling manager with current proxy health status + rolling_manager.update_proxy_health("vultr", instance_name, ip_ready, pending_ips) + + # Handle rolling deployments for age-limited instances + if instances_to_recycle and config["rolling_deployment"]["enabled"]: + rolling_config = config["rolling_deployment"] + + for inst, elapsed in instances_to_recycle: + if inst.ip_address: + instance_ip = str(inst.ip_address) + + # Check if we can recycle this instance according to rolling deployment rules + if rolling_manager.can_recycle_proxy( + provider="vultr", + instance=instance_name, + proxy_ip=instance_ip, + total_healthy=len(ip_ready), + min_available=rolling_config["min_available"], + batch_size=rolling_config["batch_size"], + rolling_enabled=True, + min_scaling=instance_config["scaling"]["min_scaling"] + ): + # Mark as recycling and delete + rolling_manager.mark_proxy_recycling("vultr", instance_name, instance_ip) + delete_proxy(inst, instance_config) + rolling_manager.mark_proxy_recycled("vultr", instance_name, instance_ip) + logger.info( + f"Rolling deployment: Recycled Vultr {display_name} instance (age limit) -> {instance_ip}" + ) + else: + logger.info( + f"Rolling deployment: Deferred recycling Vultr {display_name} instance -> {instance_ip}" + ) + elif instances_to_recycle and not config["rolling_deployment"]["enabled"]: + # Standard non-rolling recycling + for inst, elapsed in instances_to_recycle: + delete_proxy(inst, instance_config) + logger.info( + f"Recycling Vultr {display_name} instance, reached age limit -> {str(inst.ip_address)}" + ) + return ip_ready diff --git a/docs/rolling-deployments.md b/docs/rolling-deployments.md new file mode 100644 index 0000000..f60f726 --- /dev/null +++ b/docs/rolling-deployments.md @@ -0,0 +1,224 @@ +# Rolling Deployments + +CloudProxy now supports rolling deployments to ensure zero-downtime proxy recycling. When enabled, the system ensures a minimum number of healthy proxies are always available during age-based recycling operations. + +## Overview + +Rolling deployments prevent service disruption by: +- Maintaining a minimum number of healthy proxies at all times +- Limiting the number of proxies being recycled simultaneously +- Deferring proxy recycling when it would violate availability requirements +- Providing real-time visibility into the recycling process + +## Configuration + +### Environment Variables + +Configure rolling deployments using these environment variables: + +```bash +# Enable/disable rolling deployments +ROLLING_DEPLOYMENT=True + +# Minimum number of proxies to keep available during recycling +ROLLING_MIN_AVAILABLE=3 + +# Maximum number of proxies to recycle simultaneously +ROLLING_BATCH_SIZE=2 +``` + +### Configuration Details + +- **`ROLLING_DEPLOYMENT`**: Set to `True` to enable rolling deployments. Default: `False` +- **`ROLLING_MIN_AVAILABLE`**: The minimum number of healthy proxies that must remain available during recycling. The system will defer recycling if it would reduce availability below this threshold. Default: `3` +- **`ROLLING_BATCH_SIZE`**: The maximum number of proxies that can be in the recycling state simultaneously. This prevents overwhelming the system with too many concurrent deletions and creations. Default: `2` + +## How It Works + +### Standard Recycling (Rolling Deployment Disabled) + +When `ROLLING_DEPLOYMENT=False` or not set: +1. Proxies reaching the age limit are immediately recycled +2. All aged proxies are deleted simultaneously +3. New proxies are created to maintain minimum scaling +4. There may be a period with reduced availability + +### Rolling Deployment (Enabled) + +When `ROLLING_DEPLOYMENT=True`: +1. The system identifies proxies that have reached the age limit +2. Before recycling, it checks: + - Would recycling reduce healthy proxies below `ROLLING_MIN_AVAILABLE`? + - Are we already recycling `ROLLING_BATCH_SIZE` proxies? +3. If checks pass, the proxy is recycled +4. If checks fail, recycling is deferred until conditions improve +5. The process continues until all aged proxies are recycled + +### Example Scenario + +Configuration: +- `AGE_LIMIT=3600` (1 hour) +- `ROLLING_DEPLOYMENT=True` +- `ROLLING_MIN_AVAILABLE=3` +- `ROLLING_BATCH_SIZE=2` +- `DIGITALOCEAN_MIN_SCALING=5` + +Scenario: +1. You have 5 healthy DigitalOcean droplets +2. All 5 reach the age limit simultaneously +3. Rolling deployment kicks in: + - First 2 droplets are marked for recycling (batch size limit) + - Remaining 3 stay healthy (minimum availability) + - Once the first 2 are replaced and healthy, the next batch begins + - Process continues until all 5 are recycled + +## API Endpoints + +### Get Rolling Deployment Status + +```bash +# Get overall rolling deployment status +curl http://localhost:8000/rolling + +# Get status for a specific provider +curl http://localhost:8000/rolling/digitalocean + +# Get status for a specific provider instance +curl http://localhost:8000/rolling/digitalocean/default +``` + +Response example: +```json +{ + "metadata": { + "request_id": "uuid", + "timestamp": "2024-01-01T00:00:00Z" + }, + "message": "Rolling deployment status retrieved successfully", + "config": { + "enabled": true, + "min_available": 3, + "batch_size": 2 + }, + "status": { + "digitalocean/default": { + "healthy": 3, + "pending": 0, + "pending_recycle": 1, + "recycling": 1, + "last_update": "2024-01-01T00:00:00Z", + "healthy_ips": ["192.168.1.1", "192.168.1.2", "192.168.1.3"], + "pending_recycle_ips": ["192.168.1.4"], + "recycling_ips": ["192.168.1.5"] + } + } +} +``` + +### Update Rolling Deployment Configuration + +```bash +curl -X PATCH http://localhost:8000/rolling \ + -H "Content-Type: application/json" \ + -d '{ + "enabled": true, + "min_available": 5, + "batch_size": 3 + }' +``` + +## Monitoring + +### Log Messages + +Rolling deployment activities are logged with clear messages: + +``` +Rolling deployment: Marked 192.168.1.1 for recycling in digitalocean/default. Currently recycling 1/2 proxies +Rolling deployment: Cannot recycle 192.168.1.2 for digitalocean/default. Would reduce available proxies below minimum (2 < 3) +Rolling deployment: Deferred recycling digitalocean/default droplet -> 192.168.1.3 +Rolling deployment: Completed recycling 192.168.1.1 in digitalocean/default +``` + +### Status Fields + +The API provides detailed status information: +- **`healthy`**: Number of proxies currently healthy and serving traffic +- **`pending`**: Number of newly created proxies not yet healthy +- **`pending_recycle`**: Number of proxies marked for recycling but not yet started +- **`recycling`**: Number of proxies currently being deleted +- **`healthy_ips`**: List of IPs for healthy proxies +- **`pending_recycle_ips`**: List of IPs waiting to be recycled +- **`recycling_ips`**: List of IPs currently being recycled + +## Best Practices + +### Setting Minimum Available + +- Set `ROLLING_MIN_AVAILABLE` based on your traffic requirements +- Consider peak load when determining the minimum +- **IMPORTANT**: Ensure `ROLLING_MIN_AVAILABLE` is less than your `MIN_SCALING` value + - If `ROLLING_MIN_AVAILABLE >= MIN_SCALING`, the system automatically adjusts to use `MIN_SCALING - 1` + - This prevents a deadlock where no proxies can ever be recycled + - A warning will be logged when this adjustment occurs + +### Setting Batch Size + +- Smaller batch sizes provide smoother recycling but take longer +- Larger batch sizes recycle faster but may cause temporary capacity reduction +- Consider your provider's API rate limits when setting batch size + +### Age Limits with Rolling Deployment + +- Rolling deployments work best with reasonable age limits (e.g., 1-24 hours) +- Very short age limits may cause constant recycling +- Monitor the `/rolling` endpoint to ensure recycling completes successfully + +## Troubleshooting + +### Proxies Not Being Recycled + +If proxies aren't being recycled despite reaching age limit: +1. Check if rolling deployment is enabled +2. Verify you have more than `ROLLING_MIN_AVAILABLE` healthy proxies +3. Check if batch size limit is being reached +4. Review logs for "Deferred recycling" messages + +### Recycling Too Slow + +If recycling takes too long: +1. Increase `ROLLING_BATCH_SIZE` +2. Ensure new proxies become healthy quickly +3. Check provider API response times +4. Consider reducing `AGE_LIMIT` to spread recycling over time + +### API Errors + +If the rolling deployment API returns errors: +1. Ensure the provider and instance names are correct +2. Check that the CloudProxy service is running +3. Verify your configuration is valid (positive integers for limits) + +## Integration with Existing Features + +### Multi-Instance Support + +Rolling deployments work with multi-instance configurations: +- Each provider instance maintains its own rolling deployment state +- Settings apply globally but are enforced per instance +- Monitor each instance separately via the API + +### Provider Compatibility + +Rolling deployments are supported for all providers: +- DigitalOcean +- AWS (including spot instances) +- Google Cloud Platform +- Hetzner +- Vultr + +### Interaction with Manual Deletion + +- Manual proxy deletion (via `/destroy` endpoint) bypasses rolling deployment rules +- The system will create replacements to maintain minimum scaling +- Rolling deployment continues for age-based recycling \ No newline at end of file diff --git a/tests/test_providers_aws_main.py b/tests/test_providers_aws_main.py index bc4b7c9..636fc90 100644 --- a/tests/test_providers_aws_main.py +++ b/tests/test_providers_aws_main.py @@ -279,10 +279,13 @@ def test_aws_check_alive_age_limit_exceeded_directly(): """Test the aws_check_alive function directly with simulated old instance""" # Save original age limit value original_age_limit = config["age_limit"] + original_rolling = config["rolling_deployment"]["enabled"] try: # Set age limit to a small value to make instances expire quickly config["age_limit"] = 60 # 60 seconds + # Disable rolling deployment to allow immediate deletion + config["rolling_deployment"]["enabled"] = False # Create a mock instance with a launch time far in the past with patch('cloudproxy.providers.aws.main.list_instances') as mock_list_instances: @@ -309,8 +312,9 @@ def test_aws_check_alive_age_limit_exceeded_directly(): assert mock_delete_proxy.call_count == 1 # Should delete the expired instance assert len(result) == 0 # No IPs in result as the instance was deleted finally: - # Restore original age limit + # Restore original settings config["age_limit"] = original_age_limit + config["rolling_deployment"]["enabled"] = original_rolling @patch('cloudproxy.providers.aws.main.list_instances') @patch('cloudproxy.providers.aws.main.delete_proxy') diff --git a/tests/test_providers_config.py b/tests/test_providers_config.py index 9a54ff2..0f606de 100644 --- a/tests/test_providers_config.py +++ b/tests/test_providers_config.py @@ -83,8 +83,8 @@ def test_set_auth_without_auth(setup_config_test): # Verify BasicAuth line was removed assert "\nBasicAuth PROXY_USERNAME PROXY_PASSWORD\n" not in result - # The replacement seems to leave an extra newline, so we get three newlines - assert "Allow 127.0.0.1\n\n\nConnectPort" in result + # When only_host_ip is False, Allow should be changed to 0.0.0.0/0 + assert "Allow 0.0.0.0/0" in result def test_set_auth_with_host_ip(setup_config_test): diff --git a/tests/test_providers_digitalocean_main_coverage.py b/tests/test_providers_digitalocean_main_coverage.py index c9d37c6..bf4375e 100644 --- a/tests/test_providers_digitalocean_main_coverage.py +++ b/tests/test_providers_digitalocean_main_coverage.py @@ -297,15 +297,24 @@ def test_do_check_alive_active_droplets(mock_check_alive, mock_list_droplets, mo # Setup mock_list_droplets.return_value = mock_droplets mock_check_alive.return_value = True + # Disable rolling deployment and age limit to avoid interference + original_rolling = config["rolling_deployment"]["enabled"] + original_age_limit = config["age_limit"] + config["rolling_deployment"]["enabled"] = False + config["age_limit"] = 0 # Disable age-based recycling - # Execute - result = do_check_alive() - - # Verify - assert len(result) == 3 - assert "1.2.3.4" in result - assert "5.6.7.8" in result - assert "9.10.11.12" in result + try: + # Execute + result = do_check_alive() + + # Verify + assert len(result) == 3 + assert "1.2.3.4" in result + assert "5.6.7.8" in result + assert "9.10.11.12" in result + finally: + config["rolling_deployment"]["enabled"] = original_rolling + config["age_limit"] = original_age_limit mock_check_alive.assert_has_calls([ call("1.2.3.4"), call("5.6.7.8"), @@ -359,9 +368,11 @@ def test_do_check_alive_age_limit(mock_delete_proxy, mock_check_alive, mock_list mock_check_alive.return_value = True mock_delete_proxy.return_value = True - # Set age limit to 1 hour + # Set age limit to 1 hour and disable rolling deployment original_age_limit = config["age_limit"] + original_rolling = config["rolling_deployment"]["enabled"] config["age_limit"] = 3600 # 1 hour in seconds + config["rolling_deployment"]["enabled"] = False try: # Execute @@ -379,8 +390,9 @@ def test_do_check_alive_age_limit(mock_delete_proxy, mock_check_alive, mock_list args, _ = mock_delete_proxy.call_args assert args[0] == mock_droplets[2] # First parameter should be the third droplet finally: - # Restore original age limit + # Restore original settings config["age_limit"] = original_age_limit + config["rolling_deployment"]["enabled"] = original_rolling @patch('cloudproxy.providers.digitalocean.main.list_droplets') @patch('cloudproxy.providers.digitalocean.main.check_alive') @@ -390,14 +402,23 @@ def test_do_check_alive_invalid_timestamp(mock_check_alive, mock_list_droplets, mock_droplets[0].created_at = "invalid-timestamp" mock_list_droplets.return_value = mock_droplets mock_check_alive.return_value = True + # Disable rolling deployment and age limit + original_rolling = config["rolling_deployment"]["enabled"] + original_age_limit = config["age_limit"] + config["rolling_deployment"]["enabled"] = False + config["age_limit"] = 0 # Disable age-based recycling - # Execute - result = do_check_alive() - - # Verify - assert len(result) == 2 # Only two valid droplets should be in the result - assert "5.6.7.8" in result - assert "9.10.11.12" in result + try: + # Execute + result = do_check_alive() + + # Verify + assert len(result) == 2 # Only two valid droplets should be in the result + assert "5.6.7.8" in result + assert "9.10.11.12" in result + finally: + config["rolling_deployment"]["enabled"] = original_rolling + config["age_limit"] = original_age_limit @patch('cloudproxy.providers.digitalocean.main.list_droplets') @patch('cloudproxy.providers.digitalocean.main.delete_proxy') diff --git a/tests/test_providers_gcp_main.py b/tests/test_providers_gcp_main.py index c2bb445..97e3764 100644 --- a/tests/test_providers_gcp_main.py +++ b/tests/test_providers_gcp_main.py @@ -217,12 +217,19 @@ def test_gcp_check_alive_not_alive_too_long(mock_start_proxy, mock_delete_proxy, mock_check_alive.return_value = False # Instance is not alive mock_delete_proxy.return_value = True - # Execute - result = gcp_check_alive() + # Disable rolling deployment + original_rolling = config["rolling_deployment"]["enabled"] + config["rolling_deployment"]["enabled"] = False + + try: + # Execute + result = gcp_check_alive() - # Verify - assert mock_delete_proxy.call_count == 1 # Should delete the instance - assert len(result) == 0 + # Verify + assert mock_delete_proxy.call_count == 1 # Should delete the instance + assert len(result) == 0 + finally: + config["rolling_deployment"]["enabled"] = original_rolling @patch('cloudproxy.providers.gcp.main.check_alive') @patch('cloudproxy.providers.gcp.main.list_instances') @@ -230,12 +237,14 @@ def test_gcp_check_alive_not_alive_too_long(mock_start_proxy, mock_delete_proxy, @patch('cloudproxy.providers.gcp.main.start_proxy') def test_gcp_check_alive_age_limit_exceeded(mock_start_proxy, mock_delete_proxy, mock_list_instances, mock_check_alive, setup_instances): """Test checking alive for instances exceeding age limit""" - # Save original age limit value + # Save original values original_age_limit = config["age_limit"] + original_rolling = config["rolling_deployment"]["enabled"] try: - # Set age limit to a small value to make instances expire quickly + # Set age limit to a small value and disable rolling deployment config["age_limit"] = 60 # 60 seconds + config["rolling_deployment"]["enabled"] = False # Create a mock instance with a creation time far in the past old_time = datetime.datetime.now(timezone.utc) - datetime.timedelta(seconds=120) @@ -257,8 +266,9 @@ def test_gcp_check_alive_age_limit_exceeded(mock_start_proxy, mock_delete_proxy, assert mock_delete_proxy.call_count == 1 # Should delete the instance assert len(result) == 0 finally: - # Restore original age limit + # Restore original settings config["age_limit"] = original_age_limit + config["rolling_deployment"]["enabled"] = original_rolling @patch('cloudproxy.providers.gcp.main.check_alive') @patch('cloudproxy.providers.gcp.main.list_instances') @@ -276,13 +286,23 @@ def test_gcp_check_alive_type_key_error(mock_start_proxy, mock_delete_proxy, moc mock_list_instances.return_value = [invalid_instance] mock_check_alive.return_value = False - # Execute - result = gcp_check_alive() + # Disable rolling deployment and age limit to avoid the error in recycling logic + original_rolling = config["rolling_deployment"]["enabled"] + original_age_limit = config["age_limit"] + config["rolling_deployment"]["enabled"] = False + config["age_limit"] = 0 # Disable age-based recycling + + try: + # Execute + result = gcp_check_alive() - # Verify - assert mock_start_proxy.call_count == 0 - assert mock_delete_proxy.call_count == 0 - assert len(result) == 0 # No IPs should be added + # Verify + assert mock_start_proxy.call_count == 0 + assert mock_delete_proxy.call_count == 0 + assert len(result) == 0 # No IPs should be added + finally: + config["rolling_deployment"]["enabled"] = original_rolling + config["age_limit"] = original_age_limit @patch('cloudproxy.providers.gcp.main.list_instances') @patch('cloudproxy.providers.gcp.main.delete_proxy') diff --git a/tests/test_providers_hetzner_main.py b/tests/test_providers_hetzner_main.py index 214c40a..99cfa4c 100644 --- a/tests/test_providers_hetzner_main.py +++ b/tests/test_providers_hetzner_main.py @@ -76,6 +76,8 @@ def config_getitem(key): return providers_dict elif key == "age_limit": return 100 # Return an actual integer + elif key == "rolling_deployment": + return {"enabled": False, "min_available": 3, "batch_size": 2} else: return MagicMock() diff --git a/tests/test_providers_manager.py b/tests/test_providers_manager.py index c2966cf..10c971d 100644 --- a/tests/test_providers_manager.py +++ b/tests/test_providers_manager.py @@ -27,6 +27,8 @@ def test_init_schedule_all_enabled(mock_scheduler_class, setup_provider_config): # Configure all providers as enabled for provider in ["digitalocean", "aws", "gcp", "hetzner"]: settings.config["providers"][provider]["instances"]["default"]["enabled"] = True + # Disable Vultr to keep expected count + settings.config["providers"]["vultr"]["instances"]["default"]["enabled"] = False # Remove the production instance for this test if "production" in settings.config["providers"]["aws"]["instances"]: @@ -57,7 +59,7 @@ def test_init_schedule_all_disabled(mock_scheduler_class, setup_provider_config) mock_scheduler_class.return_value = mock_scheduler # Configure all providers as disabled - for provider in ["digitalocean", "aws", "gcp", "hetzner"]: + for provider in ["digitalocean", "aws", "gcp", "hetzner", "vultr"]: settings.config["providers"][provider]["instances"]["default"]["enabled"] = False # Also disable the production instance if it exists @@ -83,6 +85,7 @@ def test_init_schedule_mixed_providers(mock_scheduler_class, setup_provider_conf settings.config["providers"]["aws"]["instances"]["default"]["enabled"] = False settings.config["providers"]["gcp"]["instances"]["default"]["enabled"] = True settings.config["providers"]["hetzner"]["instances"]["default"]["enabled"] = False + settings.config["providers"]["vultr"]["instances"]["default"]["enabled"] = False # Also disable the production instance if it exists if "production" in settings.config["providers"]["aws"]["instances"]: @@ -137,6 +140,7 @@ def test_init_schedule_multiple_instances(mock_scheduler_class, setup_provider_c settings.config["providers"]["digitalocean"]["instances"]["default"]["enabled"] = False settings.config["providers"]["gcp"]["instances"]["default"]["enabled"] = False settings.config["providers"]["hetzner"]["instances"]["default"]["enabled"] = False + settings.config["providers"]["vultr"]["instances"]["default"]["enabled"] = False # Execute init_schedule() @@ -204,6 +208,7 @@ def test_init_schedule_multiple_providers_with_instances(mock_scheduler_class, s # Disable other providers for clarity settings.config["providers"]["gcp"]["instances"]["default"]["enabled"] = False settings.config["providers"]["hetzner"]["instances"]["default"]["enabled"] = False + settings.config["providers"]["vultr"]["instances"]["default"]["enabled"] = False # Execute init_schedule() diff --git a/tests/test_providers_vultr_main.py b/tests/test_providers_vultr_main.py index e616105..af1d4ef 100644 --- a/tests/test_providers_vultr_main.py +++ b/tests/test_providers_vultr_main.py @@ -37,7 +37,8 @@ def mock_config(self, mock_instance_config): } } }, - "age_limit": 3600 # 1 hour + "age_limit": 3600, # 1 hour + "rolling_deployment": {"enabled": False, "min_available": 3, "batch_size": 2} }.get(x, {}) yield mock_cfg diff --git a/tests/test_rolling_deployment.py b/tests/test_rolling_deployment.py new file mode 100644 index 0000000..94e5616 --- /dev/null +++ b/tests/test_rolling_deployment.py @@ -0,0 +1,355 @@ +""" +Unit tests for rolling deployment functionality. +""" + +import datetime +import pytest +from unittest.mock import MagicMock, patch +from cloudproxy.providers.rolling import ( + ProxyState, + ProxyInfo, + RollingDeploymentState, + RollingDeploymentManager +) + + +class TestRollingDeploymentManager: + """Test cases for the RollingDeploymentManager class.""" + + def setup_method(self): + """Set up test fixtures.""" + self.manager = RollingDeploymentManager() + + def test_get_state_creates_new(self): + """Test that get_state creates a new state if it doesn't exist.""" + state = self.manager.get_state("aws", "default") + assert state.provider == "aws" + assert state.instance == "default" + assert len(state.healthy_proxies) == 0 + assert len(state.pending_recycle) == 0 + + def test_get_state_returns_existing(self): + """Test that get_state returns existing state.""" + # Create initial state + state1 = self.manager.get_state("aws", "default") + state1.healthy_proxies.add("192.168.1.1") + + # Get state again + state2 = self.manager.get_state("aws", "default") + assert "192.168.1.1" in state2.healthy_proxies + assert state1 is state2 + + def test_can_recycle_proxy_rolling_disabled(self): + """Test that recycling is always allowed when rolling is disabled.""" + result = self.manager.can_recycle_proxy( + provider="aws", + instance="default", + proxy_ip="192.168.1.1", + total_healthy=5, + min_available=3, + batch_size=2, + rolling_enabled=False + ) + assert result is True + + def test_can_recycle_proxy_batch_size_limit(self): + """Test that batch size limits recycling.""" + state = self.manager.get_state("aws", "default") + state.recycling.add("192.168.1.1") + state.recycling.add("192.168.1.2") + + result = self.manager.can_recycle_proxy( + provider="aws", + instance="default", + proxy_ip="192.168.1.3", + total_healthy=10, + min_available=3, + batch_size=2, + rolling_enabled=True + ) + assert result is False + + def test_can_recycle_proxy_min_available_limit(self): + """Test that minimum availability prevents recycling.""" + result = self.manager.can_recycle_proxy( + provider="aws", + instance="default", + proxy_ip="192.168.1.1", + total_healthy=3, + min_available=3, + batch_size=2, + rolling_enabled=True + ) + assert result is False + + def test_can_recycle_proxy_allowed(self): + """Test successful recycling when all conditions are met.""" + result = self.manager.can_recycle_proxy( + provider="aws", + instance="default", + proxy_ip="192.168.1.1", + total_healthy=10, + min_available=3, + batch_size=2, + rolling_enabled=True + ) + assert result is True + + # Check that proxy was marked as pending recycle + state = self.manager.get_state("aws", "default") + assert "192.168.1.1" in state.pending_recycle + + def test_can_recycle_proxy_min_available_exceeds_min_scaling(self): + """Test that the system adjusts when min_available >= min_scaling.""" + # Test when min_available equals min_scaling + result = self.manager.can_recycle_proxy( + provider="aws", + instance="default", + proxy_ip="192.168.1.1", + total_healthy=5, + min_available=5, # Same as min_scaling + batch_size=2, + rolling_enabled=True, + min_scaling=5 + ) + # Should still allow recycling with adjusted minimum (min_scaling - 1 = 4) + assert result is True + + # Reset state for second test + self.manager = RollingDeploymentManager() + + # Test when min_available exceeds min_scaling + result = self.manager.can_recycle_proxy( + provider="aws", + instance="default", + proxy_ip="192.168.1.2", + total_healthy=5, + min_available=10, # Greater than min_scaling + batch_size=2, + rolling_enabled=True, + min_scaling=5 + ) + # Should still allow recycling with adjusted minimum (min_scaling - 1 = 4) + assert result is True + + def test_can_recycle_proxy_min_scaling_one(self): + """Test edge case when min_scaling is 1.""" + result = self.manager.can_recycle_proxy( + provider="aws", + instance="default", + proxy_ip="192.168.1.1", + total_healthy=1, + min_available=1, + batch_size=1, + rolling_enabled=True, + min_scaling=1 + ) + # Should not allow recycling when only one proxy exists + assert result is False + + def test_mark_proxy_recycling(self): + """Test marking a proxy as actively recycling.""" + state = self.manager.get_state("aws", "default") + state.pending_recycle.add("192.168.1.1") + + self.manager.mark_proxy_recycling("aws", "default", "192.168.1.1") + + assert "192.168.1.1" not in state.pending_recycle + assert "192.168.1.1" in state.recycling + + def test_mark_proxy_recycled(self): + """Test marking a proxy as successfully recycled.""" + state = self.manager.get_state("aws", "default") + state.recycling.add("192.168.1.1") + state.healthy_proxies.add("192.168.1.1") + + self.manager.mark_proxy_recycled("aws", "default", "192.168.1.1") + + assert "192.168.1.1" not in state.recycling + assert "192.168.1.1" not in state.healthy_proxies + assert "192.168.1.1" not in state.pending_recycle + + def test_update_proxy_health(self): + """Test updating proxy health status.""" + healthy_ips = ["192.168.1.1", "192.168.1.2"] + pending_ips = ["192.168.1.3"] + + self.manager.update_proxy_health("aws", "default", healthy_ips, pending_ips) + + state = self.manager.get_state("aws", "default") + assert state.healthy_proxies == {"192.168.1.1", "192.168.1.2"} + assert state.pending == {"192.168.1.3"} + + def test_update_proxy_health_cleans_stale_recycling(self): + """Test that update cleans up stale recycling entries.""" + state = self.manager.get_state("aws", "default") + state.recycling.add("192.168.1.1") # No longer exists + state.recycling.add("192.168.1.2") # Still exists + state.pending_recycle.add("192.168.1.3") # No longer exists + + healthy_ips = ["192.168.1.2"] + self.manager.update_proxy_health("aws", "default", healthy_ips, []) + + assert "192.168.1.1" not in state.recycling + assert "192.168.1.2" in state.recycling + assert "192.168.1.3" not in state.pending_recycle + + def test_get_recycling_status(self): + """Test getting recycling status.""" + # Set up some state + state = self.manager.get_state("aws", "default") + state.healthy_proxies = {"192.168.1.1", "192.168.1.2"} + state.pending_recycle = {"192.168.1.3"} + state.recycling = {"192.168.1.4"} + state.pending = {"192.168.1.5"} + + status = self.manager.get_recycling_status() + + assert "aws/default" in status + assert status["aws/default"]["healthy"] == 2 + assert status["aws/default"]["pending"] == 1 + assert status["aws/default"]["pending_recycle"] == 1 + assert status["aws/default"]["recycling"] == 1 + assert "192.168.1.1" in status["aws/default"]["healthy_ips"] + + def test_get_recycling_status_filtered(self): + """Test getting filtered recycling status.""" + # Set up states for multiple providers + self.manager.get_state("aws", "default").healthy_proxies = {"192.168.1.1"} + self.manager.get_state("gcp", "default").healthy_proxies = {"192.168.2.1"} + + # Filter by provider + status = self.manager.get_recycling_status(provider="aws") + assert "aws/default" in status + assert "gcp/default" not in status + + # Filter by provider and instance + status = self.manager.get_recycling_status(provider="aws", instance="default") + assert "aws/default" in status + assert len(status) == 1 + + def test_should_create_replacement(self): + """Test determining if replacement proxies should be created.""" + state = self.manager.get_state("aws", "default") + state.healthy_proxies = {"192.168.1.1", "192.168.1.2"} + state.pending = {"192.168.1.3"} + + # Total is 3, min_scaling is 5, should create replacements + should_create = self.manager.should_create_replacement("aws", "default", 5) + assert should_create is True + + # Total is 3, min_scaling is 3, should not create replacements + should_create = self.manager.should_create_replacement("aws", "default", 3) + assert should_create is False + + def test_complex_rolling_scenario(self): + """Test a complex rolling deployment scenario.""" + # Initial state: 5 healthy proxies + healthy_ips = [f"192.168.1.{i}" for i in range(1, 6)] + self.manager.update_proxy_health("aws", "default", healthy_ips, []) + + # Try to recycle 3 proxies with batch_size=2, min_available=3 + results = [] + for ip in healthy_ips[:3]: + result = self.manager.can_recycle_proxy( + provider="aws", + instance="default", + proxy_ip=ip, + total_healthy=5, + min_available=3, + batch_size=2, + rolling_enabled=True + ) + results.append(result) + if result: + self.manager.mark_proxy_recycling("aws", "default", ip) + + # First two should succeed, third should fail (batch size limit) + assert results == [True, True, False] + + # Mark first one as recycled + self.manager.mark_proxy_recycled("aws", "default", healthy_ips[0]) + + # The second proxy (192.168.1.2) is still in recycling state + # So we have: 3 healthy (3,4,5), 1 recycling (2), 1 recycled (1) + # We can't recycle another one yet because it would drop below min_available + result = self.manager.can_recycle_proxy( + provider="aws", + instance="default", + proxy_ip=healthy_ips[3], + total_healthy=4, # Proxies 2,3,4,5 are still counted as healthy + min_available=3, + batch_size=2, + rolling_enabled=True + ) + # This should fail because we still have one recycling + assert result is False + + # Now mark the second one as recycled too + self.manager.mark_proxy_recycled("aws", "default", healthy_ips[1]) + + # Now we can recycle another one + result = self.manager.can_recycle_proxy( + provider="aws", + instance="default", + proxy_ip=healthy_ips[3], + total_healthy=3, # Only 3,4,5 remain healthy + min_available=3, + batch_size=2, + rolling_enabled=True + ) + # This should fail because we're at minimum + assert result is False + + +class TestProxyState: + """Test cases for ProxyState enum.""" + + def test_proxy_states(self): + """Test that all expected proxy states exist.""" + assert ProxyState.HEALTHY.value == "healthy" + assert ProxyState.PENDING_RECYCLE.value == "pending_recycle" + assert ProxyState.RECYCLING.value == "recycling" + assert ProxyState.PENDING.value == "pending" + + +class TestProxyInfo: + """Test cases for ProxyInfo dataclass.""" + + def test_proxy_info_creation(self): + """Test creating a ProxyInfo instance.""" + now = datetime.datetime.now(datetime.timezone.utc) + info = ProxyInfo( + ip="192.168.1.1", + state=ProxyState.HEALTHY, + created_at=now, + provider="aws", + instance="default", + age_seconds=3600 + ) + + assert info.ip == "192.168.1.1" + assert info.state == ProxyState.HEALTHY + assert info.created_at == now + assert info.provider == "aws" + assert info.instance == "default" + assert info.age_seconds == 3600 + + +class TestRollingDeploymentState: + """Test cases for RollingDeploymentState dataclass.""" + + def test_rolling_deployment_state_creation(self): + """Test creating a RollingDeploymentState instance.""" + state = RollingDeploymentState( + provider="aws", + instance="default" + ) + + assert state.provider == "aws" + assert state.instance == "default" + assert len(state.healthy_proxies) == 0 + assert len(state.pending_recycle) == 0 + assert len(state.recycling) == 0 + assert len(state.pending) == 0 + assert state.last_update is not None \ No newline at end of file diff --git a/tests/test_rolling_deployment_simple.py b/tests/test_rolling_deployment_simple.py new file mode 100644 index 0000000..be9f36c --- /dev/null +++ b/tests/test_rolling_deployment_simple.py @@ -0,0 +1,236 @@ +"""Simple unit tests for rolling deployment feature.""" + +import pytest +from unittest.mock import patch, Mock +from datetime import datetime, timedelta, timezone + +from cloudproxy.providers.rolling import rolling_manager +from cloudproxy.providers.settings import config + + +class TestRollingDeploymentSimple: + """Simple tests for rolling deployment functionality.""" + + @pytest.fixture(autouse=True) + def setup(self): + """Setup before each test.""" + # Save original config + self.original_rolling = config["rolling_deployment"]["enabled"] + self.original_min_available = config["rolling_deployment"]["min_available"] + self.original_batch_size = config["rolling_deployment"]["batch_size"] + + # Reset rolling manager state + rolling_manager.states.clear() + + yield + + # Restore original config + config["rolling_deployment"]["enabled"] = self.original_rolling + config["rolling_deployment"]["min_available"] = self.original_min_available + config["rolling_deployment"]["batch_size"] = self.original_batch_size + rolling_manager.states.clear() + + def test_can_recycle_with_sufficient_proxies(self): + """Test that recycling is allowed when we have sufficient proxies.""" + # Setup + config["rolling_deployment"]["enabled"] = True + + # Update health status first + rolling_manager.update_proxy_health( + "test", "default", + healthy_ips=["1.1.1.1", "2.2.2.2", "3.3.3.3", "4.4.4.4"], + pending_ips=[] + ) + + # Test recycling with sufficient proxies + can_recycle = rolling_manager.can_recycle_proxy( + provider="test", + instance="default", + proxy_ip="1.1.1.1", + total_healthy=4, + min_available=2, + batch_size=2, + rolling_enabled=True + ) + + assert can_recycle is True + + def test_cannot_recycle_below_minimum(self): + """Test that recycling is blocked when it would go below minimum.""" + # Setup + config["rolling_deployment"]["enabled"] = True + + # Update health status first + rolling_manager.update_proxy_health( + "test", "default", + healthy_ips=["1.1.1.1", "2.2.2.2"], + pending_ips=[] + ) + + # Test recycling that would go below minimum + can_recycle = rolling_manager.can_recycle_proxy( + provider="test", + instance="default", + proxy_ip="1.1.1.1", + total_healthy=2, + min_available=2, + batch_size=1, + rolling_enabled=True + ) + + assert can_recycle is False + + def test_batch_size_limit(self): + """Test that batch size limits concurrent recycling.""" + # Setup + config["rolling_deployment"]["enabled"] = True + + # Update health status first + rolling_manager.update_proxy_health( + "test", "default", + healthy_ips=["1.1.1.1", "2.2.2.2", "3.3.3.3", "4.4.4.4", "5.5.5.5"], + pending_ips=[] + ) + + # Try to recycle multiple proxies with batch_size=2 + recycled = [] + for ip in ["1.1.1.1", "2.2.2.2", "3.3.3.3"]: + if rolling_manager.can_recycle_proxy( + provider="test", + instance="default", + proxy_ip=ip, + total_healthy=5, + min_available=2, + batch_size=2, + rolling_enabled=True + ): + recycled.append(ip) + + # Should only allow 2 due to batch size + assert len(recycled) == 2 + + def test_disabled_rolling_allows_all_deletions(self): + """Test that disabling rolling deployment allows unrestricted deletions.""" + # Setup + config["rolling_deployment"]["enabled"] = False + + # Test recycling with rolling disabled + can_recycle = rolling_manager.can_recycle_proxy( + provider="test", + instance="default", + proxy_ip="1.1.1.1", + total_healthy=1, + min_available=5, # Much higher than healthy + batch_size=1, + rolling_enabled=False + ) + + assert can_recycle is True + + def test_state_tracking(self): + """Test that proxy states are tracked correctly.""" + # Update proxy health + rolling_manager.update_proxy_health( + "test", "default", + healthy_ips=["1.1.1.1", "2.2.2.2"], + pending_ips=["3.3.3.3"] + ) + + # Check state + state = rolling_manager.get_state("test", "default") + assert len(state.healthy_proxies) == 2 + assert len(state.pending) == 1 + assert "1.1.1.1" in state.healthy_proxies + assert "3.3.3.3" in state.pending + + # Mark as recycling + rolling_manager.mark_proxy_recycling("test", "default", "1.1.1.1") + state = rolling_manager.get_state("test", "default") + assert len(state.recycling) == 1 + assert "1.1.1.1" in state.recycling + + # Mark as recycled + rolling_manager.mark_proxy_recycled("test", "default", "1.1.1.1") + state = rolling_manager.get_state("test", "default") + assert len(state.recycling) == 0 + assert "1.1.1.1" not in state.recycling + + def test_min_scaling_adjustment(self): + """Test that min_scaling adjusts the effective minimum when appropriate.""" + # Setup + config["rolling_deployment"]["enabled"] = True + + # Update health status first + rolling_manager.update_proxy_health( + "test", "default", + healthy_ips=["1.1.1.1", "2.2.2.2", "3.3.3.3"], + pending_ips=[] + ) + + # Test with min_scaling that adjusts effective minimum + can_recycle = rolling_manager.can_recycle_proxy( + provider="test", + instance="default", + proxy_ip="1.1.1.1", + total_healthy=3, + min_available=3, # Would normally block all recycling + batch_size=1, + rolling_enabled=True, + min_scaling=3 # But min_scaling adjusts it to max(1, 3-1) = 2 + ) + + # Should allow recycling because effective_min_available = 2 + assert can_recycle is True + + def test_recycling_status_report(self): + """Test getting the recycling status report.""" + # Setup some state + rolling_manager.update_proxy_health( + "aws", "default", + healthy_ips=["1.1.1.1", "2.2.2.2"], + pending_ips=["3.3.3.3"] + ) + + rolling_manager.update_proxy_health( + "gcp", "production", + healthy_ips=["4.4.4.4"], + pending_ips=[] + ) + + # Get status + status = rolling_manager.get_recycling_status() + + assert "aws/default" in status + assert status["aws/default"]["healthy"] == 2 + assert status["aws/default"]["pending"] == 1 + + assert "gcp/production" in status + assert status["gcp/production"]["healthy"] == 1 + assert status["gcp/production"]["pending"] == 0 + + def test_should_create_replacement(self): + """Test determining if replacement proxies should be created.""" + # Setup state with proxies being recycled + rolling_manager.update_proxy_health( + "test", "default", + healthy_ips=["1.1.1.1"], + pending_ips=[] + ) + + # Test with min_scaling higher than current count + should_create = rolling_manager.should_create_replacement( + provider="test", + instance="default", + min_scaling=3 + ) + + assert should_create is True + + # Test with min_scaling met + should_create = rolling_manager.should_create_replacement( + provider="test", + instance="default", + min_scaling=1 + ) + + assert should_create is False \ No newline at end of file diff --git a/tests/test_user_data.sh b/tests/test_user_data.sh index fa31596..73b2976 100644 --- a/tests/test_user_data.sh +++ b/tests/test_user_data.sh @@ -20,7 +20,7 @@ MinSpareServers 5 MaxSpareServers 20 StartServers 10 MaxRequestsPerChild 0 -Allow 127.0.0.1 +Allow 0.0.0.0/0 ViaProxyName "tinyproxy" ConnectPort 443 ConnectPort 563