diff --git a/.env.example b/.env.example index 572b0cea..6f2e7058 100644 --- a/.env.example +++ b/.env.example @@ -37,9 +37,6 @@ VALIDATOR_PORT=8090 # Port for the TEE service TEE_PORT=8080 -# Port for the VPN service (if needed for credentials mining) -VPN_PORT=3128 - # ========== API CONFIGURATION ========== # API key to protect /monitor endpoints (for validator) API_KEY="" @@ -91,7 +88,7 @@ MASA_TEE_API=https://tee-api.masa.ai # WORKER_MEMORY_LIMIT=8G # WORKER_CPU_LIMIT=2 # WORKER_MEMORY_RESERVATION=4G -# WORKER_MEMORY_LIMIT=1 +# WORKER_CPU_RESERVATION=1 # ====== MEMORY LIMITS ONLY (16GB RAM) ====== # Use this if you just want to prevent OOM errors but don't want to limit CPU @@ -100,12 +97,6 @@ MASA_TEE_API=https://tee-api.masa.ai # WORKER_MEMORY_LIMIT=8G # WORKER_MEMORY_RESERVATION=4G -# Cookie Updater Configuration -COOKIES_REMOTE_HOST= -COOKIES_REMOTE_USER=azureuser -COOKIES_REMOTE_DIR=/tmp/cookies-upload - - # ============ PERMANENT TELEMETRY STORAGE ========= # POSTGRES_HOST=your-postgres-host.example.com # POSTGRES_PORT=5432 diff --git a/.github/workflows/docker-build-extra.yml b/.github/workflows/docker-build-extra.yml deleted file mode 100644 index 2296651f..00000000 --- a/.github/workflows/docker-build-extra.yml +++ /dev/null @@ -1,45 +0,0 @@ -name: Build Extra Docker Images (main/tags) - -on: - push: - branches: - - main - tags: - - "v*" - -concurrency: - group: docker-build-extra-${{ github.ref }} - cancel-in-progress: true - -permissions: - contents: read - -jobs: - build: - name: Build (${{ matrix.dockerfile }}) - runs-on: ubuntu-latest - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - include: - - dockerfile: Dockerfile.vpn - - dockerfile: Dockerfile.cookies.generator - - dockerfile: Dockerfile.cookies.updater.docker - - dockerfile: Dockerfile.cookies.updater.kubernetes - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Build (no push) - uses: docker/build-push-action@v5 - with: - context: . - file: ./${{ matrix.dockerfile }} - platforms: linux/amd64 - push: false - cache-from: type=gha,scope=${{ matrix.dockerfile }} - cache-to: type=gha,mode=max,scope=${{ matrix.dockerfile }} diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 282f424d..9f1d2063 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -22,10 +22,6 @@ jobs: matrix: include: - dockerfile: Dockerfile - - dockerfile: Dockerfile.vpn - - dockerfile: Dockerfile.cookies.generator - - dockerfile: Dockerfile.cookies.updater.docker - - dockerfile: Dockerfile.cookies.updater.kubernetes steps: - name: Checkout uses: actions/checkout@v4 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..8ae9954b --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,40 @@ +name: Tests + +on: + pull_request: + branches: + - main + push: + branches: + - main + +concurrency: + group: test-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + test: + name: Run Tests + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: "pip" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install .[dev] + + - name: Run tests + run: pytest -v --tb=short diff --git a/.gitignore b/.gitignore index 899aa070..90cb0780 100644 --- a/.gitignore +++ b/.gitignore @@ -6,29 +6,21 @@ userstories.md /bittensor /protocol/__pycache__ /neurons/__pycache__ -.env nodes.json *.encrypted /tests/__pycache__ /protocol/x/__pycache__ +/protocol/data_processing/__pycache__ *.pyc data/* .DS_Store -/protocol/data_processing/__pycache__ -/protocol/data_processing/__pycache__ -*.pyc notebooks/.DS_Store .conda -config.ovpn -auth.txt -cookies/ - .env* !.env.example key.pem *.conf - -kubeconfig*.yaml \ No newline at end of file +*.egg-info \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 2d709531..00000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "[python]": { - "editor.defaultFormatter": "ms-python.black-formatter", - "editor.formatOnSave": true - } -} diff --git a/Dockerfile.cookies.generator b/Dockerfile.cookies.generator deleted file mode 100644 index 8c72a439..00000000 --- a/Dockerfile.cookies.generator +++ /dev/null @@ -1,60 +0,0 @@ -FROM python:3.11-slim - -# Update package repositories and install Chromium dependencies -RUN apt-get update && apt-get install -y \ - curl \ - wget \ - gnupg \ - ca-certificates \ - chromium \ - chromium-driver \ - fonts-liberation \ - xdg-utils \ - xvfb \ - dbus-x11 \ - && rm -rf /var/lib/apt/lists/* - -# Set Chrome path for undetected-chromedriver -ENV CHROME_PATH=/usr/bin/chromium -ENV CHROMIUM_PATH=/usr/bin/chromium -ENV CHROMEDRIVER_PATH=/usr/bin/chromedriver -ENV PYTHONUNBUFFERED=1 -ENV DISPLAY=:99 - -# Set up working directory -WORKDIR /app - -# Script to start Xvfb for a proper virtual display with good resolution -RUN echo '#!/bin/bash\nXvfb :99 -screen 0 1920x1080x24 -ac +extension GLX +render -noreset &\nsleep 2\nexec "$@"' > /entrypoint.sh && \ - chmod +x /entrypoint.sh - -# Install Python dependencies with exact versions to ensure compatibility -RUN pip install --no-cache-dir \ - requests==2.31.0 \ - python-dotenv==1.0.0 \ - selenium==4.14.0 \ - webdriver-manager==4.0.1 \ - selenium-stealth==1.0.6 \ - undetected-chromedriver==3.5.3 \ - pyvirtualdisplay \ - asyncio - -# Create cookies directory -RUN mkdir -p /app/cookies - -# Copy the script -COPY scripts/cookie_grabber.py /app/cookie_grabber.py - -# Set default OUTPUT_DIR -ENV OUTPUT_DIR=/app/cookies - -# Add an environment variable to indicate this is running in Docker -ENV RUNNING_IN_DOCKER=true - -# Add VNC for optional remote viewing if needed -RUN apt-get update && apt-get install -y x11vnc -RUN echo '#!/bin/bash\nXvfb :99 -screen 0 1920x1080x24 -ac +extension GLX +render -noreset &\nsleep 2\nif [ "$ENABLE_VNC" = "true" ]; then\n x11vnc -display :99 -forever -nopw -quiet &\n echo "VNC server started on port 5900"\nfi\nexec "$@"' > /entrypoint.sh && \ - chmod +x /entrypoint.sh - -# Use the Xvfb entrypoint -ENTRYPOINT ["/entrypoint.sh", "python", "/app/cookie_grabber.py"] \ No newline at end of file diff --git a/Dockerfile.cookies.updater.docker b/Dockerfile.cookies.updater.docker deleted file mode 100644 index 636c3521..00000000 --- a/Dockerfile.cookies.updater.docker +++ /dev/null @@ -1,103 +0,0 @@ -FROM alpine:latest - -# Install required packages -RUN apk add --no-cache \ - bash \ - openssh-client \ - docker-cli \ - curl - -# Create necessary directories -RUN mkdir -p /app/cookies /root/.ssh - -# Copy the update script -COPY ./scripts/update_cookies_docker.sh /app/ -RUN chmod +x /app/update_cookies_docker.sh - -# Set working directory -WORKDIR /app - -# Create entrypoint script -RUN echo '#!/bin/bash' > /app/entrypoint.sh && \ - echo 'set -e' >> /app/entrypoint.sh && \ - echo '' >> /app/entrypoint.sh && \ - echo '# Wait for the cookies to be generated' >> /app/entrypoint.sh && \ - echo 'echo "Waiting for cookies to be available..."' >> /app/entrypoint.sh && \ - echo 'while [ ! "$(ls -A /app/cookies 2>/dev/null)" ]; do' >> /app/entrypoint.sh && \ - echo ' sleep 2' >> /app/entrypoint.sh && \ - echo 'done' >> /app/entrypoint.sh && \ - echo '' >> /app/entrypoint.sh && \ - echo '# Copy and set permissions for SSH key' >> /app/entrypoint.sh && \ - echo 'if [ -f "/app/key.pem" ]; then' >> /app/entrypoint.sh && \ - echo ' echo "Found SSH key at /app/key.pem"' >> /app/entrypoint.sh && \ - echo ' cp "/app/key.pem" /root/.ssh/id_rsa' >> /app/entrypoint.sh && \ - echo ' chmod 600 /root/.ssh/id_rsa' >> /app/entrypoint.sh && \ - echo '' >> /app/entrypoint.sh && \ - echo ' # Disable host key checking for non-interactive use' >> /app/entrypoint.sh && \ - echo ' echo "StrictHostKeyChecking no" > /root/.ssh/config' >> /app/entrypoint.sh && \ - echo 'else' >> /app/entrypoint.sh && \ - echo ' echo "SSH key not found at /app/key.pem"' >> /app/entrypoint.sh && \ - echo ' exit 1' >> /app/entrypoint.sh && \ - echo 'fi' >> /app/entrypoint.sh && \ - echo '' >> /app/entrypoint.sh && \ - echo '# If we are running locally (on the same host as the Docker daemon)' >> /app/entrypoint.sh && \ - echo 'if [ "$REMOTE_HOST" = "localhost" ] || [ "$REMOTE_HOST" = "127.0.0.1" ]; then' >> /app/entrypoint.sh && \ - echo ' echo "Running in local mode..."' >> /app/entrypoint.sh && \ - echo ' volume_name="cookies-volume"' >> /app/entrypoint.sh && \ - echo ' echo "Copying cookies to volume $volume_name"' >> /app/entrypoint.sh && \ - echo ' # Get the volume mount point within the Docker daemon' >> /app/entrypoint.sh && \ - echo ' volume_path=$(docker volume inspect --format "{{ .Mountpoint }}" $volume_name)' >> /app/entrypoint.sh && \ - echo ' if [ -z "$volume_path" ]; then' >> /app/entrypoint.sh && \ - echo ' echo "Error: Could not find mount point for volume $volume_name"' >> /app/entrypoint.sh && \ - echo ' exit 1' >> /app/entrypoint.sh && \ - echo ' fi' >> /app/entrypoint.sh && \ - echo ' echo "Volume path: $volume_path"' >> /app/entrypoint.sh && \ - echo ' # Copy the cookies directly to the volume location' >> /app/entrypoint.sh && \ - echo ' # Note: This assumes Docker is running in privileged mode or has access to volume paths' >> /app/entrypoint.sh && \ - echo ' echo "Finding compatible container to help with copy..."' >> /app/entrypoint.sh && \ - echo ' # Find another container that already has the volume mounted' >> /app/entrypoint.sh && \ - echo ' container_id=$(docker ps -q --filter volume=$volume_name | head -n 1)' >> /app/entrypoint.sh && \ - echo ' if [ -n "$container_id" ]; then' >> /app/entrypoint.sh && \ - echo ' echo "Found container $container_id with volume $volume_name"' >> /app/entrypoint.sh && \ - echo ' # Get the mount point inside the container' >> /app/entrypoint.sh && \ - echo ' container_mount=$(docker inspect --format "{{ range .Mounts }}{{ if eq .Name \"$volume_name\" }}{{ .Destination }}{{ end }}{{ end }}" $container_id)' >> /app/entrypoint.sh && \ - echo ' echo "Container mount point: $container_mount"' >> /app/entrypoint.sh && \ - echo ' # Copy files via the container' >> /app/entrypoint.sh && \ - echo ' for file in /app/cookies/*; do' >> /app/entrypoint.sh && \ - echo ' if [ -f "$file" ]; then' >> /app/entrypoint.sh && \ - echo ' filename=$(basename "$file")' >> /app/entrypoint.sh && \ - echo ' echo "Copying $filename..."' >> /app/entrypoint.sh && \ - echo ' cat "$file" | docker exec -i $container_id sh -c "cat > $container_mount/$filename"' >> /app/entrypoint.sh && \ - echo ' docker exec $container_id chown 1000:1000 "$container_mount/$filename"' >> /app/entrypoint.sh && \ - echo ' fi' >> /app/entrypoint.sh && \ - echo ' done' >> /app/entrypoint.sh && \ - echo ' echo "Cookies copied successfully via container $container_id"' >> /app/entrypoint.sh && \ - echo ' else' >> /app/entrypoint.sh && \ - echo ' echo "No containers with this volume found. Will try direct volume copying instead."' >> /app/entrypoint.sh && \ - echo ' # Try to copy files directly to the volume (this may fail depending on Docker configuration)' >> /app/entrypoint.sh && \ - echo ' # Create a small temporary container for copying' >> /app/entrypoint.sh && \ - echo ' docker run --rm -v $volume_name:/volume alpine sh -c "ls -la /volume"' >> /app/entrypoint.sh && \ - echo ' docker run --rm -v $volume_name:/volume alpine sh -c "mkdir -p /volume/test && echo test > /volume/test/test.txt"' >> /app/entrypoint.sh && \ - echo ' echo "Created test file. Attempting actual copy..."' >> /app/entrypoint.sh && \ - echo ' docker run --rm -v $volume_name:/volume alpine ls -la /volume' >> /app/entrypoint.sh && \ - echo ' # Note: We are avoiding the direct path mount that was causing issues' >> /app/entrypoint.sh && \ - echo ' for file in /app/cookies/*; do' >> /app/entrypoint.sh && \ - echo ' if [ -f "$file" ]; then' >> /app/entrypoint.sh && \ - echo ' filename=$(basename "$file")' >> /app/entrypoint.sh && \ - echo ' echo "Copying $filename..."' >> /app/entrypoint.sh && \ - echo ' cat "$file" | docker run --rm -i -v $volume_name:/volume alpine sh -c "cat > /volume/$filename && chmod 644 /volume/$filename && chown 1000:1000 /volume/$filename"' >> /app/entrypoint.sh && \ - echo ' fi' >> /app/entrypoint.sh && \ - echo ' done' >> /app/entrypoint.sh && \ - echo ' echo "Files copied to volume $volume_name"' >> /app/entrypoint.sh && \ - echo ' fi' >> /app/entrypoint.sh && \ - echo 'else' >> /app/entrypoint.sh && \ - echo ' echo "Running in remote mode - transferring to hosts defined in COOKIES_REMOTE_HOSTS..."' >> /app/entrypoint.sh && \ - echo ' # Execute the update script' >> /app/entrypoint.sh && \ - echo ' /app/update_cookies_docker.sh' >> /app/entrypoint.sh && \ - echo 'fi' >> /app/entrypoint.sh && \ - echo '' >> /app/entrypoint.sh && \ - echo 'echo "Cookie update process completed!"' >> /app/entrypoint.sh - -RUN chmod +x /app/entrypoint.sh - -ENTRYPOINT ["/app/entrypoint.sh"] \ No newline at end of file diff --git a/Dockerfile.cookies.updater.kubernetes b/Dockerfile.cookies.updater.kubernetes deleted file mode 100644 index 7cfe4b8a..00000000 --- a/Dockerfile.cookies.updater.kubernetes +++ /dev/null @@ -1,44 +0,0 @@ -FROM alpine:latest - -# Install required packages for Kubernetes operations -RUN apk add --no-cache \ - bash \ - curl \ - kubectl - -# Create necessary directories -RUN mkdir -p /app/cookies - -# Copy the Kubernetes update script -COPY ./scripts/update_cookies_kubernetes.sh /app/ -RUN chmod +x /app/update_cookies_kubernetes.sh - -# Set working directory -WORKDIR /app - -# Create entrypoint script -RUN echo '#!/bin/bash' > /app/entrypoint.sh && \ - echo 'set -e' >> /app/entrypoint.sh && \ - echo '' >> /app/entrypoint.sh && \ - echo '# Wait for the cookies to be generated' >> /app/entrypoint.sh && \ - echo 'echo "Waiting for cookies to be available..."' >> /app/entrypoint.sh && \ - echo 'while [ ! "$(ls -A /app/cookies 2>/dev/null)" ]; do' >> /app/entrypoint.sh && \ - echo ' sleep 2' >> /app/entrypoint.sh && \ - echo 'done' >> /app/entrypoint.sh && \ - echo '' >> /app/entrypoint.sh && \ - echo '# Check if kubeconfig is available' >> /app/entrypoint.sh && \ - echo 'if [ ! -f "/app/kubeconfig.yaml" ]; then' >> /app/entrypoint.sh && \ - echo ' echo "Error: kubeconfig.yaml file not found at /app/kubeconfig.yaml"' >> /app/entrypoint.sh && \ - echo ' echo "Please mount your kubeconfig file to /app/kubeconfig.yaml"' >> /app/entrypoint.sh && \ - echo ' exit 1' >> /app/entrypoint.sh && \ - echo 'fi' >> /app/entrypoint.sh && \ - echo '' >> /app/entrypoint.sh && \ - echo '# Execute the Kubernetes update script' >> /app/entrypoint.sh && \ - echo 'echo "Running Kubernetes cookie update..."' >> /app/entrypoint.sh && \ - echo '/app/update_cookies_kubernetes.sh' >> /app/entrypoint.sh && \ - echo '' >> /app/entrypoint.sh && \ - echo 'echo "Kubernetes cookie update process completed!"' >> /app/entrypoint.sh - -RUN chmod +x /app/entrypoint.sh - -ENTRYPOINT ["/app/entrypoint.sh"] \ No newline at end of file diff --git a/Dockerfile.vpn b/Dockerfile.vpn deleted file mode 100644 index 367da012..00000000 --- a/Dockerfile.vpn +++ /dev/null @@ -1,14 +0,0 @@ -FROM ubuntu:22.04 - -RUN apt-get update && \ - apt-get install -y openvpn curl iptables tinyproxy - -# Configure system -RUN echo "net.ipv4.ip_forward=1" > /etc/sysctl.conf && \ - echo "Allow 0.0.0.0/0" >> /etc/tinyproxy/tinyproxy.conf - -# Setup entrypoint script -COPY entrypoint-vpn.sh /entrypoint.sh -RUN chmod +x /entrypoint.sh - -ENTRYPOINT ["/entrypoint.sh"] \ No newline at end of file diff --git a/README.vpn.md b/README.vpn.md deleted file mode 100644 index 4ebc200d..00000000 --- a/README.vpn.md +++ /dev/null @@ -1,216 +0,0 @@ -# ๐Ÿš€ Subnet-42 Miner with VPN Setup Guide - -This guide helps you set up your Subnet-42 miner with a VPN for residential IP routing. This allows your miner to be publicly accessible while your worker routes outbound traffic through a residential VPN IP. - -## ๐Ÿ“‹ Prerequisites - -- Docker installed -- **TorGuard VPN subscription** (strongly recommended for residential IPs) -- Twitter account credentials - -## ๐Ÿ”ง Setup Steps - -### 1๏ธโƒฃ Prepare Your VPN Configuration - -#### TorGuard Setup for Residential IPs - -TorGuard is specifically recommended because they offer residential IP addresses, which are crucial for this setup. - -1. **Subscribe to TorGuard with Residential IP add-on**: - - - Sign up for TorGuard VPN - - Purchase the "Residential IP" add-on - - Request residential IPs in your desired location - -2. Create the required directories: - - ```bash - mkdir -p vpn cookies - ``` - -3. **Create auth.txt file**: - - Create a file with your TorGuard credentials: - - ``` - your_torguard_username - your_torguard_password - ``` - - Save this to `vpn/auth.txt` - -4. **Configure OpenVPN**: - - - Log into your TorGuard account - - Download the OpenVPN configuration files - - Create a `config.ovpn` file in `vpn/` with your residential servers: - - ``` - client - dev tun - proto udp - # Multiple residential servers for redundancy - remote - remote - remote-random - remote-cert-tls server - auth SHA256 - key-direction 1 - # Add your TorGuard certificates and keys below - ... (rest of configuration) ... - ``` - -### 2๏ธโƒฃ Generate Twitter Cookies - -You have two options for generating Twitter cookies: - -#### Option 1: Use the Docker Cookie Generator Service (Automated) - -1. **Configure Twitter Credentials**: - - Add your Twitter account credentials to your .env file: - - ``` - # Add your Twitter accounts in this format - TWITTER_ACCOUNTS="username1:password1,username2:password2" - # Add backup email for verification (REQUIRED) - TWITTER_EMAIL="your_email@example.com" - ``` - - The `TWITTER_EMAIL` is used for verification challenges during login. - -2. **Run the Cookie Generator Service**: - - ```bash - docker compose --profile cookies up - ``` - - This service will: - - - Log in to your Twitter accounts - - Generate authentication cookies - - Save them to the `cookies/` directory in your project - - Handle verification challenges with manual intervention if needed - -3. **Verify Cookie Generation**: - - ```bash - ls -la ./cookies/ - ``` - - You should see files named `_twitter_cookies.json` for each account. - -#### Option 2: Run the Manual CAPTCHA Intervention Script (Recommended) - -If you're encountering CAPTCHA challenges or authentication issues with the automated method, use this approach: - -1. **Install Required Dependencies**: - - ```bash - # For running cookie_grabber.py directly with Python (non-headless mode) - pip install selenium selenium-stealth python-dotenv - ``` - - > **Note**: Running the script with Python directly opens a visible Chrome browser window, allowing you to interact with CAPTCHAs and verification challenges. This is different from the Docker approach which runs in headless mode. - -2. **Set Environment Variables**: - - ```bash - export TWITTER_ACCOUNTS="username1:password1,username2:password2" - export TWITTER_EMAIL="your_email@example.com" - ``` - - Or create a `.env` file in the scripts directory with these variables. - -3. **Run the Cookie Grabber Script**: - - ```bash - cd scripts - python cookie_grabber.py - ``` - -4. **Manual CAPTCHA Solving**: - - - The script opens a visible browser window - - When a CAPTCHA/authentication challenge appears, you'll see: - ``` - ================================================================================ - MANUAL INTERVENTION REQUIRED for account: username - Please solve the CAPTCHA or authentication challenge in the browser window - ================================================================================ - ``` - - Manually solve the challenge in the browser window - - The script will detect when you've solved it and continue automatically - - Cookies will be saved to the `../cookies` directory - -This manual approach is more reliable for accounts that frequently encounter verification challenges, as you can directly interact with the browser to complete any verification steps. - -### 3๏ธโƒฃ Launch Everything with One Command - -Once you have: - -- VPN files in `vpn/` (auth.txt and config.ovpn) -- Cookie files in the `cookies/` directory - -You can start the full system: - -```bash -docker compose --profile miner-vpn up -d -``` - -This command will: - -1. Start the VPN service using your TorGuard residential IPs -2. Launch the TEE worker using the VPN -3. Start your subnet-42 miner - -## ๐Ÿงช Testing Your Setup - -### Verify Worker-VPN Routing - -To make sure your worker-vpn container is properly routing through the VPN: - -1. **Install curl in the worker-vpn container**: - - ```bash - docker exec -it worker-vpn apt-get update && docker exec -it worker-vpn apt-get install -y curl - ``` - -2. **Check the IP address of the worker-vpn container**: - - ```bash - # Get the worker-vpn IP - docker exec worker-vpn curl -s https://ifconfig.me - ``` - - Verify that this shows a different IP than your host machine, confirming traffic is routing through the VPN. - -### Why Residential IPs Matter - -Regular datacenter VPN IPs are often flagged and blocked by services. Residential IPs are much less likely to be detected, making them essential for reliable operation. - -## ๐Ÿ› ๏ธ Troubleshooting - -### Cookie Generator Issues - -- If the Docker cookie generator fails with `'ChromeOptions' object has no attribute 'headless'` error, use the manual script approach (Option 2) -- If manual cookie generation fails with timeout errors, you can modify the `WAITING_TIME` constant in the script (default: 3600 seconds) -- For accounts that require verification, ensure you've set the `TWITTER_EMAIL` environment variable correctly -- If using email verification, check that the email account is accessible and can receive Twitter verification codes -- Make sure Chrome is properly installed on your system when using the manual script - -### Monitoring the Cookie Generation Process - -- When using the Docker cookie generator, you can enable VNC to view the browser: - ``` - ENABLE_VNC=true docker compose --profile cookies up - ``` - Then connect to the container using a VNC viewer on port 5900 - -### Advanced Email Verification - -For accounts that frequently require email verification: - -- The script supports a special password format: `himynameisjohn` -- It will use your `TWITTER_EMAIL` with plus addressing, like: `your_email+john@example.com` -- This helps manage multiple verification emails in a single inbox diff --git a/docker-compose.yml b/docker-compose.yml index 2d6f02de..e931d272 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ services: neuron: profiles: - ["miner", "miner-tee", "miner-tee-vpn", "validator", "validator-tee"] + ["miner", "miner-tee", "validator", "validator-tee"] image: masaengineering/subnet-42:latest pull_policy: always restart: unless-stopped @@ -66,167 +66,12 @@ services: mode: "non-blocking" max-buffer-size: "4m" - worker-vpn: - profiles: ["miner-tee-vpn"] - image: masaengineering/tee-worker:latest - pull_policy: always - restart: unless-stopped - deploy: - resources: - limits: - memory: ${WORKER_MEMORY_LIMIT:-0} - cpus: ${WORKER_CPU_LIMIT:-0} - reservations: - memory: ${WORKER_MEMORY_RESERVATION:-0} - cpus: ${WORKER_CPU_RESERVATION:-0} - ports: - - "${TEE_PORT:-8080}:${TEE_PORT:-8080}" - environment: - - http_proxy=http://vpn:${VPN_PORT:-3128} - - https_proxy=http://vpn:${VPN_PORT:-3128} - devices: - - /dev/sgx_enclave - - /dev/sgx_provision - volumes: - - cookies-volume:/home/masa - - ./.env:/home/masa/.env - depends_on: - vpn: - condition: service_healthy - logging: - driver: json-file - options: - max-size: "5m" - max-file: "2" - compress: "true" - mode: "non-blocking" - max-buffer-size: "4m" - - vpn: - profiles: ["vpn", "miner-tee-vpn"] - build: - context: . - dockerfile: Dockerfile.vpn - image: vpn-service:latest - pull_policy: never - restart: unless-stopped - cap_add: - - NET_ADMIN - devices: - - /dev/net/tun - ports: - - "${VPN_PORT:-3128}:${VPN_PORT:-3128}" - volumes: - - ./vpn:/etc/openvpn/config - env_file: .env - healthcheck: - test: ["CMD", "bash", "-c", '[ "$(cat /tmp/vpn_ready)" = "1" ]'] - interval: 5s - timeout: 5s - retries: 10 - start_period: 30s - logging: - driver: json-file - options: - max-size: "5m" - max-file: "2" - compress: "true" - mode: "non-blocking" - max-buffer-size: "4m" - - cookies-generator: - profiles: ["cookies"] - image: cookie-generator:latest - pull_policy: never - build: - context: . - dockerfile: Dockerfile.cookies.generator - volumes: - - ./.env:/app/.env - - ./cookies:/app/cookies - environment: - - PYTHONUNBUFFERED=1 - - ENABLE_VNC=true - - RUNNING_IN_DOCKER=true - restart: "no" - env_file: .env - logging: - driver: json-file - options: - max-size: "5m" - max-file: "2" - compress: "true" - mode: "non-blocking" - max-buffer-size: "4m" - - cookies-updater-docker: - profiles: ["cookies"] - image: cookies-updater-docker:latest - pull_policy: never - build: - context: . - dockerfile: Dockerfile.cookies.updater.docker - volumes: - - ./.env:/app/.env - - ./cookies:/app/cookies - - ./key.pem:/app/key.pem:ro - - /var/run/docker.sock:/var/run/docker.sock - environment: - - COOKIES_REMOTE_HOSTS=${COOKIES_REMOTE_HOSTS:-localhost} - - COOKIES_REMOTE_USER=${COOKIES_REMOTE_USER:-azureuser} - - COOKIES_REMOTE_DIR=${COOKIES_REMOTE_DIR:-/tmp/cookies-upload} - - PYTHONUNBUFFERED=1 - restart: "no" - env_file: .env - logging: - driver: json-file - options: - max-size: "5m" - max-file: "2" - compress: "true" - mode: "non-blocking" - max-buffer-size: "4m" - depends_on: - cookies-generator: - condition: service_completed_successfully - - cookies-updater-kubernetes: - profiles: ["cookies"] - image: cookies-updater-kubernetes:latest - pull_policy: never - build: - context: . - dockerfile: Dockerfile.cookies.updater.kubernetes - volumes: - - ./.env:/app/.env - - ./cookies:/app/cookies - - ./kubeconfig.yaml:/app/kubeconfig.yaml:ro - environment: - - DEPLOYMENTS=${DEPLOYMENTS} - - NAMESPACE=${NAMESPACE} - - PYTHONUNBUFFERED=1 - restart: "no" - env_file: .env - logging: - driver: json-file - options: - max-size: "5m" - max-file: "2" - compress: "true" - mode: "non-blocking" - max-buffer-size: "4m" - depends_on: - cookies-generator: - condition: service_completed_successfully - watchtower: profiles: - ["miner", "miner-tee", "miner-tee-vpn", "validator", "validator-tee"] + ["miner", "miner-tee", "validator", "validator-tee"] image: containrrr/watchtower:latest volumes: - /var/run/docker.sock:/var/run/docker.sock restart: unless-stopped command: --interval 7200 --cleanup -volumes: - cookies-volume: diff --git a/entrypoint-vpn.sh b/entrypoint-vpn.sh deleted file mode 100644 index e6b8ab1c..00000000 --- a/entrypoint-vpn.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash -set -e - -echo "Starting VPN service..." - -# Update tinyproxy port configuration -echo "Configuring tinyproxy..." -sed -i "s/Port 8888/Port ${VPN_PORT:-3128}/g" /etc/tinyproxy/tinyproxy.conf - -# Make sure tinyproxy will accept connections from all IPs -grep -q "^Allow 0.0.0.0/0" /etc/tinyproxy/tinyproxy.conf || echo "Allow 0.0.0.0/0" >> /etc/tinyproxy/tinyproxy.conf - -# Set additional tinyproxy configs for stability -sed -i 's/^#DisableViaHeader Yes/DisableViaHeader Yes/' /etc/tinyproxy/tinyproxy.conf -sed -i 's/^MaxClients 100/MaxClients 200/' /etc/tinyproxy/tinyproxy.conf -sed -i 's/^Timeout 600/Timeout 1800/' /etc/tinyproxy/tinyproxy.conf - -# Apply sysctl setting -echo "Setting up IP forwarding..." -sysctl -p - -# Start tinyproxy in background -echo "Starting tinyproxy service..." -service tinyproxy restart - -# Setup NAT for VPN tunnel -echo "Setting up NAT routing..." -iptables -t nat -A POSTROUTING -o tun0 -j MASQUERADE - -# Create a status file that the healthcheck will check -touch /tmp/vpn_ready -echo "0" > /tmp/vpn_ready - -# Start a background process to watch for the initialization message -echo "Starting OpenVPN log monitor..." -openvpn --config /etc/openvpn/config/config.ovpn --auth-user-pass /etc/openvpn/config/auth.txt --auth-nocache --verb 3 2>&1 | tee /var/log/openvpn.log | while read line; do - echo "$line" - if [[ "$line" == *"Initialization Sequence Completed"* ]]; then - echo "VPN CONNECTED SUCCESSFULLY!" - echo "1" > /tmp/vpn_ready - fi -done & - -# Keep container running -echo "VPN setup complete, keeping container alive..." -tail -f /dev/null \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index d5a3b1fc..cc7a5a55 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,10 @@ dev = [ "pytest-asyncio==0.25.3" ] +[tool.setuptools.packages.find] +include = ["validator*", "miner*", "interfaces*", "neurons*", "db*"] + [tool.pytest.ini_options] asyncio_mode = "auto" testpaths = ["tests"] -python_files = ["test_*.py"] \ No newline at end of file +python_files = ["test_*.py"] \ No newline at end of file diff --git a/scripts/cookie_grabber.py b/scripts/cookie_grabber.py deleted file mode 100644 index b9a58864..00000000 --- a/scripts/cookie_grabber.py +++ /dev/null @@ -1,2158 +0,0 @@ -#!/usr/bin/env python3 -import json -import time -import os -import logging -import datetime -import platform -import undetected_chromedriver as uc -from selenium.webdriver.common.by import By -from selenium.common.exceptions import WebDriverException -import random -import subprocess -import re -import shutil - -from selenium.webdriver.common.keys import Keys -from dotenv import load_dotenv - -# Setup logging first -logging.basicConfig( - level=logging.INFO, - format="%(asctime)s - %(levelname)s - %(message)s", - handlers=[logging.StreamHandler()], -) -logger = logging.getLogger(__name__) - -# Set output directory based on environment (robust to current working directory) -running_in_docker = os.environ.get("RUNNING_IN_DOCKER", "false").lower() == "true" -if running_in_docker: - OUTPUT_DIR = "/app/cookies" - logger.info(f"Docker environment detected, saving cookies to {OUTPUT_DIR}") -else: - # Resolve project root as the parent of the directory containing this script - script_dir = os.path.dirname(os.path.abspath(__file__)) - project_root = os.path.dirname(script_dir) - OUTPUT_DIR = os.path.join(project_root, "cookies") - logger.info(f"Local environment detected, saving cookies to {OUTPUT_DIR}") - -# Ensure output directory exists -os.makedirs(OUTPUT_DIR, exist_ok=True) - -# Cookie names we commonly observe from X sessions (informational). -COOKIE_NAMES = ["personalization_id", "kdt", "twid", "ct0", "auth_token", "att"] - -# Twitter domains to handle - We will only use x.com -TWITTER_DOMAINS = ["x.com"] - -# Twitter login URL -TWITTER_LOGIN_URL = "https://x.com/i/flow/login" - -# Constants -POLLING_INTERVAL = 1 # Check every 1 second -WAITING_TIME = 300 # Wait up to 5 minutes for manual verification -CLICK_WAIT = 5 # Wait 5 seconds after clicking buttons -POST_LOGIN_COOKIE_WAIT = 15 # Wait up to 15s for session cookies -POST_LOGIN_COOKIE_POLL_INTERVAL = 1 # Poll every second -REQUIRED_SESSION_COOKIES = ["auth_token", "ct0"] - -# Rate limit tracking -rate_limit_hits = {} # Track rate limit hits per account for exponential backoff -MAX_BACKOFF_SECONDS = 300 # Max 5 minutes between attempts - - -def get_account_backoff(username): - """Get current backoff time for an account based on recent rate limit hits.""" - hits = rate_limit_hits.get(username, 0) - if hits == 0: - return 0 - # Exponential backoff: 10s, 20s, 40s, 80s, 160s, 300s (max) - backoff = min(10 * (2 ** (hits - 1)), MAX_BACKOFF_SECONDS) - return backoff - - -def record_rate_limit_hit(username): - """Record a rate limit hit for an account.""" - rate_limit_hits[username] = rate_limit_hits.get(username, 0) + 1 - backoff = get_account_backoff(username) - logger.warning( - f"Rate limit hit for {username}. This is attempt {rate_limit_hits[username]}. " - f"Next backoff will be {backoff}s" - ) - - -def clear_rate_limit_hits(username): - """Clear rate limit hits after successful login.""" - if username in rate_limit_hits: - del rate_limit_hits[username] - - -def capture_screenshot(driver, username, reason="failure"): - """Capture a screenshot for debugging when something goes wrong.""" - try: - screenshots_dir = os.path.join(OUTPUT_DIR, "screenshots") - os.makedirs(screenshots_dir, exist_ok=True) - - timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") - filename = f"{username}_{reason}_{timestamp}.png" - filepath = os.path.join(screenshots_dir, filename) - - driver.save_screenshot(filepath) - logger.info(f"Screenshot saved: {filepath}") - return filepath - except Exception as e: - logger.warning(f"Failed to capture screenshot: {str(e)}") - return None - - -def is_rate_limited(driver): - """Check if X/Twitter is showing rate limit or "too many attempts" messages.""" - try: - rate_limit_indicators = [ - "rate limit", - "too many attempts", - "try again later", - "unusual activity", - "automated behavior", - "suspicious activity", - "temporarily locked", - "account temporarily", - "please wait", - "slow down", - "429", # HTTP status sometimes shown - ] - - page_text = driver.find_element(By.TAG_NAME, "body").text.lower() - current_url = driver.current_url.lower() - - for indicator in rate_limit_indicators: - if indicator in page_text or indicator in current_url: - logger.warning(f"Rate limit detected: '{indicator}' found") - return True - - # Check for specific error codes or states - error_selectors = [ - '[data-testid="error"]', - '[role="alert"]', - '.error-message', - '[class*="error"]', - '[class*="blocked"]', - ] - - for selector in error_selectors: - try: - elements = driver.find_elements(By.CSS_SELECTOR, selector) - for elem in elements: - if elem.is_displayed(): - text = elem.text.lower() - if any(ind in text for ind in rate_limit_indicators): - logger.warning(f"Rate limit element found: {text[:100]}") - return True - except: - pass - - return False - except Exception as e: - logger.debug(f"Error checking rate limit status: {str(e)}") - return False - - -def get_future_date(days=7, hours=0, minutes=0, seconds=0): - """ - Generate a slightly randomized ISO 8601 date string for a specified time in the future. - - Args: - days: Number of days in the future - hours: Number of hours to add - minutes: Number of minutes to add - seconds: Number of seconds to add - - Returns: - ISO 8601 formatted date string with slight randomization - """ - # Add slight randomization to make cookies appear more natural - random_seconds = random.uniform(0, 3600) # Random seconds (up to 1 hour) - random_minutes = random.uniform(0, 60) # Random minutes (up to 1 hour) - - future_date = datetime.datetime.now() + datetime.timedelta( - days=days, - hours=hours, - minutes=minutes + random_minutes, - seconds=seconds + random_seconds, - ) - - # Format in ISO 8601 format with timezone information - return future_date.strftime("%Y-%m-%dT%H:%M:%SZ") - - -def create_cookie_template(name, value, domain="x.com", expires=None): - """ - Create a standard cookie template with the given name and value. - Note: Cookie values should not contain double quotes as they cause errors in Go's HTTP client. - - Args: - name: Name of the cookie - value: Value of the cookie - domain: Domain for the cookie - expires: Optional expiration date string in ISO 8601 format - """ - # Ensure no quotes in cookie value to prevent HTTP header issues - if value.startswith('"') and value.endswith('"'): - value = value[1:-1] - value = value.replace('"', "") - - # If no expiration date is provided, use the default "0001-01-01T00:00:00Z" - if expires is None: - expires = "0001-01-01T00:00:00Z" - - return { - "Name": name, - "Value": value, - "Path": "", - "Domain": domain, - "Expires": expires, - "RawExpires": "", - "MaxAge": 0, - "Secure": False, - "HttpOnly": False, - "SameSite": 0, - "Raw": "", - "Unparsed": None, - } - - -def setup_realistic_profile(temp_profile): - """Set up a more realistic browser profile with history and common extensions.""" - - # Create history file structure - history_dir = os.path.join(temp_profile, "Default") - os.makedirs(history_dir, exist_ok=True) - - # Sample visited sites for history (just structure, not actual data) - common_sites = [ - "google.com", - "youtube.com", - "facebook.com", - "amazon.com", - "wikipedia.org", - ] - - # Create a dummy history file - history_file = os.path.join(history_dir, "History") - try: - with open(history_file, "w") as f: - # Just create an empty file to simulate history presence - f.write("") - - # Create bookmark file with common sites - bookmarks_file = os.path.join(history_dir, "Bookmarks") - bookmarks_data = { - "roots": { - "bookmark_bar": { - "children": [ - {"name": site, "url": f"https://{site}"} - for site in common_sites - ], - "date_added": str(int(time.time())), - "date_modified": str(int(time.time())), - "name": "Bookmarks Bar", - "type": "folder", - } - }, - "version": 1, - } - with open(bookmarks_file, "w") as f: - json.dump(bookmarks_data, f) - - # Create preferences file with some realistic settings - preferences_file = os.path.join(history_dir, "Preferences") - preferences_data = { - "browser": { - "last_known_google_url": "https://www.google.com/", - "last_prompted_google_url": "https://www.google.com/", - "show_home_button": True, - "custom_chrome_frame": False, - }, - "homepage": "https://www.google.com", - "session": { - "restore_on_startup": 1, - "startup_urls": [f"https://{random.choice(common_sites)}"], - }, - "search": {"suggest_enabled": True}, - "translate": {"enabled": True}, - } - with open(preferences_file, "w") as f: - json.dump(preferences_data, f) - - logger.info("Created realistic browser profile with history and preferences") - except Exception as e: - logger.warning(f"Failed to create history files: {str(e)}") - - # Add a dummy extension folder to simulate common extensions - ext_dir = os.path.join(temp_profile, "Default", "Extensions") - os.makedirs(ext_dir, exist_ok=True) - - # Create dummy extension folders for common extensions - common_extensions = [ - "aapbdbdomjkkjkaonfhkkikfgjllcleb", # Google Translate - "ghbmnnjooekpmoecnnnilnnbdlolhkhi", # Google Docs - "cjpalhdlnbpafiamejdnhcphjbkeiagm", # uBlock Origin - ] - - for ext_id in common_extensions: - ext_path = os.path.join(ext_dir, ext_id) - os.makedirs(ext_path, exist_ok=True) - # Create a minimal manifest file - manifest_path = os.path.join(ext_path, "manifest.json") - try: - with open(manifest_path, "w") as f: - f.write("{}") - except Exception as e: - logger.warning(f"Failed to create extension manifest: {str(e)}") - - return temp_profile - - -def resolve_chrome_binary() -> str | None: - """Resolve a usable Chrome binary path. - - Priority: - 1) CHROME_BINARY env var - 2) Known platform defaults - 3) First match on PATH - """ - env_path = os.environ.get("CHROME_BINARY") - if env_path and os.path.exists(env_path): - return env_path - - # macOS default - mac_path = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" - if os.path.exists(mac_path): - return mac_path - - # Linux common paths - for candidate in [ - shutil.which("google-chrome"), - shutil.which("google-chrome-stable"), - shutil.which("chromium"), - shutil.which("chromium-browser"), - shutil.which("chrome"), - ]: - if candidate and os.path.exists(candidate): - return candidate - - # Windows typical locations (best-effort, user can set CHROME_BINARY) - win_paths = [ - os.path.expandvars(r"%ProgramFiles%/Google/Chrome/Application/chrome.exe"), - os.path.expandvars(r"%ProgramFiles(x86)%/Google/Chrome/Application/chrome.exe"), - os.path.expandvars(r"%LocalAppData%/Google/Chrome/Application/chrome.exe"), - ] - for p in win_paths: - if os.path.exists(p): - return p - - return None - - -def get_chrome_major_version(chrome_binary: str | None) -> int | None: - """Return local Chrome major version (e.g., 141) by invoking --version. - - If chrome_binary is None, attempts common commands on PATH. - """ - candidates = [] - if chrome_binary: - candidates.append([chrome_binary, "--version"]) - # Common CLI names - for name in [ - "google-chrome", - "google-chrome-stable", - "chromium", - "chromium-browser", - "chrome", - ]: - exe = shutil.which(name) - if exe: - candidates.append([exe, "--version"]) - - version_regex = re.compile(r"(Chrome|Chromium)\s+([0-9]+)\.") - - for cmd in candidates: - try: - out = subprocess.check_output( - cmd, stderr=subprocess.STDOUT, text=True - ).strip() - m = version_regex.search(out) - if m: - return int(m.group(2)) - except Exception: - continue - - return None - - -def kill_orphan_chrome_processes(): - """Kill any orphaned Chrome processes that might be locking the profile.""" - try: - import signal - # Find Chrome processes using undetected_chromedriver paths - result = subprocess.run( - ["pgrep", "-f", "undetected_chromedriver"], - capture_output=True, - text=True - ) - if result.stdout.strip(): - pids = result.stdout.strip().split('\n') - for pid in pids: - try: - os.kill(int(pid), signal.SIGTERM) - logger.info(f"Killed orphaned chromedriver process: {pid}") - except (ProcessLookupError, ValueError): - pass - except Exception as e: - logger.debug(f"Error checking for orphaned processes: {e}") - - -def clear_chromedriver_cache(): - """Clear undetected_chromedriver cache to force re-download of correct version.""" - try: - import undetected_chromedriver as uc - # Get the cache directory used by undetected_chromedriver - cache_dir = os.path.expanduser("~/.undetected_chromedriver") - if os.path.exists(cache_dir): - logger.info(f"Clearing chromedriver cache: {cache_dir}") - shutil.rmtree(cache_dir, ignore_errors=True) - - # Also clear the selenium webdriver cache - selenium_cache = os.path.expanduser("~/.wdm") - if os.path.exists(selenium_cache): - logger.info(f"Clearing selenium webdriver cache: {selenium_cache}") - shutil.rmtree(selenium_cache, ignore_errors=True) - - # Clear Library/Application Support path on macOS - mac_cache = os.path.expanduser("~/Library/Application Support/undetected_chromedriver") - if os.path.exists(mac_cache): - logger.info(f"Clearing macOS chromedriver cache: {mac_cache}") - shutil.rmtree(mac_cache, ignore_errors=True) - - logger.info("Chromedriver cache cleared successfully") - except Exception as e: - logger.warning(f"Could not clear chromedriver cache: {str(e)}") - - -def clear_profile_cache(profile_dir, clear_heavy_cache=False): - """Clear profile lock files always; clear heavy caches only when requested.""" - cache_dirs = [ - os.path.join(profile_dir, "Default", "Cache"), - os.path.join(profile_dir, "Default", "Code Cache"), - os.path.join(profile_dir, "Default", "GPUCache"), - os.path.join(profile_dir, "Default", "Service Worker"), - os.path.join(profile_dir, "Default", "DawnCache"), - os.path.join(profile_dir, "Default", "ShaderCache"), - os.path.join(profile_dir, "GrShaderCache"), - os.path.join(profile_dir, "ShaderCache"), - ] - - # Also clear lock files that can cause issues - lock_files = [ - os.path.join(profile_dir, "SingletonLock"), - os.path.join(profile_dir, "SingletonCookie"), - os.path.join(profile_dir, "SingletonSocket"), - ] - - for lock_file in lock_files: - try: - if os.path.exists(lock_file): - os.remove(lock_file) - logger.debug(f"Removed lock file: {lock_file}") - except Exception as e: - logger.debug(f"Could not remove lock file {lock_file}: {e}") - - if clear_heavy_cache: - for cache_dir in cache_dirs: - try: - if os.path.exists(cache_dir): - shutil.rmtree(cache_dir) - logger.debug(f"Cleared cache directory: {cache_dir}") - except Exception as e: - logger.debug(f"Could not clear cache {cache_dir}: {e}") - - -def setup_driver(username, aggressive_cleanup=False): - """Set up and return an undetected Chrome driver with a persistent profile per account.""" - logger.info("Setting up undetected Chrome driver...") - - # Kill any orphaned Chrome processes first - kill_orphan_chrome_processes() - - # Build a persistent per-account profile directory - profile_dir = os.path.join(OUTPUT_DIR, "profiles", username) - os.makedirs(profile_dir, exist_ok=True) - - # Always clear lock files; only clear heavy caches on recovery paths. - clear_profile_cache(profile_dir, clear_heavy_cache=aggressive_cleanup) - - # Enhance profile with realistic history/bookmarks if it's a new profile - if not os.path.exists(os.path.join(profile_dir, "Default")): - try: - setup_realistic_profile(profile_dir) - except Exception as e: - logger.warning(f"Failed to setup realistic profile bits: {e}") - - logger.info(f"Using persistent Chrome profile at: {profile_dir}") - - options = uc.ChromeOptions() - - # Essentials only; avoid suspicious flags - options.add_argument(f"--user-data-dir={profile_dir}") - options.add_argument("--no-sandbox") - options.add_argument("--disable-dev-shm-usage") - - # CRITICAL: Disable blink features that expose automation - # This makes navigator.webdriver return undefined instead of true - # Note: excludeSwitches doesn't work with undetected_chromedriver, - # so we rely on the CDP script injection below - options.add_argument("--disable-blink-features=AutomationControlled") - - # Additional anti-detection measures via arguments - # These are safer than experimental options with undetected_chromedriver - options.add_argument("--disable-automation") - options.add_argument("--disable-web-security") - options.add_argument("--disable-features=IsolateOrigins,site-per-process") - options.add_argument("--disable-blink-features=AutomationControlled") - - # Performance and stability improvements - options.add_argument("--disable-background-networking") - options.add_argument("--disable-background-timer-throttling") - options.add_argument("--disable-backgrounding-occluded-windows") - options.add_argument("--disable-renderer-backgrounding") - options.add_argument("--disable-features=TranslateUI") - options.add_argument("--disable-sync") - options.add_argument("--disable-extensions-except=") - options.add_argument("--disable-default-apps") - options.add_argument("--no-first-run") - options.add_argument("--no-default-browser-check") - - # Reduce memory/resource usage - options.add_argument("--disable-component-update") - options.add_argument("--disable-domain-reliability") - - # Resolve Chrome binary path - chrome_binary = resolve_chrome_binary() - if chrome_binary: - options.binary_location = chrome_binary - logger.info(f"Using Chrome binary: {chrome_binary}") - else: - logger.warning("Could not resolve Chrome binary. Relying on system default.") - - # Determine local Chrome major version - env_force_version = os.environ.get("UC_FORCE_VERSION_MAIN") - detected_major = None - if env_force_version and env_force_version.isdigit(): - detected_major = int(env_force_version) - logger.info( - f"UC_FORCE_VERSION_MAIN set; forcing driver for Chrome {detected_major}" - ) - else: - detected_major = get_chrome_major_version(chrome_binary) - if detected_major: - logger.info(f"Detected local Chrome major version: {detected_major}") - - # Randomize viewport size a bit - width = random.randint(1050, 1200) - height = random.randint(800, 950) - options.add_argument(f"--window-size={width},{height}") - - # Language header - options.add_argument("--accept-lang=en-US,en;q=0.9") - - # Proxy support (VPN/backends) - proxy_http = os.environ.get("http_proxy") - proxy_https = os.environ.get("https_proxy") - if proxy_http or proxy_https: - proxy_to_use = proxy_http or proxy_https - logger.info(f"Detected proxy settings: {proxy_to_use}") - if proxy_to_use.startswith("http://"): - proxy_to_use = proxy_to_use[7:] - options.add_argument(f"--proxy-server={proxy_to_use}") - options.add_argument("--ignore-certificate-errors") - - # Prefer a modern, real Chrome UA; CDP override will ensure full hints - # We'll derive version from driver after launch for consistency - driver = None - version_retry_count = 0 - max_version_retries = 2 - - while version_retry_count < max_version_retries: - try: - logger.info(f"Initializing undetected Chrome driver (attempt {version_retry_count + 1})...") - - # Force kill any existing Chrome processes before creating new driver - kill_orphan_chrome_processes() - - if detected_major: - logger.info(f"Requesting ChromeDriver for Chrome version {detected_major}") - driver = uc.Chrome(options=options, version_main=detected_major) - else: - driver = uc.Chrome(options=options) - - logger.info("Successfully initialized undetected Chrome driver") - break # Success - exit the retry loop - - except TypeError as te: - if "version_main" in str(te): - logger.warning( - "Your undetected_chromedriver is outdated and lacks version_main. " - "Please upgrade: pip install -U undetected-chromedriver" - ) - driver = uc.Chrome(options=options) - break - else: - raise - - except WebDriverException as we: - error_msg = str(we).lower() - - # Check for version mismatch error - if "only supports chrome version" in error_msg or "session not created" in error_msg: - version_retry_count += 1 - - if version_retry_count < max_version_retries: - logger.warning( - f"ChromeDriver version mismatch detected. " - f"Clearing cache and retrying (attempt {version_retry_count + 1}/{max_version_retries})..." - ) - - # Clear the chromedriver cache to force re-download - clear_chromedriver_cache() - - # Small delay before retry - time.sleep(2) - - # Continue to next iteration - continue - else: - logger.error("ChromeDriver version mismatch persists after clearing cache") - # Try one more time without specifying version_main - try: - logger.info("Trying without version specification...") - driver = uc.Chrome(options=options) - logger.info("Successfully initialized undetected Chrome driver (fallback)") - break - except Exception as fallback_e: - logger.error(f"Fallback initialization also failed: {str(fallback_e)}") - raise - else: - # Not a version error - re-raise - raise - - if driver is None: - raise RuntimeError("Failed to initialize Chrome driver after all retries") - - # CRITICAL: Use CDP to hide navigator.webdriver flag - # This is essential to bypass X/Twitter's bot detection - try: - driver.execute_cdp_cmd( - "Page.addScriptToEvaluateOnNewDocument", - { - "source": """ - Object.defineProperty(navigator, 'webdriver', { - get: () => undefined - }); - - // Also hide other automation indicators - Object.defineProperty(navigator, 'plugins', { - get: () => [1, 2, 3, 4, 5] - }); - - // Hide Chrome's automation extension - window.chrome = { - runtime: {} - }; - - // Remove CDC properties that ChromeDriver adds - Object.defineProperty(window, 'cdc_adoQpoasnfa76pfcZLmcfl_Array', { - get: () => undefined - }); - Object.defineProperty(window, 'cdc_adoQpoasnfa76pfcZLmcfl_Promise', { - get: () => undefined - }); - Object.defineProperty(window, 'cdc_adoQpoasnfa76pfcZLmcfl_Symbol', { - get: () => undefined - }); - """ - }, - ) - logger.info("Successfully injected anti-detection script via CDP") - except Exception as e: - logger.warning(f"Failed to inject anti-detection script: {str(e)}") - - # Derive browser version to craft consistent Client Hints - browser_version = None - try: - caps = getattr(driver, "capabilities", {}) or {} - browser_version = caps.get("browserVersion") or caps.get("version") - except Exception: - pass - if not browser_version: - browser_version = "126.0.6478.61" - major_version = browser_version.split(".")[0] - - # Compose realistic UA (macOS Sonoma-ish) - ua_string = ( - f"Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) " - f"AppleWebKit/537.36 (KHTML, like Gecko) " - f"Chrome/{browser_version} Safari/537.36" - ) - - # Client Hints with brands and fullVersionList - brands = [ - {"brand": "Not.A/Brand", "version": "24"}, - {"brand": "Chromium", "version": major_version}, - {"brand": "Google Chrome", "version": major_version}, - ] - full_version_list = [ - {"brand": "Not.A/Brand", "version": "24.0.0.0"}, - {"brand": "Chromium", "version": browser_version}, - {"brand": "Google Chrome", "version": browser_version}, - ] - cpu_arch = platform.machine().lower() - if "arm" in cpu_arch or "aarch64" in cpu_arch: - client_hint_arch = "arm" - else: - client_hint_arch = "x86" - - try: - driver.execute_cdp_cmd( - "Network.setUserAgentOverride", - { - "userAgent": ua_string, - "platform": "macOS", - "userAgentMetadata": { - "brands": brands, - "fullVersionList": full_version_list, - "platform": "macOS", - "platformVersion": "14.5.0", - "architecture": client_hint_arch, - "model": "", - "mobile": False, - "bitness": "64", - }, - }, - ) - except Exception as e: - logger.warning(f"Failed to set UA override via CDP: {str(e)}") - - # Timezone override only when explicitly provided to avoid proxy/locale mismatch. - timezone_id = os.environ.get("TIMEZONE_ID") - if timezone_id: - try: - driver.execute_cdp_cmd( - "Emulation.setTimezoneOverride", {"timezoneId": timezone_id} - ) - except Exception as e: - logger.warning(f"Failed to set timezone override: {str(e)}") - - # Geolocation override only when explicitly provided to avoid mismatched signals. - geo_lat = os.environ.get("GEO_LAT") - geo_lon = os.environ.get("GEO_LON") - if geo_lat and geo_lon: - try: - lat = float(geo_lat) - lon = float(geo_lon) - acc = float(os.environ.get("GEO_ACC", "100")) - # Grant permission for Twitter origin - try: - driver.execute_cdp_cmd( - "Browser.grantPermissions", - {"permissions": ["geolocation"], "origin": "https://x.com"}, - ) - except Exception: - pass - driver.execute_cdp_cmd( - "Emulation.setGeolocationOverride", - {"latitude": lat, "longitude": lon, "accuracy": acc}, - ) - except Exception as e: - logger.warning(f"Failed to set geolocation override: {str(e)}") - - return driver - - -def human_like_typing(element, text): - """Simulate human-like typing with random delays between keypresses.""" - for char in text: - element.send_keys(char) - time.sleep(random.uniform(0.05, 0.25)) # Random delay between keypresses - - -def find_and_fill_input(driver, input_type, value): - """Find and fill an input field of a specific type.""" - selectors = { - "username": [ - 'input[autocomplete="username"]', - 'input[name="text"]', - 'input[name="username"]', - 'input[placeholder*="username" i]', - 'input[placeholder*="phone" i]', - 'input[placeholder*="email" i]', - ], - "password": [ - 'input[type="password"]', - 'input[name="password"]', - 'input[placeholder*="password" i]', - ], - "email": [ - 'input[type="email"]', - 'input[name="email"]', - 'input[placeholder*="email" i]', - 'input[autocomplete="email"]', - ], - "phone": [ - 'input[type="tel"]', - 'input[name="phone"]', - 'input[placeholder*="phone" i]', - 'input[autocomplete="tel"]', - ], - "code": [ - 'input[autocomplete="one-time-code"]', - 'input[name="code"]', - 'input[placeholder*="code" i]', - 'input[placeholder*="verification" i]', - ], - } - - if input_type not in selectors: - logger.warning(f"Unknown input type: {input_type}") - return False - - input_found = False - - for selector in selectors[input_type]: - try: - inputs = driver.find_elements(By.CSS_SELECTOR, selector) - for input_field in inputs: - if input_field.is_displayed(): - # Clear the field first (sometimes needed) - try: - input_field.clear() - except: - pass - - # Type the value - human_like_typing(input_field, value) - logger.info(f"Filled {input_type} field with value: {value}") - - # Add a small delay after typing - time.sleep(random.uniform(0.5, 1.5)) - input_found = True - return True - except Exception as e: - logger.debug( - f"Couldn't find or fill {input_type} field with selector {selector}: {str(e)}" - ) - - if not input_found: - logger.info(f"No {input_type} input field found") - - return False - - -def click_next_button(driver): - """Try to click a 'Next' or submit button.""" - button_clicked = False - - # Try buttons with "Next" text - try: - next_buttons = driver.find_elements( - By.XPATH, '//*[contains(text(), "Next") or contains(text(), "next")]' - ) - for button in next_buttons: - if button.is_displayed(): - button.click() - logger.info("Clicked Next button by text") - button_clicked = True - break - except Exception as e: - logger.debug(f"Couldn't click Next button by text: {str(e)}") - - # Try buttons with "Continue" text - if not button_clicked: - try: - continue_buttons = driver.find_elements( - By.XPATH, - '//*[contains(text(), "Continue") or contains(text(), "continue")]', - ) - for button in continue_buttons: - if button.is_displayed(): - button.click() - logger.info("Clicked Continue button by text") - button_clicked = True - break - except Exception as e: - logger.debug(f"Couldn't click Continue button by text: {str(e)}") - - # Try buttons with "Log in" or "Sign in" text - if not button_clicked: - try: - login_buttons = driver.find_elements( - By.XPATH, - '//*[contains(text(), "Log in") or contains(text(), "Login") or contains(text(), "Sign in")]', - ) - for button in login_buttons: - if button.is_displayed(): - button.click() - logger.info("Clicked Login button by text") - button_clicked = True - break - except Exception as e: - logger.debug(f"Couldn't click Login button by text: {str(e)}") - - # Try generic button elements by role - if not button_clicked: - try: - buttons = driver.find_elements(By.CSS_SELECTOR, 'div[role="button"]') - for button in buttons: - if button.is_displayed(): - button.click() - logger.info("Clicked button by role") - button_clicked = True - break - except Exception as e: - logger.debug(f"Couldn't click button by role: {str(e)}") - - # Try submitting the form with Enter key (last resort) - if not button_clicked: - try: - active_element = driver.switch_to.active_element - active_element.send_keys(Keys.ENTER) - logger.info("Pressed Enter key on active element") - button_clicked = True - except Exception as e: - logger.debug(f"Couldn't press Enter key: {str(e)}") - - return button_clicked - - -def is_logged_in(driver): - """Check if user is logged in to Twitter.""" - try: - current_url = driver.current_url.lower() - logger.debug(f"Checking login status, current URL: {current_url}") - - # URL check (most reliable) - check for /home anywhere in URL - if "/home" in current_url and ("twitter.com" in current_url or "x.com" in current_url): - logger.info(f"Detected logged in via URL: {current_url}") - return True - - # Also check if we're on the main feed (sometimes URL is just x.com/) - if current_url.rstrip('/') in ["https://x.com", "https://twitter.com"]: - # Check if we see home timeline elements - pass # Fall through to element checks - - # Home timeline check - home_timeline = driver.find_elements( - By.CSS_SELECTOR, 'div[aria-label="Timeline: Your Home Timeline"]' - ) - if home_timeline and any(elem.is_displayed() for elem in home_timeline): - logger.info("Detected logged in via home timeline element") - return True - - # Tweet/Post button check - updated selectors for current X.com - tweet_buttons = driver.find_elements( - By.CSS_SELECTOR, - 'a[data-testid="SideNav_NewTweet_Button"], [data-testid="tweetButtonInline"], a[href="/compose/post"], [data-testid="SideNav_NewPost_Button"]', - ) - if tweet_buttons and any(btn.is_displayed() for btn in tweet_buttons): - logger.info("Detected logged in via tweet/post button") - return True - - # Navigation elements check - updated selectors - nav_elements = driver.find_elements( - By.CSS_SELECTOR, - 'nav[role="navigation"], a[data-testid="AppTabBar_Home_Link"], a[href="/home"]', - ) - if nav_elements and any(elem.is_displayed() for elem in nav_elements): - logger.info("Detected logged in via navigation elements") - return True - - # Check for the main timeline/feed area (generic check) - main_content = driver.find_elements( - By.CSS_SELECTOR, - '[data-testid="primaryColumn"], main[role="main"]' - ) - # Also check we're not on login flow - login_elements = driver.find_elements( - By.CSS_SELECTOR, - 'input[name="text"], input[name="password"], [data-testid="LoginForm"]' - ) - if main_content and any(elem.is_displayed() for elem in main_content): - if not (login_elements and any(elem.is_displayed() for elem in login_elements)): - logger.info("Detected logged in via main content area (no login form visible)") - return True - - return False - except Exception as e: - logger.error(f"Error checking login status: {str(e)}") - return False - - -def needs_verification(driver): - """Check if the page is showing a verification or authentication screen.""" - try: - # Check for verification text - verification_texts = [ - "Authenticate your account", - "Enter your phone number", - "Enter your email", - "Check your phone", - "Check your email", - "Verification code", - "verify your identity", - "unusual login activity", - "suspicious activity", - "Help us keep your account safe", - "Verify your identity", - "keep your account safe", - ] - - for text in verification_texts: - try: - elements = driver.find_elements( - By.XPATH, f"//*[contains(text(), '{text}')]" - ) - if elements and any(elem.is_displayed() for elem in elements): - logger.info(f"Verification needed: Found text '{text}'") - return True - except: - pass - - # Check for verification URLs - current_url = driver.current_url.lower() - verification_url_patterns = [ - "verify", - "challenge", - "confirm", - "auth", - "login_challenge", - ] - - for pattern in verification_url_patterns: - if pattern in current_url: - logger.info(f"Verification needed: URL contains '{pattern}'") - return True - - return False - except Exception as e: - logger.error(f"Error checking for verification: {str(e)}") - return False - - -def is_account_locked_or_suspended(driver): - """Check if the account is locked, suspended, or disabled.""" - try: - lockout_indicators = [ - "account suspended", - "account locked", - "account disabled", - "permanently suspended", - "temporarily locked", - "unusual activity detected", - "automated behavior", - "captcha", - "prove you're not a robot", - "verify you're human", - "phone verification required", - "additional verification required", - "security check", - ] - - page_text = driver.find_element(By.TAG_NAME, "body").text.lower() - current_url = driver.current_url.lower() - - for indicator in lockout_indicators: - if indicator in page_text or indicator in current_url: - logger.error(f"Account appears locked/suspended: '{indicator}' detected") - return True - - # Check for specific locked account elements - lockout_selectors = [ - '[data-testid="lockedAccount"]', - '[data-testid="suspendedAccount"]', - '[class*="suspended"]', - '[class*="locked"]', - '[class*="captcha"]', - 'iframe[src*="captcha"]', - 'iframe[src*="recaptcha"]', - ] - - for selector in lockout_selectors: - try: - elements = driver.find_elements(By.CSS_SELECTOR, selector) - for elem in elements: - if elem.is_displayed(): - logger.error(f"Lockout/suspension element found: {selector}") - return True - except: - pass - - return False - except Exception as e: - logger.debug(f"Error checking account lockout status: {str(e)}") - return False - - -def extract_email_from_password(password): - """Extract email from password assuming format 'himynameis'.""" - # Get base email from environment variable - required - base_email = os.environ.get("TWITTER_EMAIL") - if not base_email: - logger.error("TWITTER_EMAIL environment variable not set. This is required.") - # Return a placeholder that will likely fail but doesn't expose personal info - return "email_not_configured@example.com" - - # Extract the username part from base email for plus addressing - base_username = base_email.split("@")[0] - domain = base_email.split("@")[1] - - try: - # Check if password starts with 'himynameis' or 'himynamewas' - if password.startswith("himynameis"): - name = password[10:] # Extract everything after 'himynameis' - return f"{base_username}+{name}@{domain}" - elif password.startswith("himynamewas"): - name = password[11:] # Extract everything after 'himynamewas' - return f"{base_username}+{name}@{domain}" - except: - pass - - # Fall back to the base email - return base_email - - -def extract_cookies(driver, log_cookie_names=True): - """Extract cookies from the browser.""" - logger.info("Extracting cookies") - browser_cookies = driver.get_cookies() - logger.info(f"Found {len(browser_cookies)} cookies total") - - cookie_values = {} - # Always use x.com domain, no conditional check - used_domain = "x.com" - - for cookie in browser_cookies: - value = cookie["value"] - if value.startswith('"') and value.endswith('"'): - value = value[1:-1] # Remove surrounding quotes - value = value.replace('"', "") # Replace any remaining quotes - - cookie_values[cookie["name"]] = value - if log_cookie_names: - logger.info(f"Found cookie: {cookie['name']}") - - # Log missing required cookies (critical for scraper auth). - missing_cookies = [ - name for name in REQUIRED_SESSION_COOKIES if name not in cookie_values - ] - if missing_cookies: - logger.warning(f"Missing required cookies: {', '.join(missing_cookies)}") - else: - logger.info("All required cookies found") - - return cookie_values, used_domain - - -def wait_for_required_session_cookies(driver, timeout_seconds=POST_LOGIN_COOKIE_WAIT): - """Wait for required session cookies to appear after login.""" - deadline = time.time() + timeout_seconds - while time.time() < deadline: - cookie_values, _ = extract_cookies(driver, log_cookie_names=False) - missing_required = [ - name - for name in REQUIRED_SESSION_COOKIES - if not cookie_values.get(name) - ] - if not missing_required: - logger.info( - f"Required session cookies are present: {', '.join(REQUIRED_SESSION_COOKIES)}" - ) - return cookie_values - - logger.info( - f"Waiting for required session cookies: {', '.join(missing_required)}" - ) - time.sleep(POST_LOGIN_COOKIE_POLL_INTERVAL) - - logger.warning( - f"Timed out waiting for required session cookies: {', '.join(REQUIRED_SESSION_COOKIES)}" - ) - cookie_values, _ = extract_cookies(driver) - return cookie_values - - -def generate_cookies_json(cookie_values, domain="x.com"): - """Generate the cookies JSON from the provided cookie values.""" - # Always use x.com domain regardless of what's passed in - domain = "x.com" - logger.info(f"Generating cookies JSON for domain: {domain}") - - # Determine expiration dates for different cookie types - one_week_future = get_future_date(days=7) - one_month_future = get_future_date(days=30) - - cookies = [] - - # Process all found cookies - for name, value in cookie_values.items(): - if value == "": - logger.warning(f"Using empty string for cookie: {name}") - - # Set appropriate expiration date based on cookie type - if name in ["personalization_id", "kdt"]: - # 1 month expiration for these cookies - expires = one_month_future - logger.debug(f"Setting {name} cookie to expire in 1 month: {expires}") - elif name in ["auth_token", "ct0"]: - # 1 week expiration for these cookies - expires = one_week_future - logger.debug(f"Setting {name} cookie to expire in 1 week: {expires}") - else: - # Default 1 week for all other cookies - expires = one_week_future - logger.debug( - f"Setting {name} cookie to default expiration (1 week): {expires}" - ) - - cookies.append(create_cookie_template(name, value, domain, expires)) - - return cookies - - -def is_page_stuck_loading(driver): - """Detect if the page is stuck showing a loading spinner.""" - try: - # Check for loading spinners or progress indicators - spinner_selectors = [ - 'svg[style*="animation"]', # Animated SVG spinners - '[role="progressbar"]', - '.loading', - '[class*="spinner"]', - '[class*="loading"]', - # X/Twitter specific selectors - 'svg circle[cx]', # Common spinner pattern - '[data-testid="loading"]', - ] - - for selector in spinner_selectors: - try: - spinners = driver.find_elements(By.CSS_SELECTOR, selector) - for spinner in spinners: - if spinner.is_displayed(): - # Check if it's actually visible in viewport - try: - rect = driver.execute_script( - "return arguments[0].getBoundingClientRect();", spinner - ) - if rect["width"] > 20 and rect["height"] > 20: - return True - except: - pass - except: - pass - - # Check for "Loading..." text or similar - loading_texts = ["Loading", "loading", "Please wait", "please wait"] - for text in loading_texts: - try: - elements = driver.find_elements( - By.XPATH, f"//*[contains(text(), '{text}')]" - ) - if any(e.is_displayed() for e in elements): - return True - except: - pass - - return False - except Exception as e: - logger.debug(f"Error checking for stuck loading: {str(e)}") - return False - - -def wait_for_login_page_ready(driver, max_wait=30, refresh_on_stuck=True): - """Wait until login page controls are interactive. - - Args: - driver: The Chrome driver instance - max_wait: Maximum time to wait in seconds - refresh_on_stuck: If True, refresh the page if it appears stuck loading - """ - wait_start = time.time() - stuck_check_interval = 5 # Check for stuck state every 5 seconds - last_stuck_check = wait_start - is_stuck = False - js_ready_checks = 0 # Track how many times we've seen JS ready - - while time.time() - wait_start < max_wait: - try: - ready_state = driver.execute_script("return document.readyState") - - # Check for stuck loading state periodically - if refresh_on_stuck and time.time() - last_stuck_check >= stuck_check_interval: - if is_page_stuck_loading(driver): - logger.warning("Page appears stuck on loading spinner, refreshing...") - is_stuck = True - break # Exit loop to trigger refresh - last_stuck_check = time.time() - - # Check for login form elements - login_elements = driver.find_elements( - By.CSS_SELECTOR, - 'input[name="text"], div[role="button"], form[data-testid="LoginForm"]', - ) - - # Also check if React/Vue apps have mounted by looking for dynamic content - js_complete = driver.execute_script(""" - // Check if any JS frameworks have rendered content - var reactRoot = document.querySelector('[data-reactroot]'); - var vueApp = document.querySelector('[data-v-app]') || document.querySelector('#app'); - var mainContent = document.querySelector('main') || document.querySelector('[role="main"]'); - var bodyContent = document.body && document.body.innerHTML.length > 500; - - // Return true if we have meaningful content - return !!(reactRoot || vueApp || mainContent || bodyContent); - """) - - if ready_state == "complete" and any( - elem.is_displayed() for elem in login_elements if login_elements - ): - # Require JS to be ready multiple times to avoid false positives - if js_complete: - js_ready_checks += 1 - if js_ready_checks >= 2: # Must be ready 2 consecutive checks - if is_stuck: - logger.info("Page recovered from stuck state and is now ready") - logger.info(f"Login page ready after {(time.time() - wait_start):.1f}s") - return True - else: - js_ready_checks = 0 # Reset if JS not ready - - except WebDriverException as e: - if "no such window" in str(e).lower() or "no such session" in str(e).lower(): - raise - logger.warning(f"Error checking page load: {str(e)}") - time.sleep(0.5) - - # If we detected stuck state, return False so caller can refresh - if is_stuck: - return False - - return False - - -def wait_for_post_click_transition(driver, previous_url, timeout=CLICK_WAIT): - """Wait for login UI to advance and return as soon as it does.""" - start = time.time() - while time.time() - start < timeout: - try: - current_url = driver.current_url - if current_url != previous_url: - return True - - markers = driver.find_elements( - By.CSS_SELECTOR, - 'input[name="password"], input[type="password"], input[name="text"], input[type="tel"], input[type="email"], input[autocomplete="one-time-code"], form[data-testid="LoginForm"]', - ) - if markers and any(elem.is_displayed() for elem in markers): - return True - except WebDriverException as e: - if "no such window" in str(e).lower() or "no such session" in str(e).lower(): - raise - time.sleep(0.25) - return False - - -def human_like_post_action_pause(): - """Small natural pause after a user action.""" - time.sleep(random.uniform(0.35, 0.9)) - - -def safe_navigate(driver, url, max_retries=3, retry_delay=2): - """Navigate to URL with retry logic for transient network errors.""" - for attempt in range(1, max_retries + 1): - try: - driver.get(url) - return True - except WebDriverException as e: - error_str = str(e).lower() - - # Check for network-related errors that warrant retry - network_errors = [ - "net::err_name_not_resolved", - "net::err_internet_disconnected", - "net::err_connection_reset", - "net::err_connection_refused", - "net::err_connection_timed_out", - "net::err_timed_out", - "dns", - "timeout", - "unable to connect", - ] - - is_network_error = any(err in error_str for err in network_errors) - - if is_network_error and attempt < max_retries: - logger.warning( - f"Network error on attempt {attempt}/{max_retries}: {str(e)[:100]}. " - f"Retrying in {retry_delay}s..." - ) - time.sleep(retry_delay * attempt) # Increasing backoff - else: - # Non-network error or last attempt - re-raise - raise - - return False - - -def process_account_state_machine(driver, username, password): - """Process an account using a state machine approach with continuous polling.""" - logger.info(f"==========================================") - logger.info(f"Starting to process account: {username}") - output_file = f"{username}_twitter_cookies.json" - - # Create output directory if it doesn't exist - os.makedirs(OUTPUT_DIR, exist_ok=True) - - # Extract email from password if needed for verification - email = extract_email_from_password(password) - logger.info(f"Using email {email} for account {username}") - - # Navigate to login page with short in-session retries before full restart. - try: - nav_ready = False - nav_start = time.time() - for nav_attempt in range(1, 4): - attempt_start = time.time() - - # Strategy: Try different entry points to bypass bot detection - if nav_attempt == 1: - # Attempt 1: Direct login URL - logger.info("Attempt 1: Navigating directly to login URL") - safe_navigate(driver, TWITTER_LOGIN_URL, max_retries=3) - elif nav_attempt == 2: - # Attempt 2: Visit home page first, then click login - logger.info("Attempt 2: Visiting home page first, then navigating to login") - try: - safe_navigate(driver, "https://x.com", max_retries=3) - time.sleep(3) # Let home page JS execute - - # Look for and click login button on home page - login_buttons = driver.find_elements( - By.CSS_SELECTOR, - 'a[href="/login"], a[href="/i/flow/login"], [data-testid="loginButton"]' - ) - if login_buttons: - for btn in login_buttons: - if btn.is_displayed(): - logger.info("Found login button on home page, clicking...") - btn.click() - time.sleep(2) - break - else: - # No login button found, navigate directly - logger.info("No login button found, navigating to login URL") - safe_navigate(driver, TWITTER_LOGIN_URL, max_retries=3) - except Exception as e: - logger.warning(f"Home page approach failed: {str(e)}, trying direct URL") - safe_navigate(driver, TWITTER_LOGIN_URL, max_retries=3) - else: - # Attempt 3: Try with a different user agent string - logger.info("Attempt 3: Trying alternative navigation approach") - safe_navigate(driver, TWITTER_LOGIN_URL, max_retries=3) - - # Wait for page with stuck-detection - if wait_for_login_page_ready(driver, max_wait=30, refresh_on_stuck=True): - logger.info( - f"Login page loaded successfully (attempt {nav_attempt}, " - f"{time.time() - attempt_start:.1f}s)" - ) - nav_ready = True - break - - # If we got here, page may be stuck or not ready - try refresh - logger.warning( - f"Login page not ready on attempt {nav_attempt} " - f"({time.time() - attempt_start:.1f}s), refreshing..." - ) - - # Try refreshing the page before giving up on this attempt - if nav_attempt < 3: - # CRITICAL: If we detected stuck spinner, the profile may be poisoned - # Clear it completely and restart browser with fresh profile - if is_page_stuck_loading(driver): - logger.warning( - "Detected stuck loading spinner - profile may be poisoned by bot detection. " - "Clearing profile and restarting browser..." - ) - try: - # Close current browser - driver.quit() - except: - pass - - # Clear the entire profile directory - try: - profile_dir = os.path.join(OUTPUT_DIR, "profiles", username) - if os.path.exists(profile_dir): - logger.info(f"Removing poisoned profile: {profile_dir}") - shutil.rmtree(profile_dir) - logger.info("Profile cleared successfully") - except Exception as clear_err: - logger.warning(f"Could not clear profile: {str(clear_err)}") - - # Restart browser with fresh profile - try: - logger.info("Restarting browser with fresh profile...") - driver = setup_driver(username, aggressive_cleanup=True) - logger.info("Browser restarted successfully") - # Don't increment nav_attempt - try again with fresh profile - continue - except Exception as restart_err: - logger.error(f"Failed to restart browser: {str(restart_err)}") - # Fall through to normal refresh attempt - - # Normal refresh attempt - try: - driver.refresh() - # Wait a bit after refresh - time.sleep(random.uniform(2.0, 3.5)) - - # Check if refresh helped - if wait_for_login_page_ready(driver, max_wait=20, refresh_on_stuck=False): - logger.info( - f"Login page ready after refresh (attempt {nav_attempt})" - ) - nav_ready = True - break - except WebDriverException as e: - if "no such window" in str(e).lower() or "no such session" in str(e).lower(): - raise - logger.warning(f"Refresh failed: {str(e)}") - - # Brief pause before next attempt - time.sleep(random.uniform(1.0, 2.5)) - - logger.info(f"Login page stage duration: {time.time() - nav_start:.1f}s") - if not nav_ready: - logger.warning("Proceeding despite login page readiness timeouts") - except WebDriverException as e: - # Check if window was closed - if so, propagate this up immediately - if "no such window" in str(e).lower() or "no such session" in str(e).lower(): - logger.info( - "Browser window was closed during navigation. Might be for VPN switching." - ) - raise - logger.error(f"Failed to navigate to login page: {str(e)}") - return False - - # Setup state machine variables - start_time = time.time() - last_action_time = start_time - last_url = driver.current_url - login_successful = False - manual_intervention_active = False - last_filled_at = {"username": 0.0, "password": 0.0, "email": 0.0} - last_filled_url = {"username": "", "password": "", "email": ""} - - # State machine loop - loop_count = 0 - last_progress_time = start_time - last_progress_url = "" - - while time.time() - start_time < WAITING_TIME: - loop_count += 1 - try: - current_url = driver.current_url - - # Log every 10 iterations to show we're still alive - if loop_count % 10 == 0: - logger.info(f"State machine loop iteration {loop_count}, URL: {current_url}") - - # Detect if we're stuck on a loading spinner during the flow - if loop_count % 20 == 0 and is_page_stuck_loading(driver): - logger.warning("Detected stuck loading spinner during login flow") - - # Check if we've made progress recently - time_since_progress = time.time() - last_progress_time - url_changed = current_url != last_progress_url - - if time_since_progress > 45 and not url_changed: - logger.warning( - f"No progress for {time_since_progress:.0f}s, attempting recovery refresh" - ) - try: - driver.refresh() - time.sleep(3) - last_progress_time = time.time() # Reset timer after refresh - except WebDriverException as e: - if "no such window" in str(e).lower() or "no such session" in str(e).lower(): - raise - logger.warning(f"Recovery refresh failed: {str(e)}") - else: - logger.info(f"Progress check: {time_since_progress:.0f}s since last action, URL changed: {url_changed}") - - # Check if already logged in - if is_logged_in(driver): - logger.info("Login successful!") - login_successful = True - break - - # Check for rate limiting - if detected, we'll need to back off - if loop_count % 15 == 0 and is_rate_limited(driver): - record_rate_limit_hit(username) - backoff = get_account_backoff(username) - logger.warning(f"Rate limited. Backing off for {backoff}s...") - - # Capture screenshot for debugging - capture_screenshot(driver, username, "rate_limited") - - # Wait out the backoff period - time.sleep(backoff) - - # Try refreshing after backoff - try: - driver.refresh() - time.sleep(3) - last_progress_time = time.time() - except WebDriverException as e: - if "no such window" in str(e).lower() or "no such session" in str(e).lower(): - raise - logger.warning(f"Post-backoff refresh failed: {str(e)}") - continue - - # Check for account lockout/suspension (permanent failure for this account) - if loop_count % 25 == 0 and is_account_locked_or_suspended(driver): - logger.error(f"Account {username} appears to be locked or suspended. Aborting.") - capture_screenshot(driver, username, "account_locked") - # Don't retry this account - it's a permanent failure - return False - - # Check if URL changed since last check - if current_url != last_url: - logger.info(f"URL changed to: {current_url}") - last_url = current_url - last_action_time = time.time() # Reset the idle timer when URL changes - last_progress_time = time.time() # Track progress for stuck detection - last_progress_url = current_url - - # Check if we need verification - if needs_verification(driver): - if not manual_intervention_active: - logger.info("Manual verification required") - manual_intervention_active = True - - # Try to help with the verification by filling known fields - # Check for phone/email verification screen - verification_inputs = driver.find_elements( - By.CSS_SELECTOR, - 'input[placeholder*="Phone or email"], input[placeholder*="phone number or email"], input[aria-label*="phone"], input[aria-label*="email"], input[name="text"], input.r-30o5oe, input[placeholder*="Email address"]', - ) - if verification_inputs and any( - inp.is_displayed() for inp in verification_inputs - ): - logger.info( - "Phone/email verification screen detected - filling with email" - ) - for input_field in verification_inputs: - if input_field.is_displayed(): - try: - # Clear the field completely - input_field.clear() - input_field.send_keys(Keys.CONTROL + "a") - input_field.send_keys(Keys.DELETE) - time.sleep(0.5) - except: - pass - # Only type the email, nothing else - human_like_typing(input_field, email) - logger.info( - f"Filled verification input with email: {email}" - ) - time.sleep(1) - before_click_url = driver.current_url - click_next_button(driver) - human_like_post_action_pause() - wait_for_post_click_transition( - driver, before_click_url, timeout=CLICK_WAIT - ) - last_action_time = time.time() - last_progress_time = time.time() - continue - - # Check specifically for the "Help us keep your account safe" screen - help_safe_elements = driver.find_elements( - By.XPATH, "//*[contains(text(), 'Help us keep your account safe')]" - ) - if help_safe_elements and any( - elem.is_displayed() for elem in help_safe_elements - ): - logger.info("Account safety verification screen detected") - # Try to find email input field - email_inputs = driver.find_elements( - By.CSS_SELECTOR, 'input[placeholder="Email address"]' - ) - if email_inputs and any(inp.is_displayed() for inp in email_inputs): - for input_field in email_inputs: - if input_field.is_displayed(): - try: - # Clear the field completely - input_field.clear() - input_field.send_keys(Keys.CONTROL + "a") - input_field.send_keys(Keys.DELETE) - time.sleep(0.5) - except: - pass - # Type the email address - human_like_typing(input_field, email) - logger.info( - f"Filled account safety email with: {email}" - ) - time.sleep(1) - # Look for the Next button - next_buttons = driver.find_elements( - By.XPATH, - '//div[@role="button" and contains(text(), "Next")]', - ) - if next_buttons and any( - btn.is_displayed() for btn in next_buttons - ): - for btn in next_buttons: - if btn.is_displayed(): - before_click_url = driver.current_url - btn.click() - logger.info( - "Clicked Next button on account safety screen" - ) - human_like_post_action_pause() - wait_for_post_click_transition( - driver, - before_click_url, - timeout=CLICK_WAIT, - ) - last_action_time = time.time() - last_progress_time = time.time() - break - else: - # If can't find specific Next button, try generic button click - before_click_url = driver.current_url - click_next_button(driver) - human_like_post_action_pause() - wait_for_post_click_transition( - driver, before_click_url, timeout=CLICK_WAIT - ) - last_action_time = time.time() - last_progress_time = time.time() - continue - - # Check for email input (older style) - can_refill_email = ( - current_url != last_filled_url["email"] - or time.time() - last_filled_at["email"] > 12 - ) - if can_refill_email and find_and_fill_input(driver, "email", email): - last_filled_at["email"] = time.time() - last_filled_url["email"] = current_url - before_click_url = driver.current_url - click_next_button(driver) - human_like_post_action_pause() - wait_for_post_click_transition( - driver, before_click_url, timeout=CLICK_WAIT - ) - last_action_time = time.time() - last_progress_time = time.time() - continue - - # Check for phone input (we'll let the user handle this) - phone_inputs = driver.find_elements( - By.CSS_SELECTOR, 'input[type="tel"], input[placeholder*="phone" i]' - ) - if phone_inputs and any(inp.is_displayed() for inp in phone_inputs): - logger.info( - "Phone verification required - waiting for manual completion" - ) - # Just continue polling, user needs to complete this manually - time.sleep(POLLING_INTERVAL) - continue - else: - # If we no longer need verification, update the flag - if manual_intervention_active: - logger.info("Manual verification appears to be completed") - manual_intervention_active = False - - # Normal login flow - try to identify and fill inputs - # Username field - can_refill_username = ( - current_url != last_filled_url["username"] - or time.time() - last_filled_at["username"] > 12 - ) - if can_refill_username and find_and_fill_input(driver, "username", username): - last_filled_at["username"] = time.time() - last_filled_url["username"] = current_url - before_click_url = driver.current_url - click_next_button(driver) - human_like_post_action_pause() - wait_for_post_click_transition( - driver, before_click_url, timeout=CLICK_WAIT - ) - last_action_time = time.time() - last_progress_time = time.time() - continue - - # Password field - can_refill_password = ( - current_url != last_filled_url["password"] - or time.time() - last_filled_at["password"] > 12 - ) - if can_refill_password and find_and_fill_input(driver, "password", password): - last_filled_at["password"] = time.time() - last_filled_url["password"] = current_url - before_click_url = driver.current_url - click_next_button(driver) - human_like_post_action_pause() - wait_for_post_click_transition( - driver, before_click_url, timeout=CLICK_WAIT - ) - last_action_time = time.time() - last_progress_time = time.time() - continue - - # If we haven't taken any action for a while, try clicking a button - if time.time() - last_action_time > 30: # 30 seconds of no action - if click_next_button(driver): - logger.info("Clicked a button after 30 seconds of inactivity") - human_like_post_action_pause() - last_action_time = time.time() - last_progress_time = time.time() - continue - - # If we're not logged in and can't find any inputs, wait - time.sleep(POLLING_INTERVAL) - - except WebDriverException as e: - # Immediately propagate window closing exceptions - if ( - "no such window" in str(e).lower() - or "no such session" in str(e).lower() - ): - logger.info("Browser window was closed. Might be for VPN switching.") - raise - - # Handle other WebDriver exceptions - logger.error(f"WebDriver error: {str(e)}") - # Continue the loop to try again - - except Exception as e: - logger.error(f"Unexpected error: {str(e)}") - # Continue the loop to try again - - # After the loop, check if login was successful - if login_successful: - try: - # Keep a small natural pause, then rely on adaptive cookie checks. - logger.info("Login detected, verifying cookies...") - human_like_post_action_pause() - - # Ensure we are on a stable post-login page before cookie extraction. - current = driver.current_url.lower() - if ( - "x.com/i/flow/login" in current - or ("x.com" not in current and "twitter.com" not in current) - ): - logger.info("Navigating to https://x.com/home to stabilize authenticated session") - try: - driver.get("https://x.com/home") - nav_wait_start = time.time() - while time.time() - nav_wait_start < 4: - try: - if driver.execute_script("return document.readyState") == "complete": - break - except Exception: - pass - time.sleep(0.25) - except WebDriverException as e: - if ( - "no such window" in str(e).lower() - or "no such session" in str(e).lower() - ): - logger.info( - "Browser window was closed after login. Might be for VPN switching." - ) - raise - logger.warning(f"Failed to navigate: {str(e)}") - else: - logger.info(f"On X/Twitter domain, extracting cookies from: {current}") - - # Poll for required post-login session cookies before final extraction. - cookie_wait_start = time.time() - cookie_values = wait_for_required_session_cookies(driver) - logger.info( - f"Cookie readiness stage duration: {time.time() - cookie_wait_start:.1f}s" - ) - - # Clear rate limit hits on successful login - clear_rate_limit_hits(username) - - domain = "x.com" - cookies_json = generate_cookies_json(cookie_values, domain) - - # Save cookies to file - output_path = os.path.join(OUTPUT_DIR, output_file) - with open(output_path, "w") as f: - f.write(json.dumps(cookies_json, indent=2)) - logger.info(f"Saved cookies for {username} to {output_path}") - - return True - except WebDriverException as e: - # Check if window was closed - if ( - "no such window" in str(e).lower() - or "no such session" in str(e).lower() - ): - logger.info( - "Browser window was closed after login. Might be for VPN switching." - ) - raise - logger.error(f"Error after successful login: {str(e)}") - return False - except Exception as e: - logger.error(f"Error after successful login: {str(e)}") - return False - else: - logger.error(f"Failed to login for {username} within the time limit") - - # Capture screenshot for debugging the failure - try: - capture_screenshot(driver, username, "login_failed") - except Exception as e: - logger.debug(f"Could not capture failure screenshot: {str(e)}") - - return False - - -def main(): - """Main function to process Twitter accounts from environment variable.""" - logger.info("Starting cookie grabber") - - # Check for required environment variables - if not os.environ.get("TWITTER_EMAIL"): - logger.error("TWITTER_EMAIL environment variable is not set.") - logger.error("This is required for email verification during login.") - return - - # Get Twitter accounts from environment variable - twitter_accounts_str = os.environ.get("TWITTER_ACCOUNTS", "") - - if not twitter_accounts_str: - logger.error("TWITTER_ACCOUNTS environment variable is not set.") - logger.error("Format should be: username1:password1,username2:password2") - return - - account_pairs = twitter_accounts_str.split(",") - logger.info(f"Found {len(account_pairs)} accounts to process") - logger.info( - "Browser reset between accounts is disabled to reduce verification challenges" - ) - - # Create the output directory if it doesn't exist - os.makedirs(OUTPUT_DIR, exist_ok=True) - - # Process accounts one by one - current_account_index = 0 - while current_account_index < len(account_pairs): - # Maximum number of retries for account processing - max_retries = 5 # Increased retries to allow for VPN switches - retry_count = 0 - consecutive_window_closes = 0 - driver = None - - account_pair = account_pairs[current_account_index] - if ":" not in account_pair: - logger.error( - f"Invalid account format: {account_pair}. Expected format: username:password" - ) - current_account_index += 1 - continue - - username, password = account_pair.split(":", 1) - username = username.strip() - password = password.strip() - - logger.info( - f"Processing account {current_account_index+1}/{len(account_pairs)}: {username}" - ) - - # Check if we should use fresh profiles (clears any poisoned profile data) - use_fresh_profiles = os.environ.get("COOKIE_GRABBER_FRESH_PROFILES", "false").lower() == "true" - if use_fresh_profiles: - profile_dir = os.path.join(OUTPUT_DIR, "profiles", username) - if os.path.exists(profile_dir): - logger.info(f"FRESH_PROFILES enabled: Clearing existing profile for {username}") - try: - shutil.rmtree(profile_dir) - logger.info("Profile cleared successfully") - except Exception as e: - logger.warning(f"Could not clear profile: {str(e)}") - - # Process account with potential window closing for VPN switching - success = False - while retry_count < max_retries and not success: - try: - # Apply backoff delay if this is a retry (not the first attempt) - if retry_count > 0: - backoff = get_account_backoff(username) - # Add some jitter to avoid thundering herd - jitter = random.uniform(0, 5) - total_delay = backoff + jitter - if total_delay > 0: - logger.info( - f"Waiting {total_delay:.1f}s before retry {retry_count} " - f"({backoff}s backoff + {jitter:.1f}s jitter)" - ) - time.sleep(total_delay) - - # Initialize a new driver for each retry - if driver is not None: - try: - driver.quit() - except: - pass - - driver_setup_start = time.time() - driver = setup_driver(username, aggressive_cleanup=(retry_count > 0)) - logger.info( - f"Driver setup duration: {time.time() - driver_setup_start:.1f}s" - ) - logger.info( - f"Browser initialized for account: {username} (attempt {retry_count+1}/{max_retries})" - ) - - # Process the current account - success = process_account_state_machine(driver, username, password) - - if success: - logger.info(f"Successfully processed account: {username}") - else: - retry_count += 1 - logger.info( - f"Account processing unsuccessful. Retries left: {max_retries - retry_count}" - ) - time.sleep(10) # Brief pause before retry - - except WebDriverException as e: - # Special handling for closed window (VPN switching) - if ( - "no such window" in str(e).lower() - or "no such session" in str(e).lower() - ): - consecutive_window_closes += 1 - - if consecutive_window_closes > 3: - logger.error(f"Window closed unexpectedly {consecutive_window_closes} times in a row. Treating as failure.") - - # Handle potential profile corruption by moving the profile directory - try: - profile_dir = os.path.join(OUTPUT_DIR, "profiles", username) - if os.path.exists(profile_dir): - timestamp = int(time.time()) - backup_path = f"{profile_dir}_corrupted_{timestamp}" - logger.warning(f"Profile likely corrupted. Moving {profile_dir} to {backup_path}") - # Close any lingering file handles before moving (best effort) - if driver: - try: - driver.quit() - except: - pass - driver = None - os.rename(profile_dir, backup_path) - logger.info("Profile directory reset. Next attempt will start fresh.") - except Exception as e: - logger.error(f"Failed to move corrupted profile: {str(e)}") - - retry_count += 1 - # We don't continue here, so it will fall through to cleanup and loop check - else: - logger.info( - f"Browser window was closed (occurrence {consecutive_window_closes}). This might be for VPN switching." - ) - logger.info( - "Waiting 30 seconds for VPN to stabilize before retrying..." - ) - - # Clean up the driver - try: - if driver: - driver.quit() - except: - pass - - # Wait for VPN switch to complete - time.sleep(30) - - # Don't increment retry count for intentional window closing - # This allows unlimited VPN switches - logger.info(f"Resuming after window close for account: {username}") - continue - else: - consecutive_window_closes = 0 # Reset on different error - # Handle other WebDriver exceptions - retry_count += 1 - logger.error( - f"WebDriver error (attempt {retry_count}/{max_retries}): {str(e)}" - ) - time.sleep(15) - - except Exception as e: - consecutive_window_closes = 0 # Reset on different error - retry_count += 1 - logger.error( - f"Unexpected error (attempt {retry_count}/{max_retries}): {str(e)}" - ) - time.sleep(15) - - try: - if driver: - driver.quit() - except: - pass - - # Clean up the driver - try: - if driver: - driver.quit() - except: - pass - - # Move to next account only if successful or max retries reached - if success or retry_count >= max_retries: - if success: - logger.info(f"Successfully completed account: {username}") - else: - logger.warning( - f"Failed to process account after {max_retries} attempts: {username}" - ) - - current_account_index += 1 - - # Cooldown between accounts - if current_account_index < len(account_pairs): - cool_down = random.uniform(5, 10) # 5-10 seconds cooldown - logger.info( - f"Cooling down for {cool_down:.1f} seconds before next account" - ) - time.sleep(cool_down) - - logger.info("All accounts processed") - - -if __name__ == "__main__": - load_dotenv() # Load environment variables - logger.info("Starting cookie grabber script") - main() diff --git a/scripts/update_cookies_docker.sh b/scripts/update_cookies_docker.sh deleted file mode 100755 index 25e3eb83..00000000 --- a/scripts/update_cookies_docker.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/bash - -# Environment variables are passed from docker-compose -# REMOTE_HOST or COOKIES_REMOTE_HOSTS should be set -# REMOTE_USER or COOKIES_REMOTE_USER should be set -# REMOTE_DIR or COOKIES_REMOTE_DIR should be set - -# Support both singular and plural env var names -HOSTS=${COOKIES_REMOTE_HOSTS:-$REMOTE_HOST} -USER=${COOKIES_REMOTE_USER:-$REMOTE_USER} -DIR=${COOKIES_REMOTE_DIR:-$REMOTE_DIR} - -# Split hosts by commas if multiple are provided -IFS=',' read -ra HOST_ARRAY <<< "$HOSTS" - -# Process each host individually -for CURRENT_HOST in "${HOST_ARRAY[@]}"; do - # Trim any whitespace - CURRENT_HOST=$(echo "$CURRENT_HOST" | xargs) - echo "Transferring cookies from /app/cookies to $CURRENT_HOST..." - - # Create a temporary directory on the remote server - ssh -o StrictHostKeyChecking=no -i /root/.ssh/id_rsa $USER@$CURRENT_HOST "mkdir -p $DIR" - - # Copy cookies to the remote server - scp -o StrictHostKeyChecking=no -i /root/.ssh/id_rsa -r /app/cookies/*.json $USER@$CURRENT_HOST:$DIR/ - - # Copy cookies from temporary directory to each worker's volume - ssh -o StrictHostKeyChecking=no -i /root/.ssh/id_rsa $USER@$CURRENT_HOST " - # List files to make sure they were transferred - echo 'Files in the temporary directory:' - ls -la $DIR/ - - # Find all cookies-volume volumes using a pattern match (both with and without project prefix) - echo 'Finding all cookies volumes...' - worker_volumes=\$(docker volume ls --format '{{.Name}}' | grep -E '(miner-[0-9]+_)?cookies-volume') - - if [ -z \"\$worker_volumes\" ]; then - echo 'No cookies volumes found! Are the miners running?' - else - # Update each volume found - echo \"\$worker_volumes\" | while read volume; do - echo \"Updating volume: \$volume\" - docker run --rm -v \"\$volume\":/volume -v $DIR:/source --user root alpine sh -c \" - # Copy JSON files to volume - echo 'Copying JSON files to volume...' - cp -v /source/*.json /volume/ 2>/dev/null || echo 'No JSON files to copy' - echo 'Files in the volume:' - ls -la /volume/*.json 2>/dev/null || echo 'No JSON files found' - \" - done - fi - - # Clean up temporary directory - rm -rf $DIR || echo 'Could not remove temporary directory - you may need to clean it up manually' - " - - echo "Cookies successfully updated in all miner volumes on $CURRENT_HOST!" -done - -echo "All remote cookie updates completed!" \ No newline at end of file diff --git a/scripts/update_cookies_kubernetes.sh b/scripts/update_cookies_kubernetes.sh deleted file mode 100755 index ac0ed51f..00000000 --- a/scripts/update_cookies_kubernetes.sh +++ /dev/null @@ -1,366 +0,0 @@ -#!/bin/bash - -# Load environment variables from .env file (robustly) -if [ -f .env ]; then - # Load only non-comment, non-empty lines; support KEY=VALUE with spaces - while IFS= read -r line || [ -n "$line" ]; do - # Trim leading/trailing whitespace - trimmed=$(echo "$line" | sed -e 's/^\s\+//' -e 's/\s\+$//') - # Skip empty or comment lines - if [ -z "$trimmed" ] || echo "$trimmed" | grep -qE '^#'; then - continue - fi - # Export KEY=VALUE - eval export "$trimmed" - done < .env -else - echo "Error: .env file not found" - exit 1 -fi - -# Check if DEPLOYMENTS variable is set -if [ -z "$DEPLOYMENTS" ]; then - echo "Error: DEPLOYMENTS variable not set in .env file" - echo "Please add DEPLOYMENTS=deployment1,deployment2,deployment3 to your .env file" - exit 1 -fi - -# Check if NAMESPACE variable is set -if [ -z "$NAMESPACE" ]; then - echo "Error: NAMESPACE variable not set in .env file" - echo "Please add NAMESPACE=your-namespace to your .env file" - exit 1 -fi - -# Configuration for timeouts and retries -KUBECTL_TIMEOUT=${KUBECTL_TIMEOUT:-300} # 5 minutes default -MAX_RETRIES=${MAX_RETRIES:-3} -RETRY_DELAY=${RETRY_DELAY:-5} -MAX_PARALLEL_COPIES=${MAX_PARALLEL_COPIES:-10} # Maximum parallel copy operations (per-file mode only) - -# Bulk copy: tar all JSON files and copy one tarball per pod, then extract (much faster than N separate kubectl cp). -USE_TAR_BULK_COPY=${USE_TAR_BULK_COPY:-true} - -# Container selection configuration -# If CONTAINER_NAME is provided, it will be used exactly. -# Otherwise, the first container whose name contains CONTAINER_SUBSTRING (default: "worker") will be used. -# If no match is found and FALLBACK_TO_FIRST_CONTAINER is true (default: true), the first container will be used. -CONTAINER_NAME=${CONTAINER_NAME:-} -CONTAINER_SUBSTRING=${CONTAINER_SUBSTRING:-worker} -FALLBACK_TO_FIRST_CONTAINER=${FALLBACK_TO_FIRST_CONTAINER:-true} - -echo "Deployments to process: $DEPLOYMENTS" -echo "Namespace: $NAMESPACE" -echo "Kubectl timeout: ${KUBECTL_TIMEOUT}s" -echo "Max retries: $MAX_RETRIES" -echo "Max parallel copies: $MAX_PARALLEL_COPIES" -echo "Bulk tar copy: $USE_TAR_BULK_COPY" - -# Resolve kubeconfig path from env (fallback to ./kubeconfig.yaml) -KUBECONFIG_PATH="${KUBE_CONFIG_FILE:-./kubeconfig.yaml}" - -# Debug: Check if kubeconfig exists and kubectl works -echo "" -echo "=== DEBUGGING KUBECTL ACCESS ===" -if [ -f "$KUBECONFIG_PATH" ]; then - echo "โœ“ kubeconfig file exists: $KUBECONFIG_PATH" - echo "File size: $(stat -c%s "$KUBECONFIG_PATH" 2>/dev/null || stat -f%z "$KUBECONFIG_PATH" 2>/dev/null || echo 'unknown') bytes" -else - echo "โœ— kubeconfig file NOT found: $KUBECONFIG_PATH" -fi - -echo "Testing kubectl connection..." -kubectl --kubeconfig "$KUBECONFIG_PATH" --insecure-skip-tls-verify cluster-info --request-timeout=10s 2>&1 | head -3 - -echo "Listing all pods in namespace $NAMESPACE..." -kubectl --kubeconfig "$KUBECONFIG_PATH" --insecure-skip-tls-verify get pods -n "$NAMESPACE" --request-timeout=10s 2>&1 | head -10 - -echo "=== END DEBUG ===" -echo "" - -# Select target container for a given pod -get_target_container() { - local pod_name="$1" - local namespace="$2" - - # List all container names in the pod - local containers - containers=$(kubectl --kubeconfig "$KUBECONFIG_PATH" --insecure-skip-tls-verify get pod "$pod_name" -n "$namespace" -o jsonpath='{.spec.containers[*].name}' 2>/dev/null) - - if [ -z "$containers" ]; then - echo "" - return 0 - fi - - # If an explicit container name is provided and exists, use it - if [ -n "$CONTAINER_NAME" ]; then - if echo "$containers" | tr ' ' '\n' | grep -qx "$CONTAINER_NAME"; then - echo "$CONTAINER_NAME" - return 0 - fi - fi - - # Try to find a container matching the substring (case-insensitive) - if [ -n "$CONTAINER_SUBSTRING" ]; then - local match - match=$(echo "$containers" | tr ' ' '\n' | grep -i "$CONTAINER_SUBSTRING" | head -1) - if [ -n "$match" ]; then - echo "$match" - return 0 - fi - fi - - # Fallback to the first container if allowed - if [ "$FALLBACK_TO_FIRST_CONTAINER" = "true" ]; then - echo "$containers" | awk '{print $1}' - return 0 - fi - - echo "" -} - -# Copy all cookie JSON files to a pod in one shot: tar locally, kubectl cp tarball, extract in pod. -# Much faster than one kubectl cp per file when many JSON files exist. -copy_bulk_tar_to_pod() { - local pod_name="$1" - local namespace="$2" - local container_name="$3" - local cookie_dir="$4" - local kubeconfig="$5" - local tmp_tar="/tmp/cookies_$$.tar" - local remote_tar="/home/masa/cookies_$$.tar" - - if [ ! -d "$cookie_dir" ] || [ -z "$(ls -A "$cookie_dir"/*.json 2>/dev/null)" ]; then - echo "No JSON files in $cookie_dir, skipping bulk copy" - return 0 - fi - - echo "Creating tarball of cookie JSON files..." - if ! ( cd "$cookie_dir" && tar -cf "$tmp_tar" *.json ); then - echo "Failed to create tarball" - return 1 - fi - local tar_size - tar_size=$(stat -c%s "$tmp_tar" 2>/dev/null || stat -f%z "$tmp_tar" 2>/dev/null) - echo "Tarball size: ${tar_size:-?} bytes" - - kubectl_cp() { - if [ -n "$container_name" ]; then - timeout "$KUBECTL_TIMEOUT" kubectl --kubeconfig "$kubeconfig" --insecure-skip-tls-verify cp "$tmp_tar" "$namespace/$pod_name:$remote_tar" -c "$container_name" --request-timeout="${KUBECTL_TIMEOUT}s" - else - timeout "$KUBECTL_TIMEOUT" kubectl --kubeconfig "$kubeconfig" --insecure-skip-tls-verify cp "$tmp_tar" "$namespace/$pod_name:$remote_tar" --request-timeout="${KUBECTL_TIMEOUT}s" - fi - } - kubectl_exec() { - local cmd="$1" - if [ -n "$container_name" ]; then - kubectl --kubeconfig "$kubeconfig" --insecure-skip-tls-verify exec -n "$namespace" "$pod_name" -c "$container_name" --request-timeout="${KUBECTL_TIMEOUT}s" -- /bin/sh -c "$cmd" - else - kubectl --kubeconfig "$kubeconfig" --insecure-skip-tls-verify exec -n "$namespace" "$pod_name" --request-timeout="${KUBECTL_TIMEOUT}s" -- /bin/sh -c "$cmd" - fi - } - - for ((attempt=1; attempt<=MAX_RETRIES; attempt++)); do - echo "Copying tarball to pod (attempt $attempt/$MAX_RETRIES)..." - if kubectl_cp; then - break - fi - if [ "$attempt" -lt "$MAX_RETRIES" ]; then - echo "Waiting ${RETRY_DELAY}s before retry..." - sleep "$RETRY_DELAY" - else - rm -f "$tmp_tar" - echo "โœ— Failed to copy tarball after $MAX_RETRIES attempts" - return 1 - fi - done - - echo "Extracting tarball in pod..." - if ! kubectl_exec "tar -xf $remote_tar -C /home/masa && rm -f $remote_tar"; then - kubectl_exec "rm -f $remote_tar" 2>/dev/null - rm -f "$tmp_tar" - echo "โœ— Failed to extract tarball in pod" - return 1 - fi - rm -f "$tmp_tar" - echo "โœ“ Bulk copy completed successfully" - return 0 -} - -# Function to copy file with retry logic -copy_file_with_retry() { - local file="$1" - local pod_name="$2" - local namespace="$3" - local max_retries="$4" - local container_name="$5" - local filename=$(basename "$file") - - for ((i=1; i<=max_retries; i++)); do - echo "Copying $filename (attempt $i/$max_retries)..." - - # Use timeout command to limit kubectl execution time - # Prefer provided/detected container; otherwise try common worker patterns - success=false - - # Method 0: If a container_name was provided, try that first - if [ -n "$container_name" ] && timeout $KUBECTL_TIMEOUT kubectl --kubeconfig "$KUBECONFIG_PATH" --insecure-skip-tls-verify cp "$file" "$namespace/$pod_name:/home/masa/" -c "$container_name" --request-timeout=${KUBECTL_TIMEOUT}s 2>/dev/null; then - success=true - else - # Method 1: Try exact "worker" container name - if timeout $KUBECTL_TIMEOUT kubectl --kubeconfig "$KUBECONFIG_PATH" --insecure-skip-tls-verify cp "$file" "$namespace/$pod_name:/home/masa/" -c worker --request-timeout=${KUBECTL_TIMEOUT}s 2>/dev/null; then - success=true - else - # Method 2: Try to find a container whose name contains "worker" - local containers - containers=$(kubectl --kubeconfig "$KUBECONFIG_PATH" --insecure-skip-tls-verify get pod "$pod_name" -n "$namespace" -o jsonpath='{.spec.containers[*].name}' 2>/dev/null) - if [ -n "$containers" ]; then - for container in $(echo "$containers" | tr ' ' '\n' | grep -i worker); do - if timeout $KUBECTL_TIMEOUT kubectl --kubeconfig "$KUBECONFIG_PATH" --insecure-skip-tls-verify cp "$file" "$namespace/$pod_name:/home/masa/" -c "$container" --request-timeout=${KUBECTL_TIMEOUT}s 2>/dev/null; then - success=true - break - fi - done - fi - fi - fi - - if [ "$success" = true ]; then - echo "โœ“ Successfully copied $filename" - return 0 - else - local exit_code=$? - if [ $exit_code -eq 124 ]; then - echo "โš  Timeout occurred while copying $filename (attempt $i/$max_retries)" - else - echo "โš  Failed to copy $filename (attempt $i/$max_retries, exit code: $exit_code)" - fi - - if [ $i -lt $max_retries ]; then - echo "Waiting ${RETRY_DELAY}s before retry..." - sleep $RETRY_DELAY - fi - fi - done - - echo "โœ— Failed to copy $filename after $max_retries attempts" - return 1 -} - -# Convert comma-separated list to array -IFS=',' read -ra DEPLOYMENT_ARRAY <<< "$DEPLOYMENTS" - -# Track overall success -OVERALL_SUCCESS=true -FAILED_FILES=() - -# Process each deployment -for deployment in "${DEPLOYMENT_ARRAY[@]}"; do - echo "" - echo "Processing deployment: $deployment" - - # Try multiple methods to find the pod - POD_NAME="" - - # Method 1: Try to find pod by app label matching deployment name - POD_NAME=$(kubectl --kubeconfig "$KUBECONFIG_PATH" --insecure-skip-tls-verify get pods -n "$NAMESPACE" -l app="$deployment" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) - - # Method 2: If not found, try to find pod by name containing the deployment name - if [ -z "$POD_NAME" ]; then - POD_NAME=$(kubectl --kubeconfig "$KUBECONFIG_PATH" --insecure-skip-tls-verify get pods -n "$NAMESPACE" -o jsonpath='{.items[?(@.metadata.name contains "'$deployment'")].metadata.name}' 2>/dev/null | head -1) - fi - - # Method 3: If still not found, try to find any pod with "worker" in the name that matches part of deployment - if [ -z "$POD_NAME" ]; then - # Extract the worker identifier (e.g., "juno" from "tee-worker-juno") - worker_id=$(echo "$deployment" | sed 's/.*worker-//') - POD_NAME=$(kubectl --kubeconfig "$KUBECONFIG_PATH" --insecure-skip-tls-verify get pods -n "$NAMESPACE" -o jsonpath='{.items[?(@.metadata.name contains "worker")].metadata.name}' 2>/dev/null | grep "$worker_id" | head -1) - fi - - # Method 4: If still not found, try to find any pod containing "worker" - if [ -z "$POD_NAME" ]; then - POD_NAME=$(kubectl --kubeconfig "$KUBECONFIG_PATH" --insecure-skip-tls-verify get pods -n "$NAMESPACE" -o jsonpath='{.items[?(@.metadata.name contains "worker")].metadata.name}' 2>/dev/null | head -1) - fi - - if [ -z "$POD_NAME" ]; then - echo "Warning: No pod found for deployment $deployment in namespace $NAMESPACE, skipping..." - continue - fi - - echo "Found pod: $POD_NAME in namespace: $NAMESPACE" - - # Show available containers for debugging - echo "Available containers in pod:" - kubectl --kubeconfig "$KUBECONFIG_PATH" --insecure-skip-tls-verify get pod "$POD_NAME" -n "$NAMESPACE" -o jsonpath='{.spec.containers[*].name}' 2>/dev/null || echo "Could not list containers" - - # Detect target container - TARGET_CONTAINER=$(get_target_container "$POD_NAME" "$NAMESPACE") - if [ -n "$TARGET_CONTAINER" ]; then - echo "Using container: $TARGET_CONTAINER" - else - echo "Warning: Could not auto-detect target container; will try common worker patterns" - fi - echo "Copying cookie files to /home/masa/..." - - COOKIES_DIR="${COOKIES_DIR:-./cookies}" - # Collect all files (for count and for fallback per-file mode) - declare -a file_list=() - for file in "$COOKIES_DIR"/*.json; do - if [ -f "$file" ]; then - file_list+=("$file") - fi - done - - if [ ${#file_list[@]} -eq 0 ]; then - echo "No cookie files found to copy" - elif [ "$USE_TAR_BULK_COPY" = "true" ]; then - # Fast path: one tarball per pod, single copy + extract - if ! copy_bulk_tar_to_pod "$POD_NAME" "$NAMESPACE" "$TARGET_CONTAINER" "$COOKIES_DIR" "$KUBECONFIG_PATH"; then - OVERALL_SUCCESS=false - FAILED_FILES+=("(bulk tarball) -> $deployment") - fi - else - # Per-file path: parallel kubectl cp (slower, more retries per file) - echo "Starting parallel copy of ${#file_list[@]} files (max ${MAX_PARALLEL_COPIES} concurrent)..." - for ((i=0; i<${#file_list[@]}; i+=MAX_PARALLEL_COPIES)); do - batch_pids=() - for ((j=i; j "/tmp/failed_$(basename "$file")_$$" - fi - ) & - batch_pids+=($!) - done - for pid in "${batch_pids[@]}"; do - wait "$pid" - done - done - for file in "${file_list[@]}"; do - if [ -f "/tmp/failed_$(basename "$file")_$$" ]; then - OVERALL_SUCCESS=false - FAILED_FILES+=("$(basename "$file") -> $deployment") - rm -f "/tmp/failed_$(basename "$file")_$$" - fi - done - fi - - echo "Cookie files processing completed for $deployment!" -done - -echo "" -echo "All deployments processed!" - -# Report final status -if [ "$OVERALL_SUCCESS" = true ]; then - echo "โœ“ All cookie files copied successfully!" - exit 0 -else - echo "โš  Some files failed to copy:" - for failed in "${FAILED_FILES[@]}"; do - echo " - $failed" - done - echo "Consider increasing KUBECTL_TIMEOUT or MAX_RETRIES environment variables" - exit 1 -fi \ No newline at end of file diff --git a/tests/test_metagraph_e2e.py b/tests/test_metagraph_e2e.py deleted file mode 100644 index 24be1c6e..00000000 --- a/tests/test_metagraph_e2e.py +++ /dev/null @@ -1,21 +0,0 @@ -import pytest -from neurons.validator import Validator -from fiber.logging_utils import get_logger - -logger = get_logger(__name__) - - -@pytest.mark.asyncio -async def test_metagraph_e2e(): - # Initialize a real Validator instance - validator = Validator() - - # Run the sync_metagraph method - await validator.metagraph_manager.sync_metagraph() - - # Verify that nodes are populated - assert ( - len(validator.metagraph.nodes) > 0 - ), "Nodes should be populated after sync_metagraph" - - logger.info(f"Successfully synced {len(validator.metagraph.nodes)} nodes") diff --git a/tests/test_miner_proxy.py b/tests/test_miner_proxy.py deleted file mode 100644 index e530e651..00000000 --- a/tests/test_miner_proxy.py +++ /dev/null @@ -1,79 +0,0 @@ -import os -import unittest -from fastapi.testclient import TestClient -from miner.routes_manager import MinerAPI -from fastapi import FastAPI -from threading import Thread -import uvicorn -import time - -# Mock server setup -mock_app = FastAPI() - - -@mock_app.get("/test-path") -async def read_root(): - return {"message": "GET request received"} - - -@mock_app.post("/test-path") -async def create_item(item: dict): - return {"message": "POST request received", "item": item} - - -# Function to run the mock server -def run_server(app, port): - uvicorn.run(app, host="127.0.0.1", port=port) - - -class TestMinerProxy(unittest.TestCase): - - @classmethod - def setUpClass(cls): - - # Set up the test client - cls.api = MinerAPI(None) - cls.client = TestClient(cls.api.app) - - # Start the miner server in a separate thread - cls.miner_server_thread = Thread( - target=lambda: run_server(cls.api.app, 9000), daemon=True - ) - cls.miner_server_thread.start() - - # Start the mock server in a separate thread - cls.mock_server_thread = Thread( - target=lambda: run_server(mock_app, 8000), daemon=True - ) - cls.mock_server_thread.start() - # Add a delay to ensure the server is ready - time.sleep(2) - - # Mock the TEE address in the environment - os.environ["TEE_ADDRESS"] = "http://127.0.0.1:8000" - - def test_proxy_get_request(self): - response = self.client.get("http://127.0.0.1:9000/proxy/test-path") - - response_body = response.json() - content = response_body["content"] - - self.assertEqual(response.status_code, 200) - self.assertEqual(content, {"message": "GET request received"}) - - def test_proxy_post_request(self): - response = self.client.post( - "http://127.0.0.1:9000/proxy/test-path", json={"key": "value"} - ) - response_body = response.json() - content = response_body["content"] - - self.assertEqual(response.status_code, 200) - self.assertEqual( - content, - {"message": "POST request received", "item": {"key": "value"}}, - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_monitoring.py b/tests/test_monitoring.py index 6b259fcb..770d2741 100644 --- a/tests/test_monitoring.py +++ b/tests/test_monitoring.py @@ -160,15 +160,9 @@ async def test_error_rate_calculation_in_weights(self): boot_time=0, last_operation_time=0, current_time=1000, - twitter_auth_errors=0, - twitter_errors=0, - twitter_ratelimit_errors=0, - twitter_returned_other=0, - twitter_returned_profiles=0, # Not used in scoring anymore - twitter_returned_tweets=200, - twitter_scrapes=0, - web_errors=0, - web_success=0, # Not used in scoring anymore + stats_json={ + "twitter_returned_tweets": 200, + }, ), # Low tweets, high errors NodeData( @@ -179,15 +173,9 @@ async def test_error_rate_calculation_in_weights(self): boot_time=0, last_operation_time=0, current_time=1000, - twitter_auth_errors=0, - twitter_errors=0, - twitter_ratelimit_errors=0, - twitter_returned_other=0, - twitter_returned_profiles=0, # Not used in scoring anymore - twitter_returned_tweets=20, - twitter_scrapes=0, - web_errors=0, - web_success=0, # Not used in scoring anymore + stats_json={ + "twitter_returned_tweets": 20, + }, ), ] @@ -241,15 +229,9 @@ async def test_configurable_weights(self): boot_time=0, last_operation_time=0, current_time=1000, - twitter_auth_errors=0, - twitter_errors=0, - twitter_ratelimit_errors=0, - twitter_returned_other=0, - twitter_returned_profiles=0, - twitter_returned_tweets=100, - twitter_scrapes=0, - web_errors=0, - web_success=0, + stats_json={ + "twitter_returned_tweets": 100, + }, ) test_node.time_span_seconds = 3600 # 1 hour @@ -290,15 +272,11 @@ def test_error_rate_edge_cases(self): boot_time=0, last_operation_time=0, current_time=1000, - twitter_auth_errors=0, - twitter_errors=0, - twitter_ratelimit_errors=0, - twitter_returned_other=0, - twitter_returned_profiles=50, - twitter_returned_tweets=100, - twitter_scrapes=0, - web_errors=0, - web_success=75, + stats_json={ + "twitter_returned_profiles": 50, + "twitter_returned_tweets": 100, + "web_processed_pages": 75, + }, ) # Zero time span should be handled gracefully @@ -363,15 +341,9 @@ async def test_priority_miners_generation(self): boot_time=0, last_operation_time=0, current_time=0, - twitter_auth_errors=0, - twitter_errors=0, - twitter_ratelimit_errors=0, - twitter_returned_other=0, - twitter_returned_profiles=0, - twitter_returned_tweets=100, # High tweet count - twitter_scrapes=0, - web_errors=0, - web_success=0, + stats_json={ + "twitter_returned_tweets": 100, # High tweet count + }, ), NodeData( hotkey="hotkey2", @@ -381,15 +353,9 @@ async def test_priority_miners_generation(self): boot_time=0, last_operation_time=0, current_time=0, - twitter_auth_errors=0, - twitter_errors=0, - twitter_ratelimit_errors=0, - twitter_returned_other=0, - twitter_returned_profiles=0, - twitter_returned_tweets=50, # Medium tweet count - twitter_scrapes=0, - web_errors=0, - web_success=0, + stats_json={ + "twitter_returned_tweets": 50, # Medium tweet count + }, ), NodeData( hotkey="hotkey3", @@ -399,15 +365,9 @@ async def test_priority_miners_generation(self): boot_time=0, last_operation_time=0, current_time=0, - twitter_auth_errors=0, - twitter_errors=0, - twitter_ratelimit_errors=0, - twitter_returned_other=0, - twitter_returned_profiles=0, - twitter_returned_tweets=10, # Low tweet count - twitter_scrapes=0, - web_errors=0, - web_success=0, + stats_json={ + "twitter_returned_tweets": 10, # Low tweet count + }, ), ] @@ -424,16 +384,19 @@ async def test_priority_miners_generation(self): delta_node_data, simulation=True ) - # Verify we got addresses in priority order - assert len(priority_miners) == 3 - # Should be ordered by score (highest first) - # hotkey1 should be first (highest tweets), hotkey3 should be last (lowest tweets) - assert "192.168.1.1" in priority_miners # hotkey1 - assert "192.168.1.2" in priority_miners # hotkey2 - assert "192.168.1.3" in priority_miners # hotkey3 - - # First address should correspond to highest scoring node (hotkey1) - assert priority_miners[0] == "192.168.1.1" + # get_priority_miners_by_score returns a weighted list (256 total) + # with higher-scoring nodes appearing more frequently + assert len(priority_miners) == 256 + + # Higher scoring nodes should appear in the list + unique_addresses = set(priority_miners) + assert "192.168.1.1" in unique_addresses # hotkey1 (highest tweets) + assert "192.168.1.2" in unique_addresses # hotkey2 (medium tweets) + + # The highest scorer should appear more frequently + count_hotkey1 = priority_miners.count("192.168.1.1") + count_hotkey2 = priority_miners.count("192.168.1.2") + assert count_hotkey1 > count_hotkey2 # More tweets = higher frequency if __name__ == "__main__": diff --git a/tests/test_nats_client.py b/tests/test_nats_client.py deleted file mode 100644 index b8bf1eed..00000000 --- a/tests/test_nats_client.py +++ /dev/null @@ -1,65 +0,0 @@ -import unittest -import asyncio -import os -from unittest.mock import MagicMock -from miner.nats_client import NatsClient -from nats.aio.client import Client as NATS - - -class TestNatsClient(unittest.TestCase): - def setUp(self): - self.loop = asyncio.new_event_loop() - asyncio.set_event_loop(self.loop) - self.nats_client = NatsClient() - self.nc = NATS() - - self.nodes = None - - async def async_message_handler(self, msg): - subject = msg.subject - reply = msg.reply - data = msg.data.decode() - print(f"Received a message on '{subject} {reply}': {data}") - - self.nodes = data - # Process the message here - - def test_send_connected_nodes(self): - async def run_test(): - # Mock the message handler to track calls - self.nats_client.message_handler = MagicMock() - - # Connect to the NATS server - nats_url = os.getenv("NATS_URL", "nats://127.0.0.1:4222") - await self.nc.connect(nats_url) - - # Subscribe to the channel - channel_name = os.getenv("TEE_NATS_CHANNEL_NAME", "miners") - await self.nc.subscribe(channel_name, cb=self.async_message_handler) - - # Send a test message - miners = ["0.0.0.0:8080", "0.0.0.0:8081"] - await self.nats_client.send_connected_nodes(miners) - - # Allow some time for the message to be processed - await asyncio.sleep(1) - - # Check if the message handler was called with the expected message - # self.nats_client.message_handler.assert_called() - - await asyncio.sleep(2) - - # Verify the received message contains the expected miners - received_data = eval(self.nodes) - self.assertEqual(received_data["Miners"], miners) - # Close the connection - await self.nc.close() - - self.loop.run_until_complete(run_test()) - - def tearDown(self): - self.loop.close() - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_node_manager.py b/tests/test_node_manager.py index 865af018..135e510b 100644 --- a/tests/test_node_manager.py +++ b/tests/test_node_manager.py @@ -132,25 +132,6 @@ async def asyncTearDown(self): with suppress(Exception): await task - async def test_connect_with_miner_success(self): - # Mock the handshake function to return a valid key and UUID - self.mock_validator.http_client_manager.client = AsyncMock() - self.mock_validator.keypair = MagicMock() - self.mock_node.hotkey = "test_hotkey" - - # Mock the perform_handshake function - with unittest.mock.patch( - "fiber.encrypted.validator.handshake.perform_handshake", - new=AsyncMock(return_value=("symmetric_key_str", "symmetric_key_uuid")), - ): - result = await self.node_manager.connect_with_miner( - miner_address="test_address", - miner_hotkey="test_hotkey", - node=self.mock_node, - ) - self.assertTrue(result) - self.assertIn("test_hotkey", self.node_manager.connected_nodes) - async def test_connect_with_miner_failure(self): # Mock the handshake function to return None self.mock_validator.http_client_manager.client = AsyncMock() @@ -305,44 +286,5 @@ async def direct(func, *args, **kwargs): hotkey ) - async def test_register_tee_address_telemetry_wipe_failure_logs_and_continues(self): - hotkey = "hk5" - new_address = "tee://new-failure" - - routing_table = MagicMock() - routing_table.get_miner_addresses.side_effect = [ - [], - [(new_address, "w1")], - ] - routing_table.register_worker = MagicMock() - routing_table.add_miner_address = MagicMock() - - self.mock_validator.telemetry_storage.delete_telemetry_by_hotkey = MagicMock( - side_effect=RuntimeError("telemetry deletion failed") - ) - - async def direct(func, *args, **kwargs): - return func(*args, **kwargs) - - with patch("validator.node_manager.asyncio.to_thread", new=AsyncMock(side_effect=direct)): - verified = set() - with self.assertLogs("validator.node_manager", level="ERROR") as cm: - await self.node_manager._register_tee_address( - routing_table=routing_table, - hotkey=hotkey, - node=self.mock_node, - tee_address=new_address, - worker_id="worker-1", - worker_hotkey="existing-hotkey", - verified_entries=verified, - ) - - routing_table.add_miner_address.assert_called_once() - self.assertIn((hotkey, new_address), verified) - self.assertTrue( - any("Failed to delete telemetry for hotkey" in msg for msg in cm.output), - msg="Expected telemetry deletion failure to be logged as an error.", - ) - if __name__ == "__main__": unittest.main() diff --git a/tests/test_weights_e2e.py b/tests/test_weights_e2e.py deleted file mode 100644 index 277f506c..00000000 --- a/tests/test_weights_e2e.py +++ /dev/null @@ -1,60 +0,0 @@ -import pytest -from neurons.validator import Validator -from validator.weights import WeightsManager -from interfaces.types import NodeData - - -@pytest.mark.asyncio -async def test_weights_e2e(): - # Initialize a real Validator instance - validator = Validator() - weights_manager = WeightsManager(validator=validator) - - # Simulate node data - node_data = [ - NodeData( - hotkey="node1", - worker_id="worker1", - uid=1, - boot_time=0, - last_operation_time=0, - current_time=0, - twitter_auth_errors=0, - twitter_errors=0, - twitter_ratelimit_errors=0, - twitter_returned_other=0, - twitter_returned_profiles=0, - twitter_returned_tweets=0, - twitter_scrapes=0, - web_errors=0, - web_success=10, - timestamp=0, - ), - NodeData( - hotkey="node2", - worker_id="worker2", - uid=2, - boot_time=0, - last_operation_time=0, - current_time=0, - twitter_auth_errors=0, - twitter_errors=0, - twitter_ratelimit_errors=0, - twitter_returned_other=0, - twitter_returned_profiles=0, - twitter_returned_tweets=20, - twitter_scrapes=0, - web_errors=0, - web_success=20, - timestamp=0, - ), - ] - - # Calculate weights - uids, weights = weights_manager.calculate_weights(node_data) - assert len(uids) == len(weights) > 0, "Weights should be calculated for nodes" - - # Set weights - await weights_manager.set_weights(node_data) - - # Here you would verify the weights were set correctly, possibly by querying the substrate diff --git a/tests/test_weights_unit.py b/tests/test_weights_unit.py index 7df93932..f73446dd 100644 --- a/tests/test_weights_unit.py +++ b/tests/test_weights_unit.py @@ -20,8 +20,11 @@ def weights_manager(mock_validator): return WeightsManager(validator=mock_validator) -def test_calculate_weights(weights_manager): +@pytest.mark.asyncio +async def test_calculate_weights(weights_manager, mock_validator): # Test calculate_weights method + mock_validator.node_manager.send_score_report = AsyncMock() + node_data = [ NodeData( hotkey="node1", @@ -30,16 +33,11 @@ def test_calculate_weights(weights_manager): boot_time=0, last_operation_time=0, current_time=0, - twitter_auth_errors=0, - twitter_errors=0, - twitter_ratelimit_errors=0, - twitter_returned_other=0, - twitter_returned_profiles=0, - twitter_returned_tweets=0, - twitter_scrapes=0, - web_errors=0, - web_success=10, timestamp=0, + stats_json={ + "twitter_returned_tweets": 0, + "web_processed_pages": 10, + }, ), NodeData( hotkey="node2", @@ -48,33 +46,46 @@ def test_calculate_weights(weights_manager): boot_time=0, last_operation_time=0, current_time=0, - twitter_auth_errors=0, - twitter_errors=0, - twitter_ratelimit_errors=0, - twitter_returned_other=0, - twitter_returned_profiles=0, - twitter_returned_tweets=20, - twitter_scrapes=0, - web_errors=0, - web_success=20, timestamp=0, + stats_json={ + "twitter_returned_tweets": 20, + "web_processed_pages": 20, + }, ), ] - uids, weights = weights_manager.calculate_weights(node_data) + uids, weights = await weights_manager.calculate_weights(node_data, simulation=True) assert len(uids) == len(weights) == 2 assert weights[0] < weights[1] # Assuming node2 has more activity @pytest.mark.asyncio async def test_set_weights(weights_manager, mock_validator): - # Mock the async method and dependencies - mock_validator.substrate.query = MagicMock(return_value=MagicMock(value=1)) + # Mock the required validator attributes + mock_validator.substrate.url = "ws://localhost:9944" + mock_validator.netuid = 42 + mock_validator.keypair.ss58_address = "validator1" mock_validator.metagraph.nodes = { - "node1": MagicMock(node_id=1), - "node2": MagicMock(node_id=2), + "validator1": MagicMock(node_id=0, hotkey="validator1"), + "node1": MagicMock(node_id=1, hotkey="node1"), + "node2": MagicMock(node_id=2, hotkey="node2"), } - with patch( - "validator.weights.weights.set_node_weights", return_value=True - ) as mock_set_node_weights: - await weights_manager.set_weights([]) + + # Mock telemetry storage to return empty data (simpler test path) + mock_validator.telemetry_storage = MagicMock() + mock_validator.telemetry_storage.get_all_telemetry = MagicMock(return_value={}) + + # Mock async methods + mock_validator.scorer = MagicMock() + mock_validator.scorer.fetch_active_worker_version = AsyncMock() + mock_validator.node_manager = MagicMock() + mock_validator.node_manager.send_score_report = AsyncMock() + + with patch("validator.weights.interface.get_substrate") as mock_get_substrate, \ + patch("validator.weights.weights.blocks_since_last_update", return_value=1000), \ + patch("validator.weights.weights.min_interval_to_set_weights", return_value=100), \ + patch("validator.weights.weights.set_node_weights", return_value=True) as mock_set_node_weights: + mock_get_substrate.return_value = mock_validator.substrate + await weights_manager.set_weights() mock_set_node_weights.assert_called_once() + + diff --git a/validator/platform_config.py b/validator/platform_config.py index 685bea76..919094e3 100644 --- a/validator/platform_config.py +++ b/validator/platform_config.py @@ -68,6 +68,8 @@ def __init__(self): metrics=[ "scrapes", "tweets", + "profiles", + "other", "errors", "auth_errors", "ratelimit_errors", @@ -78,6 +80,8 @@ def __init__(self): # Map raw telemetry field names to clean platform metric names "twitter_scrapes": "scrapes", "twitter_returned_tweets": "tweets", + "twitter_returned_profiles": "profiles", + "twitter_returned_other": "other", "twitter_errors": "errors", "twitter_auth_errors": "auth_errors", "twitter_ratelimit_errors": "ratelimit_errors", @@ -88,13 +92,11 @@ def __init__(self): emission_weight=0.05, metrics=[ "followers", - "errors", ], - error_metrics=["errors"], + error_metrics=[], success_metrics=["followers"], field_mappings={ "twitter_returned_followers": "followers", - "twitter_errors": "errors", }, ), "tiktok-transcription": PlatformConfig( @@ -112,26 +114,28 @@ def __init__(self): "tiktok-search": PlatformConfig( name="tiktok-search", emission_weight=0.05, - metrics=["queries", "videos", "errors"], - error_metrics=["errors"], + metrics=["queries", "videos", "errors", "auth_errors"], + error_metrics=["errors", "auth_errors"], success_metrics=["videos"], field_mappings={ # Map raw telemetry field names to clean platform metric names "tiktok_queries": "queries", "tiktok_returned_videos": "videos", "tiktok_errors": "errors", + "tiktok_auth_errors": "auth_errors", }, ), "web": PlatformConfig( name="web", emission_weight=0.05, - metrics=["queries", "pages", "errors"], + metrics=["queries", "pages", "scraped_pages", "errors"], error_metrics=["errors"], - success_metrics=["pages"], + success_metrics=["pages", "scraped_pages"], field_mappings={ # Map raw telemetry field names to clean platform metric names "web_queries": "queries", "web_processed_pages": "pages", + "web_scraped_pages": "scraped_pages", "web_errors": "errors", }, ), @@ -161,6 +165,19 @@ def __init__(self): "linkedin_queries": "queries", }, ), + "llm": PlatformConfig( + name="llm", + emission_weight=0.0, # Not scored currently, but tracked for metrics visibility + metrics=["queries", "processed_items", "errors"], + error_metrics=["errors"], + success_metrics=["processed_items"], + field_mappings={ + # Map raw telemetry field names to clean platform metric names + "llm_queries": "queries", + "llm_processed_items": "processed_items", + "llm_errors": "errors", + }, + ), } # Validate that emission weights sum to 1.0 diff --git a/validator/weights.py b/validator/weights.py index 298ac987..d52db13c 100644 --- a/validator/weights.py +++ b/validator/weights.py @@ -261,7 +261,11 @@ def _get_delta_node_data(self, telemetry_data: List[NodeData]) -> List[NodeData] hotkey = node_data.hotkey all_hotkeys.append((node_data.node_id, hotkey)) - print(f"all hotkeys: {all_hotkeys}") + logger.debug(f"All hotkeys: {all_hotkeys}") + + # Reuse the PlatformManager instance from __init__ + all_raw_fields = self.platform_manager.get_all_raw_field_names() + # Process hotkeys with telemetry data processed_hotkeys = set() for hotkey, telemetry_list in telemetry_by_hotkey.items(): @@ -272,21 +276,19 @@ def _get_delta_node_data(self, telemetry_data: List[NodeData]) -> List[NodeData] key=lambda x: self._convert_timestamp_to_int(x.timestamp), ) - # Split telemetry into chunks based on worker restarts - # Use twitter_returned_tweets as the restart indicator (legacy compatibility) + # Split telemetry into chunks based on worker restarts. + # Use boot_time as the definitive restart indicator (set once at worker startup). + # Note: Requires tee-worker versions that report boot_time. Workers with boot_time=0 + # will not have restart detection and may be undercounted if they restart mid-window. chunks = [] chunk_start = 0 for i in range(1, len(sorted_telemetry)): - # Check for restart by looking at twitter_returned_tweets decrease - current_tweets = sorted_telemetry[i].get_stat_value( - "twitter_returned_tweets", 0 - ) - prev_tweets = sorted_telemetry[i - 1].get_stat_value( - "twitter_returned_tweets", 0 - ) + current_boot_time = sorted_telemetry[i].boot_time + prev_boot_time = sorted_telemetry[i - 1].boot_time - if current_tweets < prev_tweets: + # A different boot_time indicates a restart + if current_boot_time != prev_boot_time and current_boot_time > 0: # Worker restart detected, end current chunk chunks.append( (chunk_start, i - 1) @@ -294,7 +296,7 @@ def _get_delta_node_data(self, telemetry_data: List[NodeData]) -> List[NodeData] chunk_start = i # Start new chunk at current record logger.debug( f"Worker restart detected for {hotkey} at timestamp " - f"{sorted_telemetry[i].timestamp}, creating new chunk" + f"{sorted_telemetry[i].timestamp} (boot_time changed: {prev_boot_time} -> {current_boot_time}), creating new chunk" ) # Add the final chunk @@ -303,10 +305,7 @@ def _get_delta_node_data(self, telemetry_data: List[NodeData]) -> List[NodeData] logger.debug(f"Created {len(chunks)} chunks for {hotkey}: {chunks}") # Calculate deltas for each chunk and sum them up dynamically - from validator.platform_config import PlatformManager - - platform_manager = PlatformManager() - all_raw_fields = platform_manager.get_all_raw_field_names() + # (PlatformManager already imported above for restart detection) # Initialize dynamic delta totals total_deltas = {}