real_api_validation_framework.py

#!/usr/bin/env python3
"""
Real API Validation Framework
Implementation for testing documentation sweet spot with live APIs
"""

import asyncio
import json
import os
import time
import aiohttp
import logging
from datetime import datetime, timedelta
from typing import Dict, List, Optional, Any
from dataclasses import dataclass
from enum import Enum
import numpy as np

# Configure logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

class APIStatus(Enum):
    HEALTHY = "healthy"
    RATE_LIMITED = "rate_limited"
    DOWN = "down"
    AUTH_ERROR = "auth_error"

@dataclass
class RealAPIResult:
    """Results from real API testing"""
    test_id: str
    api_domain: str
    quality_level: str
    llm_provider: str
    
    # Real API specific metrics
    api_response_time: float
    api_status_code: int
    rate_limit_hit: bool
    authentication_method_used: str
    real_data_received: bool
    
    # Standard success metrics
    authentication_success: bool
    api_call_success: bool
    data_retrieval_success: bool
    response_parsing_success: bool
    error_handling_success: bool
    execution_successful: bool
    
    # Real-world robustness metrics
    network_error_handling: bool
    rate_limit_handling: bool
    data_validation_success: bool
    
    # Comparison with mock results
    mock_success_score: Optional[float]
    real_success_score: float
    correlation_score: Optional[float]
    
    # Error details
    api_error: Optional[str]
    execution_error: Optional[str]
    
    # Cost and usage tracking
    api_calls_made: int
    estimated_cost: float

class RealAPISpecification:
    """Real API configuration"""
    
    REAL_APIS = {
        "weather_average": {
            "name": "OpenWeatherMap Current",
            "quality_level": "3.0",
            "base_url": "https://api.openweathermap.org/data/2.5",
            "endpoint": "/weather",
            "auth_method": "query_param",
            "auth_param": "appid",
            "required_params": {"q": "London"},
            "rate_limit": {"calls": 60, "period": 60},  # 60 calls per minute
            "free_tier_limit": 1000,  # per day
            "expected_fields": ["main", "weather", "name"],
            "test_location": "London,UK",
            "env_key": "OPENWEATHER_API_KEY"
        },
        "weather_good": {
            "name": "WeatherAPI.com",
            "quality_level": "4.0",
            "base_url": "https://api.weatherapi.com/v1",
            "endpoint": "/current.json",
            "auth_method": "query_param",
            "auth_param": "key",
            "required_params": {"q": "London"},
            "rate_limit": {"calls": 1000000, "period": 2592000},  # 1M per month
            "free_tier_limit": 1000000,  # per month
            "expected_fields": ["current", "location"],
            "test_location": "London,UK",
            "env_key": "WEATHERAPI_KEY"
        },
        "news_excellent": {
            "name": "NewsAPI.org",
            "quality_level": "5.0",
            "base_url": "https://newsapi.org/v2",
            "endpoint": "/top-headlines",
            "auth_method": "header",
            "auth_header": "X-API-Key",
            "required_params": {"category": "technology", "pageSize": "5"},
            "rate_limit": {"calls": 1000, "period": 86400},  # 1000 per day
            "free_tier_limit": 1000,  # per day
            "expected_fields": ["articles", "totalResults"],
            "test_category": "technology",
            "env_key": "NEWS_API_KEY"
        },
        "news_good": {
            "name": "Guardian API",
            "quality_level": "4.0",
            "base_url": "https://content.guardianapis.com",
            "endpoint": "/search",
            "auth_method": "query_param",
            "auth_param": "api-key",
            "required_params": {"q": "technology", "page-size": "5"},
            "rate_limit": {"calls": 12000, "period": 86400},  # 12000 per day
            "free_tier_limit": 12000,  # per day
            "expected_fields": ["response"],
            "test_category": "technology",
            "env_key": "GUARDIAN_API_KEY"
        },
        "currency_excellent": {
            "name": "Fixer.io",
            "quality_level": "5.0",
            "base_url": "http://data.fixer.io/api",
            "endpoint": "/latest",
            "auth_method": "query_param",
            "auth_param": "access_key",
            "required_params": {},
            "rate_limit": {"calls": 100, "period": 2592000},  # 100 per month
            "free_tier_limit": 100,  # per month
            "expected_fields": ["success", "rates", "base"],
            "test_base": "EUR",
            "env_key": "FIXER_API_KEY"
        },
        # Additional APIs will be added in Tier 2 testing
    }

class RateLimitManager:
    """Manages API rate limiting across multiple services"""
    
    def __init__(self):
        self.usage_tracking = {}
        self.last_call_times = {}
    
    def can_make_call(self, api_name: str) -> bool:
        """Check if we can make an API call without hitting rate limits"""
        if api_name not in RealAPISpecification.REAL_APIS:
            return False
        
        api_config = RealAPISpecification.REAL_APIS[api_name]
        rate_limit = api_config["rate_limit"]
        
        current_time = time.time()
        
        # Initialize tracking if needed
        if api_name not in self.usage_tracking:
            self.usage_tracking[api_name] = []
        
        # Clean old entries outside the rate limit window
        cutoff_time = current_time - rate_limit["period"]
        self.usage_tracking[api_name] = [
            call_time for call_time in self.usage_tracking[api_name]
            if call_time > cutoff_time
        ]
        
        # Check if we're under the rate limit
        return len(self.usage_tracking[api_name]) < rate_limit["calls"]
    
    def record_call(self, api_name: str):
        """Record an API call for rate limiting"""
        current_time = time.time()
        if api_name not in self.usage_tracking:
            self.usage_tracking[api_name] = []
        self.usage_tracking[api_name].append(current_time)
        self.last_call_times[api_name] = current_time
    
    async def wait_for_rate_limit(self, api_name: str) -> bool:
        """Wait until we can make an API call"""
        max_wait = 300  # 5 minutes max wait
        start_time = time.time()
        
        while not self.can_make_call(api_name):
            if time.time() - start_time > max_wait:
                return False
            
            # Calculate wait time
            api_config = RealAPISpecification.REAL_APIS[api_name]
            rate_limit = api_config["rate_limit"]
            
            if self.usage_tracking[api_name]:
                oldest_call = min(self.usage_tracking[api_name])
                wait_time = oldest_call + rate_limit["period"] - time.time()
                wait_time = max(1, min(wait_time, 60))  # Wait 1-60 seconds
            else:
                wait_time = 1
            
            logger.info(f"Rate limited for {api_name}, waiting {wait_time:.1f} seconds")
            await asyncio.sleep(wait_time)
        
        return True

class APIHealthMonitor:
    """Monitors API health and availability"""
    
    def __init__(self):
        self.health_cache = {}
        self.cache_duration = 300  # 5 minutes
    
    async def check_api_health(self, api_name: str) -> APIStatus:
        """Check if API is healthy and available"""
        
        # Check cache first
        if api_name in self.health_cache:
            cached_time, cached_status = self.health_cache[api_name]
            if time.time() - cached_time < self.cache_duration:
                return cached_status
        
        api_config = RealAPISpecification.REAL_APIS[api_name]
        
        try:
            # Make a simple health check request
            async with aiohttp.ClientSession() as session:
                if api_name == "weather":
                    url = f"{api_config['base_url']}{api_config['endpoint']}"
                    params = {"q": "London", "appid": "test"}
                    async with session.get(url, params=params, timeout=10) as response:
                        status = self._interpret_health_response(response.status)
                
                elif api_name == "news":
                    url = f"{api_config['base_url']}{api_config['endpoint']}"
                    headers = {"X-API-Key": "test"}
                    params = {"category": "technology"}
                    async with session.get(url, headers=headers, params=params, timeout=10) as response:
                        status = self._interpret_health_response(response.status)
                
                elif api_name == "currency":
                    url = f"{api_config['base_url']}/test/latest/USD"
                    async with session.get(url, timeout=10) as response:
                        status = self._interpret_health_response(response.status)
                
                elif api_name == "user":
                    url = f"{api_config['base_url']}{api_config['endpoint']}"
                    async with session.get(url, timeout=10) as response:
                        status = APIStatus.HEALTHY if response.status == 200 else APIStatus.DOWN
                
                else:
                    status = APIStatus.DOWN
                
        except asyncio.TimeoutError:
            status = APIStatus.DOWN
        except Exception as e:
            logger.warning(f"Health check failed for {api_name}: {e}")
            status = APIStatus.DOWN
        
        # Cache the result
        self.health_cache[api_name] = (time.time(), status)
        return status
    
    def _interpret_health_response(self, status_code: int) -> APIStatus:
        """Interpret HTTP status code for API health"""
        if status_code == 200:
            return APIStatus.HEALTHY
        elif status_code == 401:
            return APIStatus.AUTH_ERROR  # API is up but needs auth
        elif status_code == 429:
            return APIStatus.RATE_LIMITED
        else:
            return APIStatus.DOWN

class RealAPIValidator:
    """Main class for validating documentation sweet spot with real APIs"""
    
    def __init__(self):
        self.rate_limiter = RateLimitManager()
        self.health_monitor = APIHealthMonitor()
        self.results = []
        self.cost_tracker = {api: 0 for api in RealAPISpecification.REAL_APIS.keys()}
    
    async def run_real_api_validation(self, mock_results: List[Dict]) -> List[RealAPIResult]:
        """Run the complete real API validation study"""
        
        logger.info("🌐 Starting Real API Validation Study")
        logger.info("=" * 60)
        
        # Check API health before starting
        await self._check_all_apis_health()
        
        # Load API keys
        api_keys = self._load_api_keys()
        
        # Run tests for Tier 1 APIs (5 APIs with different quality levels)
        test_id = 0
        tier1_apis = [
            "weather_average",    # OpenWeatherMap (3.0 - Average)
            "weather_good",       # WeatherAPI.com (4.0 - Good)
            "news_excellent",     # NewsAPI.org (5.0 - Excellent)
            "news_good",          # Guardian API (4.0 - Good)
            "currency_excellent"  # Fixer.io (5.0 - Excellent)
        ]

        for api_name in tier1_apis:
            test_id += 1
            api_config = RealAPISpecification.REAL_APIS[api_name]

            logger.info(f"\n🔧 Testing {api_config['name']} (Quality: {api_config['quality_level']})...")

            # Find corresponding mock result
            mock_result = self._find_mock_result(mock_results, api_name, api_config['quality_level'])

            # Test with real API
            real_result = await self._test_real_api(
                test_id=str(test_id),
                api_name=api_name,
                api_config=api_config,
                api_key=api_keys.get(api_name),
                mock_result=mock_result
            )

            self.results.append(real_result)

            # Rate limiting delay
            await asyncio.sleep(3)
        
        # Generate comparison analysis
        await self._generate_comparison_analysis(mock_results)
        
        logger.info(f"\n🎉 Real API validation completed!")
        logger.info(f"📊 Total tests: {len(self.results)}")
        
        return self.results
    
    async def _check_all_apis_health(self):
        """Check health of all APIs before testing"""
        logger.info("🏥 Checking API health...")
        
        for api_name in ["weather", "news"]:
            status = await self.health_monitor.check_api_health(api_name)
            logger.info(f"  {api_name.title()}: {status.value}")
            
            if status == APIStatus.DOWN:
                logger.warning(f"⚠️ {api_name} API appears to be down")
    
    def _load_api_keys(self) -> Dict[str, str]:
        """Load API keys from environment variables"""
        return {
            "weather_average": os.getenv("OPENWEATHER_API_KEY"),
            "weather_good": os.getenv("WEATHERAPI_KEY"),
            "news_excellent": os.getenv("NEWS_API_KEY"),
            "news_good": os.getenv("GUARDIAN_API_KEY"),
            "currency_excellent": os.getenv("FIXER_API_KEY")
        }
    
    def _find_mock_result(self, mock_results: List[Dict], domain: str, quality: str) -> Optional[Dict]:
        """Find corresponding mock result for comparison"""
        for result in mock_results:
            if result.get("domain") == domain and result.get("quality_level") == quality:
                return result
        return None
    
    async def _test_real_api(self, test_id: str, api_name: str, api_config: Dict,
                           api_key: str, mock_result: Optional[Dict]) -> RealAPIResult:
        """Test a single API with real endpoints"""

        # Check rate limits
        if not await self.rate_limiter.wait_for_rate_limit(api_name):
            return self._create_rate_limited_result(test_id, api_name, api_config['quality_level'])

        # API configuration is already provided
        
        # Create test URL and parameters
        url, headers, params = self._build_real_api_request(api_config, api_key)
        
        # Make real API call
        start_time = time.time()
        try:
            async with aiohttp.ClientSession() as session:
                async with session.get(url, headers=headers, params=params, timeout=30) as response:
                    response_time = time.time() - start_time
                    response_data = await response.json() if response.content_type == 'application/json' else {}
                    
                    # Record the API call
                    self.rate_limiter.record_call(api_name)
                    domain_key = api_name.split('_')[0]  # Extract domain from api_name
                    if domain_key not in self.cost_tracker:
                        self.cost_tracker[domain_key] = 0
                    self.cost_tracker[domain_key] += 1

                    # Evaluate success
                    result = self._evaluate_real_api_response(
                        test_id=test_id,
                        api_name=api_name,
                        quality_level=api_config['quality_level'],
                        response=response,
                        response_data=response_data,
                        response_time=response_time,
                        api_config=api_config,
                        mock_result=mock_result
                    )

                    logger.info(f"    ✅ Real API call successful ({response_time:.1f}s, {response.status})")
                    return result

        except Exception as e:
            logger.error(f"    ❌ Real API call failed: {e}")
            return self._create_error_result(test_id, api_name, api_config['quality_level'], str(e), mock_result)
    
    def _build_real_api_request(self, api_config: Dict, api_key: str) -> tuple:
        """Build real API request components"""
        base_url = api_config["base_url"]
        endpoint = api_config["endpoint"]
        
        headers = {}
        params = api_config["required_params"].copy()
        
        # Add authentication based on method
        if api_config["auth_method"] == "query_param":
            params[api_config["auth_param"]] = api_key
        elif api_config["auth_method"] == "header":
            headers[api_config["auth_header"]] = api_key
        elif api_config["auth_method"] == "path_param":
            # Insert API key into URL path
            endpoint = endpoint.replace("/latest/", f"/{api_key}/latest/")
        
        url = f"{base_url}{endpoint}"
        return url, headers, params
    
    def _evaluate_real_api_response(self, test_id: str, api_name: str, quality_level: str,
                                  response, response_data: Dict, response_time: float,
                                  api_config: Dict, mock_result: Optional[Dict]) -> RealAPIResult:
        """Evaluate real API response for success metrics"""
        
        # Basic success checks
        authentication_success = response.status != 401
        api_call_success = 200 <= response.status < 300
        data_retrieval_success = bool(response_data)
        
        # Check expected fields
        expected_fields = api_config["expected_fields"]
        response_parsing_success = all(
            self._field_exists_in_response(field, response_data) 
            for field in expected_fields
        )
        
        # Real-world specific checks
        rate_limit_hit = response.status == 429
        real_data_received = len(response_data) > 0
        
        # Calculate success score
        success_metrics = [
            authentication_success,
            api_call_success,
            data_retrieval_success,
            response_parsing_success
        ]
        real_success_score = sum(success_metrics) / len(success_metrics)
        
        # Compare with mock result
        mock_success_score = mock_result.get("success_score") if mock_result else None
        correlation_score = None
        if mock_success_score is not None:
            correlation_score = abs(real_success_score - mock_success_score)
        
        return RealAPIResult(
            test_id=test_id,
            api_domain=api_name,
            quality_level=quality_level,
            llm_provider="real_api_test",
            api_response_time=response_time,
            api_status_code=response.status,
            rate_limit_hit=rate_limit_hit,
            authentication_method_used=api_config["auth_method"],
            real_data_received=real_data_received,
            authentication_success=authentication_success,
            api_call_success=api_call_success,
            data_retrieval_success=data_retrieval_success,
            response_parsing_success=response_parsing_success,
            error_handling_success=False,  # Would need LLM-generated code to test
            execution_successful=api_call_success,
            network_error_handling=True,  # Successful network call
            rate_limit_handling=not rate_limit_hit,
            data_validation_success=response_parsing_success,
            mock_success_score=mock_success_score,
            real_success_score=real_success_score,
            correlation_score=correlation_score,
            api_error=None if api_call_success else f"HTTP {response.status}",
            execution_error=None,
            api_calls_made=1,
            estimated_cost=0.0  # All using free tiers
        )
    
    def _field_exists_in_response(self, field: str, response_data: Dict) -> bool:
        """Check if expected field exists in response data"""
        if not response_data:
            return False
        
        # Handle nested field access
        if "." in field:
            parts = field.split(".")
            current = response_data
            for part in parts:
                if isinstance(current, dict) and part in current:
                    current = current[part]
                else:
                    return False
            return True
        else:
            return field in response_data
    
    def _create_rate_limited_result(self, test_id: str, api_name: str, quality_level: str) -> RealAPIResult:
        """Create result for rate-limited scenario"""
        return RealAPIResult(
            test_id=test_id,
            api_domain=api_name,
            quality_level=quality_level,
            llm_provider="real_api_test",
            api_response_time=0.0,
            api_status_code=429,
            rate_limit_hit=True,
            authentication_method_used="",
            real_data_received=False,
            authentication_success=False,
            api_call_success=False,
            data_retrieval_success=False,
            response_parsing_success=False,
            error_handling_success=False,
            execution_successful=False,
            network_error_handling=False,
            rate_limit_handling=False,
            data_validation_success=False,
            mock_success_score=None,
            real_success_score=0.0,
            correlation_score=None,
            api_error="Rate limit exceeded",
            execution_error=None,
            api_calls_made=0,
            estimated_cost=0.0
        )
    
    def _create_error_result(self, test_id: str, api_name: str, quality_level: str,
                           error: str, mock_result: Optional[Dict]) -> RealAPIResult:
        """Create result for error scenario"""
        mock_success_score = mock_result.get("success_score") if mock_result else None
        
        return RealAPIResult(
            test_id=test_id,
            api_domain=api_name,
            quality_level=quality_level,
            llm_provider="real_api_test",
            api_response_time=0.0,
            api_status_code=0,
            rate_limit_hit=False,
            authentication_method_used="",
            real_data_received=False,
            authentication_success=False,
            api_call_success=False,
            data_retrieval_success=False,
            response_parsing_success=False,
            error_handling_success=False,
            execution_successful=False,
            network_error_handling=False,
            rate_limit_handling=False,
            data_validation_success=False,
            mock_success_score=mock_success_score,
            real_success_score=0.0,
            correlation_score=None,
            api_error=error,
            execution_error=error,
            api_calls_made=0,
            estimated_cost=0.0
        )
    
    async def _generate_comparison_analysis(self, mock_results: List[Dict]):
        """Generate analysis comparing mock vs real API results"""
        
        logger.info("\n📊 Generating Mock vs Real API Comparison...")
        
        # Calculate correlations
        mock_scores = []
        real_scores = []
        
        for result in self.results:
            if result.mock_success_score is not None:
                mock_scores.append(result.mock_success_score)
                real_scores.append(result.real_success_score)
        
        if len(mock_scores) > 1:
            correlation = np.corrcoef(mock_scores, real_scores)[0, 1]
            logger.info(f"📈 Mock vs Real Correlation: {correlation:.3f}")
        
        # Save results
        results_data = [
            {
                "test_id": r.test_id,
                "api_domain": r.api_domain,
                "quality_level": r.quality_level,
                "real_success_score": r.real_success_score,
                "mock_success_score": r.mock_success_score,
                "correlation_score": r.correlation_score,
                "api_response_time": r.api_response_time,
                "api_status_code": r.api_status_code,
                "rate_limit_hit": r.rate_limit_hit,
                "real_data_received": r.real_data_received
            }
            for r in self.results
        ]
        
        with open("experiment_results/real_api_validation_results.json", "w") as f:
            json.dump(results_data, f, indent=2)
        
        logger.info("💾 Real API validation results saved")

async def main():
    """Run real API validation study"""
    
    # Load mock results for comparison
    try:
        with open("experiment_results/multi_domain_functional_success_reprocessed.json", "r") as f:
            mock_results = json.load(f)
    except FileNotFoundError:
        logger.error("Mock results not found. Run mock experiment first.")
        return
    
    # Run real API validation
    validator = RealAPIValidator()
    real_results = await validator.run_real_api_validation(mock_results)
    
    logger.info(f"🎉 Real API validation completed with {len(real_results)} tests")

if __name__ == "__main__":
    asyncio.run(main())