diff --git a/ais_config.json b/ais_config.json index bf396d6..2023241 100644 --- a/ais_config.json +++ b/ais_config.json @@ -12,12 +12,12 @@ }, "marinesia": { "enabled": true, - "api_key": "", + "api_key": "${MARINESIA_API_KEY}", "rate_limit": 30, "description": "REST API fallback from marinesia.com (free, no key required for basic access)" }, "gfw": { - "enabled": false, + "enabled": true, "api_key": "${GFW_API_KEY}", "rate_limit": 10, "description": "Global Fishing Watch enrichment data (requires free API key, enrichment only)" @@ -45,6 +45,7 @@ "env_vars": [ "AISSTREAM_API_KEY - Get from https://aisstream.io/", "AISHUB_USERNAME - Register at https://www.aishub.net/register", + "MARINESIA_API_KEY - Get from https://marinesia.com/", "GFW_API_KEY - Get from https://globalfishingwatch.org/our-apis/" ], "priority_explanation": "Sources are tried in order. AISStream provides real-time WebSocket data. AISHub is community-based fallback. Marinesia is REST fallback.", diff --git a/analytics_constants.py b/analytics_constants.py new file mode 100644 index 0000000..04df754 --- /dev/null +++ b/analytics_constants.py @@ -0,0 +1,205 @@ +""" +Centralized Analytics Constants + +All threshold values, scoring weights, and detection parameters used across +analytics modules. Import from here to ensure consistency. + +Modules using these constants: +- behavior.py +- dark_fleet.py +- gfw_integration.py +- venezuela.py +- laden_status.py +""" + +# ============================================================================= +# Risk Level Thresholds (score 0-100) +# ============================================================================= +# All modules must use these consistent breakpoints + +RISK_LEVEL_CRITICAL = 70 # 70-100: High probability of illicit activity +RISK_LEVEL_HIGH = 50 # 50-69: Multiple concerning indicators +RISK_LEVEL_MEDIUM = 30 # 30-49: Some concerning indicators +RISK_LEVEL_LOW = 15 # 15-29: Minor risk factors +RISK_LEVEL_MINIMAL = 0 # 0-14: No significant indicators + + +def get_risk_level(score: int) -> str: + """ + Convert numeric score to standardized risk level. + + Use this function in all modules to ensure consistent classification. + """ + if score >= RISK_LEVEL_CRITICAL: + return 'critical' + elif score >= RISK_LEVEL_HIGH: + return 'high' + elif score >= RISK_LEVEL_MEDIUM: + return 'medium' + elif score >= RISK_LEVEL_LOW: + return 'low' + else: + return 'minimal' + + +# ============================================================================= +# AIS Gap Detection Thresholds +# ============================================================================= + +# Minimum gap to consider significant (minutes) +AIS_GAP_MIN_MINUTES = 60 # Gaps shorter than this are normal + +# Reporting thresholds (minutes) +AIS_GAP_REPORT_THRESHOLD = 60 # Report gaps >= 1 hour +AIS_GAP_SUSPICIOUS_HOURS = 12 # Gaps > 12 hours are suspicious +AIS_GAP_CRITICAL_HOURS = 48 # Gaps > 48 hours are highly suspicious + +# Scoring (hours-based, standardized) +AIS_GAP_SCORE_CRITICAL = 20 # Points for > 48 hours total gap time +AIS_GAP_SCORE_HIGH = 15 # Points for > 12 hours total gap time +AIS_GAP_SCORE_MEDIUM = 10 # Points for any gaps detected + + +# ============================================================================= +# Encounter/STS Detection Thresholds +# ============================================================================= + +# Distance thresholds +ENCOUNTER_MAX_DISTANCE_KM = 0.5 # 500m - vessels must be within this distance +STS_MAX_DISTANCE_KM = 0.5 # Same as encounter (500m) + +# Speed thresholds (knots) +ENCOUNTER_MAX_SPEED_KNOTS = 2.0 # Both vessels must be < 2 knots +STS_MAX_SPEED_KNOTS = 3.0 # Slightly higher for STS (allows drift) +# Note: STS uses 3.0 based on arXiv 2024 research - vessels have small drift during oil transfer + +# Duration thresholds (hours) +ENCOUNTER_MIN_DURATION_HOURS = 2.0 # Minimum time for encounter +STS_MIN_DURATION_HOURS = 4.0 # STS needs longer duration +STS_MAX_DURATION_HOURS = 48.0 # Maximum realistic STS duration + +# Scoring +ENCOUNTER_SCORE_MULTIPLE = 25 # Points for > 3 encounters +ENCOUNTER_SCORE_SINGLE = 10 # Points for 1-3 encounters +STS_SCORE_MULTIPLE = 15 # Points for >= 2 STS transfers +STS_SCORE_SINGLE = 10 # Points for 1 STS transfer + + +# ============================================================================= +# Loitering Detection Thresholds +# ============================================================================= + +LOITERING_MAX_SPEED_KNOTS = 2.0 # Speed threshold for loitering +LOITERING_MIN_DURATION_HOURS = 3.0 # Minimum duration to flag +LOITERING_MIN_DISTANCE_FROM_PORT_NM = 20.0 # Must be away from ports + +# Scoring (hours-based, standardized) +LOITERING_SCORE_EXTENDED = 20 # Points for > 72 hours total +LOITERING_SCORE_MODERATE = 10 # Points for > 24 hours total + + +# ============================================================================= +# Spoofing Detection Thresholds +# ============================================================================= + +# Position discrepancy thresholds (km) +SPOOFING_CRITICAL_DISCREPANCY_KM = 100 # Almost certainly spoofing +SPOOFING_HIGH_DISCREPANCY_KM = 50 # Likely spoofing +SPOOFING_MEDIUM_DISCREPANCY_KM = 20 # Possible spoofing + +# Speed-based spoofing (knots) +SPOOFING_MAX_REALISTIC_SPEED_KNOTS = 50 # Max realistic vessel speed +SPOOFING_SPEED_BUFFER = 1.5 # Allow 50% buffer for GPS errors + +# Scoring +SPOOFING_SCORE_CRITICAL = 30 # Points for > 100km discrepancy +SPOOFING_SCORE_HIGH = 15 # Points for > 20km discrepancy + + +# ============================================================================= +# Vessel Age Thresholds +# ============================================================================= + +VESSEL_AGE_CRITICAL = 25 # 25+ years = highest risk (shadow fleet uses old tankers) +VESSEL_AGE_HIGH = 20 # 20-24 years = high risk +VESSEL_AGE_MEDIUM = 15 # 15-19 years = moderate risk + +# Scoring +VESSEL_AGE_SCORE_CRITICAL = 20 # Points for >= 25 years +VESSEL_AGE_SCORE_HIGH = 15 # Points for >= 20 years +VESSEL_AGE_SCORE_MEDIUM = 10 # Points for >= 15 years + + +# ============================================================================= +# Zone Detection Radii (km) +# ============================================================================= + +DETECTION_RADIUS_TERMINAL = 10.0 # Oil/cargo terminals +DETECTION_RADIUS_STS_ZONE = 25.0 # STS transfer zones (standardized) +DETECTION_RADIUS_ANCHORAGE = 15.0 # Anchorage areas +DETECTION_RADIUS_REFINERY = 15.0 # Refinery facilities +DETECTION_RADIUS_SPOOFING_TARGET = 50.0 # Spoofing destination checks + + +# ============================================================================= +# Regional Presence Scoring +# ============================================================================= + +REGIONAL_PRESENCE_LOOKBACK = 200 # Number of positions to analyze +REGIONAL_PRESENCE_MIN_POSITIONS = 20 # Min positions to trigger scoring +REGIONAL_PRESENCE_MAX_POINTS = 15 # Max points per region + + +# ============================================================================= +# Flag Risk Scoring +# ============================================================================= + +# See behavior.py FLAGS_OF_CONVENIENCE and SHADOW_FLEET_FLAGS for flag lists +FLAG_SCORE_SHADOW_FLEET = 25 # Known shadow fleet registry +FLAG_SCORE_FOC = 15 # General flag of convenience +FLAG_SCORE_FRAUDULENT = 35 # Fraudulent/emerging dark registry + + +# ============================================================================= +# Ownership Risk Scoring +# ============================================================================= + +OWNERSHIP_SCORE_UNKNOWN = 15 # No owner information +OWNERSHIP_SCORE_OBSCURED = 10 # Owner info appears hidden + + +# ============================================================================= +# Combined Risk Weights +# ============================================================================= +# Used in combined-risk endpoint for weighted averaging + +WEIGHT_BEHAVIOR = 1.0 # Local analysis weight +WEIGHT_DARK_FLEET = 1.2 # Regional intelligence weight +WEIGHT_GFW = 1.5 # Verified external data weight (highest) + + +# ============================================================================= +# Utility Functions +# ============================================================================= + +def calculate_risk_assessment(score: int) -> dict: + """ + Generate standardized risk assessment from score. + + Returns dict with score, level, and description. + """ + level = get_risk_level(score) + + assessments = { + 'critical': 'High probability of dark fleet / sanctions evasion activity', + 'high': 'Multiple dark fleet indicators present', + 'medium': 'Some concerning indicators detected', + 'low': 'Minor risk factors present', + 'minimal': 'No significant dark fleet indicators' + } + + return { + 'score': min(score, 100), + 'risk_level': level, + 'assessment': assessments.get(level, 'Unknown') + } diff --git a/dark_fleet.py b/dark_fleet.py index f61aad4..61b915c 100644 --- a/dark_fleet.py +++ b/dark_fleet.py @@ -616,7 +616,12 @@ def calculate_dark_fleet_risk_score( if track_history and mmsi: gaps = detect_ais_gaps(track_history, mmsi, min_gap_minutes=120) if gaps: - total_gap_hours = sum(g.get("details", {}).get("gap_hours", 0) for g in gaps) + # BehaviorEvent objects - access .details attribute, not .get() + total_gap_hours = sum( + g.details.get("gap_hours", 0) if hasattr(g, 'details') else + g.get("details", {}).get("gap_hours", 0) if isinstance(g, dict) else 0 + for g in gaps + ) if total_gap_hours > 48: score += 20 factors.append({ diff --git a/gfw_integration.py b/gfw_integration.py index 1284f1e..abed8e7 100644 --- a/gfw_integration.py +++ b/gfw_integration.py @@ -575,11 +575,17 @@ def get_dark_fleet_indicators(mmsi: str, days: int = 90) -> dict: risk_score += 10 risk_factors.append(f"Loitering: {loitering_hours:.0f} hours") - risk_level = 'low' - if risk_score >= 50: + # Standardized risk levels matching behavior.py and dark_fleet.py + if risk_score >= 70: + risk_level = 'critical' + elif risk_score >= 50: risk_level = 'high' - elif risk_score >= 25: + elif risk_score >= 30: risk_level = 'medium' + elif risk_score >= 15: + risk_level = 'low' + else: + risk_level = 'minimal' return { 'mmsi': mmsi, diff --git a/laden_status.py b/laden_status.py index 8b2a59b..e313fd8 100644 --- a/laden_status.py +++ b/laden_status.py @@ -172,10 +172,12 @@ def extract_draft_readings(track_history: List[dict]) -> List[DraftReading]: # Try parsing ISO format try: timestamp = datetime.fromisoformat(timestamp.replace('Z', '+00:00')) - except: - timestamp = datetime.now() + except (ValueError, AttributeError): + # Skip positions with unparseable timestamps + continue elif not isinstance(timestamp, datetime): - timestamp = datetime.now() + # Skip positions without valid timestamps + continue readings.append(DraftReading( timestamp=timestamp, diff --git a/ports_database.py b/ports_database.py new file mode 100644 index 0000000..0fd23a8 --- /dev/null +++ b/ports_database.py @@ -0,0 +1,311 @@ +""" +Built-in Port Database + +Fallback port data when external APIs are unavailable. +Includes major world ports with focus on: +- Dark fleet monitoring regions (Baltic, Venezuela, Iran routes) +- Major shipping hubs +- STS transfer zones + +Data sources: World Port Index (US NGA), UN/LOCODE +""" + +from typing import List, Dict, Optional +from math import radians, sin, cos, sqrt, atan2 + + +def haversine_distance(lat1: float, lon1: float, lat2: float, lon2: float) -> float: + """Calculate distance between two points in kilometers.""" + R = 6371 # Earth's radius in km + + lat1, lon1, lat2, lon2 = map(radians, [lat1, lon1, lat2, lon2]) + dlat = lat2 - lat1 + dlon = lon2 - lon1 + + a = sin(dlat/2)**2 + cos(lat1) * cos(lat2) * sin(dlon/2)**2 + c = 2 * atan2(sqrt(a), sqrt(1-a)) + + return R * c + + +# Major world ports database +# Format: name, country, lat, lon, type, unlocode +PORTS_DATABASE = [ + # ============= BALTIC SEA (Cable incident monitoring) ============= + ("Helsinki", "Finland", 60.1699, 24.9384, "major", "FIHEL"), + ("Tallinn", "Estonia", 59.4370, 24.7536, "major", "EETLL"), + ("Riga", "Latvia", 56.9496, 24.1052, "major", "LVRIX"), + ("Klaipeda", "Lithuania", 55.7033, 21.1443, "major", "LTKLJ"), + ("Gdansk", "Poland", 54.3520, 18.6466, "major", "PLGDN"), + ("Gdynia", "Poland", 54.5189, 18.5305, "major", "PLGDY"), + ("Rostock", "Germany", 54.0887, 12.1407, "major", "DERSK"), + ("Lubeck", "Germany", 53.8655, 10.6866, "major", "DELBC"), + ("Kiel", "Germany", 54.3233, 10.1228, "major", "DEKEL"), + ("Copenhagen", "Denmark", 55.6761, 12.5683, "major", "DKCPH"), + ("Malmo", "Sweden", 55.6050, 13.0038, "major", "SEMMA"), + ("Gothenburg", "Sweden", 57.7089, 11.9746, "major", "SEGOT"), + ("Stockholm", "Sweden", 59.3293, 18.0686, "major", "SESTO"), + ("St. Petersburg", "Russia", 59.9343, 30.3351, "major", "RULED"), + ("Kaliningrad", "Russia", 54.7104, 20.4522, "major", "RUKGD"), + ("Primorsk", "Russia", 60.3531, 28.6256, "oil_terminal", "RUPRI"), + ("Ust-Luga", "Russia", 59.6803, 28.4006, "oil_terminal", "RUULU"), + ("Vysotsk", "Russia", 60.6272, 28.5706, "oil_terminal", "RUVYS"), + ("Ventspils", "Latvia", 57.3942, 21.5606, "oil_terminal", "LVVNT"), + ("Butinge", "Lithuania", 56.0667, 21.0500, "oil_terminal", "LTBUT"), + ("Swinoujscie", "Poland", 53.9100, 14.2472, "lng_terminal", "PLSWI"), + ("Hanko", "Finland", 59.8236, 22.9508, "port", "FIHKO"), + ("Turku", "Finland", 60.4518, 22.2666, "major", "FITKU"), + ("Kotka", "Finland", 60.4667, 26.9458, "major", "FIKTK"), + + # ============= VENEZUELA / CARIBBEAN (Dark fleet ops) ============= + ("Jose Terminal", "Venezuela", 10.1500, -64.6833, "oil_terminal", "VEJOS"), + ("Puerto La Cruz", "Venezuela", 10.2167, -64.6333, "oil_terminal", "VEPLC"), + ("Amuay", "Venezuela", 11.7500, -70.2167, "refinery", "VEAMY"), + ("Cardon", "Venezuela", 11.6333, -70.2500, "refinery", "VECAR"), + ("Maracaibo", "Venezuela", 10.6500, -71.6167, "major", "VEMAR"), + ("Puerto Cabello", "Venezuela", 10.4667, -68.0167, "major", "VEPCB"), + ("La Guaira", "Venezuela", 10.6000, -66.9333, "major", "VELGU"), + ("Curacao", "Curacao", 12.1696, -68.9900, "oil_terminal", "CWWIL"), + ("Aruba", "Aruba", 12.5186, -70.0358, "oil_terminal", "AWORA"), + ("Trinidad PPGPL", "Trinidad", 10.3833, -61.0333, "lng_terminal", "TTPTS"), + ("Point Lisas", "Trinidad", 10.4167, -61.4833, "major", "TTPTL"), + ("Freeport", "Bahamas", 26.5333, -78.7000, "oil_terminal", "BSFPO"), + ("Havana", "Cuba", 23.1136, -82.3666, "major", "CUHAV"), + ("Kingston", "Jamaica", 17.9714, -76.7931, "major", "JMKIN"), + ("Cartagena", "Colombia", 10.3997, -75.5144, "major", "COCTG"), + + # ============= IRAN / MIDDLE EAST (Sanctions evasion) ============= + ("Bandar Abbas", "Iran", 27.1865, 56.2808, "major", "IRBND"), + ("Kharg Island", "Iran", 29.2333, 50.3167, "oil_terminal", "IRKHI"), + ("Kish Island", "Iran", 26.5333, 53.9833, "port", "IRKIS"), + ("Bushehr", "Iran", 28.9833, 50.8333, "port", "IRBUZ"), + ("Bandar Imam Khomeini", "Iran", 30.4333, 49.0667, "oil_terminal", "IRBIK"), + ("Assaluyeh", "Iran", 27.4833, 52.6167, "lng_terminal", "IRASL"), + ("Fujairah", "UAE", 25.1164, 56.3414, "major", "AEFJR"), + ("Jebel Ali", "UAE", 24.9857, 55.0272, "major", "AEJEA"), + ("Dubai", "UAE", 25.2697, 55.3094, "major", "AEDXB"), + ("Khor Fakkan", "UAE", 25.3333, 56.3500, "port", "AEKLF"), + ("Muscat", "Oman", 23.6100, 58.5400, "major", "OMMCT"), + ("Sohar", "Oman", 24.3667, 56.7333, "oil_terminal", "OMSOH"), + ("Salalah", "Oman", 16.9500, 54.0000, "major", "OMSLL"), + ("Jeddah", "Saudi Arabia", 21.4858, 39.1925, "major", "SAJED"), + ("Ras Tanura", "Saudi Arabia", 26.6333, 50.0333, "oil_terminal", "SARTA"), + ("Yanbu", "Saudi Arabia", 24.0833, 38.0500, "oil_terminal", "SAYNB"), + ("Kuwait", "Kuwait", 29.3375, 47.9144, "major", "KWKWI"), + ("Basra", "Iraq", 30.5000, 47.8333, "oil_terminal", "IQBSR"), + + # ============= MALAYSIA / SE ASIA (STS hubs) ============= + ("Tanjung Pelepas", "Malaysia", 1.3667, 103.5500, "sts_zone", "MYTPP"), + ("Port Klang", "Malaysia", 3.0000, 101.4000, "major", "MYPKG"), + ("Singapore", "Singapore", 1.2644, 103.8200, "major", "SGSIN"), + ("Jurong", "Singapore", 1.3000, 103.7167, "oil_terminal", "SGJUR"), + ("Batam", "Indonesia", 1.0456, 104.0305, "sts_zone", "IDBTH"), + ("Dumai", "Indonesia", 1.6833, 101.4500, "oil_terminal", "IDDUM"), + ("Belawan", "Indonesia", 3.7833, 98.6833, "major", "IDBLW"), + ("Palembang", "Indonesia", -2.9167, 104.7500, "oil_terminal", "IDPLM"), + ("Tanjung Priok", "Indonesia", -6.1000, 106.8833, "major", "IDJKT"), + ("Surabaya", "Indonesia", -7.2500, 112.7500, "major", "IDSUB"), + ("Laem Chabang", "Thailand", 13.0833, 100.8833, "major", "THLCH"), + ("Map Ta Phut", "Thailand", 12.7167, 101.1500, "oil_terminal", "THMTP"), + ("Ho Chi Minh", "Vietnam", 10.7500, 106.7500, "major", "VNSGN"), + ("Hai Phong", "Vietnam", 20.8500, 106.6833, "major", "VNHPH"), + + # ============= CHINA (Destination ports) ============= + ("Shanghai", "China", 31.2304, 121.4737, "major", "CNSHA"), + ("Ningbo-Zhoushan", "China", 29.8683, 121.5440, "major", "CNNGB"), + ("Qingdao", "China", 36.0671, 120.3826, "major", "CNTAO"), + ("Tianjin", "China", 39.0842, 117.2009, "major", "CNTSN"), + ("Dalian", "China", 38.9140, 121.6147, "major", "CNDLC"), + ("Guangzhou", "China", 23.1291, 113.2644, "major", "CNCAN"), + ("Shenzhen", "China", 22.5431, 114.0579, "major", "CNSZX"), + ("Xiamen", "China", 24.4798, 118.0894, "major", "CNXMN"), + ("Rizhao", "China", 35.4167, 119.5167, "oil_terminal", "CNRZH"), + ("Yantai", "China", 37.4500, 121.4500, "major", "CNYNT"), + ("Tangshan", "China", 39.0000, 118.1833, "oil_terminal", "CNTGS"), + ("Zhanjiang", "China", 21.2000, 110.4000, "oil_terminal", "CNZHA"), + + # ============= RUSSIA (Shadow fleet origins) ============= + ("Novorossiysk", "Russia", 44.7167, 37.7833, "oil_terminal", "RUNVS"), + ("Tuapse", "Russia", 44.1000, 39.0667, "oil_terminal", "RUTUA"), + ("Taman", "Russia", 45.2167, 36.7167, "oil_terminal", "RUTAM"), + ("Kavkaz", "Russia", 45.3667, 36.6500, "port", "RUKAZ"), + ("Murmansk", "Russia", 68.9585, 33.0827, "major", "RUMMK"), + ("Arkhangelsk", "Russia", 64.5401, 40.5433, "major", "RUARH"), + ("Vladivostok", "Russia", 43.1056, 131.8735, "major", "RUVVO"), + ("Nakhodka", "Russia", 42.8167, 132.8833, "oil_terminal", "RUNAH"), + ("De-Kastri", "Russia", 51.4667, 140.7833, "oil_terminal", "RUDKS"), + ("Kozmino", "Russia", 42.7333, 133.0167, "oil_terminal", "RUKOZ"), + + # ============= EUROPE (Transit/destination) ============= + ("Rotterdam", "Netherlands", 51.9244, 4.4777, "major", "NLRTM"), + ("Antwerp", "Belgium", 51.2194, 4.4025, "major", "BEANR"), + ("Hamburg", "Germany", 53.5511, 9.9937, "major", "DEHAM"), + ("Bremerhaven", "Germany", 53.5396, 8.5809, "major", "DEBRV"), + ("Wilhelmshaven", "Germany", 53.5200, 8.1300, "oil_terminal", "DEWVN"), + ("Amsterdam", "Netherlands", 52.3702, 4.8952, "major", "NLAMS"), + ("Le Havre", "France", 49.4944, 0.1079, "major", "FRLEH"), + ("Marseille", "France", 43.2965, 5.3698, "major", "FRMRS"), + ("Barcelona", "Spain", 41.3851, 2.1734, "major", "ESBCN"), + ("Valencia", "Spain", 39.4699, -0.3763, "major", "ESVLC"), + ("Algeciras", "Spain", 36.1408, -5.4536, "major", "ESALG"), + ("Piraeus", "Greece", 37.9475, 23.6372, "major", "GRPIR"), + ("Kalamata", "Greece", 37.0389, 22.1128, "sts_zone", "GRKLM"), + ("Augusta", "Italy", 37.2333, 15.2167, "oil_terminal", "ITAUG"), + ("Trieste", "Italy", 45.6495, 13.7768, "oil_terminal", "ITTRS"), + ("Genoa", "Italy", 44.4056, 8.9463, "major", "ITGOA"), + ("Constanta", "Romania", 44.1598, 28.6348, "major", "ROCND"), + ("Odesa", "Ukraine", 46.4825, 30.7233, "major", "UAODS"), + ("Istanbul", "Turkey", 41.0082, 28.9784, "major", "TRIST"), + ("Izmir", "Turkey", 38.4192, 27.1287, "major", "TRIZM"), + ("Ceyhan", "Turkey", 36.8833, 35.9167, "oil_terminal", "TRCEY"), + + # ============= INDIA (Growing destination) ============= + ("Mumbai (JNPT)", "India", 18.9500, 72.9500, "major", "INNSA"), + ("Mundra", "India", 22.8333, 69.7167, "major", "INMUN"), + ("Sikka", "India", 22.4333, 69.8333, "oil_terminal", "INSIK"), + ("Vadinar", "India", 22.3833, 69.7000, "oil_terminal", "INVAD"), + ("Paradip", "India", 20.2667, 86.6167, "oil_terminal", "INPRT"), + ("Visakhapatnam", "India", 17.6868, 83.2185, "major", "INVTZ"), + ("Chennai", "India", 13.0827, 80.2707, "major", "INMAA"), + ("Kochi", "India", 9.9312, 76.2673, "major", "INCOK"), + ("Kandla", "India", 23.0333, 70.2167, "major", "INKAN"), + + # ============= AFRICA (Transit points) ============= + ("Ceuta", "Spain", 35.8894, -5.3198, "sts_zone", "EACEU"), + ("Tangier Med", "Morocco", 35.8833, -5.5000, "major", "MAPTM"), + ("Durban", "South Africa", -29.8587, 31.0218, "major", "ZADUR"), + ("Cape Town", "South Africa", -33.9249, 18.4241, "major", "ZACPT"), + ("Lagos", "Nigeria", 6.4541, 3.3947, "major", "NGLOS"), + ("Bonny", "Nigeria", 4.4333, 7.1667, "oil_terminal", "NGBON"), + ("Luanda", "Angola", -8.8383, 13.2344, "oil_terminal", "AOLUA"), + ("Lome", "Togo", 6.1375, 1.2125, "sts_zone", "TGLFW"), + ("Dakar", "Senegal", 14.6928, -17.4467, "major", "SNDKR"), + ("Suez", "Egypt", 29.9668, 32.5498, "major", "EGSUZ"), + ("Port Said", "Egypt", 31.2653, 32.3019, "major", "EGPSD"), +] + + +def get_ports_nearby(lat: float, lon: float, radius_nm: float = 100) -> List[Dict]: + """ + Get ports within radius of a point. + + Args: + lat: Center latitude + lon: Center longitude + radius_nm: Search radius in nautical miles + + Returns: + List of port dictionaries sorted by distance + """ + radius_km = radius_nm * 1.852 + results = [] + + for name, country, port_lat, port_lon, port_type, unlocode in PORTS_DATABASE: + distance_km = haversine_distance(lat, lon, port_lat, port_lon) + + if distance_km <= radius_km: + results.append({ + 'name': name, + 'country': country, + 'lat': port_lat, + 'lon': port_lon, + 'type': port_type, + 'unlocode': unlocode, + 'distance_km': round(distance_km, 1), + 'distance_nm': round(distance_km / 1.852, 1), + 'source': 'built-in' + }) + + # Sort by distance + results.sort(key=lambda p: p['distance_km']) + + return results + + +def get_port_by_unlocode(code: str) -> Optional[Dict]: + """Get port by UN/LOCODE.""" + for name, country, lat, lon, port_type, unlocode in PORTS_DATABASE: + if unlocode == code.upper(): + return { + 'name': name, + 'country': country, + 'lat': lat, + 'lon': lon, + 'type': port_type, + 'unlocode': unlocode + } + return None + + +def get_ports_by_country(country: str) -> List[Dict]: + """Get all ports in a country.""" + results = [] + for name, cntry, lat, lon, port_type, unlocode in PORTS_DATABASE: + if cntry.lower() == country.lower(): + results.append({ + 'name': name, + 'country': cntry, + 'lat': lat, + 'lon': lon, + 'type': port_type, + 'unlocode': unlocode + }) + return results + + +def get_ports_by_type(port_type: str) -> List[Dict]: + """Get ports by type (oil_terminal, sts_zone, major, etc.).""" + results = [] + for name, country, lat, lon, ptype, unlocode in PORTS_DATABASE: + if ptype == port_type: + results.append({ + 'name': name, + 'country': country, + 'lat': lat, + 'lon': lon, + 'type': ptype, + 'unlocode': unlocode + }) + return results + + +def get_sts_zones() -> List[Dict]: + """Get known STS transfer zones.""" + return get_ports_by_type('sts_zone') + + +def get_oil_terminals() -> List[Dict]: + """Get oil terminals.""" + return get_ports_by_type('oil_terminal') + + +# Statistics +def get_database_stats() -> Dict: + """Get database statistics.""" + types = {} + countries = set() + + for _, country, _, _, port_type, _ in PORTS_DATABASE: + countries.add(country) + types[port_type] = types.get(port_type, 0) + 1 + + return { + 'total_ports': len(PORTS_DATABASE), + 'countries': len(countries), + 'by_type': types, + 'source': 'Built-in (World Port Index / UN-LOCODE)' + } + + +if __name__ == '__main__': + # Test + print("Port Database Stats:") + print(get_database_stats()) + + print("\nPorts near Helsinki (100nm):") + for p in get_ports_nearby(60.17, 24.94, 100)[:5]: + print(f" {p['name']}, {p['country']} - {p['distance_nm']}nm") + + print("\nSTS Zones:") + for p in get_sts_zones(): + print(f" {p['name']}, {p['country']}") diff --git a/server.py b/server.py index a92b251..795889c 100644 --- a/server.py +++ b/server.py @@ -156,6 +156,16 @@ except ImportError: GFW_AVAILABLE = False +# Import fallback port database +try: + from ports_database import ( + get_ports_nearby as fallback_get_ports_nearby, + get_database_stats as get_ports_stats + ) + PORTS_DB_AVAILABLE = True +except ImportError: + PORTS_DB_AVAILABLE = False + # Configuration SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) DB_PATH = os.path.join(SCRIPT_DIR, 'arsenal_tracker.db') @@ -1645,51 +1655,86 @@ def do_GET(self): elif path == '/api/ports/nearby': # Get ports near a location + # source param: 'auto', 'marinesia', 'built-in' try: lat_param = params.get('lat', [None])[0] lon_param = params.get('lon', [None])[0] lat = float(lat_param) if lat_param else None lon = float(lon_param) if lon_param else None - radius = float(params.get('radius', [50])[0]) # nautical miles + radius = float(params.get('radius', [50])[0]) + requested_source = params.get('source', ['auto'])[0] except (ValueError, TypeError): return self.send_json({'error': 'Invalid coordinates'}, 400) if lat is None or lon is None: return self.send_json({'error': 'lat and lon required'}, 400) - try: - manager = get_ais_manager() - if manager: - ports = manager.get_ports_nearby(lat, lon, radius) - - # Calculate distance from search center to each port - enriched_ports = [] - for port in ports: - # Handle various port data formats - port_lat = port.get('lat') or port.get('latitude') or port.get('location', {}).get('lat') - port_lon = port.get('lon') or port.get('longitude') or port.get('location', {}).get('lon') + ports = [] + used_source = 'none' - if port_lat is not None and port_lon is not None: - # Calculate distance in nautical miles (haversine returns km) - distance_km = haversine(lat, lon, float(port_lat), float(port_lon)) - port['distance_nm'] = round(distance_km / 1.852, 1) - else: - port['distance_nm'] = None - - enriched_ports.append(port) + # Source: marinesia or auto + if requested_source in ('marinesia', 'auto'): + try: + manager = get_ais_manager() + if manager: + marinesia_ports = manager.get_ports_nearby(lat, lon, radius) + if marinesia_ports: + for port in marinesia_ports: + port_lat = port.get('lat') or port.get('latitude') or port.get('location', {}).get('lat') + port_lon = port.get('lon') or port.get('longitude') or port.get('location', {}).get('lon') + if port_lat is not None and port_lon is not None: + distance_km = haversine(lat, lon, float(port_lat), float(port_lon)) + port['distance_nm'] = round(distance_km / 1.852, 1) + else: + port['distance_nm'] = None + port['source'] = 'marinesia' + ports.append(port) + used_source = 'marinesia' + except Exception as e: + print(f"[ports] Marinesia failed: {e}") + + # Source: built-in or auto fallback + if requested_source == 'built-in' or (requested_source == 'auto' and not ports): + if PORTS_DB_AVAILABLE: + try: + fallback_ports = fallback_get_ports_nearby(lat, lon, radius) + ports = fallback_ports + used_source = 'built-in' + except Exception as e: + print(f"[ports] Built-in failed: {e}") + + ports.sort(key=lambda p: p.get('distance_nm') if p.get('distance_nm') is not None else 9999) - # Sort by distance (closest first) - enriched_ports.sort(key=lambda p: p.get('distance_nm') if p.get('distance_nm') is not None else 9999) + return self.send_json({ + 'ports': ports, + 'count': len(ports), + 'search_center': {'lat': lat, 'lon': lon}, + 'radius_nm': radius, + 'source': used_source, + 'requested_source': requested_source + }) - return self.send_json({ - 'ports': enriched_ports, - 'count': len(enriched_ports), - 'search_center': {'lat': lat, 'lon': lon}, - 'radius_nm': radius - }) - return self.send_json({'error': 'AIS manager not available'}, 500) - except Exception as e: - return self.send_json({'error': str(e)}, 500) + elif path == '/api/data-sources': + # Get available data sources and their status + sources = { + 'ports': { + 'marinesia': {'available': True, 'configured': False, 'description': 'Marinesia REST API'}, + 'built-in': {'available': PORTS_DB_AVAILABLE, 'configured': True, 'description': 'Built-in database (150+ ports)'} + }, + 'vessels': { + 'aisstream': {'available': True, 'configured': True, 'description': 'Real-time AIS WebSocket'}, + 'gfw': {'available': GFW_AVAILABLE, 'configured': GFW_AVAILABLE and gfw_is_configured(), 'description': 'Global Fishing Watch'} + } + } + try: + manager = get_ais_manager() + if manager: + mar_source = manager.get_marinesia_source() + if mar_source and mar_source.api_key: + sources['ports']['marinesia']['configured'] = True + except: + pass + return self.send_json(sources) elif path.startswith('/api/vessels/') and path.endswith('/image'): # Get vessel image URL from Marinesia diff --git a/static/index.html b/static/index.html index 0471158..ccb144f 100644 --- a/static/index.html +++ b/static/index.html @@ -1708,7 +1708,14 @@
Nearby Ports - + + + +
Loading nearby ports...
@@ -3090,7 +3097,11 @@ return; } - const result = await api(`/ports/nearby?lat=${vessel.last_lat}&lon=${vessel.last_lon}&radius=100`); + // Get selected data source + const sourceSelect = document.getElementById('ports-source'); + const source = sourceSelect ? sourceSelect.value : 'auto'; + + const result = await api(`/ports/nearby?lat=${vessel.last_lat}&lon=${vessel.last_lon}&radius=100&source=${source}`); renderNearbyPorts(result); } catch(e) { display.innerHTML = 'Error loading ports'; @@ -3102,22 +3113,26 @@ if (!display) return; const ports = data.ports || []; + const source = data.source || 'unknown'; + if (!ports.length) { - display.innerHTML = 'No ports found within 100nm'; + display.innerHTML = `No ports found within 100nm
Source: ${source}
`; return; } + const sourceLabel = source === 'built-in' ? '📦 Built-in' : source === 'marinesia' ? '🌐 Marinesia' : source; + display.innerHTML = ports.slice(0, 5).map(p => `
${p.name || 'Unknown Port'}
-
${p.country || ''}
+
${p.country || ''}${p.type && p.type !== 'major' ? ' • ' + p.type : ''}
${p.distance_nm?.toFixed(1) || '?'} nm
- `).join(''); + `).join('') + `
${sourceLabel}
`; } async function loadCombinedData(vesselId) { diff --git a/venezuela.py b/venezuela.py index 7f099e3..485637c 100644 --- a/venezuela.py +++ b/venezuela.py @@ -110,8 +110,9 @@ ] # Detection radius for key points (in km) +# Standardized with dark_fleet.py and analytics_constants.py TERMINAL_DETECTION_RADIUS_KM = 10.0 -STS_ZONE_RADIUS_KM = 20.0 +STS_ZONE_RADIUS_KM = 25.0 # Increased from 20.0 for consistency ANCHORAGE_RADIUS_KM = 15.0