From 8423ed3fe0c4d81c3f0542fc5138fc81f3d5dd19 Mon Sep 17 00:00:00 2001 From: David <17435126+DavidLMS@users.noreply.github.com> Date: Thu, 19 Jun 2025 12:36:40 +0200 Subject: [PATCH 1/5] add camera management --- app/camera_detection.py | 404 ++++++++++++++++++++++++++++++++++++++++ app/config.py | 92 +++++++++ app/main.py | 281 ++++++++++++++++++++++++++++ 3 files changed, 777 insertions(+) create mode 100644 app/camera_detection.py diff --git a/app/camera_detection.py b/app/camera_detection.py new file mode 100644 index 0000000..8b36f95 --- /dev/null +++ b/app/camera_detection.py @@ -0,0 +1,404 @@ +import logging +from typing import List, Dict, Any +import traceback + +logger = logging.getLogger(__name__) + + +def find_opencv_cameras_fallback() -> List[Dict[str, Any]]: + """ + Find available OpenCV cameras using direct OpenCV detection. + + Returns: + List of camera information dictionaries + """ + try: + import cv2 + + logger.info("Searching for OpenCV cameras using direct method...") + cameras = [] + + # Test only first 5 indices to avoid spam + for index in range(5): + cap = cv2.VideoCapture(index) + + if cap.isOpened(): + # Get camera properties + width = cap.get(cv2.CAP_PROP_FRAME_WIDTH) + height = cap.get(cv2.CAP_PROP_FRAME_HEIGHT) + fps = cap.get(cv2.CAP_PROP_FPS) + backend = cap.get(cv2.CAP_PROP_BACKEND) + + # Give camera time to initialize (especially for built-in cameras) + import time + time.sleep(0.3) + + # Try multiple captures for built-in cameras that may need warmup + ret, frame = None, None + for attempt in range(3): + ret, frame = cap.read() + if ret and frame is not None and frame.sum() > 0: # Check if not all black + break + time.sleep(0.2) + + if ret and frame is not None: + # Capture a preview image immediately + import base64 + _, buffer = cv2.imencode('.jpg', frame, [cv2.IMWRITE_JPEG_QUALITY, 70]) + preview_image = base64.b64encode(buffer).decode('utf-8') + + camera_info = { + "name": f"Camera {index}", + "type": "OpenCV", + "id": index, + "backend_api": get_backend_name_fallback(backend), + "default_stream_profile": { + "width": int(width) if width > 0 else 640, + "height": int(height) if height > 0 else 480, + "fps": float(fps) if fps > 0 else 30.0, + "format": "RGB" + }, + "preview_image": preview_image # Add preview image + } + cameras.append(camera_info) + logger.info(f"Found working camera at index {index} with preview") + + cap.release() + + logger.info(f"Found {len(cameras)} OpenCV cameras using direct method") + return cameras + + except ImportError: + logger.error("OpenCV not available for camera detection") + return [] + except Exception as e: + logger.error(f"Error in direct camera detection: {e}") + return [] + + +def get_backend_name_fallback(backend_id: float) -> str: + """Convert OpenCV backend ID to readable name""" + backend_names = { + 0: "ANY", + 200: "V4L2", + 700: "AVFOUNDATION", + 1400: "DSHOW", + 1500: "GSTREAMER", + 1600: "FFMPEG" + } + return backend_names.get(int(backend_id), f"UNKNOWN_{int(backend_id)}") + + +def find_opencv_cameras() -> List[Dict[str, Any]]: + """ + Find available OpenCV cameras using direct detection (avoiding LeRobot spam). + + Returns: + List of camera information dictionaries + """ + # Force use of direct detection to avoid the 60-index spam from LeRobot + logger.info("Using direct OpenCV detection to avoid LeRobot spam...") + return find_opencv_cameras_fallback() + + +def find_realsense_cameras() -> List[Dict[str, Any]]: + """ + Find available RealSense cameras using LeRobot's camera detection. + + Returns: + List of camera information dictionaries + """ + try: + # Check if pyrealsense2 is properly available first + try: + import pyrealsense2 as rs + except ImportError: + logger.info("pyrealsense2 not installed - skipping RealSense camera detection") + return [] + except Exception as e: + logger.info(f"pyrealsense2 not available: {e} - skipping RealSense camera detection") + return [] + + # Try to import LeRobot RealSense support + try: + from lerobot.common.cameras.realsense.camera_realsense import RealSenseCamera + except ImportError: + logger.info("LeRobot RealSense support not available - skipping RealSense camera detection") + return [] + + logger.info("Searching for RealSense cameras using LeRobot...") + realsense_cameras = RealSenseCamera.find_cameras() + + logger.info(f"Found {len(realsense_cameras)} RealSense cameras") + return realsense_cameras + + except Exception as e: + logger.info(f"RealSense camera detection skipped: {e}") + return [] + + +def find_all_cameras(camera_type_filter: str = None) -> List[Dict[str, Any]]: + """ + Find all available cameras, optionally filtered by type. + + Args: + camera_type_filter: Optional filter ("opencv", "realsense", or None for all) + + Returns: + List of all available cameras matching the filter + """ + all_cameras = [] + + if camera_type_filter is None or camera_type_filter.lower() == "opencv": + all_cameras.extend(find_opencv_cameras()) + + if camera_type_filter is None or camera_type_filter.lower() == "realsense": + all_cameras.extend(find_realsense_cameras()) + + return all_cameras + + +def test_camera_configuration(camera_info: Dict[str, Any]) -> Dict[str, Any]: + """ + Test a camera configuration to ensure it works. + + Args: + camera_info: Camera information dictionary from find_cameras + + Returns: + Test result with success status and details + """ + try: + camera_type = camera_info.get("type", "").lower() + camera_id = camera_info.get("id") + + if camera_type == "opencv": + from lerobot.common.cameras.opencv.camera_opencv import OpenCVCamera + from lerobot.common.cameras.opencv.configuration_opencv import OpenCVCameraConfig + from lerobot.common.cameras.configs import ColorMode + + config = OpenCVCameraConfig( + index_or_path=camera_id, + color_mode=ColorMode.RGB, + ) + camera = OpenCVCamera(config) + + elif camera_type == "realsense": + from lerobot.common.cameras.realsense.camera_realsense import RealSenseCamera + from lerobot.common.cameras.realsense.configuration_realsense import RealSenseCameraConfig + from lerobot.common.cameras.configs import ColorMode + + config = RealSenseCameraConfig( + serial_number_or_name=camera_id, + color_mode=ColorMode.RGB, + ) + camera = RealSenseCamera(config) + + else: + return { + "success": False, + "error": f"Unsupported camera type: {camera_type}", + "details": f"Camera type '{camera_type}' is not supported" + } + + # Test connection + logger.info(f"Testing {camera_type} camera: {camera_id}") + camera.connect(warmup=False) + + # Try to read a frame + frame = camera.read() + + # Disconnect + camera.disconnect() + + return { + "success": True, + "message": f"Camera {camera_id} ({camera_type}) tested successfully", + "frame_shape": frame.shape if frame is not None else None + } + + except Exception as e: + logger.error(f"Camera test failed for {camera_info}: {e}") + logger.error(f"Traceback: {traceback.format_exc()}") + return { + "success": False, + "error": f"Camera test failed: {str(e)}", + "details": "Camera may be in use by another application or not properly configured" + } + + +def create_camera_config_for_lerobot(camera_info: Dict[str, Any], custom_settings: Dict[str, Any] = None) -> Dict[str, Any]: + """ + Create a camera configuration dictionary suitable for LeRobot robots. + + Args: + camera_info: Camera information from find_cameras + custom_settings: Optional custom settings to override defaults + + Returns: + Camera configuration dictionary for robot config + """ + camera_type = camera_info.get("type", "").lower() + camera_id = camera_info.get("id") + + # Get default stream profile + default_profile = camera_info.get("default_stream_profile", {}) + + if camera_type == "opencv": + config = { + "type": "opencv", + "index_or_path": camera_id, + "width": default_profile.get("width", 640), + "height": default_profile.get("height", 480), + "fps": default_profile.get("fps", 30), + } + elif camera_type == "realsense": + config = { + "type": "realsense", + "serial_number_or_name": camera_id, + "width": default_profile.get("width", 640), + "height": default_profile.get("height", 480), + "fps": default_profile.get("fps", 30), + "use_depth": False, # Default to color only + } + else: + # Fallback to opencv format + config = { + "type": "opencv", + "index_or_path": camera_id, + "width": 640, + "height": 480, + "fps": 30, + } + + # Apply custom settings if provided + if custom_settings: + config.update(custom_settings) + + return config + + +def get_camera_summary() -> Dict[str, Any]: + """ + Get a summary of all available cameras for display purposes. + + Returns: + Summary dictionary with camera counts and basic info + """ + try: + all_cameras = find_all_cameras() + + opencv_count = sum(1 for cam in all_cameras if cam.get("type", "").lower() == "opencv") + realsense_count = sum(1 for cam in all_cameras if cam.get("type", "").lower() == "realsense") + + return { + "total_cameras": len(all_cameras), + "opencv_cameras": opencv_count, + "realsense_cameras": realsense_count, + "cameras": all_cameras, + "success": True + } + + except Exception as e: + logger.error(f"Error getting camera summary: {e}") + return { + "total_cameras": 0, + "opencv_cameras": 0, + "realsense_cameras": 0, + "cameras": [], + "success": False, + "error": str(e) + } + + +def capture_image_from_camera(camera_info: Dict[str, Any]) -> Dict[str, Any]: + """ + Capture an image from a specific camera and return as base64. + + Args: + camera_info: Camera information dictionary from find_cameras + + Returns: + Result with success status and base64 image data + """ + try: + import cv2 + import base64 + + camera_type = camera_info.get("type", "").lower() + camera_id = camera_info.get("id") + + logger.info(f"Capturing image from {camera_type} camera: {camera_id}") + logger.info(f"Camera info received: {camera_info}") + + # Validate camera_id + if camera_id is None or camera_id == "": + return { + "success": False, + "error": "Invalid camera ID", + "details": f"Received camera_id: {camera_id}" + } + + # Ensure camera_id is an integer for OpenCV + try: + camera_index = int(camera_id) + except (ValueError, TypeError): + return { + "success": False, + "error": f"Camera ID must be an integer, got: {camera_id}", + "details": f"Camera info: {camera_info}" + } + + # For now, use OpenCV directly since LeRobot camera modules may not be available + if camera_type == "opencv" or True: # Fallback to OpenCV for all cameras + cap = cv2.VideoCapture(camera_index) + + if not cap.isOpened(): + return { + "success": False, + "error": f"Cannot open camera at index {camera_index}", + "details": "Camera may be in use by another application" + } + + # Give camera time to initialize (especially for built-in cameras) + import time + time.sleep(0.5) + + # Try multiple captures for built-in cameras that may need warmup + ret, frame = None, None + for attempt in range(3): + ret, frame = cap.read() + if ret and frame is not None: + break + time.sleep(0.2) + + cap.release() + + if not ret or frame is None: + return { + "success": False, + "error": "Cannot capture frame from camera", + "details": "Camera opened but frame capture failed" + } + + # Convert frame to JPEG + _, buffer = cv2.imencode('.jpg', frame, [cv2.IMWRITE_JPEG_QUALITY, 70]) + + # Convert to base64 + image_base64 = base64.b64encode(buffer).decode('utf-8') + + return { + "success": True, + "image_data": image_base64, + "message": f"Image captured from camera {camera_index}", + "frame_shape": frame.shape + } + + except Exception as e: + logger.error(f"Image capture failed for {camera_info}: {e}") + logger.error(f"Traceback: {traceback.format_exc()}") + return { + "success": False, + "error": f"Image capture failed: {str(e)}", + "details": "Unexpected error during image capture" + } \ No newline at end of file diff --git a/app/config.py b/app/config.py index 98a884b..fbe58e4 100644 --- a/app/config.py +++ b/app/config.py @@ -27,6 +27,10 @@ LEADER_CONFIG_FILE = os.path.join(CONFIG_STORAGE_PATH, "leader_config.txt") FOLLOWER_CONFIG_FILE = os.path.join(CONFIG_STORAGE_PATH, "follower_config.txt") +# Define camera configuration storage path +CAMERA_CONFIG_PATH = os.path.expanduser("~/.cache/huggingface/lerobot/camera_configs") +CAMERA_CONFIG_FILE = os.path.join(CAMERA_CONFIG_PATH, "camera_config.json") + def setup_calibration_files(leader_config: str, follower_config: str): """Setup calibration files in the correct locations for teleoperation and recording""" # Extract config names from file paths (remove .json extension) @@ -300,3 +304,91 @@ def get_default_robot_config(robot_type: str, available_configs: list): return available_configs[0] return None + + +def save_camera_config(camera_config: dict): + """Save camera configuration to file""" + try: + import json + + # Create the camera config directory if it doesn't exist + os.makedirs(CAMERA_CONFIG_PATH, exist_ok=True) + + # Save the camera configuration as JSON + with open(CAMERA_CONFIG_FILE, 'w') as f: + json.dump(camera_config, f, indent=2) + + logger.info(f"Saved camera configuration with {len(camera_config.get('cameras', {}))} cameras") + return True + + except Exception as e: + logger.error(f"Error saving camera configuration: {e}") + return False + + +def get_saved_camera_config(): + """Get the saved camera configuration from file""" + try: + import json + + if os.path.exists(CAMERA_CONFIG_FILE): + with open(CAMERA_CONFIG_FILE, 'r') as f: + camera_config = json.load(f) + if camera_config: + logger.info(f"Found saved camera configuration with {len(camera_config.get('cameras', {}))} cameras") + return camera_config + + logger.info("No saved camera configuration found") + return None + + except Exception as e: + logger.error(f"Error reading saved camera configuration: {e}") + return None + + +def get_default_camera_config(): + """Get the default camera configuration, checking saved configs first""" + saved_config = get_saved_camera_config() + if saved_config: + return saved_config + + # Return empty config as fallback + return {"cameras": {}} + + +def update_camera_in_config(camera_name: str, camera_config: dict): + """Update a specific camera in the saved configuration""" + try: + # Get current config or create new one + full_config = get_saved_camera_config() or {"cameras": {}} + + # Update the specific camera + full_config["cameras"][camera_name] = camera_config + + # Save updated config + return save_camera_config(full_config) + + except Exception as e: + logger.error(f"Error updating camera {camera_name} in config: {e}") + return False + + +def remove_camera_from_config(camera_name: str): + """Remove a specific camera from the saved configuration""" + try: + # Get current config + full_config = get_saved_camera_config() + if not full_config or "cameras" not in full_config: + return True # Nothing to remove + + # Remove the camera if it exists + if camera_name in full_config["cameras"]: + del full_config["cameras"][camera_name] + logger.info(f"Removed camera {camera_name} from configuration") + + # Save updated config + return save_camera_config(full_config) + + except Exception as e: + logger.error(f"Error removing camera {camera_name} from config: {e}") + return False diff --git a/app/main.py b/app/main.py index e5f69e7..a065415 100644 --- a/app/main.py +++ b/app/main.py @@ -56,6 +56,17 @@ handle_replay_logs, ) +# Import camera detection functionality +from .camera_detection import ( + find_all_cameras, + find_opencv_cameras, + find_realsense_cameras, + test_camera_configuration, + create_camera_config_for_lerobot, + get_camera_summary, + capture_image_from_camera +) + # Set up logging logging.basicConfig(level=logging.INFO) @@ -631,6 +642,276 @@ def get_robot_config(robot_type: str, available_configs: str = ""): return {"status": "error", "message": str(e)} +# Camera detection endpoints +@app.get("/cameras/detect") +def detect_cameras(camera_type: str = None): + """Detect available cameras, optionally filtered by type (opencv/realsense)""" + try: + logger.info(f"Detecting cameras, filter: {camera_type}") + cameras = find_all_cameras(camera_type_filter=camera_type) + + return { + "status": "success", + "cameras": cameras, + "total_found": len(cameras), + "message": f"Found {len(cameras)} cameras" + } + except Exception as e: + logger.error(f"Error detecting cameras: {e}") + return {"status": "error", "message": str(e), "cameras": []} + + +@app.get("/cameras/summary") +def get_cameras_summary(): + """Get a summary of all available cameras""" + try: + summary = get_camera_summary() + return {"status": "success", **summary} + except Exception as e: + logger.error(f"Error getting camera summary: {e}") + return {"status": "error", "message": str(e)} + + +@app.post("/cameras/test") +def test_camera(camera_info: dict): + """Test a specific camera configuration""" + try: + result = test_camera_configuration(camera_info) + return {"status": "success" if result["success"] else "error", **result} + except Exception as e: + logger.error(f"Error testing camera: {e}") + return {"status": "error", "message": str(e)} + + +@app.post("/cameras/capture") +def capture_camera_image(data: dict): + """Capture an image from a specific camera""" + try: + # Extract camera_info from the request data + camera_info = data.get("camera_info", {}) + logger.info(f"Capture endpoint called with camera_info: {camera_info}") + + if not camera_info: + return {"status": "error", "message": "camera_info is required"} + + result = capture_image_from_camera(camera_info) + logger.info(f"Capture result: success={result.get('success')}, error={result.get('error', 'None')}") + + if result.get("success"): + logger.info(f"Successfully captured image from camera {camera_info.get('id')}") + else: + logger.warning(f"Failed to capture from camera {camera_info.get('id')}: {result.get('error')}") + + return {"status": "success" if result["success"] else "error", **result} + except Exception as e: + logger.error(f"Error capturing image: {e}") + logger.error(f"Traceback: {traceback.format_exc()}") + return {"status": "error", "message": str(e)} + + +@app.post("/cameras/create-config") +def create_camera_config(data: dict): + """Create a camera configuration for LeRobot""" + try: + camera_info = data.get("camera_info") + custom_settings = data.get("custom_settings", {}) + + if not camera_info: + return {"status": "error", "message": "camera_info is required"} + + config = create_camera_config_for_lerobot(camera_info, custom_settings) + + return { + "status": "success", + "camera_config": config, + "message": "Camera configuration created successfully" + } + except Exception as e: + logger.error(f"Error creating camera config: {e}") + return {"status": "error", "message": str(e)} + + +@app.get("/cameras/config") +def get_camera_config(): + """Get the saved camera configuration""" + try: + camera_config = config.get_default_camera_config() + return { + "status": "success", + "camera_config": camera_config, + "message": "Camera configuration retrieved successfully" + } + except Exception as e: + logger.error(f"Error getting camera config: {e}") + return {"status": "error", "message": str(e)} + + +@app.post("/cameras/config/save") +def save_camera_config_endpoint(data: dict): + """Save camera configuration""" + try: + camera_config = data.get("camera_config") + + if not camera_config: + return {"status": "error", "message": "camera_config is required"} + + success = config.save_camera_config(camera_config) + + if success: + return { + "status": "success", + "message": "Camera configuration saved successfully" + } + else: + return {"status": "error", "message": "Failed to save camera configuration"} + + except Exception as e: + logger.error(f"Error saving camera config: {e}") + return {"status": "error", "message": str(e)} + + +@app.post("/cameras/config/update") +def update_camera_config_endpoint(data: dict): + """Update a specific camera in the configuration""" + try: + camera_name = data.get("camera_name") + camera_config = data.get("camera_config") + + if not camera_name or not camera_config: + return {"status": "error", "message": "camera_name and camera_config are required"} + + success = config.update_camera_in_config(camera_name, camera_config) + + if success: + return { + "status": "success", + "message": f"Camera {camera_name} updated successfully" + } + else: + return {"status": "error", "message": f"Failed to update camera {camera_name}"} + + except Exception as e: + logger.error(f"Error updating camera config: {e}") + return {"status": "error", "message": str(e)} + + +@app.delete("/cameras/config/{camera_name}") +def remove_camera_config_endpoint(camera_name: str): + """Remove a specific camera from the configuration""" + try: + success = config.remove_camera_from_config(camera_name) + + if success: + return { + "status": "success", + "message": f"Camera {camera_name} removed successfully" + } + else: + return {"status": "error", "message": f"Failed to remove camera {camera_name}"} + + except Exception as e: + logger.error(f"Error removing camera config: {e}") + return {"status": "error", "message": str(e)} + + +@app.get("/cameras/stream/{camera_name}") +def stream_camera(camera_name: str): + """Stream a specific camera feed""" + try: + from fastapi.responses import StreamingResponse + import cv2 + import time + from . import config + + logger.info(f"Starting camera stream for: {camera_name}") + + # Get camera configuration + camera_config = config.get_saved_camera_config() + if not camera_config or "cameras" not in camera_config: + logger.error("No camera configuration found") + return {"status": "error", "message": "No camera configuration found"} + + if camera_name not in camera_config["cameras"]: + logger.error(f"Camera '{camera_name}' not found in configuration") + available_cameras = list(camera_config["cameras"].keys()) + logger.info(f"Available cameras: {available_cameras}") + return {"status": "error", "message": f"Camera '{camera_name}' not found in configuration"} + + camera_info = camera_config["cameras"][camera_name] + logger.info(f"Camera config: {camera_info}") + + def generate_frames(): + cap = None + try: + # Initialize camera based on type + if camera_info.get("type") == "opencv": + camera_index = camera_info.get("index_or_path", 0) + logger.info(f"Opening camera at index: {camera_index}") + + cap = cv2.VideoCapture(camera_index) + + if not cap.isOpened(): + logger.error(f"Failed to open camera at index {camera_index}") + return + + logger.info(f"Camera opened successfully at index {camera_index}") + + # Set camera properties from config + width = camera_info.get("width", 640) + height = camera_info.get("height", 480) + fps = camera_info.get("fps", 30) + + cap.set(cv2.CAP_PROP_FRAME_WIDTH, width) + cap.set(cv2.CAP_PROP_FRAME_HEIGHT, height) + cap.set(cv2.CAP_PROP_FPS, fps) + + logger.info(f"Camera settings: {width}x{height} @ {fps}fps") + + frame_count = 0 + while True: + ret, frame = cap.read() + if not ret: + logger.warning("Failed to read frame from camera") + break + + if frame_count % 30 == 0: # Log every 30 frames + logger.info(f"Streaming frame {frame_count} for {camera_name}") + + # Encode frame as JPEG + _, buffer = cv2.imencode('.jpg', frame, [cv2.IMWRITE_JPEG_QUALITY, 80]) + frame_bytes = buffer.tobytes() + + # Yield frame in multipart format + yield (b'--frame\r\n' + b'Content-Type: image/jpeg\r\n\r\n' + frame_bytes + b'\r\n') + + frame_count += 1 + + # Small delay to control frame rate + time.sleep(1.0 / fps) + else: + logger.error(f"Unsupported camera type: {camera_info.get('type')}") + return + + except Exception as e: + logger.error(f"Error in camera stream for {camera_name}: {e}") + import traceback + logger.error(f"Traceback: {traceback.format_exc()}") + finally: + if cap: + cap.release() + logger.info(f"Released camera {camera_name}") + + return StreamingResponse( + generate_frames(), + media_type="multipart/x-mixed-replace; boundary=frame" + ) + + except Exception as e: + logger.error(f"Error setting up camera stream for {camera_name}: {e}") + return {"status": "error", "message": str(e)} + + @app.on_event("shutdown") async def shutdown_event(): """Clean up resources when FastAPI shuts down""" From 3ab06d707e2a95f40d56b71bf303c86cb8e5f4e8 Mon Sep 17 00:00:00 2001 From: David <17435126+DavidLMS@users.noreply.github.com> Date: Thu, 19 Jun 2025 12:36:40 +0200 Subject: [PATCH 2/5] add camera management --- app/camera_detection.py | 404 ++++++++++++++++++++++++++++++++++++++++ app/config.py | 92 +++++++++ app/main.py | 281 ++++++++++++++++++++++++++++ 3 files changed, 777 insertions(+) create mode 100644 app/camera_detection.py diff --git a/app/camera_detection.py b/app/camera_detection.py new file mode 100644 index 0000000..8b36f95 --- /dev/null +++ b/app/camera_detection.py @@ -0,0 +1,404 @@ +import logging +from typing import List, Dict, Any +import traceback + +logger = logging.getLogger(__name__) + + +def find_opencv_cameras_fallback() -> List[Dict[str, Any]]: + """ + Find available OpenCV cameras using direct OpenCV detection. + + Returns: + List of camera information dictionaries + """ + try: + import cv2 + + logger.info("Searching for OpenCV cameras using direct method...") + cameras = [] + + # Test only first 5 indices to avoid spam + for index in range(5): + cap = cv2.VideoCapture(index) + + if cap.isOpened(): + # Get camera properties + width = cap.get(cv2.CAP_PROP_FRAME_WIDTH) + height = cap.get(cv2.CAP_PROP_FRAME_HEIGHT) + fps = cap.get(cv2.CAP_PROP_FPS) + backend = cap.get(cv2.CAP_PROP_BACKEND) + + # Give camera time to initialize (especially for built-in cameras) + import time + time.sleep(0.3) + + # Try multiple captures for built-in cameras that may need warmup + ret, frame = None, None + for attempt in range(3): + ret, frame = cap.read() + if ret and frame is not None and frame.sum() > 0: # Check if not all black + break + time.sleep(0.2) + + if ret and frame is not None: + # Capture a preview image immediately + import base64 + _, buffer = cv2.imencode('.jpg', frame, [cv2.IMWRITE_JPEG_QUALITY, 70]) + preview_image = base64.b64encode(buffer).decode('utf-8') + + camera_info = { + "name": f"Camera {index}", + "type": "OpenCV", + "id": index, + "backend_api": get_backend_name_fallback(backend), + "default_stream_profile": { + "width": int(width) if width > 0 else 640, + "height": int(height) if height > 0 else 480, + "fps": float(fps) if fps > 0 else 30.0, + "format": "RGB" + }, + "preview_image": preview_image # Add preview image + } + cameras.append(camera_info) + logger.info(f"Found working camera at index {index} with preview") + + cap.release() + + logger.info(f"Found {len(cameras)} OpenCV cameras using direct method") + return cameras + + except ImportError: + logger.error("OpenCV not available for camera detection") + return [] + except Exception as e: + logger.error(f"Error in direct camera detection: {e}") + return [] + + +def get_backend_name_fallback(backend_id: float) -> str: + """Convert OpenCV backend ID to readable name""" + backend_names = { + 0: "ANY", + 200: "V4L2", + 700: "AVFOUNDATION", + 1400: "DSHOW", + 1500: "GSTREAMER", + 1600: "FFMPEG" + } + return backend_names.get(int(backend_id), f"UNKNOWN_{int(backend_id)}") + + +def find_opencv_cameras() -> List[Dict[str, Any]]: + """ + Find available OpenCV cameras using direct detection (avoiding LeRobot spam). + + Returns: + List of camera information dictionaries + """ + # Force use of direct detection to avoid the 60-index spam from LeRobot + logger.info("Using direct OpenCV detection to avoid LeRobot spam...") + return find_opencv_cameras_fallback() + + +def find_realsense_cameras() -> List[Dict[str, Any]]: + """ + Find available RealSense cameras using LeRobot's camera detection. + + Returns: + List of camera information dictionaries + """ + try: + # Check if pyrealsense2 is properly available first + try: + import pyrealsense2 as rs + except ImportError: + logger.info("pyrealsense2 not installed - skipping RealSense camera detection") + return [] + except Exception as e: + logger.info(f"pyrealsense2 not available: {e} - skipping RealSense camera detection") + return [] + + # Try to import LeRobot RealSense support + try: + from lerobot.common.cameras.realsense.camera_realsense import RealSenseCamera + except ImportError: + logger.info("LeRobot RealSense support not available - skipping RealSense camera detection") + return [] + + logger.info("Searching for RealSense cameras using LeRobot...") + realsense_cameras = RealSenseCamera.find_cameras() + + logger.info(f"Found {len(realsense_cameras)} RealSense cameras") + return realsense_cameras + + except Exception as e: + logger.info(f"RealSense camera detection skipped: {e}") + return [] + + +def find_all_cameras(camera_type_filter: str = None) -> List[Dict[str, Any]]: + """ + Find all available cameras, optionally filtered by type. + + Args: + camera_type_filter: Optional filter ("opencv", "realsense", or None for all) + + Returns: + List of all available cameras matching the filter + """ + all_cameras = [] + + if camera_type_filter is None or camera_type_filter.lower() == "opencv": + all_cameras.extend(find_opencv_cameras()) + + if camera_type_filter is None or camera_type_filter.lower() == "realsense": + all_cameras.extend(find_realsense_cameras()) + + return all_cameras + + +def test_camera_configuration(camera_info: Dict[str, Any]) -> Dict[str, Any]: + """ + Test a camera configuration to ensure it works. + + Args: + camera_info: Camera information dictionary from find_cameras + + Returns: + Test result with success status and details + """ + try: + camera_type = camera_info.get("type", "").lower() + camera_id = camera_info.get("id") + + if camera_type == "opencv": + from lerobot.common.cameras.opencv.camera_opencv import OpenCVCamera + from lerobot.common.cameras.opencv.configuration_opencv import OpenCVCameraConfig + from lerobot.common.cameras.configs import ColorMode + + config = OpenCVCameraConfig( + index_or_path=camera_id, + color_mode=ColorMode.RGB, + ) + camera = OpenCVCamera(config) + + elif camera_type == "realsense": + from lerobot.common.cameras.realsense.camera_realsense import RealSenseCamera + from lerobot.common.cameras.realsense.configuration_realsense import RealSenseCameraConfig + from lerobot.common.cameras.configs import ColorMode + + config = RealSenseCameraConfig( + serial_number_or_name=camera_id, + color_mode=ColorMode.RGB, + ) + camera = RealSenseCamera(config) + + else: + return { + "success": False, + "error": f"Unsupported camera type: {camera_type}", + "details": f"Camera type '{camera_type}' is not supported" + } + + # Test connection + logger.info(f"Testing {camera_type} camera: {camera_id}") + camera.connect(warmup=False) + + # Try to read a frame + frame = camera.read() + + # Disconnect + camera.disconnect() + + return { + "success": True, + "message": f"Camera {camera_id} ({camera_type}) tested successfully", + "frame_shape": frame.shape if frame is not None else None + } + + except Exception as e: + logger.error(f"Camera test failed for {camera_info}: {e}") + logger.error(f"Traceback: {traceback.format_exc()}") + return { + "success": False, + "error": f"Camera test failed: {str(e)}", + "details": "Camera may be in use by another application or not properly configured" + } + + +def create_camera_config_for_lerobot(camera_info: Dict[str, Any], custom_settings: Dict[str, Any] = None) -> Dict[str, Any]: + """ + Create a camera configuration dictionary suitable for LeRobot robots. + + Args: + camera_info: Camera information from find_cameras + custom_settings: Optional custom settings to override defaults + + Returns: + Camera configuration dictionary for robot config + """ + camera_type = camera_info.get("type", "").lower() + camera_id = camera_info.get("id") + + # Get default stream profile + default_profile = camera_info.get("default_stream_profile", {}) + + if camera_type == "opencv": + config = { + "type": "opencv", + "index_or_path": camera_id, + "width": default_profile.get("width", 640), + "height": default_profile.get("height", 480), + "fps": default_profile.get("fps", 30), + } + elif camera_type == "realsense": + config = { + "type": "realsense", + "serial_number_or_name": camera_id, + "width": default_profile.get("width", 640), + "height": default_profile.get("height", 480), + "fps": default_profile.get("fps", 30), + "use_depth": False, # Default to color only + } + else: + # Fallback to opencv format + config = { + "type": "opencv", + "index_or_path": camera_id, + "width": 640, + "height": 480, + "fps": 30, + } + + # Apply custom settings if provided + if custom_settings: + config.update(custom_settings) + + return config + + +def get_camera_summary() -> Dict[str, Any]: + """ + Get a summary of all available cameras for display purposes. + + Returns: + Summary dictionary with camera counts and basic info + """ + try: + all_cameras = find_all_cameras() + + opencv_count = sum(1 for cam in all_cameras if cam.get("type", "").lower() == "opencv") + realsense_count = sum(1 for cam in all_cameras if cam.get("type", "").lower() == "realsense") + + return { + "total_cameras": len(all_cameras), + "opencv_cameras": opencv_count, + "realsense_cameras": realsense_count, + "cameras": all_cameras, + "success": True + } + + except Exception as e: + logger.error(f"Error getting camera summary: {e}") + return { + "total_cameras": 0, + "opencv_cameras": 0, + "realsense_cameras": 0, + "cameras": [], + "success": False, + "error": str(e) + } + + +def capture_image_from_camera(camera_info: Dict[str, Any]) -> Dict[str, Any]: + """ + Capture an image from a specific camera and return as base64. + + Args: + camera_info: Camera information dictionary from find_cameras + + Returns: + Result with success status and base64 image data + """ + try: + import cv2 + import base64 + + camera_type = camera_info.get("type", "").lower() + camera_id = camera_info.get("id") + + logger.info(f"Capturing image from {camera_type} camera: {camera_id}") + logger.info(f"Camera info received: {camera_info}") + + # Validate camera_id + if camera_id is None or camera_id == "": + return { + "success": False, + "error": "Invalid camera ID", + "details": f"Received camera_id: {camera_id}" + } + + # Ensure camera_id is an integer for OpenCV + try: + camera_index = int(camera_id) + except (ValueError, TypeError): + return { + "success": False, + "error": f"Camera ID must be an integer, got: {camera_id}", + "details": f"Camera info: {camera_info}" + } + + # For now, use OpenCV directly since LeRobot camera modules may not be available + if camera_type == "opencv" or True: # Fallback to OpenCV for all cameras + cap = cv2.VideoCapture(camera_index) + + if not cap.isOpened(): + return { + "success": False, + "error": f"Cannot open camera at index {camera_index}", + "details": "Camera may be in use by another application" + } + + # Give camera time to initialize (especially for built-in cameras) + import time + time.sleep(0.5) + + # Try multiple captures for built-in cameras that may need warmup + ret, frame = None, None + for attempt in range(3): + ret, frame = cap.read() + if ret and frame is not None: + break + time.sleep(0.2) + + cap.release() + + if not ret or frame is None: + return { + "success": False, + "error": "Cannot capture frame from camera", + "details": "Camera opened but frame capture failed" + } + + # Convert frame to JPEG + _, buffer = cv2.imencode('.jpg', frame, [cv2.IMWRITE_JPEG_QUALITY, 70]) + + # Convert to base64 + image_base64 = base64.b64encode(buffer).decode('utf-8') + + return { + "success": True, + "image_data": image_base64, + "message": f"Image captured from camera {camera_index}", + "frame_shape": frame.shape + } + + except Exception as e: + logger.error(f"Image capture failed for {camera_info}: {e}") + logger.error(f"Traceback: {traceback.format_exc()}") + return { + "success": False, + "error": f"Image capture failed: {str(e)}", + "details": "Unexpected error during image capture" + } \ No newline at end of file diff --git a/app/config.py b/app/config.py index 98a884b..fbe58e4 100644 --- a/app/config.py +++ b/app/config.py @@ -27,6 +27,10 @@ LEADER_CONFIG_FILE = os.path.join(CONFIG_STORAGE_PATH, "leader_config.txt") FOLLOWER_CONFIG_FILE = os.path.join(CONFIG_STORAGE_PATH, "follower_config.txt") +# Define camera configuration storage path +CAMERA_CONFIG_PATH = os.path.expanduser("~/.cache/huggingface/lerobot/camera_configs") +CAMERA_CONFIG_FILE = os.path.join(CAMERA_CONFIG_PATH, "camera_config.json") + def setup_calibration_files(leader_config: str, follower_config: str): """Setup calibration files in the correct locations for teleoperation and recording""" # Extract config names from file paths (remove .json extension) @@ -300,3 +304,91 @@ def get_default_robot_config(robot_type: str, available_configs: list): return available_configs[0] return None + + +def save_camera_config(camera_config: dict): + """Save camera configuration to file""" + try: + import json + + # Create the camera config directory if it doesn't exist + os.makedirs(CAMERA_CONFIG_PATH, exist_ok=True) + + # Save the camera configuration as JSON + with open(CAMERA_CONFIG_FILE, 'w') as f: + json.dump(camera_config, f, indent=2) + + logger.info(f"Saved camera configuration with {len(camera_config.get('cameras', {}))} cameras") + return True + + except Exception as e: + logger.error(f"Error saving camera configuration: {e}") + return False + + +def get_saved_camera_config(): + """Get the saved camera configuration from file""" + try: + import json + + if os.path.exists(CAMERA_CONFIG_FILE): + with open(CAMERA_CONFIG_FILE, 'r') as f: + camera_config = json.load(f) + if camera_config: + logger.info(f"Found saved camera configuration with {len(camera_config.get('cameras', {}))} cameras") + return camera_config + + logger.info("No saved camera configuration found") + return None + + except Exception as e: + logger.error(f"Error reading saved camera configuration: {e}") + return None + + +def get_default_camera_config(): + """Get the default camera configuration, checking saved configs first""" + saved_config = get_saved_camera_config() + if saved_config: + return saved_config + + # Return empty config as fallback + return {"cameras": {}} + + +def update_camera_in_config(camera_name: str, camera_config: dict): + """Update a specific camera in the saved configuration""" + try: + # Get current config or create new one + full_config = get_saved_camera_config() or {"cameras": {}} + + # Update the specific camera + full_config["cameras"][camera_name] = camera_config + + # Save updated config + return save_camera_config(full_config) + + except Exception as e: + logger.error(f"Error updating camera {camera_name} in config: {e}") + return False + + +def remove_camera_from_config(camera_name: str): + """Remove a specific camera from the saved configuration""" + try: + # Get current config + full_config = get_saved_camera_config() + if not full_config or "cameras" not in full_config: + return True # Nothing to remove + + # Remove the camera if it exists + if camera_name in full_config["cameras"]: + del full_config["cameras"][camera_name] + logger.info(f"Removed camera {camera_name} from configuration") + + # Save updated config + return save_camera_config(full_config) + + except Exception as e: + logger.error(f"Error removing camera {camera_name} from config: {e}") + return False diff --git a/app/main.py b/app/main.py index 668bb97..dad75ab 100644 --- a/app/main.py +++ b/app/main.py @@ -56,6 +56,17 @@ handle_replay_logs, ) +# Import camera detection functionality +from .camera_detection import ( + find_all_cameras, + find_opencv_cameras, + find_realsense_cameras, + test_camera_configuration, + create_camera_config_for_lerobot, + get_camera_summary, + capture_image_from_camera +) + # Set up logging logging.basicConfig(level=logging.INFO) @@ -665,6 +676,276 @@ def get_robot_config(robot_type: str, available_configs: str = ""): return {"status": "error", "message": str(e)} +# Camera detection endpoints +@app.get("/cameras/detect") +def detect_cameras(camera_type: str = None): + """Detect available cameras, optionally filtered by type (opencv/realsense)""" + try: + logger.info(f"Detecting cameras, filter: {camera_type}") + cameras = find_all_cameras(camera_type_filter=camera_type) + + return { + "status": "success", + "cameras": cameras, + "total_found": len(cameras), + "message": f"Found {len(cameras)} cameras" + } + except Exception as e: + logger.error(f"Error detecting cameras: {e}") + return {"status": "error", "message": str(e), "cameras": []} + + +@app.get("/cameras/summary") +def get_cameras_summary(): + """Get a summary of all available cameras""" + try: + summary = get_camera_summary() + return {"status": "success", **summary} + except Exception as e: + logger.error(f"Error getting camera summary: {e}") + return {"status": "error", "message": str(e)} + + +@app.post("/cameras/test") +def test_camera(camera_info: dict): + """Test a specific camera configuration""" + try: + result = test_camera_configuration(camera_info) + return {"status": "success" if result["success"] else "error", **result} + except Exception as e: + logger.error(f"Error testing camera: {e}") + return {"status": "error", "message": str(e)} + + +@app.post("/cameras/capture") +def capture_camera_image(data: dict): + """Capture an image from a specific camera""" + try: + # Extract camera_info from the request data + camera_info = data.get("camera_info", {}) + logger.info(f"Capture endpoint called with camera_info: {camera_info}") + + if not camera_info: + return {"status": "error", "message": "camera_info is required"} + + result = capture_image_from_camera(camera_info) + logger.info(f"Capture result: success={result.get('success')}, error={result.get('error', 'None')}") + + if result.get("success"): + logger.info(f"Successfully captured image from camera {camera_info.get('id')}") + else: + logger.warning(f"Failed to capture from camera {camera_info.get('id')}: {result.get('error')}") + + return {"status": "success" if result["success"] else "error", **result} + except Exception as e: + logger.error(f"Error capturing image: {e}") + logger.error(f"Traceback: {traceback.format_exc()}") + return {"status": "error", "message": str(e)} + + +@app.post("/cameras/create-config") +def create_camera_config(data: dict): + """Create a camera configuration for LeRobot""" + try: + camera_info = data.get("camera_info") + custom_settings = data.get("custom_settings", {}) + + if not camera_info: + return {"status": "error", "message": "camera_info is required"} + + config = create_camera_config_for_lerobot(camera_info, custom_settings) + + return { + "status": "success", + "camera_config": config, + "message": "Camera configuration created successfully" + } + except Exception as e: + logger.error(f"Error creating camera config: {e}") + return {"status": "error", "message": str(e)} + + +@app.get("/cameras/config") +def get_camera_config(): + """Get the saved camera configuration""" + try: + camera_config = config.get_default_camera_config() + return { + "status": "success", + "camera_config": camera_config, + "message": "Camera configuration retrieved successfully" + } + except Exception as e: + logger.error(f"Error getting camera config: {e}") + return {"status": "error", "message": str(e)} + + +@app.post("/cameras/config/save") +def save_camera_config_endpoint(data: dict): + """Save camera configuration""" + try: + camera_config = data.get("camera_config") + + if not camera_config: + return {"status": "error", "message": "camera_config is required"} + + success = config.save_camera_config(camera_config) + + if success: + return { + "status": "success", + "message": "Camera configuration saved successfully" + } + else: + return {"status": "error", "message": "Failed to save camera configuration"} + + except Exception as e: + logger.error(f"Error saving camera config: {e}") + return {"status": "error", "message": str(e)} + + +@app.post("/cameras/config/update") +def update_camera_config_endpoint(data: dict): + """Update a specific camera in the configuration""" + try: + camera_name = data.get("camera_name") + camera_config = data.get("camera_config") + + if not camera_name or not camera_config: + return {"status": "error", "message": "camera_name and camera_config are required"} + + success = config.update_camera_in_config(camera_name, camera_config) + + if success: + return { + "status": "success", + "message": f"Camera {camera_name} updated successfully" + } + else: + return {"status": "error", "message": f"Failed to update camera {camera_name}"} + + except Exception as e: + logger.error(f"Error updating camera config: {e}") + return {"status": "error", "message": str(e)} + + +@app.delete("/cameras/config/{camera_name}") +def remove_camera_config_endpoint(camera_name: str): + """Remove a specific camera from the configuration""" + try: + success = config.remove_camera_from_config(camera_name) + + if success: + return { + "status": "success", + "message": f"Camera {camera_name} removed successfully" + } + else: + return {"status": "error", "message": f"Failed to remove camera {camera_name}"} + + except Exception as e: + logger.error(f"Error removing camera config: {e}") + return {"status": "error", "message": str(e)} + + +@app.get("/cameras/stream/{camera_name}") +def stream_camera(camera_name: str): + """Stream a specific camera feed""" + try: + from fastapi.responses import StreamingResponse + import cv2 + import time + from . import config + + logger.info(f"Starting camera stream for: {camera_name}") + + # Get camera configuration + camera_config = config.get_saved_camera_config() + if not camera_config or "cameras" not in camera_config: + logger.error("No camera configuration found") + return {"status": "error", "message": "No camera configuration found"} + + if camera_name not in camera_config["cameras"]: + logger.error(f"Camera '{camera_name}' not found in configuration") + available_cameras = list(camera_config["cameras"].keys()) + logger.info(f"Available cameras: {available_cameras}") + return {"status": "error", "message": f"Camera '{camera_name}' not found in configuration"} + + camera_info = camera_config["cameras"][camera_name] + logger.info(f"Camera config: {camera_info}") + + def generate_frames(): + cap = None + try: + # Initialize camera based on type + if camera_info.get("type") == "opencv": + camera_index = camera_info.get("index_or_path", 0) + logger.info(f"Opening camera at index: {camera_index}") + + cap = cv2.VideoCapture(camera_index) + + if not cap.isOpened(): + logger.error(f"Failed to open camera at index {camera_index}") + return + + logger.info(f"Camera opened successfully at index {camera_index}") + + # Set camera properties from config + width = camera_info.get("width", 640) + height = camera_info.get("height", 480) + fps = camera_info.get("fps", 30) + + cap.set(cv2.CAP_PROP_FRAME_WIDTH, width) + cap.set(cv2.CAP_PROP_FRAME_HEIGHT, height) + cap.set(cv2.CAP_PROP_FPS, fps) + + logger.info(f"Camera settings: {width}x{height} @ {fps}fps") + + frame_count = 0 + while True: + ret, frame = cap.read() + if not ret: + logger.warning("Failed to read frame from camera") + break + + if frame_count % 30 == 0: # Log every 30 frames + logger.info(f"Streaming frame {frame_count} for {camera_name}") + + # Encode frame as JPEG + _, buffer = cv2.imencode('.jpg', frame, [cv2.IMWRITE_JPEG_QUALITY, 80]) + frame_bytes = buffer.tobytes() + + # Yield frame in multipart format + yield (b'--frame\r\n' + b'Content-Type: image/jpeg\r\n\r\n' + frame_bytes + b'\r\n') + + frame_count += 1 + + # Small delay to control frame rate + time.sleep(1.0 / fps) + else: + logger.error(f"Unsupported camera type: {camera_info.get('type')}") + return + + except Exception as e: + logger.error(f"Error in camera stream for {camera_name}: {e}") + import traceback + logger.error(f"Traceback: {traceback.format_exc()}") + finally: + if cap: + cap.release() + logger.info(f"Released camera {camera_name}") + + return StreamingResponse( + generate_frames(), + media_type="multipart/x-mixed-replace; boundary=frame" + ) + + except Exception as e: + logger.error(f"Error setting up camera stream for {camera_name}: {e}") + return {"status": "error", "message": str(e)} + + @app.on_event("shutdown") async def shutdown_event(): """Clean up resources when FastAPI shuts down""" From c41abdda6f2a6c62975f0c5802c5332957eb4f5b Mon Sep 17 00:00:00 2001 From: David <17435126+DavidLMS@users.noreply.github.com> Date: Fri, 20 Jun 2025 10:45:31 +0200 Subject: [PATCH 3/5] Enhance /available-cameras endpoint with advanced detection --- app/main.py | 53 +++++++++++++++++++++++++++++------------------------ 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/app/main.py b/app/main.py index dad75ab..0bc743c 100644 --- a/app/main.py +++ b/app/main.py @@ -6,6 +6,7 @@ import logging import glob import asyncio +import traceback from typing import List, Dict, Any import threading import queue @@ -543,33 +544,37 @@ def get_available_ports(): @app.get("/available-cameras") def get_available_cameras(): - """Get all available cameras""" + """Get all available cameras using advanced detection""" try: - # Try to detect cameras using OpenCV - import cv2 - cameras = [] + # Use the advanced camera detection with preview images and better error handling + cameras = find_all_cameras() - # Test up to 10 camera indices - for i in range(10): - cap = cv2.VideoCapture(i) - if cap.isOpened(): - ret, frame = cap.read() - if ret: - cameras.append({ - "index": i, - "name": f"Camera {i}", - "available": True, - "width": int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)), - "height": int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)), - "fps": int(cap.get(cv2.CAP_PROP_FPS)), - }) - cap.release() + # Transform to maintain backward compatibility with the original format + compatible_cameras = [] + for cam in cameras: + compatible_cam = { + "index": cam.get("id"), + "name": cam.get("name", f"Camera {cam.get('id')}"), + "available": True, + "type": cam.get("type", "OpenCV").lower(), + "backend_api": cam.get("backend_api", "UNKNOWN"), + } - return {"status": "success", "cameras": cameras} - except ImportError: - # OpenCV not available, return empty list - logger.warning("OpenCV not available for camera detection") - return {"status": "success", "cameras": []} + # Add stream profile information + profile = cam.get("default_stream_profile", {}) + compatible_cam.update({ + "width": profile.get("width", 640), + "height": profile.get("height", 480), + "fps": profile.get("fps", 30), + }) + + # Add preview image if available + if "preview_image" in cam: + compatible_cam["preview_image"] = cam["preview_image"] + + compatible_cameras.append(compatible_cam) + + return {"status": "success", "cameras": compatible_cameras} except Exception as e: logger.error(f"Error detecting cameras: {e}") return {"status": "error", "message": str(e), "cameras": []} From 61d6ac690dbe86b76dd2ece7af022e90b7c370af Mon Sep 17 00:00:00 2001 From: David <17435126+DavidLMS@users.noreply.github.com> Date: Mon, 23 Jun 2025 18:13:02 +0200 Subject: [PATCH 4/5] improve camera management with WebRTC --- app/main.py | 269 ++++++++++++++++++++++++++++++-------- app/webrtc_signaling.py | 280 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 497 insertions(+), 52 deletions(-) create mode 100644 app/webrtc_signaling.py diff --git a/app/main.py b/app/main.py index 0bc743c..eb5e900 100644 --- a/app/main.py +++ b/app/main.py @@ -274,6 +274,26 @@ def websocket_test(): return {"websocket_endpoint": "/ws/joint-data", "status": "available"} +@app.get("/webrtc/status") +def webrtc_status(): + """Get WebRTC signaling server status""" + from .webrtc_signaling import signaling_server + try: + stats = signaling_server.get_stats() + return { + "status": "success", + "signaling_server": "active", + "stats": stats, + "endpoints": { + "signaling": "/ws/webrtc", + "status": "/webrtc/status" + } + } + except Exception as e: + logger.error(f"Error getting WebRTC status: {e}") + return {"status": "error", "message": str(e)} + + @app.websocket("/ws/joint-data") async def websocket_endpoint(websocket: WebSocket): logger.info("🔗 New WebSocket connection attempt") @@ -306,6 +326,48 @@ async def websocket_endpoint(websocket: WebSocket): logger.info("🧹 WebSocket connection cleaned up") +@app.websocket("/ws/webrtc") +async def webrtc_signaling_endpoint(websocket: WebSocket): + """WebRTC signaling server endpoint""" + from .webrtc_signaling import signaling_server + import json + + logger.info("🔗 New WebRTC signaling connection attempt") + connection_id = None + + try: + connection_id = await signaling_server.connect(websocket) + logger.info(f"✅ WebRTC signaling connection established: {connection_id}") + + while True: + try: + # Wait for signaling messages + data = await websocket.receive_text() + message = json.loads(data) + + # Handle signaling message + await signaling_server.handle_message(connection_id, message) + + except WebSocketDisconnect: + logger.info(f"🔌 WebRTC signaling client disconnected: {connection_id}") + break + except json.JSONDecodeError as e: + logger.error(f"❌ Invalid JSON received from {connection_id}: {e}") + await signaling_server._send_error(connection_id, "Invalid JSON format") + except Exception as e: + logger.error(f"❌ Error processing WebRTC message from {connection_id}: {e}") + await signaling_server._send_error(connection_id, str(e)) + + except WebSocketDisconnect: + logger.info(f"🔌 WebRTC signaling disconnected normally: {connection_id}") + except Exception as e: + logger.error(f"❌ WebRTC signaling error: {e}") + finally: + if connection_id: + signaling_server.disconnect(connection_id) + logger.info(f"🧹 WebRTC signaling connection cleaned up: {connection_id}") + + @app.post("/start-recording") def start_recording(request: RecordingRequest): """Start a dataset recording session""" @@ -853,84 +915,187 @@ def remove_camera_config_endpoint(camera_name: str): return {"status": "error", "message": str(e)} -@app.get("/cameras/stream/{camera_name}") -def stream_camera(camera_name: str): - """Stream a specific camera feed""" +@app.get("/cameras/stream/{camera_identifier}") +def stream_camera(camera_identifier: str): + """Stream a specific camera feed using hash or name""" try: from fastapi.responses import StreamingResponse import cv2 import time from . import config - logger.info(f"Starting camera stream for: {camera_name}") + logger.info(f"Starting camera stream for identifier: {camera_identifier}") # Get camera configuration camera_config = config.get_saved_camera_config() if not camera_config or "cameras" not in camera_config: logger.error("No camera configuration found") return {"status": "error", "message": "No camera configuration found"} - - if camera_name not in camera_config["cameras"]: - logger.error(f"Camera '{camera_name}' not found in configuration") + + # Find camera by hash or name for backward compatibility + camera_info = None + camera_name = None + + # First try to find by exact name match (backward compatibility) + if camera_identifier in camera_config["cameras"]: + camera_info = camera_config["cameras"][camera_identifier] + camera_name = camera_identifier + logger.info(f"Found camera by name: {camera_name}") + else: + # Try to find by hash (new robust system) + for name, config_data in camera_config["cameras"].items(): + if config_data.get("hash") == camera_identifier: + camera_info = config_data + camera_name = name + logger.info(f"Found camera by hash: {camera_identifier} -> {camera_name}") + break + + if not camera_info: + logger.error(f"Camera '{camera_identifier}' not found in configuration") available_cameras = list(camera_config["cameras"].keys()) logger.info(f"Available cameras: {available_cameras}") - return {"status": "error", "message": f"Camera '{camera_name}' not found in configuration"} + return {"status": "error", "message": f"Camera '{camera_identifier}' not found in configuration"} - camera_info = camera_config["cameras"][camera_name] - logger.info(f"Camera config: {camera_info}") + logger.info(f"Camera config for '{camera_name}': {camera_info}") + + # CRITICAL: Use device_id as primary identifier for consistency with preview + device_id = camera_info.get("device_id") + camera_hash = camera_info.get("hash") + index_or_path = camera_info.get("index_or_path", 0) + logger.info(f"Robust camera identifiers - device_id: {device_id}, hash: {camera_hash}, fallback_index: {index_or_path}") def generate_frames(): cap = None try: - # Initialize camera based on type - if camera_info.get("type") == "opencv": - camera_index = camera_info.get("index_or_path", 0) - logger.info(f"Opening camera at index: {camera_index}") + # ROBUST APPROACH: Use device_id enumeration to find correct OpenCV index + # This ensures the SAME camera is opened in both preview and streaming + + if device_id and not device_id.startswith("fallback_"): + # Try to map device_id to current OpenCV index + logger.info(f"Attempting to map device_id {device_id[:16]}... to OpenCV index") - cap = cv2.VideoCapture(camera_index) + # Get current camera enumeration (same as frontend does) + import subprocess + import json - if not cap.isOpened(): - logger.error(f"Failed to open camera at index {camera_index}") - return + try: + # Use Python to enumerate cameras the same way frontend does + enum_script = ''' +import cv2 +import json +cameras = [] +for i in range(10): # Check first 10 indices + cap = cv2.VideoCapture(i) + if cap.isOpened(): + cameras.append({"index": i, "available": True}) + cap.release() + else: + break +print(json.dumps(cameras)) +''' + result = subprocess.run(['python3', '-c', enum_script], + capture_output=True, text=True, timeout=10) + + if result.returncode == 0: + available_cameras = json.loads(result.stdout) + logger.info(f"Available camera indices: {[c['index'] for c in available_cameras]}") + + # Try each available index to see which matches our device + camera_index = None + for cam_info in available_cameras: + test_index = cam_info["index"] + test_cap = cv2.VideoCapture(test_index) + if test_cap.isOpened(): + # For now, use the fallback index from config + # In a more sophisticated version, we could try to match device properties + if test_index == index_or_path: + camera_index = test_index + logger.info(f"Matched device_id to OpenCV index {camera_index}") + test_cap.release() + break + test_cap.release() + + if camera_index is None: + # Use the stored index as fallback + camera_index = index_or_path + logger.warning(f"Could not map device_id, using fallback index {camera_index}") + else: + camera_index = index_or_path + logger.warning(f"Camera enumeration failed, using fallback index {camera_index}") + + except Exception as e: + camera_index = index_or_path + logger.warning(f"Error during camera mapping: {e}, using fallback index {camera_index}") + + else: + # No device_id or fallback device_id, use stored index + camera_index = index_or_path + logger.info(f"Using stored camera index: {camera_index}") + + logger.info(f"Opening camera at index {camera_index} for streaming (device_id: {device_id[:16] if device_id else 'none'}...)") + cap = cv2.VideoCapture(camera_index) + identifier_used = f"robust_index_{camera_index}" + + if not cap.isOpened(): + logger.error(f"Failed to open camera with identifier: {identifier_used}") + return + + logger.info(f"Camera opened successfully with identifier: {identifier_used}") + + # Set camera properties from config + width = camera_info.get("width", 640) + height = camera_info.get("height", 480) + fps = camera_info.get("fps", 30) + + cap.set(cv2.CAP_PROP_FRAME_WIDTH, width) + cap.set(cv2.CAP_PROP_FRAME_HEIGHT, height) + cap.set(cv2.CAP_PROP_FPS, fps) + + logger.info(f"Camera settings: {width}x{height} @ {fps}fps") + + # Add camera warmup + logger.info("Warming up camera...") + time.sleep(0.5) + + # Try multiple captures for camera warmup + for attempt in range(3): + ret, test_frame = cap.read() + if ret and test_frame is not None and test_frame.sum() > 0: + logger.info(f"Camera warmed up successfully after {attempt + 1} attempts") + break + time.sleep(0.2) + else: + logger.error("Camera warmup failed - no valid frames received") + return + + frame_count = 0 + while True: + ret, frame = cap.read() + if not ret: + logger.warning("Failed to read frame from camera") + break - logger.info(f"Camera opened successfully at index {camera_index}") + # Validate frame is not black/empty + if frame is None or frame.sum() == 0: + logger.warning("Received black/empty frame, skipping") + continue - # Set camera properties from config - width = camera_info.get("width", 640) - height = camera_info.get("height", 480) - fps = camera_info.get("fps", 30) + # Encode frame as JPEG - CRITICAL FIX: This should happen for EVERY frame, not just every 30th + _, buffer = cv2.imencode('.jpg', frame, [cv2.IMWRITE_JPEG_QUALITY, 80]) + frame_bytes = buffer.tobytes() - cap.set(cv2.CAP_PROP_FRAME_WIDTH, width) - cap.set(cv2.CAP_PROP_FRAME_HEIGHT, height) - cap.set(cv2.CAP_PROP_FPS, fps) + # Yield frame in multipart format + yield (b'--frame\r\n' + b'Content-Type: image/jpeg\r\n\r\n' + frame_bytes + b'\r\n') - logger.info(f"Camera settings: {width}x{height} @ {fps}fps") + # Log every 30 frames for debugging + if frame_count % 30 == 0: + logger.info(f"Streaming frame {frame_count} for {camera_name}") - frame_count = 0 - while True: - ret, frame = cap.read() - if not ret: - logger.warning("Failed to read frame from camera") - break - - if frame_count % 30 == 0: # Log every 30 frames - logger.info(f"Streaming frame {frame_count} for {camera_name}") - - # Encode frame as JPEG - _, buffer = cv2.imencode('.jpg', frame, [cv2.IMWRITE_JPEG_QUALITY, 80]) - frame_bytes = buffer.tobytes() - - # Yield frame in multipart format - yield (b'--frame\r\n' - b'Content-Type: image/jpeg\r\n\r\n' + frame_bytes + b'\r\n') - - frame_count += 1 - - # Small delay to control frame rate - time.sleep(1.0 / fps) - else: - logger.error(f"Unsupported camera type: {camera_info.get('type')}") - return + frame_count += 1 + + # Small delay to control frame rate + time.sleep(1.0 / fps) except Exception as e: logger.error(f"Error in camera stream for {camera_name}: {e}") diff --git a/app/webrtc_signaling.py b/app/webrtc_signaling.py new file mode 100644 index 0000000..4b10d03 --- /dev/null +++ b/app/webrtc_signaling.py @@ -0,0 +1,280 @@ +# WebRTC Signaling Server for Unified Camera System + +import json +import logging +import asyncio +from typing import Dict, Set, Optional, Any +from fastapi import WebSocket, WebSocketDisconnect +import uuid + +logger = logging.getLogger(__name__) + +class WebRTCSignalingServer: + """WebRTC signaling server for camera streaming""" + + def __init__(self): + # Active WebSocket connections + self.connections: Dict[str, WebSocket] = {} + + # Camera sources registry + self.camera_sources: Dict[str, Dict[str, Any]] = {} + + # Peer-to-peer mappings (for future remote cameras) + self.peer_mappings: Dict[str, str] = {} + + # Connection metadata + self.connection_metadata: Dict[str, Dict[str, Any]] = {} + + async def connect(self, websocket: WebSocket) -> str: + """Accept a new WebSocket connection and return connection ID""" + await websocket.accept() + connection_id = str(uuid.uuid4()) + + self.connections[connection_id] = websocket + self.connection_metadata[connection_id] = { + "connected_at": asyncio.get_event_loop().time(), + "camera_sources": [], + "peer_id": None + } + + logger.info(f"🔗 WebRTC client connected: {connection_id}") + logger.info(f"📊 Total connections: {len(self.connections)}") + + return connection_id + + def disconnect(self, connection_id: str): + """Handle client disconnection""" + if connection_id in self.connections: + # Clean up camera sources for this connection + metadata = self.connection_metadata.get(connection_id, {}) + camera_sources = metadata.get("camera_sources", []) + + for source_id in camera_sources: + if source_id in self.camera_sources: + logger.info(f"🗑️ Removing camera source: {source_id}") + del self.camera_sources[source_id] + + # Remove connection + del self.connections[connection_id] + if connection_id in self.connection_metadata: + del self.connection_metadata[connection_id] + + logger.info(f"🔌 WebRTC client disconnected: {connection_id}") + logger.info(f"📊 Total connections: {len(self.connections)}") + + async def handle_message(self, connection_id: str, message: Dict[str, Any]): + """Handle incoming signaling message""" + try: + message_type = message.get("type") + source_id = message.get("sourceId") + + logger.info(f"📨 Received {message_type} for source {source_id} from {connection_id}") + + if message_type == "register-camera": + await self._handle_register_camera(connection_id, message) + elif message_type == "offer": + await self._handle_offer(connection_id, message) + elif message_type == "answer": + await self._handle_answer(connection_id, message) + elif message_type == "ice-candidate": + await self._handle_ice_candidate(connection_id, message) + elif message_type == "request-camera-list": + await self._handle_camera_list_request(connection_id) + elif message_type == "request-camera-stream": + await self._handle_camera_stream_request(connection_id, message) + else: + logger.warning(f"⚠️ Unknown message type: {message_type}") + + except Exception as e: + logger.error(f"❌ Error handling message from {connection_id}: {e}") + await self._send_error(connection_id, str(e)) + + async def _handle_register_camera(self, connection_id: str, message: Dict[str, Any]): + """Register a new camera source""" + source_id = message.get("sourceId") + payload = message.get("payload", {}) + + camera_info = { + "id": source_id, + "name": payload.get("name", "Unknown Camera"), + "type": payload.get("type", "local"), + "width": payload.get("width", 640), + "height": payload.get("height", 480), + "fps": payload.get("fps", 30), + "owner_connection": connection_id, + "status": "available", + "registered_at": asyncio.get_event_loop().time() + } + + self.camera_sources[source_id] = camera_info + + # Add to connection metadata + if connection_id in self.connection_metadata: + self.connection_metadata[connection_id]["camera_sources"].append(source_id) + + logger.info(f"📹 Registered camera: {camera_info['name']} ({source_id})") + + # Broadcast camera list update to all connections + await self._broadcast_camera_list() + + # Confirm registration + await self._send_message(connection_id, { + "type": "camera-registered", + "sourceId": source_id, + "payload": {"status": "success"}, + "timestamp": int(asyncio.get_event_loop().time() * 1000) + }) + + async def _handle_offer(self, connection_id: str, message: Dict[str, Any]): + """Handle WebRTC offer""" + source_id = message.get("sourceId") + target_id = message.get("targetId") + + if target_id and target_id in self.connections: + # Forward offer to target connection (for remote cameras) + await self._send_message(target_id, message) + else: + # For local cameras, this might be an offer to the server + # We can handle it here or forward to all other connections + logger.info(f"📤 Broadcasting offer for camera {source_id}") + await self._broadcast_message(message, exclude=connection_id) + + async def _handle_answer(self, connection_id: str, message: Dict[str, Any]): + """Handle WebRTC answer""" + source_id = message.get("sourceId") + target_id = message.get("targetId") + + if target_id and target_id in self.connections: + # Forward answer to target connection + await self._send_message(target_id, message) + else: + # Broadcast answer + await self._broadcast_message(message, exclude=connection_id) + + async def _handle_ice_candidate(self, connection_id: str, message: Dict[str, Any]): + """Handle ICE candidate""" + source_id = message.get("sourceId") + target_id = message.get("targetId") + + if target_id and target_id in self.connections: + # Forward ICE candidate to target connection + await self._send_message(target_id, message) + else: + # Broadcast ICE candidate + await self._broadcast_message(message, exclude=connection_id) + + async def _handle_camera_list_request(self, connection_id: str): + """Send available camera list to requesting connection""" + camera_list = [] + for source_id, camera_info in self.camera_sources.items(): + camera_list.append({ + "id": source_id, + "name": camera_info["name"], + "type": camera_info["type"], + "available": camera_info["status"] == "available", + "width": camera_info["width"], + "height": camera_info["height"], + "fps": camera_info["fps"] + }) + + await self._send_message(connection_id, { + "type": "camera-list", + "sourceId": "server", + "payload": {"cameras": camera_list}, + "timestamp": int(asyncio.get_event_loop().time() * 1000) + }) + + async def _handle_camera_stream_request(self, connection_id: str, message: Dict[str, Any]): + """Handle request for camera stream""" + requested_source_id = message.get("payload", {}).get("sourceId") + + if requested_source_id in self.camera_sources: + camera_info = self.camera_sources[requested_source_id] + owner_connection = camera_info["owner_connection"] + + if owner_connection in self.connections: + # Forward stream request to camera owner + stream_request = { + "type": "stream-request", + "sourceId": requested_source_id, + "targetId": connection_id, + "payload": {"requesterId": connection_id}, + "timestamp": int(asyncio.get_event_loop().time() * 1000) + } + await self._send_message(owner_connection, stream_request) + else: + await self._send_error(connection_id, f"Camera owner not connected") + else: + await self._send_error(connection_id, f"Camera {requested_source_id} not found") + + async def _broadcast_camera_list(self): + """Broadcast updated camera list to all connections""" + camera_list = [] + for source_id, camera_info in self.camera_sources.items(): + camera_list.append({ + "id": source_id, + "name": camera_info["name"], + "type": camera_info["type"], + "available": camera_info["status"] == "available" + }) + + broadcast_message = { + "type": "camera-list-update", + "sourceId": "server", + "payload": {"cameras": camera_list}, + "timestamp": int(asyncio.get_event_loop().time() * 1000) + } + + await self._broadcast_message(broadcast_message) + + async def _send_message(self, connection_id: str, message: Dict[str, Any]): + """Send message to specific connection""" + if connection_id in self.connections: + try: + websocket = self.connections[connection_id] + await websocket.send_text(json.dumps(message)) + except Exception as e: + logger.error(f"❌ Failed to send message to {connection_id}: {e}") + # Remove dead connection + self.disconnect(connection_id) + + async def _broadcast_message(self, message: Dict[str, Any], exclude: Optional[str] = None): + """Broadcast message to all connections except excluded one""" + dead_connections = [] + + for connection_id, websocket in self.connections.items(): + if connection_id == exclude: + continue + + try: + await websocket.send_text(json.dumps(message)) + except Exception as e: + logger.error(f"❌ Failed to broadcast to {connection_id}: {e}") + dead_connections.append(connection_id) + + # Clean up dead connections + for dead_id in dead_connections: + self.disconnect(dead_id) + + async def _send_error(self, connection_id: str, error_message: str): + """Send error message to connection""" + error_msg = { + "type": "error", + "sourceId": "server", + "payload": {"error": error_message}, + "timestamp": int(asyncio.get_event_loop().time() * 1000) + } + await self._send_message(connection_id, error_msg) + + def get_stats(self) -> Dict[str, Any]: + """Get signaling server statistics""" + return { + "total_connections": len(self.connections), + "total_cameras": len(self.camera_sources), + "local_cameras": len([c for c in self.camera_sources.values() if c["type"] == "local"]), + "remote_cameras": len([c for c in self.camera_sources.values() if c["type"] == "remote"]), + "active_cameras": len([c for c in self.camera_sources.values() if c["status"] == "available"]) + } + +# Global signaling server instance +signaling_server = WebRTCSignalingServer() \ No newline at end of file From e92f9e2c290cf85c07f5d9164076ec6ce802ba2d Mon Sep 17 00:00:00 2001 From: David <17435126+DavidLMS@users.noreply.github.com> Date: Wed, 25 Jun 2025 13:38:18 +0200 Subject: [PATCH 5/5] cameras working --- app/config.py | 54 +++ app/main.py | 847 +++++++++++++++++++++++++++++++++++++++- app/webrtc_signaling.py | 255 +++++++++++- scripts/backend.py | 68 +++- 4 files changed, 1217 insertions(+), 7 deletions(-) diff --git a/app/config.py b/app/config.py index fbe58e4..d7d9695 100644 --- a/app/config.py +++ b/app/config.py @@ -31,6 +31,60 @@ CAMERA_CONFIG_PATH = os.path.expanduser("~/.cache/huggingface/lerobot/camera_configs") CAMERA_CONFIG_FILE = os.path.join(CAMERA_CONFIG_PATH, "camera_config.json") +# Define external URL configuration storage path +EXTERNAL_URL_FILE = os.path.join(CONFIG_STORAGE_PATH, "external_url.txt") + +# Global variable to store current external URL +_external_url = None + +def set_external_url(url: str): + """Set the external URL for QR code generation (ngrok, etc.)""" + global _external_url + _external_url = url.rstrip('/') if url else None + + # Ensure directory exists + os.makedirs(CONFIG_STORAGE_PATH, exist_ok=True) + + # Save to file for persistence + try: + with open(EXTERNAL_URL_FILE, 'w') as f: + f.write(url if url else '') + logger.info(f"External URL saved: {url}") + except Exception as e: + logger.error(f"Failed to save external URL: {e}") + +def get_external_url() -> str: + """Get the current external URL for QR code generation""" + global _external_url + + # Return cached value if available + if _external_url is not None: + return _external_url + + # Try to load from file + try: + if os.path.exists(EXTERNAL_URL_FILE): + with open(EXTERNAL_URL_FILE, 'r') as f: + url = f.read().strip() + _external_url = url if url else None + return _external_url + except Exception as e: + logger.error(f"Failed to load external URL: {e}") + + return None + +def clear_external_url(): + """Clear the external URL configuration""" + global _external_url + _external_url = None + + try: + if os.path.exists(EXTERNAL_URL_FILE): + os.remove(EXTERNAL_URL_FILE) + logger.info("External URL configuration cleared") + except Exception as e: + logger.error(f"Failed to clear external URL: {e}") + def setup_calibration_files(leader_config: str, follower_config: str): """Setup calibration files in the correct locations for teleoperation and recording""" # Extract config names from file paths (remove .json extension) diff --git a/app/main.py b/app/main.py index eb5e900..cdd7d67 100644 --- a/app/main.py +++ b/app/main.py @@ -1,6 +1,6 @@ from fastapi import FastAPI, WebSocket, WebSocketDisconnect from fastapi.staticfiles import StaticFiles -from fastapi.responses import FileResponse +from fastapi.responses import FileResponse, HTMLResponse from fastapi.middleware.cors import CORSMiddleware import os import logging @@ -110,6 +110,9 @@ save_robot_port, get_saved_robot_port, get_default_robot_port, + set_external_url, + get_external_url, + clear_external_url, ) @@ -250,6 +253,42 @@ def stop_teleoperation(): return handle_stop_teleoperation() +@app.post("/api/config/external-url") +def configure_external_url(data: dict): + """Configure external URL for QR code generation (ngrok, tunneling, etc.)""" + try: + external_url = data.get("external_url", "").strip() + + if external_url: + # Validate URL format + if not external_url.startswith(("http://", "https://")): + return {"success": False, "error": "URL must start with http:// or https://"} + + set_external_url(external_url) + logger.info(f"✅ External URL configured: {external_url}") + return {"success": True, "external_url": external_url} + else: + # Clear external URL configuration + clear_external_url() + logger.info("✅ External URL configuration cleared") + return {"success": True, "external_url": None} + + except Exception as e: + logger.error(f"❌ Failed to configure external URL: {e}") + return {"success": False, "error": str(e)} + + +@app.get("/api/config/external-url") +def get_current_external_url(): + """Get the current external URL configuration""" + try: + external_url = get_external_url() + return {"success": True, "external_url": external_url} + except Exception as e: + logger.error(f"❌ Failed to get external URL: {e}") + return {"success": False, "error": str(e)} + + @app.get("/teleoperation-status") def teleoperation_status(): """Get the current teleoperation status""" @@ -915,6 +954,812 @@ def remove_camera_config_endpoint(camera_name: str): return {"status": "error", "message": str(e)} +@app.get("/remote_cam/{session_id}", response_class=HTMLResponse) +async def external_camera_page(session_id: str): + """Serve the external camera page for WebRTC connection""" + + # HTML page for external camera capture + html_content = f""" + + + + + + External Camera - LeLab + + + +
+
+

External Camera

+
LeLab External Camera Connection
+
+ +
+ + Connecting to session... +
+ +
+ + + +
+ +
+
📹
+
+ +
+ Instructions: +
• Allow camera access when prompted +
• Tap "Start Camera" to begin streaming +
• Keep this page open while using the robot +
+
+ + + + + + + + """ + + return HTMLResponse(content=html_content) + + @app.get("/cameras/stream/{camera_identifier}") def stream_camera(camera_identifier: str): """Stream a specific camera feed using hash or name""" diff --git a/app/webrtc_signaling.py b/app/webrtc_signaling.py index 4b10d03..20da024 100644 --- a/app/webrtc_signaling.py +++ b/app/webrtc_signaling.py @@ -3,12 +3,27 @@ import json import logging import asyncio +import socket from typing import Dict, Set, Optional, Any from fastapi import WebSocket, WebSocketDisconnect import uuid logger = logging.getLogger(__name__) +def get_local_ip(): + """Get the local IP address of this machine""" + try: + # Connect to a remote address to determine the local IP + # This doesn't actually send data, just determines the local IP used for routing + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + s.connect(("8.8.8.8", 80)) + local_ip = s.getsockname()[0] + s.close() + return local_ip + except Exception as e: + logger.warning(f"Could not determine local IP, using localhost: {e}") + return "localhost" + class WebRTCSignalingServer: """WebRTC signaling server for camera streaming""" @@ -24,6 +39,27 @@ def __init__(self): # Connection metadata self.connection_metadata: Dict[str, Dict[str, Any]] = {} + + # External camera sessions (legacy: phone_sessions for backward compatibility) + self.external_sessions: Dict[str, Dict[str, Any]] = {} + self.phone_sessions: Dict[str, Dict[str, Any]] = {} # Legacy support + + def _get_session(self, session_id: str) -> Dict[str, Any]: + """Get session from either external_sessions or legacy phone_sessions""" + return self.external_sessions.get(session_id) or self.phone_sessions.get(session_id) + + def _set_session(self, session_id: str, session_data: Dict[str, Any]): + """Set session, preferring external_sessions for new sessions""" + if session_id.startswith('external_'): + self.external_sessions[session_id] = session_data + else: + # Legacy support for phone_ prefixed sessions + self.phone_sessions[session_id] = session_data + + def _remove_session(self, session_id: str): + """Remove session from both locations""" + self.external_sessions.pop(session_id, None) + self.phone_sessions.pop(session_id, None) async def connect(self, websocket: WebSocket) -> str: """Accept a new WebSocket connection and return connection ID""" @@ -82,6 +118,10 @@ async def handle_message(self, connection_id: str, message: Dict[str, Any]): await self._handle_camera_list_request(connection_id) elif message_type == "request-camera-stream": await self._handle_camera_stream_request(connection_id, message) + elif message_type == "create-session": + await self._handle_create_session(connection_id, message) + elif message_type == "join-session": + await self._handle_join_session(connection_id, message) else: logger.warning(f"⚠️ Unknown message type: {message_type}") @@ -130,7 +170,24 @@ async def _handle_offer(self, connection_id: str, message: Dict[str, Any]): source_id = message.get("sourceId") target_id = message.get("targetId") - if target_id and target_id in self.connections: + logger.info(f"📤 Processing offer from {connection_id} for session {source_id}") + + # Check if this is an external camera session offer + session = self._get_session(source_id) + if session: + desktop_client = session.get("desktop_client") + + logger.info(f"📤 External camera offer for session {source_id}, forwarding to desktop {desktop_client}") + logger.info(f"📤 Session status: {session.get('status')}") + logger.info(f"📤 External device client: {session.get('external_client') or session.get('phone_client')}") + + if desktop_client and desktop_client in self.connections: + # Forward offer to desktop client + await self._send_message(desktop_client, message) + else: + logger.error(f"❌ Desktop client {desktop_client} not found for session {source_id}") + await self._send_error(connection_id, f"Desktop client not connected for session {source_id}") + elif target_id and target_id in self.connections: # Forward offer to target connection (for remote cameras) await self._send_message(target_id, message) else: @@ -144,7 +201,22 @@ async def _handle_answer(self, connection_id: str, message: Dict[str, Any]): source_id = message.get("sourceId") target_id = message.get("targetId") - if target_id and target_id in self.connections: + logger.info(f"📥 Processing answer from {connection_id} for session {source_id}") + + # Check if this is an external camera session answer + session = self._get_session(source_id) + if session: + external_client = session.get("external_client") or session.get("phone_client") # Legacy support + + logger.info(f"📥 Desktop answer for session {source_id}, forwarding to external device {external_client}") + + if external_client and external_client in self.connections: + # Forward answer to external device client + await self._send_message(external_client, message) + else: + logger.error(f"❌ External device client {external_client} not found for session {source_id}") + await self._send_error(connection_id, f"External device client not connected for session {source_id}") + elif target_id and target_id in self.connections: # Forward answer to target connection await self._send_message(target_id, message) else: @@ -155,8 +227,39 @@ async def _handle_ice_candidate(self, connection_id: str, message: Dict[str, Any """Handle ICE candidate""" source_id = message.get("sourceId") target_id = message.get("targetId") + candidate_info = message.get("payload", {}).get("candidate", {}) + + logger.info(f"🧊 Processing ICE candidate from {connection_id} for session {source_id}") + logger.info(f"🧊 Candidate type: {candidate_info.get('type', 'unknown')}") - if target_id and target_id in self.connections: + # Check if this is an external camera session ICE candidate + session = self._get_session(source_id) + if session: + external_client = session.get("external_client") or session.get("phone_client") # Legacy support + + # Determine if this is from external device or desktop + if connection_id == external_client: + # From external device to desktop + desktop_client = session.get("desktop_client") + logger.info(f"🧊 External device ICE candidate for session {source_id}, forwarding to desktop {desktop_client}") + + if desktop_client and desktop_client in self.connections: + await self._send_message(desktop_client, message) + else: + logger.error(f"❌ Desktop client {desktop_client} not found for ICE candidate") + + elif connection_id == session.get("desktop_client"): + # From desktop to external device + logger.info(f"🧊 Desktop ICE candidate for session {source_id}, forwarding to external device {external_client}") + + if external_client and external_client in self.connections: + await self._send_message(external_client, message) + else: + logger.error(f"❌ External device client {external_client} not found for ICE candidate") + else: + logger.warning(f"⚠️ ICE candidate from unknown connection {connection_id} for session {source_id}") + + elif target_id and target_id in self.connections: # Forward ICE candidate to target connection await self._send_message(target_id, message) else: @@ -207,6 +310,152 @@ async def _handle_camera_stream_request(self, connection_id: str, message: Dict[ else: await self._send_error(connection_id, f"Camera {requested_source_id} not found") + async def _handle_create_session(self, connection_id: str, message: Dict[str, Any]): + """Handle external camera session creation""" + session_id = message.get("sourceId") + payload = message.get("payload", {}) + + if not session_id: + await self._send_error(connection_id, "Session ID is required") + return + + # Check if session already exists + existing_session = self._get_session(session_id) + session_existed = existing_session is not None + + if existing_session: + logger.info(f"🌐 Session {session_id} already exists, updating desktop client and re-sending QR URL") + # Update the desktop client (in case it's a reconnection) + existing_session["desktop_client"] = connection_id + existing_session["name"] = payload.get("name", existing_session.get("name", "External Camera")) + self._set_session(session_id, existing_session) + + # Log session status for debugging + external_client = existing_session.get("external_client") or existing_session.get("phone_client") + logger.info(f"🌐 Existing session status - external_client: {external_client}, status: {existing_session.get('status')}") + else: + # Create new external camera session + session_info = { + "session_id": session_id, + "name": payload.get("name", "External Camera"), + "desktop_client": connection_id, + "external_client": None, + "phone_client": None, # Legacy support + "status": "waiting_for_device", + "created_at": asyncio.get_event_loop().time() + } + + self._set_session(session_id, session_info) + logger.info(f"🌐 Created new external camera session: {session_id}") + + # Generate QR URL - only use external URL (ngrok), no local fallback + from .config import get_external_url + external_url = get_external_url() + + if external_url: + # Use external URL (ngrok, tunneling, etc.) + qr_url = f"{external_url}/remote_cam/{session_id}" + logger.info(f"📱 Using external URL for QR: {external_url}") + + # Send confirmation to desktop with QR URL and session info + session_info = existing_session if existing_session else self._get_session(session_id) + external_client = session_info.get("external_client") or session_info.get("phone_client") if session_info else None + device_connected = external_client is not None and external_client in self.connections + + await self._send_message(connection_id, { + "type": "session-created", + "sourceId": session_id, + "payload": { + "sessionId": session_id, + "qrCodeUrl": qr_url, + "sessionExisted": session_existed, + "deviceConnected": device_connected, + "sessionStatus": session_info.get("status") if session_info else "new" + }, + "timestamp": int(asyncio.get_event_loop().time() * 1000) + }) + else: + # No external URL configured - don't generate local URLs due to HTTPS issues + logger.warning(f"🌐 No external URL configured - external camera requires ngrok setup") + + # Send confirmation to desktop without QR URL (will show ngrok required message) + session_info = existing_session if existing_session else self._get_session(session_id) + external_client = session_info.get("external_client") or session_info.get("phone_client") if session_info else None + device_connected = external_client is not None and external_client in self.connections + + await self._send_message(connection_id, { + "type": "session-created", + "sourceId": session_id, + "payload": { + "sessionId": session_id, + "sessionExisted": session_existed, + "deviceConnected": device_connected, + "sessionStatus": session_info.get("status") if session_info else "new" + # No qrCodeUrl - frontend will show "configure ngrok" message + }, + "timestamp": int(asyncio.get_event_loop().time() * 1000) + }) + + async def _handle_join_session(self, connection_id: str, message: Dict[str, Any]): + """Handle external device joining a session""" + session_id = message.get("sourceId") + payload = message.get("payload", {}) + + session = self._get_session(session_id) + if not session_id or not session: + await self._send_error(connection_id, "Session not found") + return + + # Enhanced debugging for multiple sessions + logger.info(f"🌐 External device joining session: {session_id}") + logger.info(f"🌐 Device details: {payload}") + logger.info(f"🌐 Active sessions: {len(self.external_sessions) + len(self.phone_sessions)}") + logger.info(f"🌐 Total connections: {len(self.connections)}") + + # Check for existing external device connection (Android multi-session issue) + existing_client = session.get("external_client") or session.get("phone_client") # Legacy support + if existing_client and existing_client in self.connections: + logger.warning(f"⚠️ Session {session_id} already has a device connected: {existing_client}") + logger.warning(f"⚠️ New device connection: {connection_id}") + logger.warning(f"⚠️ This might be an Android multi-session conflict") + + session["external_client"] = connection_id + session["phone_client"] = connection_id # Legacy support + session["status"] = "device_connected" + session["device_info"] = { + "userAgent": payload.get("userAgent", "unknown"), + "platform": payload.get("platform", "unknown"), + "webrtcSupported": payload.get("webrtcSupported", False), + "connected_at": asyncio.get_event_loop().time() + } + + logger.info(f"🌐 External device joined session: {session_id} from {payload.get('platform', 'unknown')}") + + # Notify desktop that external device connected + desktop_client = session.get("desktop_client") + if desktop_client and desktop_client in self.connections: + await self._send_message(desktop_client, { + "type": "external-device-connected", + "sourceId": session_id, + "payload": { + "sessionId": session_id, + "deviceInfo": session["device_info"] + }, + "timestamp": int(asyncio.get_event_loop().time() * 1000) + }) + + # Confirm to external device + await self._send_message(connection_id, { + "type": "session-joined", + "sourceId": session_id, + "payload": { + "sessionId": session_id, + "activeSessions": len(self.external_sessions) + len(self.phone_sessions), + "totalConnections": len(self.connections) + }, + "timestamp": int(asyncio.get_event_loop().time() * 1000) + }) + async def _broadcast_camera_list(self): """Broadcast updated camera list to all connections""" camera_list = [] diff --git a/scripts/backend.py b/scripts/backend.py index 325ae95..fc9c6f7 100644 --- a/scripts/backend.py +++ b/scripts/backend.py @@ -5,18 +5,80 @@ import logging import uvicorn +import ssl +import os # Set up logging logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) +def create_ssl_context(cert_file: str, key_file: str): + """Create an SSL context optimized for iOS compatibility""" + try: + # Create SSL context with iOS-compatible settings + ssl_context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) + ssl_context.load_cert_chain(cert_file, key_file) + + # Set minimum and maximum TLS versions for iOS compatibility + ssl_context.minimum_version = ssl.TLSVersion.TLSv1_2 + ssl_context.maximum_version = ssl.TLSVersion.TLSv1_3 + + # Use iOS-compatible cipher suites + ssl_context.set_ciphers('ECDHE+AESGCM:ECDHE+CHACHA20:DHE+AESGCM:DHE+CHACHA20:!aNULL:!MD5:!DSS') + + # Additional iOS-friendly settings + ssl_context.check_hostname = False # For self-signed certificates + ssl_context.verify_mode = ssl.CERT_NONE # For self-signed certificates + + logger.info("✅ SSL context created with iOS-compatible settings") + return ssl_context + + except Exception as e: + logger.error(f"❌ Failed to create SSL context: {e}") + return None + + def main(): """Start the FastAPI backend server only""" logger.info("🚀 Starting LeLab FastAPI backend server...") - uvicorn.run( - "app.main:app", host="0.0.0.0", port=8000, reload=True, log_level="info" - ) + + # Check if SSL certificates exist + cert_file = "cert.pem" + key_file = "key.pem" + + if os.path.exists(cert_file) and os.path.exists(key_file): + logger.info("🔒 SSL certificates found, starting HTTPS server...") + + # Try to create iOS-compatible SSL context + ssl_context = create_ssl_context(cert_file, key_file) + + if ssl_context: + logger.info("🍎 Using iOS-compatible SSL configuration") + uvicorn.run( + "app.main:app", + host="0.0.0.0", + port=8000, + reload=True, + log_level="info", + ssl_context=ssl_context + ) + else: + logger.warning("⚠️ Falling back to basic SSL configuration") + uvicorn.run( + "app.main:app", + host="0.0.0.0", + port=8000, + reload=True, + log_level="info", + ssl_keyfile=key_file, + ssl_certfile=cert_file + ) + else: + logger.info("🔓 No SSL certificates found, starting HTTP server...") + uvicorn.run( + "app.main:app", host="0.0.0.0", port=8000, reload=True, log_level="info" + ) if __name__ == "__main__":