diff --git a/.dockerignore b/.dockerignore index 4ab19f48..e81825de 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,8 @@ # Node node_modules -.npmrc +**/node_modules +apps/*/node_modules +packages/*/node_modules .pnpm-store # Python diff --git a/.gitignore b/.gitignore index a74161b3..58f7f06f 100644 --- a/.gitignore +++ b/.gitignore @@ -30,4 +30,7 @@ test-results/* .env.testing .env.dev docker/.env.system -.env.system \ No newline at end of file +.env.system +docker/docker-compose.override.yml +docker/start.ps1 +docker/stop.ps1 diff --git a/WARP.md b/WARP.md new file mode 100644 index 00000000..9570f52f --- /dev/null +++ b/WARP.md @@ -0,0 +1,121 @@ +# WARP.md + +This file provides guidance to WARP (warp.dev) when working with code in this repository. + +# Project Structure + +This is a monorepo managed by **pnpm**. + +- **apps/** + - `web`: React/Vite application (Frontend). + - `desktop`: Electron application (Desktop Client). + - `background-jobs`: Node.js service (Express, BullMQ) that orchestrates video processing and communicates with the Python service. +- **packages/** + - `prisma`: Database schema, migrations, and client generation. + - `shared`: Shared utilities, types, constants, and services (including `pythonService`). + - `ui`: Shared UI component library. +- **python/**: Python service for AI/ML tasks (Video Analysis, Transcription, Face Recognition). +- **docker/**: Docker Compose configuration and Dockerfiles. + +# Development Setup + +## Prerequisites +- Node.js (v22+) +- pnpm (v10+) +- Python (3.11+) +- Docker & Docker Compose (Recommended for running services like Postgres, Redis, ChromaDB) + +## Initial Setup +1. Install dependencies: + ```bash + pnpm install + ``` +2. Set up environment variables: + - Copy `.env.example` to `.env` in the root. + - Configure `HOST_MEDIA_PATH`, `GEMINI_API_KEY` (if used), etc. +3. Start infrastructure services (Postgres, Redis, ChromaDB): + ```bash + docker compose -f docker/docker-compose.yml up -d postgres redis chroma + ``` +4. Initialize Database: + ```bash + pnpm prisma generate + pnpm prisma migrate dev + pnpm prisma seed + ``` + +## Python Environment (Local Development) +If running `background-jobs` locally (not in Docker), you must set up the Python environment: + +1. Create a virtual environment in `python/venv` (or similar): + ```bash + cd python + python -m venv venv + # Activate venv (Windows: venv\Scripts\activate, Unix: source venv/bin/activate) + pip install -r requirements.txt + cd .. + ``` +2. Update `.env` to point to the local Python setup: + ```ini + PYTHON_SCRIPT="./python/analysis_service.py" + VENV_PATH="./python/venv" + PYTHON_PORT="8765" + ``` + +# Common Commands + +## Running Applications +- **Web App**: + ```bash + pnpm --filter web dev + ``` +- **Desktop App**: + ```bash + pnpm --filter desktop dev + ``` +- **Background Jobs**: + ```bash + pnpm --filter background-jobs dev + ``` + +## Database Management (via `packages/prisma`) +- Generate Client: `pnpm prisma generate` +- Run Migrations: `pnpm prisma migrate dev` +- Seed Database: `pnpm prisma seed` +- Open Studio: `pnpm prisma studio` + +## Testing +- Run all tests: + ```bash + pnpm test + ``` +- Run specific package tests: + ```bash + pnpm --filter web test + pnpm --filter desktop test + pnpm --filter shared test + ``` + +## Linting & Formatting +- Lint: `pnpm --filter lint` +- Format: `pnpm --filter format` (or `prettier --write .`) + +# Architecture & Key Components + +## Video Processing Pipeline +1. **Ingestion**: `background-jobs` watches `HOST_MEDIA_PATH` or receives upload events. +2. **Queue**: BullMQ manages processing jobs (`video-indexing`). +3. **Orchestration**: `videoIndexer.ts` in `background-jobs` coordinates the workflow. +4. **Analysis**: `pythonService` (in `packages/shared`) spawns the Python process and communicates via **WebSockets** (`ws://localhost:8765`). + - **Transcription**: OpenAI Whisper. + - **Vision**: OpenCV, YOLO, Gemini (optional). + - **Embeddings**: Stored in ChromaDB for semantic search. + +## Communication +- **Node <-> Python**: WebSocket connection. The Node.js service manages the Python process lifecycle (spawn/kill). +- **Frontend <-> Backend**: REST API (Express) and likely WebSockets or Polling for job status. + +## Database +- **Postgres**: Stores metadata, video info, jobs, scenes, and faces. Managed by Prisma. +- **ChromaDB**: Vector database for semantic search embeddings. +- **Redis**: Backend for BullMQ job queues. diff --git a/apps/background-jobs/src/jobs/videoIndexer.ts b/apps/background-jobs/src/jobs/videoIndexer.ts index 191893de..8db34f40 100644 --- a/apps/background-jobs/src/jobs/videoIndexer.ts +++ b/apps/background-jobs/src/jobs/videoIndexer.ts @@ -235,9 +235,10 @@ async function processVideo(job: Job<{ videoPath: string; jobId: string; forceRe export const videoIndexerWorker = new Worker('video-indexing', processVideo, { connection, concurrency: 1, - lockDuration: 15 * 60 * 1000, - stalledInterval: 5 * 60 * 1000, - maxStalledCount: 2, + // Extended timeouts for long videos (up to 4 hours) + lockDuration: 4 * 60 * 60 * 1000, // 4 hours - job can run this long + stalledInterval: 30 * 60 * 1000, // Check for stalled jobs every 30 minutes + maxStalledCount: 3, // Allow 3 stall events before marking failed }) videoIndexerWorker.on('failed', async (job: Job | undefined, err: Error) => { diff --git a/apps/background-jobs/src/queue.ts b/apps/background-jobs/src/queue.ts index f46c7f88..79add09f 100644 --- a/apps/background-jobs/src/queue.ts +++ b/apps/background-jobs/src/queue.ts @@ -1,11 +1,64 @@ import { Queue } from 'bullmq' import { config } from './config' import IORedis from 'ioredis' +import { logger } from '@shared/services/logger' +// Resilient Redis connection with exponential backoff +// Addresses Docker Desktop networking instability on Windows/WSL2 export const connection = new IORedis({ host: config.redisHost, port: config.redisPort, - maxRetriesPerRequest: null, + maxRetriesPerRequest: null, // Required by BullMQ + + // Connection resilience settings + retryStrategy: (times: number) => { + // Exponential backoff: 100ms, 200ms, 400ms, ... up to 30s max + const delay = Math.min(Math.pow(2, times) * 100, 30000) + logger.warn(`Redis connection retry #${times}, next attempt in ${delay}ms`) + return delay + }, + + // Reconnect on error + reconnectOnError: (err: Error) => { + const targetErrors = ['READONLY', 'ETIMEDOUT', 'ECONNRESET', 'ECONNREFUSED'] + if (targetErrors.some(e => err.message.includes(e))) { + logger.warn(`Redis reconnecting due to: ${err.message}`) + return true // Reconnect for these errors + } + return false + }, + + // Connection timeouts + connectTimeout: 30000, // 30s to establish connection + commandTimeout: 60000, // 60s for commands (long jobs) + + // Keep-alive settings + keepAlive: 30000, // Send TCP keepalive every 30s + + // Auto-reconnect + enableReadyCheck: true, + enableOfflineQueue: true, // Queue commands while disconnected +}) + +// Log connection events +connection.on('connect', () => { + logger.info('Redis connected') +}) + +connection.on('ready', () => { + logger.info('Redis ready') +}) + +connection.on('error', (err) => { + logger.error(`Redis error: ${err.message}`) +}) + +connection.on('close', () => { + logger.warn('Redis connection closed') +}) + +connection.on('reconnecting', () => { + logger.info('Redis reconnecting...') }) export const videoQueue = new Queue('video-indexing', { diff --git a/apps/background-jobs/src/routes/folders.ts b/apps/background-jobs/src/routes/folders.ts index 6239bf87..dca6b9c0 100644 --- a/apps/background-jobs/src/routes/folders.ts +++ b/apps/background-jobs/src/routes/folders.ts @@ -1,46 +1,52 @@ import express from 'express' import { prisma } from '../services/db' -import { findVideoFiles } from '@shared/utils/videos'; +import { findVideoFiles, findAudioFiles } from '@shared/utils/videos'; import { watchFolder } from '../watcher' import { videoQueue } from 'src/queue' import { getVideosNotEmbedded } from '@shared/services/vectorDb'; const router = express.Router() +function isAudioFolder(p: string): boolean { + const name = p.toLowerCase() + return name.includes('/x_audio') || name.endsWith('/audio') || name.includes('xaudio') +} + router.post('/trigger', async (req, res) => { const { folderPath } = req.body if (!folderPath) return res.status(400).json({ error: 'folderPath required' }) try { - const videos = await findVideoFiles(folderPath) - const uniqueVideos = await getVideosNotEmbedded(videos.map((video) => video.path)) + const useAudio = isAudioFolder(folderPath) + const mediaFiles = useAudio ? await findAudioFiles(folderPath) : await findVideoFiles(folderPath) + const uniqueMedia = await getVideosNotEmbedded(mediaFiles.map((f) => f.path)) const folder = await prisma.folder.update({ where: { path: folderPath }, data: { - videoCount: uniqueVideos.length, + videoCount: uniqueMedia.length, lastScanned: new Date(), }, }) watchFolder(folderPath) - for (const video of uniqueVideos) { + for (const mediaPath of uniqueMedia) { const job = await prisma.job.upsert({ - where: { videoPath: video, id: '' }, + where: { videoPath: mediaPath, id: '' }, create: { - videoPath: video, + videoPath: mediaPath, userId: folder?.userId, folderId: folder.id, }, update: { folderId: folder.id}, }) - await videoQueue.add('index-video', { videoPath: video, jobId: job.id, folderId: folder.id }) + await videoQueue.add('index-video', { videoPath: mediaPath, jobId: job.id, folderId: folder.id }) } res.json({ - message: 'Folder added and videos queued for processing', + message: useAudio ? 'Audio folder scanned and files queued' : 'Folder added and videos queued for processing', folder, - queuedVideos: uniqueVideos.length, + queuedFiles: uniqueMedia.length, }) } catch (error) { console.error(error) diff --git a/apps/background-jobs/src/watcher.ts b/apps/background-jobs/src/watcher.ts index 5428b021..965fd4a4 100644 --- a/apps/background-jobs/src/watcher.ts +++ b/apps/background-jobs/src/watcher.ts @@ -2,15 +2,36 @@ import chokidar from 'chokidar' import path from 'path' import { videoQueue } from './queue.js' import { prisma } from './services/db.js' -import { SUPPORTED_VIDEO_EXTENSIONS } from '@shared/constants/index' +import { SUPPORTED_VIDEO_EXTENSIONS, SUPPORTED_AUDIO_EXTENSIONS, WATCHER_IGNORED } from '@shared/constants/index' import { logger } from '@shared/services/logger.js' +function isAudioFolder(p: string): boolean { + const name = p.toLowerCase() + return name.includes('/x_audio') || name.endsWith('/audio') || name.includes('xaudio') +} + export function watchFolder(folderPath: string) { - const watcher = chokidar.watch(folderPath, { ignored: /^\./, persistent: true, ignoreInitial: true }) + const useAudio = isAudioFolder(folderPath) + const extPattern = useAudio ? SUPPORTED_AUDIO_EXTENSIONS : SUPPORTED_VIDEO_EXTENSIONS + + const watcher = chokidar.watch(folderPath, { + ignored: (p: string) => WATCHER_IGNORED.some((re) => re.test(p)), + persistent: true, + ignoreInitial: true, + depth: 0, // top-level only; avoids heavy recursion (Downloads subdirs, Syncthing internals) + awaitWriteFinish: { stabilityThreshold: 3000, pollInterval: 100 }, + ignorePermissionErrors: true, + atomic: true, + }) + + watcher.on('error', (error) => { + logger.error(`Watcher error for ${folderPath}: ${error.message}`) + // Do not crash. For EIO/ENOMEM we keep the watcher alive; new file events may still arrive. + }) watcher.on('add', async (filePath) => { try { - if (!SUPPORTED_VIDEO_EXTENSIONS.test(filePath)) return + if (!extPattern.test(filePath)) return const folder = await prisma.folder.findFirst({ where: { @@ -25,7 +46,7 @@ export function watchFolder(folderPath: string) { }) await videoQueue.add('index-video', { videoPath: filePath, jobId: job.id, folderId: folder.id }) } catch (error) { - console.error('Error adding new video file while watching for new folder changes: ', error) + logger.error(`Error adding file from watcher for ${folderPath}: ${(error as Error).message}`) } }) } @@ -41,12 +62,12 @@ export async function initializeWatchers() { watchFolder(folder.path) } } catch (error) { - console.error('Failed to initialize watchers:', error) + logger.error(`Failed to initialize watchers: ${(error as Error).message}`) } } export function stopWatcher(folderPath: string) { - const watcher = chokidar.watch(folderPath, { ignored: /^\./, persistent: true, ignoreInitial: true }) + const watcher = chokidar.watch(folderPath, { ignored: (p: string) => WATCHER_IGNORED.some((re) => re.test(p)), persistent: true, ignoreInitial: true }) if (watcher) { watcher.close() } diff --git a/apps/desktop/electron.vite.config.ts b/apps/desktop/electron.vite.config.ts index b2cfaaf2..49aae132 100644 --- a/apps/desktop/electron.vite.config.ts +++ b/apps/desktop/electron.vite.config.ts @@ -25,8 +25,9 @@ export default defineConfig({ input: { main: resolve(__dirname, 'lib/main/main.ts'), }, - external: ['chromadb', '@shared', 'onnxruntime-node', '@ffmpeg-installer/ffmpeg', '@ffprobe-installer/ffprobe', 'sharp', 'egm96-universal'], + external: ['chromadb', '@shared', 'onnxruntime-node', '@ffmpeg-installer/ffmpeg', '@ffprobe-installer/ffprobe', 'sharp', 'egm96-universal', 'node-llama-cpp'], }, + outDir: 'dist', }, }, preload: { diff --git a/apps/desktop/package.json b/apps/desktop/package.json index 4dd30930..114f1b90 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -2,14 +2,14 @@ "name": "desktop", "version": "0.1.1", "description": "AI-Powered Video Indexing and Search", - "main": "./out/main/main.js", + "main": "./dist/main.js", "license": "MIT", "author": { "name": "ilias", "url": "https://github.com/iliashad" }, "scripts": { - "dev": "pnpm run build:shared && cross-env ELECTRON_DISABLE_SANDBOX=1 electron-vite dev -w", + "dev": "cross-env ELECTRON_DISABLE_SANDBOX=1 electron-vite dev -w --config electron.vite.config.ts", "format": "prettier --write .", "lint": "eslint . --ext .ts,.tsx --fix", "start": "electron-vite preview", diff --git a/docker-compose.yml b/docker-compose.yml index 1f23890e..1fab11eb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,32 +1,53 @@ + services: background-jobs: - image: ghcr.io/iliashad/edit-mind-background-jobs:${VERSION:-latest} - container_name: edit-mind-background-jobs + build: + target: development + context: . + dockerfile: docker/Dockerfile.background-jobs + args: + NODE_VERSION: 22.20.0 + PNPM_VERSION: 10.20.0 + image: edit-mind-background-jobs:dev + container_name: edit-mind-background-jobs-dev ports: - "${BACKGROUND_JOBS_PORT:-4000}:4000" - "${BACKGROUND_JOBS_WS_PORT:-8765}:8765" depends_on: postgres: condition: service_healthy - redis: - condition: service_healthy chroma: condition: service_started + redis: + condition: service_started env_file: - .env - - .env.system + - docker/.env.system + environment: + NODE_ENV: development networks: - app-network volumes: - - ${HOST_MEDIA_PATH:-./media}:/media/videos:ro - - whisper-models:/app/models:rw - - yolo-models:/app/models/yolo:rw + - ./apps/background-jobs:/app/apps/background-jobs + - ./packages:/app/packages + - ./python:/app/python + - ./data:/app/data + - ./tsconfig.json:/app/tsconfig.json + - ${HOST_MEDIA_PATH:-./media}:/media/videos + - whisper-models:/app/models + - yolo-models:/app/models/yolo + - background_jobs_node_modules:/app/node_modules web: - image: ghcr.io/iliashad/edit-mind-web:${VERSION:-latest} - container_name: edit-mind-web - environment: - NODE_ENV: "production" + container_name: edit-mind-web-dev + build: + context: . + dockerfile: docker/Dockerfile.web + target: development + args: + NODE_VERSION: 22.20.0 + PNPM_VERSION: 10.20.0 + image: edit-mind-web:dev ports: - "${WEB_PORT:-3745}:3745" depends_on: @@ -35,12 +56,22 @@ services: chroma: condition: service_started redis: - condition: service_healthy + condition: service_started env_file: - .env - - .env.system + - docker/.env.system + environment: + NODE_ENV: development + DATABASE_URL: postgresql://user:password@postgres:5432/app + CHROMA_URL: http://chroma:8000 + REDIS_URL: redis://redis:6379 volumes: - - ${HOST_MEDIA_PATH:-./media}:/media/videos:ro + - ./apps/web:/app/apps/web + - ./packages:/app/packages + - ./data:/app/data + - ./tsconfig.json:/app/tsconfig.json + - ${HOST_MEDIA_PATH:-./media}:/media/videos + - web_node_modules:/app/node_modules networks: - app-network @@ -50,20 +81,27 @@ services: restart: unless-stopped environment: CHROMA_CORS_ALLOW_ORIGINS: '["http://localhost:3745", "http://localhost:4000", "http://web:3745", "http://background-jobs:4000"]' + CHROMA_HOST: ${CHROMA_HOST:-0.0.0.0} + CHROMA_PORT: ${CHROMA_PORT:-8000} ports: - "${CHROMA_PORT:-8000}:8000" volumes: - - chroma-data:/chroma/chroma + - ./.chroma_db:/data networks: - app-network + healthcheck: + test: [ "CMD", "curl", "-f", "http://localhost:8000/api/v1/heartbeat" ] + interval: 30s + timeout: 10s + retries: 3 redis: - image: redis:7 - container_name: edit-mind-redis + image: redis:7-alpine + container_name: edit-mind-redis-dev ports: - "${REDIS_PORT:-6379}:6379" volumes: - - redis_data:/data + - redis_data_dev:/data networks: - app-network healthcheck: @@ -74,8 +112,7 @@ services: postgres: image: postgres:16-alpine - container_name: edit-mind-postgres - restart: unless-stopped + container_name: edit-mind-postgres-dev ports: - "${POSTGRES_PORT:-5433}:5432" environment: @@ -84,7 +121,7 @@ services: POSTGRES_DB: ${POSTGRES_DB:-app} PGDATA: /var/lib/postgresql/data/pgdata volumes: - - pgdata_edit_mind:/var/lib/postgresql/data + - pgdata_dev:/var/lib/postgresql/data networks: - app-network healthcheck: @@ -92,18 +129,22 @@ services: interval: 10s timeout: 5s retries: 5 + shm_size: 128mb volumes: - pgdata_edit_mind: + pgdata_dev: driver: local - redis_data: + redis_data_dev: driver: local whisper-models: driver: local yolo-models: driver: local - chroma-data: + web_node_modules: + driver: local + background_jobs_node_modules: driver: local networks: - app-network: \ No newline at end of file + app-network: + driver: bridge diff --git a/docker/.env.system.example b/docker/.env.system.example new file mode 100644 index 00000000..41b1c271 --- /dev/null +++ b/docker/.env.system.example @@ -0,0 +1,7 @@ +# System-specific environment variables +# Copy this file to .env.system and fill in your values + +# SMB/CIFS mount credentials (optional - for network drive access) +SMB_USERNAME=your_username +SMB_PASSWORD=your_password +SMB_DEVICE=//your-server/share diff --git a/docker/Dockerfile.background-jobs b/docker/Dockerfile.background-jobs index 6aa97cc8..8647add5 100644 --- a/docker/Dockerfile.background-jobs +++ b/docker/Dockerfile.background-jobs @@ -1,36 +1,34 @@ ARG NODE_VERSION=22.20.0 -ARG PNPM_VERSION=10.20.0 +ARG PNPM_VERSION=10.13.1 +ARG CUDA_VERSION=12.4.1 -FROM node:${NODE_VERSION}-bookworm-slim AS base +# Use NVIDIA CUDA base image for GPU support +FROM nvidia/cuda:${CUDA_VERSION}-cudnn-runtime-ubuntu22.04 AS base +ARG NODE_VERSION ARG PNPM_VERSION -ENV PNPM_HOME="/pnpm" +# Install Node.js +RUN apt-get update && apt-get install -y --no-install-recommends \ + curl \ + ca-certificates \ + && curl -fsSL https://deb.nodesource.com/setup_22.x | bash - \ + && apt-get install -y nodejs \ + && rm -rf /var/lib/apt/lists/* + +ENV PNPM_HOME="/pnpm" ENV PATH="$PNPM_HOME:$PATH" +ENV NPM_CONFIG_NODE_LINKER=hoisted RUN corepack enable && corepack prepare pnpm@${PNPM_VERSION} --activate WORKDIR /app -RUN mkdir -p /app/models /app/models/yolo /tmp - - - -FROM base AS node-deps - -COPY pnpm-workspace.yaml pnpm-lock.yaml package.json prisma.config.ts ./ -COPY apps/background-jobs/package.json ./apps/background-jobs/ -COPY apps/web/package.json ./apps/web/ -COPY packages/shared/package.json ./packages/shared/ -COPY packages/prisma ./packages/prisma/ +RUN mkdir -p /app/models /app/models/yolo /tmp -RUN --mount=type=cache,id=pnpm,target=/pnpm/store \ - pnpm install --frozen-lockfile - - -FROM base AS python-deps +FROM base AS dependencies RUN apt-get update && apt-get install -y --no-install-recommends \ - python3=3.11.* \ + python3 \ python3-venv \ python3-pip \ build-essential \ @@ -45,63 +43,69 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && rm -rf /var/lib/apt/lists/* \ && apt-get clean +COPY pnpm-workspace.yaml pnpm-lock.yaml package.json prisma.config.ts .npmrc ./ +COPY apps/background-jobs/package.json ./apps/background-jobs/ +COPY apps/web/package.json ./apps/web/ +COPY packages/shared/package.json ./packages/shared/ +COPY packages/prisma ./packages/prisma/ -RUN python3 -m venv /app/.venv -ENV PATH="/app/.venv/bin:$PATH" \ - VIRTUAL_ENV="/app/.venv" +RUN --mount=type=cache,id=pnpm,target=/pnpm/store \ + pnpm install -COPY python/requirements.txt ./python/ +FROM dependencies AS python-deps -RUN --mount=type=cache,target=/root/.cache/pip \ - pip install --upgrade pip setuptools wheel +COPY python/requirements.txt ./python/ -RUN --mount=type=cache,target=/root/.cache/pip \ - pip install face-recognition +RUN python3 -m venv /app/.venv -RUN --mount=type=cache,target=/root/.cache/pip \ - pip install -r python/requirements.txt +ENV PATH="/app/.venv/bin:$PATH" +ENV VIRTUAL_ENV="/app/.venv" +RUN --mount=type=cache,target=/root/.cache/pip \ + pip install --no-cache-dir --upgrade pip setuptools wheel && \ + pip install --no-cache-dir -r python/requirements.txt -FROM node-deps AS builder +FROM python-deps AS builder -COPY --from=node-deps ./app/node_modules /app/node_modules -COPY apps ./apps -COPY packages ./packages -COPY python ./python -COPY tsconfig.json ./ +COPY apps ./apps +COPY packages ./packages +COPY python ./python +COPY tsconfig.json ./ RUN pnpm rebuild @tailwindcss/oxide rollup sharp -RUN pnpm --filter shared build + RUN pnpm --filter background-jobs build FROM base AS production RUN apt-get update && apt-get install -y --no-install-recommends \ - python3=3.11.* \ + python3 \ python3-venv \ - libopenblas0 \ - liblapack3 \ - libx11-6 \ - libgtk-3-0 \ - libboost-python1.74.0 \ - libboost-thread1.74.0 \ - libboost-filesystem1.74.0 \ + python3-pip \ + build-essential \ + cmake \ + libopenblas-dev \ + liblapack-dev \ + libx11-dev \ + libgtk-3-dev \ + libboost-all-dev \ ffmpeg \ - libsm6 \ - libxext6 \ - libxrender1 \ - libgomp1 \ && ln -sf /usr/bin/python3 /usr/bin/python \ && rm -rf /var/lib/apt/lists/* \ && apt-get clean -COPY --from=builder /app /app -COPY --from=python-deps ./app/.venv /app/.venv -COPY --from=python-deps ./app/python /app/python +# Set CUDA environment variables +ENV NVIDIA_VISIBLE_DEVICES=all +ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility + + +COPY --from=builder /app /app ENV NODE_ENV=production \ PATH="/app/.venv/bin:$PATH" \ - VIRTUAL_ENV="/app/.venv" + VIRTUAL_ENV="/app/.venv" + +RUN pnpm --filter background-jobs build WORKDIR /app/ @@ -115,11 +119,33 @@ CMD ["sh", "-c", "\ "] -FROM builder AS development +FROM base AS development + +RUN apt-get update && apt-get install -y --no-install-recommends \ + python3 \ + python3-venv \ + python3-pip \ + build-essential \ + cmake \ + libopenblas-dev \ + liblapack-dev \ + libx11-dev \ + libgtk-3-dev \ + libboost-all-dev \ + ffmpeg \ + && ln -sf /usr/bin/python3 /usr/bin/python \ + && rm -rf /var/lib/apt/lists/* \ + && apt-get clean + +ENV NVIDIA_VISIBLE_DEVICES=all +ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility + + +COPY --from=builder /app /app ENV NODE_ENV=development \ PATH="/app/.venv/bin:$PATH" \ - VIRTUAL_ENV="/app/.venv" + VIRTUAL_ENV="/app/.venv" RUN pnpm --filter background-jobs build diff --git a/docker/Dockerfile.web b/docker/Dockerfile.web index 6f45df1d..b8303355 100644 --- a/docker/Dockerfile.web +++ b/docker/Dockerfile.web @@ -1,5 +1,5 @@ ARG NODE_VERSION=22.20.0 -ARG PNPM_VERSION=10.20.0 +ARG PNPM_VERSION=10.13.1 FROM node:${NODE_VERSION}-slim AS base @@ -7,6 +7,7 @@ ARG PNPM_VERSION ENV PNPM_HOME="/pnpm" ENV PATH="$PNPM_HOME:$PATH" +ENV NPM_CONFIG_NODE_LINKER=hoisted RUN corepack enable && corepack prepare pnpm@${PNPM_VERSION} --activate WORKDIR /app @@ -19,14 +20,14 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ curl \ && rm -rf /var/lib/apt/lists/* -COPY pnpm-workspace.yaml pnpm-lock.yaml package.json prisma.config.ts ./ +COPY pnpm-workspace.yaml pnpm-lock.yaml package.json prisma.config.ts .npmrc ./ COPY apps/web/package.json ./apps/web/ COPY apps/background-jobs/package.json ./apps/background-jobs/ COPY packages/shared/package.json ./packages/shared/ COPY packages/prisma ./packages/prisma/ RUN --mount=type=cache,id=pnpm,target=/pnpm/store \ - pnpm install --frozen-lockfile + pnpm install FROM dependencies AS builder @@ -41,7 +42,7 @@ RUN pnpm rebuild @tailwindcss/oxide rollup sharp RUN pnpm --filter shared build RUN --mount=type=cache,id=pnpm,target=/pnpm/store \ - pnpm install --frozen-lockfile + pnpm install RUN pnpm --filter web build @@ -71,7 +72,7 @@ COPY --from=builder /app/packages/shared/dist ./packages/shared/dist COPY --from=builder /app/packages/shared/package.json ./packages/shared/ RUN --mount=type=cache,id=pnpm,target=/pnpm/store \ - pnpm install --frozen-lockfile + pnpm install COPY --from=builder /app/apps/web ./apps/web @@ -116,7 +117,7 @@ COPY --from=builder /app/packages/prisma /app/packages/prisma COPY --from=builder /app/packages/shared/dist /app/packages/shared/dist RUN --mount=type=cache,id=pnpm,target=/pnpm/store \ - pnpm install --frozen-lockfile + pnpm install COPY --from=builder /app/apps/web/build /app/apps/web/build COPY --from=builder /app/apps/web/app ./apps/web/app diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index a568fd0c..2904e47f 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -12,6 +12,14 @@ services: ports: - "${BACKGROUND_JOBS_PORT:-4000}:4000" - "${BACKGROUND_JOBS_WS_PORT:-8765}:8765" + # Optional GPU acceleration (set GPU_COUNT=-1 for all GPUs, or specific count) + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: ${GPU_COUNT:-0} + capabilities: [gpu] depends_on: postgres: condition: service_healthy diff --git a/packages/prisma/package.json b/packages/prisma/package.json index c5fec216..6dd2db5c 100644 --- a/packages/prisma/package.json +++ b/packages/prisma/package.json @@ -25,13 +25,13 @@ }, "dependencies": { "@prisma/client": "6.19.0", - "bcryptjs": "^2.4.3" - }, - "devDependencies": { - "@types/bcryptjs": "^2.4.6", - "@types/node": "^22.0.0", + "bcryptjs": "^2.4.3", "prisma": "6.19.0", "ts-node": "^10.9.2", "typescript": "5.9.2" + }, + "devDependencies": { + "@types/bcryptjs": "^2.4.6", + "@types/node": "^22.0.0" } } \ No newline at end of file diff --git a/packages/shared/constants/index.ts b/packages/shared/constants/index.ts index 503e5472..5ed7d2d2 100644 --- a/packages/shared/constants/index.ts +++ b/packages/shared/constants/index.ts @@ -46,6 +46,15 @@ export const USE_LOCAL = process.env.USE_LOCAL_MODEL === 'true' export const GEMINI_MODEL_NAME = 'gemini-2.5-pro' // Files export const SUPPORTED_VIDEO_EXTENSIONS = /\.(mp4|mov|avi|mkv)$/i +export const SUPPORTED_AUDIO_EXTENSIONS = /\.(mp3|m4a|aac|flac|wav|ogg|opus|mka)$/i +export const SUPPORTED_MEDIA_EXTENSIONS = /\.(mp4|mov|avi|mkv|mp3|m4a|aac|flac|wav|ogg|opus|mka)$/i +// Common temporary/ignored patterns for watchers (Syncthing, partial downloads, hidden) +export const WATCHER_IGNORED = [ + /^\./, // dotfiles + /(^|\\|\/)\.st(folder|ignore)/i, // Syncthing markers + /(~$|\.tmp$|\.part$|\.crdownload$|\.partial$)/i, // temp/partials + /(^|\\|\/)node_modules(\\|\/|$)/i, // node_modules +] export const DEFAULT_FPS = 30 export const THUMBNAIL_SCALE = '320:-1' export const THUMBNAIL_QUALITY = '4' @@ -54,7 +63,7 @@ export const BATCH_THUMBNAIL_QUALITY = '3' export const PYTHON_SCRIPT = path.resolve(process.env.PYTHON_SCRIPT || './python') export const VENV_PATH = path.resolve(process.env.VENV_PATH || './venv') -export const MEDIA_BASE_PATH = '/media/videos' +export const MEDIA_BASE_PATH = '/media' export const STITCHED_VIDEOS_DIR = process.env.STITCHED_VIDEOS_DIR diff --git a/packages/shared/services/pythonService.ts b/packages/shared/services/pythonService.ts index a980b8af..49f558eb 100644 --- a/packages/shared/services/pythonService.ts +++ b/packages/shared/services/pythonService.ts @@ -284,7 +284,7 @@ class PythonService { this.client = null } reject(new Error('WebSocket connection timeout')) - }, 3000) + }, 10000) // Increased timeout for slow starts this.client = new WebSocket(this.serviceUrl) @@ -299,11 +299,17 @@ class PythonService { const message = JSON.parse(data.toString()) const { type, payload, job_id } = message + // Handle ping/pong heartbeat from Python service + if (type === 'ping') { + this.client?.send(JSON.stringify({ type: 'pong' })) + return + } + const callback = this.messageCallbacks[type as PythonMessageType] if (callback) { callback({ ...payload, job_id }) - } else { + } else if (type !== 'pong') { logger.warn(`⚠️ No callback registered for message type: ${type}`) } } catch (error) { @@ -313,18 +319,41 @@ class PythonService { this.client.on('error', (error) => { clearTimeout(timeout) + logger.error({ error }, 'WebSocket error') this.client = null reject(error) }) - this.client.on('close', () => { + this.client.on('close', (code, reason) => { clearTimeout(timeout) + logger.warn(`WebSocket closed: code=${code}, reason=${reason?.toString() || 'none'}`) this.client = null this.isRunning = false + // Auto-reconnect after unexpected close + if (code !== 1000) { + logger.info('Attempting auto-reconnect in 2 seconds...') + setTimeout(() => { + this.start().catch((err) => logger.error('Auto-reconnect failed:', err)) + }, 2000) + } }) }) } + // Ensure connection is alive, reconnect if needed + public async ensureConnected(): Promise { + if (this.client && this.client.readyState === WebSocket.OPEN) { + return true + } + try { + await this.start() + return true + } catch (error) { + logger.error({ error }, 'Failed to ensure connection') + return false + } + } + private handleCrash() { if (this.restartCount < MAX_RESTARTS) { this.restartCount++ diff --git a/packages/shared/utils/frameAnalyze.ts b/packages/shared/utils/frameAnalyze.ts index 96bae616..ac64c44f 100644 --- a/packages/shared/utils/frameAnalyze.ts +++ b/packages/shared/utils/frameAnalyze.ts @@ -1,6 +1,13 @@ import { Analysis, AnalysisProgress } from '../types/analysis' import { pythonService } from '../services/pythonService' import { logger } from '../services/logger' +import { existsSync, readFileSync, statSync } from 'fs' +import path from 'path' + +// Timeout for analysis based on video complexity +// Long videos with many frames can take a while +const ANALYSIS_TIMEOUT_MS = 4 * 60 * 60 * 1000 // 4 hours +const FILE_CHECK_INTERVAL_MS = 30 * 1000 // Check every 30 seconds /** * Analyzes a video file using the persistent Python analysis service. @@ -13,6 +20,72 @@ export function analyzeVideo( onProgress: (progress: AnalysisProgress) => void ): Promise<{ analysis: Analysis; category: string }> { return new Promise((resolve, reject) => { + let resolved = false + let lastFileSize = 0 + let fileCheckInterval: NodeJS.Timeout | null = null + + // Python saves analysis to analysis_results/_analysis.json + // This is relative to Python's cwd which is /app/apps/background-jobs + const videoBasename = path.basename(videoPath, path.extname(videoPath)) + const analysisResultPath = `/app/apps/background-jobs/analysis_results/${videoBasename}_analysis.json` + + // Parse analysis result and extract category + const parseAnalysisResult = (result: Analysis): { analysis: Analysis; category: string } => { + let category = 'Uncategorized' + if (result?.scene_analysis?.environment) { + const env = result.scene_analysis.environment + category = env.charAt(0).toUpperCase() + env.slice(1).replace(/_/g, ' ') + } + return { analysis: result, category } + } + + // File-based completion check as fallback + const checkFileComplete = (): boolean => { + try { + if (existsSync(analysisResultPath)) { + const stats = statSync(analysisResultPath) + // File exists and hasn't changed in size (analysis complete) + if (stats.size > 0 && stats.size === lastFileSize) { + logger.info(`Analysis file detected as complete: ${analysisResultPath}`) + const content = readFileSync(analysisResultPath, 'utf-8') + const result = JSON.parse(content) as Analysis + cleanup() + if (!resolved) { + resolved = true + resolve(parseAnalysisResult(result)) + } + return true + } + lastFileSize = stats.size + } + } catch (error) { + // File doesn't exist yet or JSON parsing failed, keep waiting + logger.debug(`Analysis file check: ${error}`) + } + return false + } + + const cleanup = () => { + if (fileCheckInterval) { + clearInterval(fileCheckInterval) + fileCheckInterval = null + } + } + + // Start periodic file check as fallback + fileCheckInterval = setInterval(checkFileComplete, FILE_CHECK_INTERVAL_MS) + + // Overall timeout + const timeout = setTimeout(() => { + cleanup() + if (!resolved) { + // Last check before giving up + if (checkFileComplete()) return + resolved = true + reject(new Error(`Analysis timeout after ${ANALYSIS_TIMEOUT_MS / 1000}s`)) + } + }, ANALYSIS_TIMEOUT_MS) + pythonService.analyzeVideo( videoPath, jobId, @@ -26,16 +99,23 @@ export function analyzeVideo( } }, (result) => { - let category = 'Uncategorized' - if (result?.scene_analysis?.environment) { - const env = result.scene_analysis.environment - category = env.charAt(0).toUpperCase() + env.slice(1).replace(/_/g, ' ') + clearTimeout(timeout) + cleanup() + if (!resolved) { + resolved = true + resolve(parseAnalysisResult(result)) } - resolve({ analysis: result, category }) }, (error) => { - logger.error('❌ ERROR CALLBACK EXECUTED: ' + error) - reject(error) + clearTimeout(timeout) + cleanup() + // Before rejecting, check if file was actually created + if (checkFileComplete()) return + if (!resolved) { + resolved = true + logger.error('❌ ERROR CALLBACK EXECUTED: ' + error) + reject(error) + } } ) }) diff --git a/packages/shared/utils/transcribe.ts b/packages/shared/utils/transcribe.ts index 3b14b9d8..12576e62 100644 --- a/packages/shared/utils/transcribe.ts +++ b/packages/shared/utils/transcribe.ts @@ -1,9 +1,15 @@ +import { existsSync, statSync } from 'fs' import { logger } from '../services/logger' import { pythonService } from '../services/pythonService' import { TranscriptionProgress } from '../types/transcription' type ProgressCallback = (progress: TranscriptionProgress) => void +// Timeout for transcription based on video duration estimate +// Most videos transcribe at 10-30x realtime, so 4 hours should cover most cases +const TRANSCRIPTION_TIMEOUT_MS = 4 * 60 * 60 * 1000 // 4 hours +const FILE_CHECK_INTERVAL_MS = 30 * 1000 // Check every 30 seconds + export function transcribeAudio( videoPath: string, jsonFilePath: string, @@ -11,6 +17,54 @@ export function transcribeAudio( onProgress?: ProgressCallback ): Promise { return new Promise((resolve, reject) => { + let resolved = false + let lastFileSize = 0 + let fileCheckInterval: NodeJS.Timeout | null = null + + // File-based completion check as fallback + const checkFileComplete = () => { + try { + if (existsSync(jsonFilePath)) { + const stats = statSync(jsonFilePath) + // File exists and hasn't changed in size (transcription complete) + if (stats.size > 0 && stats.size === lastFileSize) { + logger.info(`Transcription file detected as complete: ${jsonFilePath}`) + cleanup() + if (!resolved) { + resolved = true + resolve() + } + return true + } + lastFileSize = stats.size + } + } catch (error) { + // File doesn't exist yet, keep waiting + } + return false + } + + const cleanup = () => { + if (fileCheckInterval) { + clearInterval(fileCheckInterval) + fileCheckInterval = null + } + } + + // Start periodic file check as fallback + fileCheckInterval = setInterval(checkFileComplete, FILE_CHECK_INTERVAL_MS) + + // Overall timeout + const timeout = setTimeout(() => { + cleanup() + if (!resolved) { + // Last check before giving up + if (checkFileComplete()) return + resolved = true + reject(new Error(`Transcription timeout after ${TRANSCRIPTION_TIMEOUT_MS / 1000}s`)) + } + }, TRANSCRIPTION_TIMEOUT_MS) + pythonService.transcribe( videoPath, jsonFilePath, @@ -25,10 +79,22 @@ export function transcribeAudio( } }, (result) => { - resolve(result) + clearTimeout(timeout) + cleanup() + if (!resolved) { + resolved = true + resolve(result) + } }, (error) => { - reject(error) + clearTimeout(timeout) + cleanup() + // Before rejecting, check if file was actually created + if (checkFileComplete()) return + if (!resolved) { + resolved = true + reject(error) + } } ) }) diff --git a/packages/shared/utils/videos.ts b/packages/shared/utils/videos.ts index 550030c2..1cb02374 100644 --- a/packages/shared/utils/videos.ts +++ b/packages/shared/utils/videos.ts @@ -6,6 +6,7 @@ import { DEFAULT_FPS, MAX_DEPTH, SUPPORTED_VIDEO_EXTENSIONS, + SUPPORTED_AUDIO_EXTENSIONS, THUMBNAIL_QUALITY, THUMBNAIL_SCALE, THUMBNAILS_DIR, @@ -124,6 +125,23 @@ export async function findVideoFiles( dirPath: string, currentDepth: number = 0, maxDepth: number = MAX_DEPTH +): Promise { + return findMediaFiles(dirPath, SUPPORTED_VIDEO_EXTENSIONS, currentDepth, maxDepth) +} + +export async function findAudioFiles( + dirPath: string, + currentDepth: number = 0, + maxDepth: number = MAX_DEPTH +): Promise { + return findMediaFiles(dirPath, SUPPORTED_AUDIO_EXTENSIONS, currentDepth, maxDepth) +} + +export async function findMediaFiles( + dirPath: string, + extensionPattern: RegExp, + currentDepth: number = 0, + maxDepth: number = MAX_DEPTH ): Promise { if (currentDepth > maxDepth) { return [] @@ -139,8 +157,8 @@ export async function findVideoFiles( const stats = await fs.promises.stat(fullPath) if (stats.isDirectory()) { - results.push(await findVideoFiles(fullPath, currentDepth + 1, maxDepth)) - } else if (stats.isFile() && SUPPORTED_VIDEO_EXTENSIONS.test(item)) { + results.push(await findMediaFiles(fullPath, extensionPattern, currentDepth + 1, maxDepth)) + } else if (stats.isFile() && extensionPattern.test(item)) { results.push([{ path: fullPath, mtime: stats.mtime }]) } } catch (error) { diff --git a/python/analysis_service.py b/python/analysis_service.py index a4dbdf95..a9b17ba5 100644 --- a/python/analysis_service.py +++ b/python/analysis_service.py @@ -901,7 +901,16 @@ async def start( elif host and port: logger.info(f"Starting service on {host}:{port}") - async with websockets.serve(self.handler.handle_connection, host, port,ping_interval=60,ping_timeout=120,close_timeout=30,max_queue=None): + # Extended timeouts for long-running transcriptions (hours-long videos) + async with websockets.serve( + self.handler.handle_connection, + host, + port, + ping_interval=30, # Send ping every 30 seconds + ping_timeout=3600, # Allow 1 hour without pong response + close_timeout=60, # Wait 60s for close handshake + max_queue=None + ): logger.info(f"Server listening on {host}:{port}") await asyncio.Future() # Run forever diff --git a/python/requirements.txt b/python/requirements.txt index 3b33ac88..e3f960fb 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -3,10 +3,10 @@ opencv-python>=4.8.0 numpy>=1.24.0,<2.0.0 tqdm>=4.66.0 -# ML/AI frameworks (CPU ONLY) ---extra-index-url https://download.pytorch.org/whl/cpu -torch==2.1.0 -torchvision==0.16.0 +# ML/AI frameworks with CUDA 12.4 support +--extra-index-url https://download.pytorch.org/whl/cu124 +torch>=2.5.0 +torchvision>=0.20.0 --index-url https://pypi.org/simple # Emotion detection (optional - can be heavy) @@ -23,6 +23,10 @@ scikit-learn>=1.3.0 easyocr>=1.7.0 Pillow>=9.3.0 +# Face recognition +face_recognition>=1.3.0 +dlib>=19.24.0 + # Monitoring psutil>=5.9.0