From cbf65cab219f612a64644ad986dbdb5a143128b0 Mon Sep 17 00:00:00 2001 From: Yondon Fu Date: Tue, 16 Dec 2025 15:14:31 -0500 Subject: [PATCH 01/11] Add plugins module Signed-off-by: Yondon Fu --- pyproject.toml | 1 + src/scope/core/__init__.py | 5 ++++ src/scope/core/pipelines/registry.py | 21 +++++++++++++++- src/scope/core/plugins/__init__.py | 6 +++++ src/scope/core/plugins/hookspecs.py | 24 +++++++++++++++++++ src/scope/core/plugins/manager.py | 36 ++++++++++++++++++++++++++++ uv.lock | 2 ++ 7 files changed, 94 insertions(+), 1 deletion(-) create mode 100644 src/scope/core/plugins/__init__.py create mode 100644 src/scope/core/plugins/hookspecs.py create mode 100644 src/scope/core/plugins/manager.py diff --git a/pyproject.toml b/pyproject.toml index 7f021c8de..57619c69e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,6 +47,7 @@ dependencies = [ "sageattention==2.2.0; sys_platform == 'linux' or sys_platform == 'win32'", "safetensors>=0.6.2", "huggingface_hub>=0.25.0", + "pluggy>=1.5.0", "peft>=0.17.1", "torchao==0.13.0", "kernels>=0.10.4", diff --git a/src/scope/core/__init__.py b/src/scope/core/__init__.py index e69de29bb..38baa9211 100644 --- a/src/scope/core/__init__.py +++ b/src/scope/core/__init__.py @@ -0,0 +1,5 @@ +"""Core functionality for Scope.""" + +from scope.core.plugins import hookimpl + +__all__ = ["hookimpl"] diff --git a/src/scope/core/pipelines/registry.py b/src/scope/core/pipelines/registry.py index 493804be4..2959c0ab6 100644 --- a/src/scope/core/pipelines/registry.py +++ b/src/scope/core/pipelines/registry.py @@ -5,12 +5,15 @@ metadata retrieval. """ +import logging from typing import TYPE_CHECKING if TYPE_CHECKING: from .interface import Pipeline from .schema import BasePipelineConfig +logger = logging.getLogger(__name__) + class PipelineRegistry: """Registry for managing available pipelines.""" @@ -86,5 +89,21 @@ def _register_pipelines(): PipelineRegistry.register(config_class.pipeline_id, pipeline_class) +def _initialize_registry(): + """Initialize registry with built-in pipelines and plugins.""" + # Register built-in pipelines first + _register_pipelines() + + # Load and register plugin pipelines + from scope.core.plugins import load_plugins, register_plugin_pipelines + + load_plugins() + register_plugin_pipelines(PipelineRegistry) + + logger.info( + f"Registry initialized with {len(PipelineRegistry.list_pipelines())} pipeline(s)" + ) + + # Auto-register pipelines on module import -_register_pipelines() +_initialize_registry() diff --git a/src/scope/core/plugins/__init__.py b/src/scope/core/plugins/__init__.py new file mode 100644 index 000000000..0109c708f --- /dev/null +++ b/src/scope/core/plugins/__init__.py @@ -0,0 +1,6 @@ +"""Plugin system for Scope.""" + +from .hookspecs import hookimpl +from .manager import load_plugins, register_plugin_pipelines + +__all__ = ["hookimpl", "load_plugins", "register_plugin_pipelines"] diff --git a/src/scope/core/plugins/hookspecs.py b/src/scope/core/plugins/hookspecs.py new file mode 100644 index 000000000..62ce4f672 --- /dev/null +++ b/src/scope/core/plugins/hookspecs.py @@ -0,0 +1,24 @@ +"""Hook specifications for the Scope plugin system.""" + +import pluggy + +hookspec = pluggy.HookspecMarker("scope") +hookimpl = pluggy.HookimplMarker("scope") + + +class ScopeHookSpec: + """Hook specifications for Scope plugins.""" + + @hookspec + def register_pipelines(self, register): + """Register custom pipeline implementations. + + Args: + register: Callback to register pipeline classes. + Usage: register(PipelineClass) + + Example: + @scope.core.hookimpl + def register_pipelines(register): + register(MyPipeline) + """ diff --git a/src/scope/core/plugins/manager.py b/src/scope/core/plugins/manager.py new file mode 100644 index 000000000..9a724a42f --- /dev/null +++ b/src/scope/core/plugins/manager.py @@ -0,0 +1,36 @@ +"""Plugin manager for discovering and loading Scope plugins.""" + +import logging + +import pluggy + +from .hookspecs import ScopeHookSpec + +logger = logging.getLogger(__name__) + +# Create the plugin manager singleton +pm = pluggy.PluginManager("scope") +pm.add_hookspecs(ScopeHookSpec) + + +def load_plugins(): + """Discover and load all plugins via entry points.""" + pm.load_setuptools_entrypoints("scope") + logger.info(f"Loaded {len(pm.get_plugins())} plugin(s)") + + +def register_plugin_pipelines(registry): + """Call register_pipelines hook for all plugins. + + Args: + registry: PipelineRegistry to register pipelines with + """ + + def register_callback(pipeline_class): + """Callback function passed to plugins.""" + config_class = pipeline_class.get_config_class() + pipeline_id = config_class.pipeline_id + registry.register(pipeline_id, pipeline_class) + logger.info(f"Registered plugin pipeline: {pipeline_id}") + + pm.hook.register_pipelines(register=register_callback) diff --git a/uv.lock b/uv.lock index 400751bac..c226b6c2f 100644 --- a/uv.lock +++ b/uv.lock @@ -636,6 +636,7 @@ dependencies = [ { name = "lmdb" }, { name = "omegaconf" }, { name = "peft" }, + { name = "pluggy" }, { name = "pyopengl", marker = "sys_platform == 'win32'" }, { name = "safetensors" }, { name = "sageattention", version = "2.2.0", source = { url = "https://github.com/daydreamlive/SageAttention/releases/download/v2.2.0-linux/sageattention-2.2.0-cp310-cp310-linux_x86_64.whl" }, marker = "sys_platform == 'linux'" }, @@ -681,6 +682,7 @@ requires-dist = [ { name = "lmdb", specifier = ">=1.7.3" }, { name = "omegaconf", specifier = ">=2.3.0" }, { name = "peft", specifier = ">=0.17.1" }, + { name = "pluggy", specifier = ">=1.5.0" }, { name = "pyopengl", marker = "sys_platform == 'win32'", specifier = ">=3.1.10" }, { name = "safetensors", specifier = ">=0.6.2" }, { name = "sageattention", marker = "sys_platform == 'linux'", url = "https://github.com/daydreamlive/SageAttention/releases/download/v2.2.0-linux/sageattention-2.2.0-cp310-cp310-linux_x86_64.whl" }, From 3e5041985cb996958f3144ef11d2e4663a169add Mon Sep 17 00:00:00 2001 From: Yondon Fu Date: Tue, 16 Dec 2025 15:15:58 -0500 Subject: [PATCH 02/11] Support loading plugin pipelines in PipelineManager Signed-off-by: Yondon Fu --- src/scope/server/pipeline_manager.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/scope/server/pipeline_manager.py b/src/scope/server/pipeline_manager.py index 383eb9951..2877150f8 100644 --- a/src/scope/server/pipeline_manager.py +++ b/src/scope/server/pipeline_manager.py @@ -318,6 +318,27 @@ def _load_pipeline_implementation( self, pipeline_id: str, load_params: dict | None = None ): """Synchronous pipeline loading (runs in thread executor).""" + from scope.core.pipelines.registry import PipelineRegistry + + # Check if pipeline is in registry + pipeline_class = PipelineRegistry.get(pipeline_id) + + # List of built-in pipelines with custom initialization + BUILTIN_PIPELINES = { + "streamdiffusionv2", + "passthrough", + "longlive", + "krea-realtime-video", + "reward-forcing", + } + + if pipeline_class is not None and pipeline_id not in BUILTIN_PIPELINES: + # Plugin pipeline - instantiate generically with load_params + logger.info(f"Loading plugin pipeline: {pipeline_id}") + load_params = load_params or {} + return pipeline_class(**load_params) + + # Fall through to built-in pipeline initialization if pipeline_id == "streamdiffusionv2": from scope.core.pipelines import ( StreamDiffusionV2Pipeline, From 7f60d48491fe0030f4bb5ed64af6edec958aa366 Mon Sep 17 00:00:00 2001 From: Yondon Fu Date: Tue, 16 Dec 2025 15:44:28 -0500 Subject: [PATCH 03/11] Incl plugin pipelines when listing from server Signed-off-by: Yondon Fu --- src/scope/server/app.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/scope/server/app.py b/src/scope/server/app.py index 16f0e4536..049b43cbf 100644 --- a/src/scope/server/app.py +++ b/src/scope/server/app.py @@ -346,15 +346,17 @@ async def get_pipeline_schemas(): The frontend should use this as the source of truth for parameter defaults. """ - from scope.core.pipelines.schema import PIPELINE_CONFIGS + from scope.core.pipelines.registry import PipelineRegistry pipelines: dict = {} - for pipeline_id, config_class in PIPELINE_CONFIGS.items(): - # get_schema_with_metadata() now includes supported_modes, default_mode, - # and mode_defaults directly from the config class - schema_data = config_class.get_schema_with_metadata() - pipelines[pipeline_id] = schema_data + for pipeline_id in PipelineRegistry.list_pipelines(): + config_class = PipelineRegistry.get_config_class(pipeline_id) + if config_class: + # get_schema_with_metadata() includes supported_modes, default_mode, + # and mode_defaults directly from the config class + schema_data = config_class.get_schema_with_metadata() + pipelines[pipeline_id] = schema_data return PipelineSchemasResponse(pipelines=pipelines) From d1e914fcea01e0c0ec16942dbae70da4a07497fb Mon Sep 17 00:00:00 2001 From: Yondon Fu Date: Tue, 16 Dec 2025 15:58:36 -0500 Subject: [PATCH 04/11] Rely on server for pipeline list in frontend Signed-off-by: Yondon Fu --- frontend/src/components/DownloadDialog.tsx | 6 +- .../src/components/InputAndControlsPanel.tsx | 7 +- frontend/src/components/SettingsPanel.tsx | 27 +++--- frontend/src/hooks/usePipelines.ts | 83 +++++++++++++++++++ frontend/src/pages/StreamPage.tsx | 35 ++++---- 5 files changed, 126 insertions(+), 32 deletions(-) create mode 100644 frontend/src/hooks/usePipelines.ts diff --git a/frontend/src/components/DownloadDialog.tsx b/frontend/src/components/DownloadDialog.tsx index d94362e84..556b7c26a 100644 --- a/frontend/src/components/DownloadDialog.tsx +++ b/frontend/src/components/DownloadDialog.tsx @@ -9,11 +9,12 @@ import { DialogTitle, } from "./ui/dialog"; import { Progress } from "./ui/progress"; -import { PIPELINES } from "../data/pipelines"; import type { PipelineId, DownloadProgress } from "../types"; +import type { PipelineInfo } from "../hooks/usePipelines"; interface DownloadDialogProps { open: boolean; + pipelines: Record | null; pipelineId: PipelineId; onClose: () => void; onDownload: () => void; @@ -23,13 +24,14 @@ interface DownloadDialogProps { export function DownloadDialog({ open, + pipelines, pipelineId, onClose, onDownload, isDownloading = false, progress = null, }: DownloadDialogProps) { - const pipelineInfo = PIPELINES[pipelineId]; + const pipelineInfo = pipelines?.[pipelineId]; if (!pipelineInfo) return null; return ( diff --git a/frontend/src/components/InputAndControlsPanel.tsx b/frontend/src/components/InputAndControlsPanel.tsx index ca3a0e060..f83f5da45 100644 --- a/frontend/src/components/InputAndControlsPanel.tsx +++ b/frontend/src/components/InputAndControlsPanel.tsx @@ -15,7 +15,7 @@ import { LabelWithTooltip } from "./ui/label-with-tooltip"; import type { VideoSourceMode } from "../hooks/useVideoSource"; import type { PromptItem, PromptTransition } from "../lib/api"; import type { InputMode } from "../types"; -import { pipelineIsMultiMode } from "../data/pipelines"; +import type { PipelineInfo } from "../hooks/usePipelines"; import { PromptInput } from "./PromptInput"; import { TimelinePromptEditor } from "./TimelinePromptEditor"; import type { TimelinePrompt } from "./PromptTimeline"; @@ -24,6 +24,7 @@ import { Button } from "./ui/button"; interface InputAndControlsPanelProps { className?: string; + pipelines: Record | null; localStream: MediaStream | null; isInitializing: boolean; error: string | null; @@ -73,6 +74,7 @@ interface InputAndControlsPanelProps { export function InputAndControlsPanel({ className = "", + pipelines, localStream, isInitializing, error, @@ -128,7 +130,8 @@ export function InputAndControlsPanel({ const videoRef = useRef(null); // Check if this pipeline supports multiple input modes - const isMultiMode = pipelineIsMultiMode(pipelineId); + const pipeline = pipelines?.[pipelineId]; + const isMultiMode = (pipeline?.supportedModes?.length ?? 0) > 1; useEffect(() => { if (videoRef.current && localStream) { diff --git a/frontend/src/components/SettingsPanel.tsx b/frontend/src/components/SettingsPanel.tsx index d72e6768c..f3a96ffdc 100644 --- a/frontend/src/components/SettingsPanel.tsx +++ b/frontend/src/components/SettingsPanel.tsx @@ -20,11 +20,6 @@ import { Button } from "./ui/button"; import { Toggle } from "./ui/toggle"; import { SliderWithInput } from "./ui/slider-with-input"; import { Hammer, Info, Minus, Plus, RotateCcw } from "lucide-react"; -import { - PIPELINES, - pipelineSupportsLoRA, - pipelineSupportsVACE, -} from "../data/pipelines"; import { PARAMETER_METADATA } from "../data/parameterMetadata"; import { DenoisingStepsSlider } from "./DenoisingStepsSlider"; import { useLocalSliderValue } from "../hooks/useLocalSliderValue"; @@ -35,12 +30,14 @@ import type { SettingsState, InputMode, } from "../types"; +import type { PipelineInfo } from "../hooks/usePipelines"; import { LoRAManager } from "./LoRAManager"; const MIN_DIMENSION = 16; interface SettingsPanelProps { className?: string; + pipelines: Record | null; pipelineId: PipelineId; onPipelineIdChange?: (pipelineId: PipelineId) => void; isStreaming?: boolean; @@ -89,6 +86,7 @@ interface SettingsPanelProps { export function SettingsPanel({ className = "", + pipelines, pipelineId, onPipelineIdChange, isStreaming = false, @@ -141,7 +139,7 @@ export function SettingsPanel({ const [seedError, setSeedError] = useState(null); const handlePipelineIdChange = (value: string) => { - if (value in PIPELINES) { + if (pipelines && value in pipelines) { onPipelineIdChange?.(value as PipelineId); } }; @@ -235,7 +233,7 @@ export function SettingsPanel({ handleSeedChange(newValue); }; - const currentPipeline = PIPELINES[pipelineId]; + const currentPipeline = pipelines?.[pipelineId]; return ( @@ -254,11 +252,12 @@ export function SettingsPanel({ - {Object.keys(PIPELINES).map(id => ( - - {id} - - ))} + {pipelines && + Object.keys(pipelines).map(id => ( + + {id} + + ))} @@ -337,7 +336,7 @@ export function SettingsPanel({ )} {/* VACE Toggle */} - {pipelineSupportsVACE(pipelineId) && ( + {currentPipeline?.supportsVACE && (
)} - {pipelineSupportsLoRA(pipelineId) && ( + {currentPipeline?.supportsLoRA && (
| null>(null); + const [isLoading, setIsLoading] = useState(true); + const [error, setError] = useState(null); + + useEffect(() => { + let mounted = true; + + async function fetchPipelines() { + try { + setIsLoading(true); + const schemas = await getPipelineSchemas(); + + if (!mounted) return; + + // Transform schemas into PipelineInfo format + const transformed: Record = {}; + + // TODO: Just rely on the backend for everything + for (const [id, schema] of Object.entries(schemas.pipelines)) { + transformed[id] = { + name: schema.name, + about: schema.description, + supportedModes: schema.supported_modes as InputMode[], + defaultMode: schema.default_mode as InputMode, + // Defaults for optional fields + docsUrl: undefined, + modified: false, + estimatedVram: undefined, + requiresModels: false, + defaultTemporalInterpolationMethod: "slerp", + defaultTemporalInterpolationSteps: 0, + supportsLoRA: false, + supportsVACE: false, + }; + } + + setPipelines(transformed); + setError(null); + } catch (err) { + if (!mounted) return; + const errorMessage = + err instanceof Error ? err.message : "Failed to fetch pipelines"; + setError(errorMessage); + console.error("Failed to fetch pipelines:", err); + } finally { + if (mounted) { + setIsLoading(false); + } + } + } + + fetchPipelines(); + + return () => { + mounted = false; + }; + }, []); + + return { pipelines, isLoading, error }; +} diff --git a/frontend/src/pages/StreamPage.tsx b/frontend/src/pages/StreamPage.tsx index a583ec5c4..c0680c236 100644 --- a/frontend/src/pages/StreamPage.tsx +++ b/frontend/src/pages/StreamPage.tsx @@ -12,12 +12,8 @@ import { useVideoSource } from "../hooks/useVideoSource"; import { useWebRTCStats } from "../hooks/useWebRTCStats"; import { usePipeline } from "../hooks/usePipeline"; import { useStreamState } from "../hooks/useStreamState"; -import { - PIPELINES, - getPipelineDefaultMode, - getDefaultPromptForMode, - pipelineSupportsVACE, -} from "../data/pipelines"; +import { usePipelines } from "../hooks/usePipelines"; +import { getDefaultPromptForMode } from "../data/pipelines"; import type { InputMode, PipelineId, @@ -66,6 +62,14 @@ function getVaceParams( } export function StreamPage() { + // Fetch available pipelines dynamically + const { pipelines } = usePipelines(); + + // Helper to get default mode for a pipeline + const getPipelineDefaultMode = (pipelineId: string): InputMode => { + return pipelines?.[pipelineId]?.defaultMode ?? "text"; + }; + // Use the stream state hook for settings management const { settings, @@ -253,7 +257,7 @@ export function StreamPage() { stopStream(); } - const newPipeline = PIPELINES[pipelineId]; + const newPipeline = pipelines?.[pipelineId]; const modeToUse = newPipeline?.defaultMode || "text"; const currentMode = settings.inputMode || "text"; @@ -339,7 +343,7 @@ export function StreamPage() { // Preserve the current input mode that the user selected before download // Only fall back to pipeline's default mode if no mode is currently set - const newPipeline = PIPELINES[pipelineId]; + const newPipeline = pipelines?.[pipelineId]; const currentMode = settings.inputMode || newPipeline?.defaultMode || "text"; const defaults = getDefaults(pipelineId, currentMode); @@ -612,7 +616,7 @@ export function StreamPage() { // Update temporal interpolation defaults when pipeline changes useEffect(() => { - const pipeline = PIPELINES[settings.pipelineId]; + const pipeline = pipelines?.[settings.pipelineId]; if (pipeline) { const defaultMethod = pipeline.defaultTemporalInterpolationMethod || "slerp"; @@ -621,7 +625,7 @@ export function StreamPage() { setTemporalInterpolationMethod(defaultMethod); setTransitionSteps(defaultSteps); } - }, [settings.pipelineId]); + }, [settings.pipelineId, pipelines]); const handlePlayPauseToggle = () => { const newPausedState = !settings.paused; @@ -661,7 +665,7 @@ export function StreamPage() { try { // Check if models are needed but not downloaded - const pipelineInfo = PIPELINES[pipelineIdToUse]; + const pipelineInfo = pipelines?.[pipelineIdToUse]; if (pipelineInfo?.requiresModels) { try { const status = await checkModelStatus(pipelineIdToUse); @@ -693,7 +697,7 @@ export function StreamPage() { // Compute VACE enabled state once - enabled by default for text mode on VACE-supporting pipelines const vaceEnabled = settings.vaceEnabled ?? - (pipelineSupportsVACE(pipelineIdToUse) && currentMode !== "video"); + (pipelines?.[pipelineIdToUse]?.supportsVACE && currentMode !== "video"); if (pipelineIdToUse === "streamdiffusionv2" && resolution) { loadParams = { @@ -872,6 +876,7 @@ export function StreamPage() {
Date: Tue, 16 Dec 2025 16:48:31 -0500 Subject: [PATCH 05/11] Add plugins, pipelines CLI cmd with click Signed-off-by: Yondon Fu --- pyproject.toml | 4 + src/scope/core/plugins/__init__.py | 4 +- src/scope/server/app.py | 154 ++++++++++++++++++++--------- uv.lock | 8 +- 4 files changed, 121 insertions(+), 49 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 57619c69e..3de03f39b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,7 @@ dependencies = [ "safetensors>=0.6.2", "huggingface_hub>=0.25.0", "pluggy>=1.5.0", + "click>=8.3.1", "peft>=0.17.1", "torchao==0.13.0", "kernels>=0.10.4", @@ -67,6 +68,9 @@ Homepage = "https://github.com/daydreamlive/scope" Repository = "https://github.com/daydreamlive/scope" Issues = "https://github.com/daydreamlive/scope/issues" +[tool.uv] +preview = true + [tool.uv.extra-build-dependencies] flash-attn = [{ requirement = "torch", match-runtime = true, marker = "sys_platform == 'linux' or sys_platform == 'win32'" }] diff --git a/src/scope/core/plugins/__init__.py b/src/scope/core/plugins/__init__.py index 0109c708f..03b4ac6f5 100644 --- a/src/scope/core/plugins/__init__.py +++ b/src/scope/core/plugins/__init__.py @@ -1,6 +1,6 @@ """Plugin system for Scope.""" from .hookspecs import hookimpl -from .manager import load_plugins, register_plugin_pipelines +from .manager import load_plugins, pm, register_plugin_pipelines -__all__ = ["hookimpl", "load_plugins", "register_plugin_pipelines"] +__all__ = ["hookimpl", "load_plugins", "pm", "register_plugin_pipelines"] diff --git a/src/scope/server/app.py b/src/scope/server/app.py index 049b43cbf..6df81598c 100644 --- a/src/scope/server/app.py +++ b/src/scope/server/app.py @@ -1,18 +1,22 @@ -import argparse import asyncio +import contextlib +import io import logging import os import subprocess import sys import threading import time +import warnings import webbrowser from contextlib import asynccontextmanager from datetime import datetime +from functools import wraps from importlib.metadata import version from logging.handlers import RotatingFileHandler from pathlib import Path +import click import torch import uvicorn from fastapi import Depends, FastAPI, HTTPException, Query, Request @@ -117,6 +121,28 @@ def filter(self, record): logger = logging.getLogger(__name__) +def suppress_init_output(func): + """Decorator to suppress all initialization output (logging, warnings, stdout/stderr).""" + + @wraps(func) + def wrapper(*args, **kwargs): + with ( + contextlib.redirect_stdout(io.StringIO()), + contextlib.redirect_stderr(io.StringIO()), + warnings.catch_warnings(), + ): + warnings.simplefilter("ignore") + # Temporarily disable all logging + logging.disable(logging.CRITICAL) + try: + return func(*args, **kwargs) + finally: + # Re-enable logging + logging.disable(logging.NOTSET) + + return wrapper + + def get_git_commit_hash() -> str: """ Get the current git commit hash. @@ -847,41 +873,8 @@ def open_browser_when_ready(host: str, port: int, server): logger.info(f"🌐 UI is available at: {url}") -def main(): - """Main entry point for the daydream-scope command.""" - parser = argparse.ArgumentParser( - description="A tool for running and customizing real-time, interactive generative AI pipelines and models" - ) - parser.add_argument( - "--version", - action="store_true", - help="Show version information and exit", - ) - parser.add_argument( - "--reload", - action="store_true", - help="Enable auto-reload for development (default: False)", - ) - parser.add_argument( - "--host", default="0.0.0.0", help="Host to bind to (default: 0.0.0.0)" - ) - parser.add_argument( - "--port", type=int, default=8000, help="Port to bind to (default: 8000)" - ) - parser.add_argument( - "-N", - "--no-browser", - action="store_true", - help="Do not automatically open a browser window after the server starts", - ) - - args = parser.parse_args() - - # Handle version flag - if args.version: - print_version_info() - sys.exit(0) - +def run_server(reload: bool, host: str, port: int, no_browser: bool): + """Run the Daydream Scope server.""" # Configure static file serving configure_static_files() @@ -893,18 +886,18 @@ def main(): # Create server instance for production mode config = uvicorn.Config( "scope.server.app:app", - host=args.host, - port=args.port, - reload=args.reload, + host=host, + port=port, + reload=reload, log_config=None, # Use our logging config, don't override it ) server = uvicorn.Server(config) # Start browser opening thread (unless disabled) - if not args.no_browser: + if not no_browser: browser_thread = threading.Thread( target=open_browser_when_ready, - args=(args.host, args.port, server), + args=(host, port, server), daemon=True, ) browser_thread.start() @@ -920,12 +913,85 @@ def main(): # Development mode - just run normally uvicorn.run( "scope.server.app:app", - host=args.host, - port=args.port, - reload=args.reload, + host=host, + port=port, + reload=reload, log_config=None, # Use our logging config, don't override it ) +@click.group(invoke_without_command=True) +@click.option("--version", is_flag=True, help="Show version information and exit") +@click.option( + "--reload", is_flag=True, help="Enable auto-reload for development (default: False)" +) +@click.option("--host", default="0.0.0.0", help="Host to bind to (default: 0.0.0.0)") +@click.option("--port", default=8000, help="Port to bind to (default: 8000)") +@click.option( + "-N", + "--no-browser", + is_flag=True, + help="Do not automatically open a browser window after the server starts", +) +@click.pass_context +def main(ctx, version: bool, reload: bool, host: str, port: int, no_browser: bool): + # Handle version flag + if version: + print_version_info() + sys.exit(0) + + # If no subcommand was invoked, run the server + if ctx.invoked_subcommand is None: + run_server(reload, host, port, no_browser) + + +@main.command() +def plugins(): + """List all installed plugins.""" + + @suppress_init_output + def _load_plugins(): + from scope.core.plugins import load_plugins, pm + + load_plugins() + return pm.get_plugins() + + plugin_list = _load_plugins() + + if not plugin_list: + click.echo("No plugins installed.") + return + + click.echo(f"{len(plugin_list)} plugin(s) installed:\n") + + # List each plugin + for plugin in plugin_list: + plugin_name = plugin.__name__ if hasattr(plugin, "__name__") else str(plugin) + click.echo(f" • {plugin_name}") + + +@main.command() +def pipelines(): + """List all available pipelines.""" + + @suppress_init_output + def _load_pipelines(): + from scope.core.pipelines.registry import PipelineRegistry + + return PipelineRegistry.list_pipelines() + + all_pipelines = _load_pipelines() + + if not all_pipelines: + click.echo("No pipelines available.") + return + + click.echo(f"{len(all_pipelines)} pipeline(s) available:\n") + + # List all pipelines + for pipeline_id in all_pipelines: + click.echo(f" • {pipeline_id}") + + if __name__ == "__main__": main() diff --git a/uv.lock b/uv.lock index c226b6c2f..a2952b5b5 100644 --- a/uv.lock +++ b/uv.lock @@ -532,14 +532,14 @@ wheels = [ [[package]] name = "click" -version = "8.3.0" +version = "8.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/46/61/de6cd827efad202d7057d93e0fed9294b96952e188f7384832791c7b2254/click-8.3.0.tar.gz", hash = "sha256:e7b8232224eba16f4ebe410c25ced9f7875cb5f3263ffc93cc3e8da705e229c4", size = 276943, upload-time = "2025-09-18T17:32:23.696Z" } +sdist = { url = "https://files.pythonhosted.org/packages/3d/fa/656b739db8587d7b5dfa22e22ed02566950fbfbcdc20311993483657a5c0/click-8.3.1.tar.gz", hash = "sha256:12ff4785d337a1bb490bb7e9c2b1ee5da3112e94a8622f26a6c77f5d2fc6842a", size = 295065, upload-time = "2025-11-15T20:45:42.706Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/db/d3/9dcc0f5797f070ec8edf30fbadfb200e71d9db6b84d211e3b2085a7589a0/click-8.3.0-py3-none-any.whl", hash = "sha256:9b9f285302c6e3064f4330c05f05b81945b2a39544279343e6e7c5f27a9baddc", size = 107295, upload-time = "2025-09-18T17:32:22.42Z" }, + { url = "https://files.pythonhosted.org/packages/98/78/01c019cdb5d6498122777c1a43056ebb3ebfeef2076d9d026bfe15583b2b/click-8.3.1-py3-none-any.whl", hash = "sha256:981153a64e25f12d547d3426c367a4857371575ee7ad18df2a6183ab0545b2a6", size = 108274, upload-time = "2025-11-15T20:45:41.139Z" }, ] [[package]] @@ -623,6 +623,7 @@ source = { editable = "." } dependencies = [ { name = "accelerate" }, { name = "aiortc" }, + { name = "click" }, { name = "diffusers" }, { name = "easydict" }, { name = "einops" }, @@ -669,6 +670,7 @@ dev = [ requires-dist = [ { name = "accelerate", specifier = ">=1.1.1" }, { name = "aiortc", specifier = ">=1.13.0" }, + { name = "click", specifier = ">=8.3.1" }, { name = "diffusers", specifier = ">=0.31.0" }, { name = "easydict", specifier = ">=1.13" }, { name = "einops", specifier = ">=0.8.1" }, From e245628002c3cc6e4eb7c4200861b748d1a6c8cb Mon Sep 17 00:00:00 2001 From: Yondon Fu Date: Tue, 16 Dec 2025 17:03:09 -0500 Subject: [PATCH 06/11] Add install, uninstall (plugins) cmd in CLI Signed-off-by: Yondon Fu --- src/scope/server/app.py | 48 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/src/scope/server/app.py b/src/scope/server/app.py index 6df81598c..46975c557 100644 --- a/src/scope/server/app.py +++ b/src/scope/server/app.py @@ -993,5 +993,53 @@ def _load_pipelines(): click.echo(f" • {pipeline_id}") +@main.command() +@click.argument("packages", nargs=-1, required=False) +@click.option("--upgrade", is_flag=True, help="Upgrade packages to the latest version") +@click.option( + "-e", "--editable", help="Install a project in editable mode from this path" +) +@click.option("--force-reinstall", is_flag=True, help="Force reinstall packages") +@click.option("--no-cache-dir", is_flag=True, help="Disable the cache") +@click.option( + "--pre", is_flag=True, help="Include pre-release and development versions" +) +def install(packages, upgrade, editable, force_reinstall, no_cache_dir, pre): + """Install a plugin.""" + args = ["uv", "pip", "install"] + if upgrade: + args.append("--upgrade") + if editable: + args += ["--editable", editable] + if force_reinstall: + args.append("--force-reinstall") + if no_cache_dir: + args.append("--no-cache-dir") + if pre: + args.append("--pre") + args += list(packages) + + result = subprocess.run(args, capture_output=False) + + if result.returncode != 0: + sys.exit(result.returncode) + + +@main.command() +@click.argument("packages", nargs=-1, required=True) +@click.option("-y", "--yes", is_flag=True, help="Don't ask for confirmation") +def uninstall(packages, yes): + """Uninstall a plugin.""" + args = ["uv", "pip", "uninstall"] + args += list(packages) + if yes: + args.append("-y") + + result = subprocess.run(args, capture_output=False) + + if result.returncode != 0: + sys.exit(result.returncode) + + if __name__ == "__main__": main() From 955bf815d8f897ca9d932448d96357863d6181d2 Mon Sep 17 00:00:00 2001 From: Yondon Fu Date: Tue, 16 Dec 2025 18:04:13 -0500 Subject: [PATCH 07/11] PipelineRegistry on server startup Signed-off-by: Yondon Fu --- src/scope/server/app.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/scope/server/app.py b/src/scope/server/app.py index 46975c557..564d4b29f 100644 --- a/src/scope/server/app.py +++ b/src/scope/server/app.py @@ -875,6 +875,11 @@ def open_browser_when_ready(host: str, port: int, server): def run_server(reload: bool, host: str, port: int, no_browser: bool): """Run the Daydream Scope server.""" + + from scope.core.pipelines.registry import ( + PipelineRegistry, # noqa: F401 - imported for side effects (registry initialization) + ) + # Configure static file serving configure_static_files() From 7c8b8593e5044c9c9b065bd791858ce684917fde Mon Sep 17 00:00:00 2001 From: Yondon Fu Date: Tue, 16 Dec 2025 18:20:40 -0500 Subject: [PATCH 08/11] Rely on backend for all pipeline metadata and add conditional render for prompts box Signed-off-by: Yondon Fu --- .../src/components/InputAndControlsPanel.tsx | 6 +- frontend/src/components/SettingsPanel.tsx | 26 +--- frontend/src/data/pipelines.ts | 123 +----------------- frontend/src/hooks/usePipelines.ts | 29 ++--- frontend/src/hooks/useStreamState.ts | 29 +++-- frontend/src/lib/api.ts | 10 ++ frontend/src/pages/StreamPage.tsx | 16 +-- frontend/src/types/index.ts | 2 +- src/scope/core/pipelines/schema.py | 86 ++++++++++-- 9 files changed, 135 insertions(+), 192 deletions(-) diff --git a/frontend/src/components/InputAndControlsPanel.tsx b/frontend/src/components/InputAndControlsPanel.tsx index f83f5da45..2d00d5d79 100644 --- a/frontend/src/components/InputAndControlsPanel.tsx +++ b/frontend/src/components/InputAndControlsPanel.tsx @@ -332,6 +332,11 @@ export function InputAndControlsPanel({ // The Input can have two states: Append (default) and Edit (when a prompt is selected and the video is paused) const isEditMode = selectedTimelinePrompt && isVideoPaused; + // Hide prompts section if pipeline doesn't support prompts + if (pipeline?.supportsPrompts === false) { + return null; + } + return (
@@ -361,7 +366,6 @@ export function InputAndControlsPanel({ onPromptsSubmit={onPromptsSubmit} onTransitionSubmit={onTransitionSubmit} disabled={ - pipelineId === "passthrough" || (_isTimelinePlaying && !isVideoPaused && !isAtEndOfTimeline()) || diff --git a/frontend/src/components/SettingsPanel.tsx b/frontend/src/components/SettingsPanel.tsx index f3a96ffdc..a59776620 100644 --- a/frontend/src/components/SettingsPanel.tsx +++ b/frontend/src/components/SettingsPanel.tsx @@ -19,7 +19,7 @@ import { Input } from "./ui/input"; import { Button } from "./ui/button"; import { Toggle } from "./ui/toggle"; import { SliderWithInput } from "./ui/slider-with-input"; -import { Hammer, Info, Minus, Plus, RotateCcw } from "lucide-react"; +import { Info, Minus, Plus, RotateCcw } from "lucide-react"; import { PARAMETER_METADATA } from "../data/parameterMetadata"; import { DenoisingStepsSlider } from "./DenoisingStepsSlider"; import { useLocalSliderValue } from "../hooks/useLocalSliderValue"; @@ -272,9 +272,7 @@ export function SettingsPanel({
- {(currentPipeline.about || - currentPipeline.docsUrl || - currentPipeline.modified) && ( + {(currentPipeline.about || currentPipeline.docsUrl) && (
{currentPipeline.about && ( @@ -293,26 +291,6 @@ export function SettingsPanel({ )} - {currentPipeline.modified && ( - - - - - - - - -

- This pipeline contains modifications based on the - original project. -

- - - - )} {currentPipeline.docsUrl && ( = { text: "A 3D animated scene. A **panda** walks along a path towards the camera in a park on a spring day.", video: "A 3D animated scene. A **panda** sitting in the grass, looking around.", }; -export interface PipelineInfo { - name: string; - about: string; - docsUrl?: string; - modified?: boolean; - estimatedVram?: number; // GB - requiresModels?: boolean; // Whether this pipeline requires models to be downloaded - defaultTemporalInterpolationMethod?: "linear" | "slerp"; // Default method for temporal interpolation - defaultTemporalInterpolationSteps?: number; // Default number of steps for temporal interpolation - supportsLoRA?: boolean; // Whether this pipeline supports LoRA adapters - supportsVACE?: boolean; // Whether this pipeline supports VACE (Video All-In-One Creation and Editing) - - // Multi-mode support - supportedModes: InputMode[]; - defaultMode: InputMode; -} - -export const PIPELINES: Record = { - streamdiffusionv2: { - name: "StreamDiffusionV2", - docsUrl: - "https://github.com/daydreamlive/scope/blob/main/src/scope/core/pipelines/streamdiffusionv2/docs/usage.md", - about: - "A streaming pipeline and autoregressive video diffusion model from the creators of the original StreamDiffusion project. The model is trained using Self-Forcing on Wan2.1 1.3b with modifications to support streaming. Includes VACE (All-In-One Video Creation and Editing) for reference image conditioning and structural guidance (depth, flow, pose).", - modified: true, - estimatedVram: 20, - requiresModels: true, - defaultTemporalInterpolationMethod: "slerp", - defaultTemporalInterpolationSteps: 0, - supportsLoRA: true, - supportsVACE: true, - // Multi-mode support - supportedModes: ["text", "video"], - defaultMode: "video", - }, - longlive: { - name: "LongLive", - docsUrl: - "https://github.com/daydreamlive/scope/blob/main/src/scope/core/pipelines/longlive/docs/usage.md", - about: - "A streaming pipeline and autoregressive video diffusion model from Nvidia, MIT, HKUST, HKU and THU. The model is trained using Self-Forcing on Wan2.1 1.3b with modifications to support smoother prompt switching and improved quality over longer time periods while maintaining fast generation. Includes VACE (All-In-One Video Creation and Editing) for reference image conditioning and structural guidance (depth, flow, pose).", - modified: true, - estimatedVram: 20, - requiresModels: true, - defaultTemporalInterpolationMethod: "slerp", - defaultTemporalInterpolationSteps: 0, - supportsLoRA: true, - supportsVACE: true, - // Multi-mode support - supportedModes: ["text", "video"], - defaultMode: "text", - }, - "krea-realtime-video": { - name: "Krea Realtime Video", - docsUrl: - "https://github.com/daydreamlive/scope/blob/main/src/scope/core/pipelines/krea_realtime_video/docs/usage.md", - about: - "A streaming pipeline and autoregressive video diffusion model from Krea. The model is trained using Self-Forcing on Wan2.1 14b.", - modified: true, - estimatedVram: 32, - requiresModels: true, - defaultTemporalInterpolationMethod: "linear", - defaultTemporalInterpolationSteps: 4, - supportsLoRA: true, - // Multi-mode support - supportedModes: ["text", "video"], - defaultMode: "text", - }, - "reward-forcing": { - name: "RewardForcing", - docsUrl: - "https://github.com/daydreamlive/scope/blob/main/src/scope/core/pipelines/reward_forcing/docs/usage.md", - about: - "A streaming pipeline and autoregressive video diffusion model from ZJU, Ant Group, SIAS-ZJU, HUST and SJTU. The model is trained with Rewarded Distribution Matching Distillation using Wan2.1 1.3b as the base model. Includes VACE (All-In-One Video Creation and Editing) for reference image conditioning and structural guidance (depth, flow, pose).", - modified: true, - estimatedVram: 20, - requiresModels: true, - defaultTemporalInterpolationMethod: "slerp", - defaultTemporalInterpolationSteps: 0, - supportsLoRA: true, - supportsVACE: true, - // Multi-mode support - supportedModes: ["text", "video"], - defaultMode: "text", - }, - passthrough: { - name: "Passthrough", - about: - "A pipeline that returns the input video without any processing that is useful for testing and debugging.", - requiresModels: false, - // Video-only pipeline - supportedModes: ["video"], - defaultMode: "video", - }, -}; - -export function pipelineSupportsLoRA(pipelineId: string): boolean { - return PIPELINES[pipelineId]?.supportsLoRA === true; -} - -export function pipelineSupportsVACE(pipelineId: string): boolean { - return PIPELINES[pipelineId]?.supportsVACE === true; -} - -export function pipelineSupportsMode( - pipelineId: string, - mode: InputMode -): boolean { - return PIPELINES[pipelineId]?.supportedModes?.includes(mode) ?? false; -} - -export function pipelineIsMultiMode(pipelineId: string): boolean { - const modes = PIPELINES[pipelineId]?.supportedModes ?? []; - return modes.length > 1; -} - -export function getPipelineDefaultMode(pipelineId: string): InputMode { - return PIPELINES[pipelineId]?.defaultMode ?? "text"; -} - export function getDefaultPromptForMode(mode: InputMode): string { return DEFAULT_PROMPTS[mode]; } diff --git a/frontend/src/hooks/usePipelines.ts b/frontend/src/hooks/usePipelines.ts index d81235ca0..c07d24f66 100644 --- a/frontend/src/hooks/usePipelines.ts +++ b/frontend/src/hooks/usePipelines.ts @@ -5,10 +5,10 @@ import type { InputMode } from "../types"; export interface PipelineInfo { name: string; about: string; - docsUrl?: string; - modified?: boolean; - estimatedVram?: number; + docsUrl?: string | null; + estimatedVram?: number | null; requiresModels?: boolean; + supportsPrompts?: boolean; defaultTemporalInterpolationMethod?: "linear" | "slerp"; defaultTemporalInterpolationSteps?: number; supportsLoRA?: boolean; @@ -35,25 +35,24 @@ export function usePipelines() { if (!mounted) return; - // Transform schemas into PipelineInfo format + // Transform to camelCase for TypeScript conventions const transformed: Record = {}; - - // TODO: Just rely on the backend for everything for (const [id, schema] of Object.entries(schemas.pipelines)) { transformed[id] = { name: schema.name, about: schema.description, supportedModes: schema.supported_modes as InputMode[], defaultMode: schema.default_mode as InputMode, - // Defaults for optional fields - docsUrl: undefined, - modified: false, - estimatedVram: undefined, - requiresModels: false, - defaultTemporalInterpolationMethod: "slerp", - defaultTemporalInterpolationSteps: 0, - supportsLoRA: false, - supportsVACE: false, + supportsPrompts: schema.supports_prompts, + defaultTemporalInterpolationMethod: + schema.default_temporal_interpolation_method, + defaultTemporalInterpolationSteps: + schema.default_temporal_interpolation_steps, + docsUrl: schema.docs_url, + estimatedVram: schema.estimated_vram_gb, + requiresModels: schema.requires_models, + supportsLoRA: schema.supports_lora, + supportsVACE: schema.supports_vace, }; } diff --git a/frontend/src/hooks/useStreamState.ts b/frontend/src/hooks/useStreamState.ts index 95c7e60cb..b804ca229 100644 --- a/frontend/src/hooks/useStreamState.ts +++ b/frontend/src/hooks/useStreamState.ts @@ -13,11 +13,9 @@ import { type HardwareInfoResponse, type PipelineSchemasResponse, } from "../lib/api"; -import { getPipelineDefaultMode } from "../data/pipelines"; // Generic fallback defaults used before schemas are loaded. -// Resolution and denoising steps use conservative values; mode-specific -// values are derived from pipelines.ts when possible. +// Resolution and denoising steps use conservative values. const BASE_FALLBACK = { height: 512, width: 512, @@ -26,9 +24,9 @@ const BASE_FALLBACK = { }; // Get fallback defaults for a pipeline before schemas are loaded -// Derives mode from pipelines.ts to stay in sync with frontend definitions -function getFallbackDefaults(pipelineId: PipelineId, mode?: InputMode) { - const effectiveMode = mode ?? getPipelineDefaultMode(pipelineId); +function getFallbackDefaults(mode?: InputMode) { + // Default to text mode if no mode specified (will be corrected when schemas load) + const effectiveMode = mode ?? "text"; const isVideoMode = effectiveMode === "video"; // Video mode gets noise controls, text mode doesn't @@ -107,8 +105,7 @@ export function useStreamState() { }; } // Fallback to derived defaults if schemas not loaded - // Mode is derived from pipelines.ts to stay in sync - return getFallbackDefaults(pipelineId, mode); + return getFallbackDefaults(mode); }, [pipelineSchemas] ); @@ -137,7 +134,7 @@ export function useStreamState() { ); // Get initial defaults (use fallback since schemas haven't loaded yet) - const initialDefaults = getFallbackDefaults("streamdiffusionv2"); + const initialDefaults = getFallbackDefaults(); const [settings, setSettings] = useState({ pipelineId: "streamdiffusionv2", @@ -201,6 +198,20 @@ export function useStreamState() { fetchInitialData(); }, []); + // Update inputMode when schemas load for the first time + // This corrects the initial fallback mode to match the pipeline's actual default mode + useEffect(() => { + if (pipelineSchemas) { + const schema = pipelineSchemas.pipelines[settings.pipelineId]; + if (schema?.default_mode && settings.inputMode !== schema.default_mode) { + setSettings(prev => ({ + ...prev, + inputMode: schema.default_mode, + })); + } + } + }, [pipelineSchemas, settings.pipelineId, settings.inputMode]); + // Set recommended quantization when krea-realtime-video is selected // Reset to null when switching to other pipelines useEffect(() => { diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index bdd19f7f0..c36f320f7 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -423,10 +423,20 @@ export interface PipelineSchemaInfo { name: string; description: string; version: string; + docs_url: string | null; + estimated_vram_gb: number | null; + requires_models: boolean; + supports_lora: boolean; + supports_vace: boolean; + // Pipeline config schema config_schema: PipelineConfigSchema; // Mode support - comes from config class supported_modes: ("text" | "video")[]; default_mode: "text" | "video"; + // Prompt and temporal interpolation support + supports_prompts: boolean; + default_temporal_interpolation_method: "linear" | "slerp"; + default_temporal_interpolation_steps: number; // Mode-specific default overrides (optional) mode_defaults?: Record<"text" | "video", ModeDefaults>; } diff --git a/frontend/src/pages/StreamPage.tsx b/frontend/src/pages/StreamPage.tsx index c0680c236..1b30c9b02 100644 --- a/frontend/src/pages/StreamPage.tsx +++ b/frontend/src/pages/StreamPage.tsx @@ -614,7 +614,7 @@ export function StreamPage() { return () => document.removeEventListener("keydown", handleKeyDown); }, [selectedTimelinePrompt]); - // Update temporal interpolation defaults when pipeline changes + // Update temporal interpolation defaults and clear prompts when pipeline changes useEffect(() => { const pipeline = pipelines?.[settings.pipelineId]; if (pipeline) { @@ -624,6 +624,11 @@ export function StreamPage() { setTemporalInterpolationMethod(defaultMethod); setTransitionSteps(defaultSteps); + + // Clear prompts if pipeline doesn't support them + if (pipeline.supportsPrompts === false) { + setPromptItems([{ text: "", weight: 1.0 }]); + } } }, [settings.pipelineId, pipelines]); @@ -805,7 +810,7 @@ export function StreamPage() { }; // Common parameters for pipelines that support prompts - if (pipelineIdToUse !== "passthrough") { + if (pipelineInfo?.supportsPrompts !== false) { initialParameters.prompts = promptItems; initialParameters.prompt_interpolation_method = interpolationMethod; initialParameters.denoising_step_list = settings.denoisingSteps || [ @@ -1041,12 +1046,7 @@ export function StreamPage() { }); } }} - disabled={ - settings.pipelineId === "passthrough" || - isPipelineLoading || - isConnecting || - showDownloadDialog - } + disabled={isPipelineLoading || isConnecting || showDownloadDialog} isStreaming={isStreaming} isVideoPaused={settings.paused} timelineRef={timelineRef} diff --git a/frontend/src/types/index.ts b/frontend/src/types/index.ts index c16a06353..7f2e66a76 100644 --- a/frontend/src/types/index.ts +++ b/frontend/src/types/index.ts @@ -84,13 +84,13 @@ export interface PipelineInfo { about: string; projectUrl?: string; docsUrl?: string; - modified?: boolean; defaultPrompt?: string; estimatedVram?: number; requiresModels?: boolean; defaultTemporalInterpolationMethod?: "linear" | "slerp"; defaultTemporalInterpolationSteps?: number; supportsLoRA?: boolean; + supportsVACE?: boolean; // Multi-mode support supportedModes: InputMode[]; diff --git a/src/scope/core/pipelines/schema.py b/src/scope/core/pipelines/schema.py index 73b9e68aa..9cfec81ab 100644 --- a/src/scope/core/pipelines/schema.py +++ b/src/scope/core/pipelines/schema.py @@ -59,11 +59,23 @@ class BasePipelineConfig(BaseModel): pipeline_name: ClassVar[str] = "Base Pipeline" pipeline_description: ClassVar[str] = "Base pipeline configuration" pipeline_version: ClassVar[str] = "1.0.0" + docs_url: ClassVar[str | None] = None + estimated_vram_gb: ClassVar[float | None] = None + requires_models: ClassVar[bool] = False + supports_lora: ClassVar[bool] = False + supports_vace: ClassVar[bool] = False # Mode support - override in subclasses supported_modes: ClassVar[list[InputMode]] = ["text"] default_mode: ClassVar[InputMode] = "text" + # Prompt and temporal interpolation support + supports_prompts: ClassVar[bool] = True + default_temporal_interpolation_method: ClassVar[Literal["linear", "slerp"]] = ( + "slerp" + ) + default_temporal_interpolation_steps: ClassVar[int] = 0 + # Resolution settings height: int = Field(default=512, ge=1, description="Output height in pixels") width: int = Field(default=512, ge=1, description="Output width in pixels") @@ -154,16 +166,23 @@ def get_schema_with_metadata(cls) -> dict[str, Any]: This is the primary method for API/UI schema generation. Returns: - Dict containing: - - Pipeline metadata (id, name, description, version) - - supported_modes: List of supported input modes - - default_mode: Default input mode - - mode_defaults: Dict of mode-specific default overrides - - config_schema: Full JSON schema for the config model + Dict containing pipeline metadata """ metadata = cls.get_pipeline_metadata() metadata["supported_modes"] = cls.supported_modes metadata["default_mode"] = cls.default_mode + metadata["supports_prompts"] = cls.supports_prompts + metadata["default_temporal_interpolation_method"] = ( + cls.default_temporal_interpolation_method + ) + metadata["default_temporal_interpolation_steps"] = ( + cls.default_temporal_interpolation_steps + ) + metadata["docs_url"] = cls.docs_url + metadata["estimated_vram_gb"] = cls.estimated_vram_gb + metadata["requires_models"] = cls.requires_models + metadata["supports_lora"] = cls.supports_lora + metadata["supports_vace"] = cls.supports_vace metadata["config_schema"] = cls.model_json_schema() # Include mode-specific defaults if defined @@ -198,8 +217,17 @@ class LongLiveConfig(BasePipelineConfig): pipeline_id: ClassVar[str] = "longlive" pipeline_name: ClassVar[str] = "LongLive" pipeline_description: ClassVar[str] = ( - "Long-form video generation with temporal consistency" + "A streaming pipeline and autoregressive video diffusion model from Nvidia, MIT, HKUST, HKU and THU. " + "The model is trained using Self-Forcing on Wan2.1 1.3b with modifications to support smoother prompt " + "switching and improved quality over longer time periods while maintaining fast generation." + ) + docs_url: ClassVar[str | None] = ( + "https://github.com/daydreamlive/scope/blob/main/src/scope/core/pipelines/longlive/docs/usage.md" ) + estimated_vram_gb: ClassVar[float | None] = 20.0 + requires_models: ClassVar[bool] = True + supports_lora: ClassVar[bool] = True + supports_vace: ClassVar[bool] = True # Mode support supported_modes: ClassVar[list[InputMode]] = ["text", "video"] @@ -258,10 +286,19 @@ class StreamDiffusionV2Config(BasePipelineConfig): """ pipeline_id: ClassVar[str] = "streamdiffusionv2" - pipeline_name: ClassVar[str] = "StreamDiffusion V2" + pipeline_name: ClassVar[str] = "StreamDiffusionV2" pipeline_description: ClassVar[str] = ( - "Real-time video-to-video generation with temporal consistency" + "A streaming pipeline and autoregressive video diffusion model from the creators of the original " + "StreamDiffusion project. The model is trained using Self-Forcing on Wan2.1 1.3b with modifications " + "to support streaming." ) + docs_url: ClassVar[str | None] = ( + "https://github.com/daydreamlive/scope/blob/main/src/scope/core/pipelines/streamdiffusionv2/docs/usage.md" + ) + estimated_vram_gb: ClassVar[float | None] = 20.0 + requires_models: ClassVar[bool] = True + supports_lora: ClassVar[bool] = True + supports_vace: ClassVar[bool] = True # Mode support supported_modes: ClassVar[list[InputMode]] = ["text", "video"] @@ -329,8 +366,20 @@ class KreaRealtimeVideoConfig(BasePipelineConfig): pipeline_id: ClassVar[str] = "krea-realtime-video" pipeline_name: ClassVar[str] = "Krea Realtime Video" pipeline_description: ClassVar[str] = ( - "High-quality real-time video generation with 14B model" + "A streaming pipeline and autoregressive video diffusion model from Krea. " + "The model is trained using Self-Forcing on Wan2.1 14b." + ) + docs_url: ClassVar[str | None] = ( + "https://github.com/daydreamlive/scope/blob/main/src/scope/core/pipelines/krea_realtime_video/docs/usage.md" ) + estimated_vram_gb: ClassVar[float | None] = 32.0 + requires_models: ClassVar[bool] = True + supports_lora: ClassVar[bool] = True + + default_temporal_interpolation_method: ClassVar[Literal["linear", "slerp"]] = ( + "linear" + ) + default_temporal_interpolation_steps: ClassVar[int] = 4 # Mode support supported_modes: ClassVar[list[InputMode]] = ["text", "video"] @@ -379,8 +428,16 @@ class RewardForcingConfig(BasePipelineConfig): pipeline_id: ClassVar[str] = "reward-forcing" pipeline_name: ClassVar[str] = "RewardForcing" pipeline_description: ClassVar[str] = ( - "Efficient streaming video generation with rewarded distribution matching distillation" + "A streaming pipeline and autoregressive video diffusion model from ZJU, Ant Group, SIAS-ZJU, HUST and SJTU. " + "The model is trained with Rewarded Distribution Matching Distillation using Wan2.1 1.3b as the base model." + ) + docs_url: ClassVar[str | None] = ( + "https://github.com/daydreamlive/scope/blob/main/src/scope/core/pipelines/reward_forcing/docs/usage.md" ) + estimated_vram_gb: ClassVar[float | None] = 20.0 + requires_models: ClassVar[bool] = True + supports_lora: ClassVar[bool] = True + supports_vace: ClassVar[bool] = True # Mode support supported_modes: ClassVar[list[InputMode]] = ["text", "video"] @@ -439,12 +496,17 @@ class PassthroughConfig(BasePipelineConfig): pipeline_id: ClassVar[str] = "passthrough" pipeline_name: ClassVar[str] = "Passthrough" - pipeline_description: ClassVar[str] = "Passthrough pipeline for testing" + pipeline_description: ClassVar[str] = ( + "A pipeline that returns the input video without any processing that is useful for testing and debugging." + ) # Mode support - video only supported_modes: ClassVar[list[InputMode]] = ["video"] default_mode: ClassVar[InputMode] = "video" + # Does not support prompts + supports_prompts: ClassVar[bool] = False + # Passthrough defaults - requires video input (distinct from StreamDiffusionV2) height: int = Field(default=512, ge=1, description="Output height in pixels") width: int = Field(default=512, ge=1, description="Output width in pixels") From bd78ab759780bdfd576975e48b13787cc6f88b52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Leszko?= Date: Wed, 17 Dec 2025 10:36:14 +0100 Subject: [PATCH 09/11] Use default pipeline only if it meets the hardware requirements MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rafał Leszko Signed-off-by: Yondon Fu --- frontend/src/hooks/useStreamState.ts | 22 ++++- src/scope/core/pipelines/registry.py | 124 ++++++++++++++++++++++----- 2 files changed, 124 insertions(+), 22 deletions(-) diff --git a/frontend/src/hooks/useStreamState.ts b/frontend/src/hooks/useStreamState.ts index b804ca229..d889fad5f 100644 --- a/frontend/src/hooks/useStreamState.ts +++ b/frontend/src/hooks/useStreamState.ts @@ -174,7 +174,27 @@ export function useStreamState() { ]); if (schemasResult.status === "fulfilled") { - setPipelineSchemas(schemasResult.value); + const schemas = schemasResult.value; + setPipelineSchemas(schemas); + + // Check if the default pipeline (streamdiffusionv2) is available + // If not, switch to the first available pipeline + const availablePipelines = Object.keys(schemas.pipelines); + const preferredPipeline = "streamdiffusionv2"; + + if ( + !availablePipelines.includes(preferredPipeline) && + availablePipelines.length > 0 + ) { + const firstPipelineId = availablePipelines[0] as PipelineId; + const firstPipelineSchema = schemas.pipelines[firstPipelineId]; + + setSettings(prev => ({ + ...prev, + pipelineId: firstPipelineId, + inputMode: firstPipelineSchema.default_mode, + })); + } } else { console.error( "useStreamState: Failed to fetch pipeline schemas:", diff --git a/src/scope/core/pipelines/registry.py b/src/scope/core/pipelines/registry.py index 2959c0ab6..8c43b949b 100644 --- a/src/scope/core/pipelines/registry.py +++ b/src/scope/core/pipelines/registry.py @@ -5,9 +5,12 @@ metadata retrieval. """ +import importlib import logging from typing import TYPE_CHECKING +import torch + if TYPE_CHECKING: from .interface import Pipeline from .schema import BasePipelineConfig @@ -67,26 +70,106 @@ def list_pipelines(cls) -> list[str]: return list(cls._pipelines.keys()) +def _get_gpu_vram_gb() -> float | None: + """Get total GPU VRAM in GB if available. + + Returns: + Total VRAM in GB if GPU is available, None otherwise + """ + try: + if torch.cuda.is_available(): + _, total_mem = torch.cuda.mem_get_info(0) + return total_mem / (1024**3) + except Exception as e: + logger.warning(f"Failed to get GPU VRAM info: {e}") + return None + + +def _should_register_pipeline( + estimated_vram_gb: float | None, vram_gb: float | None +) -> bool: + """Determine if a pipeline should be registered based on GPU requirements. + + Args: + estimated_vram_gb: Estimated/required VRAM in GB from pipeline config, + or None if no requirement + vram_gb: Total GPU VRAM in GB, or None if no GPU + + Returns: + True if the pipeline should be registered, False otherwise + """ + return estimated_vram_gb is None or vram_gb is not None + + # Register all available pipelines def _register_pipelines(): - """Register all built-in pipelines.""" - # Import lazily to avoid circular imports and heavy dependencies - from .krea_realtime_video.pipeline import KreaRealtimeVideoPipeline - from .longlive.pipeline import LongLivePipeline - from .passthrough.pipeline import PassthroughPipeline - from .reward_forcing.pipeline import RewardForcingPipeline - from .streamdiffusionv2.pipeline import StreamDiffusionV2Pipeline - - # Register each pipeline with its ID from its config class - for pipeline_class in [ - LongLivePipeline, - KreaRealtimeVideoPipeline, - StreamDiffusionV2Pipeline, - PassthroughPipeline, - RewardForcingPipeline, - ]: - config_class = pipeline_class.get_config_class() - PipelineRegistry.register(config_class.pipeline_id, pipeline_class) + """Register pipelines based on GPU availability and requirements.""" + # Check GPU VRAM + vram_gb = _get_gpu_vram_gb() + + if vram_gb is not None: + logger.info(f"GPU detected with {vram_gb:.1f} GB VRAM") + else: + logger.info("No GPU detected") + + # Define pipeline imports with their module paths and class names + pipeline_configs = [ + ("passthrough", ".passthrough.pipeline", "PassthroughPipeline"), + ("longlive", ".longlive.pipeline", "LongLivePipeline"), + ( + "krea_realtime_video", + ".krea_realtime_video.pipeline", + "KreaRealtimeVideoPipeline", + ), + ( + "streamdiffusionv2", + ".streamdiffusionv2.pipeline", + "StreamDiffusionV2Pipeline", + ), + ( + "reward_forcing", + ".reward_forcing.pipeline", + "RewardForcingPipeline", + ), + ] + + # Try to import and register each pipeline + for pipeline_name, module_path, class_name in pipeline_configs: + # Try to import the pipeline first to get its config + try: + module = importlib.import_module(module_path, package=__package__) + pipeline_class = getattr(module, class_name) + + # Get the config class to check VRAM requirements + config_class = pipeline_class.get_config_class() + estimated_vram_gb = config_class.estimated_vram_gb + + # Check if pipeline meets GPU requirements + should_register = _should_register_pipeline(estimated_vram_gb, vram_gb) + if not should_register: + logger.debug( + f"Skipping {pipeline_name} pipeline - " + f"does not meet GPU requirements " + f"(required: {estimated_vram_gb} GB, " + f"available: {vram_gb} GB)" + ) + continue + + # Register the pipeline + PipelineRegistry.register(config_class.pipeline_id, pipeline_class) + logger.debug( + f"Registered {pipeline_name} pipeline (ID: {config_class.pipeline_id})" + ) + except ImportError as e: + logger.warning( + f"Could not import {pipeline_name} pipeline: {e}. " + f"This pipeline will not be available." + ) + except Exception as e: + logger.warning( + f"Error loading {pipeline_name} pipeline: {e}. " + f"This pipeline will not be available." + ) def _initialize_registry(): @@ -100,9 +183,8 @@ def _initialize_registry(): load_plugins() register_plugin_pipelines(PipelineRegistry) - logger.info( - f"Registry initialized with {len(PipelineRegistry.list_pipelines())} pipeline(s)" - ) + pipeline_count = len(PipelineRegistry.list_pipelines()) + logger.info(f"Registry initialized with {pipeline_count} pipeline(s)") # Auto-register pipelines on module import From 47ac7f559619155e1ce3c03e2fc3bb828e7e72ef Mon Sep 17 00:00:00 2001 From: Yondon Fu Date: Wed, 17 Dec 2025 19:02:56 -0500 Subject: [PATCH 10/11] Lazy import heavy model imports in pipeline init Signed-off-by: Yondon Fu --- src/scope/core/pipelines/krea_realtime_video/pipeline.py | 3 ++- src/scope/core/pipelines/longlive/pipeline.py | 3 ++- src/scope/core/pipelines/reward_forcing/pipeline.py | 3 ++- src/scope/core/pipelines/streamdiffusionv2/pipeline.py | 3 ++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/scope/core/pipelines/krea_realtime_video/pipeline.py b/src/scope/core/pipelines/krea_realtime_video/pipeline.py index 0db8065c7..5ab26ccaa 100644 --- a/src/scope/core/pipelines/krea_realtime_video/pipeline.py +++ b/src/scope/core/pipelines/krea_realtime_video/pipeline.py @@ -21,7 +21,6 @@ from ..wan2_1.lora.mixin import LoRAEnabledPipeline from ..wan2_1.vae import WanVAEWrapper from .modular_blocks import KreaRealtimeVideoBlocks -from .modules.causal_model import CausalWanModel if TYPE_CHECKING: from ..schema import BasePipelineConfig @@ -49,6 +48,8 @@ def __init__( device: torch.device | None = None, dtype: torch.dtype = torch.bfloat16, ): + from .modules.causal_model import CausalWanModel + model_dir = getattr(config, "model_dir", None) generator_path = getattr(config, "generator_path", None) text_encoder_path = getattr(config, "text_encoder_path", None) diff --git a/src/scope/core/pipelines/longlive/pipeline.py b/src/scope/core/pipelines/longlive/pipeline.py index 67d76b9a8..b2dfe99fd 100644 --- a/src/scope/core/pipelines/longlive/pipeline.py +++ b/src/scope/core/pipelines/longlive/pipeline.py @@ -23,7 +23,6 @@ from ..wan2_1.vace import VACEEnabledPipeline from ..wan2_1.vae import WanVAEWrapper from .modular_blocks import LongLiveBlocks -from .modules.causal_model import CausalWanModel if TYPE_CHECKING: from ..schema import BasePipelineConfig @@ -45,6 +44,8 @@ def __init__( device: torch.device | None = None, dtype: torch.dtype = torch.bfloat16, ): + from .modules.causal_model import CausalWanModel + model_dir = getattr(config, "model_dir", None) generator_path = getattr(config, "generator_path", None) lora_path = getattr(config, "lora_path", None) diff --git a/src/scope/core/pipelines/reward_forcing/pipeline.py b/src/scope/core/pipelines/reward_forcing/pipeline.py index 094f40bc8..ddb281928 100644 --- a/src/scope/core/pipelines/reward_forcing/pipeline.py +++ b/src/scope/core/pipelines/reward_forcing/pipeline.py @@ -22,7 +22,6 @@ from ..wan2_1.vace.mixin import VACEEnabledPipeline from ..wan2_1.vae import WanVAEWrapper from .modular_blocks import RewardForcingBlocks -from .modules.causal_model import CausalWanModel if TYPE_CHECKING: from ..schema import BasePipelineConfig @@ -44,6 +43,8 @@ def __init__( device: torch.device | None = None, dtype: torch.dtype = torch.bfloat16, ): + from .modules.causal_model import CausalWanModel + model_dir = getattr(config, "model_dir", None) generator_path = getattr(config, "generator_path", None) text_encoder_path = getattr(config, "text_encoder_path", None) diff --git a/src/scope/core/pipelines/streamdiffusionv2/pipeline.py b/src/scope/core/pipelines/streamdiffusionv2/pipeline.py index 55085eeaa..0055f96f0 100644 --- a/src/scope/core/pipelines/streamdiffusionv2/pipeline.py +++ b/src/scope/core/pipelines/streamdiffusionv2/pipeline.py @@ -22,7 +22,6 @@ from ..wan2_1.vace import VACEEnabledPipeline from .components import StreamDiffusionV2WanVAEWrapper from .modular_blocks import StreamDiffusionV2Blocks -from .modules.causal_model import CausalWanModel if TYPE_CHECKING: from ..schema import BasePipelineConfig @@ -44,6 +43,8 @@ def __init__( device: torch.device | None = None, dtype: torch.dtype = torch.bfloat16, ): + from .modules.causal_model import CausalWanModel + model_dir = getattr(config, "model_dir", None) generator_path = getattr(config, "generator_path", None) text_encoder_path = getattr(config, "text_encoder_path", None) From ceba9d0d4bc3d9b526c59d5a34e30578e347097e Mon Sep 17 00:00:00 2001 From: Yondon Fu Date: Wed, 17 Dec 2025 19:24:01 -0500 Subject: [PATCH 11/11] Fix input mode bug when switching pipelines Signed-off-by: Yondon Fu --- frontend/src/hooks/useStreamState.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/frontend/src/hooks/useStreamState.ts b/frontend/src/hooks/useStreamState.ts index d889fad5f..c8c0fe310 100644 --- a/frontend/src/hooks/useStreamState.ts +++ b/frontend/src/hooks/useStreamState.ts @@ -218,19 +218,20 @@ export function useStreamState() { fetchInitialData(); }, []); - // Update inputMode when schemas load for the first time - // This corrects the initial fallback mode to match the pipeline's actual default mode + // Update inputMode when schemas load or pipeline changes + // This sets the correct default mode for the pipeline useEffect(() => { if (pipelineSchemas) { const schema = pipelineSchemas.pipelines[settings.pipelineId]; - if (schema?.default_mode && settings.inputMode !== schema.default_mode) { + if (schema?.default_mode) { setSettings(prev => ({ ...prev, inputMode: schema.default_mode, })); } } - }, [pipelineSchemas, settings.pipelineId, settings.inputMode]); + // Only run when schemas load or pipeline changes, NOT when inputMode changes + }, [pipelineSchemas, settings.pipelineId]); // Set recommended quantization when krea-realtime-video is selected // Reset to null when switching to other pipelines