From a9069a31678008d67f5108de54c53f0a8ddabb0d Mon Sep 17 00:00:00 2001 From: Jack Tinker Date: Thu, 6 Mar 2025 16:45:29 -0500 Subject: [PATCH 01/19] fix: added scipy dependency --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index e2b8865c0..f7c79b91f 100644 --- a/setup.py +++ b/setup.py @@ -94,6 +94,7 @@ def _build_frontend(self): "Requests>=2.31.0", # NOTE: maybe need to make this server only as well? "setuptools>=69.5.1", "tomli>=2.0.1", # TODO: standardize alongside toml/tomllib + "scipy>=1.15.2" ] # Define additional dependencies for development From 30934ca0705dce042e24a98c7368f5145f7411eb Mon Sep 17 00:00:00 2001 From: joshlavroff Date: Fri, 4 Apr 2025 15:13:35 -0400 Subject: [PATCH 02/19] add: load api key for chat component from secrets.toml --- frontend/src/components/DynamicComponents.jsx | 1 + .../src/components/widgets/ChatWidget.jsx | 8 +- preswald/interfaces/components.py | 511 ++++++------------ 3 files changed, 187 insertions(+), 333 deletions(-) diff --git a/frontend/src/components/DynamicComponents.jsx b/frontend/src/components/DynamicComponents.jsx index 76eea7c9f..bc99be909 100644 --- a/frontend/src/components/DynamicComponents.jsx +++ b/frontend/src/components/DynamicComponents.jsx @@ -218,6 +218,7 @@ const MemoizedComponent = memo( {...props} sourceId={component.config?.source || null} sourceData={component.config?.data || null} + apiKey={component.config?.apiKey || null} value={component.value || component.state || { messages: [] }} onChange={(value) => { handleUpdate(componentId, value); diff --git a/frontend/src/components/widgets/ChatWidget.jsx b/frontend/src/components/widgets/ChatWidget.jsx index 7400d9a1d..8ed509780 100644 --- a/frontend/src/components/widgets/ChatWidget.jsx +++ b/frontend/src/components/widgets/ChatWidget.jsx @@ -14,6 +14,7 @@ import { createChatCompletion } from '@/services/openai'; const ChatWidget = ({ sourceId = null, sourceData = null, + apiKey = null, value = { messages: [] }, onChange, className, @@ -23,9 +24,14 @@ const ChatWidget = ({ const messagesEndRef = useRef(null); const chatContainerRef = useRef(null); + // Load API key from secrets.toml if present + if (apiKey) { + sessionStorage.setItem('openai_api_key', apiKey.trim()); + } + const [inputValue, setInputValue] = useState(''); const [showSettings, setShowSettings] = useState(false); - const [apiKey, setApiKey] = useState(''); + // const [apiKey, setApiKey] = useState(''); const [isLoading, setIsLoading] = useState(false); const [error, setError] = useState(null); const hasApiKey = useMemo(() => !!sessionStorage.getItem('openai_api_key'), []); diff --git a/preswald/interfaces/components.py b/preswald/interfaces/components.py index 943c33056..b56701f17 100644 --- a/preswald/interfaces/components.py +++ b/preswald/interfaces/components.py @@ -4,24 +4,18 @@ import io import json import logging -import re import uuid from typing import Dict, List, Optional # Third-Party +import fastplotlib as fplt import matplotlib.pyplot as plt +import msgpack import numpy as np import pandas as pd +import tomllib +from PIL import Image -# from PIL import Image -# try: -# import fastplotlib as fplt -# import msgpack -# -# FASTPLOTLIB_AVAILABLE = True -# except ImportError: -# FASTPLOTLIB_AVAILABLE = False -# fplt = None # Internal from preswald.engine.service import PreswaldService from preswald.interfaces.workflow import Workflow @@ -53,36 +47,16 @@ def alert(message: str, level: str = "info", size: float = 1.0) -> str: return message -def button( - label: str, - variant: str = "default", - disabled: bool = False, - loading: bool = False, - size: float = 1.0, -) -> bool: - """Create a button component that returns True when clicked.""" +# TODO: Add button functionality +def button(label: str, size: float = 1.0): + """Create a button component.""" service = PreswaldService.get_instance() - component_id = generate_id_by_label("button", label) - - # Get current state or use default - current_value = service.get_component_state(component_id) - if current_value is None: - current_value = False - - component = { - "type": "button", - "id": component_id, - "label": label, - "variant": variant, - "disabled": disabled, - "loading": loading, - "size": size, - "value": current_value, - "onClick": True, # Always enable click handling - } - + id = generate_id("button") + logger.debug(f"Creating button component with id {id}, label: {label}") + component = {"type": "button", "id": id, "label": label, "size": size} + logger.debug(f"Created component: {component}") service.append_component(component) - return current_value + return component def chat(source: str, table: Optional[str] = None) -> Dict: @@ -97,6 +71,16 @@ def chat(source: str, table: Optional[str] = None) -> Dict: if current_state is None: current_state = {"messages": [], "source": source} + # Get API key from secrets.toml + with open("secrets.toml", "rb") as toml: + secrets = tomllib.load(toml) + + if secrets["data"]["openai"]["api_key"]: + api_key = secrets["data"]["openai"]["api_key"] + + else: + api_key = None + # Get dataframe from source df = ( service.data_manager.get_df(source) @@ -132,6 +116,7 @@ def chat(source: str, table: Optional[str] = None) -> Dict: "config": { "source": source, "data": serializable_data, + "apiKey": api_key, }, } @@ -165,44 +150,38 @@ def checkbox(label: str, default: bool = False, size: float = 1.0) -> bool: return current_value -# def fastplotlib(fig: "fplt.Figure", size: float = 1.0) -> str: +# def fastplotlib(fig: fplt.Figure, size: float = 1.0) -> str: # """ # Render a Fastplotlib figure and asynchronously stream the resulting image to the frontend. -# + # This component leverages Fastplotlib's GPU-accelerated offscreen rendering capabilities. # Rendering and transmission are triggered only when the figure state or the client changes, # ensuring efficient updates. The rendered image is encoded as a PNG and sent to the frontend # via WebSocket using MessagePack. -# + # Args: # fig (fplt.Figure): A configured Fastplotlib figure ready to be rendered. # size (float, optional): Width of the rendered component relative to its container (0.0-1.0). # Defaults to 1.0. -# + # Returns: # str: A deterministic component ID used to reference the figure on the frontend. -# + # Notes: # - The figure must have '_label' and '_client_id' attributes set externally. # - Rendering occurs asynchronously if the figure state or client_id changes. # - If client_id is not provided, a warning is logged and no rendering task is triggered. # """ -# if not FASTPLOTLIB_AVAILABLE: -# logger.warning( -# "fastplotlib is not available. Please install it with 'pip install fastplotlib'" -# ) -# return None -# # service = PreswaldService.get_instance() -# + # label = getattr(fig, "_label", "fastplotlib") # component_id = generate_id_by_label("fastplotlib", label) -# + # try: # state = fig.get_state() # except Exception: # state = label -# + # # hash input data early and use hash to avoid unnecessary rendering # client_id = getattr(fig, "_client_id", None) # hashable_data = { @@ -212,7 +191,7 @@ def checkbox(label: str, default: bool = False, size: float = 1.0) -> bool: # "size": size, # } # data_hash = hashlib.sha256(msgpack.packb(hashable_data)).hexdigest() -# + # component = { # "id": component_id, # "type": "fastplotlib_component", @@ -222,7 +201,7 @@ def checkbox(label: str, default: bool = False, size: float = 1.0) -> bool: # "value": None, # "hash": data_hash[:8], # } -# + # # skip rendering if unchanged # if data_hash != service.get_component_state(f"{component_id}_img_hash"): # if client_id: @@ -234,7 +213,7 @@ def checkbox(label: str, default: bool = False, size: float = 1.0) -> bool: # ) # else: # logger.warning(f"No client_id provided for {component_id}") -# + # service.append_component(component) # return component_id @@ -280,106 +259,6 @@ def matplotlib(fig: Optional[plt.Figure] = None, label: str = "plot") -> str: return component_id # Returning ID for potential tracking -def playground( - label: str, query: str, source: str | None = None, size: float = 1.0 -) -> pd.DataFrame: - """ - Create a playground component for interactive data querying and visualization. - - Args: - label (str): The label for the playground component (used for identification). - query (str): The SQL query string to be executed. - source (str, optional): The name of the data source to query from. All data sources are considered by default. - size (float, optional): The visual size/scale of the component. Defaults to 1.0. - - Returns: - pd.DataFrame: The queried data as a pandas DataFrame. - """ - - # Get the singleton instance of the PreswaldService - service = PreswaldService.get_instance() - - # Generate a unique component ID using the label's hash - component_id = f"playground-{hashlib.md5(label.encode()).hexdigest()[:8]}" - - logger.debug( - f"Creating playground component with id {component_id}, label: {label}" - ) - - # Retrieve the current query state (if previously modified by the user) - # If no previous state, use the provided query - current_query_value = service.get_component_state(component_id) - if current_query_value is None: - current_query_value = query - - # Initialize data_source with the provided source or auto-detect it - data_source = source - if source is None: - # Auto-extract the first table name from the SQL query using regex - # Handles 'FROM' and 'JOIN' clauses with optional backticks or quotes - fetched_sources = re.findall( - r'(?:FROM|JOIN)\s+[`"]?([a-zA-Z0-9_\.]+)[`"]?', - current_query_value, - re.IGNORECASE | re.DOTALL, - ) - # Use the first detected source as the data source - data_source = fetched_sources[0] if fetched_sources else None - - # Initialize placeholders for data and error - data = None - error = None - processed_data = None - column_defs = [] - - # Attempt to execute the query against the determined data source - try: - data = service.data_manager.query(current_query_value, data_source) - logger.debug(f"Successfully queried data source: {data_source}") - - # Process data for the table - if isinstance(data, pd.DataFrame): - data = data.reset_index(drop=True) - processed_data = data.to_dict("records") - column_defs = [ - {"headerName": str(col), "field": str(col)} for col in data.columns - ] - - # Process each row to ensure JSON serialization - processed_data = [] - for _, row in data.iterrows(): - processed_row = { - str(key): ( - value.item() - if isinstance(value, (np.integer, np.floating)) - else value - ) - if value is not None - else "" # Ensure no None values - for key, value in row.items() - } - processed_data.append(processed_row) - except Exception as e: - error = str(e) - logger.error(f"Error querying data source: {e}") - - component = { - "type": "playground", - "id": component_id, - "label": label, - "source": source, - "value": current_query_value, - "size": size, - "error": error, - "data": {"columnDefs": column_defs, "rowData": processed_data or []}, - } - - logger.debug(f"Created component: {component}") - service.append_component(component) - - # Return the raw DataFrame - return data - - def plotly(fig, size: float = 1.0) -> Dict: # noqa: C901 """ Render a Plotly figure. @@ -609,37 +488,19 @@ def slider( return current_value -def spinner( - label: str = "Loading...", - variant: str = "default", - show_label: bool = True, - size: float = 1.0, -) -> None: - """Create a loading spinner component. - - Args: - label: Text to show below the spinner - variant: Visual style ("default" or "card") - show_label: Whether to show the label text - size: Component width (1.0 = full width) - """ +# TODO: requires testing +def spinner(label: str, size: float = 1.0): + """Create a spinner component.""" service = PreswaldService.get_instance() - component_id = generate_id("spinner") - - component = { - "type": "spinner", - "id": component_id, - "label": label, - "variant": variant, - "showLabel": show_label, - "size": size, - } - + id = generate_id("spinner") + logger.debug(f"Creating spinner component with id {id}, label: {label}") + component = {"type": "spinner", "id": id, "label": label, "size": size} + logger.debug(f"Created component: {component}") service.append_component(component) - return None + return component -def sidebar(defaultopen: bool = False): +def sidebar(defaultopen: bool): """Create a sidebar component.""" service = PreswaldService.get_instance() id = generate_id("sidebar") @@ -650,25 +511,24 @@ def sidebar(defaultopen: bool = False): return component -def table( +def table( # noqa: C901 data: pd.DataFrame, title: Optional[str] = None, limit: Optional[int] = None ) -> Dict: """Create a table component that renders data using TableViewerWidget. Args: - data: Pandas DataFrame or list of dictionaries to display. - title: Optional title for the table. - limit: Optional limit for rows displayed. + data: Pandas DataFrame or list of dictionaries to display + title: Optional title for the table Returns: - Dict: Component metadata and processed data. + Dict: Component metadata and processed data """ id = generate_id("table") logger.debug(f"Creating table component with id {id}") service = PreswaldService.get_instance() try: - # Convert pandas DataFrame to a list of dictionaries if needed + # Convert pandas DataFrame to list of dictionaries if needed if hasattr(data, "to_dict"): if isinstance(data, pd.DataFrame): data = data.reset_index(drop=True) @@ -680,49 +540,49 @@ def table( if not isinstance(data, list): data = [data] if data else [] - # Ensure data is not empty before accessing keys - if data and isinstance(data[0], dict): - column_defs = [ - {"headerName": str(col), "field": str(col)} for col in data[0].keys() - ] - else: - column_defs = [] - - # Process each row to ensure JSON serialization + # Convert each row to ensure JSON serialization processed_data = [] for row in data: - processed_row = { - str(key): ( - value.item() - if isinstance(value, (np.integer, np.floating)) - else value - ) - if value is not None - else "" # Ensure no None values - for key, value in row.items() - } - processed_data.append(processed_row) - - # Log debug info - logger.debug(f"Column Definitions: {column_defs}") - logger.debug( - f"Processed Data (first 5 rows): {processed_data[:5]}" - ) # Limit logs + if isinstance(row, dict): + processed_row = {} + for key, value in row.items(): + # Convert key to string to ensure it's serializable + key_str = str(key) + + # Handle special cases and convert value + if pd.isna(value): + processed_row[key_str] = None + elif isinstance(value, (pd.Timestamp, pd.DatetimeTZDtype)): + processed_row[key_str] = str(value) + elif isinstance(value, (np.integer, np.floating)): + processed_row[key_str] = value.item() + elif isinstance(value, (list, np.ndarray)): + processed_row[key_str] = convert_to_serializable(value) + else: + try: + # Try to serialize to test if it's JSON-compatible + json.dumps(value) + processed_row[key_str] = value + except: # noqa: E722 + # If serialization fails, convert to string + processed_row[key_str] = str(value) + processed_data.append(processed_row) + else: + # If row is not a dict, convert it to a simple dict + processed_data.append({"value": str(row)}) - # Create AG Grid compatible component structure + # Create the component structure component = { "type": "table", "id": id, - "props": { - "columnDefs": column_defs, - "rowData": processed_data, - "title": str(title) if title else None, - }, + "data": processed_data, + "title": str(title) if title is not None else None, } # Verify JSON serialization before returning json.dumps(component) - logger.debug(f"Created AG Grid table component: {component}") + + logger.debug(f"Created table component: {component}") service.append_component(component) return component @@ -731,11 +591,8 @@ def table( error_component = { "type": "table", "id": id, - "props": { - "columnDefs": [], - "rowData": [], - "title": f"Error: {e!s}", - }, + "data": [], + "title": f"Error: {e!s}", } service.append_component(error_component) return error_component @@ -758,31 +615,21 @@ def text(markdown_str: str, size: float = 1.0) -> str: return markdown_str -def text_input( - label: str, - placeholder: str = "", - default: str = "", - size: float = 1.0, -) -> str: - """Create a text input component. - - Args: - label: Label text shown above the input - placeholder: Placeholder text shown when input is empty - default: Initial value for the input - size: Component width (1.0 = full width) - - Returns: - str: Current value of the input - """ +def text_input(label: str, placeholder: str = "", size: float = 1.0) -> str: + """Create a text input component with consistent ID based on label.""" service = PreswaldService.get_instance() + + # Create a consistent ID based on the label component_id = generate_id_by_label("text_input", label) # Get current state or use default current_value = service.get_component_state(component_id) if current_value is None: - current_value = default + current_value = "" + logger.debug( + f"Creating text input component with id {component_id}, label: {label}" + ) component = { "type": "text_input", "id": component_id, @@ -791,7 +638,7 @@ def text_input( "value": current_value, "size": size, } - + logger.debug(f"Created component: {component}") service.append_component(component) return current_value @@ -922,90 +769,90 @@ def generate_id_by_label(prefix: str, label: str) -> str: return f"{prefix}-{hashed}" -# async def render_and_send_fastplotlib( -# fig: "fplt.Figure", -# component_id: str, -# label: str, -# size: float, -# client_id: str, -# data_hash: str, -# ) -> Optional[str]: -# """ -# Asynchronously renders a Fastplotlib figure to an offscreen canvas, encodes it as a PNG, -# and streams the resulting image data via WebSocket to the connected frontend client. -# -# This helper function handles rendering logic, alpha-blending, and ensures robust error -# handling. It updates the component state after successfully sending the image data. -# -# Args: -# fig (fplt.Figure): The fully configured Fastplotlib figure instance to render. -# component_id (str): Unique identifier for the component instance receiving this image. -# label (str): Human-readable label describing the component (for logging/debugging). -# size (float): Relative size of the component in the UI layout (0.0-1.0). -# client_id (str): WebSocket client identifier to route the rendered image correctly. -# data_hash (str): SHA-256 hash representing the figure state, used for cache invalidation. -# -# Returns: -# str: Returns "Render failed" if framebuffer blending fails, otherwise None. -# -# Raises: -# Logs and handles any exceptions internally without raising further. -# """ -# service = PreswaldService.get_instance() -# -# fig.show() # must call even in offscreen mode to initialize GPU resources -# -# # manually render the scene for all subplots -# for subplot in fig: -# subplot.viewport.render(subplot.scene, subplot.camera) -# -# # read from the framebuffer -# try: -# fig.canvas.request_draw() -# raw_img = np.asarray(fig.renderer.target.draw()) -# -# if raw_img.ndim != 3 or raw_img.shape[2] != 4: -# raise ValueError(f"Unexpected image shape: {raw_img.shape}") -# -# # handle alpha blending -# alpha = raw_img[..., 3:4] / 255.0 -# rgb = (raw_img[..., :3] * alpha + (1 - alpha) * 255).astype(np.uint8) -# -# except Exception as e: -# logger.error(f"Framebuffer blending failed for {component_id}: {e}") -# return "Render failed" -# -# # encode image to PNG -# img_buf = io.BytesIO() -# Image.fromarray(rgb).save(img_buf, format="PNG") -# png_bytes = img_buf.getvalue() -# -# # handle websocket communication -# client_websocket = service.websocket_connections.get(client_id) -# if client_websocket: -# packed_msg = msgpack.packb( -# { -# "type": "image_update", -# "component_id": component_id, -# "format": "png", -# "label": label, -# "size": size, -# "data": png_bytes, -# }, -# use_bin_type=True, -# ) -# -# try: -# await client_websocket.send_bytes(packed_msg) -# await service.handle_client_message( -# client_id, -# { -# "type": "component_update", -# "states": {f"{component_id}_img_hash": data_hash}, -# }, -# ) -# logger.debug(f"✅ Sent {component_id} image to client {client_id}") -# except Exception as e: -# logger.error(f"WebSocket send failed for {component_id}: {e}") -# else: -# logger.warning(f"No active WebSocket found for client ID: {client_id}") +async def render_and_send_fastplotlib( + fig: fplt.Figure, + component_id: str, + label: str, + size: float, + client_id: str, + data_hash: str, +) -> Optional[str]: + """ + Asynchronously renders a Fastplotlib figure to an offscreen canvas, encodes it as a PNG, + and streams the resulting image data via WebSocket to the connected frontend client. + + This helper function handles rendering logic, alpha-blending, and ensures robust error + handling. It updates the component state after successfully sending the image data. + + Args: + fig (fplt.Figure): The fully configured Fastplotlib figure instance to render. + component_id (str): Unique identifier for the component instance receiving this image. + label (str): Human-readable label describing the component (for logging/debugging). + size (float): Relative size of the component in the UI layout (0.0-1.0). + client_id (str): WebSocket client identifier to route the rendered image correctly. + data_hash (str): SHA-256 hash representing the figure state, used for cache invalidation. + + Returns: + str: Returns "Render failed" if framebuffer blending fails, otherwise None. + + Raises: + Logs and handles any exceptions internally without raising further. + """ + service = PreswaldService.get_instance() + + fig.show() # must call even in offscreen mode to initialize GPU resources + + # manually render the scene for all subplots + for subplot in fig: + subplot.viewport.render(subplot.scene, subplot.camera) + + # read from the framebuffer + try: + fig.canvas.request_draw() + raw_img = np.asarray(fig.renderer.target.draw()) + + if raw_img.ndim != 3 or raw_img.shape[2] != 4: + raise ValueError(f"Unexpected image shape: {raw_img.shape}") + + # handle alpha blending + alpha = raw_img[..., 3:4] / 255.0 + rgb = (raw_img[..., :3] * alpha + (1 - alpha) * 255).astype(np.uint8) + + except Exception as e: + logger.error(f"Framebuffer blending failed for {component_id}: {e}") + return "Render failed" + + # encode image to PNG + img_buf = io.BytesIO() + Image.fromarray(rgb).save(img_buf, format="PNG") + png_bytes = img_buf.getvalue() + + # handle websocket communication + client_websocket = service.websocket_connections.get(client_id) + if client_websocket: + packed_msg = msgpack.packb( + { + "type": "image_update", + "component_id": component_id, + "format": "png", + "label": label, + "size": size, + "data": png_bytes, + }, + use_bin_type=True, + ) + + try: + await client_websocket.send_bytes(packed_msg) + await service.handle_client_message( + client_id, + { + "type": "component_update", + "states": {f"{component_id}_img_hash": data_hash}, + }, + ) + logger.debug(f"✅ Sent {component_id} image to client {client_id}") + except Exception as e: + logger.error(f"WebSocket send failed for {component_id}: {e}") + else: + logger.warning(f"No active WebSocket found for client ID: {client_id}") From f149b8c71adf2c52d27b201d0fa07fb21720d640 Mon Sep 17 00:00:00 2001 From: joshlavroff Date: Sat, 5 Apr 2025 16:10:55 -0400 Subject: [PATCH 03/19] fix: update components.py to newest release --- preswald/interfaces/components.py | 501 ++++++++++++++++++++---------- 1 file changed, 333 insertions(+), 168 deletions(-) diff --git a/preswald/interfaces/components.py b/preswald/interfaces/components.py index b56701f17..dc8282c84 100644 --- a/preswald/interfaces/components.py +++ b/preswald/interfaces/components.py @@ -4,18 +4,25 @@ import io import json import logging +import re import uuid from typing import Dict, List, Optional # Third-Party -import fastplotlib as fplt import matplotlib.pyplot as plt -import msgpack import numpy as np import pandas as pd import tomllib -from PIL import Image +# from PIL import Image +# try: +# import fastplotlib as fplt +# import msgpack +# +# FASTPLOTLIB_AVAILABLE = True +# except ImportError: +# FASTPLOTLIB_AVAILABLE = False +# fplt = None # Internal from preswald.engine.service import PreswaldService from preswald.interfaces.workflow import Workflow @@ -47,16 +54,36 @@ def alert(message: str, level: str = "info", size: float = 1.0) -> str: return message -# TODO: Add button functionality -def button(label: str, size: float = 1.0): - """Create a button component.""" +def button( + label: str, + variant: str = "default", + disabled: bool = False, + loading: bool = False, + size: float = 1.0, +) -> bool: + """Create a button component that returns True when clicked.""" service = PreswaldService.get_instance() - id = generate_id("button") - logger.debug(f"Creating button component with id {id}, label: {label}") - component = {"type": "button", "id": id, "label": label, "size": size} - logger.debug(f"Created component: {component}") + component_id = generate_id_by_label("button", label) + + # Get current state or use default + current_value = service.get_component_state(component_id) + if current_value is None: + current_value = False + + component = { + "type": "button", + "id": component_id, + "label": label, + "variant": variant, + "disabled": disabled, + "loading": loading, + "size": size, + "value": current_value, + "onClick": True, # Always enable click handling + } + service.append_component(component) - return component + return current_value def chat(source: str, table: Optional[str] = None) -> Dict: @@ -75,7 +102,7 @@ def chat(source: str, table: Optional[str] = None) -> Dict: with open("secrets.toml", "rb") as toml: secrets = tomllib.load(toml) - if secrets["data"]["openai"]["api_key"]: + if secrets and secrets["data"]["openai"]["api_key"]: api_key = secrets["data"]["openai"]["api_key"] else: @@ -150,38 +177,44 @@ def checkbox(label: str, default: bool = False, size: float = 1.0) -> bool: return current_value -# def fastplotlib(fig: fplt.Figure, size: float = 1.0) -> str: +# def fastplotlib(fig: "fplt.Figure", size: float = 1.0) -> str: # """ # Render a Fastplotlib figure and asynchronously stream the resulting image to the frontend. - +# # This component leverages Fastplotlib's GPU-accelerated offscreen rendering capabilities. # Rendering and transmission are triggered only when the figure state or the client changes, # ensuring efficient updates. The rendered image is encoded as a PNG and sent to the frontend # via WebSocket using MessagePack. - +# # Args: # fig (fplt.Figure): A configured Fastplotlib figure ready to be rendered. # size (float, optional): Width of the rendered component relative to its container (0.0-1.0). # Defaults to 1.0. - +# # Returns: # str: A deterministic component ID used to reference the figure on the frontend. - +# # Notes: # - The figure must have '_label' and '_client_id' attributes set externally. # - Rendering occurs asynchronously if the figure state or client_id changes. # - If client_id is not provided, a warning is logged and no rendering task is triggered. # """ +# if not FASTPLOTLIB_AVAILABLE: +# logger.warning( +# "fastplotlib is not available. Please install it with 'pip install fastplotlib'" +# ) +# return None +# # service = PreswaldService.get_instance() - +# # label = getattr(fig, "_label", "fastplotlib") # component_id = generate_id_by_label("fastplotlib", label) - +# # try: # state = fig.get_state() # except Exception: # state = label - +# # # hash input data early and use hash to avoid unnecessary rendering # client_id = getattr(fig, "_client_id", None) # hashable_data = { @@ -191,7 +224,7 @@ def checkbox(label: str, default: bool = False, size: float = 1.0) -> bool: # "size": size, # } # data_hash = hashlib.sha256(msgpack.packb(hashable_data)).hexdigest() - +# # component = { # "id": component_id, # "type": "fastplotlib_component", @@ -201,7 +234,7 @@ def checkbox(label: str, default: bool = False, size: float = 1.0) -> bool: # "value": None, # "hash": data_hash[:8], # } - +# # # skip rendering if unchanged # if data_hash != service.get_component_state(f"{component_id}_img_hash"): # if client_id: @@ -213,7 +246,7 @@ def checkbox(label: str, default: bool = False, size: float = 1.0) -> bool: # ) # else: # logger.warning(f"No client_id provided for {component_id}") - +# # service.append_component(component) # return component_id @@ -259,6 +292,106 @@ def matplotlib(fig: Optional[plt.Figure] = None, label: str = "plot") -> str: return component_id # Returning ID for potential tracking +def playground( + label: str, query: str, source: str | None = None, size: float = 1.0 +) -> pd.DataFrame: + """ + Create a playground component for interactive data querying and visualization. + + Args: + label (str): The label for the playground component (used for identification). + query (str): The SQL query string to be executed. + source (str, optional): The name of the data source to query from. All data sources are considered by default. + size (float, optional): The visual size/scale of the component. Defaults to 1.0. + + Returns: + pd.DataFrame: The queried data as a pandas DataFrame. + """ + + # Get the singleton instance of the PreswaldService + service = PreswaldService.get_instance() + + # Generate a unique component ID using the label's hash + component_id = f"playground-{hashlib.md5(label.encode()).hexdigest()[:8]}" + + logger.debug( + f"Creating playground component with id {component_id}, label: {label}" + ) + + # Retrieve the current query state (if previously modified by the user) + # If no previous state, use the provided query + current_query_value = service.get_component_state(component_id) + if current_query_value is None: + current_query_value = query + + # Initialize data_source with the provided source or auto-detect it + data_source = source + if source is None: + # Auto-extract the first table name from the SQL query using regex + # Handles 'FROM' and 'JOIN' clauses with optional backticks or quotes + fetched_sources = re.findall( + r'(?:FROM|JOIN)\s+[`"]?([a-zA-Z0-9_\.]+)[`"]?', + current_query_value, + re.IGNORECASE | re.DOTALL, + ) + # Use the first detected source as the data source + data_source = fetched_sources[0] if fetched_sources else None + + # Initialize placeholders for data and error + data = None + error = None + processed_data = None + column_defs = [] + + # Attempt to execute the query against the determined data source + try: + data = service.data_manager.query(current_query_value, data_source) + logger.debug(f"Successfully queried data source: {data_source}") + + # Process data for the table + if isinstance(data, pd.DataFrame): + data = data.reset_index(drop=True) + processed_data = data.to_dict("records") + column_defs = [ + {"headerName": str(col), "field": str(col)} for col in data.columns + ] + + # Process each row to ensure JSON serialization + processed_data = [] + for _, row in data.iterrows(): + processed_row = { + str(key): ( + value.item() + if isinstance(value, (np.integer, np.floating)) + else value + ) + if value is not None + else "" # Ensure no None values + for key, value in row.items() + } + processed_data.append(processed_row) + except Exception as e: + error = str(e) + logger.error(f"Error querying data source: {e}") + + component = { + "type": "playground", + "id": component_id, + "label": label, + "source": source, + "value": current_query_value, + "size": size, + "error": error, + "data": {"columnDefs": column_defs, "rowData": processed_data or []}, + } + + logger.debug(f"Created component: {component}") + service.append_component(component) + + # Return the raw DataFrame + return data + + def plotly(fig, size: float = 1.0) -> Dict: # noqa: C901 """ Render a Plotly figure. @@ -488,19 +621,37 @@ def slider( return current_value -# TODO: requires testing -def spinner(label: str, size: float = 1.0): - """Create a spinner component.""" +def spinner( + label: str = "Loading...", + variant: str = "default", + show_label: bool = True, + size: float = 1.0, +) -> None: + """Create a loading spinner component. + + Args: + label: Text to show below the spinner + variant: Visual style ("default" or "card") + show_label: Whether to show the label text + size: Component width (1.0 = full width) + """ service = PreswaldService.get_instance() - id = generate_id("spinner") - logger.debug(f"Creating spinner component with id {id}, label: {label}") - component = {"type": "spinner", "id": id, "label": label, "size": size} - logger.debug(f"Created component: {component}") + component_id = generate_id("spinner") + + component = { + "type": "spinner", + "id": component_id, + "label": label, + "variant": variant, + "showLabel": show_label, + "size": size, + } + service.append_component(component) - return component + return None -def sidebar(defaultopen: bool): +def sidebar(defaultopen: bool = False): """Create a sidebar component.""" service = PreswaldService.get_instance() id = generate_id("sidebar") @@ -511,24 +662,25 @@ def sidebar(defaultopen: bool): return component -def table( # noqa: C901 +def table( data: pd.DataFrame, title: Optional[str] = None, limit: Optional[int] = None ) -> Dict: """Create a table component that renders data using TableViewerWidget. Args: - data: Pandas DataFrame or list of dictionaries to display - title: Optional title for the table + data: Pandas DataFrame or list of dictionaries to display. + title: Optional title for the table. + limit: Optional limit for rows displayed. Returns: - Dict: Component metadata and processed data + Dict: Component metadata and processed data. """ id = generate_id("table") logger.debug(f"Creating table component with id {id}") service = PreswaldService.get_instance() try: - # Convert pandas DataFrame to list of dictionaries if needed + # Convert pandas DataFrame to a list of dictionaries if needed if hasattr(data, "to_dict"): if isinstance(data, pd.DataFrame): data = data.reset_index(drop=True) @@ -540,49 +692,49 @@ def table( # noqa: C901 if not isinstance(data, list): data = [data] if data else [] - # Convert each row to ensure JSON serialization + # Ensure data is not empty before accessing keys + if data and isinstance(data[0], dict): + column_defs = [ + {"headerName": str(col), "field": str(col)} for col in data[0].keys() + ] + else: + column_defs = [] + + # Process each row to ensure JSON serialization processed_data = [] for row in data: - if isinstance(row, dict): - processed_row = {} - for key, value in row.items(): - # Convert key to string to ensure it's serializable - key_str = str(key) - - # Handle special cases and convert value - if pd.isna(value): - processed_row[key_str] = None - elif isinstance(value, (pd.Timestamp, pd.DatetimeTZDtype)): - processed_row[key_str] = str(value) - elif isinstance(value, (np.integer, np.floating)): - processed_row[key_str] = value.item() - elif isinstance(value, (list, np.ndarray)): - processed_row[key_str] = convert_to_serializable(value) - else: - try: - # Try to serialize to test if it's JSON-compatible - json.dumps(value) - processed_row[key_str] = value - except: # noqa: E722 - # If serialization fails, convert to string - processed_row[key_str] = str(value) - processed_data.append(processed_row) - else: - # If row is not a dict, convert it to a simple dict - processed_data.append({"value": str(row)}) + processed_row = { + str(key): ( + value.item() + if isinstance(value, (np.integer, np.floating)) + else value + ) + if value is not None + else "" # Ensure no None values + for key, value in row.items() + } + processed_data.append(processed_row) + + # Log debug info + logger.debug(f"Column Definitions: {column_defs}") + logger.debug( + f"Processed Data (first 5 rows): {processed_data[:5]}" + ) # Limit logs - # Create the component structure + # Create AG Grid compatible component structure component = { "type": "table", "id": id, - "data": processed_data, - "title": str(title) if title is not None else None, + "props": { + "columnDefs": column_defs, + "rowData": processed_data, + "title": str(title) if title else None, + }, } # Verify JSON serialization before returning json.dumps(component) - - logger.debug(f"Created table component: {component}") + logger.debug(f"Created AG Grid table component: {component}") service.append_component(component) return component @@ -591,8 +743,11 @@ def table( # noqa: C901 error_component = { "type": "table", "id": id, - "data": [], - "title": f"Error: {e!s}", + "props": { + "columnDefs": [], + "rowData": [], + "title": f"Error: {e!s}", + }, } service.append_component(error_component) return error_component @@ -615,21 +770,31 @@ def text(markdown_str: str, size: float = 1.0) -> str: return markdown_str -def text_input(label: str, placeholder: str = "", size: float = 1.0) -> str: - """Create a text input component with consistent ID based on label.""" - service = PreswaldService.get_instance() +def text_input( + label: str, + placeholder: str = "", + default: str = "", + size: float = 1.0, +) -> str: + """Create a text input component. - # Create a consistent ID based on the label + Args: + label: Label text shown above the input + placeholder: Placeholder text shown when input is empty + default: Initial value for the input + size: Component width (1.0 = full width) + + Returns: + str: Current value of the input + """ + service = PreswaldService.get_instance() component_id = generate_id_by_label("text_input", label) # Get current state or use default current_value = service.get_component_state(component_id) if current_value is None: - current_value = "" + current_value = default - logger.debug( - f"Creating text input component with id {component_id}, label: {label}" - ) component = { "type": "text_input", "id": component_id, @@ -638,7 +803,7 @@ def text_input(label: str, placeholder: str = "", size: float = 1.0) -> str: "value": current_value, "size": size, } - logger.debug(f"Created component: {component}") + service.append_component(component) return current_value @@ -769,90 +934,90 @@ def generate_id_by_label(prefix: str, label: str) -> str: return f"{prefix}-{hashed}" -async def render_and_send_fastplotlib( - fig: fplt.Figure, - component_id: str, - label: str, - size: float, - client_id: str, - data_hash: str, -) -> Optional[str]: - """ - Asynchronously renders a Fastplotlib figure to an offscreen canvas, encodes it as a PNG, - and streams the resulting image data via WebSocket to the connected frontend client. - - This helper function handles rendering logic, alpha-blending, and ensures robust error - handling. It updates the component state after successfully sending the image data. - - Args: - fig (fplt.Figure): The fully configured Fastplotlib figure instance to render. - component_id (str): Unique identifier for the component instance receiving this image. - label (str): Human-readable label describing the component (for logging/debugging). - size (float): Relative size of the component in the UI layout (0.0-1.0). - client_id (str): WebSocket client identifier to route the rendered image correctly. - data_hash (str): SHA-256 hash representing the figure state, used for cache invalidation. - - Returns: - str: Returns "Render failed" if framebuffer blending fails, otherwise None. - - Raises: - Logs and handles any exceptions internally without raising further. - """ - service = PreswaldService.get_instance() - - fig.show() # must call even in offscreen mode to initialize GPU resources - - # manually render the scene for all subplots - for subplot in fig: - subplot.viewport.render(subplot.scene, subplot.camera) - - # read from the framebuffer - try: - fig.canvas.request_draw() - raw_img = np.asarray(fig.renderer.target.draw()) - - if raw_img.ndim != 3 or raw_img.shape[2] != 4: - raise ValueError(f"Unexpected image shape: {raw_img.shape}") - - # handle alpha blending - alpha = raw_img[..., 3:4] / 255.0 - rgb = (raw_img[..., :3] * alpha + (1 - alpha) * 255).astype(np.uint8) - - except Exception as e: - logger.error(f"Framebuffer blending failed for {component_id}: {e}") - return "Render failed" - - # encode image to PNG - img_buf = io.BytesIO() - Image.fromarray(rgb).save(img_buf, format="PNG") - png_bytes = img_buf.getvalue() - - # handle websocket communication - client_websocket = service.websocket_connections.get(client_id) - if client_websocket: - packed_msg = msgpack.packb( - { - "type": "image_update", - "component_id": component_id, - "format": "png", - "label": label, - "size": size, - "data": png_bytes, - }, - use_bin_type=True, - ) - - try: - await client_websocket.send_bytes(packed_msg) - await service.handle_client_message( - client_id, - { - "type": "component_update", - "states": {f"{component_id}_img_hash": data_hash}, - }, - ) - logger.debug(f"✅ Sent {component_id} image to client {client_id}") - except Exception as e: - logger.error(f"WebSocket send failed for {component_id}: {e}") - else: - logger.warning(f"No active WebSocket found for client ID: {client_id}") +# async def render_and_send_fastplotlib( +# fig: "fplt.Figure", +# component_id: str, +# label: str, +# size: float, +# client_id: str, +# data_hash: str, +# ) -> Optional[str]: +# """ +# Asynchronously renders a Fastplotlib figure to an offscreen canvas, encodes it as a PNG, +# and streams the resulting image data via WebSocket to the connected frontend client. +# +# This helper function handles rendering logic, alpha-blending, and ensures robust error +# handling. It updates the component state after successfully sending the image data. +# +# Args: +# fig (fplt.Figure): The fully configured Fastplotlib figure instance to render. +# component_id (str): Unique identifier for the component instance receiving this image. +# label (str): Human-readable label describing the component (for logging/debugging). +# size (float): Relative size of the component in the UI layout (0.0-1.0). +# client_id (str): WebSocket client identifier to route the rendered image correctly. +# data_hash (str): SHA-256 hash representing the figure state, used for cache invalidation. +# +# Returns: +# str: Returns "Render failed" if framebuffer blending fails, otherwise None. +# +# Raises: +# Logs and handles any exceptions internally without raising further. +# """ +# service = PreswaldService.get_instance() +# +# fig.show() # must call even in offscreen mode to initialize GPU resources +# +# # manually render the scene for all subplots +# for subplot in fig: +# subplot.viewport.render(subplot.scene, subplot.camera) +# +# # read from the framebuffer +# try: +# fig.canvas.request_draw() +# raw_img = np.asarray(fig.renderer.target.draw()) +# +# if raw_img.ndim != 3 or raw_img.shape[2] != 4: +# raise ValueError(f"Unexpected image shape: {raw_img.shape}") +# +# # handle alpha blending +# alpha = raw_img[..., 3:4] / 255.0 +# rgb = (raw_img[..., :3] * alpha + (1 - alpha) * 255).astype(np.uint8) +# +# except Exception as e: +# logger.error(f"Framebuffer blending failed for {component_id}: {e}") +# return "Render failed" +# +# # encode image to PNG +# img_buf = io.BytesIO() +# Image.fromarray(rgb).save(img_buf, format="PNG") +# png_bytes = img_buf.getvalue() +# +# # handle websocket communication +# client_websocket = service.websocket_connections.get(client_id) +# if client_websocket: +# packed_msg = msgpack.packb( +# { +# "type": "image_update", +# "component_id": component_id, +# "format": "png", +# "label": label, +# "size": size, +# "data": png_bytes, +# }, +# use_bin_type=True, +# ) +# +# try: +# await client_websocket.send_bytes(packed_msg) +# await service.handle_client_message( +# client_id, +# { +# "type": "component_update", +# "states": {f"{component_id}_img_hash": data_hash}, +# }, +# ) +# logger.debug(f"✅ Sent {component_id} image to client {client_id}") +# except Exception as e: +# logger.error(f"WebSocket send failed for {component_id}: {e}") +# else: +# logger.warning(f"No active WebSocket found for client ID: {client_id}") From 674084e8e98f07aecbc76f53aa3b6c1944f7a31d Mon Sep 17 00:00:00 2001 From: joshlavroff Date: Sat, 5 Apr 2025 16:17:21 -0400 Subject: [PATCH 04/19] add: update chat documentation to include secrets.toml API key support --- docs/sdk/chat.mdx | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/sdk/chat.mdx b/docs/sdk/chat.mdx index ea0e54426..65719d74d 100644 --- a/docs/sdk/chat.mdx +++ b/docs/sdk/chat.mdx @@ -47,6 +47,11 @@ Implemented in `services/openai.js` - OpenAI chat completions integration - GPT-3.5-turbo implementation - Conversation threading support +- API key can be loaded from secrets.toml project file by adding a [data.openai] field and setting api_key to your API key like so +``` +[data.openai] +api_key = "sk-YOUR-KEY" +``` ## Basic Usage From 52e477aeab2672d86379b52a43cfda625bd3b5a6 Mon Sep 17 00:00:00 2001 From: joshlavroff Date: Thu, 17 Apr 2025 20:58:23 -0400 Subject: [PATCH 05/19] add: logic for debug panel --- frontend/src/App.jsx | 13 +++++- frontend/src/components/common/DebugPanel.jsx | 44 +++++++++++++++++++ frontend/src/utils/websocket.js | 6 +++ preswald/engine/managers/branding.py | 6 +++ preswald/engine/server_service.py | 21 ++++++++- preswald/service.py | 8 ++++ 6 files changed, 95 insertions(+), 3 deletions(-) create mode 100644 frontend/src/components/common/DebugPanel.jsx create mode 100644 preswald/service.py diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index fae9a0950..437448ebc 100644 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -1,7 +1,9 @@ -import { useEffect, useState } from 'react'; +import React, { useEffect, useState } from 'react'; import { Route, Routes } from 'react-router-dom'; import { BrowserRouter as Router } from 'react-router-dom'; +import DebugPanel from '@/components/common/DebugPanel'; + import Layout from './components/Layout'; import LoadingState from './components/LoadingState'; import Dashboard from './components/pages/Dashboard'; @@ -13,6 +15,7 @@ const App = () => { const [config, setConfig] = useState(null); const [isConnected, setIsConnected] = useState(false); const [areComponentsLoading, setAreComponentsLoading] = useState(true); + const [debugMode, setDebugMode] = useState(false); useEffect(() => { comm.connect(); @@ -38,6 +41,13 @@ const App = () => { return () => document.removeEventListener('visibilitychange', updateTitle); }, [config]); + useEffect(() => { + fetch('/api/config') + .then((res) => res.json()) + .then((config) => setDebugMode(config.debug || false)) + .catch((err) => console.error('Error fetching config:', err)); + }, []); + const handleMessage = (message) => { console.log('[App] Received message:', message); @@ -162,6 +172,7 @@ const App = () => { handleComponentUpdate={handleComponentUpdate} /> )} + {debugMode && } ); diff --git a/frontend/src/components/common/DebugPanel.jsx b/frontend/src/components/common/DebugPanel.jsx new file mode 100644 index 000000000..ae9a6adc2 --- /dev/null +++ b/frontend/src/components/common/DebugPanel.jsx @@ -0,0 +1,44 @@ +import React, { useEffect, useState } from 'react'; + +import { Button } from '@/components/ui/button'; +import { Card } from '@/components/ui/card'; +import { ScrollArea } from '@/components/ui/scroll-area'; + +const DebugPanel = () => { + const [visible, setVisible] = useState(false); + const [state, setState] = useState({}); + + useEffect(() => { + const handleDebugMessage = (event) => { + const msg = JSON.parse(event.data); + if (msg.channel === '__debug__') { + setState(msg.payload); + } + }; + + const socket = new WebSocket('ws://localhost:8000/ws'); + socket.addEventListener('message', handleDebugMessage); + + return () => { + socket.removeEventListener('message', handleDebugMessage); + socket.close(); + }; + }, []); + + return ( +
+ + {visible && ( + + + 
{JSON.stringify(state, null, 2)}
+ + + )} +
+ ); +}; + +export default DebugPanel; diff --git a/frontend/src/utils/websocket.js b/frontend/src/utils/websocket.js index cc6c34bf6..439d7e016 100644 --- a/frontend/src/utils/websocket.js +++ b/frontend/src/utils/websocket.js @@ -62,6 +62,7 @@ class WebSocketClient { this.socket.onmessage = async (event) => { try { if (typeof event.data === 'string') { + // Normal text message — parse as JSON // Normal text message — parse as JSON const data = JSON.parse(event.data); console.log('[WebSocket] JSON Message received:', { @@ -105,6 +106,11 @@ class WebSocketClient { this.connections = data.connections || []; console.log('[WebSocket] Connections updated:', this.connections); break; + + case '__debug__': + window.__DEBUG_STATE__ = data.payload; + console.log('[WebSocket] Debug state updated:', window.__DEBUG_STATE__); + break; } this._notifySubscribers(data); diff --git a/preswald/engine/managers/branding.py b/preswald/engine/managers/branding.py index a9fa1b93a..6b2828555 100644 --- a/preswald/engine/managers/branding.py +++ b/preswald/engine/managers/branding.py @@ -28,6 +28,7 @@ def get_branding_config(self, script_path: str | None = None) -> dict[str, Any]: "logo": "/images/logo.png", "favicon": f"/images/favicon.ico?timestamp={time.time()}", "primaryColor": "#000000", + "debug": False, # Default value for debug flag } if script_path: @@ -46,6 +47,11 @@ def get_branding_config(self, script_path: str | None = None) -> dict[str, Any]: branding["primaryColor"] = branding_config.get( "primaryColor", branding["primaryColor"] ) + + # Read the debug flag from the [debug] section + if "debug" in config: + branding["debug"] = config["debug"].get("enabled", False) + except Exception as e: logger.error(f"Error loading branding config: {e}") self._ensure_default_assets() diff --git a/preswald/engine/server_service.py b/preswald/engine/server_service.py index 9a62a499b..507f9918b 100644 --- a/preswald/engine/server_service.py +++ b/preswald/engine/server_service.py @@ -1,6 +1,6 @@ import asyncio import logging -from typing import Any, Dict +from typing import Any from fastapi import WebSocket, WebSocketDisconnect @@ -34,7 +34,7 @@ def __init__(self): self.branding_manager = None # set during server creation # Initialize session tracking - self.websocket_connections: Dict[str, WebSocket] = {} + self.websocket_connections: dict[str, WebSocket] = {} async def register_client( self, client_id: str, websocket: WebSocket @@ -82,3 +82,20 @@ async def _broadcast_connections(self): except Exception as e: logger.error(f"Error broadcasting connections: {e}") # Don't raise the exception to prevent disrupting the main flow + + async def _broadcast_debug_state(self): + """Broadcast debug state snapshots to all clients""" + if not self.debug_mode: + return + + try: + state_snapshot = self.get_state_snapshot() + for websocket in self.websocket_connections.values(): + await websocket.send_json( + { + "channel": "__debug__", + "payload": state_snapshot, + } + ) + except Exception as e: + logger.error(f"Error broadcasting debug state: {e}") diff --git a/preswald/service.py b/preswald/service.py new file mode 100644 index 000000000..7a5be02c1 --- /dev/null +++ b/preswald/service.py @@ -0,0 +1,8 @@ +def get_state_snapshot(cls): + """Get all info about current state of instance for debug panel""" + return { + "components": cls.get_instance().components, + "variables": cls.get_instance().variable_store, + "errors": cls.get_instance().error_log, + "execution_graph": cls.get_instance().workflow_graph.serialize(), + } From e2a7ccd15a8f7215ecaa5bdfa66a32b8a8b0ade1 Mon Sep 17 00:00:00 2001 From: joshlavroff Date: Mon, 21 Apr 2025 09:33:26 -0400 Subject: [PATCH 06/19] add: debug flag in config endpoint --- examples/iris/preswald.toml | 5 ++++- frontend/src/App.jsx | 3 ++- frontend/src/components/common/DebugPanel.jsx | 2 +- preswald/engine/managers/branding.py | 2 +- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/examples/iris/preswald.toml b/examples/iris/preswald.toml index 7bb0960f4..168115978 100644 --- a/examples/iris/preswald.toml +++ b/examples/iris/preswald.toml @@ -17,4 +17,7 @@ path = "data/iris.csv" [logging] level = "INFO" # Options: DEBUG, INFO, WARNING, ERROR, CRITICAL -format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s" \ No newline at end of file +format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s" + +[debug] +enabled = true \ No newline at end of file diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index 437448ebc..0c9c57335 100644 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -44,7 +44,7 @@ const App = () => { useEffect(() => { fetch('/api/config') .then((res) => res.json()) - .then((config) => setDebugMode(config.debug || false)) + .then((config) => setDebugMode(config?.debug || false)) .catch((err) => console.error('Error fetching config:', err)); }, []); @@ -68,6 +68,7 @@ const App = () => { case 'config': setConfig(message.config); + setDebugMode(message.config?.debug || false); break; case 'initial_state': diff --git a/frontend/src/components/common/DebugPanel.jsx b/frontend/src/components/common/DebugPanel.jsx index ae9a6adc2..245e47a87 100644 --- a/frontend/src/components/common/DebugPanel.jsx +++ b/frontend/src/components/common/DebugPanel.jsx @@ -5,7 +5,7 @@ import { Card } from '@/components/ui/card'; import { ScrollArea } from '@/components/ui/scroll-area'; const DebugPanel = () => { - const [visible, setVisible] = useState(false); + const [visible, setVisible] = useState(true); const [state, setState] = useState({}); useEffect(() => { diff --git a/preswald/engine/managers/branding.py b/preswald/engine/managers/branding.py index 6b2828555..f3634e0c7 100644 --- a/preswald/engine/managers/branding.py +++ b/preswald/engine/managers/branding.py @@ -28,7 +28,7 @@ def get_branding_config(self, script_path: str | None = None) -> dict[str, Any]: "logo": "/images/logo.png", "favicon": f"/images/favicon.ico?timestamp={time.time()}", "primaryColor": "#000000", - "debug": False, # Default value for debug flag + "debug": False, } if script_path: From 435f3df4949fc259ffa1530ff92ec865ade2ce25 Mon Sep 17 00:00:00 2001 From: Jack Tinker Date: Mon, 21 Apr 2025 13:18:25 -0400 Subject: [PATCH 07/19] Get debug value from config --- frontend/src/App.jsx | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index 0c9c57335..5ec59e10a 100644 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -43,8 +43,28 @@ const App = () => { useEffect(() => { fetch('/api/config') - .then((res) => res.json()) - .then((config) => setDebugMode(config?.debug || false)) + .then((res) => res.text()) // 👈 read as text, NOT JSON + .then((htmlText) => { + // Create a fake DOM element to parse it + const parser = new DOMParser(); + const doc = parser.parseFromString(htmlText, 'text/html'); + + // Find the script tag that defines window.PRESWALD_BRANDING + const scripts = Array.from(doc.scripts); + + for (let script of scripts) { + if (script.textContent.includes('window.PRESWALD_BRANDING')) { + const match = script.textContent.match(/window\.PRESWALD_BRANDING\s*=\s*(\{.*\});/s); + if (match && match[1]) { + const brandingConfig = JSON.parse(match[1]); + setDebugMode(brandingConfig?.debug || false); + return; + } + } + } + + throw new Error('PRESWALD_BRANDING not found'); + }) .catch((err) => console.error('Error fetching config:', err)); }, []); From f40f68dc4788e7d6e87722834a6862ce3de43612 Mon Sep 17 00:00:00 2001 From: joshlavroff Date: Mon, 21 Apr 2025 13:43:37 -0400 Subject: [PATCH 08/19] fix: move snapshot state function to base_service --- preswald/engine/base_service.py | 31 +++++++++++++++++++++++-------- preswald/engine/server_service.py | 4 +--- preswald/engine/service.py | 10 ++++++++++ preswald/service.py | 8 -------- 4 files changed, 34 insertions(+), 19 deletions(-) delete mode 100644 preswald/service.py diff --git a/preswald/engine/base_service.py b/preswald/engine/base_service.py index 40efa973d..b83159e38 100644 --- a/preswald/engine/base_service.py +++ b/preswald/engine/base_service.py @@ -2,9 +2,9 @@ import os import time from collections.abc import Callable -from threading import Lock -from typing import Any, Callable, Dict, Optional from contextlib import contextmanager +from threading import Lock +from typing import Any from preswald.engine.runner import ScriptRunner from preswald.engine.utils import ( @@ -13,7 +13,8 @@ compress_data, optimize_plotly_data, ) -from preswald.interfaces.workflow import Workflow, Atom +from preswald.interfaces.workflow import Atom, Workflow + from .managers.data import DataManager from .managers.layout import LayoutManager @@ -43,7 +44,7 @@ def __init__(self): # DAG workflow engine self._workflow = Workflow() - self._current_atom: Optional[str] = None + self._current_atom: str | None = None # Initialize session tracking self.script_runners: dict[str, ScriptRunner] = {} @@ -75,6 +76,14 @@ def initialize(cls, script_path=None): cls._instance._initialize_data_manager(script_path) return cls._instance + @classmethod + def get_state_snapshot(cls): + """Get all info about current state of instance for debug panel""" + return { + "components": "FJDASKSLFJDSAKLFJADSKL", + "execution_graph": "FJDKLFJDSAKLFJDASKL", + } + @property def script_path(self) -> str | None: return self._script_path @@ -117,10 +126,14 @@ def append_component(self, component): ) with self.active_atom(component_id): if component_id not in self._workflow.atoms: - self._workflow.atoms[component_id] = Atom(name=component_id, func=lambda: None) + self._workflow.atoms[component_id] = Atom( + name=component_id, func=lambda: None + ) self._layout_manager.add_component(cleaned_component) if logger.isEnabledFor(logging.DEBUG): - logger.debug(f"Added component with state: {cleaned_component}") + logger.debug( + f"Added component with state: {cleaned_component}" + ) else: # Components without IDs are added as-is self._layout_manager.add_component(cleaned_component) @@ -167,7 +180,9 @@ def get_component_state(self, component_id: str, default: Any = None) -> Any: if self._current_atom: logger.debug(f"[DAG] {self._current_atom} depends on {component_id}") if self._current_atom not in self._workflow.atoms: - self._workflow.atoms[self._current_atom] = Atom(name=self._current_atom, func=lambda: None) + self._workflow.atoms[self._current_atom] = Atom( + name=self._current_atom, func=lambda: None + ) self._workflow.atoms[self._current_atom].dependencies.add(component_id) return value @@ -180,7 +195,7 @@ def get_rendered_components(self): def get_workflow(self) -> Workflow: return self._workflow - async def handle_client_message(self, client_id: str, message: Dict[str, Any]): + async def handle_client_message(self, client_id: str, message: dict[str, Any]): """Process incoming messages from clients""" start_time = time.time() try: diff --git a/preswald/engine/server_service.py b/preswald/engine/server_service.py index 507f9918b..f84a13c38 100644 --- a/preswald/engine/server_service.py +++ b/preswald/engine/server_service.py @@ -85,11 +85,9 @@ async def _broadcast_connections(self): async def _broadcast_debug_state(self): """Broadcast debug state snapshots to all clients""" - if not self.debug_mode: - return - try: state_snapshot = self.get_state_snapshot() + logger.critical(state_snapshot) for websocket in self.websocket_connections.values(): await websocket.send_json( { diff --git a/preswald/engine/service.py b/preswald/engine/service.py index 84cdddcac..e429688ce 100644 --- a/preswald/engine/service.py +++ b/preswald/engine/service.py @@ -40,3 +40,13 @@ def initialize(cls, script_path=None): def get_instance(cls): """Get the service instance""" return ServiceImpl.get_instance() + + @classmethod + def get_state_snapshot(cls): + """Get all info about current state of instance for debug panel""" + return { + "components": cls.get_instance().components, + "variables": cls.get_instance().variable_store, + "errors": cls.get_instance().error_log, + "execution_graph": cls.get_instance().workflow_graph.serialize(), + } diff --git a/preswald/service.py b/preswald/service.py deleted file mode 100644 index 7a5be02c1..000000000 --- a/preswald/service.py +++ /dev/null @@ -1,8 +0,0 @@ -def get_state_snapshot(cls): - """Get all info about current state of instance for debug panel""" - return { - "components": cls.get_instance().components, - "variables": cls.get_instance().variable_store, - "errors": cls.get_instance().error_log, - "execution_graph": cls.get_instance().workflow_graph.serialize(), - } From 451fe8ddaca94ad5217b44a821a6c6797acbcc9b Mon Sep 17 00:00:00 2001 From: Jack Tinker Date: Mon, 21 Apr 2025 14:02:11 -0400 Subject: [PATCH 09/19] Quick comment change --- frontend/src/App.jsx | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index 5ec59e10a..5ec7b921a 100644 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -43,13 +43,12 @@ const App = () => { useEffect(() => { fetch('/api/config') - .then((res) => res.text()) // 👈 read as text, NOT JSON + .then((res) => res.text()) .then((htmlText) => { - // Create a fake DOM element to parse it + //parse html text for debug flag const parser = new DOMParser(); const doc = parser.parseFromString(htmlText, 'text/html'); - // Find the script tag that defines window.PRESWALD_BRANDING const scripts = Array.from(doc.scripts); for (let script of scripts) { From 3796bd78b7e1eed5edb40db6f6f0543d9efdf0a2 Mon Sep 17 00:00:00 2001 From: joshlavroff Date: Mon, 21 Apr 2025 17:21:23 -0400 Subject: [PATCH 10/19] fix: broadcasting debug info --- frontend/src/components/common/DebugPanel.jsx | 24 +++++++++---------- preswald/engine/base_service.py | 4 ++-- preswald/engine/server_service.py | 9 +++++-- preswald/main.py | 1 + 4 files changed, 21 insertions(+), 17 deletions(-) diff --git a/frontend/src/components/common/DebugPanel.jsx b/frontend/src/components/common/DebugPanel.jsx index 245e47a87..a75389276 100644 --- a/frontend/src/components/common/DebugPanel.jsx +++ b/frontend/src/components/common/DebugPanel.jsx @@ -4,24 +4,22 @@ import { Button } from '@/components/ui/button'; import { Card } from '@/components/ui/card'; import { ScrollArea } from '@/components/ui/scroll-area'; +import { comm } from '@/utils/websocket'; + const DebugPanel = () => { - const [visible, setVisible] = useState(true); - const [state, setState] = useState({}); + const [visible, setVisible] = useState(false); + const [debugState, setDebugState] = useState({}); useEffect(() => { - const handleDebugMessage = (event) => { - const msg = JSON.parse(event.data); - if (msg.channel === '__debug__') { - setState(msg.payload); - } - }; + const unsubscribe = comm.subscribe((payload) => { + console.log('[DebugPanel] Received debug state:', payload); + setDebugState(payload); + }); - const socket = new WebSocket('ws://localhost:8000/ws'); - socket.addEventListener('message', handleDebugMessage); + comm.connect(); return () => { - socket.removeEventListener('message', handleDebugMessage); - socket.close(); + unsubscribe(); }; }, []); @@ -33,7 +31,7 @@ const DebugPanel = () => { {visible && ( - 
{JSON.stringify(state, null, 2)}
+ 
{JSON.stringify(debugState, null, 2)}
 )} diff --git a/preswald/engine/base_service.py b/preswald/engine/base_service.py index b83159e38..fc446ea9c 100644 --- a/preswald/engine/base_service.py +++ b/preswald/engine/base_service.py @@ -80,8 +80,8 @@ def initialize(cls, script_path=None): def get_state_snapshot(cls): """Get all info about current state of instance for debug panel""" return { - "components": "FJDASKSLFJDSAKLFJADSKL", - "execution_graph": "FJDKLFJDSAKLFJDASKL", + "components": cls.get_instance().get_rendered_components, + "execution_graph": cls.get_instance().get_workflow(), } @property diff --git a/preswald/engine/server_service.py b/preswald/engine/server_service.py index f84a13c38..71c04a3ed 100644 --- a/preswald/engine/server_service.py +++ b/preswald/engine/server_service.py @@ -87,13 +87,18 @@ async def _broadcast_debug_state(self): """Broadcast debug state snapshots to all clients""" try: state_snapshot = self.get_state_snapshot() - logger.critical(state_snapshot) + + # Log the state snapshot as critical + logger.critical(f"[Debug] State snapshot: {state_snapshot}") + + # Broadcast the state snapshot to all connected WebSocket clients for websocket in self.websocket_connections.values(): await websocket.send_json( { - "channel": "__debug__", + "type": "__debug__", "payload": state_snapshot, } ) + logger.info("[Debug] Debug state broadcasted to all clients") except Exception as e: logger.error(f"Error broadcasting debug state: {e}") diff --git a/preswald/main.py b/preswald/main.py index d8a607ed8..763a2aad7 100644 --- a/preswald/main.py +++ b/preswald/main.py @@ -90,6 +90,7 @@ def _register_websocket_routes(app: FastAPI): async def websocket_endpoint(websocket: WebSocket, client_id: str): """Handle WebSocket connections""" try: + await app.state.service._broadcast_debug_state() await app.state.service.register_client(client_id, websocket) try: while not app.state.service._is_shutting_down: From 46f992452696a7dbb3bc0f8442151658dfc01994 Mon Sep 17 00:00:00 2001 From: joshlavroff Date: Mon, 21 Apr 2025 17:28:04 -0400 Subject: [PATCH 11/19] fix: await debug broadcast as app runs --- preswald/engine/base_service.py | 5 +---- preswald/main.py | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/preswald/engine/base_service.py b/preswald/engine/base_service.py index fc446ea9c..287e8eaae 100644 --- a/preswald/engine/base_service.py +++ b/preswald/engine/base_service.py @@ -79,10 +79,7 @@ def initialize(cls, script_path=None): @classmethod def get_state_snapshot(cls): """Get all info about current state of instance for debug panel""" - return { - "components": cls.get_instance().get_rendered_components, - "execution_graph": cls.get_instance().get_workflow(), - } + return {"components": "TEST"} @property def script_path(self) -> str | None: diff --git a/preswald/main.py b/preswald/main.py index 763a2aad7..1a46558cc 100644 --- a/preswald/main.py +++ b/preswald/main.py @@ -90,10 +90,10 @@ def _register_websocket_routes(app: FastAPI): async def websocket_endpoint(websocket: WebSocket, client_id: str): """Handle WebSocket connections""" try: - await app.state.service._broadcast_debug_state() await app.state.service.register_client(client_id, websocket) try: while not app.state.service._is_shutting_down: + await app.state.service._broadcast_debug_state() message = await websocket.receive_json() await app.state.service.handle_client_message(client_id, message) except WebSocketDisconnect: From b23480a48fea8ac1ea957f1ce1aaa510eabf63f6 Mon Sep 17 00:00:00 2001 From: Jack Tinker Date: Mon, 21 Apr 2025 17:40:56 -0400 Subject: [PATCH 12/19] Changed message parsing logic in DebugPanel.jsx --- frontend/src/components/common/DebugPanel.jsx | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/frontend/src/components/common/DebugPanel.jsx b/frontend/src/components/common/DebugPanel.jsx index a75389276..37cdb0be2 100644 --- a/frontend/src/components/common/DebugPanel.jsx +++ b/frontend/src/components/common/DebugPanel.jsx @@ -11,9 +11,11 @@ const DebugPanel = () => { const [debugState, setDebugState] = useState({}); useEffect(() => { - const unsubscribe = comm.subscribe((payload) => { - console.log('[DebugPanel] Received debug state:', payload); - setDebugState(payload); + const unsubscribe = comm.subscribe((message) => { + if (message.type === '__debug__') { + console.log('[DebugPanel] Received debug state:', message); + setDebugState(message.payload); + } }); comm.connect(); From 766df072761afaba1e2825b66f8ed33168ebfacf Mon Sep 17 00:00:00 2001 From: joshlavroff Date: Mon, 21 Apr 2025 18:56:16 -0400 Subject: [PATCH 13/19] add: debug tracking for errors and components --- frontend/src/components/common/DebugPanel.jsx | 7 +++- preswald/engine/base_service.py | 40 ++++++++++++++++++- preswald/main.py | 20 ++++++++-- 3 files changed, 59 insertions(+), 8 deletions(-) diff --git a/frontend/src/components/common/DebugPanel.jsx b/frontend/src/components/common/DebugPanel.jsx index a75389276..6c7eb3499 100644 --- a/frontend/src/components/common/DebugPanel.jsx +++ b/frontend/src/components/common/DebugPanel.jsx @@ -12,8 +12,11 @@ const DebugPanel = () => { useEffect(() => { const unsubscribe = comm.subscribe((payload) => { - console.log('[DebugPanel] Received debug state:', payload); - setDebugState(payload); + if (payload.type === '__debug__') { + // Only process messages with type "__debug__" + console.log('[DebugPanel] Received debug state:', payload); + setDebugState(payload.payload); // Use the "payload" field for the debug state + } }); comm.connect(); diff --git a/preswald/engine/base_service.py b/preswald/engine/base_service.py index 287e8eaae..d49c06d65 100644 --- a/preswald/engine/base_service.py +++ b/preswald/engine/base_service.py @@ -78,8 +78,44 @@ def initialize(cls, script_path=None): @classmethod def get_state_snapshot(cls): - """Get all info about current state of instance for debug panel""" - return {"components": "TEST"} + """Get all info about the current state of the instance for the debug panel.""" + try: + instance = cls.get_instance() + + # Get components hierarchy + components = instance.get_rendered_components() + components_cleaned = [] + rows = components.get("rows", []) + for row in rows: + for component in row: + component_id = component.get("id") + if not component_id: + continue + else: + components_cleaned.append( + { + "id": component_id, + "state": cls.get_instance()._component_states.get( + component_id + ), + } + ) + # Get tracked variables + variables = ( + instance._workflow.context.variables if instance._workflow else {} + ) + + # Get errors + errors = instance.error_log if hasattr(instance, "error_log") else [] + + return { + "components": components_cleaned, + "variables": variables, + "errors": errors, + } + except Exception as e: + logger.error(f"Error generating state snapshot: {e}", exc_info=True) + return {"components": {}, "variables": {}, "errors": [{"message": str(e)}]} @property def script_path(self) -> str | None: diff --git a/preswald/main.py b/preswald/main.py index 1a46558cc..62eb54b86 100644 --- a/preswald/main.py +++ b/preswald/main.py @@ -92,10 +92,22 @@ async def websocket_endpoint(websocket: WebSocket, client_id: str): try: await app.state.service.register_client(client_id, websocket) try: - while not app.state.service._is_shutting_down: - await app.state.service._broadcast_debug_state() - message = await websocket.receive_json() - await app.state.service.handle_client_message(client_id, message) + branding = app.state.service.branding_manager.get_branding_config( + app.state.service.script_path + ) + if branding["debug"]: + while not app.state.service._is_shutting_down: + await app.state.service._broadcast_debug_state() + message = await websocket.receive_json() + await app.state.service.handle_client_message( + client_id, message + ) + else: + while not app.state.service._is_shutting_down: + message = await websocket.receive_json() + await app.state.service.handle_client_message( + client_id, message + ) except WebSocketDisconnect: logger.info(f"Client disconnected: {client_id}") finally: From ff32487d3b214f8a41930f70591f9bb7ab171701 Mon Sep 17 00:00:00 2001 From: joshlavroff Date: Mon, 21 Apr 2025 19:06:52 -0400 Subject: [PATCH 14/19] add: component type to debug panel --- preswald/engine/base_service.py | 1 + 1 file changed, 1 insertion(+) diff --git a/preswald/engine/base_service.py b/preswald/engine/base_service.py index d49c06d65..b0f2d9e98 100644 --- a/preswald/engine/base_service.py +++ b/preswald/engine/base_service.py @@ -95,6 +95,7 @@ def get_state_snapshot(cls): components_cleaned.append( { "id": component_id, + "type": component.get("type"), "state": cls.get_instance()._component_states.get( component_id ), From f0b431bdc1e42b4b2bf9a89914c9f17a28418c8a Mon Sep 17 00:00:00 2001 From: joshlavroff Date: Mon, 21 Apr 2025 19:19:50 -0400 Subject: [PATCH 15/19] fix: remove critical logging for debug state update --- preswald/engine/server_service.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/preswald/engine/server_service.py b/preswald/engine/server_service.py index 71c04a3ed..78c3fe13a 100644 --- a/preswald/engine/server_service.py +++ b/preswald/engine/server_service.py @@ -88,9 +88,6 @@ async def _broadcast_debug_state(self): try: state_snapshot = self.get_state_snapshot() - # Log the state snapshot as critical - logger.critical(f"[Debug] State snapshot: {state_snapshot}") - # Broadcast the state snapshot to all connected WebSocket clients for websocket in self.websocket_connections.values(): await websocket.send_json( @@ -99,6 +96,8 @@ async def _broadcast_debug_state(self): "payload": state_snapshot, } ) - logger.info("[Debug] Debug state broadcasted to all clients") + logger.info( + f"[Debug] Debug state {state_snapshot} broadcasted to all clients" + ) except Exception as e: logger.error(f"Error broadcasting debug state: {e}") From 194c393b29691ef9b0f67d29f129b39e8982efad Mon Sep 17 00:00:00 2001 From: Jack Tinker Date: Mon, 21 Apr 2025 19:28:56 -0400 Subject: [PATCH 16/19] Added dropdowns to debug panel --- frontend/src/components/common/DebugPanel.jsx | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/frontend/src/components/common/DebugPanel.jsx b/frontend/src/components/common/DebugPanel.jsx index 37cdb0be2..b0fe62f30 100644 --- a/frontend/src/components/common/DebugPanel.jsx +++ b/frontend/src/components/common/DebugPanel.jsx @@ -25,6 +25,15 @@ const DebugPanel = () => { }; }, []); + const [openSections, setOpenSections] = useState({}); + + const toggleSection = (key) => { + setOpenSections((prev) => ({ + ...prev, + [key]: !prev[key], + })); + }; + return (
+ {openSections[key] && ( + 
+                    {JSON.stringify(value, null, 2)}
+
+ )} +
+ ))} )} From ca39ccd112b3e310c6ff25a33cd7953056be8ef3 Mon Sep 17 00:00:00 2001 From: Jack Tinker Date: Tue, 6 May 2025 22:07:43 -0400 Subject: [PATCH 17/19] Draft of debug panel documentation --- docs/usage/troubleshooting.mdx | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/usage/troubleshooting.mdx b/docs/usage/troubleshooting.mdx index 83eb90127..864cb7c37 100644 --- a/docs/usage/troubleshooting.mdx +++ b/docs/usage/troubleshooting.mdx @@ -120,6 +120,14 @@ Dependency errors may arise if the required Python packages are not installed or 4. **Reinstall Dependencies**: If dependency issues persist, delete the `env` folder, recreate the virtual environment, and reinstall dependencies. + +5. **Debug Panel** + You can view live app state, tracked variables, and error messages to assist in real-time debugging with the debug panel. In `preswald.toml`, set: + ```toml + [debug] + enabled = true + ``` + and click the toggle that appears in the app. --- ## **Need More Help?** From e1aa686fced5a2a3c3dfd7878034f7edd5d68c88 Mon Sep 17 00:00:00 2001 From: joshlavroff Date: Wed, 7 May 2025 17:09:54 -0400 Subject: [PATCH 18/19] fix: track local variables --- preswald/engine/base_service.py | 24 +++++++++++++++++++++--- preswald/engine/runner.py | 8 ++++++-- preswald/engine/server_service.py | 4 +--- 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/preswald/engine/base_service.py b/preswald/engine/base_service.py index b0f2d9e98..4ba16aabd 100644 --- a/preswald/engine/base_service.py +++ b/preswald/engine/base_service.py @@ -1,6 +1,7 @@ import logging import os import time +import types from collections.abc import Callable from contextlib import contextmanager from threading import Lock @@ -76,6 +77,19 @@ def initialize(cls, script_path=None): cls._instance._initialize_data_manager(script_path) return cls._instance + @classmethod + def get_tracked_variables(cls): + """Aggregate variables from all active ScriptRunner instances.""" + tracked_variables = [] + for runner in cls.get_instance().script_runners.values(): + local_vars = runner._script_locals + for key, value in local_vars.items(): + if not isinstance(value, types.ModuleType) and not isinstance( + value, types.FunctionType + ): + tracked_variables.append({"name": key, "value": str(value)}) + return tracked_variables + @classmethod def get_state_snapshot(cls): """Get all info about the current state of the instance for the debug panel.""" @@ -101,10 +115,14 @@ def get_state_snapshot(cls): ), } ) + # Get tracked variables - variables = ( - instance._workflow.context.variables if instance._workflow else {} - ) + workflow_variables = [ + instance._workflow.context.variables if instance._workflow else None + ] + + script_variables = instance.get_tracked_variables() + variables = script_variables + workflow_variables # Get errors errors = instance.error_log if hasattr(instance, "error_log") else [] diff --git a/preswald/engine/runner.py b/preswald/engine/runner.py index 841a8692f..73fee2127 100644 --- a/preswald/engine/runner.py +++ b/preswald/engine/runner.py @@ -47,8 +47,12 @@ def __init__( self._run_count = 0 self._lock = threading.Lock() self._script_globals = {} + self._script_locals = {} + + from .service import ( + PreswaldService, + ) # deferred import to avoid cyclic dependency - from .service import PreswaldService # deferred import to avoid cyclic dependency self._service = PreswaldService.get_instance() logger.info(f"[ScriptRunner] Initialized with session_id: {session_id}") @@ -245,7 +249,7 @@ async def run_script(self): os.chdir(script_dir) code = compile(f.read(), self.script_path, "exec") logger.debug("[ScriptRunner] Script compiled") - exec(code, self._script_globals) + exec(code, self._script_globals, self._script_locals) logger.debug("[ScriptRunner] Script executed") # Change back to original working dir os.chdir(current_working_dir) diff --git a/preswald/engine/server_service.py b/preswald/engine/server_service.py index 78c3fe13a..2fa772e96 100644 --- a/preswald/engine/server_service.py +++ b/preswald/engine/server_service.py @@ -96,8 +96,6 @@ async def _broadcast_debug_state(self): "payload": state_snapshot, } ) - logger.info( - f"[Debug] Debug state {state_snapshot} broadcasted to all clients" - ) + logger.info("[Debug] Debug state broadcasted to all clients") except Exception as e: logger.error(f"Error broadcasting debug state: {e}") From 2233680813ec14c76a08bc000eb50f07f70f0270 Mon Sep 17 00:00:00 2001 From: joshlavroff Date: Wed, 7 May 2025 18:14:12 -0400 Subject: [PATCH 19/19] fix: remove loading from secrets toml --- docs/sdk/chat.mdx | 5 ----- frontend/src/components/widgets/ChatWidget.jsx | 10 ++-------- preswald/interfaces/components.py | 14 +------------- 3 files changed, 3 insertions(+), 26 deletions(-) diff --git a/docs/sdk/chat.mdx b/docs/sdk/chat.mdx index 46203c9d3..4fb29e042 100644 --- a/docs/sdk/chat.mdx +++ b/docs/sdk/chat.mdx @@ -47,11 +47,6 @@ Implemented in `services/openai.js` - OpenAI chat completions integration - GPT-3.5-turbo implementation - Conversation threading support -- API key can be loaded from secrets.toml project file by adding a [data.openai] field and setting api_key to your API key like so -``` -[data.openai] -api_key = "sk-YOUR-KEY" -``` ## Basic Usage diff --git a/frontend/src/components/widgets/ChatWidget.jsx b/frontend/src/components/widgets/ChatWidget.jsx index c05e66462..16f9f0d91 100644 --- a/frontend/src/components/widgets/ChatWidget.jsx +++ b/frontend/src/components/widgets/ChatWidget.jsx @@ -15,7 +15,6 @@ const ChatWidget = ({ id, sourceId = null, sourceData = null, - apiKey = null, value = { messages: [] }, onChange, className, @@ -25,14 +24,9 @@ const ChatWidget = ({ const messagesEndRef = useRef(null); const chatContainerRef = useRef(null); - // Load API key from secrets.toml if present - if (apiKey) { - sessionStorage.setItem('openai_api_key', apiKey.trim()); - } - const [inputValue, setInputValue] = useState(''); const [showSettings, setShowSettings] = useState(false); - // const [apiKey, setApiKey] = useState(''); + const [apiKey, setApiKey] = useState(''); const [isLoading, setIsLoading] = useState(false); const [error, setError] = useState(null); const hasApiKey = useMemo(() => !!sessionStorage.getItem('openai_api_key'), []); @@ -345,4 +339,4 @@ const ChatWidget = ({ ); }; -export default ChatWidget; +export default ChatWidget; \ No newline at end of file diff --git a/preswald/interfaces/components.py b/preswald/interfaces/components.py index fdb26eba9..3e1a69ad8 100644 --- a/preswald/interfaces/components.py +++ b/preswald/interfaces/components.py @@ -10,7 +10,6 @@ import matplotlib.pyplot as plt import numpy as np import pandas as pd -import tomllib # from PIL import Image # try: @@ -130,16 +129,6 @@ def chat(source: str, table: str | None = None, component_id: str | None = None, if current_state is None: current_state = {"messages": [], "source": source} - # Get API key from secrets.toml - with open("secrets.toml", "rb") as toml: - secrets = tomllib.load(toml) - - if secrets and secrets["data"]["openai"]["api_key"]: - api_key = secrets["data"]["openai"]["api_key"] - - else: - api_key = None - # Get dataframe from source df = ( service.data_manager.get_df(source) @@ -175,7 +164,6 @@ def chat(source: str, table: str | None = None, component_id: str | None = None, "config": { "source": source, "data": serializable_data, - "apiKey": api_key, }, } @@ -1113,4 +1101,4 @@ def convert_to_serializable(obj): # except Exception as e: # logger.error(f"WebSocket send failed for {component_id}: {e}") # else: -# logger.warning(f"No active WebSocket found for client ID: {client_id}") +# logger.warning(f"No active WebSocket found for client ID: {client_id}") \ No newline at end of file