feat: add real-time progress tracking for note extraction

src/electron/main.ts

@@ -13,6 +13,7 @@ import {
 import {
   extractAndEmbedNotes,
   countNotes,
+  ProgressEvent,
 } from "./noteProcessing/extractAndEmbedNotes.js";
 import { DatabaseService } from "./database/databaseService.js";
 
@@ -106,10 +107,18 @@ async function createWindow() {
 
     ipcMain.handle("extractAndEmbedNotes", async () => {
       try {
-        await extractAndEmbedNotes(dbService);
+        await extractAndEmbedNotes(dbService, (event: ProgressEvent) => {
+          // Send progress events to renderer
+          mainWindow?.webContents.send("extraction-progress", event);
+        });
         return { success: true };
       } catch (error) {
         console.error("Error handling extractAndEmbedNotes:", error);
+        mainWindow?.webContents.send("extraction-progress", {
+          type: 'error',
+          error: `Extraction failed: ${error}`,
+          message: 'Failed to extract and embed notes'
+        });
         throw new Error("Failed to extract and embed notes");
       }
     });

src/electron/noteProcessing/extractAndEmbedNotes.ts

@@ -3,6 +3,18 @@ import * as crypto from "crypto";
 import { DatabaseService } from "../database/databaseService.js";
 import { CharChunker } from "./CharChunker.js";
 
+export interface ProgressEvent {
+  type: 'start' | 'progress' | 'note' | 'chunk' | 'complete' | 'error';
+  totalNotes?: number;
+  processedNotes?: number;
+  totalChunks?: number;
+  processedChunks?: number;
+  currentNoteTitle?: string;
+  currentChunkIndex?: number;
+  message?: string;
+  error?: string;
+}
+
 const COUNT_SCRIPT = `
 tell application "Notes"
     set noteCount to count of notes
@@ -99,6 +111,7 @@ export interface ChunkMetadata {
 
 export async function extractAndEmbedNotes(
   dbService: DatabaseService,
+  onProgress?: (event: ProgressEvent) => void,
 ): Promise<void> {
   console.log("Cleaning existing data from all tables...");
   dbService.cleanAllTables();
@@ -108,6 +121,13 @@ export async function extractAndEmbedNotes(
   let processedNotes = 1; // because of NOT NULL constraint from sqlite-vec
   let processedChunks = 0;
 
+  // Send start event
+  onProgress?.({
+    type: 'start',
+    totalNotes,
+    message: `Starting extraction of ${totalNotes} notes...`
+  });
+
   const split = crypto.randomBytes(8).toString("hex");
   const process = spawn("osascript", [
     "-e",
@@ -192,6 +212,16 @@ export async function extractAndEmbedNotes(
     try {
       processedNotes++;
       console.log(`\nProcessing Note ${processedNotes}/${totalNotes}:`);
+
+      // Send note progress event
+      onProgress?.({
+        type: 'note',
+        totalNotes,
+        processedNotes,
+        currentNoteTitle: note.title || 'Untitled',
+        message: `Processing note: ${note.title || 'Untitled'}`
+      });
+
       const processedNote: ProcessedNote = {
         id: note.id!,
         title: note.title || "",
@@ -236,6 +266,19 @@ export async function extractAndEmbedNotes(
         for (const [index, chunk] of chunks.entries()) {
           try {
             console.log(`Processing chunk ${index + 1}/${chunks.length}`);
+
+            // Send chunk progress event
+            onProgress?.({
+              type: 'chunk',
+              totalNotes,
+              processedNotes,
+              totalChunks: chunks.length,
+              processedChunks: processedChunks + 1,
+              currentChunkIndex: index + 1,
+              currentNoteTitle: processedNote.title,
+              message: `Processing chunk ${index + 1}/${chunks.length} for "${processedNote.title}"`
+            });
+
             const embedding = await generateEmbedding(chunk);
             processedChunks++;
 
@@ -265,6 +308,14 @@ export async function extractAndEmbedNotes(
               `Error processing chunk ${index + 1} of note ${processedNote.id}:`,
               error,
             );
+
+            // Send error event
+            onProgress?.({
+              type: 'error',
+              error: `Failed to process chunk ${index + 1} of note "${processedNote.title}": ${error}`,
+              currentNoteTitle: processedNote.title,
+              message: `Error processing chunk ${index + 1}/${chunks.length}`
+            });
           }
         }
 
@@ -283,9 +334,24 @@ export async function extractAndEmbedNotes(
           title: processedNote.title,
           folderName: processedNote.folderName,
         });
+
+        // Send error event
+        onProgress?.({
+          type: 'error',
+          error: `Failed to save note "${processedNote.title}": ${error}`,
+          currentNoteTitle: processedNote.title,
+          message: `Error saving note`
+        });
       }
     } catch (error) {
       console.error(`Error processing note ${note.id}:`, error);
+
+      // Send error event
+      onProgress?.({
+        type: 'error',
+        error: `Error processing note: ${error}`,
+        message: `Error processing note`
+      });
     } finally {
       note = {};
       body = [];
@@ -305,13 +371,33 @@ export async function extractAndEmbedNotes(
         console.log(`Total notes processed: ${processedNotes - 1}/${totalNotes}`);
         console.log(`Total chunks processed: ${processedChunks}`);
 
+        // Send completion event
+        onProgress?.({
+          type: 'complete',
+          totalNotes,
+          processedNotes: processedNotes - 1,
+          totalChunks: processedChunks,
+          message: `Completed extraction: ${processedNotes - 1} notes, ${processedChunks} chunks processed`
+        });
+
         if (code === 0) {
           resolve();
         } else {
-          reject(new Error(`Notes extraction failed with code ${code}`));
+          const errorMsg = `Notes extraction failed with code ${code}`;
+          onProgress?.({
+            type: 'error',
+            error: errorMsg,
+            message: 'Extraction failed'
+          });
+          reject(new Error(errorMsg));
         }
       } catch (error) {
         console.error("Error during process completion:", error);
+        onProgress?.({
+          type: 'error',
+          error: `Process completion error: ${error}`,
+          message: 'Error during completion'
+        });
         reject(error);
       }
     });

src/electron/preload.cts

@@ -14,13 +14,26 @@ interface SimilarChunk {
   distance: number;
 }
 
+interface ProgressEvent {
+  type: 'start' | 'progress' | 'note' | 'chunk' | 'complete' | 'error';
+  totalNotes?: number;
+  processedNotes?: number;
+  totalChunks?: number;
+  processedChunks?: number;
+  currentNoteTitle?: string;
+  currentChunkIndex?: number;
+  message?: string;
+  error?: string;
+}
+
 interface ElectronAPI {
   checkOllamaStatus: () => Promise<OllamaStatus>;
   startOllama: () => Promise<boolean>;
   setupOllama: () => Promise<boolean>;
   checkOllamaModel: (modelName: string) => Promise<boolean>;
   pullOllamaModel: (modelName: string) => Promise<boolean>;
   onSetupMessage: (callback: (message: string) => void) => () => void;
+  onExtractionProgress: (callback: (event: ProgressEvent) => void) => () => void;
   findSimilarChunks: (
     queryText: string,
     limit?: number,
@@ -55,6 +68,12 @@ contextBridge.exposeInMainWorld("electron", {
     );
     return () => ipcRenderer.removeAllListeners("setup-message");
   },
+  onExtractionProgress: (callback: (event: ProgressEvent) => void) => {
+    ipcRenderer.on("extraction-progress", (_: unknown, event: ProgressEvent) =>
+      callback(event),
+    );
+    return () => ipcRenderer.removeAllListeners("extraction-progress");
+  },
   findSimilarChunks: (
     queryText: string,
     limit?: number,

src/ui/App.tsx

@@ -1,9 +1,10 @@
-import { useState } from "react";
+import { useState, useEffect } from "react";
 import Header, { ChatMode } from "./components/Header";
 import "./App.css";
 import Chat from "./components/Chat";
 import Notification from "./components/Notification";
 import FeedbackModal from "./components/FeedbackModal";
+import ExtractionProgress from "./components/ExtractionProgress";
 
 function App() {
   const [mode, setMode] = useState<ChatMode>("local");
@@ -12,27 +13,49 @@ function App() {
   const [isSetupComplete, setIsSetupComplete] = useState(false);
   const [lastUpdateTime, setLastUpdateTime] = useState<Date | null>(null);
   const [showFeedback, setShowFeedback] = useState(false);
+  const [progressEvent, setProgressEvent] = useState<any>(null);
 
   const handleExtractNotes = async () => {
     setIsExtracting(true);
+    setProgressEvent(null);
     try {
       let count = await window.electron.countNotes();
       if (count > 0) {
         setExtractionMessage(
           `Adding ${count} Notes to knowledgebase, this might take a couple of minutes...`
         );
+
+        // Set up progress listener
+        const removeProgressListener = window.electron.onExtractionProgress((event) => {
+          setProgressEvent(event);
+
+          if (event.type === 'complete') {
+            setLastUpdateTime(new Date());
+            setExtractionMessage("Successfully added notes to knowledgebase!");
+            setTimeout(() => {
+              setProgressEvent(null);
+              setIsExtracting(false);
+            }, 3000);
+          } else if (event.type === 'error') {
+            setExtractionMessage("Failed to extract notes. Please try again.");
+            setTimeout(() => {
+              setProgressEvent(null);
+              setIsExtracting(false);
+            }, 5000);
+          }
+        });
+
         await window.electron.extractAndEmbedNotes();
-        setLastUpdateTime(new Date());
-        setExtractionMessage("Successfully added notes to knowledgebase!");
+        removeProgressListener();
       } else {
         setExtractionMessage("No notes found to add to knowledgebase.");
+        setIsExtracting(false);
       }
     } catch (error) {
       console.error("Failed to extract notes:", error);
       setExtractionMessage("Failed to extract notes. Please try again.");
-    } finally {
+      setProgressEvent(null);
       setIsExtracting(false);
-      setTimeout(() => setExtractionMessage(""), 3000);
     }
   };
 
@@ -41,6 +64,15 @@ function App() {
     setIsSetupComplete(status);
   }
 
+  // Clean up progress event on unmount
+  useEffect(() => {
+    return () => {
+      if (progressEvent?.type === 'complete' || progressEvent?.type === 'error') {
+        setProgressEvent(null);
+      }
+    };
+  }, []);
+
   return (
     <div className="h-screen w-screen flex flex-col bg-apple-notes">
       <Header 
@@ -56,11 +88,12 @@ function App() {
       <main className="flex-1 relative bg-apple-notes pt-[88px]">
         <Chat onSetupComplete={handleSetupComplete} />
       </main>
-      {extractionMessage && (
+      {extractionMessage && !progressEvent && (
         <div className="fixed top-4 right-4 z-50">
           <Notification message={extractionMessage} />
         </div>
       )}
+      <ExtractionProgress event={progressEvent} isVisible={isExtracting} />
       <FeedbackModal 
         showFeedback={showFeedback}
         setShowFeedback={setShowFeedback}