feat: add support for Notion database source

Author: onematchfox

README.md

@@ -17,6 +17,8 @@ The primary goal is to prepare documentation content for Retrieval-Augmented Gen
     * **Knowledge Base Articles:** Converts help center articles from HTML to clean Markdown.
     * **Incremental Updates:** Only processes tickets/articles updated since the last run.
     * **Flexible Filtering:** Filter tickets by status and priority.
+*   **Notion Integration:** Fetches entries from a Notion database, converting them to searchable chunks.
+    * **Flexible Filtering:** filter returned entries from the database using specified criteria.
 *   **Local Directory Processing:** Scans local directories for files, converts content to searchable chunks.
     * **PDF Support:** Automatically extracts text from PDF files and converts them to Markdown format using Mozilla's PDF.js.
 *   **Content Extraction:** Uses Puppeteer for rendering JavaScript-heavy pages and `@mozilla/readability` to extract the main article content.
@@ -114,6 +116,12 @@ Configuration is managed through two files:
         *   `start_date`: (Optional) Only process tickets/articles updated since this date (e.g., `'2025-01-01'`).
         *   `ticket_status`: (Optional) Filter tickets by status (defaults to `['new', 'open', 'pending', 'hold', 'solved']`).
         *   `ticket_priority`: (Optional) Filter tickets by priority (defaults to all priorities).
+        
+        For Notion (`type: 'notion'`):
+        *   `api_token`: Your Notion API token (reference environment variable
+            as `'${NOTION_API_TOKEN}'`).
+        *   `database_id`: The ID of the Notion database to query
+        *   `filter`: Filter to apply when querying the database
 
         Common configuration for all types:
         *   `product_name`: A string identifying the product (used in metadata).
@@ -186,6 +194,21 @@ Configuration is managed through two files:
           type: 'sqlite'
           params:
             db_path: './zendesk-kb.db'
+
+      # Notion example
+      - type: notion
+        product_name: 'notion-database'
+        version: 'latest'
+        database_id: '897e5a76ae524b489fdfe71f5945d1af'
+        api_token: '${NOTION_API_TOKEN}'
+        filter:
+          property: 'Status'
+          status:
+            equals: 'Current'
+        database_config:
+          type: 'sqlite'
+          params:
+            db_path: './notion-database.db'
       
       # Qdrant example
       - type: 'website'

doc2vec.ts

@@ -12,13 +12,17 @@ import { Logger, LogLevel } from './logger';
 import { Utils } from './utils';
 import { DatabaseManager } from './database';
 import { ContentProcessor } from './content-processor';
+
+import { isFullPage, Client as NotionClient } from '@notionhq/client';
+import { NotionConverter } from 'notion-to-md';
+
 import { 
     Config, 
-    SourceConfig, 
     GithubSourceConfig, 
     WebsiteSourceConfig, 
     LocalDirectorySourceConfig,
     ZendeskSourceConfig,
+    NotionSourceConfig,
     DatabaseConnection,
     DocumentChunk
 } from './types';
@@ -35,7 +39,6 @@ class Doc2Vec {
 
     constructor(configPath: string) {
         this.logger = new Logger('Doc2Vec', {
-            level: LogLevel.DEBUG,
             useTimestamp: true,
             useColor: true,
             prettyPrint: true
@@ -92,6 +95,8 @@ class Doc2Vec {
                 await this.processLocalDirectory(sourceConfig, sourceLogger);
             } else if (sourceConfig.type === 'zendesk') {
                 await this.processZendesk(sourceConfig, sourceLogger);
+            } else if (sourceConfig.type === 'notion') {
+                await this.processNotionDatabase(sourceConfig, sourceLogger);
             } else {
                 sourceLogger.error(`Unknown source type: ${(sourceConfig as any).type}`);
             }
@@ -656,6 +661,112 @@ class Doc2Vec {
         logger.info(`Successfully processed ${processedArticles} of ${totalArticles} articles (filtered by date >= ${startDate})`);
     }
 
+    private async processNotionDatabase(config: NotionSourceConfig, parentLogger: Logger): Promise<void> {
+        const logger = parentLogger.child('process');
+        logger.info(`Starting processing for Notion: ${config.database_id}`);
+        
+        const dbConnection = await DatabaseManager.initDatabase(config, logger);
+        
+        // Initialize metadata storage
+        await DatabaseManager.initDatabaseMetadata(dbConnection, logger);
+
+        const notion = new NotionClient({
+            auth: config.api_token,
+        });
+    
+        // Create a NotionConverter instance
+        const n2m = new NotionConverter(notion);
+
+        // Chunk the markdown content   
+        const pageConfig = {
+            ...config,
+            product_name: config.product_name || 'notion',
+            max_size: config.max_size || Infinity
+        };
+
+        const getMarkdownForPage = async (page_id: string): Promise<[string, string]> => {          
+            let pageUrl = '';
+            try {
+                logger.debug(`Retrieving Notion page: ${page_id}`);
+                
+                // Retrieve the page and convert to markdown
+                const page = await notion.pages.retrieve({ page_id: page_id });
+                if (!isFullPage(page)) {
+                    logger.info(`Skipping partial page #${page_id}`);
+                    return ['', ''];
+                }
+    
+                const nameProperty = page.properties.Name;
+                const pageTitle = nameProperty && nameProperty.type === 'title' && nameProperty.title?.[0]?.plain_text || 'Untitled';
+                
+                pageUrl = page.url;
+
+                logger.debug(`Generating markdown for page: ${pageUrl}`);
+                
+                const md = await n2m.convert(page_id);
+                const mdWithNoImages = md.content.replace(/!\[.*?\]\(.*?\)[\s\n]*/g, ''); 
+
+                return [page.url, `# ${pageTitle}${mdWithNoImages}`];
+            } catch (error) {
+                logger.error(`Failed to generate markdown for Notion page ${page_id} (${pageUrl}):`, error);
+                return ['', ''];
+            }
+        }
+
+        const processPage = async (page_id: string): Promise<void> => {
+            logger.info(`Processing Notion page: ${page_id}`);
+            
+            const [url, md] = await getMarkdownForPage(page_id);
+            if (!md) {
+                logger.info(`No markdown for Notion page: ${page_id}`);
+                return
+            }
+
+            const chunks = await this.contentProcessor.chunkMarkdown(md, pageConfig, url);
+            
+            // Process and store each chunk immediately
+            await this.processAndStoreChunks(page_id, chunks, dbConnection, logger);
+            
+            logger.debug(`Finished processing Notion page: ${page_id}`);
+        }
+
+        const processDatabase = async (database_id: string): Promise<void> => {
+            logger.info(`Processing Notion database: ${database_id}`);
+            
+            let next_cursor = undefined;
+
+            try {
+                do {
+                    const response = await notion.databases.query({
+                        database_id: database_id,
+                        start_cursor: next_cursor,
+                        filter: config.filter,
+                    });
+        
+                    for (const res of response.results) {
+                        if (res.object === "database") {
+                            await processDatabase(res.id);
+                        } else if (res.object === "page") {
+                            await processPage(res.id);
+                        } else {
+                            logger.error("unknown database object: ${res.object}");
+                        }
+                    }
+        
+                    next_cursor = response.next_cursor;
+                } while (next_cursor)
+
+                logger.debug(`Finished processing Notion database: ${database_id}`);
+            } catch (error) {
+                logger.error(`Failed to process Notion database ${database_id}:`, error);
+            }
+        }
+
+        await processDatabase(config.database_id);
+
+        logger.info(`Completed processing Notion database: ${config.database_id}`);
+    }
+
     private async createEmbeddings(texts: string[]): Promise<number[][]> {
         const logger = this.logger.child('embeddings');
         try {