✨(app) add progress bar for Notion import

Co-authored-by: Clara Ni <clara.ni@outlook.fr>

Author: emersion

src/backend/core/api/viewsets.py

@@ -39,10 +39,17 @@
 from core.services.ai_services import AIService
 from core.services.collaboration_services import CollaborationService
 from core.services.converter_services import YdocConverter
-from core.services.notion_import import import_notion
+from core.services.notion_import import (
+    ImportedDocument,
+    build_notion_session,
+    fetch_all_pages,
+    import_page,
+    link_child_page_to_parent,
+)
 from core.tasks.mail import send_ask_for_access_mail
 from core.utils import extract_attachments, filter_descendants
 
+from ..notion_schemas.notion_page import NotionPage
 from . import permissions, serializers, utils
 from .filters import DocumentFilter, ListDocumentFilter
 
@@ -2134,7 +2141,7 @@ def _import_notion_doc_content(imported_doc, obj, user):
     obj.save()
 
 
-def _import_notion_child_page(imported_doc, parent_doc, user, imported_docs_by_page_id):
+def _import_notion_child_page(imported_doc, parent_doc, user, imported_ids):
     obj = parent_doc.add_child(
         creator=user,
         title=imported_doc.page.get_title() or "Child page",
@@ -2148,13 +2155,13 @@ def _import_notion_child_page(imported_doc, parent_doc, user, imported_docs_by_p
 
     _import_notion_doc_content(imported_doc, obj, user)
 
-    imported_docs_by_page_id[imported_doc.page.id] = obj
+    imported_ids.append(imported_doc.page.id)
 
     for child in imported_doc.children:
-        _import_notion_child_page(child, obj, user, imported_docs_by_page_id)
+        _import_notion_child_page(child, obj, user, imported_ids)
 
 
-def _import_notion_root_page(imported_doc, user, imported_docs_by_page_id):
+def _import_notion_root_page(imported_doc, user) -> list[str]:
     obj = models.Document.add_root(
         depth=1,
         creator=user,
@@ -2168,23 +2175,81 @@ def _import_notion_root_page(imported_doc, user, imported_docs_by_page_id):
         role=models.RoleChoices.OWNER,
     )
 
-    _import_notion_doc_content(imported_doc, obj, user)
+    imported_ids = [imported_doc.page.id]
 
-    imported_docs_by_page_id[imported_doc.page.id] = obj
+    _import_notion_doc_content(imported_doc, obj, user)
 
     for child in imported_doc.children:
-        _import_notion_child_page(child, obj, user, imported_docs_by_page_id)
+        _import_notion_child_page(child, obj, user, imported_ids)
 
+    return imported_ids
 
-@drf.decorators.api_view(["GET", "POST"])  # TODO: drop GET (used for testing)
-def notion_import_run(request):
-    if "notion_token" not in request.session:
-        raise drf.exceptions.PermissionDenied()
 
-    imported_docs = import_notion(request.session["notion_token"])
+def _generate_notion_progress(
+    all_pages: list[NotionPage], page_statuses: dict[str, str]
+) -> str:
+    raw = json.dumps(
+        [
+            {
+                "title": page.get_title(),
+                "status": page_statuses[page.id],
+            }
+            for page in all_pages
+        ]
+    )
+    return f"data: {raw}\n\n"
+
 
-    imported_docs_by_page_id = {}
-    for imported_doc in imported_docs:
-        _import_notion_root_page(imported_doc, request.user, imported_docs_by_page_id)
+def _notion_import_event_stream(request):
+    session = build_notion_session(request.session["notion_token"])
+    all_pages = fetch_all_pages(session)
 
-    return drf.response.Response({})
+    page_statuses = {}
+    for page in all_pages:
+        page_statuses[page.id] = "pending"
+
+    yield _generate_notion_progress(all_pages, page_statuses)
+
+    docs_by_page_id: dict[str, ImportedDocument] = {}
+    child_page_blocs_ids_to_parent_page_ids: dict[str, str] = {}
+
+    for page in all_pages:
+        docs_by_page_id[page.id] = import_page(
+            session, page, child_page_blocs_ids_to_parent_page_ids
+        )
+        page_statuses[page.id] = "fetched"
+        yield _generate_notion_progress(all_pages, page_statuses)
+
+    for page in all_pages:
+        link_child_page_to_parent(
+            page, docs_by_page_id, child_page_blocs_ids_to_parent_page_ids
+        )
+
+    root_docs = [doc for doc in docs_by_page_id.values() if doc.page.is_root()]
+
+    for root_doc in root_docs:
+        imported_ids = _import_notion_root_page(root_doc, request.user)
+        for imported_id in imported_ids:
+            page_statuses[imported_id] = "imported"
+
+        yield _generate_notion_progress(all_pages, page_statuses)
+
+
+class IgnoreClientContentNegotiation(drf.negotiation.BaseContentNegotiation):
+    def select_parser(self, request, parsers):
+        return parsers[0]
+
+    def select_renderer(self, request, renderers, format_suffix):
+        return (renderers[0], renderers[0].media_type)
+
+
+class NotionImportRunView(drf.views.APIView):
+    content_negotiation_class = IgnoreClientContentNegotiation
+
+    def get(self, request, format=None):
+        if "notion_token" not in request.session:
+            raise drf.exceptions.PermissionDenied()
+
+        return StreamingHttpResponse(
+            _notion_import_event_stream(request), content_type="text/event-stream"
+        )

src/backend/core/notion_schemas/notion_page.py

@@ -56,3 +56,6 @@ def get_title(self) -> str | None:
         # This could be parsed using NotionRichText
         rich_text = title_property["title"][0]
         return rich_text["plain_text"]
+
+    def is_root(self):
+        return isinstance(self.parent, NotionParentWorkspace)

src/backend/core/services/notion_import.py

@@ -460,35 +460,18 @@ def import_page(
     )
 
 
-def import_notion(token: str) -> list[ImportedDocument]:
-    """Recursively imports all Notion pages and blocks accessible using the given token."""
-    session = build_notion_session(token)
-    all_pages = fetch_all_pages(session)
-    docs_by_page_id: dict[str, ImportedDocument] = {}
-    child_page_blocs_ids_to_parent_page_ids: dict[str, str] = {}
-    for page in all_pages:
-        docs_by_page_id[page.id] = import_page(
-            session, page, child_page_blocs_ids_to_parent_page_ids
-        )
-
-    root_pages = []
-    for page in all_pages:
-        if isinstance(page.parent, NotionParentPage):
-            docs_by_page_id[page.parent.page_id].children.append(
-                docs_by_page_id[page.id]
+def link_child_page_to_parent(
+    page: NotionPage,
+    docs_by_page_id: dict[str, ImportedDocument],
+    child_page_blocs_ids_to_parent_page_ids: dict[str, str],
+):
+    if isinstance(page.parent, NotionParentPage):
+        docs_by_page_id[page.parent.page_id].children.append(docs_by_page_id[page.id])
+    elif isinstance(page.parent, NotionParentBlock):
+        parent_page_id = child_page_blocs_ids_to_parent_page_ids.get(page.id)
+        if parent_page_id:
+            docs_by_page_id[parent_page_id].children.append(docs_by_page_id[page.id])
+        else:
+            logger.warning(
+                f"Page {page.id} has a parent block, but no parent page found."
             )
-        elif isinstance(page.parent, NotionParentBlock):
-            parent_page_id = child_page_blocs_ids_to_parent_page_ids.get(page.id)
-            if parent_page_id:
-                docs_by_page_id[parent_page_id].children.append(
-                    docs_by_page_id[page.id]
-                )
-            else:
-                logger.warning(
-                    f"Page {page.id} has a parent block, but no parent page found."
-                )
-        elif isinstance(page.parent, NotionParentWorkspace):
-            # This is a root page, not a child of another page
-            root_pages.append(docs_by_page_id[page.id])
-
-    return root_pages

src/backend/core/urls.py

@@ -64,7 +64,7 @@
                         [
                             path("redirect", viewsets.notion_import_redirect),
                             path("callback", viewsets.notion_import_callback),
-                            path("run", viewsets.notion_import_run),
+                            path("run", viewsets.NotionImportRunView.as_view()),
                         ]
                     ),
                 ),

src/frontend/apps/impress/src/features/docs/doc-management/api/useImportNotion.tsx

@@ -1,33 +1,73 @@
-import { useMutation, useQueryClient } from '@tanstack/react-query';
 import { useRouter } from 'next/navigation';
+import { useEffect, useState } from 'react';
 
-import { APIError, errorCauses, fetchAPI } from '@/api';
+import { baseApiUrl } from '@/api';
 
-import { KEY_LIST_DOC } from './useDocs';
+type ImportState = {
+  title: string;
+  status: 'pending' | 'fetched' | 'imported';
+}[];
 
-export const importNotion = async (): Promise<void> => {
-  const response = await fetchAPI('notion_import/run', {
-    method: 'POST',
-  });
+const computeSuccessPercentage = (importState?: ImportState) => {
+  if (!importState) {
+    return 0;
+  }
+  if (!importState.length) {
+    return 100;
+  }
 
-  if (!response.ok) {
-    throw new APIError(
-      'Failed to import the Notion',
-      await errorCauses(response),
-    );
+  let fetchedFiles = 0;
+  let importedFiles = 0;
+
+  for (const file of importState) {
+    if (file.status === 'fetched') {
+      fetchedFiles += 1;
+    } else if (file.status === 'imported') {
+      fetchedFiles += 1;
+      importedFiles += 1;
+    }
   }
+
+  const filesNb = importState.length;
+
+  return Math.round(((fetchedFiles + importedFiles) / (2 * filesNb)) * 100);
 };
 
 export function useImportNotion() {
   const router = useRouter();
-  const queryClient = useQueryClient();
-  return useMutation<void, APIError, void>({
-    mutationFn: importNotion,
-    onSuccess: () => {
-      void queryClient.resetQueries({
-        queryKey: [KEY_LIST_DOC],
-      });
-      router.push('/');
-    },
-  });
+
+  const [importState, setImportState] = useState<ImportState>();
+
+  useEffect(() => {
+    // send the request with an Event Source
+    const eventSource = new EventSource(
+      `${baseApiUrl('1.0')}notion_import/run`,
+      {
+        withCredentials: true,
+      },
+    );
+
+    eventSource.onmessage = (event) => {
+      const files = JSON.parse(event.data as string) as ImportState;
+
+      // si tous les fichiers sont chargés, rediriger vers la home page
+      if (files.some((file) => file.status === 'imported')) {
+        eventSource.close();
+        router.push('/');
+      }
+
+      // mettre à jour le state d'import
+      setImportState(files);
+    };
+
+    return () => {
+      eventSource.close();
+    };
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, []);
+
+  return {
+    importState,
+    percentageValue: computeSuccessPercentage(importState),
+  };
 }

src/frontend/apps/impress/src/i18n/translations.json

@@ -690,6 +690,9 @@
       "No text selected": "Aucun texte sélectionné",
       "No versions": "Aucune version",
       "Notion import in progress...": "Import Notion en cours...",
+      "Notion import fetched": "🔄 Page Notion récupérée",
+      "Notion import imported": "✅️ Importé",
+      "Notion import pending": "⏸️ En attente",
       "OK": "OK",
       "Offline ?!": "Hors-ligne ?!",
       "Only invited people can access": "Seules les personnes invitées peuvent accéder",

src/frontend/apps/impress/src/pages/import-notion/index.tsx

@@ -1,5 +1,5 @@
 import { Loader } from '@openfun/cunningham-react';
-import { ReactElement, useEffect } from 'react';
+import { ReactElement } from 'react';
 import { useTranslation } from 'react-i18next';
 
 import { Box, Text } from '@/components';
@@ -10,12 +10,7 @@ import { NextPageWithLayout } from '@/types/next';
 const Page: NextPageWithLayout = () => {
   const { t } = useTranslation();
 
-  const { mutate: importNotion } = useImportNotion();
-
-  useEffect(() => {
-    importNotion();
-    // eslint-disable-next-line react-hooks/exhaustive-deps
-  }, []);
+  const { importState, percentageValue } = useImportNotion();
 
   return (
     <Box
@@ -31,6 +26,16 @@ const Page: NextPageWithLayout = () => {
         {t('Please stay on this page and be patient')}
       </Text>
       <Loader />
+      <Text as="p" $margin={{ top: '10px', bottom: '30px' }}>
+        {percentageValue}%
+      </Text>
+      <Box>
+        {importState?.map((page) => (
+          <Text
+            key={page.title}
+          >{`${page.title} - ${t(`Notion import ${page.status}`)}`}</Text>
+        ))}
+      </Box>
     </Box>
   );
 };