diff --git a/apps/test/app/page.tsx b/apps/test/app/page.tsx index 7d2b9820..da91c112 100644 --- a/apps/test/app/page.tsx +++ b/apps/test/app/page.tsx @@ -1,6 +1,9 @@ import { gateway } from "@ai-sdk/gateway"; import { Chat } from "./components/chat"; +// ISR: Generate on-demand, cache for 5 minutes +export const revalidate = 300; + export default async function Page() { const { models } = await gateway.getAvailableModels(); const list = models diff --git a/apps/website/components/code-block.tsx b/apps/website/components/code-block.tsx index 7a49007a..6c39ddf4 100644 --- a/apps/website/components/code-block.tsx +++ b/apps/website/components/code-block.tsx @@ -6,11 +6,14 @@ import { createContext, type HTMLAttributes, useContext, - useEffect, - useRef, useState, } from "react"; -import { type BundledLanguage, codeToHtml } from "shiki"; +import { + bundledLanguages, + type BundledLanguage, + type SpecialLanguage, +} from "shiki"; +import { createJavaScriptRegexEngine, useShikiHighlighter } from "react-shiki"; import { cn } from "@/lib/utils"; type CodeBlockProps = HTMLAttributes & { @@ -26,19 +29,6 @@ const CodeBlockContext = createContext({ code: "", }); -export async function highlightCode(code: string, language: BundledLanguage) { - return Promise.all([ - await codeToHtml(code, { - lang: language, - theme: "github-light", - }), - await codeToHtml(code, { - lang: language, - theme: "github-dark", - }), - ]); -} - export const CodeBlock = ({ code, language, @@ -46,43 +36,32 @@ export const CodeBlock = ({ children, ...props }: CodeBlockProps) => { - const [html, setHtml] = useState(""); - const [darkHtml, setDarkHtml] = useState(""); - const mounted = useRef(false); + const isLanguageSupported = (lang: string): lang is BundledLanguage => { + return Object.hasOwn(bundledLanguages, lang); + }; - useEffect(() => { - highlightCode(code, language).then(([light, dark]) => { - if (!mounted.current) { - setHtml(light); - setDarkHtml(dark); - mounted.current = true; - } - }); + const langToUse = isLanguageSupported(language) + ? language + : ("text" as SpecialLanguage); - return () => { - mounted.current = false; - }; - }, [code, language]); + const html = useShikiHighlighter( + code, + langToUse, + { light: "github-light", dark: "github-dark" }, + { + outputFormat: "html", + defaultColor: "light-dark()", + engine: createJavaScriptRegexEngine({ forgiving: true }), + } + ); return (
pre]:bg-transparent!", - className - )} + className={cn("overflow-x-auto [&>pre]:bg-transparent!", className)} // biome-ignore lint/security/noDangerouslySetInnerHtml: "this is needed." - dangerouslySetInnerHTML={{ __html: html }} - {...props} - /> -
pre]:bg-transparent!", - className - )} - // biome-ignore lint/security/noDangerouslySetInnerHtml: "this is needed." - dangerouslySetInnerHTML={{ __html: darkHtml }} + dangerouslySetInnerHTML={{ __html: (html as string) || "" }} {...props} /> {children} @@ -115,10 +94,12 @@ export const CodeBlockCopyButton = ({ } try { - await navigator.clipboard.writeText(code); - setIsCopied(true); - onCopy?.(); - setTimeout(() => setIsCopied(false), timeout); + if (!isCopied) { + await navigator.clipboard.writeText(code); + setIsCopied(true); + onCopy?.(); + setTimeout(() => setIsCopied(false), timeout); + } } catch (error) { onError?.(error as Error); } diff --git a/apps/website/package.json b/apps/website/package.json index 503ba054..b63e12c2 100644 --- a/apps/website/package.json +++ b/apps/website/package.json @@ -21,6 +21,7 @@ "react": "19.1.1", "react-dom": "19.1.1", "react-markdown": "^10.1.0", + "react-shiki": "^0.9.0", "rehype-harden": "^1.1.5", "shiki": "^3.12.2", "streamdown": "workspace:*", diff --git a/packages/streamdown/__tests__/code-block.test.tsx b/packages/streamdown/__tests__/code-block.test.tsx index 504324e4..23633f55 100644 --- a/packages/streamdown/__tests__/code-block.test.tsx +++ b/packages/streamdown/__tests__/code-block.test.tsx @@ -313,4 +313,212 @@ describe("CodeBlock with multiple languages", () => { { timeout: 5000 } ); }); + + it("should actually highlight code with syntax colors", async () => { + const jsCode = "const x = 1;"; + const { container } = render( + + + + ); + + await waitFor( + () => { + // Check for shiki-themed pre element with CSS variables + const pre = container.querySelector("pre.shiki"); + expect(pre).toBeTruthy(); + expect(pre).toHaveClass("shiki-themes"); + + // Verify CSS variables for multi-theme support + const style = pre?.getAttribute("style"); + expect(style).toBeTruthy(); + expect(style).toContain("--shiki-light"); + expect(style).toContain("--shiki-dark"); + + // Verify actual syntax highlighting tokens exist + // Code should be wrapped in span elements (tokens) + const tokens = container.querySelectorAll("pre.shiki span.line span"); + expect(tokens.length).toBeGreaterThan(0); + + // At least one token should have style attribute with color + const tokensWithStyle = Array.from(tokens).filter( + (token) => token.getAttribute("style") + ); + expect(tokensWithStyle.length).toBeGreaterThan(0); + }, + { timeout: 5000 } + ); + }); + + it("should remove background styles from pre element", async () => { + const { container } = render( + + + + ); + + await waitFor( + () => { + const pre = container.querySelector("pre.shiki"); + expect(pre).toBeTruthy(); + + const style = pre?.getAttribute("style"); + expect(style).toBeTruthy(); + + // Should NOT contain ANY background properties + expect(style).not.toMatch(/background[^;]*;?/); + expect(style).not.toMatch(/background-color/); + expect(style).not.toMatch(/background-image/); + expect(style).not.toMatch(/background:/); + + // Should still have CSS variables for theming (proves we didn't remove all styles) + expect(style).toContain("--shiki-light"); + expect(style).toContain("--shiki-dark"); + + // Verify it actually has syntax highlighting tokens + const tokens = pre.querySelectorAll("span.line span"); + expect(tokens.length).toBeGreaterThan(0); + }, + { timeout: 5000 } + ); + }); + + it("should apply preClassName to pre element", async () => { + const { container } = render( + + + + ); + + await waitFor( + () => { + const pre = container.querySelector("pre.shiki"); + expect(pre).toBeTruthy(); + + // Verify the actual class attribute contains our custom class + const classAttr = pre?.getAttribute("class"); + expect(classAttr).toContain("custom-pre-class"); + expect(classAttr).toContain("shiki"); + + // Also use toHaveClass for convenience + expect(pre).toHaveClass("custom-pre-class"); + expect(pre).toHaveClass("shiki"); + expect(pre).toHaveClass("shiki-themes"); + + // Verify highlighting actually happened (not just an empty pre) + const code = pre?.querySelector("code"); + expect(code).toBeTruthy(); + expect(code?.textContent).toContain("const x = 1"); + }, + { timeout: 5000 } + ); + }); + + it("should handle preClassName with multiple existing classes", async () => { + const { container } = render( + + + + ); + + await waitFor( + () => { + const pre = container.querySelector("pre.shiki"); + expect(pre).toBeTruthy(); + + // Should merge all classes + expect(pre?.className).toContain("shiki"); + expect(pre?.className).toContain("my-custom-class another-class"); + }, + { timeout: 5000 } + ); + }); + + it("should work without preClassName (transformer handles undefined)", async () => { + const { container } = render( + + + + ); + + await waitFor( + () => { + const pre = container.querySelector("pre.shiki"); + expect(pre).toBeTruthy(); + + // Should only have Shiki's classes + const classAttr = pre?.getAttribute("class"); + expect(classAttr).toContain("shiki"); + expect(classAttr).not.toContain("undefined"); + expect(classAttr).not.toContain("null"); + + // Highlighting should still work + const tokens = pre?.querySelectorAll("span.line span"); + expect(tokens.length).toBeGreaterThan(0); + }, + { timeout: 5000 } + ); + }); + + it("should produce valid HTML with transformers applied (integration test)", async () => { + const { container } = render( + + + + ); + + await waitFor( + () => { + const pre = container.querySelector("pre"); + expect(pre).toBeTruthy(); + + // Get actual rendered HTML + const preHTML = pre?.outerHTML; + expect(preHTML).toBeTruthy(); + + // Verify structure: pre > code > span.line > span (tokens) + expect(preHTML).toContain(")/; - type CodeBlockProps = HTMLAttributes & { code: string; language: BundledLanguage; @@ -37,143 +34,60 @@ const CodeBlockContext = createContext({ code: "", }); -class HighlighterManager { - private lightHighlighter: Awaited< - ReturnType - > | null = null; - private darkHighlighter: Awaited< - ReturnType - > | null = null; - private lightTheme: BundledTheme | null = null; - private darkTheme: BundledTheme | null = null; - private readonly loadedLanguages: Set = new Set(); - private initializationPromise: Promise | null = null; - - private isLanguageSupported(language: string): language is BundledLanguage { - return Object.hasOwn(bundledLanguages, language); - } - - private getFallbackLanguage(): SpecialLanguage { - return "text"; - } - - private async ensureHighlightersInitialized( - themes: [BundledTheme, BundledTheme], - language: BundledLanguage - ): Promise { - const [lightTheme, darkTheme] = themes; - const jsEngine = createJavaScriptRegexEngine({ forgiving: true }); - - // Check if we need to recreate highlighters due to theme change - const needsLightRecreation = - !this.lightHighlighter || this.lightTheme !== lightTheme; - const needsDarkRecreation = - !this.darkHighlighter || this.darkTheme !== darkTheme; - - if (needsLightRecreation || needsDarkRecreation) { - // If themes changed, reset loaded languages - this.loadedLanguages.clear(); - } - - // Check if we need to load the language - const isLanguageSupported = this.isLanguageSupported(language); - const needsLanguageLoad = - !this.loadedLanguages.has(language) && isLanguageSupported; - - // Create or recreate light highlighter if needed - if (needsLightRecreation) { - this.lightHighlighter = await createHighlighter({ - themes: [lightTheme], - langs: isLanguageSupported ? [language] : [], - engine: jsEngine, - }); - this.lightTheme = lightTheme; - if (isLanguageSupported) { - this.loadedLanguages.add(language); - } - } else if (needsLanguageLoad) { - // Load the language if not already loaded - await this.lightHighlighter?.loadLanguage(language); - } - - // Create or recreate dark highlighter if needed - if (needsDarkRecreation) { - // If recreating dark highlighter, load all previously loaded languages plus the new one - const langsToLoad = needsLanguageLoad - ? [...this.loadedLanguages].concat( - isLanguageSupported ? [language] : [] - ) - : Array.from(this.loadedLanguages); - - this.darkHighlighter = await createHighlighter({ - themes: [darkTheme], - langs: - langsToLoad.length > 0 - ? langsToLoad - : isLanguageSupported - ? [language] - : [], - engine: jsEngine, - }); - this.darkTheme = darkTheme; - } else if (needsLanguageLoad) { - // Load the language if not already loaded - await this.darkHighlighter?.loadLanguage(language); +// shiki transformers for background removal and pre class injection +// operates on AST, more reliable than regex +const createPreClassTransformer = (className?: string): ShikiTransformer => ({ + name: "streamdown:pre-class", + pre(node) { + if (!className) { + return; } - // Mark language as loaded after both highlighters have it - if (needsLanguageLoad) { - this.loadedLanguages.add(language); + const existingClass = node.properties?.class; + if (typeof existingClass === "string") { + node.properties.class = `${existingClass} ${className}`; + } else { + node.properties.class = className; } - } + }, +}); - async highlightCode( - code: string, - language: BundledLanguage, - themes: [BundledTheme, BundledTheme], - preClassName?: string - ): Promise<[string, string]> { - // Ensure only one initialization happens at a time - if (this.initializationPromise) { - await this.initializationPromise; +// original implementation removed the entire style attribute, didn't +// support use of Shiki's native dark mode/multi-theme support +// this parses AST with more targeted regex to only remove background* properties +const removeBackgroundTransformer: ShikiTransformer = { + name: "streamdown:remove-background", + pre(node) { + const style = node.properties?.style; + if (typeof style === "string") { + node.properties.style = style.replace(/background[^;]*;?/g, "").trim(); } - // Initialize or load language - this.initializationPromise = this.ensureHighlightersInitialized( - themes, - language - ); - await this.initializationPromise; - this.initializationPromise = null; - - const [lightTheme, darkTheme] = themes; - - const lang = this.isLanguageSupported(language) - ? language - : this.getFallbackLanguage(); - - const light = this.lightHighlighter?.codeToHtml(code, { - lang, - theme: lightTheme, - }); - - const dark = this.darkHighlighter?.codeToHtml(code, { - lang, - theme: darkTheme, - }); - - const addPreClass = (html: string) => { - if (!preClassName) { - return html; - } - return html.replace(PRE_TAG_REGEX, `
 {
+//   if (!rawHtml || typeof rawHtml !== 'string') return "";
+//   return addPreClassToHtml(
+//     removeBackgroundFromHtml(rawHtml),
+//     preClassName
+//   );
+// }, [rawHtml, preClassName]);
+//
+// const removeBackgroundFromHtml = (html: string) =>
+//   html.replace(/style="([^"]*)"/g, (_, styles) => {
+//     const cleaned = styles.replace(/background[^;]*;?/g, '').trim();
+//     return cleaned ? `style="${cleaned}"` : '';
+//   });
+//
+// const addPreClassToHtml = (html: string, className?: string) => {
+//   if (!className) return html;
+//   return html.replace(
+//     /
+//       `
 {
-  const [html, setHtml] = useState("");
-  const [darkHtml, setDarkHtml] = useState("");
-  const mounted = useRef(false);
   const [lightTheme, darkTheme] = useContext(ShikiThemeContext);
 
-  useEffect(() => {
-    mounted.current = true;
-
-    highlighterManager
-      .highlightCode(code, language, [lightTheme, darkTheme], preClassName)
-      .then(([light, dark]) => {
-        if (mounted.current) {
-          setHtml(light);
-          setDarkHtml(dark);
-        }
-      });
+  const isLanguageSupported = (lang: string): lang is BundledLanguage => {
+    return Object.hasOwn(bundledLanguages, lang);
+  };
 
-    return () => {
-      mounted.current = false;
-    };
-  }, [code, language, lightTheme, darkTheme, preClassName]);
+  const langToUse = isLanguageSupported(language)
+    ? language
+    : ("text" as SpecialLanguage);
+
+  // no longer uses two highlighter for light/dark. relies on shiki's native multi-theme and light-dark() support
+  // WARN: BREAKING - does not automatically work for class based dark mode when color-scheme is not set
+  const html = useShikiHighlighter(
+    code,
+    langToUse,
+    { light: lightTheme, dark: darkTheme },
+    {
+      outputFormat: "html",
+      defaultColor: "light-dark()",
+      engine: createJavaScriptRegexEngine({ forgiving: true }), // PR #77 - js engine prevents csp errors
+      transformers: [
+        createPreClassTransformer(preClassName),
+        removeBackgroundTransformer,
+      ],
+    }
+  );
 
   return (
     
@@ -223,17 +142,9 @@ export const CodeBlock = ({
         
-
{ - const { code: contextCode } = useContext(CodeBlockContext); + const contextCode = useContext(CodeBlockContext).code; const { isAnimating } = useContext(StreamdownRuntimeContext); const code = propCode ?? contextCode; const extension = @@ -624,7 +535,7 @@ export const CodeBlockCopyButton = ({ }: CodeBlockCopyButtonProps & { code?: string }) => { const [isCopied, setIsCopied] = useState(false); const timeoutRef = useRef(0); - const { code: contextCode } = useContext(CodeBlockContext); + const contextCode = useContext(CodeBlockContext).code; const { isAnimating } = useContext(StreamdownRuntimeContext); const code = propCode ?? contextCode; diff --git a/packages/streamdown/lib/table.tsx b/packages/streamdown/lib/table.tsx index 4cc4cb32..0aefb7fd 100644 --- a/packages/streamdown/lib/table.tsx +++ b/packages/streamdown/lib/table.tsx @@ -197,7 +197,6 @@ export const TableDownloadButton = ({ filename, }: TableDownloadButtonProps) => { const { isAnimating } = useContext(StreamdownRuntimeContext); - const downloadTableData = (event: React.MouseEvent) => { try { // Find the closest table element diff --git a/packages/streamdown/package.json b/packages/streamdown/package.json index 52cf198d..4f9e2b3e 100644 --- a/packages/streamdown/package.json +++ b/packages/streamdown/package.json @@ -55,12 +55,14 @@ "marked": "^16.2.1", "mermaid": "^11.11.0", "react-markdown": "^10.1.0", + "react-shiki": "^0.9.0", "rehype-harden": "^1.1.5", "rehype-katex": "^7.0.1", "rehype-raw": "^7.0.0", "remark-gfm": "^4.0.1", "remark-math": "^6.0.0", - "shiki": "^3.12.2", - "tailwind-merge": "^3.3.1" + "shiki": "^3.13.0", + "tailwind-merge": "^3.3.1", + "unified": "^11.0.5" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4a46fd58..b64a4367 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -168,6 +168,9 @@ importers: react-markdown: specifier: ^10.1.0 version: 10.1.0(@types/react@19.1.10)(react@19.1.1) + react-shiki: + specifier: ^0.9.0 + version: 0.9.0(@types/react-dom@19.1.7(@types/react@19.1.10))(@types/react@19.1.10)(react-dom@19.1.1(react@19.1.1))(react@19.1.1) rehype-harden: specifier: ^1.1.5 version: 1.1.5 @@ -223,6 +226,9 @@ importers: react-markdown: specifier: ^10.1.0 version: 10.1.0(@types/react@19.1.12)(react@19.1.1) + react-shiki: + specifier: ^0.9.0 + version: 0.9.0(@types/react-dom@19.1.9(@types/react@19.1.12))(@types/react@19.1.12)(react-dom@19.1.1(react@19.1.1))(react@19.1.1) rehype-harden: specifier: ^1.1.5 version: 1.1.5 @@ -239,11 +245,14 @@ importers: specifier: ^6.0.0 version: 6.0.0 shiki: - specifier: ^3.12.2 - version: 3.12.2 + specifier: ^3.13.0 + version: 3.13.0 tailwind-merge: specifier: ^3.3.1 version: 3.3.1 + unified: + specifier: ^11.0.5 + version: 11.0.5 devDependencies: '@testing-library/jest-dom': specifier: ^6.8.0 @@ -275,9 +284,6 @@ importers: tsup: specifier: ^8.5.0 version: 8.5.0(jiti@2.5.1)(postcss@8.5.6)(tsx@4.20.4)(typescript@5.9.2) - unified: - specifier: ^11.0.5 - version: 11.0.5 vitest: specifier: ^3.2.4 version: 3.2.4(@types/debug@4.1.12)(@types/node@24.3.0)(jiti@2.5.1)(jsdom@26.1.0)(lightningcss@1.30.1)(tsx@4.20.4) @@ -1804,21 +1810,39 @@ packages: '@shikijs/core@3.12.2': resolution: {integrity: sha512-L1Safnhra3tX/oJK5kYHaWmLEBJi1irASwewzY3taX5ibyXyMkkSDZlq01qigjryOBwrXSdFgTiZ3ryzSNeu7Q==} + '@shikijs/core@3.13.0': + resolution: {integrity: sha512-3P8rGsg2Eh2qIHekwuQjzWhKI4jV97PhvYjYUzGqjvJfqdQPz+nMlfWahU24GZAyW1FxFI1sYjyhfh5CoLmIUA==} + '@shikijs/engine-javascript@3.12.2': resolution: {integrity: sha512-Nm3/azSsaVS7hk6EwtHEnTythjQfwvrO5tKqMlaH9TwG1P+PNaR8M0EAKZ+GaH2DFwvcr4iSfTveyxMIvXEHMw==} + '@shikijs/engine-javascript@3.13.0': + resolution: {integrity: sha512-Ty7xv32XCp8u0eQt8rItpMs6rU9Ki6LJ1dQOW3V/56PKDcpvfHPnYFbsx5FFUP2Yim34m/UkazidamMNVR4vKg==} + '@shikijs/engine-oniguruma@3.12.2': resolution: {integrity: sha512-hozwnFHsLvujK4/CPVHNo3Bcg2EsnG8krI/ZQ2FlBlCRpPZW4XAEQmEwqegJsypsTAN9ehu2tEYe30lYKSZW/w==} + '@shikijs/engine-oniguruma@3.13.0': + resolution: {integrity: sha512-O42rBGr4UDSlhT2ZFMxqM7QzIU+IcpoTMzb3W7AlziI1ZF7R8eS2M0yt5Ry35nnnTX/LTLXFPUjRFCIW+Operg==} + '@shikijs/langs@3.12.2': resolution: {integrity: sha512-bVx5PfuZHDSHoBal+KzJZGheFuyH4qwwcwG/n+MsWno5cTlKmaNtTsGzJpHYQ8YPbB5BdEdKU1rga5/6JGY8ww==} + '@shikijs/langs@3.13.0': + resolution: {integrity: sha512-672c3WAETDYHwrRP0yLy3W1QYB89Hbpj+pO4KhxK6FzIrDI2FoEXNiNCut6BQmEApYLfuYfpgOZaqbY+E9b8wQ==} + '@shikijs/themes@3.12.2': resolution: {integrity: sha512-fTR3QAgnwYpfGczpIbzPjlRnxyONJOerguQv1iwpyQZ9QXX4qy/XFQqXlf17XTsorxnHoJGbH/LXBvwtqDsF5A==} + '@shikijs/themes@3.13.0': + resolution: {integrity: sha512-Vxw1Nm1/Od8jyA7QuAenaV78BG2nSr3/gCGdBkLpfLscddCkzkL36Q5b67SrLLfvAJTOUzW39x4FHVCFriPVgg==} + '@shikijs/types@3.12.2': resolution: {integrity: sha512-K5UIBzxCyv0YoxN3LMrKB9zuhp1bV+LgewxuVwHdl4Gz5oePoUFrr9EfgJlGlDeXCU1b/yhdnXeuRvAnz8HN8Q==} + '@shikijs/types@3.13.0': + resolution: {integrity: sha512-oM9P+NCFri/mmQ8LoFGVfVyemm5Hi27330zuOBp0annwJdKH1kOLndw3zCtAVDehPLg9fKqoEx3Ht/wNZxolfw==} + '@shikijs/vscode-textmate@10.0.2': resolution: {integrity: sha512-83yeghZ2xxin3Nj8z1NMd/NCuca+gsYXswywDy5bHvwlWL8tpTQmzGeUuHd9FC3E/SBEMvzJRwWEOz5gGes9Qg==} @@ -3686,6 +3710,19 @@ packages: react: ^16.14.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc react-dom: ^16.14.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc + react-shiki@0.9.0: + resolution: {integrity: sha512-5t+vHGglJioG3LU6uTKFaiOC+KNW7haL8e22ZHSP7m174ZD/X2KgCVJcxvcUOM3FiqjPQD09AyS9/+RcOh3PmA==} + peerDependencies: + '@types/react': '>=16.8.0' + '@types/react-dom': '>=16.8.0' + react: '>= 16.8.0' + react-dom: '>= 16.8.0' + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + react-smooth@4.0.4: resolution: {integrity: sha512-gnGKTpYwqL0Iii09gHobNolvX4Kiq4PKx6eWBCYYix+8cdw+cGo3do906l1NBPKkSWx1DghC1dlWG9L2uGd61Q==} peerDependencies: @@ -3832,6 +3869,9 @@ packages: shiki@3.12.2: resolution: {integrity: sha512-uIrKI+f9IPz1zDT+GMz+0RjzKJiijVr6WDWm9Pe3NNY6QigKCfifCEv9v9R2mDASKKjzjQ2QpFLcxaR3iHSnMA==} + shiki@3.13.0: + resolution: {integrity: sha512-aZW4l8Og16CokuCLf8CF8kq+KK2yOygapU5m3+hoGw0Mdosc6fPitjM+ujYarppj5ZIKGyPDPP1vqmQhr+5/0g==} + siginfo@2.0.0: resolution: {integrity: sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==} @@ -5927,30 +5967,61 @@ snapshots: '@types/hast': 3.0.4 hast-util-to-html: 9.0.5 + '@shikijs/core@3.13.0': + dependencies: + '@shikijs/types': 3.13.0 + '@shikijs/vscode-textmate': 10.0.2 + '@types/hast': 3.0.4 + hast-util-to-html: 9.0.5 + '@shikijs/engine-javascript@3.12.2': dependencies: '@shikijs/types': 3.12.2 '@shikijs/vscode-textmate': 10.0.2 oniguruma-to-es: 4.3.3 + '@shikijs/engine-javascript@3.13.0': + dependencies: + '@shikijs/types': 3.13.0 + '@shikijs/vscode-textmate': 10.0.2 + oniguruma-to-es: 4.3.3 + '@shikijs/engine-oniguruma@3.12.2': dependencies: '@shikijs/types': 3.12.2 '@shikijs/vscode-textmate': 10.0.2 + '@shikijs/engine-oniguruma@3.13.0': + dependencies: + '@shikijs/types': 3.13.0 + '@shikijs/vscode-textmate': 10.0.2 + '@shikijs/langs@3.12.2': dependencies: '@shikijs/types': 3.12.2 + '@shikijs/langs@3.13.0': + dependencies: + '@shikijs/types': 3.13.0 + '@shikijs/themes@3.12.2': dependencies: '@shikijs/types': 3.12.2 + '@shikijs/themes@3.13.0': + dependencies: + '@shikijs/types': 3.13.0 + '@shikijs/types@3.12.2': dependencies: '@shikijs/vscode-textmate': 10.0.2 '@types/hast': 3.0.4 + '@shikijs/types@3.13.0': + dependencies: + '@shikijs/vscode-textmate': 10.0.2 + '@types/hast': 3.0.4 + '@shikijs/vscode-textmate@10.0.2': {} '@standard-schema/spec@1.0.0': {} @@ -8173,6 +8244,36 @@ snapshots: react: 19.1.1 react-dom: 19.1.1(react@19.1.1) + react-shiki@0.9.0(@types/react-dom@19.1.7(@types/react@19.1.10))(@types/react@19.1.10)(react-dom@19.1.1(react@19.1.1))(react@19.1.1): + dependencies: + clsx: 2.1.1 + dequal: 2.0.3 + hast-util-to-jsx-runtime: 2.3.6 + react: 19.1.1 + react-dom: 19.1.1(react@19.1.1) + shiki: 3.13.0 + unist-util-visit: 5.0.0 + optionalDependencies: + '@types/react': 19.1.10 + '@types/react-dom': 19.1.7(@types/react@19.1.10) + transitivePeerDependencies: + - supports-color + + react-shiki@0.9.0(@types/react-dom@19.1.9(@types/react@19.1.12))(@types/react@19.1.12)(react-dom@19.1.1(react@19.1.1))(react@19.1.1): + dependencies: + clsx: 2.1.1 + dequal: 2.0.3 + hast-util-to-jsx-runtime: 2.3.6 + react: 19.1.1 + react-dom: 19.1.1(react@19.1.1) + shiki: 3.13.0 + unist-util-visit: 5.0.0 + optionalDependencies: + '@types/react': 19.1.12 + '@types/react-dom': 19.1.9(@types/react@19.1.12) + transitivePeerDependencies: + - supports-color + react-smooth@4.0.4(react-dom@19.1.1(react@19.1.1))(react@19.1.1): dependencies: fast-equals: 5.2.2 @@ -8411,6 +8512,17 @@ snapshots: '@shikijs/vscode-textmate': 10.0.2 '@types/hast': 3.0.4 + shiki@3.13.0: + dependencies: + '@shikijs/core': 3.13.0 + '@shikijs/engine-javascript': 3.13.0 + '@shikijs/engine-oniguruma': 3.13.0 + '@shikijs/langs': 3.13.0 + '@shikijs/themes': 3.13.0 + '@shikijs/types': 3.13.0 + '@shikijs/vscode-textmate': 10.0.2 + '@types/hast': 3.0.4 + siginfo@2.0.0: {} signal-exit@4.1.0: {} diff --git a/react-shiki-migration-context.md b/react-shiki-migration-context.md new file mode 100644 index 00000000..3eb6958c --- /dev/null +++ b/react-shiki-migration-context.md @@ -0,0 +1,155 @@ +# React-Shiki Migration + +Migration from custom dual-highlighter implementation to react-shiki with Shiki singleton. Reduces ~100 lines of custom highlighter management to ~30 lines while improving language persistence and eliminating race conditions. + +--- + +## 1. Multi-Theme Highlighting + +**Original (PR #40, Aug 2025)**: Two highlighter instances (light/dark), generated two HTML outputs, used `dark:hidden`/`dark:block` for theme switching. + +**Migrated**: Single `getSingletonHighlighter` with both themes, one HTML output with CSS variables, `light-dark()` function for theme switching. + +**Breaking change**: Requires `color-scheme` property (not just `.dark` class). `next-themes` sets both automatically, but standard Tailwind dark mode users must add `color-scheme` management. + +**Solutions**: +1. Document `color-scheme` requirement +2. Add MutationObserver to sync `.dark` class → `color-scheme` + +--- + +## 2. HTML Transformations + +**Original**: Regex-based string manipulation for `preClassName` injection and background removal. + +**Migrated**: HAST transformers (`preClassTransformer`, `removeBackgroundTransformer`) operating on abstract syntax tree. + +--- + +## 3. Language Loading + +**Original (Issue #78, PR #80)**: `HighlighterManager` class (~100 lines) with manual `Set` tracking, synchronized loading across dual highlighters, `initializationPromise` for race conditions. Theme changes cleared languages, recreated highlighters, pre-loaded all languages. Used `createJavaScriptRegexEngine` per PR #77. + +**Migrated**: `getSingletonHighlighter` with manual language loading (required when passing custom highlighter to react-shiki). + +```typescript +async function getStreamdownHighlighter(themes, lang) { + const highlighter = await getSingletonHighlighter({ + themes, + langs: [], + engine: createJavaScriptRegexEngine({ forgiving: true }) + }); + + if (lang !== "text" && Object.hasOwn(bundledLanguages, lang)) { + if (!highlighter.getLoadedLanguages().includes(lang)) { + await highlighter.loadLanguage(lang); + } + } + + return highlighter; +} +``` + +**Why manual loading**: React-shiki's `highlighterFactory` auto-loads languages, but skips when custom highlighter provided. Falls back to `plaintext` if language not loaded. + +**Improvements**: Languages persist in Registry (never cleared), no manual tracking, no synchronization, no race conditions. ~100 lines → ~8 lines. + +--- + +## 4. Further Simplification (react-shiki 0.9.0) + +**Migration to react-shiki 0.9.0**: Upgraded from 0.8.0 to leverage new `engine` parameter support, eliminating custom highlighter creation. + +**Before (0.8.0)**: +```typescript +// Create custom highlighter to use JS engine +const highlighter = await getSingletonHighlighter({ + themes: [lightTheme, darkTheme], + langs: [], + engine: createJavaScriptRegexEngine({ forgiving: true }), +}); + +// Manually load language +if (!highlighter.getLoadedLanguages().includes(lang)) { + await highlighter.loadLanguage(lang); +} + +// Pass custom highlighter +useShikiHighlighter(code, lang, themes, { highlighter }); +``` + +**After (0.9.0)**: +```typescript +// Just pass engine as an option - react-shiki handles the rest! +useShikiHighlighter(code, lang, themes, { + engine: createJavaScriptRegexEngine({ forgiving: true }), + transformers: [...], +}); +``` + +**Improvements**: No highlighter state management, no manual language loading, no custom highlighter creation. React-shiki's internal factory handles everything. ~30 lines → ~15 lines of highlighter-related code. + +--- + +## 5. Testing + +**All 253 package tests passing**: Multi-language rendering, concurrent blocks, component functionality, data attributes, language fallback. + +**Website verified**: Theme toggle, system preferences, SSR, syntax highlighting all working with `next-themes` (sets both `.dark` class and `color-scheme`). + +--- + +## 6. v1.4 Integration (October 2025) + +**Integrated Features**: +- `isAnimating` prop to disable copy/download buttons during streaming +- rehype-harden plugin system (replaced harden-react-markdown wrapper) +- Exported `defaultRehypePlugins` and `defaultRemarkPlugins` for customization +- Proper timeout cleanup with `useRef` in button components +- StreamdownRuntimeContext for runtime state management + +**Breaking Changes from v1.3**: +- `allowedImagePrefixes`, `allowedLinkPrefixes`, `defaultOrigin` props removed → use `rehypePlugins` with harden config +- Props now passed directly to ReactMarkdown instead of through wrapper + +**Migration Path**: +```typescript +// v1.3 (deprecated) + + +// v1.4+ (recommended) +import { defaultRehypePlugins } from 'streamdown'; +import { harden } from 'rehype-harden'; + + p[0] !== harden), + ]} +/> +``` + +**Testing**: All 258 package tests passing ✅ + +--- + +## Architecture + +| Aspect | Original | Migrated (0.8.0) | Optimized (0.9.0) | Integrated (v1.4) | +|--------|----------|------------------|-------------------|-------------------| +| Highlighters | Dual instances | Singleton (custom) | Singleton (react-shiki managed) | Singleton (react-shiki managed) | +| HTML output | Two (light/dark) | One + CSS variables | One + CSS variables | One + CSS variables | +| Theme switching | DOM visibility | `color-scheme` / `light-dark()` | `color-scheme` / `light-dark()` | `color-scheme` / `light-dark()` | +| Language loading | Manual `Set` + sync | Manual on-demand | Automatic (react-shiki) | Automatic (react-shiki) | +| Highlighter creation | Manual dual | Manual singleton | react-shiki factory | react-shiki factory | +| Streaming UX | N/A | N/A | N/A | Disabled buttons during streaming | +| Plugin System | harden-react-markdown | harden-react-markdown | harden-react-markdown | rehype-harden + exports | +| Code complexity | ~100 lines | ~30 lines | ~15 lines | ~20 lines (w/ runtime context) |