Add CDN-based syntax highlighting for 18 languages

scripts/install-bundled-lsp.ts

@@ -1,14 +1,18 @@
 /**
  * Install LSP dependencies for all bundled extensions
+ * and copy tree-sitter WASM files to public directory
  * Runs automatically after `bun install` via postinstall hook
  */
 
 import { existsSync } from "node:fs";
-import { readdir } from "node:fs/promises";
+import { copyFile, mkdir, readdir } from "node:fs/promises";
 import { join } from "node:path";
 import { $ } from "bun";
 
 const BUNDLED_EXTENSIONS_DIR = "src/extensions/bundled";
+const TREE_SITTER_WASMS_DIR = "node_modules/tree-sitter-wasms/out";
+const PUBLIC_PARSERS_DIR = "public/tree-sitter/parsers";
+const PUBLIC_QUERIES_DIR = "public/tree-sitter/queries";
 
 async function installBundledLspDependencies() {
   console.log("Installing bundled extension LSP dependencies...");
@@ -42,4 +46,83 @@ async function installBundledLspDependencies() {
   console.log("Bundled LSP installation complete.");
 }
 
+/**
+ * Copy tree-sitter WASM files from node_modules to public directory
+ * Only copies parsers that are actually used by bundled extensions
+ */
+async function copyTreeSitterWasms() {
+  console.log("Copying tree-sitter WASM files...");
+
+  const wasmsDir = join(process.cwd(), TREE_SITTER_WASMS_DIR);
+  const publicDir = join(process.cwd(), PUBLIC_PARSERS_DIR);
+
+  if (!existsSync(wasmsDir)) {
+    console.log("  tree-sitter-wasms not installed, skipping.");
+    return;
+  }
+
+  // Ensure public directory exists
+  await mkdir(publicDir, { recursive: true });
+
+  // List of parsers to copy (used by bundled extensions)
+  // Note: Rust uses CDN (see src/extensions/languages/parser-cdn.ts)
+  const parsersToInstall = ["tree-sitter-tsx.wasm"];
+
+  for (const parser of parsersToInstall) {
+    const src = join(wasmsDir, parser);
+    const dest = join(publicDir, parser);
+
+    if (existsSync(src)) {
+      try {
+        await copyFile(src, dest);
+        console.log(`  Copied ${parser}`);
+      } catch (error) {
+        console.error(`  Failed to copy ${parser}:`, error);
+      }
+    } else {
+      console.warn(`  ${parser} not found in tree-sitter-wasms`);
+    }
+  }
+
+  console.log("Tree-sitter WASM copy complete.");
+}
+
+/**
+ * Copy highlight queries from bundled extensions to public directory
+ */
+async function copyHighlightQueries() {
+  console.log("Copying highlight queries...");
+
+  const bundledDir = join(process.cwd(), BUNDLED_EXTENSIONS_DIR);
+  const publicQueriesDir = join(process.cwd(), PUBLIC_QUERIES_DIR);
+
+  // Map of bundled extension to query folder name
+  // Note: Rust uses CDN (see src/extensions/languages/parser-cdn.ts)
+  const extensionQueryMap: Record<string, string> = {
+    typescript: "tsx", // TypeScript extension provides tsx queries
+  };
+
+  for (const [extName, queryFolder] of Object.entries(extensionQueryMap)) {
+    const srcQuery = join(bundledDir, extName, "queries", "highlights.scm");
+    const destDir = join(publicQueriesDir, queryFolder);
+    const destQuery = join(destDir, "highlights.scm");
+
+    if (existsSync(srcQuery)) {
+      try {
+        await mkdir(destDir, { recursive: true });
+        await copyFile(srcQuery, destQuery);
+        console.log(`  Copied ${extName} queries to ${queryFolder}/`);
+      } catch (error) {
+        console.error(`  Failed to copy ${extName} queries:`, error);
+      }
+    } else {
+      console.warn(`  No queries found for ${extName}`);
+    }
+  }
+
+  console.log("Highlight queries copy complete.");
+}
+
 await installBundledLspDependencies();
+await copyTreeSitterWasms();
+await copyHighlightQueries();

src/extensions/bundled/typescript/queries/highlights.scm

@@ -1,9 +1,5 @@
-; Combined highlights for TSX (JavaScript + JSX + TypeScript)
-; Based on tree-sitter-javascript and tree-sitter-typescript queries
-
-; ====================
-; JavaScript Base
-; ====================
+; Highlights for TSX/TypeScript/JavaScript
+; Simplified query compatible with tree-sitter-wasms
 
 ; Variables
 (identifier) @variable
@@ -23,19 +19,10 @@
   key: (property_identifier) @function.method
   value: [(function_expression) (arrow_function)])
 
-(assignment_expression
-  left: (member_expression
-    property: (property_identifier) @function.method)
-  right: [(function_expression) (arrow_function)])
-
 (variable_declarator
   name: (identifier) @function
   value: [(function_expression) (arrow_function)])
 
-(assignment_expression
-  left: (identifier) @function
-  right: [(function_expression) (arrow_function)])
-
 ; Function and method calls
 (call_expression
   function: (identifier) @function)
@@ -55,14 +42,6 @@
  ] @constant
  (#match? @constant "^[A-Z_][A-Z\\d_]+$"))
 
-((identifier) @variable.builtin
- (#match? @variable.builtin "^(arguments|module|console|window|document)$")
- (#is-not? local))
-
-((identifier) @function.builtin
- (#eq? @function.builtin "require")
- (#is-not? local))
-
 ; Literals
 (this) @variable.builtin
 (super) @variable.builtin
@@ -84,7 +63,7 @@
 (regex) @string.special
 (number) @number
 
-; Tokens
+; Punctuation
 [
   ";"
   (optional_chain)
@@ -196,24 +175,7 @@
   "yield"
 ] @keyword
 
-; ====================
-; JSX Extensions
-; ====================
-
-(jsx_opening_element (identifier) @tag (#match? @tag "^[a-z][^.]*$"))
-(jsx_closing_element (identifier) @tag (#match? @tag "^[a-z][^.]*$"))
-(jsx_self_closing_element (identifier) @tag (#match? @tag "^[a-z][^.]*$"))
-
-(jsx_attribute (property_identifier) @attribute)
-(jsx_opening_element (["<" ">"]) @punctuation.bracket)
-(jsx_closing_element (["</" ">"]) @punctuation.bracket)
-(jsx_self_closing_element (["<" "/>"]) @punctuation.bracket)
-
-; ====================
 ; TypeScript Extensions
-; ====================
-
-; Types
 (type_identifier) @type
 (predefined_type) @type.builtin
 
@@ -224,11 +186,9 @@
   "<" @punctuation.bracket
   ">" @punctuation.bracket)
 
-; Variables
 (required_parameter (identifier) @variable.parameter)
 (optional_parameter (identifier) @variable.parameter)
 
-; TypeScript Keywords
 [
   "abstract"
   "declare"

src/extensions/languages/language-packager.ts

@@ -4,10 +4,11 @@
  */
 
 import type { ExtensionManifest } from "../types/extension-manifest";
+import { getQueryCdnUrl, getWasmCdnUrl } from "./parser-cdn";
 
-// CDN base URL for downloading WASM parsers and highlight queries
+// Fallback CDN base URL for languages not in parser-cdn.ts
 // Can be configured via environment variable
-const CDN_BASE_URL = import.meta.env.VITE_PARSER_CDN_URL || "https://athas.dev/extensions";
+const FALLBACK_CDN_BASE_URL = import.meta.env.VITE_PARSER_CDN_URL || "https://athas.dev/extensions";
 
 // Old manifest format from JSON files
 interface LanguageManifestFile {
@@ -40,10 +41,6 @@ function convertLanguageManifest(manifest: LanguageManifestFile): ExtensionManif
     ext.startsWith(".") ? ext : `.${ext}`,
   );
 
-  // Extract parser name from wasmPath
-  // "/tree-sitter/parsers/tree-sitter-javascript.wasm" -> "tree-sitter-javascript.wasm"
-  const wasmFileName = languageProvider.wasmPath.split("/").pop() || "";
-
   return {
     id: manifest.id,
     name: manifest.name,
@@ -62,7 +59,7 @@ function convertLanguageManifest(manifest: LanguageManifestFile): ExtensionManif
     activationEvents: [`onLanguage:${languageProvider.id}`],
     // Extension is downloadable from CDN
     installation: {
-      downloadUrl: `${CDN_BASE_URL}/parsers/${wasmFileName}`,
+      downloadUrl: getWasmUrlForLanguage(languageProvider.id),
       size: 0, // Will be determined during download
       checksum: "", // Will be calculated after download
       minEditorVersion: "0.1.0",
@@ -116,14 +113,28 @@ export function getLanguageExtensionByFileExt(fileExt: string): ExtensionManifes
 
 /**
  * Get WASM download URL for a language
+ * Uses GitHub/jsdelivr CDN if configured, otherwise falls back to athas.dev
  */
 export function getWasmUrlForLanguage(languageId: string): string {
-  return `${CDN_BASE_URL}/parsers/tree-sitter-${languageId}.wasm`;
+  // Check if this language has a version-pinned CDN config
+  const cdnUrl = getWasmCdnUrl(languageId);
+  if (cdnUrl) {
+    return cdnUrl;
+  }
+  // Fall back to default CDN
+  return `${FALLBACK_CDN_BASE_URL}/parsers/tree-sitter-${languageId}.wasm`;
 }
 
 /**
  * Get highlight query URL for a language
+ * Uses GitHub CDN if configured, otherwise falls back to athas.dev
  */
 export function getHighlightQueryUrl(languageId: string): string {
-  return `${CDN_BASE_URL}/queries/${languageId}/highlights.scm`;
+  // Check if this language has a version-pinned CDN config
+  const cdnUrl = getQueryCdnUrl(languageId);
+  if (cdnUrl) {
+    return cdnUrl;
+  }
+  // Fall back to default CDN
+  return `${FALLBACK_CDN_BASE_URL}/queries/${languageId}/highlights.scm`;
 }