Add ruff parser backend as experimental alternative to tree-sitter

pylight/Cargo.toml

@@ -26,6 +26,10 @@ futures-lite = "1.13"
 rayon = "1.9.0"
 tree-sitter = "0.20.10"
 tree-sitter-python = "0.20.4"
+ruff_python_parser = { git = "https://github.com/astral-sh/ruff", tag = "0.12.1" }
+ruff_python_ast = { git = "https://github.com/astral-sh/ruff", tag = "0.12.1" }
+ruff_text_size = { git = "https://github.com/astral-sh/ruff", tag = "0.12.1" }
+ruff_source_file = { git = "https://github.com/astral-sh/ruff", tag = "0.12.1" }
 fuzzy-matcher = "0.3"
 bincode = "1.3"
 flate2 = "1.0"

pylight/benches/parallel_indexing.rs

@@ -69,7 +69,7 @@ fn benchmark_parallel_indexing(c: &mut Criterion) {
 
                 pool.install(|| {
                     // Create a fresh index for each iteration
-                    let index = Arc::new(SymbolIndex::new());
+                    let index = Arc::new(SymbolIndex::default());
 
                     // Parse and index files
                     // Return the result to prevent optimization
@@ -107,7 +107,7 @@ fn benchmark_parallel_vs_sequential(c: &mut Criterion) {
                 .unwrap();
 
             pool.install(|| {
-                let index = Arc::new(SymbolIndex::new());
+                let index = Arc::new(SymbolIndex::default());
                 index.parse_and_index_files(subset.clone()).unwrap()
             });
         });
@@ -122,7 +122,7 @@ fn benchmark_parallel_vs_sequential(c: &mut Criterion) {
                 .unwrap();
 
             pool.install(|| {
-                let index = Arc::new(SymbolIndex::new());
+                let index = Arc::new(SymbolIndex::default());
                 index.parse_and_index_files(subset.clone()).unwrap()
             });
         });

pylight/src/bin/pylight.rs

@@ -1,7 +1,7 @@
 //! Pylight LSP server binary
 
 use clap::Parser;
-use pylight::{LspServer, Result};
+use pylight::{parser::ParserBackend, LspServer, Result};
 use tracing_subscriber::EnvFilter;
 
 #[derive(Parser, Debug)]
@@ -18,6 +18,10 @@ struct Args {
     /// Search query in standalone mode
     #[arg(short, long, requires = "standalone")]
     query: Option<String>,
+
+    /// Parser backend to use (tree-sitter or ruff)
+    #[arg(long, default_value = "tree-sitter")]
+    parser: String,
 }
 
 fn main() -> Result<()> {
@@ -53,18 +57,30 @@ fn main() -> Result<()> {
 
     let args = Args::parse();
 
+    // Parse the parser backend
+    let parser_backend = args
+        .parser
+        .parse::<ParserBackend>()
+        .map_err(pylight::Error::Parse)?;
+
+    tracing::info!("Using parser backend: {:?}", parser_backend);
+
     if args.standalone {
         // Standalone mode for testing
-        run_standalone(args.directory, args.query)
+        run_standalone(args.directory, args.query, parser_backend)
     } else {
         // LSP server mode
         tracing::info!("Starting pylight LSP server");
-        let server = LspServer::new()?;
+        let server = LspServer::new(parser_backend)?;
         server.run()
     }
 }
 
-fn run_standalone(directory: Option<std::path::PathBuf>, query: Option<String>) -> Result<()> {
+fn run_standalone(
+    directory: Option<std::path::PathBuf>,
+    query: Option<String>,
+    parser_backend: ParserBackend,
+) -> Result<()> {
     use pylight::{SearchEngine, SymbolIndex};
     use std::sync::Arc;
 
@@ -73,7 +89,7 @@ fn run_standalone(directory: Option<std::path::PathBuf>, query: Option<String>)
     tracing::info!("Indexing directory: {}", dir.display());
 
     // Create index and use the parallel index_workspace method
-    let index = Arc::new(SymbolIndex::new());
+    let index = Arc::new(SymbolIndex::new(parser_backend));
     index.clone().index_workspace(&dir)?;
 
     if let Some(query) = query {

pylight/src/bin/pylight_devtools.rs

@@ -1,3 +1,4 @@
+use pylight::parser::{create_parser, ParserBackend};
 use serde::{Deserialize, Serialize};
 use serde_json::{json, Value};
 use std::io::{BufRead, BufReader, Read, Write};
@@ -19,6 +20,12 @@ struct PylightInstance {
 #[derive(Serialize, Deserialize)]
 struct IndexRequest {
     path: String,
+    #[serde(default = "default_parser")]
+    parser: String,
+}
+
+fn default_parser() -> String {
+    "tree-sitter".to_string()
 }
 
 #[derive(Serialize, Deserialize)]
@@ -88,8 +95,8 @@ fn main() {
                             .respond(
                                 response.with_header(
                                     tiny_http::Header::from_bytes(
-                                        &b"Content-Type"[..],
-                                        &b"application/json"[..],
+                                        b"Content-Type",
+                                        b"application/json",
                                     )
                                     .unwrap(),
                                 ),
@@ -99,9 +106,12 @@ fn main() {
                     }
                 };
 
-                info!("Indexing codebase at: {}", index_req.path);
+                info!(
+                    "Indexing codebase at: {} with parser: {}",
+                    index_req.path, index_req.parser
+                );
 
-                let result = spawn_pylight(&index_req.path, pylight.clone());
+                let result = spawn_pylight(&index_req.path, &index_req.parser, pylight.clone());
                 let response = if result.is_ok() {
                     info!("Successfully spawned pylight for {}", index_req.path);
                     Response::from_string(json!({"status": "success"}).to_string())
@@ -180,6 +190,56 @@ fn main() {
                     )
                     .unwrap();
             }
+            ("POST", "/compare-parsers") => {
+                let mut content = String::new();
+                request.as_reader().read_to_string(&mut content).unwrap();
+
+                let req: serde_json::Value = match serde_json::from_str(&content) {
+                    Ok(req) => req,
+                    Err(e) => {
+                        error!("Failed to parse compare request: {}", e);
+                        let response = Response::from_string(
+                            json!({"status": "error", "message": format!("Invalid request: {e}")})
+                                .to_string(),
+                        )
+                        .with_status_code(400);
+                        request
+                            .respond(
+                                response.with_header(
+                                    tiny_http::Header::from_bytes(
+                                        b"Content-Type",
+                                        b"application/json",
+                                    )
+                                    .unwrap(),
+                                ),
+                            )
+                            .unwrap();
+                        continue;
+                    }
+                };
+
+                let test_code = req.get("code").and_then(|v| v.as_str()).unwrap_or(
+                    r#"
+def test_function():
+    pass
+
+class TestClass:
+    def method(self):
+        pass
+"#,
+                );
+
+                let response = compare_parsers(test_code);
+                request
+                    .respond(
+                        Response::from_string(serde_json::to_string(&response).unwrap())
+                            .with_header(
+                                tiny_http::Header::from_bytes(b"Content-Type", b"application/json")
+                                    .unwrap(),
+                            ),
+                    )
+                    .unwrap();
+            }
             _ => {
                 warn!("404 Not Found: {} {}", method, url);
                 request
@@ -190,8 +250,11 @@ fn main() {
     }
 }
 
-fn spawn_pylight(workspace_path: &str, pylight: SharedPylight) -> Result<(), String> {
-    info!("spawn_pylight called with path: {}", workspace_path);
+fn spawn_pylight(workspace_path: &str, parser: &str, pylight: SharedPylight) -> Result<(), String> {
+    info!(
+        "spawn_pylight called with path: {} and parser: {}",
+        workspace_path, parser
+    );
 
     // Kill existing instance if any
     {
@@ -202,14 +265,22 @@ fn spawn_pylight(workspace_path: &str, pylight: SharedPylight) -> Result<(), Str
         }
     }
 
-    info!("Spawning new pylight instance");
+    info!("Spawning new pylight instance with {} parser", parser);
 
-    // Spawn new pylight instance
+    // Spawn new pylight instance with parser argument
     let mut cmd = Command::new("cargo");
-    cmd.args(["run", "--release", "--bin", "pylight"])
-        .stdin(Stdio::piped())
-        .stdout(Stdio::piped())
-        .stderr(Stdio::piped());
+    cmd.args([
+        "run",
+        "--release",
+        "--bin",
+        "pylight",
+        "--",
+        "--parser",
+        parser,
+    ])
+    .stdin(Stdio::piped())
+    .stdout(Stdio::piped())
+    .stderr(Stdio::piped());
 
     let mut child = cmd
         .spawn()
@@ -441,3 +512,78 @@ fn read_lsp_message(instance: &mut PylightInstance) -> Result<Value, String> {
 
     Ok(response)
 }
+
+fn compare_parsers(code: &str) -> Value {
+    use std::path::Path;
+
+    let test_path = Path::new("test.py");
+
+    // Parse with tree-sitter
+    let ts_start = Instant::now();
+    let ts_parser = create_parser(ParserBackend::TreeSitter).unwrap();
+    let ts_symbols = ts_parser.parse_file(test_path, code).unwrap_or_default();
+    let ts_duration = ts_start.elapsed();
+
+    // Parse with ruff
+    let ruff_start = Instant::now();
+    let ruff_parser = create_parser(ParserBackend::Ruff).unwrap();
+    let ruff_symbols = ruff_parser.parse_file(test_path, code).unwrap_or_default();
+    let ruff_duration = ruff_start.elapsed();
+
+    // Compare results
+    let ts_symbol_info: Vec<Value> = ts_symbols
+        .iter()
+        .map(|s| {
+            json!({
+                "name": s.name,
+                "kind": format!("{:?}", s.kind),
+                "line": s.line,
+                "column": s.column,
+                "container": s.container_name.as_ref(),
+            })
+        })
+        .collect();
+
+    let ruff_symbol_info: Vec<Value> = ruff_symbols
+        .iter()
+        .map(|s| {
+            json!({
+                "name": s.name,
+                "kind": format!("{:?}", s.kind),
+                "line": s.line,
+                "column": s.column,
+                "container": s.container_name.as_ref(),
+            })
+        })
+        .collect();
+
+    let same_count = ts_symbols.len() == ruff_symbols.len();
+    let same_symbols = ts_symbols
+        .iter()
+        .zip(ruff_symbols.iter())
+        .all(|(ts, ruff)| {
+            ts.name == ruff.name && ts.kind == ruff.kind && ts.container_name == ruff.container_name
+        });
+
+    json!({
+        "tree_sitter": {
+            "symbols": ts_symbol_info,
+            "count": ts_symbols.len(),
+            "duration_ms": ts_duration.as_secs_f64() * 1000.0,
+        },
+        "ruff": {
+            "symbols": ruff_symbol_info,
+            "count": ruff_symbols.len(),
+            "duration_ms": ruff_duration.as_secs_f64() * 1000.0,
+        },
+        "comparison": {
+            "same_count": same_count,
+            "same_symbols": same_symbols,
+            "differences": if !same_symbols {
+                Some("Symbols differ in name, kind, or container")
+            } else {
+                None
+            }
+        }
+    })
+}

pylight/src/index/symbol_index.rs

@@ -1,6 +1,7 @@
 //! Symbol index implementation
 
-use crate::{PythonParser, Result, Symbol};
+use crate::parser::{create_parser, ParserBackend};
+use crate::{Result, Symbol};
 use parking_lot::RwLock;
 use rayon::prelude::*;
 use std::collections::HashMap;
@@ -13,6 +14,7 @@ pub struct SymbolIndex {
     symbols: Arc<RwLock<HashMap<PathBuf, Vec<Arc<Symbol>>>>>,
     all_symbols: Arc<RwLock<Vec<Arc<Symbol>>>>,
     file_metadata: Arc<RwLock<HashMap<PathBuf, FileMetadata>>>,
+    parser_backend: ParserBackend,
 }
 
 #[derive(Debug, Clone)]
@@ -22,14 +24,20 @@ pub struct FileMetadata {
 }
 
 impl SymbolIndex {
-    pub fn new() -> Self {
+    pub fn new(parser_backend: ParserBackend) -> Self {
         Self {
             symbols: Arc::new(RwLock::new(HashMap::new())),
             all_symbols: Arc::new(RwLock::new(Vec::new())),
             file_metadata: Arc::new(RwLock::new(HashMap::new())),
+            parser_backend,
         }
     }
 
+    /// Get the parser backend used by this index
+    pub fn parser_backend(&self) -> ParserBackend {
+        self.parser_backend
+    }
+
     pub fn add_file(&self, path: PathBuf, symbols: Vec<Symbol>) -> Result<()> {
         // Canonicalize the path for consistent comparison
         let canonical_path = path.canonicalize().unwrap_or(path.clone());
@@ -238,8 +246,8 @@ impl SymbolIndex {
                     thread_id
                 );
 
-                // Each thread gets its own parser
-                let mut parser = match PythonParser::new() {
+                // Create parser instance for this thread
+                let parser = match create_parser(self.parser_backend) {
                     Ok(p) => p,
                     Err(e) => {
                         tracing::warn!("Failed to create parser: {}", e);
@@ -335,7 +343,7 @@ impl SymbolIndex {
 
 impl Default for SymbolIndex {
     fn default() -> Self {
-        Self::new()
+        Self::new(ParserBackend::TreeSitter)
     }
 }
 
@@ -345,7 +353,7 @@ mod tests {
 
     #[test]
     fn test_index_creation() {
-        let index = SymbolIndex::new();
+        let index = SymbolIndex::new(ParserBackend::TreeSitter);
         assert_eq!(index.get_all_symbols().len(), 0);
     }
 }