Type-checked data tables

backend/parsers/windmill-parser-py/src/asset_parser.rs

@@ -3,11 +3,11 @@ use rustpython_parser::{ast::Suite, Parse};
 use std::collections::HashMap;
 use windmill_parser::asset_parser::{
     asset_was_used, merge_assets, parse_asset_syntax, AssetKind, AssetUsageAccessType,
-    ParseAssetsResult,
+    ParseAssetsOutput, ParseAssetsResult,
 };
 use AssetUsageAccessType::*;
 
-pub fn parse_assets(input: &str) -> anyhow::Result<Vec<ParseAssetsResult>> {
+pub fn parse_assets(input: &str) -> anyhow::Result<ParseAssetsOutput> {
     let ast = Suite::parse(input, "main.py")
         .map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))?;
 
@@ -25,7 +25,7 @@ pub fn parse_assets(input: &str) -> anyhow::Result<Vec<ParseAssetsResult>> {
         }
     }
 
-    Ok(merge_assets(assets_finder.assets))
+    Ok(ParseAssetsOutput { assets: merge_assets(assets_finder.assets), ..Default::default() })
 }
 
 type VarAssetName = String;
@@ -216,7 +216,7 @@ impl AssetsFinder {
                 match windmill_parser_sql::parse_assets(&sql) {
                     Ok(mut sql_assets) => {
                         if let Some(schema_name) = schema {
-                            for asset in &mut sql_assets {
+                            for asset in &mut sql_assets.assets {
                                 if asset.kind == *kind && asset.path.starts_with(path.as_str()) {
                                     asset.path = format!(
                                         "{}/{}.{}",
@@ -227,7 +227,7 @@ impl AssetsFinder {
                                 }
                             }
                         }
-                        self.assets.extend(sql_assets);
+                        self.assets.extend(sql_assets.assets);
                     }
                     _ => {}
                 }
@@ -290,7 +290,7 @@ import wmill
 def main():
     wmill.load_s3_file('s3:///test.csv')
 "#;
-        let s = parse_assets(input);
+        let s = parse_assets(input).map(|o| o.assets);
         assert_eq!(
             s.map_err(|e| e.to_string()),
             Ok(vec![ParseAssetsResult {
@@ -308,7 +308,7 @@ import wmill
 def main():
     db = wmill.datatable()
 "#;
-        let s = parse_assets(input);
+        let s = parse_assets(input).map(|o| o.assets);
         assert_eq!(
             s.map_err(|e| e.to_string()),
             Ok(vec![ParseAssetsResult {
@@ -327,7 +327,7 @@ def main(x: int):
     db = wmill.datatable('dt')
     return db.query('SELECT * FROM friends WHERE age = $1', x).fetch()
 "#;
-        let s = parse_assets(input);
+        let s = parse_assets(input).map(|o| o.assets);
         assert_eq!(
             s.map_err(|e| e.to_string()),
             Ok(vec![ParseAssetsResult {
@@ -348,7 +348,7 @@ def main(x: int):
     db.query('SELECT * FROM friends WHERE age = $1', x).fetch_one()
     db.query('SELECT * FROM analytics').fetch()
 "#;
-        let s = parse_assets(input);
+        let s = parse_assets(input).map(|o| o.assets);
         assert_eq!(
             s.map_err(|e| e.to_string()),
             Ok(vec![
@@ -380,7 +380,7 @@ def main():
 def g():
     db = wmill.ducklake('another2')
 "#;
-        let s = parse_assets(input);
+        let s = parse_assets(input).map(|o| o.assets);
         assert_eq!(
             s.map_err(|e| e.to_string()),
             Ok(vec![
@@ -412,7 +412,7 @@ def main():
 def g():
     db = wmill.ducklake()
 "#;
-        let s = parse_assets(input);
+        let s = parse_assets(input).map(|o| o.assets);
         assert_eq!(
             s.map_err(|e| e.to_string()),
             Ok(vec![
@@ -438,7 +438,7 @@ def main(x: int):
     db = wmill.datatable('dt:public')
     return db.query('SELECT * FROM friends WHERE age = $1', x).fetch()
 "#;
-        let s = parse_assets(input);
+        let s = parse_assets(input).map(|o| o.assets);
         assert_eq!(
             s.map_err(|e| e.to_string()),
             Ok(vec![ParseAssetsResult {
@@ -457,7 +457,7 @@ def main():
     db = wmill.ducklake('lake1:analytics')
     return db.query('SELECT * FROM metrics').fetch()
 "#;
-        let s = parse_assets(input);
+        let s = parse_assets(input).map(|o| o.assets);
         assert_eq!(
             s.map_err(|e| e.to_string()),
             Ok(vec![ParseAssetsResult {
@@ -477,7 +477,7 @@ def main(x: int):
     db.query('INSERT INTO users VALUES ($1)', x).fetch()
     return db.query('SELECT * FROM users').fetch()
 "#;
-        let s = parse_assets(input);
+        let s = parse_assets(input).map(|o| o.assets);
         assert_eq!(
             s.map_err(|e| e.to_string()),
             Ok(vec![ParseAssetsResult {
@@ -495,7 +495,7 @@ import wmill
 def main():
     db = wmill.datatable('dt:public')
 "#;
-        let s = parse_assets(input);
+        let s = parse_assets(input).map(|o| o.assets);
         assert_eq!(
             s.map_err(|e| e.to_string()),
             Ok(vec![ParseAssetsResult {

backend/parsers/windmill-parser-sql/src/asset_parser.rs

@@ -10,11 +10,11 @@ use sqlparser::{
 };
 use windmill_parser::asset_parser::{
     asset_was_used, merge_assets, parse_asset_syntax, AssetKind, AssetUsageAccessType,
-    ParseAssetsResult,
+    ParseAssetsOutput, ParseAssetsResult,
 };
 use AssetUsageAccessType::*;
 
-pub fn parse_assets(input: &str) -> anyhow::Result<Vec<ParseAssetsResult>> {
+pub fn parse_assets(input: &str) -> anyhow::Result<ParseAssetsOutput> {
     let statements = Parser::parse_sql(&DuckDbDialect, input)?;
 
     let mut collector = AssetCollector::new();
@@ -30,7 +30,7 @@ pub fn parse_assets(input: &str) -> anyhow::Result<Vec<ParseAssetsResult>> {
         }
     }
 
-    Ok(merge_assets(collector.assets))
+    Ok(ParseAssetsOutput { assets: merge_assets(collector.assets), ..Default::default() })
 }
 
 /// Visitor that collects S3 asset literals from SQL statements
@@ -402,7 +402,7 @@ mod tests {
             SELECT * FROM read_parquet('s3:///a.parquet');
             COPY (SELECT * FROM 's3://snd/b.parquet') TO 's3:///c.parquet';
         "#;
-        let s = parse_assets(input);
+        let s = parse_assets(input).map(|s| s.assets);
         assert_eq!(
             s.map_err(|e| e.to_string()),
             Ok(vec![
@@ -432,7 +432,7 @@ mod tests {
             SELECT 2;
             USE dl;
         "#;
-        let s = parse_assets(input);
+        let s = parse_assets(input).map(|s| s.assets);
         assert_eq!(
             s.map_err(|e| e.to_string()),
             Ok(vec![ParseAssetsResult {
@@ -449,7 +449,7 @@ mod tests {
             ATTACH 'ducklake://my_dl' AS dl;
             SELECT * FROM dl.table1;
         "#;
-        let s = parse_assets(input);
+        let s = parse_assets(input).map(|s| s.assets);
         assert_eq!(
             s.map_err(|e| e.to_string()),
             Ok(vec![ParseAssetsResult {
@@ -467,7 +467,7 @@ mod tests {
             SELECT dt.read_bait FROM unrelated_table; -- dt. doesn't access the asset
             INSERT INTO dt.table1 VALUES ('test');
         "#;
-        let s = parse_assets(input);
+        let s = parse_assets(input).map(|s| s.assets);
         assert_eq!(
             s.map_err(|e| e.to_string()),
             Ok(vec![ParseAssetsResult {
@@ -485,7 +485,7 @@ mod tests {
             DETACH dl;
             SELECT * FROM dl.table1;
         "#;
-        let s = parse_assets(input);
+        let s = parse_assets(input).map(|s| s.assets);
         assert_eq!(s.map_err(|e| e.to_string()), Ok(vec![]));
     }
 
@@ -498,7 +498,7 @@ mod tests {
             USE memory;
             SELECT * FROM table1;
         "#;
-        let s = parse_assets(input);
+        let s = parse_assets(input).map(|s| s.assets);
         assert_eq!(
             s.map_err(|e| e.to_string()),
             Ok(vec![ParseAssetsResult {
@@ -515,7 +515,7 @@ mod tests {
             ATTACH 'datatable' AS dl;
             INSERT INTO dl.table1 VALUES ('test');
         "#;
-        let s = parse_assets(input);
+        let s = parse_assets(input).map(|s| s.assets);
         assert_eq!(
             s.map_err(|e| e.to_string()),
             Ok(vec![ParseAssetsResult {
@@ -537,7 +537,7 @@ mod tests {
             INSERT INTO friends VALUES ($name, $age);
             SELECT * FROM friends;
         "#;
-        let s = parse_assets(input);
+        let s = parse_assets(input).map(|s| s.assets);
         assert_eq!(
             s.map_err(|e| e.to_string()),
             Ok(vec![ParseAssetsResult {
@@ -555,7 +555,7 @@ mod tests {
             ATTACH 'ducklake' AS dl; USE dl;
             SELECT * FROM a_function('');
         "#;
-        let s = parse_assets(input);
+        let s = parse_assets(input).map(|s| s.assets);
         assert_eq!(
             s.map_err(|e| e.to_string()),
             Ok(vec![ParseAssetsResult {
@@ -573,7 +573,7 @@ mod tests {
             USE dl;
             DELETE FROM table1;
         "#;
-        let s = parse_assets(input);
+        let s = parse_assets(input).map(|s| s.assets);
         assert_eq!(
             s.map_err(|e| e.to_string()),
             Ok(vec![ParseAssetsResult {
@@ -591,7 +591,7 @@ mod tests {
             USE dl;
             UPDATE table1 SET id = NULL;
         "#;
-        let s = parse_assets(input);
+        let s = parse_assets(input).map(|s| s.assets);
         assert_eq!(
             s.map_err(|e| e.to_string()),
             Ok(vec![ParseAssetsResult {
@@ -609,7 +609,7 @@ mod tests {
             USE db;
             SELECT * FROM table1;
         "#;
-        let s = parse_assets(input);
+        let s = parse_assets(input).map(|s| s.assets);
         assert_eq!(
             s.map_err(|e| e.to_string()),
             Ok(vec![ParseAssetsResult {
@@ -626,7 +626,7 @@ mod tests {
             ATTACH 'ducklake' AS dl;
             UPDATE dl.table1 SET id = NULL;
         "#;
-        let s = parse_assets(input);
+        let s = parse_assets(input).map(|s| s.assets);
         assert_eq!(
             s.map_err(|e| e.to_string()),
             Ok(vec![ParseAssetsResult {
@@ -644,7 +644,7 @@ mod tests {
             UPDATE dl.sch.table1 SET id = NULL;
             SELECT * FROM dl.sch.table1;
         "#;
-        let s = parse_assets(input);
+        let s = parse_assets(input).map(|s| s.assets);
         assert_eq!(
             s.map_err(|e| e.to_string()),
             Ok(vec![ParseAssetsResult {
@@ -663,7 +663,7 @@ mod tests {
             UPDATE sch.table1 SET id = NULL;
             SELECT * FROM sch.table1;
         "#;
-        let s = parse_assets(input);
+        let s = parse_assets(input).map(|s| s.assets);
         assert_eq!(
             s.map_err(|e| e.to_string()),
             Ok(vec![ParseAssetsResult {