Merge branch 'main' into di/new-duckdb-asset-parser

This reverts commit 88068b1a77.

backend/Cargo.lock

@@ -2822,7 +2822,7 @@ dependencies = [
  "parquet",
  "rand 0.8.5",
  "regex",
- "sqlparser",
+ "sqlparser 0.55.0",
  "tempfile",
  "tokio",
  "url",
@@ -2899,7 +2899,7 @@ dependencies = [
  "parquet",
  "paste",
  "recursive",
- "sqlparser",
+ "sqlparser 0.55.0",
  "tokio",
  "web-time",
 ]
@@ -3075,7 +3075,7 @@ dependencies = [
  "paste",
  "recursive",
  "serde_json",
- "sqlparser",
+ "sqlparser 0.55.0",
 ]
 
 [[package]]
@@ -3371,7 +3371,7 @@ dependencies = [
  "log",
  "recursive",
  "regex",
- "sqlparser",
+ "sqlparser 0.55.0",
 ]
 
 [[package]]
@@ -8602,15 +8602,6 @@ dependencies = [
  "minimal-lexical",
 ]
 
-[[package]]
-name = "nom"
-version = "8.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405"
-dependencies = [
- "memchr",
-]
-
 [[package]]
 name = "notify"
 version = "6.1.1"
@@ -12264,6 +12255,17 @@ dependencies = [
  "sqlparser_derive",
 ]
 
+[[package]]
+name = "sqlparser"
+version = "0.59.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4591acadbcf52f0af60eafbb2c003232b2b4cd8de5f0e9437cb8b1b59046cc0f"
+dependencies = [
+ "log",
+ "recursive",
+ "sqlparser_derive",
+]
+
 [[package]]
 name = "sqlparser_derive"
 version = "0.3.0"
@@ -15184,7 +15186,6 @@ dependencies = [
  "lazy_static",
  "libloading 0.8.9",
  "memchr",
- "nom 8.0.0",
  "object_store",
  "once_cell",
  "pep440_rs",
@@ -15705,11 +15706,11 @@ version = "1.590.0"
 dependencies = [
  "anyhow",
  "lazy_static",
- "nom 8.0.0",
  "regex",
  "regex-lite",
  "serde",
  "serde_json",
+ "sqlparser 0.59.0",
  "windmill-parser",
 ]
 

backend/Cargo.toml

@@ -158,7 +158,6 @@ kube.workspace = true
 k8s-openapi.workspace = true
 libloading.workspace = true
 bitflags.workspace = true
-nom.workspace = true
 globset.workspace = true
 
 
@@ -378,7 +377,6 @@ pg_escape = "0.1.1"
 async-nats = "0.38.0"
 nkeys = "0.4.4"
 nu-parser = { version = "0.101.0", default-features = false }
-nom = "8.0.0"
 globset = "0.4.16"
 
 process-wrap = { version = "8.2.1", features = ["tokio1"] }

backend/parsers/windmill-parser-py/src/asset_parser.rs

@@ -87,7 +87,7 @@ impl Visitor for AssetsFinder {
     fn visit_expr_constant(&mut self, node: ExprConstant) {
         match node.value {
             Constant::Str(s) => {
-                if let Some((kind, path)) = parse_asset_syntax(&s) {
+                if let Some((kind, path)) = parse_asset_syntax(&s, false) {
                     self.assets.push(ParseAssetsResult {
                         kind,
                         path: path.to_string(),
@@ -108,7 +108,7 @@ impl Visitor for AssetsFinder {
                     if let Expr::Constant(ExprConstant { value: Constant::Str(s), .. }) =
                         &keyword.value
                     {
-                        if let Some((kind, path)) = parse_asset_syntax(s) {
+                        if let Some((kind, path)) = parse_asset_syntax(s, false) {
                             self.assets.push(ParseAssetsResult {
                                 kind,
                                 path: path.to_string(),
@@ -249,7 +249,9 @@ impl AssetsFinder {
 
         match arg_val {
             Some(Expr::Constant(ExprConstant { value: Constant::Str(value), .. })) => {
-                let path = parse_asset_syntax(&value).map(|(_, p)| p).unwrap_or(&value);
+                let path = parse_asset_syntax(&value, false)
+                    .map(|(_, p)| p)
+                    .unwrap_or(&value);
                 self.assets
                     .push(ParseAssetsResult { kind, path: path.to_string(), access_type });
             }

backend/parsers/windmill-parser-sql/Cargo.toml

@@ -20,4 +20,4 @@ anyhow.workspace = true
 lazy_static.workspace = true
 serde_json.workspace = true
 serde.workspace = true
-nom.workspace = true
+sqlparser = { version = "0.59.0", features = ["visitor"] }

backend/parsers/windmill-parser-sql/src/asset_parser.rs

@@ -1,151 +1,456 @@
+use std::collections::HashMap;
+
+use sqlparser::{
+    ast::{CopyTarget, Expr, ObjectName, TableFactor, Value, ValueWithSpan, Visit, Visitor},
+    dialect::DuckDbDialect,
+    parser::Parser,
+};
 use windmill_parser::asset_parser::{
-    merge_assets, AssetKind, AssetUsageAccessType, ParseAssetsResult,
+    merge_assets, parse_asset_syntax, AssetKind, AssetUsageAccessType, ParseAssetsResult,
 };
 use AssetUsageAccessType::*;
 
-use nom::{
-    branch::alt,
-    bytes::complete::{tag, tag_no_case, take_while},
-    character::complete::{char, multispace0},
-    combinator::opt,
-    sequence::preceded,
-    IResult, Parser,
-};
+pub fn parse_assets(input: &str) -> anyhow::Result<Vec<ParseAssetsResult<String>>> {
+    let statements = Parser::parse_sql(&DuckDbDialect, input)?;
 
-pub fn parse_assets<'a>(input: &str) -> anyhow::Result<Vec<ParseAssetsResult<&str>>> {
-    let mut assets = Vec::new();
-    let mut remaining = input;
+    let mut collector = AssetCollector::new();
+    for statement in statements {
+        let _ = statement.visit(&mut collector);
+    }
 
-    while !remaining.trim().is_empty() {
-        if let Ok((rest, _)) = parse_comment(remaining) {
-            remaining = rest; // skip comment
-        }
-        if let Ok((rest, res)) = parse_asset(remaining) {
-            assets.push(res);
-            remaining = rest;
-        } else {
-            remaining = &remaining[1..]; // skip 1 char and continue
+    for (_, (kind, path)) in collector.var_identifiers {
+        if collector
+            .assets
+            .iter()
+            .all(|a| a.path.as_ref() != path || a.kind != kind)
+        {
+            collector
+                .assets
+                .push(ParseAssetsResult { kind: kind, access_type: None, path: path });
         }
     }
 
-    Ok(merge_assets(assets))
+    Ok(merge_assets(collector.assets))
 }
 
-fn parse_asset(input: &str) -> IResult<&str, ParseAssetsResult<&str>> {
-    alt((
-        parse_s3_object_read.map(|path| ParseAssetsResult {
-            path,
-            kind: AssetKind::S3Object,
-            access_type: Some(R),
-        }),
-        parse_s3_object_write.map(|path| ParseAssetsResult {
-            path,
-            kind: AssetKind::S3Object,
-            access_type: Some(W),
-        }),
-        // Parse ambiguous access_types at the end if we could not find precisely read or copy
-        parse_s3_object_lit.map(|path| ParseAssetsResult {
-            path,
-            kind: AssetKind::S3Object,
-            access_type: None,
-        }),
-        parse_resource_lit.map(|path| ParseAssetsResult {
-            path,
-            kind: AssetKind::Resource,
-            access_type: None,
-        }),
-        parse_ducklake_lit.map(|path| ParseAssetsResult {
-            path,
-            kind: AssetKind::Ducklake,
-            access_type: None,
-        }),
-        parse_datatable_lit.map(|path| ParseAssetsResult {
-            path,
-            kind: AssetKind::DataTable,
-            access_type: None,
-        }),
-    ))
-    .parse(input)
+/// Visitor that collects S3 asset literals from SQL statements
+struct AssetCollector {
+    assets: Vec<ParseAssetsResult<String>>,
+    // e.g set to Read when we are inside a SELECT ... FROM ... statement
+    current_access_type_stack: Vec<AssetUsageAccessType>,
+    // e.g ATTACH 'ducklake://a' AS dl; => { "dl": (Ducklake, "a") }
+    var_identifiers: HashMap<String, (AssetKind, String)>,
+    // e.g USE dl;
+    currently_used_asset: Option<(AssetKind, String)>,
 }
 
-/// Any expression that reads an s3 asset
-fn parse_s3_object_read(input: &str) -> IResult<&str, &str> {
-    alt((parse_s3_object_read_fn, parse_s3_object_select_from)).parse(input)
+impl AssetCollector {
+    fn new() -> Self {
+        Self {
+            assets: Vec::new(),
+            current_access_type_stack: Vec::with_capacity(8),
+            var_identifiers: HashMap::new(),
+            currently_used_asset: None,
+        }
+    }
+
+    // Detect when we do 'a.b' and 'a' is associated with an asset in var_identifiers
+    // Or when we access 'b' and we did USE a;
+    fn get_associated_asset_from_obj_name(
+        &self,
+        name: &ObjectName,
+    ) -> Option<ParseAssetsResult<String>> {
+        let access_type = self.current_access_type_stack.last().copied();
+        if name.0.len() == 1 {
+            if name.0.first()?.as_ident()?.quote_style.is_some() {
+                return None;
+            }
+            // We don't want to infer that any simple identifier refers to an asset if
+            // we are not in a known R/W context
+            if access_type.is_none() {
+                return None;
+            }
+            if let Some((kind, path)) = &self.currently_used_asset {
+                return Some(ParseAssetsResult { kind: *kind, access_type, path: path.clone() });
+            }
+        }
+
+        // Check if the first part of the name (the a in a.b) is associated with an asset
+        if name.0.len() < 2 {
+            return None;
+        }
+        let ident = name.0.first()?.as_ident()?;
+        let (kind, path) = self.var_identifiers.get(&ident.value)?;
+        Some(ParseAssetsResult { kind: *kind, access_type, path: path.clone() })
+    }
+
+    fn handle_string_literal(&mut self, s: &str) {
+        // Check if the string matches our asset syntax patterns
+        if let Some((kind, path)) = parse_asset_syntax(s, false) {
+            if kind == AssetKind::S3Object {
+                self.assets.push(ParseAssetsResult {
+                    kind,
+                    path: path.to_string(),
+                    access_type: self.current_access_type_stack.last().copied(),
+                });
+            }
+        }
+    }
+
+    fn handle_obj_name_pre(&mut self, name: &ObjectName) {
+        if let Some(fname) = get_trivial_obj_name(name) {
+            if is_read_fn(fname) {
+                self.current_access_type_stack.push(R);
+            }
+        }
+        if let Some(str_lit) = get_str_lit_from_obj_name(name) {
+            self.handle_string_literal(str_lit);
+        }
+        if let Some(asset) = self.get_associated_asset_from_obj_name(name) {
+            self.assets.push(asset);
+        }
+    }
+
+    fn handle_obj_name_post(&mut self, name: &ObjectName) {
+        if self.current_access_type_stack.is_empty() {
+            return;
+        }
+        if let Some(fname) = get_trivial_obj_name(name) {
+            if is_read_fn(fname) {
+                self.current_access_type_stack.pop();
+            }
+        }
+    }
 }
 
-/// Any expression that writes to an s3 asset
-fn parse_s3_object_write(input: &str) -> IResult<&str, &str> {
-    // COPY (...) TO 's3://...'
-    let (input, _) = (tag_no_case("TO"), multispace0).parse(input)?;
-    let (input, path) = parse_s3_object_lit(input)?;
-    Ok((input, path))
+impl Visitor for AssetCollector {
+    type Break = ();
+
+    fn pre_visit_table_factor(
+        &mut self,
+        table_factor: &TableFactor,
+    ) -> std::ops::ControlFlow<Self::Break> {
+        match table_factor {
+            TableFactor::Table { name, .. } => self.handle_obj_name_pre(name),
+            _ => {}
+        }
+        std::ops::ControlFlow::Continue(())
+    }
+
+    fn post_visit_table_factor(
+        &mut self,
+        table_factor: &TableFactor,
+    ) -> std::ops::ControlFlow<Self::Break> {
+        match table_factor {
+            TableFactor::Table { name, .. } => self.handle_obj_name_post(name),
+            _ => {}
+        }
+        std::ops::ControlFlow::Continue(())
+    }
+
+    fn pre_visit_expr(&mut self, expr: &Expr) -> std::ops::ControlFlow<Self::Break> {
+        match expr {
+            Expr::Value(ValueWithSpan { value: Value::SingleQuotedString(s), .. }) => {
+                self.handle_string_literal(s)
+            }
+            Expr::Value(ValueWithSpan { value: Value::DoubleQuotedString(s), .. }) => {
+                self.handle_string_literal(s);
+            }
+            Expr::Function(func) => self.handle_obj_name_pre(&func.name),
+            _ => {}
+        }
+        std::ops::ControlFlow::Continue(())
+    }
+
+    fn post_visit_expr(&mut self, expr: &Expr) -> std::ops::ControlFlow<Self::Break> {
+        match expr {
+            Expr::Function(func) => self.handle_obj_name_post(&func.name),
+            _ => {}
+        }
+        std::ops::ControlFlow::Continue(())
+    }
+
+    fn pre_visit_statement(
+        &mut self,
+        statement: &sqlparser::ast::Statement,
+    ) -> std::ops::ControlFlow<Self::Break> {
+        if let Some(access_type) = get_stmt_access_type(statement) {
+            self.current_access_type_stack.push(access_type);
+        }
+
+        match statement {
+            sqlparser::ast::Statement::Copy { target: CopyTarget::File { filename }, .. } => {
+                self.current_access_type_stack.push(W);
+                self.handle_string_literal(filename);
+                self.current_access_type_stack.pop();
+            }
+            sqlparser::ast::Statement::AttachDuckDBDatabase {
+                database_path,
+                database_alias,
+                ..
+            } => {
+                if let Some((kind, path)) = parse_asset_syntax(&database_path.value, true) {
+                    if kind == AssetKind::Ducklake
+                        || kind == AssetKind::DataTable
+                        || kind == AssetKind::Resource
+                    {
+                        if let Some(database_alias) = database_alias {
+                            self.var_identifiers
+                                .insert(database_alias.value.clone(), (kind, path.to_string()));
+                        }
+                    }
+                }
+            }
+            sqlparser::ast::Statement::DetachDuckDBDatabase { database_alias, .. } => {
+                let asset = self.var_identifiers.remove(&database_alias.value);
+                if self.currently_used_asset == asset {
+                    self.currently_used_asset = None;
+                }
+            }
+            sqlparser::ast::Statement::Use(sqlparser::ast::Use::Object(obj_name)) => {
+                if let Some((kind, path)) = self.var_identifiers.get(&obj_name.to_string()) {
+                    self.currently_used_asset = Some((*kind, path.clone()));
+                } else {
+                    self.currently_used_asset = None;
+                }
+            }
+            _ => {}
+        }
+
+        std::ops::ControlFlow::Continue(())
+    }
+
+    fn post_visit_statement(
+        &mut self,
+        statement: &sqlparser::ast::Statement,
+    ) -> std::ops::ControlFlow<Self::Break> {
+        if let Some(_access_type) = get_stmt_access_type(statement) {
+            self.current_access_type_stack.pop();
+        }
+        std::ops::ControlFlow::Continue(())
+    }
+
+    fn pre_visit_query(
+        &mut self,
+        _query: &sqlparser::ast::Query,
+    ) -> std::ops::ControlFlow<Self::Break> {
+        self.current_access_type_stack.push(R);
+        std::ops::ControlFlow::Continue(())
+    }
+
+    fn post_visit_query(
+        &mut self,
+        _query: &sqlparser::ast::Query,
+    ) -> std::ops::ControlFlow<Self::Break> {
+        self.current_access_type_stack.pop();
+        std::ops::ControlFlow::Continue(())
+    }
+
+    fn pre_visit_relation(&mut self, relation: &ObjectName) -> std::ops::ControlFlow<Self::Break> {
+        self.handle_obj_name_pre(relation);
+        std::ops::ControlFlow::Continue(())
+    }
+
+    fn post_visit_relation(&mut self, relation: &ObjectName) -> std::ops::ControlFlow<Self::Break> {
+        self.handle_obj_name_post(relation);
+        std::ops::ControlFlow::Continue(())
+    }
 }
 
-/// read_parquet('s3://...')
-fn parse_s3_object_read_fn(input: &str) -> IResult<&str, &str> {
-    let (input, _) = alt((
-        tag_no_case("read_parquet"),
-        tag_no_case("read_csv"),
-        tag_no_case("read_json"),
-    ))
-    .parse(input)?;
-    let (input, _) = multispace0(input)?;
-    let (input, _) = char('(')(input)?;
-    let (input, _) = multispace0(input)?;
-    let (input, path) = parse_s3_object_lit(input)?;
-    let (input, _) = multispace0(input)?;
-    let (input, _) = char(')')(input)?;
-    Ok((input, path))
+fn is_read_fn(fname: &str) -> bool {
+    fname.eq_ignore_ascii_case("read_parquet")
+        || fname.eq_ignore_ascii_case("read_csv")
+        || fname.eq_ignore_ascii_case("read_json")
 }
 
-/// SELECT ... FROM 's3://...'
-fn parse_s3_object_select_from(input: &str) -> IResult<&str, &str> {
-    let (input, _) = tag_no_case("FROM").parse(input)?;
-    let (input, _) = multispace0(input)?;
-    let (input, path) = parse_s3_object_lit(input)?;
-    Ok((input, path))
-}
-/// 's3://...'
-fn parse_s3_object_lit(input: &str) -> IResult<&str, &str> {
-    let (input, _) = quote(input)?;
-    let (input, _) = tag("s3://").parse(input)?;
-    let (input, path) = take_while(|c| c != '\'' && c != '"')(input)?;
-    let (input, _) = quote(input)?;
-    Ok((input, path))
+fn get_stmt_access_type(statement: &sqlparser::ast::Statement) -> Option<AssetUsageAccessType> {
+    match statement {
+        sqlparser::ast::Statement::Query(_) => Some(R),
+        sqlparser::ast::Statement::Insert(insert) => {
+            Some(if insert.returning.is_some() { RW } else { W })
+        }
+        sqlparser::ast::Statement::Update { returning, .. } => {
+            Some(if returning.is_some() { RW } else { W })
+        }
+        sqlparser::ast::Statement::Delete(delete) => {
+            Some(if delete.returning.is_some() { RW } else { W })
+        }
+        sqlparser::ast::Statement::CreateTable { .. } => Some(W),
+        sqlparser::ast::Statement::CreateView { .. } => Some(W),
+        _ => None,
+    }
 }
 
-fn quote(input: &str) -> IResult<&str, char> {
-    alt((char('\''), char('\"'))).parse(input)
+fn get_trivial_obj_name(name: &sqlparser::ast::ObjectName) -> Option<&str> {
+    if name.0.len() != 1 {
+        return None;
+    }
+    Some(name.0.first()?.as_ident()?.value.as_str())
 }
 
-fn parse_resource_lit(input: &str) -> IResult<&str, &str> {
-    let (input, _) = quote(input)?;
-    let (input, _) = alt((tag("$res:"), tag("res://"))).parse(input)?;
-    let (input, path) = take_while(|c| c != '\'' && c != '"')(input)?;
-    let (input, _) = quote(input)?;
-    Ok((input, path))
+fn get_str_lit_from_obj_name(name: &ObjectName) -> Option<&str> {
+    if name.0.len() != 1 {
+        return None;
+    }
+    let ident = name.0.first()?.as_ident()?;
+    if ident.quote_style != Some('\'') {
+        return None;
+    }
+    Some(ident.value.as_str())
 }
 
-fn parse_ducklake_lit(input: &str) -> IResult<&str, &str> {
-    let (input, _) = quote(input)?;
-    let (input, _) = tag("ducklake").parse(input)?;
-    let (input, path) =
-        opt(preceded(tag("://"), take_while(|c| c != '\'' && c != '"'))).parse(input)?;
-    let (input, _) = quote(input)?;
-    Ok((input, path.unwrap_or("main")))
-}
+#[cfg(test)]
+mod tests {
+    use super::*;
+    #[test]
+    fn test_sql_asset_parser_s3_literals() {
+        let input = r#"
+            SELECT * FROM read_parquet('s3:///a.parquet');
+            COPY (SELECT * FROM 's3://snd/b.parquet') TO 's3:///c.parquet';
+        "#;
+        let s = parse_assets(input);
+        assert_eq!(
+            s.map_err(|e| e.to_string()),
+            Ok(vec![
+                ParseAssetsResult {
+                    kind: AssetKind::S3Object,
+                    path: "/a.parquet".to_string(),
+                    access_type: Some(R)
+                },
+                ParseAssetsResult {
+                    kind: AssetKind::S3Object,
+                    path: "/c.parquet".to_string(),
+                    access_type: Some(W)
+                },
+                ParseAssetsResult {
+                    kind: AssetKind::S3Object,
+                    path: "snd/b.parquet".to_string(),
+                    access_type: Some(R)
+                },
+            ])
+        );
+    }
 
-fn parse_datatable_lit(input: &str) -> IResult<&str, &str> {
-    let (input, _) = quote(input)?;
-    let (input, _) = tag("datatable").parse(input)?;
-    let (input, path) =
-        opt(preceded(tag("://"), take_while(|c| c != '\'' && c != '"'))).parse(input)?;
-    let (input, _) = quote(input)?;
-    Ok((input, path.unwrap_or("main")))
-}
+    #[test]
+    fn test_sql_asset_parser_attach_no_usage_is_registered_as_unknown() {
+        let input = r#"
+            ATTACH 'ducklake://my_dl' AS dl;
+            SELECT 2;
+            USE dl;
+        "#;
+        let s = parse_assets(input);
+        assert_eq!(
+            s.map_err(|e| e.to_string()),
+            Ok(vec![ParseAssetsResult {
+                kind: AssetKind::Ducklake,
+                path: "my_dl".to_string(),
+                access_type: None
+            },])
+        );
+    }
 
-fn parse_comment(input: &str) -> IResult<&str, &str> {
-    let (input, _) = tag("--").parse(input)?;
-    let (input, comment) = take_while(|c| c != '\n')(input)?;
-    Ok((input, comment))
+    #[test]
+    fn test_sql_asset_parser_attach_dot_notation_read() {
+        let input = r#"
+            ATTACH 'ducklake://my_dl' AS dl;
+            SELECT * FROM dl.table1;
+        "#;
+        let s = parse_assets(input);
+        assert_eq!(
+            s.map_err(|e| e.to_string()),
+            Ok(vec![ParseAssetsResult {
+                kind: AssetKind::Ducklake,
+                path: "my_dl".to_string(),
+                access_type: Some(R)
+            },])
+        );
+    }
+
+    #[test]
+    fn test_sql_asset_parser_attach_dot_notation_write() {
+        let input = r#"
+            ATTACH 'ducklake://my_dl' AS dl;
+            SELECT dl.read_bait FROM unrelated_table; -- dl. doesn't access the asset
+            INSERT INTO dl.table1 VALUES ('test');
+        "#;
+        let s = parse_assets(input);
+        assert_eq!(
+            s.map_err(|e| e.to_string()),
+            Ok(vec![ParseAssetsResult {
+                kind: AssetKind::Ducklake,
+                path: "my_dl".to_string(),
+                access_type: Some(W)
+            },])
+        );
+    }
+
+    #[test]
+    fn test_sql_asset_parser_detach() {
+        let input = r#"
+            ATTACH 'ducklake://my_dl' AS dl;
+            DETACH dl;
+            SELECT * FROM dl.table1;
+        "#;
+        let s = parse_assets(input);
+        assert_eq!(s.map_err(|e| e.to_string()), Ok(vec![]));
+    }
+
+    #[test]
+    fn test_sql_asset_parser_implicit_use_asset() {
+        let input = r#"
+            ATTACH 'ducklake://my_dl' AS dl;
+            USE dl;
+            INSERT INTO table1 VALUES ('test');
+            USE memory;
+            SELECT * FROM table1;
+        "#;
+        let s = parse_assets(input);
+        assert_eq!(
+            s.map_err(|e| e.to_string()),
+            Ok(vec![ParseAssetsResult {
+                kind: AssetKind::Ducklake,
+                path: "my_dl".to_string(),
+                access_type: Some(W)
+            },])
+        );
+    }
+
+    #[test]
+    fn test_sql_asset_parser_default_main() {
+        let input = r#"
+            ATTACH 'datatable' AS dl;
+            INSERT INTO dl.table1 VALUES ('test');
+        "#;
+        let s = parse_assets(input);
+        assert_eq!(
+            s.map_err(|e| e.to_string()),
+            Ok(vec![ParseAssetsResult {
+                kind: AssetKind::DataTable,
+                path: "main".to_string(),
+                access_type: Some(W)
+            },])
+        );
+    }
+
+    #[test]
+    fn test_sql_asset_parser_create_table() {
+        let input = r#"
+            ATTACH 'ducklake' AS dl; USE dl;
+            CREATE TABLE friends (
+                name text,
+                age int
+            );
+            INSERT INTO friends VALUES ($name, $age);
+            SELECT * FROM friends;
+        "#;
+        let s = parse_assets(input);
+        assert_eq!(
+            s.map_err(|e| e.to_string()),
+            Ok(vec![ParseAssetsResult {
+                kind: AssetKind::Ducklake,
+                path: "main".to_string(),
+                access_type: Some(RW)
+            },])
+        );
+    }
 }

backend/parsers/windmill-parser-ts/src/asset_parser.rs

@@ -58,7 +58,7 @@ impl Visit for AssetsFinder {
     fn visit_lit(&mut self, node: &swc_ecma_ast::Lit) {
         match node {
             swc_ecma_ast::Lit::Str(str) => {
-                if let Some((kind, path)) = parse_asset_syntax(str.value.as_str()) {
+                if let Some((kind, path)) = parse_asset_syntax(str.value.as_str(), false) {
                     self.assets.push(ParseAssetsResult {
                         kind,
                         path: path.to_string(),
@@ -221,7 +221,9 @@ impl AssetsFinder {
 
         match arg_value.map(|e| e.expr.as_ref()) {
             Some(Expr::Lit(Lit::Str(Str { value, .. }))) => {
-                let path = parse_asset_syntax(&value).map(|(_, p)| p).unwrap_or(&value);
+                let path = parse_asset_syntax(&value, false)
+                    .map(|(_, p)| p)
+                    .unwrap_or(&value);
                 self.assets
                     .push(ParseAssetsResult { kind, path: path.to_string(), access_type });
             }

backend/parsers/windmill-parser-wasm/src/lib.rs

@@ -190,41 +190,36 @@ pub fn parse_ruby(code: &str) -> String {
 #[cfg(feature = "sql-parser")]
 #[wasm_bindgen]
 pub fn parse_assets_sql(code: &str) -> String {
-    if let Ok(r) = windmill_parser_sql::parse_assets(code) {
-        return serde_json::to_string(&r).unwrap();
-    } else {
-        return "Invalid".to_string();
+    match windmill_parser_sql::parse_assets(code) {
+        Ok(r) => serde_json::to_string(&r).unwrap(),
+        Err(err) => format!("err: {:?}", err),
     }
 }
 
 #[cfg(feature = "ts-parser")]
 #[wasm_bindgen]
 pub fn parse_assets_ts(code: &str) -> String {
-    if let Ok(r) = windmill_parser_ts::parse_assets(code) {
-        return serde_json::to_string(&r).unwrap();
-    } else {
-        return "Invalid".to_string();
+    match windmill_parser_ts::parse_assets(code) {
+        Ok(r) => serde_json::to_string(&r).unwrap(),
+        Err(err) => format!("err: {:?}", err),
     }
 }
 
 #[cfg(feature = "py-parser")]
 #[wasm_bindgen]
 pub fn parse_assets_py(code: &str) -> String {
-    if let Ok(r) = windmill_parser_py::parse_assets(code) {
-        return serde_json::to_string(&r).unwrap();
-    } else {
-        return "Invalid".to_string();
+    match windmill_parser_py::parse_assets(code) {
+        Ok(r) => serde_json::to_string(&r).unwrap(),
+        Err(err) => format!("err: {:?}", err),
     }
 }
 
 #[cfg(feature = "ansible-parser")]
 #[wasm_bindgen]
 pub fn parse_assets_ansible(code: &str) -> String {
-    let o = windmill_parser_yaml::parse_assets(code);
-    if let Ok(r) = o {
-        return serde_json::to_string(&r).unwrap();
-    } else {
-        return format!("err: {:?}", o.err().unwrap());
+    match windmill_parser_yaml::parse_assets(code) {
+        Ok(r) => serde_json::to_string(&r).unwrap(),
+        Err(err) => format!("err: {:?}", err),
     }
 }
 

backend/parsers/windmill-parser/src/asset_parser.rs

@@ -1,6 +1,6 @@
 use serde::Serialize;
 
-#[derive(Serialize, PartialEq, Clone, Copy)]
+#[derive(Serialize, PartialEq, Clone, Copy, Debug)]
 #[serde(rename_all(serialize = "lowercase"))]
 pub enum AssetUsageAccessType {
     R,
@@ -10,7 +10,7 @@ pub enum AssetUsageAccessType {
 
 use AssetUsageAccessType::*;
 
-#[derive(Serialize, PartialEq, Clone, Copy)]
+#[derive(Serialize, PartialEq, Clone, Copy, Debug)]
 #[serde(rename_all(serialize = "lowercase"))]
 pub enum AssetKind {
     S3Object,
@@ -19,7 +19,7 @@ pub enum AssetKind {
     DataTable,
 }
 
-#[derive(Serialize)]
+#[derive(Serialize, Debug, PartialEq)]
 pub struct ParseAssetsResult<S: AsRef<str>> {
     pub kind: AssetKind,
     pub path: S,
@@ -58,8 +58,12 @@ pub fn merge_assets<S: AsRef<str>>(assets: Vec<ParseAssetsResult<S>>) -> Vec<Par
     arr
 }
 
-pub fn parse_asset_syntax(s: &str) -> Option<(AssetKind, &str)> {
-    if s.starts_with("s3://") {
+pub fn parse_asset_syntax(s: &str, enable_default_syntax: bool) -> Option<(AssetKind, &str)> {
+    if enable_default_syntax && s == "datatable" {
+        Some((AssetKind::DataTable, "main"))
+    } else if enable_default_syntax && s == "ducklake" {
+        Some((AssetKind::Ducklake, "main"))
+    } else if s.starts_with("s3://") {
         Some((AssetKind::S3Object, &s[5..]))
     } else if s.starts_with("res://") {
         Some((AssetKind::Resource, &s[6..]))

frontend/src/lib/components/EditorBar.svelte

@@ -741,7 +741,7 @@ JsonNode ${windmillPathToCamelCaseName(path)} = JsonNode.Parse(await client.GetS
 			}
 		}}
 		tooltip="Attach a Ducklake to your scripts. Ducklake allows you to manipulate large data on S3 blob files through a traditional SQL interface."
-		documentationLink="https://www.windmill.dev/docs/core_concepts/ducklake"
+		documentationLink="https://www.windmill.dev/docs/core_concepts/persistent_storage/ducklake"
 		itemName="ducklake"
 		loadItems={async () =>
 			(await WorkspaceService.listDucklakes({ workspace: $workspaceStore ?? 'NO_W' })).map(
@@ -783,7 +783,7 @@ JsonNode ${windmillPathToCamelCaseName(path)} = JsonNode.Parse(await client.GetS
 			}
 		}}
 		tooltip="Attach a datatable to your script."
-		documentationLink="https://www.windmill.dev/docs/core_concepts/data_tables"
+		documentationLink="https://www.windmill.dev/docs/core_concepts/persistent_storage/data_tables"
 		itemName="data table"
 		loadItems={async () =>
 			(await WorkspaceService.listDataTables({ workspace: $workspaceStore ?? 'NO_W' })).map(
@@ -836,7 +836,7 @@ JsonNode ${windmillPathToCamelCaseName(path)} = JsonNode.Parse(await client.GetS
 	onSelectAndClose={(s3obj) => {
 		let s = `'${formatS3Object(s3obj)}'`
 		if (lang === 'duckdb') {
-			editor?.insertAtCursor(`SELECT * FROM ${s}`)
+			editor?.insertAtCursor(`SELECT * FROM ${s};`)
 		} else if (lang === 'python3') {
 			if (!editor?.getCode().includes('import wmill')) {
 				editor?.insertAtBeginning('import wmill\n')

frontend/src/lib/components/ScriptEditor.svelte

@@ -58,6 +58,7 @@
 	import { copilotInfo } from '$lib/aiStore'
 	import JsonInputs from '$lib/components/JsonInputs.svelte'
 	import Toggle from './Toggle.svelte'
+	import { deepEqual } from 'fast-equals'
 
 	interface Props {
 		// Exported
@@ -158,12 +159,14 @@
 	$effect(() => {
 		;[lang, code]
 		untrack(() => {
-			inferAssets(lang, code).then((newAssets: AssetWithAltAccessType[]) => {
+			inferAssets(lang, code).then((inferAssetsResult) => {
+				if (inferAssetsResult.status === 'error') return
+				let newAssets = inferAssetsResult.assets as AssetWithAltAccessType[]
 				for (const asset of newAssets) {
 					const old = assets?.find((a) => assetEq(a, asset))
 					if (old?.alt_access_type) asset.alt_access_type = old.alt_access_type
 				}
-				assets = newAssets
+				if (!deepEqual(assets, newAssets)) assets = newAssets
 			})
 
 			if (lang === 'ansible') {

frontend/src/lib/components/assets/AssetsDropdownButton.svelte

@@ -107,7 +107,7 @@
 				'text-xs flex items-center gap-1.5 px-2 rounded-md relative',
 				'border',
 				'bg-surface hover:bg-surface-hover active:bg-surface',
-				'transition-all hover:text-primary cursor-pointer'
+				'transition-all hover:text-primary backdrop-blur-md cursor-pointer'
 			)}
 		>
 			<div
@@ -146,9 +146,7 @@
 							<div
 								class={twMerge(
 									'text-xs font-normal border text-primary w-10 p-1 text-center rounded-md',
-									!asset.access_type && !asset.alt_access_type
-										? 'text-orange-500 !border-orange-500'
-										: '',
+									!asset.access_type ? 'text-orange-600 !border-orange-600' : '',
 									!asset.access_type ? 'hover:bg-surface active:opacity-80' : ''
 								)}
 							>
@@ -162,7 +160,11 @@
 									<span class="text-xs">Please select manually </span>
 								</span>
 								<div class="flex items-center gap-2">
-									<ToggleButtonGroup bind:selected={asset.alt_access_type} class="max-w-fit">
+									<ToggleButtonGroup
+										allowEmpty
+										bind:selected={asset.alt_access_type}
+										class="max-w-fit"
+									>
 										{#snippet children({ item })}
 											<ToggleButton value="r" label="Read" {item} />
 											<ToggleButton value="w" label="Write" {item} />

frontend/src/lib/components/assets/JobAssetsViewer.svelte

@@ -35,10 +35,9 @@
 			)
 		}
 		if (job.job_kind === 'script') {
-			return [
-				...(await inferAssets(job.language!, job.raw_code ?? '')),
-				...parseInputArgsAssets(job.args ?? {})
-			]
+			let inferAssetsResult = await inferAssets(job.language!, job.raw_code ?? '')
+			let assets = inferAssetsResult.status === 'ok' ? inferAssetsResult.assets : []
+			return [...assets, ...parseInputArgsAssets(job.args ?? {})]
 		}
 		return []
 	}

frontend/src/lib/components/assets/lib.ts

@@ -93,8 +93,8 @@ export function formatAssetAccessType(accessType: AssetUsageAccessType | undefin
 }
 
 export function getAccessType(asset: AssetWithAltAccessType): AssetUsageAccessType | undefined {
-	if (asset.alt_access_type) return asset.alt_access_type
 	if (asset.access_type) return asset.access_type
+	if (asset.alt_access_type) return asset.alt_access_type
 }
 
 export function getFlowModuleAssets(

frontend/src/lib/components/flows/FlowAssetsHandler.svelte

@@ -119,14 +119,14 @@
 	})
 
 	async function parseAndUpdateRawScriptModule(v: RawScript) {
-		try {
-			let parsedAssets: AssetWithAltAccessType[] = await inferAssets(v.language, v.content)
-			for (const asset of parsedAssets) {
-				const old = v.assets?.find((a) => assetEq(a, asset))
-				if (old?.alt_access_type) asset.alt_access_type = old.alt_access_type
-			}
-			if (!deepEqual(v.assets, parsedAssets)) v.assets = parsedAssets
-		} catch (e) {}
+		let inferAssetsResult = await inferAssets(v.language, v.content)
+		if (inferAssetsResult.status === 'error') return
+		let newAssets = inferAssetsResult.assets as AssetWithAltAccessType[]
+		for (const asset of newAssets) {
+			const old = v.assets?.find((a) => assetEq(a, asset))
+			if (old?.alt_access_type) asset.alt_access_type = old.alt_access_type
+		}
+		if (!deepEqual(v.assets, newAssets)) v.assets = newAssets
 	}
 
 	// Check for raw script modules whose assets were not parsed. Useful for flows created

frontend/src/lib/components/workspaceSettings/DucklakeSettings.svelte

@@ -183,7 +183,7 @@
 <div class="flex flex-col gap-4 mb-8 mt-20">
 	<div class="flex flex-col gap-1">
 		<div class="text-primary text-lg font-semibold">Ducklake</div>
-		<Description link="https://www.windmill.dev/docs/core_concepts/ducklake">
+		<Description link="https://www.windmill.dev/docs/core_concepts/persistent_storage/ducklake">
 			Windmill has first class support for Ducklake. You can use and explore ducklakes like a normal
 			SQL database, even though the data is actually stored in parquet files in S3 !
 		</Description>

frontend/src/lib/infer.ts

@@ -89,33 +89,43 @@ async function initWasmRuby() {
 	await initRubyParser(wasmUrlRuby)
 }
 
+type InferAssetsResult =
+	| { status: 'ok'; assets: AssetWithAccessType[] }
+	| { status: 'error'; error: string }
+
 export async function inferAssets(
 	language: SupportedLanguage | undefined,
 	code: string
-): Promise<AssetWithAccessType[]> {
+): Promise<InferAssetsResult> {
+	function wrap(raw_result: string): InferAssetsResult {
+		if (raw_result.startsWith('err:')) {
+			return { status: 'error', error: raw_result.slice(4).trim() }
+		}
+		return { status: 'ok', assets: JSON.parse(raw_result) as AssetWithAccessType[] }
+	}
+
 	try {
 		if (language === 'duckdb') {
 			await initWasmRegex()
-			let r = JSON.parse(parse_assets_sql(code))
-			return r
+			return wrap(parse_assets_sql(code))
 		}
 		if (language === 'deno' || language === 'nativets' || language === 'bun') {
 			await initWasmTs()
-			return JSON.parse(parse_assets_ts(code))
+			return wrap(parse_assets_ts(code))
 		}
 		if (language === 'python3') {
 			await initWasmPython()
-			return JSON.parse(parse_assets_py(code))
+			return wrap(parse_assets_py(code))
 		}
 		if (language === 'ansible') {
 			await initWasmYaml()
-			return JSON.parse(parse_assets_ansible(code))
+			return wrap(parse_assets_ansible(code))
 		}
 	} catch (e) {
-		console.error('error parsing assets', e)
-		return []
+		return { status: 'error', error: (e as Error)?.message || JSON.stringify(e) }
 	}
-	return []
+
+	return { status: 'ok', assets: [] }
 }
 
 export async function inferAnsibleExecutionMode(code: string): Promise<any> {

frontend/src/lib/script_helpers.ts

@@ -323,7 +323,7 @@ const DUCKDB_INIT_CODE = `-- result_collection=last_statement_all_rows
 -- SELECT * FROM db.public.friends;
 
 -- Click the +Ducklake button to use a ducklake
--- https://www.windmill.dev/docs/core_concepts/ducklake
+-- https://www.windmill.dev/docs/core_concepts/persistent_storage/ducklake
 --
 -- ATTACH 'ducklake' AS dl;
 -- USE dl;