feat: Data tables (#7226)

* data tables settings ui

* install runed

* zod 4 fixes

* use new toJSONSchema

* Migrate ducklake catalogs to more generic custom instance databases

* fix compilation

* Safety conversion for old duckdb ffi

* data tables settings

* ts client basis

* inline run works

* datatables work

* Revert "datatables work"

This reverts commit 6e1588d59e.

* datatables work (without leaking pg credentials)

* println

* separate sqlUtils.ts

* nit

* Separate custom instance db Select and Wizard components

* nit

* nit wording

* add tags to custom instance dbs

* error when trying to use ducklake as datatable or opposite

* show status in dropdown

* data table instance setup works

* sqk function for ducklake

* factorize logic

* fix temp reactivity

* Data table assetexplore

* Migrate S3 permissions to modal

* Revert "Migrate S3 permissions to modal"

This reverts commit 0631d03cb0.

* nit query -> fetch

* Custom instance setup new look

* run_language_executor separate fn

* run_inline param

* nit wording

* Better typed client

* Data tables display as assets in frontend

* asset db icon

* nit

* cleaner errors

* nit

* Fix sed calls in mac

* run_inline_script_preview in python client

* basic python datatable client

* datatable and datalake parser in python

* ducklake client python

* nit fix

* Fix migration producing NULL instead of {} when no custom databases

* merge conflict fail

* python ducklake client arg fix

* parse or infer sql types in ts client

* ts asset parser, detect datatable & ducklake R/W

* fix sql repl for other read ops than select

* export type SqlTemplateFunction

* rename list_custom_instance_pg_databases

* typecheck datatable and ducklake name in Typescript

* Fix typecheck datatable and ducklake in TS

* declare module overriding instead of extending

* infer_sql_type in python client

* SqlQuery object in python

* fix merge conflicts

* update const_format

* CI fix

* factor out to var_identifiers

* sqlx prepare

* unnecessary security (admin is required)

* clearer comment

* ee repo ref

* nit snake case

* claude step 1: detect var declarations

* move detect_sql_access_type to common mod

* claude step 2: detect when saved vars are queried

* Revert "claude step 2: detect when saved vars are queried"

This reverts commit 1e1f930568.

* Revert "claude step 1: detect var declarations"

This reverts commit f866f4819d.

* remove ducklake/datatable and default

* detect data table assigns in var_identifiers

* Python parser successfully infers R/W/RW from ducklake / datatable

* still register ducklake/datatable if not used as unknown R/W

* Go to settings button in Assets Dropdown on not found

* nit

* sqlx prepare fail

* manual fix, somehow sqlx prepare won't do it

* fix frontend ci

* ee repo ref

* ducklake_user doesnt exist in unit tests

* nit fix

* ui nit

* nit

* nit missing clone

* fork ducklakes and datatables

* fix surface hover bug

* stupid mistake

* better deeply reactive mutable derived

* Ducklake picker

* Editor bar data tables

* DuckDB supports datatables

* datatable in duckdb asset parser

* duckdb asset parser var_identifiers

* Revert "duckdb asset parser var_identifiers"

This reverts commit 88068b1a77.

* sqlx prepare

* Box pin in test_workflow_as_code to fix stack overflow

* go to settings button

* ee repo ref

* fix compilation

* wording nit

python-client/wmill/wmill/client.py

@@ -276,6 +276,21 @@ class Windmill:
             cleanup=cleanup, assert_result_is_not_none=assert_result_is_not_none
         )
 
+    def run_inline_script_preview(
+        self,
+        content: str,
+        language: str,
+        args: dict = None,
+    ) -> Any:
+        """Run a script on the current worker without creating a job"""
+        endpoint = f"/w/{self.workspace}/jobs/run_inline/preview"
+        body = {
+            "content": content,
+            "language": language,
+            "args": args or {},
+        }
+        return self.post(endpoint, json=body).text
+
     def wait_job(
         self,
         job_id,
@@ -1000,6 +1015,13 @@ class Windmill:
             },
         )
 
+    def datatable(self, name: str = "main"):
+        return DataTableClient(self, name)
+
+    def ducklake(self, name: str = "main"):
+        return DucklakeClient(self, name)
+
+
 
 def init_global_client(f):
     @functools.wraps(f)
@@ -1527,6 +1549,18 @@ def run_script_by_hash(
         timeout=timeout,
     )
 
+@init_global_client
+def run_inline_script_preview(
+    content: str,
+    language: str,
+    args: dict = None,
+) -> Any:
+    """Run a script on the current worker without creating a job"""
+    return _client.run_inline_script_preview(
+        content=content,
+        language=language,
+        args=args,
+    )
 
 @init_global_client
 def username_to_email(username: str) -> str:
@@ -1538,6 +1572,14 @@ def username_to_email(username: str) -> str:
     return _client.username_to_email(username)
 
 
+@init_global_client
+def datatable(name: str = "main") -> DataTableClient:
+    return _client.datatable(name)
+
+@init_global_client
+def ducklake(name: str = "main") -> DucklakeClient:
+    return _client.ducklake(name)
+
 def task(*args, **kwargs):
     from inspect import signature
 
@@ -1635,3 +1677,78 @@ def stream_result(stream) -> None:
     """
     for text in stream:
         append_to_result_stream(text)
+
+class DataTableClient:
+    def __init__(self, client: Windmill, name: str):
+        self.client = client
+        self.name = name
+    def query(self, sql: str, *args):
+        args_dict = {}
+        args_def = ""
+        for i, arg in enumerate(args):
+            args_dict[f"arg{i+1}"] = arg
+            args_def += f"-- ${i+1} arg{i+1}\n"
+        sql = args_def + sql
+        return SqlQuery(
+            sql, 
+            lambda sql: self.client.run_inline_script_preview(
+                content=sql,
+                language="postgresql",
+                args={"database": f"datatable://{self.name}", **args_dict},
+            )
+        )
+
+class DucklakeClient:
+    def __init__(self, client: Windmill, name: str):
+        self.client = client
+        self.name = name
+    def query(self, sql: str, **kwargs):
+        args_dict = {}
+        args_def = ""
+        for key, value in kwargs.items():
+            args_dict[key] = value
+            args_def += f"-- ${key} ({infer_sql_type(value)})\n"
+        attach = f"ATTACH 'ducklake://{self.name}' AS dl;USE dl;\n"
+        sql = args_def + attach + sql
+        return SqlQuery(
+            sql, 
+            lambda sql: self.client.run_inline_script_preview(
+                content=sql,
+                language="duckdb",
+                args=args_dict,
+            )
+        )
+
+class SqlQuery:
+    def __init__(self, sql: str, fetch_fn):
+        self.sql = sql
+        self.fetch_fn = fetch_fn
+    def fetch(self, result_collection: str | None = None):
+        sql = self.sql
+        if result_collection is not None:
+            sql = f'-- result_collection={result_collection}\n{sql}'
+        return self.fetch_fn(sql)
+    def fetch_one(self):
+        return self.fetch(result_collection="last_statement_first_row")
+
+def infer_sql_type(value) -> str:
+    """
+    DuckDB executor requires explicit argument types at declaration
+    These types exist in both DuckDB and Postgres
+    Check that the types exist if you plan to extend this function for other SQL engines.
+    """
+    if isinstance(value, bool):
+        # Check bool before int since bool is a subclass of int in Python
+        return "BOOLEAN"
+    elif isinstance(value, int):
+        return "BIGINT"
+    elif isinstance(value, float):
+        return "DOUBLE PRECISION"
+    elif value is None:
+        return "TEXT"
+    elif isinstance(value, str):
+        return "TEXT"
+    elif isinstance(value, dict) or isinstance(value, list):
+        return "JSON"
+    else:
+        return "TEXT"