feat: add OSS wrapper files for all EE modules

Create OSS implementations for all *_ee.rs files following the established pattern:
- Copy function signatures from EE modules and call EE implementations
- Re-export public structs, enums, and traits to maintain API compatibility
- Enables seamless switching between Enterprise and Open Source builds

Co-authored-by: diegoimbert <diegoimbert@users.noreply.github.com>

.github/workflows/claude.yml

@@ -13,10 +13,10 @@ on:
 jobs:
   check-membership:
     if: |
-      (github.event_name == 'issue_comment' && contains(github.event.comment.body, '/aider') && !contains(github.event.comment.user.login, '[bot]')) ||
-      (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '/aider') && !contains(github.event.comment.user.login, '[bot]')) ||
-      (github.event_name == 'pull_request_review' && contains(github.event.review.body, '/aider') && !contains(github.event.review.user.login, '[bot]')) ||
-      (github.event_name == 'issues' && contains(github.event.issue.body, '/aider') && !contains(github.event.issue.user.login, '[bot]'))
+      (github.event_name == 'issue_comment' && contains(github.event.comment.body, '/ai') && !contains(github.event.comment.user.login, '[bot]')) ||
+      (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '/ai') && !contains(github.event.comment.user.login, '[bot]')) ||
+      (github.event_name == 'pull_request_review' && contains(github.event.review.body, '/ai') && !contains(github.event.review.user.login, '[bot]')) ||
+      (github.event_name == 'issues' && contains(github.event.issue.body, '/ai') && !contains(github.event.issue.user.login, '[bot]'))
     runs-on: ubicloud-standard-2
     outputs:
       is_member: ${{ steps.check-membership.outputs.is_member }}
@@ -82,4 +82,4 @@ jobs:
           - Bash(npm run generate-backend-client): Generate the backend client. You need this to run npm run check.
           - Bash(cargo check): Run the cargo check script. You should run this tool after making changes to the backend code.
           - Bash(curl https://sh.rustup.rs -sSf | sh): Install Rust. You need this to run cargo check."
-          trigger_phrase: "/aider"
+          trigger_phrase: "/ai"

CHANGELOG.md

@@ -1,5 +1,37 @@
 # Changelog
 
+## [1.493.2](https://github.com/windmill-labs/windmill/compare/v1.493.1...v1.493.2) (2025-05-28)
+
+
+### Bug Fixes
+
+* improve monaco editor memory leak ([e0f4f83](https://github.com/windmill-labs/windmill/commit/e0f4f83ebf4416c3bcc24433a7bf606349e1f75a))
+* improve monaco javascript extra lib refresh ([7b70348](https://github.com/windmill-labs/windmill/commit/7b70348b4bba3726e3fb26c964219a5a2aa6af55))
+
+## [1.493.1](https://github.com/windmill-labs/windmill/compare/v1.493.0...v1.493.1) (2025-05-28)
+
+
+### Bug Fixes
+
+* improve monaco javascript extra lib refresh ([a2c8ea6](https://github.com/windmill-labs/windmill/commit/a2c8ea69a3962a350273717cd237d8a96523fd00))
+
+## [1.493.0](https://github.com/windmill-labs/windmill/compare/v1.492.1...v1.493.0) (2025-05-27)
+
+
+### Features
+
+* add aws oidc support for instance s3 storage ([#5810](https://github.com/windmill-labs/windmill/issues/5810)) ([5b96bcc](https://github.com/windmill-labs/windmill/commit/5b96bccedd6e68fea631580dd49338301ad0305f))
+* duckdb sql lang support ([#5761](https://github.com/windmill-labs/windmill/issues/5761)) ([fdefd4b](https://github.com/windmill-labs/windmill/commit/fdefd4be9398b9610a539360353fd61b521732d4))
+* **python:** inline script metadata (PEP 723)  ([#5712](https://github.com/windmill-labs/windmill/issues/5712)) ([2622253](https://github.com/windmill-labs/windmill/commit/26222539e66bce7e88f86a7e5917e6ca99350865))
+
+
+### Bug Fixes
+
+* add missing http_trigger_version_seq grants ([#5816](https://github.com/windmill-labs/windmill/issues/5816)) ([306f3ea](https://github.com/windmill-labs/windmill/commit/306f3eabd1c03fa904b0e59438de124a0e680597))
+* avoid monaco memory leak ([0d459d5](https://github.com/windmill-labs/windmill/commit/0d459d5d223728270854e37715ecc1663ede9870))
+* error handler node rendering at top level ([feae9b0](https://github.com/windmill-labs/windmill/commit/feae9b09240ba306c007013a36d2aefb0b273766))
+* **frontend:** auto completion and render of tailwind classes in app editor ([#5817](https://github.com/windmill-labs/windmill/issues/5817)) ([5897e7e](https://github.com/windmill-labs/windmill/commit/5897e7e01b8839425c30c2a97481ef7bb9090661))
+
 ## [1.492.1](https://github.com/windmill-labs/windmill/compare/v1.492.0...v1.492.1) (2025-05-22)
 
 

backend/.gitignore

@@ -5,4 +5,5 @@ oauth2.json
 tracing.folded
 heaptrack*
 index/
-windmill-api/openapi-*.*
+windmill-api/openapi-*.*
+.duckdb/*

backend/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "windmill"
-version = "1.492.1"
+version = "1.493.2"
 authors.workspace = true
 edition.workspace = true
 
@@ -32,7 +32,7 @@ members = [
 ]
 
 [workspace.package]
-version = "1.492.1"
+version = "1.493.2"
 authors = ["Ruben Fiszel <ruben@windmill.dev>"]
 edition = "2021"
 
@@ -59,7 +59,7 @@ embedding = ["windmill-api/embedding"]
 parquet = ["windmill-api/parquet", "windmill-common/parquet", "windmill-worker/parquet", "dep:object_store"]
 prometheus = ["windmill-common/prometheus", "windmill-api/prometheus", "windmill-worker/prometheus", "windmill-queue/prometheus", "dep:prometheus"]
 flow_testing = ["windmill-worker/flow_testing"]
-openidconnect = ["windmill-api/openidconnect"]
+openidconnect = ["windmill-api/openidconnect", "windmill-common/openidconnect"]
 cloud = ["windmill-queue/cloud", "windmill-worker/cloud", "windmill-common/cloud", "windmill-api/cloud"]
 jemalloc = ["windmill-common/jemalloc", "dep:tikv-jemallocator", "dep:tikv-jemalloc-sys", "dep:tikv-jemalloc-ctl"]
 tantivy = ["dep:windmill-indexer", "windmill-api/tantivy", "windmill-indexer/enterprise", "windmill-indexer/parquet", "windmill-common/tantivy", "enterprise", "parquet"]
@@ -87,13 +87,14 @@ python = ["windmill-worker/python", "windmill-api/python"]
 rust = ["windmill-worker/rust"]
 mysql = ["windmill-worker/mysql"]
 oracledb = ["windmill-worker/oracledb"]
+duckdb = ["windmill-worker/duckdb"]
 mssql = ["windmill-worker/mssql"]
 bigquery = ["windmill-worker/bigquery"]
 php = ["windmill-worker/php"]
 csharp = ["windmill-worker/csharp"]
 nu = ["windmill-worker/nu"]
 java = ["windmill-worker/java"]
-all_languages = [ "python", "deno_core", "rust", "mysql", "oracledb", "mssql", "bigquery", "csharp", "nu", "php", "java"]
+all_languages = [ "python", "deno_core", "rust", "mysql", "oracledb", "duckdb", "mssql", "bigquery", "csharp", "nu", "php", "java"]
 
 
 [patch.crates-io]
@@ -238,6 +239,7 @@ json-pointer = "^0"
 itertools = "^0"
 regex = "^1"
 semver = "^1"
+duckdb = { version = "1.2.2", features = ["bundled"] }
 
 v8 = "=130.0.7"  # Exact version NOTE: Do not forget to update version and hash in flake.nix
 deno_fetch = "0.214.0"
@@ -346,7 +348,7 @@ openidconnect = { version = "4.0.0-rc.1" }
 aws-config = "^1"
 aws-sdk-sqs = "1.57.0"
 aws-sdk-sts = "^1"
-
+aws-smithy-types-convert = { version = "^0", features = ["convert-chrono"] }
 crc = "^3"
 tar = "^0"
 http = "^1"

backend/ee-repo-ref.txt

@@ -1 +1 @@
-72e6260ca886628cf1ba271bc058e6ecfdecdae5
+32039f675060b5996951708368bdefe14278d5cd

backend/migrations/20250515084520_duckdb_support.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20250515084520_duckdb_support.up.sql

@@ -0,0 +1,3 @@
+-- Add up migration script here
+ALTER TYPE SCRIPT_LANG ADD VALUE IF NOT EXISTS 'duckdb';
+UPDATE config set config = jsonb_set(config, '{worker_tags}', config->'worker_tags' || '["duckdb"]'::jsonb) where name = 'worker__default' and config @> '{"worker_tags": ["deno", "python3", "go", "bash", "powershell", "dependency", "flow", "hub", "other", "bun", "php", "rust", "ansible", "csharp", "nu", "java"]}'::jsonb AND NOT config->'worker_tags' @> '"duckdb"'::jsonb;

backend/migrations/20250526173025_http_trigger_seq_grants.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20250526173025_http_trigger_seq_grants.up.sql

@@ -0,0 +1,3 @@
+-- Add up migration script here
+GRANT ALL ON SEQUENCE http_trigger_version_seq TO windmill_user;
+GRANT ALL ON SEQUENCE http_trigger_version_seq TO windmill_admin;

backend/parsers/windmill-parser-sql/src/lib.rs

@@ -83,6 +83,21 @@ pub fn parse_bigquery_sig(code: &str) -> anyhow::Result<MainArgSignature> {
     }
 }
 
+pub fn parse_duckdb_sig(code: &str) -> anyhow::Result<MainArgSignature> {
+    let parsed = parse_duckdb_file(&code)?;
+    if let Some(args) = parsed {
+        Ok(MainArgSignature {
+            star_args: false,
+            star_kwargs: false,
+            args,
+            no_main_func: None,
+            has_preprocessor: None,
+        })
+    } else {
+        Err(anyhow!("Error parsing sql".to_string()))
+    }
+}
+
 pub fn parse_snowflake_sig(code: &str) -> anyhow::Result<MainArgSignature> {
     let parsed = parse_snowflake_file(&code)?;
     if let Some(x) = parsed {
@@ -212,6 +227,9 @@ lazy_static::lazy_static! {
     // -- @name (type) = default
     static ref RE_ARG_BIGQUERY: Regex = Regex::new(r#"(?m)^-- @(\w+) \((\w+(?:\[\])?)\)(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap();
 
+    // -- $name (type) = default
+    static ref RE_ARG_DUCKDB: Regex = Regex::new(r#"(?m)^-- \$(\w+) \((\w+)\)(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap();
+
     static ref RE_ARG_SNOWFLAKE: Regex = Regex::new(r#"(?m)^-- \? (\w+) \((\w+)\)(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap();
 
 
@@ -577,6 +595,35 @@ fn parse_bigquery_file(code: &str) -> anyhow::Result<Option<Vec<Arg>>> {
     Ok(Some(args))
 }
 
+fn parse_duckdb_file(code: &str) -> anyhow::Result<Option<Vec<Arg>>> {
+    let mut args: Vec<Arg> = vec![];
+
+    for cap in RE_ARG_DUCKDB.captures_iter(code) {
+        let name = cap.get(1).map(|x| x.as_str().to_string()).unwrap();
+        let typ = cap
+            .get(2)
+            .map(|x| x.as_str().to_string().to_lowercase())
+            .unwrap();
+        let default = cap.get(3).map(|x| x.as_str().to_string());
+        let has_default = default.is_some();
+        let parsed_typ = parse_duckdb_typ(typ.as_str());
+
+        let parsed_default = default.and_then(|x| parsed_default(&parsed_typ, x));
+
+        args.push(Arg {
+            name,
+            typ: parsed_typ,
+            default: parsed_default,
+            otyp: Some(typ),
+            has_default,
+            oidx: None,
+        });
+    }
+
+    args.append(&mut parse_sql_sanitized_interpolation(code));
+    Ok(Some(args))
+}
+
 fn parse_snowflake_file(code: &str) -> anyhow::Result<Option<Vec<Arg>>> {
     let mut args: Vec<Arg> = vec![];
 
@@ -729,6 +776,33 @@ pub fn parse_bigquery_typ(typ: &str) -> Typ {
     }
 }
 
+pub fn parse_duckdb_typ(typ: &str) -> Typ {
+    if typ.ends_with("[]") {
+        let base_typ = parse_duckdb_typ(typ.strip_suffix("[]").unwrap());
+        Typ::List(Box::new(base_typ))
+    } else {
+        match typ {
+            "varchar" | "char" | "bpchar" | "text" | "string" => Typ::Str(None),
+            "blob" | "bytea" | "binary" | "varbinary" | "bitstring" => Typ::Bytes,
+            "boolean" | "bool" | "bit" | "logical" => Typ::Bool,
+            "bigint" | "int8" | "long" | "integer" | "int4" | "int" | "smallint" | "int2"
+            | "short" | "tinyint" | "int1" | "signed" | "ubigint" | "uhugeint" | "uinteger"
+            | "usmallint" | "utinyint" => Typ::Int,
+            "decimal" | "numeric" | "double" | "float8" | "float" | "float4" | "real" => Typ::Float,
+            "date"
+            | "time"
+            | "timestamp with time zone"
+            | "timestamptz"
+            | "timestamp"
+            | "datetime" => Typ::Datetime,
+            "uuid" | "json" => Typ::Str(None),
+            "interval" | "hugeint" => Typ::Str(None),
+            "s3object" => Typ::Resource("S3Object".to_string()),
+            _ => Typ::Str(None),
+        }
+    }
+}
+
 pub fn parse_snowflake_typ(typ: &str) -> Typ {
     match typ {
         "varchar" => Typ::Str(None),

backend/parsers/windmill-parser-wasm/src/lib.rs

@@ -96,6 +96,12 @@ pub fn parse_oracledb(code: &str) -> String {
     wrap_sig(windmill_parser_sql::parse_oracledb_sig(code))
 }
 
+#[cfg(feature = "sql-parser")]
+#[wasm_bindgen]
+pub fn parse_duckdb(code: &str) -> String {
+    wrap_sig(windmill_parser_sql::parse_duckdb_sig(code))
+}
+
 #[cfg(feature = "sql-parser")]
 #[wasm_bindgen]
 pub fn parse_bigquery(code: &str) -> String {

backend/src/main.rs

@@ -69,7 +69,7 @@ use tikv_jemallocator::Jemalloc;
 static GLOBAL: Jemalloc = Jemalloc;
 
 #[cfg(feature = "parquet")]
-use windmill_common::global_settings::OBJECT_STORE_CACHE_CONFIG_SETTING;
+use windmill_common::global_settings::OBJECT_STORE_CONFIG_SETTING;
 
 use windmill_worker::{
     get_hub_script_content_and_requirements, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, CSHARP_CACHE_DIR,
@@ -92,7 +92,7 @@ use crate::monitor::{
 };
 
 #[cfg(feature = "parquet")]
-use crate::monitor::reload_s3_cache_setting;
+use windmill_common::s3_helpers::reload_object_store_setting;
 
 const DEFAULT_NUM_WORKERS: usize = 1;
 const DEFAULT_PORT: u16 = 8000;
@@ -907,9 +907,9 @@ Windmill Community Edition {GIT_VERSION}
                                                             reload_job_default_timeout_setting(&conn).await
                                                         },
                                                         #[cfg(feature = "parquet")]
-                                                        OBJECT_STORE_CACHE_CONFIG_SETTING => {
+                                                        OBJECT_STORE_CONFIG_SETTING => {
                                                             if !disable_s3_store {
-                                                                reload_s3_cache_setting(&db).await
+                                                                reload_object_store_setting(&db).await;
                                                             }
                                                         },
                                                         SCIM_TOKEN_SETTING => {

backend/src/monitor.rs

@@ -33,8 +33,11 @@ use windmill_common::ee::low_disk_alerts;
 #[cfg(feature = "enterprise")]
 use windmill_common::ee::{jobs_waiting_alerts, worker_groups_alerts};
 
+use windmill_common::client::AuthedClient;
 #[cfg(feature = "oauth2")]
 use windmill_common::global_settings::OAUTH_SETTING;
+#[cfg(feature = "parquet")]
+use windmill_common::s3_helpers::reload_object_store_setting;
 use windmill_common::{
     agent_workers::DECODED_AGENT_TOKEN,
     auth::create_token_for_owner,
@@ -75,19 +78,13 @@ use windmill_common::{
 };
 use windmill_queue::{cancel_job, MiniPulledJob, SameWorkerPayload};
 use windmill_worker::{
-    handle_job_error, AuthedClient, JobCompletedSender, SameWorkerSender, BUNFIG_INSTALL_SCOPES,
+    handle_job_error, JobCompletedSender, SameWorkerSender, BUNFIG_INSTALL_SCOPES,
     INSTANCE_PYTHON_VERSION, JOB_DEFAULT_TIMEOUT, KEEP_JOB_DIR, MAVEN_REPOS, NO_DEFAULT_MAVEN,
     NPM_CONFIG_REGISTRY, NUGET_CONFIG, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL,
 };
 
 #[cfg(feature = "parquet")]
-use windmill_common::s3_helpers::{
-    build_object_store_from_settings, build_s3_client_from_settings, S3Settings,
-    OBJECT_STORE_CACHE_SETTINGS,
-};
-
-#[cfg(feature = "parquet")]
-use windmill_common::global_settings::OBJECT_STORE_CACHE_CONFIG_SETTING;
+use windmill_common::s3_helpers::ObjectStoreReload;
 
 #[cfg(feature = "enterprise")]
 use crate::ee::verify_license_key;
@@ -241,7 +238,23 @@ pub async fn initial_load(
     #[cfg(feature = "parquet")]
     if !disable_s3_store {
         if let Some(db) = conn.as_sql() {
-            reload_s3_cache_setting(db).await;
+            let db2 = db.clone();
+            match reload_object_store_setting(db).await {
+                ObjectStoreReload::Later => {
+                    tokio::spawn(async move {
+                        tokio::time::sleep(Duration::from_secs(10)).await;
+                        match reload_object_store_setting(&db2).await {
+                            ObjectStoreReload::Later => {
+                                tracing::error!("Giving up on loading object store setting");
+                            }
+                            ObjectStoreReload::Never => {
+                                tracing::info!("Object store setting successfully loaded");
+                            }
+                        }
+                    });
+                }
+                ObjectStoreReload::Never => (),
+            }
         }
     }
 
@@ -631,7 +644,7 @@ async fn send_log_file_to_object_store(
         }
 
         #[cfg(feature = "parquet")]
-        let s3_client = OBJECT_STORE_CACHE_SETTINGS.read().await.clone();
+        let s3_client = windmill_common::s3_helpers::get_object_store().await;
         #[cfg(feature = "parquet")]
         if let Some(s3_client) = s3_client {
             let path = std::path::Path::new(TMP_WINDMILL_LOGS_SERVICE)
@@ -917,10 +930,7 @@ async fn delete_log_files_from_disk_and_store(
     _s3_prefix: &str,
 ) {
     #[cfg(feature = "parquet")]
-    let os = windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS
-        .read()
-        .await
-        .clone();
+    let os = windmill_common::s3_helpers::get_object_store().await;
     #[cfg(not(feature = "parquet"))]
     let os: Option<()> = None;
 
@@ -1101,64 +1111,6 @@ pub async fn reload_delete_logs_periodically_setting(conn: &Connection) {
     }
 }
 
-#[cfg(feature = "parquet")]
-pub async fn reload_s3_cache_setting(db: &DB) {
-    use windmill_common::{
-        ee::{get_license_plan, LicensePlan},
-        s3_helpers::ObjectSettings,
-    };
-
-    let s3_config = load_value_from_global_settings(db, OBJECT_STORE_CACHE_CONFIG_SETTING).await;
-    if let Err(e) = s3_config {
-        tracing::error!("Error reloading s3 cache config: {:?}", e)
-    } else {
-        if let Some(v) = s3_config.unwrap() {
-            if matches!(get_license_plan().await, LicensePlan::Pro) {
-                tracing::error!("S3 cache is not available for pro plan");
-                return;
-            }
-            let mut s3_cache_settings = OBJECT_STORE_CACHE_SETTINGS.write().await;
-            let setting = serde_json::from_value::<ObjectSettings>(v);
-            if let Err(e) = setting {
-                tracing::error!("Error parsing s3 cache config: {:?}", e)
-            } else {
-                let setting = setting.unwrap();
-                let bucket = setting.get_bucket().map(|b| b.to_string());
-                let s3_client = build_object_store_from_settings(setting).await;
-                if let Err(e) = s3_client {
-                    tracing::error!("Error building s3 client from settings: {:?}", e)
-                } else {
-                    tracing::info!("Loaded object store {:?}", bucket);
-                    *s3_cache_settings = Some(s3_client.unwrap());
-                }
-            }
-        } else {
-            let mut s3_cache_settings = OBJECT_STORE_CACHE_SETTINGS.write().await;
-            if std::env::var("S3_CACHE_BUCKET").is_ok() {
-                if matches!(get_license_plan().await, LicensePlan::Pro) {
-                    tracing::error!("S3 cache is not available for pro plan");
-                    return;
-                }
-                *s3_cache_settings = build_s3_client_from_settings(S3Settings {
-                    bucket: None,
-                    region: None,
-                    access_key: None,
-                    secret_key: None,
-                    endpoint: None,
-                    store_logs: None,
-                    path_style: None,
-                    allow_http: None,
-                    port: None,
-                })
-                .await
-                .ok();
-            } else {
-                *s3_cache_settings = None;
-            }
-        }
-    }
-}
-
 pub async fn reload_job_default_timeout_setting(conn: &Connection) {
     reload_option_setting_with_tracing(
         conn,

backend/windmill-api/Cargo.toml

@@ -18,7 +18,7 @@ benchmark = []
 embedding = ["dep:tinyvector", "dep:hf-hub", "dep:tokenizers", "dep:candle-core", "dep:candle-transformers", "dep:candle-nn"]
 parquet = ["dep:datafusion", "dep:object_store", "dep:url", "windmill-common/parquet", "windmill-worker/parquet"]
 prometheus = ["windmill-common/prometheus", "windmill-queue/prometheus", "dep:prometheus", "windmill-worker/prometheus"]
-openidconnect = ["dep:openidconnect"]
+openidconnect = ["dep:openidconnect", "windmill-common/openidconnect"]
 tantivy = ["dep:windmill-indexer"]
 kafka = ["dep:rdkafka"]
 nats = ["dep:async-nats", "dep:nkeys"]

backend/windmill-api/openapi.yaml

@@ -1,7 +1,7 @@
 openapi: "3.0.3"
 
 info:
-  version: 1.492.1
+  version: 1.493.2
   title: Windmill API
 
   contact:
@@ -14242,7 +14242,8 @@ components:
           ansible,
           csharp,
           nu,
-          java
+          java,
+          duckdb
           # for related places search: ADD_NEW_LANG
         ]
 

backend/windmill-api/src/agent_workers_oss.rs

@@ -0,0 +1,32 @@
+use crate::db::DB;
+use axum::Router;
+
+pub struct AgentAuth {
+    pub worker_group: String,
+    pub suffix: Option<String>,
+    pub tags: Vec<String>,
+    pub exp: Option<usize>,
+}
+
+pub struct AgentCache {}
+
+impl AgentCache {
+    pub fn new() -> Self {
+        crate::agent_workers_ee::AgentCache::new()
+    }
+}
+
+pub fn global_service() -> Router {
+    crate::agent_workers_ee::global_service()
+}
+
+pub fn workspaced_service(
+    db: DB,
+    base_internal_url: String,
+) -> (
+    Router,
+    Vec<tokio::task::JoinHandle<()>>,
+    Option<windmill_worker::JobCompletedSender>,
+) {
+    crate::agent_workers_ee::workspaced_service(db, base_internal_url)
+}

backend/windmill-api/src/apps_oss.rs

@@ -0,0 +1,5 @@
+use axum::Router;
+
+pub fn global_unauthed_service() -> Router {
+    crate::apps_ee::global_unauthed_service()
+}

backend/windmill-api/src/gcp_triggers_oss.rs

@@ -0,0 +1,126 @@
+use crate::db::DB;
+use axum::Router;
+use serde::{Deserialize, Serialize};
+use serde_json::value::RawValue;
+use sqlx_json::Json as SqlxJson;
+use std::collections::HashMap;
+use windmill_common::{error::WindmillError, utils::TriggerJobArgs, error::Result as WindmillResult};
+
+#[derive(Serialize, Deserialize)]
+pub enum DeliveryType {
+    Pull,
+    Push,
+}
+
+#[derive(Serialize, Deserialize)]
+pub enum SubscriptionMode {
+    Existing,
+    CreateUpdate,
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct PushConfig {
+    route_path: Option<String>,
+    audience: Option<String>,
+    authenticate: bool,
+    base_endpoint: String,
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct CreateUpdateConfig {
+    pub delivery_type: DeliveryType,
+    pub subscription_id: Option<String>,
+    pub delivery_config: Option<SqlxJson<PushConfig>>,
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct ExistingGcpSubscription {
+    pub subscription_id: String,
+    pub base_endpoint: String,
+}
+
+#[derive(Serialize, Clone)]
+pub struct GcpTrigger {
+    pub workspace_id: String,
+    pub path: String,
+    pub gcp_resource_path: String,
+    pub project_id: String,
+    pub topic_id: String,
+    pub subscription_mode: SubscriptionMode,
+    pub existing_subscription: Option<ExistingGcpSubscription>,
+    pub create_update_config: Option<CreateUpdateConfig>,
+    pub script_path: String,
+    pub is_flow: bool,
+    pub edited_by: String,
+    pub email: String,
+    pub edited_at: chrono::DateTime<chrono::Utc>,
+    pub server_id: Option<String>,
+    pub last_server_ping: Option<chrono::DateTime<chrono::Utc>>,
+    pub extra_perms: serde_json::Value,
+    pub error: Option<String>,
+    pub enabled: bool,
+}
+
+impl TriggerJobArgs<String> for GcpTrigger {
+    fn get_workspace_id(&self) -> &str {
+        &self.workspace_id
+    }
+
+    fn get_path(&self) -> &str {
+        &self.path
+    }
+
+    fn get_script_path(&self) -> &str {
+        &self.script_path
+    }
+
+    fn get_is_flow(&self) -> bool {
+        self.is_flow
+    }
+
+    fn get_extra_perms(&self) -> &serde_json::Value {
+        &self.extra_perms
+    }
+
+    fn get_args(&self) -> &String {
+        &String::new()
+    }
+}
+
+pub fn workspaced_service() -> Router {
+    crate::gcp_triggers_ee::workspaced_service()
+}
+
+pub fn start_consuming_gcp_pubsub_event(
+    _db: DB,
+    mut _killpill_rx: tokio::sync::broadcast::Receiver<()>,
+) -> () {
+    crate::gcp_triggers_ee::start_consuming_gcp_pubsub_event(_db, _killpill_rx)
+}
+
+pub async fn manage_google_subscription(
+    _authed: windmill_common::users::Authed,
+    _path: axum::extract::Path<String>,
+    _axum::extract::Json(_trigger): axum::extract::Json<GcpTrigger>,
+) -> WindmillResult<CreateUpdateConfig> {
+    crate::gcp_triggers_ee::manage_google_subscription(_authed, _path, axum::extract::Json(_trigger)).await
+}
+
+pub async fn process_google_push_request(
+    _headers: axum::http::HeaderMap,
+    _body: axum::body::Bytes,
+    _base_endpoint: String,
+) -> Result<(String, HashMap<String, Box<RawValue>>), WindmillError> {
+    crate::gcp_triggers_ee::process_google_push_request(_headers, _body, _base_endpoint).await
+}
+
+pub async fn validate_jwt_token(
+    _audience: &str,
+    _jwt_token: &str,
+) -> Result<(), windmill_common::error::Error> {
+    crate::gcp_triggers_ee::validate_jwt_token(_audience, _jwt_token).await
+}
+
+pub fn gcp_push_route_handler() -> Router {
+    crate::gcp_triggers_ee::gcp_push_route_handler()
+}

backend/windmill-api/src/git_sync_oss.rs

@@ -0,0 +1,9 @@
+use axum::Router;
+
+pub fn workspaced_service() -> Router {
+    crate::git_sync_ee::workspaced_service()
+}
+
+pub fn global_service() -> Router {
+    crate::git_sync_ee::global_service()
+}

backend/windmill-api/src/indexer_oss.rs

@@ -0,0 +1,9 @@
+use axum::Router;
+
+pub fn workspaced_service() -> Router {
+    crate::indexer_ee::workspaced_service()
+}
+
+pub fn global_service() -> Router {
+    crate::indexer_ee::global_service()
+}

backend/windmill-api/src/job_helpers_oss.rs

@@ -0,0 +1,89 @@
+use crate::db::DB;
+use axum::{Router, response::Response};
+use serde::{Deserialize, Serialize};
+use windmill_common::{error, object_store::ObjectStoreResource};
+
+#[derive(Serialize, Deserialize)]
+pub struct UploadFileResponse {
+    pub file_key: String,
+}
+
+#[derive(Deserialize)]
+pub struct LoadImagePreviewQuery;
+
+#[derive(Deserialize)]
+pub struct DownloadFileQuery;
+
+pub fn workspaced_service() -> Router {
+    crate::job_helpers_ee::workspaced_service()
+}
+
+pub async fn get_workspace_s3_resource(
+    _authed: &windmill_common::users::Authed,
+    _w_id: &str,
+    _db: &DB,
+) -> windmill_common::error::Result<(Option<bool>, Option<ObjectStoreResource>)> {
+    crate::job_helpers_ee::get_workspace_s3_resource(_authed, _w_id, _db).await
+}
+
+pub fn get_random_file_name(_file_extension: Option<String>) -> String {
+    crate::job_helpers_ee::get_random_file_name(_file_extension)
+}
+
+pub async fn get_s3_resource(
+    _authed: &windmill_common::users::Authed,
+    _w_id: &str,
+    _db: &DB,
+    _disable_s3_internal: bool,
+) -> error::Result<ObjectStoreResource> {
+    crate::job_helpers_ee::get_s3_resource(_authed, _w_id, _db, _disable_s3_internal).await
+}
+
+pub async fn upload_file_from_req(
+    _authed: windmill_common::users::Authed,
+    _w_id: axum::extract::Path<String>,
+    _db: DB,
+    _req: axum::extract::Request,
+) -> error::Result<()> {
+    crate::job_helpers_ee::upload_file_from_req(_authed, _w_id, _db, _req).await
+}
+
+pub async fn upload_file_internal(
+    _authed: &windmill_common::users::Authed,
+    _w_id: &str,
+    _db: &DB,
+    _file_content: bytes::Bytes,
+    _file_extension: Option<String>,
+    _file_name: Option<String>,
+    _s3_resource_opt: Option<ObjectStoreResource>,
+    _disable_s3_internal: bool,
+) -> error::Result<()> {
+    crate::job_helpers_ee::upload_file_internal(
+        _authed,
+        _w_id,
+        _db,
+        _file_content,
+        _file_extension,
+        _file_name,
+        _s3_resource_opt,
+        _disable_s3_internal,
+    ).await
+}
+
+pub async fn download_s3_file_internal(
+    _authed: &windmill_common::users::Authed,
+    _w_id: &str,
+    _db: &DB,
+    _file_key: &str,
+    _s3_resource_opt: Option<ObjectStoreResource>,
+    _disable_s3_internal: bool,
+) -> error::Result<Response> {
+    crate::job_helpers_ee::download_s3_file_internal(
+        _authed,
+        _w_id,
+        _db,
+        _file_key,
+        _s3_resource_opt,
+        _disable_s3_internal,
+    ).await
+}

backend/windmill-api/src/jobs.rs

@@ -83,8 +83,6 @@ use windmill_common::{
     },
 };
 
-#[cfg(all(feature = "enterprise", feature = "parquet"))]
-use windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS;
 #[cfg(feature = "prometheus")]
 use windmill_common::{METRICS_DEBUG_ENABLED, METRICS_ENABLED};
 
@@ -1058,7 +1056,7 @@ async fn get_logs_from_store(
     if log_offset > 0 {
         if let Some(file_index) = log_file_index.clone() {
             tracing::debug!("Getting logs from store: {file_index:?}");
-            if let Some(os) = OBJECT_STORE_CACHE_SETTINGS.read().await.clone() {
+            if let Some(os) = windmill_common::s3_helpers::get_object_store().await {
                 tracing::debug!("object store client present, streaming from there");
 
                 let logs = logs.to_string();
@@ -4962,10 +4960,7 @@ async fn run_bundle_preview_script(
             uploaded = true;
 
             #[cfg(all(feature = "enterprise", feature = "parquet"))]
-            let object_store = windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS
-                .read()
-                .await
-                .clone();
+            let object_store = windmill_common::s3_helpers::get_object_store().await;
 
             #[cfg(not(all(feature = "enterprise", feature = "parquet")))]
             let object_store: Option<()> = None;
@@ -5663,7 +5658,7 @@ async fn get_log_file(Path((_w_id, file_p)): Path<(String, String)>) -> error::R
     }
 
     #[cfg(all(feature = "enterprise", feature = "parquet"))]
-    if let Some(os) = OBJECT_STORE_CACHE_SETTINGS.read().await.clone() {
+    if let Some(os) = windmill_common::s3_helpers::get_object_store().await {
         let file = os
             .get(&object_store::path::Path::from(format!("logs/{file_p}")))
             .await;

backend/windmill-api/src/kafka_triggers_oss.rs

@@ -0,0 +1,13 @@
+use crate::db::DB;
+use axum::Router;
+
+pub fn workspaced_service() -> Router {
+    crate::kafka_triggers_ee::workspaced_service()
+}
+
+pub fn start_kafka_consumers(
+    _db: DB,
+    mut _killpill_rx: tokio::sync::broadcast::Receiver<()>,
+) -> () {
+    crate::kafka_triggers_ee::start_kafka_consumers(_db, _killpill_rx)
+}

backend/windmill-api/src/nats_triggers_oss.rs

@@ -0,0 +1,38 @@
+use crate::db::DB;
+use axum::Router;
+use serde::{Deserialize, Serialize};
+
+#[derive(Serialize, Deserialize)]
+pub struct NatsResourceAuth {}
+
+#[derive(Serialize, Deserialize)]
+pub enum NatsTriggerConfigConnection {}
+
+#[derive(Serialize, Clone)]
+pub struct NatsTrigger {
+    pub workspace_id: String,
+    pub path: String,
+    pub nats_resource_path: String,
+    pub subject: String,
+    pub script_path: String,
+    pub is_flow: bool,
+    pub edited_by: String,
+    pub email: String,
+    pub edited_at: chrono::DateTime<chrono::Utc>,
+    pub server_id: Option<String>,
+    pub last_server_ping: Option<chrono::DateTime<chrono::Utc>>,
+    pub extra_perms: serde_json::Value,
+    pub error: Option<String>,
+    pub enabled: bool,
+}
+
+pub fn workspaced_service() -> Router {
+    crate::nats_triggers_ee::workspaced_service()
+}
+
+pub fn start_nats_consumers(
+    _db: DB,
+    mut _killpill_rx: tokio::sync::broadcast::Receiver<()>,
+) -> () {
+    crate::nats_triggers_ee::start_nats_consumers(_db, _killpill_rx)
+}

backend/windmill-api/src/oauth2_oss.rs

@@ -0,0 +1,60 @@
+use crate::db::DB;
+use axum::Router;
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+use windmill_common::error;
+
+#[derive(Serialize, Deserialize)]
+pub struct ClientWithScopes;
+
+pub type BasicClientsMap = HashMap<String, ClientWithScopes>;
+
+#[derive(Serialize, Deserialize)]
+pub struct OAuthConfig;
+
+#[derive(Serialize, Deserialize)]
+pub struct OAuthClient;
+
+#[derive(Serialize, Deserialize)]
+pub struct AllClients;
+
+#[derive(Serialize, Deserialize)]
+pub struct TokenResponse;
+
+pub struct SlackVerifier;
+
+impl SlackVerifier {
+    pub fn new<S: AsRef<[u8]>>(secret: S) -> anyhow::Result<SlackVerifier> {
+        crate::oauth2_ee::SlackVerifier::new(secret)
+    }
+}
+
+pub fn global_service() -> Router {
+    crate::oauth2_ee::global_service()
+}
+
+pub fn workspaced_service() -> Router {
+    crate::oauth2_ee::workspaced_service()
+}
+
+pub async fn build_oauth_clients(
+    _db: &DB,
+    _w_id: &str,
+    _base_url: &str,
+) -> anyhow::Result<AllClients> {
+    crate::oauth2_ee::build_oauth_clients(_db, _w_id, _base_url).await
+}
+
+pub async fn _refresh_token(
+    _workspace_id: &str,
+    _path: &str,
+    _db: &DB,
+    _token: &str,
+    _http_client: &reqwest::Client,
+) -> error::Result<String> {
+    crate::oauth2_ee::_refresh_token(_workspace_id, _path, _db, _token, _http_client).await
+}
+
+pub async fn check_nb_of_user(db: &DB) -> error::Result<()> {
+    crate::oauth2_ee::check_nb_of_user(db).await
+}

backend/windmill-api/src/oidc_oss.rs

@@ -0,0 +1,9 @@
+use axum::Router;
+
+pub fn global_service() -> Router {
+    crate::oidc_ee::global_service()
+}
+
+pub fn workspaced_service() -> Router {
+    crate::oidc_ee::workspaced_service()
+}

backend/windmill-api/src/saml_oss.rs

@@ -0,0 +1,15 @@
+use axum::Router;
+
+pub struct ServiceProviderExt();
+
+pub async fn build_sp_extension() -> anyhow::Result<ServiceProviderExt> {
+    crate::saml_ee::build_sp_extension().await
+}
+
+pub fn global_service() -> Router {
+    crate::saml_ee::global_service()
+}
+
+pub async fn acs() -> String {
+    crate::saml_ee::acs().await
+}

backend/windmill-api/src/scim_oss.rs

@@ -0,0 +1,13 @@
+use axum::{http::Request, middleware::Next, response::Response, Router};
+
+pub fn global_service() -> Router {
+    crate::scim_ee::global_service()
+}
+
+pub async fn ee() -> String {
+    crate::scim_ee::ee().await
+}
+
+pub async fn has_scim_token<B>(_request: Request<B>, _next: Next) -> Response {
+    crate::scim_ee::has_scim_token(_request, _next).await
+}

backend/windmill-api/src/scripts.rs

@@ -410,10 +410,7 @@ async fn create_snapshot_script(
             uploaded = true;
 
             #[cfg(all(feature = "enterprise", feature = "parquet"))]
-            let object_store = windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS
-                .read()
-                .await
-                .clone();
+            let object_store = windmill_common::s3_helpers::get_object_store().await;
 
             #[cfg(not(all(feature = "enterprise", feature = "parquet")))]
             let object_store: Option<()> = None;
@@ -1327,10 +1324,12 @@ async fn raw_script_by_path_internal(
             w_id
         )
         .fetch_one(&db)
-        .await?;
-        if exists.unwrap_or(false) {
+        .await?
+        .unwrap_or(false);
+
+        if exists {
             return Err(Error::NotFound(format!(
-                "Script {path} not visible to {} but exists",
+                "Script {path} exists but {} does not have permissions to access it",
                 authed.username
             )));
         }

backend/windmill-api/src/service_logs.rs

@@ -98,10 +98,7 @@ async fn get_log_file(
     require_devops_role(&db, &email).await?;
     let path = path.to_path();
     #[cfg(feature = "parquet")]
-    let s3_client = windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS
-        .read()
-        .await
-        .clone();
+    let s3_client = windmill_common::s3_helpers::get_object_store().await;
     #[cfg(feature = "parquet")]
     if let Some(s3_client) = s3_client {
         let path = format!("{}{}", windmill_common::tracing_init::LOGS_SERVICE, path);

backend/windmill-api/src/settings.rs

@@ -120,12 +120,15 @@ use windmill_common::s3_helpers::build_object_store_from_settings;
 #[cfg(feature = "parquet")]
 pub async fn test_s3_bucket(
     _authed: ApiAuthed,
+    Extension(db): Extension<DB>,
     Json(test_s3_bucket): Json<ObjectSettings>,
 ) -> error::Result<String> {
     use bytes::Bytes;
     use futures::StreamExt;
 
-    let client = build_object_store_from_settings(test_s3_bucket).await?;
+    let client = build_object_store_from_settings(test_s3_bucket, Some(&db))
+        .await?
+        .store;
 
     let mut list = client.list(Some(&object_store::path::Path::from("".to_string())));
     let first_file = list.next().await;

backend/windmill-api/src/smtp_server_oss.rs

@@ -0,0 +1,23 @@
+use crate::{db::DB, users::AuthCache, users::UserDB};
+use std::{net::SocketAddr, sync::Arc};
+
+pub struct SmtpServer {
+    pub auth_cache: Arc<AuthCache>,
+    pub db: DB,
+    pub user_db: UserDB,
+    pub base_internal_url: String,
+}
+
+impl SmtpServer {
+    pub async fn start_listener_thread(
+        self: Arc<Self>,
+        _addr: SocketAddr,
+    ) -> anyhow::Result<()> {
+        crate::smtp_server_ee::SmtpServer {
+            auth_cache: self.auth_cache,
+            db: self.db,
+            user_db: self.user_db,
+            base_internal_url: self.base_internal_url,
+        }.start_listener_thread(_addr).await
+    }
+}

backend/windmill-api/src/sqs_triggers_oss.rs

@@ -0,0 +1,32 @@
+use crate::db::DB;
+use axum::Router;
+use serde::{Deserialize, Serialize};
+
+#[derive(Serialize, Clone)]
+pub struct SqsTrigger {
+    pub workspace_id: String,
+    pub path: String,
+    pub sqs_resource_path: String,
+    pub queue_url: String,
+    pub script_path: String,
+    pub is_flow: bool,
+    pub edited_by: String,
+    pub email: String,
+    pub edited_at: chrono::DateTime<chrono::Utc>,
+    pub server_id: Option<String>,
+    pub last_server_ping: Option<chrono::DateTime<chrono::Utc>>,
+    pub extra_perms: serde_json::Value,
+    pub error: Option<String>,
+    pub enabled: bool,
+}
+
+pub fn workspaced_service() -> Router {
+    crate::sqs_triggers_ee::workspaced_service()
+}
+
+pub fn start_sqs(
+    _db: DB,
+    mut _killpill_rx: tokio::sync::broadcast::Receiver<()>,
+) -> () {
+    crate::sqs_triggers_ee::start_sqs(_db, _killpill_rx)
+}

backend/windmill-api/src/stripe_oss.rs

@@ -0,0 +1,5 @@
+use axum::Router;
+
+pub fn add_stripe_routes(router: Router) -> Router {
+    crate::stripe_ee::add_stripe_routes(router)
+}

backend/windmill-api/src/teams_approvals_oss.rs

@@ -0,0 +1,6 @@
+use axum::http::StatusCode;
+use windmill_common::error::Error;
+
+pub async fn request_teams_approval() -> Result<StatusCode, Error> {
+    crate::teams_approvals_ee::request_teams_approval().await
+}

backend/windmill-api/src/teams_oss.rs

@@ -0,0 +1,26 @@
+use axum::{http::StatusCode, Router};
+use windmill_common::error::Error;
+
+pub async fn edit_teams_command() -> Result<StatusCode, Error> {
+    crate::teams_ee::edit_teams_command().await
+}
+
+pub async fn workspaces_list_available_teams_ids() -> Result<StatusCode, Error> {
+    crate::teams_ee::workspaces_list_available_teams_ids().await
+}
+
+pub async fn connect_teams() -> Result<StatusCode, Error> {
+    crate::teams_ee::connect_teams().await
+}
+
+pub async fn run_teams_message_test_job() -> Result<StatusCode, Error> {
+    crate::teams_ee::run_teams_message_test_job().await
+}
+
+pub async fn workspaces_list_available_teams_channels() -> Result<StatusCode, Error> {
+    crate::teams_ee::workspaces_list_available_teams_channels().await
+}
+
+pub fn teams_service() -> Router {
+    crate::teams_ee::teams_service()
+}

backend/windmill-api/src/users_oss.rs

@@ -0,0 +1,24 @@
+use axum::http::StatusCode;
+use windmill_common::error::Error;
+
+pub async fn create_user(
+    _authed: windmill_common::users::Authed,
+    _extract_path: axum::extract::Path<String>,
+    _db: crate::db::DB,
+    _new_user: axum::extract::Json<serde_json::Value>,
+) -> Result<(StatusCode, String), Error> {
+    crate::users_ee::create_user(_authed, _extract_path, _db, _new_user).await
+}
+
+pub async fn set_password(
+    _authed: windmill_common::users::Authed,
+    _extract_path: axum::extract::Path<String>,
+    _db: crate::db::DB,
+    _set_password: axum::extract::Json<serde_json::Value>,
+) -> Result<String, Error> {
+    crate::users_ee::set_password(_authed, _extract_path, _db, _set_password).await
+}
+
+pub fn send_email_if_possible(_subject: &str, _content: &str, _to: &str) {
+    crate::users_ee::send_email_if_possible(_subject, _content, _to)
+}

backend/windmill-api/src/workspaces_export.rs

@@ -360,6 +360,7 @@ pub(crate) async fn tarball_workspace(
                 ScriptLang::Bigquery => "bq.sql",
                 ScriptLang::Snowflake => "sf.sql",
                 ScriptLang::Mssql => "ms.sql",
+                ScriptLang::DuckDb => "duckdb.sql",
                 ScriptLang::Graphql => "gql",
                 ScriptLang::Nativets => "fetch.ts",
                 ScriptLang::Bun | ScriptLang::Bunnative => {

backend/windmill-api/src/workspaces_oss.rs

@@ -0,0 +1,15 @@
+use crate::db::DB;
+use windmill_common::users::ApiAuthed;
+use serde::Deserialize;
+
+#[derive(Deserialize)]
+pub struct EditAutoInvite;
+
+pub async fn edit_auto_invite(
+    _authed: ApiAuthed,
+    _db: DB,
+    _w_id: String,
+    _ea: EditAutoInvite,
+) -> windmill_common::error::Result<String> {
+    crate::workspaces_ee::edit_auto_invite(_authed, _db, _w_id, _ea).await
+}

backend/windmill-audit/src/audit_oss.rs

@@ -0,0 +1,67 @@
+use serde::{Deserialize, Serialize};
+use sqlx::{Postgres, Transaction};
+use windmill_common::error::Result;
+
+#[derive(Serialize, Deserialize)]
+pub struct AuditAuthor {
+    pub username: String,
+    pub email: String,
+    pub username_override: Option<String>,
+}
+
+pub trait AuditAuthorable {
+    fn username(&self) -> &str;
+    fn email(&self) -> &str;
+    fn username_override(&self) -> Option<&str>;
+}
+
+impl AuditAuthorable for AuditAuthor {
+    fn username(&self) -> &str {
+        &self.username
+    }
+
+    fn email(&self) -> &str {
+        &self.email
+    }
+
+    fn username_override(&self) -> Option<&str> {
+        self.username_override.as_deref()
+    }
+}
+
+pub struct AuditLog;
+
+pub async fn audit_log<A: AuditAuthorable>(
+    _tx: &mut Transaction<'_, Postgres>,
+    _author: &A,
+    _operation: &str,
+    _action_kind: &str,
+    _workspace_id: &str,
+    _resource: Option<&str>,
+    _parameters: Option<serde_json::Map<String, serde_json::Value>>,
+) -> Result<()> {
+    crate::audit_ee::audit_log(_tx, _author, _operation, _action_kind, _workspace_id, _resource, _parameters).await
+}
+
+pub async fn list_audit(
+    _tx: Transaction<'_, Postgres>,
+    _workspace_id: &str,
+    _username: Option<String>,
+    _operation: Option<String>,
+    _action_kind: Option<String>,
+    _resource: Option<String>,
+    _before: Option<chrono::DateTime<chrono::Utc>>,
+    _after: Option<chrono::DateTime<chrono::Utc>>,
+    _per_page: usize,
+    _offset: usize,
+) -> Result<Vec<AuditLog>> {
+    crate::audit_ee::list_audit(_tx, _workspace_id, _username, _operation, _action_kind, _resource, _before, _after, _per_page, _offset).await
+}
+
+pub async fn get_audit(
+    tx: Transaction<'_, Postgres>,
+    _id: i32,
+    _w_id: &str,
+) -> Result<AuditLog> {
+    crate::audit_ee::get_audit(tx, _id, _w_id).await
+}

backend/windmill-autoscaling/src/autoscaling_oss.rs

@@ -0,0 +1,5 @@
+use windmill_common::DB;
+
+pub async fn apply_all_autoscaling(_db: &DB) -> anyhow::Result<()> {
+    crate::autoscaling_ee::apply_all_autoscaling(_db).await
+}

backend/windmill-common/Cargo.toml

@@ -12,14 +12,14 @@ tantivy = []
 prometheus = ["dep:prometheus"]
 loki = ["dep:tracing-loki"]
 benchmark = []
-parquet = ["dep:object_store", "dep:aws-config", "dep:aws-sdk-sts", "dep:datafusion"]
+parquet = ["dep:object_store", "dep:aws-config", "dep:aws-sdk-sts", "dep:aws-smithy-types-convert", "dep:datafusion"]
 aws_auth = ["dep:aws-sdk-sts", "dep:aws-config"]
 otel = ["dep:opentelemetry-semantic-conventions", "dep:opentelemetry-otlp", "dep:opentelemetry_sdk", 
     "dep:opentelemetry", "dep:tracing-opentelemetry", "dep:opentelemetry-appender-tracing", "dep:tonic"]
 smtp = ["dep:mail-send"]
 scoped_cache = []
 cloud = []
-
+openidconnect = ["dep:openidconnect"]
 [lib]
 name = "windmill_common"
 path = "src/lib.rs"
@@ -62,6 +62,7 @@ object_store = { workspace = true, optional = true }
 prometheus = { workspace = true, optional = true }
 aws-config = { workspace = true, optional = true }
 aws-sdk-sts = { workspace = true, optional = true }
+aws-smithy-types-convert = { workspace = true, optional = true }
 indexmap.workspace = true
 bytes.workspace = true
 mail-send = { workspace = true, optional = true }
@@ -75,6 +76,7 @@ windmill-parser-ts.workspace = true
 windmill-parser-py.workspace = true
 jsonwebtoken.workspace = true
 backon.workspace = true
+openidconnect = { workspace = true, optional = true }
 strum.workspace = true
 strum_macros.workspace = true
 

backend/windmill-common/src/client.rs

@@ -0,0 +1,244 @@
+use anyhow::Context;
+use reqwest::{Body, Response};
+use serde::de::DeserializeOwned;
+
+use crate::{
+    error::{self, to_anyhow},
+    s3_helpers::{DuckdbConnectionSettingsQueryV2, DuckdbConnectionSettingsResponse},
+    utils::HTTP_CLIENT,
+};
+
+#[derive(Clone)]
+pub struct AuthedClient {
+    pub base_internal_url: String,
+    pub workspace: String,
+    pub token: String,
+    pub force_client: Option<reqwest::Client>,
+}
+
+impl AuthedClient {
+    pub async fn get(&self, url: &str, query: Vec<(&str, String)>) -> anyhow::Result<Response> {
+        self.force_client
+            .as_ref()
+            .unwrap_or(&HTTP_CLIENT)
+            .get(url)
+            .query(&query)
+            .header(
+                reqwest::header::ACCEPT,
+                reqwest::header::HeaderValue::from_static("application/json"),
+            )
+            .header(
+                reqwest::header::AUTHORIZATION,
+                reqwest::header::HeaderValue::from_str(&format!("Bearer {}", self.token))?,
+            )
+            .send()
+            .await
+            .map_err(|e| {
+                tracing::error!("Error executing get request from authed http client to {url} with query {query:?}: {e}");
+                anyhow::anyhow!("Error executing get request from authed http client to {url} with query {query:?}: {e}")
+            })
+    }
+
+    pub async fn get_id_token(&self, audience: &str) -> anyhow::Result<String> {
+        let url = format!(
+            "{}/api/w/{}/oidc/token/{}",
+            self.base_internal_url, self.workspace, audience
+        );
+        let response = self.get(&url, vec![]).await?;
+        match response.status().as_u16() {
+            200u16 => Ok(response
+                .json::<String>()
+                .await
+                .context("decoding oidc token as json string")?),
+            _ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())),
+        }
+    }
+
+    pub async fn get_resource_value<T: DeserializeOwned>(&self, path: &str) -> anyhow::Result<T> {
+        let url = format!(
+            "{}/api/w/{}/resources/get_value/{}",
+            self.base_internal_url, self.workspace, path
+        );
+        let response = self.get(&url, vec![]).await?;
+        match response.status().as_u16() {
+            200u16 => Ok(response
+                .json::<T>()
+                .await
+                .context("decoding resource value as json")?),
+            _ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())),
+        }
+    }
+
+    pub async fn get_variable_value(&self, path: &str) -> anyhow::Result<String> {
+        let url = format!(
+            "{}/api/w/{}/variables/get_value/{}",
+            self.base_internal_url, self.workspace, path
+        );
+        let response = self.get(&url, vec![]).await?;
+        match response.status().as_u16() {
+            200u16 => Ok(response
+                .json::<String>()
+                .await
+                .context("decoding variable value as json")?),
+            _ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())),
+        }
+    }
+
+    pub async fn get_resource_value_interpolated<T: DeserializeOwned>(
+        &self,
+        path: &str,
+        job_id: Option<String>,
+    ) -> anyhow::Result<T> {
+        let url = format!(
+            "{}/api/w/{}/resources/get_value_interpolated/{}",
+            self.base_internal_url, self.workspace, path
+        );
+        let mut query = Vec::with_capacity(1usize);
+        if let Some(v) = &job_id {
+            query.push(("job_id", v.to_string()));
+        }
+        let response = self.get(&url, query).await?;
+        match response.status().as_u16() {
+            200u16 => Ok(response
+                .json::<T>()
+                .await
+                .context("decoding interpolated resource value as json")?),
+            _ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())),
+        }
+    }
+
+    pub async fn get_completed_job_result<T: DeserializeOwned>(
+        &self,
+        path: &str,
+        json_path: Option<String>,
+    ) -> anyhow::Result<T> {
+        let url = format!(
+            "{}/api/w/{}/jobs_u/completed/get_result/{}",
+            self.base_internal_url, self.workspace, path
+        );
+        let query = if let Some(json_path) = json_path {
+            vec![("json_path", json_path)]
+        } else {
+            vec![]
+        };
+        let response = self.get(&url, query).await?;
+        match response.status().as_u16() {
+            200u16 => Ok(response
+                .json::<T>()
+                .await
+                .context("decoding completed job result as json")?),
+            _ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())),
+        }
+    }
+
+    pub async fn get_result_by_id<T: DeserializeOwned>(
+        &self,
+        flow_job_id: &str,
+        node_id: &str,
+        json_path: Option<String>,
+    ) -> anyhow::Result<T> {
+        let url = format!(
+            "{}/api/w/{}/jobs/result_by_id/{}/{}",
+            self.base_internal_url, self.workspace, flow_job_id, node_id
+        );
+        let query = if let Some(json_path) = json_path {
+            vec![("json_path", json_path)]
+        } else {
+            vec![]
+        };
+        let response = self.get(&url, query).await?;
+        match response.status().as_u16() {
+            200u16 => Ok(response
+                .json::<T>()
+                .await
+                .context("decoding result by id as json")?),
+            _ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())),
+        }
+    }
+
+    pub async fn upload_s3_file<S>(
+        &self,
+        workspace_id: &str,
+        object_key: String,
+        storage: Option<String>,
+        body: S,
+    ) -> anyhow::Result<()>
+    where
+        S: futures::stream::TryStream + Send + 'static,
+        S::Error: Into<Box<dyn std::error::Error + Send + Sync>>,
+        bytes::Bytes: From<S::Ok>,
+    {
+        let mut query = vec![("file_key", object_key)];
+        if let Some(storage) = storage {
+            query.push(("storage", storage));
+        }
+        let response = self
+            .force_client
+            .as_ref()
+            .unwrap_or(&HTTP_CLIENT)
+            .post(format!(
+                "{}/api/w/{}/job_helpers/upload_s3_file",
+                self.base_internal_url, workspace_id
+            ))
+            .query(&query)
+            .header(
+                reqwest::header::ACCEPT,
+                reqwest::header::HeaderValue::from_static("application/json"),
+            )
+            .header(
+                reqwest::header::AUTHORIZATION,
+                reqwest::header::HeaderValue::from_str(&format!("Bearer {}", self.token))
+                    .map_err(|e| anyhow::anyhow!(e.to_string()))?,
+            )
+            .body(Body::wrap_stream(body))
+            .send()
+            .await
+            .context(format!("Sent upload_s3_file request",))
+            .map_err(|e| anyhow::anyhow!(e.to_string()))?;
+
+        match response.status().as_u16() {
+            200u16 => Ok(()),
+            _ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default()))?,
+        }
+    }
+
+    pub async fn get_duckdb_connection_settings(
+        &self,
+        s3: &DuckdbConnectionSettingsQueryV2,
+    ) -> error::Result<DuckdbConnectionSettingsResponse> {
+        let url = format!(
+            "{}/api/w/{}/job_helpers/v2/duckdb_connection_settings",
+            self.base_internal_url, &self.workspace
+        );
+        let response = self
+            .force_client
+            .as_ref()
+            .unwrap_or(&HTTP_CLIENT)
+            .post(url)
+            .header(
+                reqwest::header::CONTENT_TYPE,
+                reqwest::header::HeaderValue::from_static("application/json"),
+            )
+            .header(
+                reqwest::header::ACCEPT,
+                reqwest::header::HeaderValue::from_static("application/json"),
+            )
+            .header(
+                reqwest::header::AUTHORIZATION,
+                reqwest::header::HeaderValue::from_str(&format!("Bearer {}", self.token))
+                    .map_err(|e| error::Error::BadConfig(e.to_string()))?,
+            )
+            .body(serde_json::to_string(&s3).map_err(to_anyhow)?)
+            .send()
+            .await
+            .context(format!("Sent get_duckdb_connection_settings request",))
+            .map_err(error::Error::from)?;
+        match response.status().as_u16() {
+            200u16 => Ok(response
+                .json::<DuckdbConnectionSettingsResponse>()
+                .await
+                .context("decoding duckdb_connection_settings response as json")?),
+            _ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default()))?,
+        }
+    }
+}

backend/windmill-common/src/email_oss.rs

@@ -0,0 +1,12 @@
+use crate::{error::Result, DB};
+
+pub async fn send_email(
+    _to: &str,
+    _subject: &str,
+    _body: &str,
+    _html_body: Option<&str>,
+    _workspace_id: &str,
+    _db: &DB,
+) -> Result<()> {
+    crate::email_ee::send_email(_to, _subject, _body, _html_body, _workspace_id, _db).await
+}

backend/windmill-common/src/global_settings.rs

@@ -30,7 +30,7 @@ pub const EXPOSE_METRICS_SETTING: &str = "expose_metrics";
 pub const EXPOSE_DEBUG_METRICS_SETTING: &str = "expose_debug_metrics";
 pub const KEEP_JOB_DIR_SETTING: &str = "keep_job_dir";
 pub const REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING: &str = "require_preexisting_user_for_oauth";
-pub const OBJECT_STORE_CACHE_CONFIG_SETTING: &str = "object_store_cache_config";
+pub const OBJECT_STORE_CONFIG_SETTING: &str = "object_store_cache_config";
 
 pub const AUTOMATE_USERNAME_CREATION_SETTING: &str = "automate_username_creation";
 pub const HUB_BASE_URL_SETTING: &str = "hub_base_url";

backend/windmill-common/src/job_s3_helpers_ee.rs

@@ -1,18 +1,34 @@
-use std::future::Future;
+use crate::s3_helpers::{ObjectStoreResource, StorageResourceType};
 
-use crate::{
-    error::Error,
-    s3_helpers::{ObjectStoreResource, StorageResourceType},
-};
-
-pub async fn get_s3_resource_internal<'c, F, Fut>(
+pub async fn get_s3_resource_internal<'c>(
     _resource_type: StorageResourceType,
     _s3_resource_value_raw: serde_json::Value,
-    _gen_token: F,
-) -> crate::error::Result<ObjectStoreResource>
-where
-    F: FnOnce(String) -> Fut,
-    Fut: Future<Output = Result<String, Error>> + Send + 'static,
-{
+    _gen_token: TokenGenerator<'c>,
+    _db: &crate::DB,
+) -> crate::error::Result<ObjectStoreResource> {
+    todo!()
+}
+
+pub enum TokenGenerator<'c> {
+    AsClient(&'c crate::client::AuthedClient),
+    AsServerInstance(),
+}
+
+impl<'c> TokenGenerator<'c> {
+    pub async fn gen_token(
+        &self,
+        _audience: &str,
+        _db: Option<&crate::DB>,
+    ) -> anyhow::Result<String> {
+        todo!()
+    }
+}
+
+#[cfg(feature = "parquet")]
+pub(crate) async fn generate_s3_aws_oidc_resource<'c>(
+    _clone: crate::s3_helpers::S3AwsOidcResource,
+    _token_generator: TokenGenerator<'c>,
+    _init_private_key: Option<&sqlx::Pool<sqlx::Postgres>>,
+) -> crate::error::Result<ObjectStoreResource> {
     todo!()
 }

backend/windmill-common/src/job_s3_helpers_oss.rs

@@ -0,0 +1,32 @@
+use crate::{error::Result, object_store::ObjectStoreResource, DB};
+
+pub enum TokenGenerator<'c> {
+    AsClient(&'c crate::client::AuthedClient),
+    AsServerInstance(),
+}
+
+impl<'c> TokenGenerator<'c> {
+    pub async fn gen_token(&self, _audience: &str, _db: Option<&DB>) -> anyhow::Result<String> {
+        crate::job_s3_helpers_ee::TokenGenerator::gen_token(self, _audience, _db).await
+    }
+}
+
+pub async fn get_s3_resource_internal(
+    _token_generator: &TokenGenerator<'_>,
+    _audience: &str,
+    _workspace_id: &str,
+    _resource_path: &str,
+    _db: Option<&DB>,
+) -> Result<ObjectStoreResource> {
+    crate::job_s3_helpers_ee::get_s3_resource_internal(_token_generator, _audience, _workspace_id, _resource_path, _db).await
+}
+
+pub(crate) async fn generate_s3_aws_oidc_resource(
+    _token_generator: &TokenGenerator<'_>,
+    _audience: &str,
+    _role_arn: &str,
+    _region: &str,
+    _db: Option<&DB>,
+) -> Result<ObjectStoreResource> {
+    crate::job_s3_helpers_ee::generate_s3_aws_oidc_resource(_token_generator, _audience, _role_arn, _region, _db).await
+}

backend/windmill-common/src/jobs.rs

@@ -608,11 +608,11 @@ pub async fn get_logs_from_store(
     logs: &str,
     log_file_index: &Option<Vec<String>>,
 ) -> Option<impl Stream<Item = Result<Bytes, object_store::Error>>> {
-    use crate::s3_helpers::OBJECT_STORE_CACHE_SETTINGS;
+    use crate::s3_helpers::get_object_store;
 
     if log_offset > 0 {
         if let Some(file_index) = log_file_index.clone() {
-            if let Some(os) = OBJECT_STORE_CACHE_SETTINGS.read().await.clone() {
+            if let Some(os) = get_object_store().await {
                 let logs = logs.to_string();
                 let stream = async_stream::stream! {
                     for file_p in file_index.clone() {

backend/windmill-common/src/lib.rs

@@ -30,6 +30,7 @@ pub mod auth;
 #[cfg(feature = "benchmark")]
 pub mod bench;
 pub mod cache;
+pub mod client;
 pub mod db;
 pub mod ee;
 pub mod email_ee;
@@ -43,6 +44,9 @@ pub mod job_metrics;
 #[cfg(feature = "parquet")]
 pub mod job_s3_helpers_ee;
 
+#[cfg(all(feature = "enterprise", feature = "openidconnect"))]
+pub mod oidc_ee;
+
 pub mod jobs;
 pub mod jwt;
 pub mod more_serde;

backend/windmill-common/src/oidc_ee.rs

@@ -0,0 +1,198 @@
+/*
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2023
+ * This file and its contents are licensed under the AGPLv3 License.
+ * Please see the included NOTICE for copyright information and
+ * LICENSE-AGPL for a copy of the license.
+ */
+
+use serde::{Deserialize, Serialize};
+use tokio::sync::RwLock;
+#[cfg(all(feature = "enterprise", feature = "openidconnect"))]
+use {
+    crate::db::DB,
+    crate::{auth::IdToken as WindmillIdToken, error::Result},
+    anyhow,
+    openidconnect::{
+        core::{CoreJwsSigningAlgorithm, CoreRsaPrivateSigningKey},
+        IssuerUrl, JsonWebKeyId,
+    },
+    std::process::Command,
+};
+
+#[cfg(feature = "openidconnect")]
+use openidconnect::AdditionalClaims;
+
+#[cfg(feature = "openidconnect")]
+impl AdditionalClaims for JobClaim {}
+
+#[cfg(feature = "openidconnect")]
+impl AdditionalClaims for WorkspaceClaim {}
+
+#[cfg(feature = "openidconnect")]
+impl AdditionalClaims for InstanceClaim {}
+
+#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq, Serialize)]
+pub struct WorkspaceClaim {
+    pub workspace: String,
+}
+
+#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq, Serialize)]
+pub struct InstanceClaim {}
+
+#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq, Serialize)]
+pub struct JobClaim {
+    pub job_id: String,
+    pub path: Option<String>,
+    pub flow_path: Option<String>,
+    pub groups: Vec<String>,
+    pub username: String,
+    pub email: String,
+    pub workspace: String,
+}
+
+lazy_static::lazy_static! {
+    static ref PRIVATE_KEY: RwLock<Option<String>> = RwLock::new(None);
+}
+
+pub async fn generate_id_token<T: AdditionalClaims>(
+    db: Option<&DB>,
+    claim: T,
+    audience: &str,
+    identifier: String,
+    email: Option<String>,
+) -> Result<WindmillIdToken> {
+    use chrono::{Duration, Utc};
+    use openidconnect::{
+        core::{CoreGenderClaim, CoreJweContentEncryptionAlgorithm},
+        Audience, EndUserEmail, IdToken, IdTokenClaims, StandardClaims, SubjectIdentifier,
+    };
+
+    let private_key = get_private_key(db).await?;
+
+    let issue_url = format!("{}/api/oidc/", crate::BASE_URL.read().await.clone());
+    let issue_time = Utc::now();
+    let expiration = issue_time + Duration::try_hours(48).unwrap();
+    let id_token = IdToken::<
+        T,
+        CoreGenderClaim,
+        CoreJweContentEncryptionAlgorithm,
+        CoreJwsSigningAlgorithm,
+    >::new(
+        IdTokenClaims::<T, CoreGenderClaim>::new(
+            // Specify the issuer URL for the OpenID Connect Provider.
+            IssuerUrl::new(issue_url)
+                .map_err(|e| anyhow::anyhow!("Failed to generate IssueUrl: {}", e))?,
+            // The audience is usually a single entry with the client ID of the client for whom
+            // the ID token is intended. This is a required claim.
+            vec![Audience::new(audience.to_string())],
+            // The ID token expiration is usually much shorter than that of the access or refresh
+            // tokens issued to clients.
+            expiration,
+            // The issue time is usually the current time.
+            issue_time,
+            // Set the standard claims defined by the OpenID Connect Core spec.
+            StandardClaims::new(
+                // Stable subject identifiers are recommended in place of e-mail addresses or other
+                // potentially unstable identifiers. This is the only required claim.
+                SubjectIdentifier::new(identifier),
+            )
+            // Optional: specify the user's e-mail address. This should only be provided if the
+            // client has been granted the 'profile' or 'email' scopes.
+            .set_email(email.map(|x| EndUserEmail::new(x)))
+            // Optional: specify whether the provider has verified the user's e-mail address.
+            .set_email_verified(Some(true)),
+            // OpenID Connect Providers may supply custom claims by providing a struct that
+            // implements the AdditionalClaims trait. This requires manually using the
+            // generic IdTokenClaims struct rather than the CoreIdTokenClaims type alias,
+            // however.
+            claim,
+        ),
+        // The private key used for signing the ID token. For confidential clients (those able
+        // to maintain a client secret), a CoreHmacKey can also be used, in conjunction
+        // with one of the CoreJwsSigningAlgorithm::HmacSha* signing algorithms. When using an
+        // HMAC-based signing algorithm, the UTF-8 representation of the client secret should
+        // be used as the HMAC key.
+        &CoreRsaPrivateSigningKey::from_pem(
+            &private_key,
+            Some(JsonWebKeyId::new("windmill".to_string())),
+        )
+        .map_err(|e| anyhow::anyhow!("Invalid private key: {}", e))?,
+        // Uses the RS256 signature algorithm. This crate supports any RS*, PS*, or HS*
+        // signature algorithm.
+        CoreJwsSigningAlgorithm::RsaSsaPkcs1V15Sha256,
+        // When returning the ID token alongside an access token (e.g., in the Authorization Code
+        // flow), it is recommended to pass the access token here to set the `at_hash` claim
+        // automatically.
+        None,
+        // When returning the ID token alongside an authorization code (e.g., in the implicit
+        // flow), it is recommended to pass the authorization code here to set the `c_hash` claim
+        // automatically.
+        None,
+    )
+    .map_err(|e| anyhow::anyhow!("Failed to generate token: {}", e))?;
+
+    Ok(WindmillIdToken::new(id_token.to_string(), expiration))
+}
+
+#[cfg(all(feature = "enterprise", feature = "openidconnect"))]
+pub async fn get_private_key(db: Option<&DB>) -> anyhow::Result<String> {
+    if let Some(key) = PRIVATE_KEY.read().await.clone() {
+        return Ok(key);
+    } else if let Some(db) = db {
+        let key = sqlx::query_scalar!(
+            "SELECT value->>'private_key' FROM global_settings WHERE name = 'rsa_keys'",
+        )
+        .fetch_optional(db)
+        .await?
+        .flatten();
+
+        let key = key.filter(|s| !s.is_empty());
+
+        if let Some(key) = key {
+            return Ok(key);
+        } else {
+            let keys = gen_pems(db).await?;
+            return Ok(keys.private_key);
+        }
+    } else {
+        return Err(anyhow::anyhow!("Private key not found and no db provided"));
+    }
+}
+
+#[cfg(all(feature = "enterprise", feature = "openidconnect"))]
+#[derive(Debug, Clone, serde::Serialize)]
+struct Keys {
+    private_key: String,
+}
+
+#[cfg(all(feature = "enterprise", feature = "openidconnect"))]
+async fn gen_pems(db: &DB) -> anyhow::Result<Keys> {
+    use anyhow::anyhow;
+
+    let private_key_cmd = Command::new("openssl")
+        .arg("genrsa")
+        .arg("--traditional")
+        .arg("2048")
+        .output()
+        .expect("failed to execute process");
+
+    let private_key = String::from_utf8(private_key_cmd.stdout)?;
+
+    tracing::debug!("Generated private key: {}", private_key);
+
+    if private_key.is_empty() {
+        return Err(anyhow!("Failed to generate RSA key: key is empty"));
+    }
+
+    let keys = Keys { private_key };
+
+    sqlx::query!(
+        r#"INSERT INTO global_settings (name, value) VALUES ('rsa_keys', $1)"#,
+        serde_json::to_value(&keys).unwrap()
+    )
+    .execute(db)
+    .await?;
+
+    Ok(keys)
+}

backend/windmill-common/src/oidc_oss.rs

@@ -0,0 +1,46 @@
+use crate::{error::Result as WindmillResult, DB};
+use serde::{Deserialize, Serialize};
+
+pub trait AdditionalClaims: Serialize + for<'de> Deserialize<'de> {}
+
+#[derive(Serialize, Deserialize)]
+pub struct WorkspaceClaim {
+    pub workspace: String,
+}
+
+impl AdditionalClaims for WorkspaceClaim {}
+
+#[derive(Serialize, Deserialize)]
+pub struct InstanceClaim {}
+
+impl AdditionalClaims for InstanceClaim {}
+
+#[derive(Serialize, Deserialize)]
+pub struct JobClaim {
+    pub workspace: String,
+    pub job: String,
+    pub path: Option<String>,
+    pub groups: Vec<String>,
+    pub email: String,
+    pub username: String,
+    pub is_operator: bool,
+    pub is_admin: bool,
+    pub is_super_admin: bool,
+    pub folders: Vec<String>,
+}
+
+impl AdditionalClaims for JobClaim {}
+
+pub struct WindmillIdToken;
+
+pub async fn generate_id_token<T: AdditionalClaims>(
+    _additional_claims: T,
+    _audience: &str,
+    _db: Option<&DB>,
+) -> anyhow::Result<WindmillIdToken> {
+    crate::oidc_ee::generate_id_token(_additional_claims, _audience, _db).await
+}
+
+pub async fn get_private_key(db: Option<&DB>) -> anyhow::Result<String> {
+    crate::oidc_ee::get_private_key(db).await
+}

backend/windmill-common/src/otel_oss.rs

@@ -0,0 +1,34 @@
+use crate::{jobs::QueuedJob, server::Mode};
+use std::future::Future;
+use tracing_subscriber::filter::EnvFilter;
+use uuid::Uuid;
+
+pub(crate) type OtelProvider = Option<()>;
+
+pub fn set_span_parent(_span: &tracing::Span, _rj: &Uuid) {
+    crate::otel_ee::set_span_parent(_span, _rj)
+}
+
+pub fn otel_ctx() -> () {
+    crate::otel_ee::otel_ctx()
+}
+
+pub(crate) fn init_logs_bridge(_mode: &Mode, _hostname: &str, _env: &str) -> Option<EnvFilter> {
+    crate::otel_ee::init_logs_bridge(_mode, _hostname, _env)
+}
+
+pub(crate) fn init_meter_provider(_mode: &Mode, _hostname: &str, _env: &str) -> OtelProvider {
+    crate::otel_ee::init_meter_provider(_mode, _hostname, _env)
+}
+
+pub fn add_root_flow_job_to_otlp(_queued_job: &QueuedJob, _success: bool) {
+    crate::otel_ee::add_root_flow_job_to_otlp(_queued_job, _success)
+}
+
+pub trait FutureExt: Sized {
+    fn with_context(self, _otel_cx: ()) -> Self {
+        self
+    }
+}
+
+impl<T: Future> FutureExt for T {}

backend/windmill-common/src/s3_helpers.rs

@@ -4,6 +4,7 @@ use crate::error;
 use aws_sdk_sts::config::ProvideCredentials;
 #[cfg(feature = "parquet")]
 use axum::async_trait;
+use chrono::{DateTime, Utc};
 #[cfg(feature = "parquet")]
 use object_store::aws::AwsCredential;
 #[cfg(feature = "parquet")]
@@ -17,6 +18,7 @@ use reqwest::header::HeaderMap;
 use serde::{Deserialize, Serialize};
 #[cfg(feature = "parquet")]
 use std::sync::{Arc, Mutex};
+
 #[cfg(feature = "parquet")]
 use tokio::sync::RwLock;
 
@@ -46,9 +48,170 @@ use tokio::task;
 use windmill_parser_sql::S3ModeFormat;
 
 #[cfg(feature = "parquet")]
-lazy_static::lazy_static! {
+#[derive(Clone)]
+pub struct ExpirableObjectStore {
+    pub store: Arc<dyn ObjectStore>,
+    pub refresh: Option<ObjectStoreRefresh>,
+}
 
-    pub static ref OBJECT_STORE_CACHE_SETTINGS: Arc<RwLock<Option<Arc<dyn ObjectStore>>>> = Arc::new(RwLock::new(None));
+#[cfg(feature = "parquet")]
+#[derive(Clone)]
+pub struct ObjectStoreRefresh {
+    refresh: Option<DateTime<Utc>>,
+    settings: ObjectSettings,
+}
+
+#[cfg(feature = "parquet")]
+impl ObjectStoreRefresh {
+    pub fn new(settings: ObjectSettings, refresh: Option<DateTime<Utc>>) -> Self {
+        Self { settings, refresh }
+    }
+    fn refresh_needed(&self) -> bool {
+        if let Some(refresh) = self.refresh {
+            if refresh < Utc::now() - chrono::Duration::minutes(1) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    async fn refresh(&self) -> Option<ExpirableObjectStore> {
+        return build_object_store_from_settings(self.settings.clone(), None)
+            .await
+            .map_err(|e| {
+                tracing::error!("Error building s3 client from settings: {:?}", e);
+                e
+            })
+            .ok();
+    }
+}
+
+#[cfg(feature = "parquet")]
+impl From<Arc<dyn ObjectStore>> for ExpirableObjectStore {
+    fn from(store: Arc<dyn ObjectStore>) -> Self {
+        Self { store, refresh: None }
+    }
+}
+
+// #[cfg(feature = "parquet")]
+
+// impl ExpirableObjectStore {
+//     pub fn new(store: Arc<dyn ObjectStore>, expiration: Option<DateTime<Utc>>) -> Self {
+//         Self { store, expiration }
+//     }
+// }
+
+#[cfg(feature = "parquet")]
+lazy_static::lazy_static! {
+    pub static ref OBJECT_STORE_SETTINGS: Arc<RwLock<Option<ExpirableObjectStore>>> = Arc::new(RwLock::new(None));
+}
+
+#[cfg(feature = "parquet")]
+pub async fn get_object_store() -> Option<Arc<dyn ObjectStore>> {
+    let settings = OBJECT_STORE_SETTINGS.read().await;
+    if let Some(s) = settings.as_ref() {
+        match &s.refresh {
+            Some(refresh) => {
+                if refresh.refresh_needed() {
+                    let refresh = refresh.clone();
+                    drop(settings);
+                    let new_store = refresh.refresh().await;
+                    if let Some(new_store) = new_store {
+                        let mut s3_cache_settings = OBJECT_STORE_SETTINGS.write().await;
+                        let arc = new_store.store.clone();
+                        *s3_cache_settings = Some(new_store);
+                        return Some(arc);
+                    } else {
+                        return None;
+                    }
+                } else {
+                    return Some(s.store.clone());
+                }
+            }
+            None => {
+                return Some(s.store.clone());
+            }
+        }
+    } else {
+        return None;
+    }
+}
+
+#[cfg(feature = "parquet")]
+pub enum ObjectStoreReload {
+    //if the jwks endpoints are not up yet, we should retry later soon
+    Later,
+    Never,
+}
+
+#[cfg(feature = "parquet")]
+pub async fn reload_object_store_setting(db: &crate::DB) -> ObjectStoreReload {
+    use crate::{
+        ee::{get_license_plan, LicensePlan},
+        global_settings::{load_value_from_global_settings, OBJECT_STORE_CONFIG_SETTING},
+        s3_helpers::ObjectSettings,
+    };
+
+    let s3_config = load_value_from_global_settings(db, OBJECT_STORE_CONFIG_SETTING).await;
+    if let Err(e) = s3_config {
+        tracing::error!("Error reloading s3 cache config: {:?}", e)
+    } else {
+        if let Some(v) = s3_config.unwrap() {
+            if matches!(get_license_plan().await, LicensePlan::Pro) {
+                tracing::error!("S3 cache is not available for pro plan");
+                return ObjectStoreReload::Never;
+            }
+            let setting = serde_json::from_value::<ObjectSettings>(v);
+            match setting {
+                Ok(setting) => {
+                    let is_oidc = matches!(setting, ObjectSettings::AwsOidc(_));
+                    let s3_client = build_object_store_from_settings(setting, Some(db)).await;
+                    match s3_client {
+                        Ok(s3_client) => {
+                            let mut s3_cache_settings = OBJECT_STORE_SETTINGS.write().await;
+                            *s3_cache_settings = Some(s3_client);
+                        }
+                        Err(e) => {
+                            if is_oidc {
+                                tracing::error!("Error building s3 client from oidc settings. It may be due to the jwks endpoints not being up yet, it will be attempted again in 10s to leave time for the server to be ready: {:?}", e);
+                                return ObjectStoreReload::Later;
+                            } else {
+                                tracing::error!("Error building s3 client from settings: {:?}", e);
+                            }
+                        }
+                    }
+                }
+                Err(e) => {
+                    tracing::error!("Error parsing s3 cache config: {:?}", e)
+                }
+            }
+        } else {
+            let mut s3_cache_settings = OBJECT_STORE_SETTINGS.write().await;
+            if std::env::var("S3_CACHE_BUCKET").is_ok() {
+                if matches!(get_license_plan().await, LicensePlan::Pro) {
+                    tracing::error!("S3 cache is not available for pro plan");
+                    return ObjectStoreReload::Never;
+                }
+                *s3_cache_settings = build_s3_client_from_settings(S3Settings {
+                    bucket: None,
+                    region: None,
+                    access_key: None,
+                    secret_key: None,
+                    endpoint: None,
+                    store_logs: None,
+                    path_style: None,
+                    allow_http: None,
+                    port: None,
+                })
+                .await
+                .ok()
+                .map(|x| ExpirableObjectStore::from(x))
+            } else {
+                *s3_cache_settings = None;
+            }
+        }
+    }
+    return ObjectStoreReload::Never;
 }
 
 #[derive(Serialize, Deserialize, Debug)]
@@ -81,6 +244,15 @@ pub enum ObjectStoreResource {
     Azure(AzureBlobResource),
 }
 
+impl ObjectStoreResource {
+    pub fn expiration(&self) -> Option<DateTime<Utc>> {
+        match self {
+            ObjectStoreResource::S3(s3_resource) => s3_resource.expiration,
+            _ => None,
+        }
+    }
+}
+
 #[derive(Deserialize, Debug)]
 pub enum StorageResourceType {
     S3,
@@ -104,6 +276,8 @@ pub struct S3Resource {
     #[serde(rename = "pathStyle")]
     pub path_style: Option<bool>,
     pub token: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub expiration: Option<DateTime<Utc>>,
     pub port: Option<u16>,
 }
 
@@ -126,7 +300,7 @@ pub struct AzureBlobResource {
     pub federated_token_file: Option<String>,
 }
 
-#[derive(Debug, Deserialize, Serialize, Clone)]
+#[derive(Debug, Deserialize, Serialize, Clone, Hash)]
 pub struct S3AwsOidcResource {
     #[serde(rename = "bucket")]
     pub bucket: String,
@@ -412,6 +586,7 @@ pub enum ObjectStoreSettings {
 pub enum ObjectSettings {
     S3(S3Settings),
     Azure(AzureBlobResource),
+    AwsOidc(S3AwsOidcResource),
 }
 
 impl ObjectSettings {
@@ -419,6 +594,7 @@ impl ObjectSettings {
         match self {
             ObjectSettings::S3(s3_settings) => s3_settings.bucket.as_ref(),
             ObjectSettings::Azure(azure_settings) => Some(&azure_settings.container_name),
+            ObjectSettings::AwsOidc(s3_aws_oidc_settings) => Some(&s3_aws_oidc_settings.bucket),
         }
     }
 }
@@ -426,12 +602,31 @@ impl ObjectSettings {
 #[cfg(feature = "parquet")]
 pub async fn build_object_store_from_settings(
     settings: ObjectSettings,
-) -> error::Result<Arc<dyn ObjectStore>> {
+    init_private_key: Option<&crate::DB>,
+) -> error::Result<ExpirableObjectStore> {
     match settings {
-        ObjectSettings::S3(s3_settings) => build_s3_client_from_settings(s3_settings).await,
+        ObjectSettings::S3(s3_settings) => build_s3_client_from_settings(s3_settings)
+            .await
+            .map(|x| ExpirableObjectStore::from(x)),
         ObjectSettings::Azure(azure_settings) => {
             let azure_blob_resource = azure_settings;
-            build_azure_blob_client(&azure_blob_resource)
+            build_azure_blob_client(&azure_blob_resource).map(|x| ExpirableObjectStore::from(x))
+        }
+        ObjectSettings::AwsOidc(ref s3_aws_oidc_settings) => {
+            let token_generator = crate::job_s3_helpers_ee::TokenGenerator::AsServerInstance();
+            let res = crate::job_s3_helpers_ee::generate_s3_aws_oidc_resource(
+                s3_aws_oidc_settings.clone(),
+                token_generator,
+                init_private_key,
+            )
+            .await?;
+
+            build_object_store_client(&res)
+                .await
+                .map(|x| ExpirableObjectStore {
+                    store: x,
+                    refresh: Some(ObjectStoreRefresh::new(settings.clone(), res.expiration())),
+                })
         }
     }
 }
@@ -479,6 +674,7 @@ pub async fn build_s3_client_from_settings(
         path_style: settings.path_style,
         port: settings.port,
         token: None,
+        expiration: None,
     };
 
     build_s3_client(&s3_resource).await
@@ -695,3 +891,20 @@ pub async fn convert_json_line_stream<E: Into<anyhow::Error>>(
 
     Ok(tokio_stream::wrappers::ReceiverStream::new(rx))
 }
+
+#[derive(Deserialize, Serialize)]
+pub struct DuckdbConnectionSettingsResponse {
+    pub connection_settings_str: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub azure_container_path: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub s3_bucket: Option<String>,
+}
+
+#[derive(Deserialize, Serialize)]
+pub struct DuckdbConnectionSettingsQueryV2 {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub s3_resource_path: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub storage: Option<String>,
+}

backend/windmill-common/src/schema.rs

@@ -349,7 +349,7 @@ pub fn should_validate_schema(code: &str, lang: &ScriptLang) -> bool {
     let comment = match lang {
         Nativets | Bun | Bunnative | Deno | Php | CSharp | Java => "//",
         Python3 | Go | Bash | Powershell | Graphql | Ansible | Nu => "#",
-        Postgresql | Mysql | Bigquery | Snowflake | Mssql | OracleDB => "--",
+        Postgresql | Mysql | Bigquery | Snowflake | Mssql | OracleDB | DuckDb => "--",
         Rust => "//!",
         // for related places search: ADD_NEW_LANG
     };

backend/windmill-common/src/scripts.rs

@@ -46,13 +46,13 @@ pub enum ScriptLang {
     Graphql,
     Mssql,
     OracleDB,
+    DuckDb,
     Php,
     Rust,
     Ansible,
     CSharp,
     Nu,
-    Java,
-    // for related places search: ADD_NEW_LANG
+    Java, // for related places search: ADD_NEW_LANG
 }
 
 impl ScriptLang {
@@ -73,6 +73,7 @@ impl ScriptLang {
             ScriptLang::Mssql => "mssql",
             ScriptLang::Graphql => "graphql",
             ScriptLang::OracleDB => "oracledb",
+            ScriptLang::DuckDb => "duckdb",
             ScriptLang::Php => "php",
             ScriptLang::Rust => "rust",
             ScriptLang::Ansible => "ansible",

backend/windmill-common/src/stats_oss.rs

@@ -0,0 +1,40 @@
+use crate::DB;
+use serde::{Deserialize, Serialize};
+
+#[derive(Serialize, Deserialize)]
+pub struct ActiveUserUsage {
+    pub author_count: Option<i32>,
+    pub operator_count: Option<i32>,
+}
+
+#[derive(Clone)]
+pub enum SendStatsReason {
+    Manual,
+    Schedule,
+    OnStart,
+}
+
+pub async fn get_disable_stats_setting(_db: &DB) -> bool {
+    crate::stats_ee::get_disable_stats_setting(_db).await
+}
+
+pub async fn schedule_stats(_db: &DB, _http_client: &reqwest::Client) -> () {
+    crate::stats_ee::schedule_stats(_db, _http_client).await
+}
+
+pub async fn send_stats(
+    _db: &DB,
+    _http_client: &reqwest::Client,
+    _reason: SendStatsReason,
+    _basic_auth: Option<String>,
+) -> anyhow::Result<()> {
+    crate::stats_ee::send_stats(_db, _http_client, _reason, _basic_auth).await
+}
+
+pub async fn get_user_usage(
+    _db: &DB,
+    _workspace_id: Option<&str>,
+    _days: Option<i32>,
+) -> anyhow::Result<ActiveUserUsage> {
+    crate::stats_ee::get_user_usage(_db, _workspace_id, _days).await
+}

backend/windmill-common/src/teams_oss.rs

@@ -0,0 +1 @@
+// Empty OSS implementation for teams_ee

backend/windmill-common/src/worker.rs

@@ -79,6 +79,7 @@ lazy_static::lazy_static! {
         "csharp".to_string(),
         "nu".to_string(),
         "java".to_string(),
+        "duckdb".to_string(),
         // for related places search: ADD_NEW_LANG
         "dependency".to_string(),
         "flow".to_string(),
@@ -582,11 +583,7 @@ pub async fn load_cache(bin_path: &str, _remote_path: &str, is_dir: bool) -> (bo
         (true, format!("loaded from local cache: {}\n", bin_path))
     } else {
         #[cfg(all(feature = "enterprise", feature = "parquet"))]
-        if let Some(os) = crate::s3_helpers::OBJECT_STORE_CACHE_SETTINGS
-            .read()
-            .await
-            .clone()
-        {
+        if let Some(os) = crate::s3_helpers::get_object_store().await {
             let started = std::time::Instant::now();
             use crate::s3_helpers::attempt_fetch_bytes;
 
@@ -629,11 +626,7 @@ pub async fn exists_in_cache(bin_path: &str, _remote_path: &str) -> bool {
         return true;
     } else {
         #[cfg(all(feature = "enterprise", feature = "parquet"))]
-        if let Some(os) = crate::s3_helpers::OBJECT_STORE_CACHE_SETTINGS
-            .read()
-            .await
-            .clone()
-        {
+        if let Some(os) = crate::s3_helpers::get_object_store().await {
             return os
                 .get(&object_store::path::Path::from(_remote_path))
                 .await
@@ -651,11 +644,7 @@ pub async fn save_cache(
 ) -> crate::error::Result<String> {
     let mut _cached_to_s3 = false;
     #[cfg(all(feature = "enterprise", feature = "parquet"))]
-    if let Some(os) = crate::s3_helpers::OBJECT_STORE_CACHE_SETTINGS
-        .read()
-        .await
-        .clone()
-    {
+    if let Some(os) = crate::s3_helpers::get_object_store().await {
         use object_store::path::Path;
         let file_to_cache = if is_dir {
             let tar_path = format!(

backend/windmill-git-sync/src/git_sync_oss.rs

@@ -0,0 +1,10 @@
+use sqlx::{Postgres, Transaction};
+
+pub async fn handle_deployment_metadata(
+    _tx: &mut Transaction<'_, Postgres>,
+    _workspace_id: &str,
+    _path: &str,
+    _raw_value: &serde_json::Value,
+) -> anyhow::Result<()> {
+    crate::git_sync_ee::handle_deployment_metadata(_tx, _workspace_id, _path, _raw_value).await
+}

backend/windmill-indexer/src/completed_runs_oss.rs

@@ -0,0 +1,16 @@
+use sqlx::{Pool, Postgres};
+
+pub struct IndexReader;
+pub struct IndexWriter;
+
+pub async fn init_index(_db: &Pool<Postgres>) -> Result<(IndexReader, IndexWriter), anyhow::Error> {
+    crate::completed_runs_ee::init_index(_db).await
+}
+
+pub async fn run_indexer(
+    _db: Pool<Postgres>,
+    _writer: IndexWriter,
+    _mut killpill_rx: tokio::sync::broadcast::Receiver<()>,
+) -> Result<(), anyhow::Error> {
+    crate::completed_runs_ee::run_indexer(_db, _writer, _mut killpill_rx).await
+}

backend/windmill-indexer/src/indexer_oss.rs

@@ -0,0 +1 @@
+// Empty OSS implementation for indexer_ee

backend/windmill-indexer/src/service_logs_oss.rs

@@ -0,0 +1,21 @@
+use sqlx::{Pool, Postgres};
+
+pub struct ServiceLogIndexReader;
+pub struct ServiceLogIndexWriter;
+
+pub async fn init_index(
+    _db: &Pool<Postgres>,
+    _index_name: &str,
+    _index_location: &str,
+    _tantivy_buffer_size_mb: usize,
+) -> Result<(ServiceLogIndexReader, ServiceLogIndexWriter), anyhow::Error> {
+    crate::service_logs_ee::init_index(_db, _index_name, _index_location, _tantivy_buffer_size_mb).await
+}
+
+pub async fn run_indexer(
+    _db: Pool<Postgres>,
+    _writer: ServiceLogIndexWriter,
+    _mut killpill_rx: tokio::sync::broadcast::Receiver<()>,
+) -> Result<(), anyhow::Error> {
+    crate::service_logs_ee::run_indexer(_db, _writer, _mut killpill_rx).await
+}

backend/windmill-queue/src/jobs_oss.rs

@@ -0,0 +1,11 @@
+use chrono::{DateTime, Utc};
+
+pub(crate) async fn update_concurrency_counter(
+    _tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
+    _w_id: &str,
+    _job_id: uuid::Uuid,
+    _concurrency_key: &str,
+    _cancel: bool,
+) -> anyhow::Result<(bool, Option<DateTime<Utc>>)> {
+    crate::jobs_ee::update_concurrency_counter(_tx, _w_id, _job_id, _concurrency_key, _cancel).await
+}

backend/windmill-worker/Cargo.toml

@@ -31,6 +31,7 @@ csharp = ["dep:windmill-parser-csharp"]
 rust = ["dep:windmill-parser-rust"]
 nu = ["dep:windmill-parser-nu"]
 java = ["dep:windmill-parser-java"]
+duckdb = []
 
 [dependencies]
 windmill-queue.workspace = true
@@ -92,6 +93,7 @@ deno_permissions = { workspace = true, optional = true }
 deno_io = { workspace = true, optional = true }
 deno_error = { workspace = true, optional = true }
 async-stream.workspace = true
+duckdb.workspace = true
 
 postgres-native-tls.workspace = true
 native-tls.workspace = true

backend/windmill-worker/src/ansible_executor.rs

@@ -31,9 +31,10 @@ use crate::{
     },
     handle_child::handle_child,
     python_executor::{create_dependencies_dir, handle_python_reqs, uv_pip_compile},
-    AuthedClient, PyVAlias, DISABLE_NSJAIL, DISABLE_NUSER, GIT_PATH, HOME_ENV, NSJAIL_PATH,
-    PATH_ENV, PROXY_ENVS, PY_INSTALL_DIR, TZ_ENV,
+    PyVAlias, DISABLE_NSJAIL, DISABLE_NUSER, GIT_PATH, HOME_ENV, NSJAIL_PATH, PATH_ENV, PROXY_ENVS,
+    PY_INSTALL_DIR, TZ_ENV,
 };
+use windmill_common::client::AuthedClient;
 
 lazy_static::lazy_static! {
     static ref ANSIBLE_PLAYBOOK_PATH: String =
@@ -1190,7 +1191,7 @@ async fn create_file_resources(
     job_dir: &str,
     args: Option<&HashMap<String, Box<RawValue>>>,
     r: &AnsibleRequirements,
-    client: &crate::AuthedClient,
+    client: &AuthedClient,
     conn: &Connection,
 ) -> error::Result<Vec<String>> {
     let mut logs = String::new();
@@ -1267,7 +1268,7 @@ async fn create_file_resources(
 }
 
 async fn get_resource_or_variable_content(
-    client: &crate::AuthedClient,
+    client: &AuthedClient,
     path: &ResourceOrVariablePath,
     job_id: String,
 ) -> anyhow::Result<String> {

backend/windmill-worker/src/bash_executor.rs

@@ -43,9 +43,11 @@ use crate::{
         OccupancyMetrics,
     },
     handle_child::handle_child,
-    AuthedClient, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV,
+    DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV,
     POWERSHELL_CACHE_DIR, POWERSHELL_PATH, PROXY_ENVS, TZ_ENV,
 };
+use windmill_common::client::AuthedClient;
+
 
 #[cfg(windows)]
 use crate::SYSTEM_ROOT;

backend/windmill-worker/src/bigquery_executor.rs

@@ -4,6 +4,7 @@ use futures::future::BoxFuture;
 use futures::{FutureExt, StreamExt};
 use reqwest::Client;
 use serde_json::{json, value::RawValue, Value};
+use windmill_common::client::AuthedClient;
 use windmill_common::error::to_anyhow;
 use windmill_common::s3_helpers::convert_json_line_stream;
 use windmill_common::worker::Connection;
@@ -16,15 +17,12 @@ use windmill_queue::CanceledBy;
 
 use serde::Deserialize;
 
+use crate::common::{build_args_values, resolve_job_timeout};
 use crate::common::{
     build_http_client, s3_mode_args_to_worker_data, OccupancyMetrics, S3ModeWorkerData,
 };
 use crate::handle_child::run_future_with_polling_update_job_poller;
 use crate::sanitized_sql_params::sanitize_and_interpolate_unsafe_sql_args;
-use crate::{
-    common::{build_args_values, resolve_job_timeout},
-    AuthedClient,
-};
 
 use gcp_auth::{AuthenticationManager, CustomServiceAccount};
 

backend/windmill-worker/src/bun_executor.rs

@@ -20,10 +20,11 @@ use crate::{
         read_file_content, read_result, start_child_process, write_file_binary, OccupancyMetrics,
     },
     handle_child::handle_child,
-    AuthedClient, BUNFIG_INSTALL_SCOPES, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, BUN_PATH,
-    DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NODE_BIN_PATH, NODE_PATH, NPM_CONFIG_REGISTRY,
-    NPM_PATH, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, TZ_ENV,
+    BUNFIG_INSTALL_SCOPES, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, BUN_PATH, DISABLE_NSJAIL,
+    DISABLE_NUSER, HOME_ENV, NODE_BIN_PATH, NODE_PATH, NPM_CONFIG_REGISTRY, NPM_PATH, NSJAIL_PATH,
+    PATH_ENV, PROXY_ENVS, TZ_ENV,
 };
+use windmill_common::client::AuthedClient;
 
 #[cfg(windows)]
 use crate::SYSTEM_ROOT;
@@ -612,10 +613,7 @@ pub async fn pull_codebase(w_id: &str, id: &str, job_dir: &str) -> Result<()> {
         extract_saved_codebase(job_dir, &bun_cache_path, is_tar, &dst, false)?;
     } else {
         #[cfg(all(feature = "enterprise", feature = "parquet"))]
-        let object_store = windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS
-            .read()
-            .await
-            .clone();
+        let object_store = windmill_common::s3_helpers::get_object_store().await;
 
         #[cfg(not(all(feature = "enterprise", feature = "parquet")))]
         let object_store: Option<()> = None;

backend/windmill-worker/src/common.rs

@@ -44,10 +44,8 @@ use windmill_common::{variables, DB};
 use tokio::{io::AsyncWriteExt, process::Child, time::Instant};
 
 use crate::agent_workers::UPDATE_PING_URL;
-use crate::{
-    AuthedClient, DISABLE_NSJAIL, JOB_DEFAULT_TIMEOUT, MAX_RESULT_SIZE, MAX_TIMEOUT_DURATION,
-    PATH_ENV,
-};
+use crate::{DISABLE_NSJAIL, JOB_DEFAULT_TIMEOUT, MAX_RESULT_SIZE, MAX_TIMEOUT_DURATION, PATH_ENV};
+use windmill_common::client::AuthedClient;
 
 pub async fn build_args_map<'a>(
     job: &'a MiniPulledJob,
@@ -782,19 +780,17 @@ async fn get_workspace_s3_resource_path(
         }
     };
 
-    let client2 = client.clone();
-    let token_fn = |audience: String| async move {
-        client2
-            .get_id_token(&audience)
-            .await
-            .map_err(|e| windmill_common::error::Error::from(e))
-    };
     let s3_resource_value_raw = client
         .get_resource_value::<serde_json::Value>(path.as_str())
         .await?;
-    get_s3_resource_internal(rt, s3_resource_value_raw, token_fn)
-        .await
-        .map(Some)
+    get_s3_resource_internal(
+        rt,
+        s3_resource_value_raw,
+        windmill_common::job_s3_helpers_ee::TokenGenerator::AsClient(client),
+        db,
+    )
+    .await
+    .map(Some)
 }
 
 #[cfg(feature = "parquet")]
@@ -1109,7 +1105,7 @@ pub async fn par_install_language_dependencies<'a>(
         }
 
         #[cfg(all(feature = "enterprise", feature = "parquet"))]
-        if windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS
+        if windmill_common::s3_helpers::OBJECT_STORE_SETTINGS
             .read()
             .await
             .is_none()
@@ -1264,11 +1260,7 @@ pub async fn par_install_language_dependencies<'a>(
 
         #[cfg(all(feature = "enterprise", feature = "parquet"))]
         let s3_pull_future = if is_not_pro {
-            if let Some(os) = windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS
-                .read()
-                .await
-                .clone()
-            {
+            if let Some(os) = windmill_common::s3_helpers::get_object_store().await {
                 Some(crate::global_cache::pull_from_tar(
                     os,
                     path.clone(),
@@ -1449,11 +1441,7 @@ pub async fn par_install_language_dependencies<'a>(
                 };
                 #[cfg(all(feature = "enterprise", feature = "parquet"))]
                 {
-                    if let Some(os) = windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS
-                        .read()
-                        .await
-                        .clone()
-                    {
+                    if let Some(os) = windmill_common::s3_helpers::get_object_store().await {
                         tokio::spawn(async move {
                             if let Err(e) = crate::global_cache::build_tar_and_push(
                                 os,
@@ -1541,11 +1529,7 @@ pub async fn par_install_language_dependencies<'a>(
                 };
                 #[cfg(all(feature = "enterprise", feature = "parquet"))]
                 {
-                    if let Some(os) = windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS
-                        .read()
-                        .await
-                        .clone()
-                    {
+                    if let Some(os) = windmill_common::s3_helpers::get_object_store().await {
                         let language_name = language_name.to_owned();
                         tokio::spawn(async move {
                             if let Err(e) = crate::global_cache::build_tar_and_push(
@@ -1591,7 +1575,7 @@ pub struct S3ModeWorkerData {
 }
 
 impl S3ModeWorkerData {
-    pub async fn upload<S>(&self, stream: S) -> error::Result<()>
+    pub async fn upload<S>(&self, stream: S) -> anyhow::Result<()>
     where
         S: futures::stream::TryStream + Send + 'static,
         S::Error: Into<Box<dyn std::error::Error + Send + Sync>>,

backend/windmill-worker/src/csharp_executor.rs

@@ -36,7 +36,7 @@ use crate::{
 };
 
 use crate::common::OccupancyMetrics;
-use crate::AuthedClient;
+use windmill_common::client::AuthedClient;
 
 #[cfg(windows)]
 use crate::SYSTEM_ROOT;

backend/windmill-worker/src/deno_executor.rs

@@ -11,9 +11,11 @@ use crate::{
         start_child_process, OccupancyMetrics,
     },
     handle_child::handle_child,
-    AuthedClient, DENO_CACHE_DIR, DENO_PATH, DISABLE_NSJAIL, HOME_ENV, NPM_CONFIG_REGISTRY,
+    DENO_CACHE_DIR, DENO_PATH, DISABLE_NSJAIL, HOME_ENV, NPM_CONFIG_REGISTRY,
     PATH_ENV, TZ_ENV,
 };
+use windmill_common::client::AuthedClient;
+
 use tokio::{fs::File, io::AsyncReadExt, process::Command};
 use windmill_common::{error::Result, worker::write_file, BASE_URL};
 use windmill_common::{

backend/windmill-worker/src/duckdb_executor.rs

@@ -0,0 +1,664 @@
+use std::collections::HashMap;
+use std::env;
+
+use duckdb::types::TimeUnit;
+use duckdb::{params_from_iter, Row};
+use rust_decimal::prelude::FromPrimitive;
+use rust_decimal::Decimal;
+use serde_json::value::RawValue;
+use serde_json::{json, Value};
+use tokio::fs::remove_file;
+use tokio::task;
+use uuid::Uuid;
+use windmill_common::error::{to_anyhow, Error, Result};
+use windmill_common::s3_helpers::{
+    DuckdbConnectionSettingsQueryV2, DuckdbConnectionSettingsResponse, S3Object,
+};
+use windmill_common::worker::{to_raw_value, Connection};
+use windmill_parser_sql::{parse_duckdb_sig, parse_sql_blocks};
+use windmill_queue::{CanceledBy, MiniPulledJob};
+
+use crate::common::{build_args_values, OccupancyMetrics};
+use crate::handle_child::run_future_with_polling_update_job_poller;
+#[cfg(feature = "mysql")]
+use crate::mysql_executor::MysqlDatabase;
+use crate::pg_executor::PgDatabase;
+use crate::sanitized_sql_params::sanitize_and_interpolate_unsafe_sql_args;
+use windmill_common::client::AuthedClient;
+
+fn do_duckdb_inner(
+    conn: &duckdb::Connection,
+    query: &str,
+    job_args: &HashMap<String, duckdb::types::Value>,
+    skip_collect: bool,
+    column_order: &mut Option<Vec<String>>,
+) -> Result<Box<RawValue>> {
+    let mut rows_vec = vec![];
+
+    let (query, job_args) = interpolate_named_args(query, &job_args);
+
+    let mut stmt = conn
+        .prepare(&query)
+        .map_err(|e| Error::ExecutionErr(e.to_string()))?;
+
+    let mut rows = stmt
+        .query(params_from_iter(job_args))
+        .map_err(|e| Error::ExecutionErr(e.to_string()))?;
+
+    if skip_collect {
+        return Ok(to_raw_value(&json!([])));
+    }
+
+    // Statement needs to be stepped at least once or stmt.column_names() will panic
+    let mut column_names = None;
+    loop {
+        let row = rows.next();
+        match row {
+            Ok(Some(row)) => {
+                // Set column names if not already set
+                let stmt = row.as_ref();
+                let column_names = match column_names.as_ref() {
+                    Some(column_names) => column_names,
+                    None => {
+                        column_names = Some(stmt.column_names());
+                        column_names.as_ref().unwrap()
+                    }
+                };
+
+                let row = row_to_value(row, &column_names.as_slice())
+                    .map_err(|e| Error::ExecutionErr(e.to_string()))?;
+                rows_vec.push(row);
+            }
+            Ok(None) => break,
+            Err(e) => {
+                return Err(Error::ExecutionErr(e.to_string()));
+            }
+        }
+    }
+
+    if let (Some(column_order), Some(column_names)) = (column_order.as_mut(), column_names) {
+        *column_order = column_names.clone();
+    }
+
+    return Ok(to_raw_value(&rows_vec));
+}
+
+pub async fn do_duckdb(
+    job: &MiniPulledJob,
+    client: &AuthedClient,
+    query: &str,
+    conn: &Connection,
+    mem_peak: &mut i32,
+    canceled_by: &mut Option<CanceledBy>,
+    worker_name: &str,
+    column_order_ref: &mut Option<Vec<String>>,
+    occupancy_metrics: &mut OccupancyMetrics,
+) -> Result<Box<RawValue>> {
+    let result_f = async {
+        let sig = parse_duckdb_sig(query)?.args;
+        let mut job_args = build_args_values(job, client, conn).await?;
+
+        let (query, _) = &sanitize_and_interpolate_unsafe_sql_args(query, &sig, &job_args)?;
+        // Prevent interpolate_named_args from detecting argument identifiers in the signature for
+        // the first query block
+        let query = trunc_sig(query);
+
+        let (_query_with_transformed_s3_uris, mut used_storages) =
+            transform_s3_uris(query, client).await?;
+        let query = _query_with_transformed_s3_uris.as_deref().unwrap_or(query);
+
+        let job_args = {
+            let mut m: HashMap<String, duckdb::types::Value> = HashMap::new();
+            for sig_arg in sig.into_iter() {
+                let json_value = job_args
+                    .remove(&sig_arg.name)
+                    .or_else(|| sig_arg.default)
+                    .unwrap_or_else(|| json!(null));
+
+                if matches!(&sig_arg.otyp.as_ref().map(String::as_str), Some("s3object")) {
+                    let s3_obj = serde_json::from_value::<S3Object>(json_value).map_err(|e| {
+                        Error::ExecutionErr(format!("Failed to deserialize S3Object: {}", e))
+                    })?;
+                    let duckdb_conn_settings: windmill_common::s3_helpers::DuckdbConnectionSettingsResponse = client
+                        .get_duckdb_connection_settings(&DuckdbConnectionSettingsQueryV2 {
+                            s3_resource_path: None,
+                            storage: s3_obj.storage.clone(),
+                        })
+                        .await?;
+
+                    let uri = match (
+                        &duckdb_conn_settings.s3_bucket,
+                        &duckdb_conn_settings.azure_container_path,
+                    ) {
+                        (Some(s3_bucket), None) => format!("s3://{}/{}", s3_bucket, &s3_obj.s3),
+                        (None, Some(az_container)) => format!("{}/{}", az_container, &s3_obj.s3),
+                        _ => {
+                            return Err(Error::ExecutionErr(
+                                "S3Object must have either s3_bucket or azure_container_path"
+                                    .to_string(),
+                            ));
+                        }
+                    };
+                    m.insert(sig_arg.name, duckdb::types::Value::Text(uri));
+                    used_storages.insert(s3_obj.storage, duckdb_conn_settings);
+                } else {
+                    let duckdb_value = json_value_to_duckdb_value(
+                        &json_value,
+                        sig_arg
+                            .otyp
+                            .clone()
+                            .unwrap_or_else(|| "text".to_string())
+                            .as_str(),
+                        client,
+                    )?;
+                    m.insert(sig_arg.name, duckdb_value);
+                }
+            }
+            m
+        };
+
+        let query_block_list = parse_sql_blocks(query);
+
+        // Replace windmill resource ATTACH statements with the real instructions
+        let query_block_list = {
+            let mut v = vec![];
+            for query_block in query_block_list.iter() {
+                match parse_attach_db_resource(query_block) {
+                    Some(parsed) => v.extend(
+                        transform_attach_db_resource_query(&parsed, &job.id, client).await?,
+                    ),
+                    None => v.push(query_block.to_string()),
+                };
+            }
+            v
+        };
+
+        // duckdb::Connection is not Send so we do it in a single blocking task
+        let (result, column_order) = task::spawn_blocking(move || {
+            let conn = duckdb::Connection::open_in_memory()
+                .map_err(|e| Error::ConnectingToDatabase(e.to_string()))?;
+
+            for (_, DuckdbConnectionSettingsResponse { connection_settings_str, .. }) in
+                used_storages.into_iter()
+            {
+                conn.execute_batch(&connection_settings_str)
+                    .map_err(|e| Error::ExecutionErr(e.to_string()))?;
+            }
+
+            let mut result: Option<Box<RawValue>> = None;
+            let mut column_order = None;
+            for (query_block_index, query_block) in query_block_list.iter().enumerate() {
+                result = Some(
+                    do_duckdb_inner(
+                        &conn,
+                        query_block.as_str(),
+                        &job_args,
+                        query_block_index != query_block_list.len() - 1,
+                        &mut column_order,
+                    )
+                    .map_err(|e| Error::ExecutionErr(e.to_string()))?,
+                );
+            }
+            let result = result.unwrap_or_else(|| to_raw_value(&json!([])));
+            Ok::<_, Error>((result, column_order))
+        })
+        .await
+        .map_err(to_anyhow)??;
+
+        *column_order_ref = column_order;
+
+        // BigQuery cleanup
+        let bq_credentials_path = make_bq_credentials_path(&job.id);
+        env::remove_var("GOOGLE_APPLICATION_CREDENTIALS");
+        if matches!(tokio::fs::try_exists(&bq_credentials_path).await, Ok(true)) {
+            remove_file(&bq_credentials_path).await.map_err(to_anyhow)?;
+        }
+        Ok(result)
+    };
+
+    let result = run_future_with_polling_update_job_poller(
+        job.id,
+        job.timeout,
+        conn,
+        mem_peak,
+        canceled_by,
+        result_f,
+        worker_name,
+        &job.workspace_id,
+        &mut Some(occupancy_metrics),
+        Box::pin(futures::stream::once(async { 0 })),
+    )
+    .await?;
+
+    Ok(result)
+}
+
+fn row_to_value(row: &Row<'_>, column_names: &[String]) -> Result<Box<RawValue>> {
+    let mut obj = serde_json::Map::new();
+    for (i, key) in column_names.iter().enumerate() {
+        let value: duckdb::types::Value =
+            row.get(i).map_err(|e| Error::ExecutionErr(e.to_string()))?;
+        let json_value = match value {
+            duckdb::types::Value::Null => serde_json::Value::Null,
+            duckdb::types::Value::Boolean(b) => serde_json::Value::Bool(b),
+            duckdb::types::Value::TinyInt(i) => serde_json::Value::Number(i.into()),
+            duckdb::types::Value::SmallInt(i) => serde_json::Value::Number(i.into()),
+            duckdb::types::Value::Int(i) => serde_json::Value::Number(i.into()),
+            duckdb::types::Value::BigInt(i) => serde_json::Value::Number(i.into()),
+            duckdb::types::Value::HugeInt(i) => serde_json::Value::String(i.to_string()),
+            duckdb::types::Value::UTinyInt(u) => serde_json::Value::Number(u.into()),
+            duckdb::types::Value::USmallInt(u) => serde_json::Value::Number(u.into()),
+            duckdb::types::Value::UInt(u) => serde_json::Value::Number(u.into()),
+            duckdb::types::Value::UBigInt(u) => serde_json::Value::Number(u.into()),
+            duckdb::types::Value::Float(f) => serde_json::Value::Number(
+                serde_json::Number::from_f64(f as f64)
+                    .ok_or_else(|| Error::ExecutionErr("Could not convert to f64".to_string()))?,
+            ),
+            duckdb::types::Value::Double(f) => serde_json::Value::Number(
+                serde_json::Number::from_f64(f)
+                    .ok_or_else(|| Error::ExecutionErr("Could not convert to f64".to_string()))?,
+            ),
+            duckdb::types::Value::Decimal(d) => serde_json::Value::String(d.to_string()),
+            duckdb::types::Value::Timestamp(_, ts) => serde_json::Value::String(ts.to_string()),
+            duckdb::types::Value::Text(s) => serde_json::Value::String(s),
+            duckdb::types::Value::Blob(b) => serde_json::Value::Array(
+                b.into_iter()
+                    .map(|byte| serde_json::Value::Number(byte.into()))
+                    .collect(),
+            ),
+            duckdb::types::Value::Date32(d) => serde_json::Value::Number(d.into()),
+            duckdb::types::Value::Time64(_, t) => serde_json::Value::String(t.to_string()),
+            duckdb::types::Value::Interval { months, days, nanos } => serde_json::json!({
+                "months": months,
+                "days": days,
+                "nanos": nanos
+            }),
+            duckdb::types::Value::List(values) => serde_json::Value::Array(
+                values
+                    .into_iter()
+                    .map(|v| serde_json::Value::String(format!("{:?}", v)))
+                    .collect(),
+            ),
+            duckdb::types::Value::Enum(e) => serde_json::Value::String(e),
+            duckdb::types::Value::Struct(fields) => serde_json::Value::Object(
+                fields
+                    .iter()
+                    .map(|(k, v)| (k.clone(), serde_json::Value::String(format!("{:?}", v))))
+                    .collect(),
+            ),
+            duckdb::types::Value::Array(values) => serde_json::Value::Array(
+                values
+                    .into_iter()
+                    .map(|v| serde_json::Value::String(format!("{:?}", v)))
+                    .collect(),
+            ),
+            duckdb::types::Value::Map(map) => serde_json::Value::Object(
+                map.iter()
+                    .map(|(k, v)| {
+                        (
+                            format!("{:?}", k),
+                            serde_json::Value::String(format!("{:?}", v)),
+                        )
+                    })
+                    .collect(),
+            ),
+            duckdb::types::Value::Union(value) => {
+                serde_json::Value::String(format!("{:?}", *value))
+            }
+        };
+        obj.insert(key.clone(), json_value);
+    }
+    serde_json::value::to_raw_value(&obj).map_err(|e| e.into())
+}
+
+fn json_value_to_duckdb_value(
+    json_value: &serde_json::Value,
+    arg_type: &str,
+    client: &AuthedClient,
+) -> Result<duckdb::types::Value> {
+    let arg_type = arg_type.to_lowercase();
+    let duckdb_value = match json_value {
+        serde_json::Value::Null => duckdb::types::Value::Null,
+        serde_json::Value::Bool(b) => duckdb::types::Value::Boolean(*b),
+
+        serde_json::Value::String(s)
+            if matches!(
+                arg_type.as_str(),
+                "timestamp" | "timestamptz" | "timestamp with time zone" | "datetime"
+            ) =>
+        {
+            string_to_duckdb_timestamp(&s)?
+        }
+        serde_json::Value::String(s) if arg_type.as_str() == "date" => string_to_duckdb_date(&s)?,
+        serde_json::Value::String(s) if arg_type.as_str() == "time" => string_to_duckdb_time(&s)?,
+        serde_json::Value::String(s) => duckdb::types::Value::Text(s.clone()),
+
+        serde_json::Value::Number(n) if n.is_i64() => {
+            let v = n.as_i64().unwrap();
+            match arg_type.as_str() {
+                "tinyint" | "int1" => duckdb::types::Value::TinyInt(v as i8),
+                "smallint" | "int2" | "short" => duckdb::types::Value::SmallInt(v as i16),
+                "integer" | "int4" | "int" | "signed" => duckdb::types::Value::Int(v as i32),
+                "bigint" | "int8" | "long" => duckdb::types::Value::BigInt(v),
+                "hugeint" => duckdb::types::Value::HugeInt(v as i128),
+                "float" | "float4" | "real" => duckdb::types::Value::Float(v as f32),
+                "double" | "float8" => duckdb::types::Value::Double(v as f64),
+                _ => duckdb::types::Value::BigInt(v), // default fallback
+            }
+        }
+
+        serde_json::Value::Number(n) if n.is_u64() => {
+            let v = n.as_u64().unwrap();
+            match arg_type.as_str() {
+                "utinyint" => duckdb::types::Value::UTinyInt(v as u8),
+                "usmallint" => duckdb::types::Value::USmallInt(v as u16),
+                "uinteger" => duckdb::types::Value::UInt(v as u32),
+                "ubigint" | "uhugeint" => duckdb::types::Value::UBigInt(v),
+                _ => duckdb::types::Value::UBigInt(v), // default fallback
+            }
+        }
+
+        serde_json::Value::Number(n) if n.is_f64() => {
+            let v = n.as_f64().unwrap();
+            match arg_type.as_str() {
+                "float" | "float4" | "real" => duckdb::types::Value::Float(v as f32),
+                "double" | "float8" => duckdb::types::Value::Double(v),
+                "decimal" | "numeric" => {
+                    duckdb::types::Value::Decimal(Decimal::from_f64(v).ok_or_else(|| {
+                        Error::ExecutionErr("Could not convert f64 to Decimal".to_string())
+                    })?)
+                }
+                _ => duckdb::types::Value::Double(v), // default fallback
+            }
+        }
+
+        serde_json::Value::Array(arr) => duckdb::types::Value::Array(
+            arr.iter()
+                .map(|val| json_value_to_duckdb_value(val, arg_type.as_str(), client))
+                .collect::<Result<Vec<_>>>()?,
+        ),
+        serde_json::Value::Object(map) => duckdb::types::Value::Struct(
+            map.iter()
+                .map(|(k, v)| {
+                    Ok::<_, Error>((
+                        k.clone(),
+                        json_value_to_duckdb_value(v, arg_type.as_str(), client)?,
+                    ))
+                })
+                .collect::<Result<Vec<_>>>()?
+                .into(),
+        ),
+
+        value @ _ => {
+            return Err(Error::ExecutionErr(format!(
+                "Unsupported type in query: {:?} and signature {arg_type:?}",
+                value
+            )))
+        }
+    };
+    Ok(duckdb_value)
+}
+
+fn string_to_duckdb_timestamp(s: &str) -> Result<duckdb::types::Value> {
+    let ts = chrono::DateTime::parse_from_rfc3339(s)
+        .map_err(|e: chrono::ParseError| Error::ExecutionErr(e.to_string()))?;
+    Ok(duckdb::types::Value::Timestamp(
+        TimeUnit::Millisecond,
+        ts.timestamp_millis(),
+    ))
+}
+
+fn string_to_duckdb_date(s: &str) -> Result<duckdb::types::Value> {
+    use chrono::Datelike;
+    let date = chrono::NaiveDate::parse_from_str(s, "%Y-%m-%d").unwrap();
+    Ok(duckdb::types::Value::Date32(date.num_days_from_ce()))
+}
+
+fn string_to_duckdb_time(s: &str) -> Result<duckdb::types::Value> {
+    use chrono::Timelike;
+    let time = chrono::NaiveTime::parse_from_str(s, "%H:%M:%S").unwrap();
+    Ok(duckdb::types::Value::Time64(
+        TimeUnit::Microsecond,
+        time.num_seconds_from_midnight() as i64,
+    ))
+}
+
+struct ParsedAttachDbResource<'a> {
+    resource_path: &'a str,
+    name: &'a str,
+    db_type: &'a str,
+    extra_args: Option<&'a str>,
+}
+fn parse_attach_db_resource<'a>(query: &'a str) -> Option<ParsedAttachDbResource<'a>> {
+    lazy_static::lazy_static! {
+        static ref RE: regex::Regex = regex::Regex::new(r"ATTACH '\$res:([^']+)' AS (\S+) \(TYPE (\w+)(.*)\)").unwrap();
+    }
+
+    for cap in RE.captures_iter(query) {
+        if let (Some(resource_path), Some(name), Some(db_type)) =
+            (cap.get(1), cap.get(2), cap.get(3))
+        {
+            let extra_args = cap.get(4).map(|m| query[m.start()..m.end()].trim());
+            return Some(ParsedAttachDbResource {
+                resource_path: query[resource_path.start()..resource_path.end()].trim(),
+                name: query[name.start()..name.end()].trim(),
+                db_type: query[db_type.start()..db_type.end()].trim(),
+                extra_args,
+            });
+        }
+    }
+    None
+}
+
+async fn transform_attach_db_resource_query(
+    parsed: &ParsedAttachDbResource<'_>,
+    job_id: &Uuid,
+    client: &AuthedClient,
+) -> Result<Vec<String>> {
+    match parsed.db_type.to_lowercase().as_str() {
+        "postgres" => {
+            let resource: PgDatabase = client
+                .get_resource_value_interpolated(parsed.resource_path, Some(job_id.to_string()))
+                .await?;
+
+            let attach_str = format!(
+                "ATTACH 'dbname={} {} host={} {} {}' AS {} (TYPE postgres{});",
+                resource.dbname,
+                resource
+                    .user
+                    .map(|u| format!("user={}", u))
+                    .unwrap_or_default(),
+                resource.host,
+                resource
+                    .password
+                    .map(|p| format!("password={}", p))
+                    .unwrap_or_default(),
+                resource
+                    .port
+                    .map(|p| format!("port={}", p))
+                    .unwrap_or_default(),
+                parsed.name,
+                parsed.extra_args.unwrap_or("")
+            );
+
+            Ok(vec![
+                "INSTALL postgres;".to_string(),
+                "LOAD postgres;".to_string(),
+                attach_str,
+            ])
+        }
+        "mysql" => {
+            #[cfg(not(feature = "mysql"))]
+            return Err(Error::ExecutionErr(
+                "MySQL feature is not enabled".to_string(),
+            ));
+
+            #[cfg(feature = "mysql")]
+            {
+                let resource: MysqlDatabase = client
+                    .get_resource_value_interpolated(parsed.resource_path, Some(job_id.to_string()))
+                    .await?;
+
+                let attach_str = format!(
+                    "ATTACH 'database={} host={} ssl_mode={} {} {} {}' AS {} (TYPE mysql{});",
+                    resource.database,
+                    resource.host,
+                    resource
+                        .ssl
+                        .map(|ssl| if ssl { "required" } else { "disabled" })
+                        .unwrap_or("preferred"),
+                    resource
+                        .password
+                        .map(|p| format!("password={}", p))
+                        .unwrap_or_default(),
+                    resource
+                        .port
+                        .map(|p| format!("port={}", p))
+                        .unwrap_or_default(),
+                    resource
+                        .user
+                        .map(|u| format!("user={}", u))
+                        .unwrap_or_default(),
+                    parsed.name,
+                    parsed.extra_args.unwrap_or("")
+                );
+
+                Ok(vec![
+                    "INSTALL mysql;".to_string(),
+                    "LOAD mysql;".to_string(),
+                    attach_str,
+                ])
+            }
+        }
+        "bigquery" => {
+            let resource: Value = client
+                .get_resource_value_interpolated(parsed.resource_path, Some(job_id.to_string()))
+                .await?;
+            // duckdb's bigquery extension requires a json file as credentials
+            let bq_credentials_path = make_bq_credentials_path(job_id);
+            env::set_var("GOOGLE_APPLICATION_CREDENTIALS", &bq_credentials_path);
+            tokio::fs::write(&bq_credentials_path, resource.to_string())
+                .await
+                .map_err(|e| {
+                    Error::ExecutionErr(format!(
+                        "Failed to write BigQuery credentials to {}: {}",
+                        &bq_credentials_path, e
+                    ))
+                })?;
+            let project_id: String = serde_json::from_value(
+                resource
+                    .get("project_id")
+                    .ok_or_else(|| {
+                        Error::ExecutionErr("BigQuery resource must contain project_id".to_string())
+                    })?
+                    .to_owned(),
+            )
+            .map_err(|_e| Error::ExecutionErr("failed project_id deserialize".to_string()))?;
+            let attach_str = format!(
+                "ATTACH 'project={}' as {} (TYPE bigquery{});",
+                project_id,
+                parsed.name,
+                parsed.extra_args.unwrap_or("")
+            )
+            .to_string();
+            Ok(vec![
+                "INSTALL bigquery FROM community;".to_string(),
+                "LOAD bigquery;".to_string(),
+                attach_str,
+            ])
+        }
+        _ => Err(Error::ExecutionErr(format!(
+            "Unsupported db type in DuckDB ATTACH: {}",
+            parsed.db_type
+        ))),
+    }
+}
+
+// Returns the transformed query and the set of storages used
+async fn transform_s3_uris(
+    query: &str,
+    client: &AuthedClient,
+) -> Result<(
+    Option<String>,
+    HashMap<Option<String>, DuckdbConnectionSettingsResponse>,
+)> {
+    let mut transformed_query = None;
+    lazy_static::lazy_static! {
+        static ref RE: regex::Regex = regex::Regex::new(r"'s3://([^'/]*)/([^']+)'").unwrap();
+    }
+    let mut used_storages = HashMap::new();
+    for cap in RE.captures_iter(query) {
+        if let (storage, Some(s3_path)) = (cap.get(1), cap.get(2)) {
+            let s3_path = s3_path.as_str();
+            let storage = match storage.map(|m| m.as_str()) {
+                Some("") | None => None,
+                Some(s) => Some(s.to_string()),
+            };
+            let original_str_lit =
+                format!("'s3://{}/{}'", storage.as_deref().unwrap_or(""), s3_path);
+            let duckdb_conn_settings = client
+                .get_duckdb_connection_settings(&DuckdbConnectionSettingsQueryV2 {
+                    s3_resource_path: None,
+                    storage: storage.clone(),
+                })
+                .await?;
+            let url = match &duckdb_conn_settings {
+                DuckdbConnectionSettingsResponse { s3_bucket: Some(bucket), .. } => {
+                    format!("'s3://{bucket}/{s3_path}'")
+                }
+                DuckdbConnectionSettingsResponse { azure_container_path: Some(base), .. } => {
+                    format!("'{base}/{s3_path}'")
+                }
+                _ => {
+                    return Err(Error::ExecutionErr(
+                        "DuckDB connection settings response must have either s3_bucket or azure_container_path".to_string(),
+                    ))?;
+                }
+            };
+            transformed_query = Some(
+                transformed_query
+                    .unwrap_or(query.to_string())
+                    .replace(&original_str_lit, &url),
+            );
+            used_storages.insert(storage, duckdb_conn_settings);
+        }
+    }
+    Ok((transformed_query, used_storages))
+}
+
+// BigQuery extension requires a json file as credentials
+// The file path is set as an env var by do_duckdb
+// It is created by transform_attach_db_resource_query (when bigquery is detected)
+// and deleted by do_duckdb after the query is executed
+fn make_bq_credentials_path(job_id: &Uuid) -> String {
+    format!("/tmp/service-account-credentials-{}.json", job_id)
+}
+
+// duckdb-rs does not support named parameters,
+// and it raises an error when passing unused arguments. We cannot prepare batch statements
+// but only single SQL statements so it doesn't work when all arguments are not used by
+// every single statement.
+fn interpolate_named_args<'a>(
+    query: &str,
+    args: &'a HashMap<String, duckdb::types::Value>,
+) -> (String, Vec<&'a duckdb::types::Value>) {
+    let mut query = query.to_string();
+
+    let mut values = vec![];
+    for (arg_name, arg_value) in args {
+        let pat = format!("${}", arg_name);
+        if !query.contains(&pat) {
+            continue;
+        }
+        values.push(arg_value);
+        query = query.replace(&pat, &format!("${}", values.len()));
+    }
+    (query, values)
+}
+
+fn trunc_sig(query: &str) -> &str {
+    let idx = query.rfind("-- $").unwrap_or(query.len());
+    // find next \n starting from idx and return everything after it
+    let idx = query[idx..].find('\n').map(|i| i + idx).unwrap_or(0);
+    &query[idx..]
+}

backend/windmill-worker/src/go_executor.rs

@@ -19,9 +19,10 @@ use crate::{
         start_child_process, OccupancyMetrics,
     },
     handle_child::handle_child,
-    AuthedClient, DISABLE_NSJAIL, DISABLE_NUSER, GOPRIVATE, GOPROXY, GO_BIN_CACHE_DIR,
-    GO_CACHE_DIR, HOME_ENV, NSJAIL_PATH, PATH_ENV, TZ_ENV,
+    DISABLE_NSJAIL, DISABLE_NUSER, GOPRIVATE, GOPROXY, GO_BIN_CACHE_DIR, GO_CACHE_DIR, HOME_ENV,
+    NSJAIL_PATH, PATH_ENV, TZ_ENV,
 };
+use windmill_common::client::AuthedClient;
 
 const GO_REQ_SPLITTER: &str = "//go.sum\n";
 const NSJAIL_CONFIG_RUN_GO_CONTENT: &str = include_str!("../nsjail/run.go.config.proto");

backend/windmill-worker/src/graphql_executor.rs

@@ -12,7 +12,8 @@ use serde::Deserialize;
 
 use crate::common::{build_http_client, resolve_job_timeout, OccupancyMetrics};
 use crate::handle_child::run_future_with_polling_update_job_poller;
-use crate::{common::build_args_map, AuthedClient};
+use crate::common::build_args_map;
+use windmill_common::client::AuthedClient;
 
 #[derive(Deserialize)]
 struct GraphqlApi {

backend/windmill-worker/src/java_executor.rs

@@ -24,9 +24,11 @@ use crate::{
         create_args_and_out_file, get_reserved_variables, par_install_language_dependencies,
         read_result, start_child_process, OccupancyMetrics, RequiredDependency,
     },
-    handle_child, AuthedClient, COURSIER_CACHE_DIR, DISABLE_NSJAIL, DISABLE_NUSER, JAVA_CACHE_DIR,
+    handle_child, COURSIER_CACHE_DIR, DISABLE_NSJAIL, DISABLE_NUSER, JAVA_CACHE_DIR,
     JAVA_REPOSITORY_DIR, MAVEN_REPOS, NO_DEFAULT_MAVEN, NSJAIL_PATH, PATH_ENV, PROXY_ENVS,
 };
+use windmill_common::client::AuthedClient;
+
 lazy_static::lazy_static! {
     static ref JAVA_CONCURRENT_DOWNLOADS: usize = std::env::var("JAVA_CONCURRENT_DOWNLOADS").ok().map(|flag| flag.parse().unwrap_or(20)).unwrap_or(20);
     static ref JAVA_PATH: String = std::env::var("JAVA_PATH").unwrap_or_else(|_| "/usr/bin/java".to_string());

backend/windmill-worker/src/job_logger_oss.rs

@@ -0,0 +1,30 @@
+use std::io;
+
+pub(crate) async fn s3_storage(
+    _value: &str,
+    _job_id: &uuid::Uuid,
+    _w_id: &str,
+    _db: &windmill_common::DB,
+    _offset: usize,
+) {
+    crate::job_logger_ee::s3_storage(_value, _job_id, _w_id, _db, _offset).await
+}
+
+pub(crate) async fn default_disk_log_storage(
+    _value: &str,
+    _job_id: &uuid::Uuid,
+    _w_id: &str,
+    _offset: usize,
+) {
+    crate::job_logger_ee::default_disk_log_storage(_value, _job_id, _w_id, _offset).await
+}
+
+pub(crate) fn process_streaming_log_lines(
+    _line: &str,
+    _job_id: &uuid::Uuid,
+    _w_id: &str,
+    _logs: &[String],
+    _offset: usize,
+) -> Option<Result<String, io::Error>> {
+    crate::job_logger_ee::process_streaming_log_lines(_line, _job_id, _w_id, _logs, _offset)
+}

backend/windmill-worker/src/js_eval.rs

@@ -48,7 +48,8 @@ use windmill_common::worker::{write_file, TMP_DIR};
 use windmill_common::flow_status::JobResult;
 use windmill_queue::CanceledBy;
 
-use crate::{common::OccupancyMetrics, AuthedClient};
+use crate::common::OccupancyMetrics;
+use windmill_common::client::AuthedClient;
 
 #[cfg(feature = "deno_core")]
 use crate::{common::unsafe_raw, handle_child::run_future_with_polling_update_job_poller};

backend/windmill-worker/src/lib.rs

@@ -20,6 +20,8 @@ mod csharp_executor;
 #[cfg(feature = "enterprise")]
 mod dedicated_worker;
 mod deno_executor;
+#[cfg(feature = "duckdb")]
+mod duckdb_executor;
 mod global_cache;
 mod go_executor;
 mod graphql_executor;

backend/windmill-worker/src/mssql_executor.rs

@@ -22,7 +22,7 @@ use windmill_queue::{append_logs, CanceledBy};
 use crate::common::{build_args_values, s3_mode_args_to_worker_data, OccupancyMetrics};
 use crate::handle_child::run_future_with_polling_update_job_poller;
 use crate::sanitized_sql_params::sanitize_and_interpolate_unsafe_sql_args;
-use crate::AuthedClient;
+use windmill_common::client::AuthedClient;
 
 use serde::Deserializer;
 

backend/windmill-worker/src/mysql_executor.rs

@@ -13,6 +13,7 @@ use serde_json::{json, value::RawValue, Value};
 use std::str::FromStr;
 use tokio::sync::Mutex;
 use windmill_common::{
+    client::AuthedClient,
     error::{to_anyhow, Error},
     s3_helpers::convert_json_line_stream,
     worker::{to_raw_value, Connection},
@@ -28,17 +29,16 @@ use crate::{
     common::{build_args_values, s3_mode_args_to_worker_data, OccupancyMetrics, S3ModeWorkerData},
     handle_child::run_future_with_polling_update_job_poller,
     sanitized_sql_params::sanitize_and_interpolate_unsafe_sql_args,
-    AuthedClient,
 };
 
 #[derive(Deserialize)]
-struct MysqlDatabase {
-    host: String,
-    user: Option<String>,
-    password: Option<String>,
-    port: Option<u16>,
-    database: String,
-    ssl: Option<bool>,
+pub struct MysqlDatabase {
+    pub host: String,
+    pub user: Option<String>,
+    pub password: Option<String>,
+    pub port: Option<u16>,
+    pub database: String,
+    pub ssl: Option<bool>,
 }
 
 fn do_mysql_inner<'a>(

backend/windmill-worker/src/nu_executor.rs

@@ -16,8 +16,10 @@ use crate::{
         create_args_and_out_file, get_reserved_variables, read_result, start_child_process,
         OccupancyMetrics,
     },
-    handle_child, AuthedClient, DISABLE_NSJAIL, DISABLE_NUSER, NSJAIL_PATH, PATH_ENV, PROXY_ENVS,
+    handle_child, DISABLE_NSJAIL, DISABLE_NUSER, NSJAIL_PATH, PATH_ENV, PROXY_ENVS,
 };
+use windmill_common::client::AuthedClient;
+
 
 const NSJAIL_CONFIG_RUN_NU_CONTENT: &str = include_str!("../nsjail/run.nu.config.proto");
 lazy_static::lazy_static! {

backend/windmill-worker/src/oracledb_executor.rs

@@ -27,9 +27,9 @@ use crate::{
         OccupancyMetrics, S3ModeWorkerData,
     },
     handle_child::run_future_with_polling_update_job_poller,
-    sanitized_sql_params::sanitize_and_interpolate_unsafe_sql_args,
-    AuthedClient,
+    sanitized_sql_params::sanitize_and_interpolate_unsafe_sql_args
 };
+use windmill_common::client::AuthedClient;
 
 #[derive(Deserialize)]
 struct OracleDatabase {

backend/windmill-worker/src/otel_oss.rs

@@ -0,0 +1,5 @@
+use windmill_queue::MiniPulledJob;
+
+pub fn add_root_flow_job_to_otlp(_queued_job: &MiniPulledJob, _success: bool) {
+    crate::otel_ee::add_root_flow_job_to_otlp(_queued_job, _success)
+}

backend/windmill-worker/src/pg_executor.rs

@@ -41,20 +41,20 @@ use crate::common::{
 };
 use crate::handle_child::run_future_with_polling_update_job_poller;
 use crate::sanitized_sql_params::sanitize_and_interpolate_unsafe_sql_args;
-use crate::{AuthedClient, MAX_RESULT_SIZE};
+use crate::MAX_RESULT_SIZE;
 use bytes::Buf;
 use lazy_static::lazy_static;
 use urlencoding::encode;
-
+use windmill_common::client::AuthedClient;
 #[derive(Deserialize)]
-struct PgDatabase {
-    host: String,
-    user: Option<String>,
-    password: Option<String>,
-    port: Option<u16>,
-    sslmode: Option<String>,
-    dbname: String,
-    root_certificate_pem: Option<String>,
+pub struct PgDatabase {
+    pub host: String,
+    pub user: Option<String>,
+    pub password: Option<String>,
+    pub port: Option<u16>,
+    pub sslmode: Option<String>,
+    pub dbname: String,
+    pub root_certificate_pem: Option<String>,
 }
 
 lazy_static! {

backend/windmill-worker/src/php_executor.rs

@@ -20,9 +20,10 @@ use crate::{
         read_result, start_child_process, OccupancyMetrics,
     },
     handle_child::handle_child,
-    AuthedClient, COMPOSER_CACHE_DIR, COMPOSER_PATH, DISABLE_NSJAIL, DISABLE_NUSER, NSJAIL_PATH,
+    COMPOSER_CACHE_DIR, COMPOSER_PATH, DISABLE_NSJAIL, DISABLE_NUSER, NSJAIL_PATH,
     PHP_PATH,
 };
+use windmill_common::client::AuthedClient;
 
 const NSJAIL_CONFIG_RUN_PHP_CONTENT: &str = include_str!("../nsjail/run.php.config.proto");
 

backend/windmill-worker/src/python_executor.rs

@@ -70,7 +70,7 @@ const RELATIVE_PYTHON_LOADER: &str = include_str!("../loader.py");
 use crate::global_cache::{build_tar_and_push, pull_from_tar};
 
 #[cfg(all(feature = "enterprise", feature = "parquet", unix))]
-use windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS;
+use windmill_common::s3_helpers::OBJECT_STORE_SETTINGS;
 
 use crate::{
     common::{
@@ -79,9 +79,10 @@ use crate::{
     },
     handle_child::handle_child,
     worker_utils::ping_job_status,
-    AuthedClient, PyV, PyVAlias, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV,
+    PyV, PyVAlias, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV,
     PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, PROXY_ENVS, PY_INSTALL_DIR, TZ_ENV, UV_CACHE_DIR,
 };
+use windmill_common::client::AuthedClient;
 
 #[cfg(windows)]
 use crate::SYSTEM_ROOT;
@@ -1424,7 +1425,7 @@ pub async fn handle_python_reqs(
         }
 
         #[cfg(all(feature = "enterprise", feature = "parquet", unix))]
-        if OBJECT_STORE_CACHE_SETTINGS.read().await.is_none() {
+        if OBJECT_STORE_SETTINGS.read().await.is_none() {
             (s3_pull, s3_push) = (false, false);
         }
 
@@ -1735,7 +1736,7 @@ pub async fn handle_python_reqs(
             let start = std::time::Instant::now();
             #[cfg(all(feature = "enterprise", feature = "parquet", unix))]
             if is_not_pro {
-                if let Some(os) = OBJECT_STORE_CACHE_SETTINGS.read().await.clone() {
+                if let Some(os) = windmill_common::s3_helpers::get_object_store().await {
                     tokio::select! {
                         // Cancel was called on the job
                         _ = kill_rx.recv() => return Err(anyhow::anyhow!("S3 pull was canceled")),
@@ -1889,7 +1890,7 @@ pub async fn handle_python_reqs(
 
             #[cfg(all(feature = "enterprise", feature = "parquet", unix))]
             if s3_push {
-                if let Some(os) = OBJECT_STORE_CACHE_SETTINGS.read().await.clone() {
+                if let Some(os) = windmill_common::s3_helpers::get_object_store().await {
                     tokio::spawn(build_tar_and_push(os, venv_p.clone(), py_version.to_cache_dir_top_level(), None, false));
                 }
             }

backend/windmill-worker/src/result_processor.rs

@@ -29,8 +29,7 @@ use windmill_common::{
 use windmill_common::bench::{BenchmarkInfo, BenchmarkIter};
 
 use windmill_queue::{
-    append_logs, get_queued_job,  CanceledBy, JobCompleted, MiniPulledJob,
-    WrappedError,
+    append_logs, get_queued_job, CanceledBy, JobCompleted, MiniPulledJob, WrappedError,
 };
 
 use serde_json::{json, value::RawValue};
@@ -44,9 +43,10 @@ use crate::{
     common::{error_to_value, read_result, save_in_cache},
     otel_ee::add_root_flow_job_to_otlp,
     worker_flow::update_flow_status_after_job_completion,
-    AuthedClient, JobCompletedReceiver, JobCompletedSender, SameWorkerSender, SendResult,
-    UpdateFlow, INIT_SCRIPT_TAG,
+    JobCompletedReceiver, JobCompletedSender, SameWorkerSender, SendResult, UpdateFlow,
+    INIT_SCRIPT_TAG,
 };
+use windmill_common::client::AuthedClient;
 
 async fn process_jc(
     jc: JobCompleted,
@@ -273,11 +273,7 @@ pub fn start_background_processor(
     })
 }
 
-async fn send_job_completed(
-    job_completed_tx: JobCompletedSender,
-    jc: JobCompleted,
-
-) {
+async fn send_job_completed(job_completed_tx: JobCompletedSender, jc: JobCompleted) {
     job_completed_tx
         .send_job(jc, true)
         .with_context(windmill_common::otel_ee::otel_ctx())
@@ -301,7 +297,6 @@ pub async fn process_result(
 ) -> error::Result<bool> {
     match result {
         Ok(result) => {
-
             send_job_completed(
                 job_completed_tx,
                 JobCompleted {

backend/windmill-worker/src/rust_executor.rs

@@ -19,9 +19,10 @@ use crate::{
         read_result, start_child_process, OccupancyMetrics,
     },
     handle_child::handle_child,
-    AuthedClient, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, PROXY_ENVS,
+    DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, PROXY_ENVS,
     RUST_CACHE_DIR, TZ_ENV,
 };
+use windmill_common::client::AuthedClient;
 
 #[cfg(windows)]
 use crate::SYSTEM_ROOT;

backend/windmill-worker/src/snowflake_executor.rs

@@ -26,7 +26,8 @@ use crate::common::{
 };
 use crate::handle_child::run_future_with_polling_update_job_poller;
 use crate::sanitized_sql_params::sanitize_and_interpolate_unsafe_sql_args;
-use crate::{common::build_args_values, AuthedClient};
+use crate::common::build_args_values;
+use windmill_common::client::AuthedClient;
 
 #[derive(Serialize)]
 struct Claims {

backend/windmill-worker/src/windmill-client.js

@@ -2998,6 +2998,8 @@ var $RawScript = {
         "mssql",
         "graphql",
         "nativets",
+        "duckdb",
+        // for related places search: ADD_NEW_LANG
       ],
     },
     path: {

backend/windmill-worker/src/worker.rs

@@ -11,6 +11,7 @@
 
 use anyhow::anyhow;
 use futures::TryFutureExt;
+use windmill_common::client::AuthedClient;
 use windmill_common::{
     agent_workers::DECODED_AGENT_TOKEN,
     apps::AppScriptId,
@@ -28,7 +29,7 @@ use windmill_common::{
 #[cfg(feature = "enterprise")]
 use windmill_common::ee::LICENSE_KEY_VALID;
 
-use anyhow::{Context, Result};
+use anyhow::Result;
 use const_format::concatcp;
 #[cfg(feature = "prometheus")]
 use prometheus::IntCounter;
@@ -39,8 +40,7 @@ use windmill_common::METRICS_DEBUG_ENABLED;
 #[cfg(feature = "prometheus")]
 use windmill_common::METRICS_ENABLED;
 
-use reqwest::{Body, Response};
-use serde::{de::DeserializeOwned, Deserialize, Serialize};
+use serde::{Deserialize, Serialize};
 use sqlx::types::Json;
 use std::{
     collections::HashMap,
@@ -145,6 +145,9 @@ use crate::ansible_executor::handle_ansible_job;
 #[cfg(feature = "mysql")]
 use crate::mysql_executor::do_mysql;
 
+#[cfg(feature = "duckdb")]
+use crate::duckdb_executor::do_duckdb;
+
 #[cfg(feature = "oracledb")]
 use crate::oracledb_executor::do_oracledb;
 
@@ -393,201 +396,6 @@ pub const MAX_RESULT_SIZE: usize = 1024 * 1024 * 2; // 2MB
 
 pub const INIT_SCRIPT_TAG: &str = "init_script";
 
-#[derive(Clone)]
-pub struct AuthedClient {
-    pub base_internal_url: String,
-    pub workspace: String,
-    pub token: String,
-    pub force_client: Option<reqwest::Client>,
-}
-
-impl AuthedClient {
-    pub async fn get(&self, url: &str, query: Vec<(&str, String)>) -> anyhow::Result<Response> {
-        self.force_client
-            .as_ref()
-            .unwrap_or(&HTTP_CLIENT)
-            .get(url)
-            .query(&query)
-            .header(
-                reqwest::header::ACCEPT,
-                reqwest::header::HeaderValue::from_static("application/json"),
-            )
-            .header(
-                reqwest::header::AUTHORIZATION,
-                reqwest::header::HeaderValue::from_str(&format!("Bearer {}", self.token))?,
-            )
-            .send()
-            .await
-            .map_err(|e| {
-                tracing::error!("Error executing get request from authed http client to {url} with query {query:?}: {e}");
-                anyhow::anyhow!("Error executing get request from authed http client to {url} with query {query:?}: {e}")
-            })
-    }
-
-    pub async fn get_id_token(&self, audience: &str) -> anyhow::Result<String> {
-        let url = format!(
-            "{}/api/w/{}/oidc/token/{}",
-            self.base_internal_url, self.workspace, audience
-        );
-        let response = self.get(&url, vec![]).await?;
-        match response.status().as_u16() {
-            200u16 => Ok(response
-                .json::<String>()
-                .await
-                .context("decoding oidc token as json string")?),
-            _ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())),
-        }
-    }
-
-    pub async fn get_resource_value<T: DeserializeOwned>(&self, path: &str) -> anyhow::Result<T> {
-        let url = format!(
-            "{}/api/w/{}/resources/get_value/{}",
-            self.base_internal_url, self.workspace, path
-        );
-        let response = self.get(&url, vec![]).await?;
-        match response.status().as_u16() {
-            200u16 => Ok(response
-                .json::<T>()
-                .await
-                .context("decoding resource value as json")?),
-            _ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())),
-        }
-    }
-
-    pub async fn get_variable_value(&self, path: &str) -> anyhow::Result<String> {
-        let url = format!(
-            "{}/api/w/{}/variables/get_value/{}",
-            self.base_internal_url, self.workspace, path
-        );
-        let response = self.get(&url, vec![]).await?;
-        match response.status().as_u16() {
-            200u16 => Ok(response
-                .json::<String>()
-                .await
-                .context("decoding variable value as json")?),
-            _ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())),
-        }
-    }
-
-    pub async fn get_resource_value_interpolated<T: DeserializeOwned>(
-        &self,
-        path: &str,
-        job_id: Option<String>,
-    ) -> anyhow::Result<T> {
-        let url = format!(
-            "{}/api/w/{}/resources/get_value_interpolated/{}",
-            self.base_internal_url, self.workspace, path
-        );
-        let mut query = Vec::with_capacity(1usize);
-        if let Some(v) = &job_id {
-            query.push(("job_id", v.to_string()));
-        }
-        let response = self.get(&url, query).await?;
-        match response.status().as_u16() {
-            200u16 => Ok(response
-                .json::<T>()
-                .await
-                .context("decoding interpolated resource value as json")?),
-            _ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())),
-        }
-    }
-
-    pub async fn get_completed_job_result<T: DeserializeOwned>(
-        &self,
-        path: &str,
-        json_path: Option<String>,
-    ) -> anyhow::Result<T> {
-        let url = format!(
-            "{}/api/w/{}/jobs_u/completed/get_result/{}",
-            self.base_internal_url, self.workspace, path
-        );
-        let query = if let Some(json_path) = json_path {
-            vec![("json_path", json_path)]
-        } else {
-            vec![]
-        };
-        let response = self.get(&url, query).await?;
-        match response.status().as_u16() {
-            200u16 => Ok(response
-                .json::<T>()
-                .await
-                .context("decoding completed job result as json")?),
-            _ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())),
-        }
-    }
-
-    pub async fn get_result_by_id<T: DeserializeOwned>(
-        &self,
-        flow_job_id: &str,
-        node_id: &str,
-        json_path: Option<String>,
-    ) -> anyhow::Result<T> {
-        let url = format!(
-            "{}/api/w/{}/jobs/result_by_id/{}/{}",
-            self.base_internal_url, self.workspace, flow_job_id, node_id
-        );
-        let query = if let Some(json_path) = json_path {
-            vec![("json_path", json_path)]
-        } else {
-            vec![]
-        };
-        let response = self.get(&url, query).await?;
-        match response.status().as_u16() {
-            200u16 => Ok(response
-                .json::<T>()
-                .await
-                .context("decoding result by id as json")?),
-            _ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())),
-        }
-    }
-
-    pub async fn upload_s3_file<S>(
-        &self,
-        workspace_id: &str,
-        object_key: String,
-        storage: Option<String>,
-        body: S,
-    ) -> error::Result<()>
-    where
-        S: futures::stream::TryStream + Send + 'static,
-        S::Error: Into<Box<dyn std::error::Error + Send + Sync>>,
-        bytes::Bytes: From<S::Ok>,
-    {
-        let mut query = vec![("file_key", object_key)];
-        if let Some(storage) = storage {
-            query.push(("storage", storage));
-        }
-        let response = self
-            .force_client
-            .as_ref()
-            .unwrap_or(&HTTP_CLIENT)
-            .post(format!(
-                "{}/api/w/{}/job_helpers/upload_s3_file",
-                self.base_internal_url, workspace_id
-            ))
-            .query(&query)
-            .header(
-                reqwest::header::ACCEPT,
-                reqwest::header::HeaderValue::from_static("application/json"),
-            )
-            .header(
-                reqwest::header::AUTHORIZATION,
-                reqwest::header::HeaderValue::from_str(&format!("Bearer {}", self.token))
-                    .map_err(|e| error::Error::BadConfig(e.to_string()))?,
-            )
-            .body(Body::wrap_stream(body))
-            .send()
-            .await
-            .context(format!("Sent upload_s3_file request",))
-            .map_err(error::Error::from)?;
-
-        match response.status().as_u16() {
-            200u16 => Ok(()),
-            _ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default()))?,
-        }
-    }
-}
-
 #[derive(Clone)]
 pub struct SameWorkerSender(pub Sender<SameWorkerPayload>, pub Arc<AtomicU16>);
 
@@ -2809,6 +2617,30 @@ async fn handle_code_execution_job(
             )
             .await;
         }
+    } else if language == Some(ScriptLang::DuckDb) {
+        #[allow(unreachable_code)]
+        #[cfg(not(feature = "duckdb"))]
+        {
+            return Err(Error::internal_err(
+                "Duck DB requires the duckdb feature to be enabled".to_string(),
+            ));
+        }
+
+        #[cfg(feature = "duckdb")]
+        {
+            return do_duckdb(
+                job,
+                &client,
+                &code,
+                conn,
+                mem_peak,
+                canceled_by,
+                worker_name,
+                column_order,
+                occupancy_metrics,
+            )
+            .await;
+        }
     } else if language == Some(ScriptLang::Graphql) {
         return do_graphql(
             job,
@@ -3216,6 +3048,7 @@ fn parse_sig_of_lang(
             ScriptLang::Snowflake => Some(windmill_parser_sql::parse_snowflake_sig(code)?),
             ScriptLang::Graphql => None,
             ScriptLang::Mssql => Some(windmill_parser_sql::parse_mssql_sig(code)?),
+            ScriptLang::DuckDb => Some(windmill_parser_sql::parse_duckdb_sig(code)?),
             ScriptLang::OracleDB => Some(windmill_parser_sql::parse_oracledb_sig(code)?),
             #[cfg(feature = "php")]
             ScriptLang::Php => Some(windmill_parser_php::parse_php_signature(

backend/windmill-worker/src/worker_flow.rs

@@ -15,8 +15,7 @@ use crate::common::{cached_result_path, save_in_cache};
 use crate::js_eval::{eval_timeout, IdContext};
 use crate::worker_utils::get_tag_and_concurrency;
 use crate::{
-    AuthedClient, JobCompletedSender, PreviousResult, SameWorkerSender, SendResult, UpdateFlow,
-    KEEP_JOB_DIR,
+    JobCompletedSender, PreviousResult, SameWorkerSender, SendResult, UpdateFlow, KEEP_JOB_DIR,
 };
 use anyhow::Context;
 use futures::TryFutureExt;
@@ -32,6 +31,7 @@ use windmill_common::auth::JobPerms;
 #[cfg(feature = "benchmark")]
 use windmill_common::bench::BenchmarkIter;
 use windmill_common::cache::{self, RawData};
+use windmill_common::client::AuthedClient;
 use windmill_common::db::Authed;
 use windmill_common::flow_status::{
     ApprovalConditions, FlowStatusModuleWParent, Iterator as FlowIterator, JobResult,

backend/windmill-worker/src/worker_lockfiles.rs

@@ -1964,13 +1964,12 @@ async fn ansible_dep(
 ) -> std::result::Result<String, Error> {
     use windmill_parser_yaml::add_versions_to_requirements_yaml;
 
-    use crate::{
-        ansible_executor::{
+    use crate::ansible_executor::{
             create_ansible_cfg, get_collection_locks, get_git_ssh_cmd, get_role_locks,
             install_galaxy_collections,
-        },
-        AuthedClient,
-    };
+        };
+    use windmill_common::client::AuthedClient;
+
 
     let python_lockfile = python_dep(
         reqs.python_reqs.join("\n").to_string(),

benchmarks/lib.ts

@@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts";
 import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts";
 import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts";
 
-export const VERSION = "v1.492.1";
+export const VERSION = "v1.493.2";
 
 export async function login(email: string, password: string): Promise<string> {
   return await windmill.UserService.login({

cli/bootstrap/script_bootstrap.ts

@@ -76,6 +76,7 @@ func main() (interface{}, error) {
 
   bash: `echo "Hello world"
 `,
+  duckdb: `SELECT 'Hello world' AS message`,
 
   oracledb: `SELECT 'Hello world' AS message`,
   powershell: `Write-Output "Hello world"`,
@@ -126,5 +127,5 @@ public class Main {
   }
 }
 `,
-// for related places search: ADD_NEW_LANG 
+  // for related places search: ADD_NEW_LANG
 };

cli/build.sh

@@ -1,3 +1,4 @@
 #!/bin/bash
+# Note for mac OS users: you need to install gnu-sed with `brew install gnu-sed` and use `gsed` instead of `sed`.
 ./gen_wm_client.sh
 deno run -A dnt.ts

cli/main.ts

@@ -63,7 +63,7 @@ export {
 //   }
 // });
 
-export const VERSION = "1.492.1";
+export const VERSION = "1.493.2";
 
 const command = new Command()
   .name("wmill")

cli/metadata.ts

@@ -528,6 +528,9 @@ export async function inferSchema(
       { name: "database", typ: { resource: "postgresql" } },
       ...inferedSchema.args,
     ];
+  } else if (language === "duckdb") {
+    const { parse_sql } = await import("./wasm/regex/windmill_parser_wasm.js");
+    inferedSchema = JSON.parse(parse_sql(content));
   } else if (language === "graphql") {
     const { parse_graphql } = await import(
       "./wasm/regex/windmill_parser_wasm.js"

cli/script.ts

@@ -547,6 +547,8 @@ export function filePathExtensionFromContentType(
     return ".my.sql";
   } else if (language === "bigquery") {
     return ".bq.sql";
+  } else if (language === "duckdb") {
+    return ".duckdb.sql";
   } else if (language === "oracledb") {
     return ".odb.sql";
   } else if (language === "snowflake") {
@@ -573,7 +575,7 @@ export function filePathExtensionFromContentType(
     return ".nu";
   } else if (language === "java") {
     return ".java";
-    // for related places search: ADD_NEW_LANG 
+    // for related places search: ADD_NEW_LANG
   } else {
     throw new Error("Invalid language: " + language);
   }
@@ -593,6 +595,7 @@ export const exts = [
   ".odb.sql",
   ".sf.sql",
   ".ms.sql",
+  ".duckdb.sql",
   ".sql",
   ".gql",
   ".ps1",
@@ -601,9 +604,8 @@ export const exts = [
   ".cs",
   ".nu",
   ".playbook.yml",
-  ".java"
-  // for related places search: ADD_NEW_LANG 
-
+  ".java",
+  // for related places search: ADD_NEW_LANG
 ];
 
 export function removeExtensionToPath(path: string): string {