fix: optimize S3 proxy performance (#8685)

* perf: re-export GetOptions and GetRange from object_store Needed by S3 proxy to use get_opts with range for single-request range fetches instead of HEAD + get_range. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * Avoid logging S3 proxy requests as info * Revert "Avoid logging S3 proxy requests as info" This reverts commit b6359a7a03. * Don't log s3 proxy * Revert "Don't log s3 proxy" This reverts commit 2b21ee3c78. * Update duckdb * AUTOMATIC_MIGRATION for ducklake * ee repo ref * wrong comment * chore: update ee-repo-ref to 41b0d1cb312919109407640fc4bd7060cfe0e107 This commit updates the EE repository reference after PR #505 was merged in windmill-ee-private. Previous ee-repo-ref: 9b97a1c563365006657c4c6cde6e7df31c5173c3 New ee-repo-ref: 41b0d1cb312919109407640fc4bd7060cfe0e107 Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com> Co-authored-by: windmill-internal-app[bot] <windmill-internal-app[bot]@users.noreply.github.com> Co-authored-by: Ruben Fiszel <ruben@windmill.dev>
2026-04-03 13:53:10 +02:00
parent 6656b46f10
commit 0cfa462c37
5 changed files with 54 additions and 77 deletions
--- a/backend/ee-repo-ref.txt
+++ b/backend/ee-repo-ref.txt
@@ -1 +1 @@
-94567b204a5536ec3dc7591830c58c5bdc1d8381
+41b0d1cb312919109407640fc4bd7060cfe0e107
--- a/backend/windmill-duckdb-ffi-internal/Cargo.lock
+++ b/backend/windmill-duckdb-ffi-internal/Cargo.lock
@@ -68,9 +68,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"

 [[package]]
 name = "arrow"
-version = "56.2.0"
+version = "58.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6e833808ff2d94ed40d9379848a950d995043c7fb3e81a30b383f4c6033821cc"
+checksum = "d441fdda254b65f3e9025910eb2c2066b6295d9c8ed409522b8d2ace1ff8574c"
 dependencies = [
 "arrow-arith",
 "arrow-array",
@@ -86,23 +86,23 @@ dependencies = [

 [[package]]
 name = "arrow-arith"
-version = "56.2.0"
+version = "58.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ad08897b81588f60ba983e3ca39bda2b179bdd84dced378e7df81a5313802ef8"
+checksum = "ced5406f8b720cc0bc3aa9cf5758f93e8593cda5490677aa194e4b4b383f9a59"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
 "arrow-data",
 "arrow-schema",
 "chrono",
- "num",
+ "num-traits",
 ]

 [[package]]
 name = "arrow-array"
-version = "56.2.0"
+version = "58.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8548ca7c070d8db9ce7aa43f37393e4bfcf3f2d3681df278490772fd1673d08d"
+checksum = "772bd34cacdda8baec9418d80d23d0fb4d50ef0735685bd45158b83dfeb6e62d"
 dependencies = [
 "ahash 0.8.12",
 "arrow-buffer",
@@ -111,29 +111,33 @@ dependencies = [
 "chrono",
 "half",
 "hashbrown 0.16.1",
- "num",
+ "num-complex",
+ "num-integer",
+ "num-traits",
 ]

 [[package]]
 name = "arrow-buffer"
-version = "56.2.0"
+version = "58.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e003216336f70446457e280807a73899dd822feaf02087d31febca1363e2fccc"
+checksum = "898f4cf1e9598fdb77f356fdf2134feedfd0ee8d5a4e0a5f573e7d0aec16baa4"
 dependencies = [
 "bytes",
 "half",
- "num",
+ "num-bigint",
+ "num-traits",
 ]

 [[package]]
 name = "arrow-cast"
-version = "56.2.0"
+version = "58.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "919418a0681298d3a77d1a315f625916cb5678ad0d74b9c60108eb15fd083023"
+checksum = "b0127816c96533d20fc938729f48c52d3e48f99717e7a0b5ade77d742510736d"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
 "arrow-data",
+ "arrow-ord",
 "arrow-schema",
 "arrow-select",
 "atoi",
@@ -142,27 +146,28 @@ dependencies = [
 "comfy-table",
 "half",
 "lexical-core",
- "num",
+ "num-traits",
 "ryu",
 ]

 [[package]]
 name = "arrow-data"
-version = "56.2.0"
+version = "58.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a5c64fff1d142f833d78897a772f2e5b55b36cb3e6320376f0961ab0db7bd6d0"
+checksum = "42d10beeab2b1c3bb0b53a00f7c944a178b622173a5c7bcabc3cb45d90238df4"
 dependencies = [
 "arrow-buffer",
 "arrow-schema",
 "half",
- "num",
+ "num-integer",
+ "num-traits",
 ]

 [[package]]
 name = "arrow-ord"
-version = "56.2.0"
+version = "58.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3c8f82583eb4f8d84d4ee55fd1cb306720cddead7596edce95b50ee418edf66f"
+checksum = "763a7ba279b20b52dad300e68cfc37c17efa65e68623169076855b3a9e941ca5"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -173,9 +178,9 @@ dependencies = [

 [[package]]
 name = "arrow-row"
-version = "56.2.0"
+version = "58.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9d07ba24522229d9085031df6b94605e0f4b26e099fb7cdeec37abd941a73753"
+checksum = "e14fe367802f16d7668163ff647830258e6e0aeea9a4d79aaedf273af3bdcd3e"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -186,32 +191,32 @@ dependencies = [

 [[package]]
 name = "arrow-schema"
-version = "56.2.0"
+version = "58.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b3aa9e59c611ebc291c28582077ef25c97f1975383f1479b12f3b9ffee2ffabe"
+checksum = "c30a1365d7a7dc50cc847e54154e6af49e4c4b0fddc9f607b687f29212082743"
 dependencies = [
 "bitflags",
 ]

 [[package]]
 name = "arrow-select"
-version = "56.2.0"
+version = "58.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8c41dbbd1e97bfcaee4fcb30e29105fb2c75e4d82ae4de70b792a5d3f66b2e7a"
+checksum = "78694888660a9e8ac949853db393af2a8b8fc82c19ce333132dfa2e72cc1a7fe"
 dependencies = [
 "ahash 0.8.12",
 "arrow-array",
 "arrow-buffer",
 "arrow-data",
 "arrow-schema",
- "num",
+ "num-traits",
 ]

 [[package]]
 name = "arrow-string"
-version = "56.2.0"
+version = "58.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "53f5183c150fbc619eede22b861ea7c0eebed8eaac0333eaa7f6da5205fd504d"
+checksum = "61e04a01f8bb73ce54437514c5fd3ee2aa3e8abe4c777ee5cc55853b1652f79e"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -219,7 +224,7 @@ dependencies = [
 "arrow-schema",
 "arrow-select",
 "memchr",
- "num",
+ "num-traits",
 "regex",
 "regex-syntax",
 ]
@@ -445,9 +450,9 @@ dependencies = [

 [[package]]
 name = "duckdb"
-version = "1.4.4"
+version = "1.10501.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8685352ce688883098b61a361e86e87df66fc8c444f4a2411e884c16d5243a65"
+checksum = "f13bc6d6487032fc2825a62ef8b4924b2378a2eb3166e132e5f3141ae9dd633f"
 dependencies = [
 "arrow",
 "cast",
@@ -1007,9 +1012,9 @@ checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112"

 [[package]]
 name = "libduckdb-sys"
-version = "1.4.4"
+version = "1.10501.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d78bacb8933586cee3b550c39b610d314f9b7a48701ac7a914a046165a4ad8da"
+checksum = "12096c1694924782b3fe21e790630b77bacb4fcb7ad9d7ee0fec626f985bf248"
 dependencies = [
 "cc",
 "flate2",
@@ -1090,20 +1095,6 @@ dependencies = [
 "windows-sys 0.61.2",
 ]

-[[package]]
-name = "num"
-version = "0.4.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23"
-dependencies = [
- "num-bigint",
- "num-complex",
- "num-integer",
- "num-iter",
- "num-rational",
- "num-traits",
-]
-
 [[package]]
 name = "num-bigint"
 version = "0.4.6"
@@ -1132,28 +1123,6 @@ dependencies = [
 "num-traits",
 ]

-[[package]]
-name = "num-iter"
-version = "0.1.45"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf"
-dependencies = [
- "autocfg",
- "num-integer",
- "num-traits",
-]
-
-[[package]]
-name = "num-rational"
-version = "0.4.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824"
-dependencies = [
- "num-bigint",
- "num-integer",
- "num-traits",
-]
-
 [[package]]
 name = "num-traits"
 version = "0.2.19"
--- a/backend/windmill-duckdb-ffi-internal/Cargo.toml
+++ b/backend/windmill-duckdb-ffi-internal/Cargo.toml
@@ -5,7 +5,7 @@ edition = "2024"

 [dependencies]
 chrono = "0.4.41"
-duckdb = { version = "1.4.4", features = ["bundled"] }
+duckdb = { version = "1.10501.0", features = ["bundled"] }
 regex = "1"
 rust_decimal = "1.37.2"
 serde = { version = "1.0", features = ["derive"] }
--- a/backend/windmill-object-store/src/lib.rs
+++ b/backend/windmill-object-store/src/lib.rs
@@ -65,8 +65,9 @@ pub mod object_store_reexports {
    pub use object_store::memory::InMemory;
    pub use object_store::path::Path;
    pub use object_store::{
-        Attribute, Attributes, Error as ObjectStoreError, GetResult, ObjectStore, PutMultipartOpts,
-        PutPayload, PutResult, Result as ObjectStoreResult, WriteMultipart,
+        Attribute, Attributes, Error as ObjectStoreError, GetOptions, GetRange, GetResult,
+        ObjectStore, PutMultipartOpts, PutPayload, PutResult, Result as ObjectStoreResult,
+        WriteMultipart,
    };
 }

--- a/backend/windmill-worker/src/duckdb_executor.rs
+++ b/backend/windmill-worker/src/duckdb_executor.rs
@@ -649,6 +649,12 @@ async fn transform_attach_ducklake(
        .unwrap_or(DEFAULT_STORAGE);
    let data_path = ducklake.storage.path;

+
+    let extra_args = if let Some(default_extra_args) = ducklake.extra_args {
+        format!("{},{}", extra_args, default_extra_args)
+    } else {
+        extra_args
+    };
    // Ducklake 0.3 only requires DATA_PATH at creation and then stores it internally in the catalog
    // But it will fail if DATA_PATH changes afterwards which is annoying for us
    // So we always enable override
@@ -657,12 +663,13 @@ async fn transform_attach_ducklake(
    } else {
        format!(", OVERRIDE_DATA_PATH TRUE{extra_args}")
    };
-    let extra_args = if let Some(default_extra_args) = ducklake.extra_args {
-        // premise : extra_args is always non empty (and doesn't end with a comma given it's valid)
-        format!("{},{}", extra_args, default_extra_args)
-    } else {
+    // Automatically migrate ducklake
+    let extra_args = if extra_args.contains("AUTOMATIC_MIGRATION") {
        extra_args
+    } else {
+        format!(", AUTOMATIC_MIGRATION TRUE{extra_args}")
    };
+    

    let attach_str = format!(
        "ATTACH 'ducklake:{db_type}:{db_conn_str}' AS {alias_name} (DATA_PATH 's3://{storage}/{data_path}'{extra_args});",