From fdefd4be9398b9610a539360353fd61b521732d4 Mon Sep 17 00:00:00 2001 From: Diego Imbert <70353967+diegoimbert@users.noreply.github.com> Date: Tue, 27 May 2025 15:52:57 +0200 Subject: [PATCH] feat: duckdb sql lang support (#5761) --- backend/.gitignore | 3 +- backend/Cargo.lock | 376 +++++++--- backend/Cargo.toml | 4 +- backend/ee-repo-ref.txt | 2 +- .../20250515084520_duckdb_support.down.sql | 1 + .../20250515084520_duckdb_support.up.sql | 3 + .../parsers/windmill-parser-sql/src/lib.rs | 74 ++ .../parsers/windmill-parser-wasm/src/lib.rs | 6 + backend/windmill-api/openapi.yaml | 3 +- backend/windmill-api/src/workspaces_export.rs | 1 + backend/windmill-common/src/client.rs | 46 +- backend/windmill-common/src/s3_helpers.rs | 17 + backend/windmill-common/src/schema.rs | 2 +- backend/windmill-common/src/scripts.rs | 5 +- backend/windmill-common/src/worker.rs | 1 + backend/windmill-worker/Cargo.toml | 2 + .../windmill-worker/src/duckdb_executor.rs | 664 ++++++++++++++++++ backend/windmill-worker/src/lib.rs | 2 + backend/windmill-worker/src/mysql_executor.rs | 14 +- backend/windmill-worker/src/pg_executor.rs | 16 +- .../windmill-worker/src/windmill-client.js | 2 + backend/windmill-worker/src/worker.rs | 28 + cli/bootstrap/script_bootstrap.ts | 3 +- cli/build.sh | 1 + cli/metadata.ts | 3 + cli/script.ts | 10 +- cli/script_common.ts | 7 +- cli/sync.ts | 1 + cli/wasm/regex/windmill_parser_wasm.js | 1 + frontend/package-lock.json | 8 +- frontend/package.json | 2 +- frontend/src/lib/components/Editor.svelte | 5 +- .../src/lib/components/HighlightCode.svelte | 8 +- .../common/languageIcons/LanguageIcon.svelte | 9 +- .../lib/components/icons/DuckDbIcon.svelte | 19 + frontend/src/lib/consts.ts | 50 ++ frontend/src/lib/infer.ts | 10 +- frontend/src/lib/inferArgSig.ts | 20 +- frontend/src/lib/script_helpers.ts | 18 + frontend/src/lib/scripts.ts | 7 +- .../(root)/(logged)/run/[...run]/+page.svelte | 2 +- 41 files changed, 1310 insertions(+), 146 deletions(-) create mode 100644 backend/migrations/20250515084520_duckdb_support.down.sql create mode 100644 backend/migrations/20250515084520_duckdb_support.up.sql create mode 100644 backend/windmill-worker/src/duckdb_executor.rs create mode 100644 frontend/src/lib/components/icons/DuckDbIcon.svelte diff --git a/backend/.gitignore b/backend/.gitignore index 4ca669fc53..f75b9d1eaa 100644 --- a/backend/.gitignore +++ b/backend/.gitignore @@ -5,4 +5,5 @@ oauth2.json tracing.folded heaptrack* index/ -windmill-api/openapi-*.* \ No newline at end of file +windmill-api/openapi-*.* +.duckdb/* \ No newline at end of file diff --git a/backend/Cargo.lock b/backend/Cargo.lock index eaf6ef8e37..8f6ca407af 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -286,25 +286,57 @@ dependencies = [ "serde", ] +[[package]] +name = "arrow" +version = "54.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5ec52ba94edeed950e4a41f75d35376df196e8cb04437f7280a5aa49f20f796" +dependencies = [ + "arrow-arith 54.3.1", + "arrow-array 54.3.1", + "arrow-buffer 54.3.1", + "arrow-cast 54.3.1", + "arrow-data 54.3.1", + "arrow-ord 54.3.1", + "arrow-row 54.3.1", + "arrow-schema 54.3.1", + "arrow-select 54.3.1", + "arrow-string 54.3.1", +] + [[package]] name = "arrow" version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1bb018b6960c87fd9d025009820406f74e83281185a8bdcb44880d2aa5c9a87" dependencies = [ - "arrow-arith", - "arrow-array", - "arrow-buffer", - "arrow-cast", + "arrow-arith 55.1.0", + "arrow-array 55.1.0", + "arrow-buffer 55.1.0", + "arrow-cast 55.1.0", "arrow-csv", - "arrow-data", + "arrow-data 55.1.0", "arrow-ipc", "arrow-json", - "arrow-ord", - "arrow-row", - "arrow-schema", - "arrow-select", - "arrow-string", + "arrow-ord 55.1.0", + "arrow-row 55.1.0", + "arrow-schema 55.1.0", + "arrow-select 55.1.0", + "arrow-string 55.1.0", +] + +[[package]] +name = "arrow-arith" +version = "54.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc766fdacaf804cb10c7c70580254fcdb5d55cdfda2bc57b02baf5223a3af9e" +dependencies = [ + "arrow-array 54.3.1", + "arrow-buffer 54.3.1", + "arrow-data 54.3.1", + "arrow-schema 54.3.1", + "chrono", + "num", ] [[package]] @@ -313,14 +345,30 @@ version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44de76b51473aa888ecd6ad93ceb262fb8d40d1f1154a4df2f069b3590aa7575" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 55.1.0", + "arrow-buffer 55.1.0", + "arrow-data 55.1.0", + "arrow-schema 55.1.0", "chrono", "num", ] +[[package]] +name = "arrow-array" +version = "54.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a12fcdb3f1d03f69d3ec26ac67645a8fe3f878d77b5ebb0b15d64a116c212985" +dependencies = [ + "ahash 0.8.12", + "arrow-buffer 54.3.1", + "arrow-data 54.3.1", + "arrow-schema 54.3.1", + "chrono", + "half", + "hashbrown 0.15.3", + "num", +] + [[package]] name = "arrow-array" version = "55.1.0" @@ -328,9 +376,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29ed77e22744475a9a53d00026cf8e166fe73cf42d89c4c4ae63607ee1cfcc3f" dependencies = [ "ahash 0.8.12", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-buffer 55.1.0", + "arrow-data 55.1.0", + "arrow-schema 55.1.0", "chrono", "chrono-tz", "half", @@ -338,6 +386,17 @@ dependencies = [ "num", ] +[[package]] +name = "arrow-buffer" +version = "54.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "263f4801ff1839ef53ebd06f99a56cecd1dbaf314ec893d93168e2e860e0291c" +dependencies = [ + "bytes", + "half", + "num", +] + [[package]] name = "arrow-buffer" version = "55.1.0" @@ -349,17 +408,38 @@ dependencies = [ "num", ] +[[package]] +name = "arrow-cast" +version = "54.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ede6175fbc039dfc946a61c1b6d42fd682fcecf5ab5d148fbe7667705798cac9" +dependencies = [ + "arrow-array 54.3.1", + "arrow-buffer 54.3.1", + "arrow-data 54.3.1", + "arrow-schema 54.3.1", + "arrow-select 54.3.1", + "atoi", + "base64 0.22.1", + "chrono", + "comfy-table", + "half", + "lexical-core", + "num", + "ryu", +] + [[package]] name = "arrow-cast" version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f39e1d774ece9292697fcbe06b5584401b26bd34be1bec25c33edae65c2420ff" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 55.1.0", + "arrow-buffer 55.1.0", + "arrow-data 55.1.0", + "arrow-schema 55.1.0", + "arrow-select 55.1.0", "atoi", "base64 0.22.1", "chrono", @@ -376,9 +456,9 @@ version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9055c972a07bf12c2a827debfd34f88d3b93da1941d36e1d9fee85eebe38a12a" dependencies = [ - "arrow-array", - "arrow-cast", - "arrow-schema", + "arrow-array 55.1.0", + "arrow-cast 55.1.0", + "arrow-schema 55.1.0", "chrono", "csv", "csv-core", @@ -386,14 +466,26 @@ dependencies = [ "regex", ] +[[package]] +name = "arrow-data" +version = "54.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61cfdd7d99b4ff618f167e548b2411e5dd2c98c0ddebedd7df433d34c20a4429" +dependencies = [ + "arrow-buffer 54.3.1", + "arrow-schema 54.3.1", + "half", + "num", +] + [[package]] name = "arrow-data" version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf75ac27a08c7f48b88e5c923f267e980f27070147ab74615ad85b5c5f90473d" dependencies = [ - "arrow-buffer", - "arrow-schema", + "arrow-buffer 55.1.0", + "arrow-schema 55.1.0", "half", "num", ] @@ -404,10 +496,10 @@ version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a222f0d93772bd058d1268f4c28ea421a603d66f7979479048c429292fac7b2e" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 55.1.0", + "arrow-buffer 55.1.0", + "arrow-data 55.1.0", + "arrow-schema 55.1.0", "flatbuffers", "lz4_flex", ] @@ -418,11 +510,11 @@ version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9085342bbca0f75e8cb70513c0807cc7351f1fbf5cb98192a67d5e3044acb033" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-schema", + "arrow-array 55.1.0", + "arrow-buffer 55.1.0", + "arrow-cast 55.1.0", + "arrow-data 55.1.0", + "arrow-schema 55.1.0", "chrono", "half", "indexmap 2.9.0", @@ -434,17 +526,43 @@ dependencies = [ "simdutf8", ] +[[package]] +name = "arrow-ord" +version = "54.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0a3334a743bd2a1479dbc635540617a3923b4b2f6870f37357339e6b5363c21" +dependencies = [ + "arrow-array 54.3.1", + "arrow-buffer 54.3.1", + "arrow-data 54.3.1", + "arrow-schema 54.3.1", + "arrow-select 54.3.1", +] + [[package]] name = "arrow-ord" version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab2f1065a5cad7b9efa9e22ce5747ce826aa3855766755d4904535123ef431e7" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 55.1.0", + "arrow-buffer 55.1.0", + "arrow-data 55.1.0", + "arrow-schema 55.1.0", + "arrow-select 55.1.0", +] + +[[package]] +name = "arrow-row" +version = "54.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d1d7a7291d2c5107e92140f75257a99343956871f3d3ab33a7b41532f79cb68" +dependencies = [ + "arrow-array 54.3.1", + "arrow-buffer 54.3.1", + "arrow-data 54.3.1", + "arrow-schema 54.3.1", + "half", ] [[package]] @@ -453,19 +571,42 @@ version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3703a0e3e92d23c3f756df73d2dc9476873f873a76ae63ef9d3de17fda83b2d8" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 55.1.0", + "arrow-buffer 55.1.0", + "arrow-data 55.1.0", + "arrow-schema 55.1.0", "half", ] +[[package]] +name = "arrow-schema" +version = "54.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cfaf5e440be44db5413b75b72c2a87c1f8f0627117d110264048f2969b99e9" +dependencies = [ + "bitflags 2.9.1", +] + [[package]] name = "arrow-schema" version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73a47aa0c771b5381de2b7f16998d351a6f4eb839f1e13d48353e17e873d969b" +[[package]] +name = "arrow-select" +version = "54.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69efcd706420e52cd44f5c4358d279801993846d1c2a8e52111853d61d55a619" +dependencies = [ + "ahash 0.8.12", + "arrow-array 54.3.1", + "arrow-buffer 54.3.1", + "arrow-data 54.3.1", + "arrow-schema 54.3.1", + "num", +] + [[package]] name = "arrow-select" version = "55.1.0" @@ -473,24 +614,41 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24b7b85575702b23b85272b01bc1c25a01c9b9852305e5d0078c79ba25d995d4" dependencies = [ "ahash 0.8.12", - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 55.1.0", + "arrow-buffer 55.1.0", + "arrow-data 55.1.0", + "arrow-schema 55.1.0", "num", ] +[[package]] +name = "arrow-string" +version = "54.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a21546b337ab304a32cfc0770f671db7411787586b45b78b4593ae78e64e2b03" +dependencies = [ + "arrow-array 54.3.1", + "arrow-buffer 54.3.1", + "arrow-data 54.3.1", + "arrow-schema 54.3.1", + "arrow-select 54.3.1", + "memchr", + "num", + "regex", + "regex-syntax 0.8.5", +] + [[package]] name = "arrow-string" version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9260fddf1cdf2799ace2b4c2fc0356a9789fa7551e0953e35435536fecefebbd" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 55.1.0", + "arrow-buffer 55.1.0", + "arrow-data 55.1.0", + "arrow-schema 55.1.0", + "arrow-select 55.1.0", "memchr", "num", "regex", @@ -1838,6 +1996,12 @@ dependencies = [ "syn 2.0.101", ] +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cbc" version = "0.1.2" @@ -2599,9 +2763,9 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffe060b978f74ab446be722adb8a274e052e005bf6dfd171caadc3abaad10080" dependencies = [ - "arrow", + "arrow 55.1.0", "arrow-ipc", - "arrow-schema", + "arrow-schema 55.1.0", "async-trait", "bytes", "bzip2", @@ -2654,7 +2818,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61fe34f401bd03724a1f96d12108144f8cd495a3cdda2bf5e091822fb80b7e66" dependencies = [ - "arrow", + "arrow 55.1.0", "async-trait", "dashmap 6.1.0", "datafusion-common", @@ -2680,7 +2844,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4411b8e3bce5e0fc7521e44f201def2e2d5d1b5f176fb56e8cdc9942c890f00" dependencies = [ - "arrow", + "arrow 55.1.0", "async-trait", "datafusion-catalog", "datafusion-common", @@ -2704,7 +2868,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0734015d81c8375eb5d4869b7f7ecccc2ee8d6cb81948ef737cd0e7b743bd69c" dependencies = [ "ahash 0.8.12", - "arrow", + "arrow 55.1.0", "arrow-ipc", "base64 0.22.1", "half", @@ -2738,7 +2902,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04e602dcdf2f50c2abf297cc2203c73531e6f48b29516af7695d338cf2a778b1" dependencies = [ - "arrow", + "arrow 55.1.0", "async-compression", "async-trait", "bytes", @@ -2774,7 +2938,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bb2253952dc32296ed5b84077cb2e0257fea4be6373e1c376426e17ead4ef6" dependencies = [ - "arrow", + "arrow 55.1.0", "async-trait", "bytes", "datafusion-catalog", @@ -2799,7 +2963,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b8c7f47a5d2fe03bfa521ec9bafdb8a5c82de8377f60967c3663f00c8790352" dependencies = [ - "arrow", + "arrow 55.1.0", "async-trait", "bytes", "datafusion-catalog", @@ -2824,7 +2988,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27d15868ea39ed2dc266728b554f6304acd473de2142281ecfa1294bb7415923" dependencies = [ - "arrow", + "arrow 55.1.0", "async-trait", "bytes", "datafusion-catalog", @@ -2861,7 +3025,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06f004d100f49a3658c9da6fb0c3a9b760062d96cd4ad82ccc3b7b69a9fb2f84" dependencies = [ - "arrow", + "arrow 55.1.0", "dashmap 6.1.0", "datafusion-common", "datafusion-expr", @@ -2880,7 +3044,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a4e4ce3802609be38eeb607ee72f6fe86c3091460de9dbfae9e18db423b3964" dependencies = [ - "arrow", + "arrow 55.1.0", "chrono", "datafusion-common", "datafusion-doc", @@ -2901,7 +3065,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "422ac9cf3b22bbbae8cdf8ceb33039107fde1b5492693168f13bd566b1bcc839" dependencies = [ - "arrow", + "arrow 55.1.0", "datafusion-common", "indexmap 2.9.0", "itertools 0.14.0", @@ -2914,8 +3078,8 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2ddf0a0a2db5d2918349c978d42d80926c6aa2459cd8a3c533a84ec4bb63479e" dependencies = [ - "arrow", - "arrow-buffer", + "arrow 55.1.0", + "arrow-buffer 55.1.0", "base64 0.22.1", "blake2", "blake3", @@ -2944,7 +3108,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "408a05dafdc70d05a38a29005b8b15e21b0238734dab1e98483fcb58038c5aba" dependencies = [ "ahash 0.8.12", - "arrow", + "arrow 55.1.0", "datafusion-common", "datafusion-doc", "datafusion-execution", @@ -2965,7 +3129,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "756d21da2dd6c9bef97af1504970ff56cbf35d03fbd4ffd62827f02f4d2279d4" dependencies = [ "ahash 0.8.12", - "arrow", + "arrow 55.1.0", "datafusion-common", "datafusion-expr-common", "datafusion-physical-expr-common", @@ -2977,8 +3141,8 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d8d50f6334b378930d992d801a10ac5b3e93b846b39e4a05085742572844537" dependencies = [ - "arrow", - "arrow-ord", + "arrow 55.1.0", + "arrow-ord 55.1.0", "datafusion-common", "datafusion-doc", "datafusion-execution", @@ -2998,7 +3162,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc9a97220736c8fff1446e936be90d57216c06f28969f9ffd3b72ac93c958c8a" dependencies = [ - "arrow", + "arrow 55.1.0", "async-trait", "datafusion-catalog", "datafusion-common", @@ -3052,7 +3216,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "084d9f979c4b155346d3c34b18f4256e6904ded508e9554d90fed416415c3515" dependencies = [ - "arrow", + "arrow 55.1.0", "chrono", "datafusion-common", "datafusion-expr", @@ -3072,7 +3236,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64c536062b0076f4e30084065d805f389f9fe38af0ca75bcbac86bc5e9fbab65" dependencies = [ "ahash 0.8.12", - "arrow", + "arrow 55.1.0", "datafusion-common", "datafusion-expr", "datafusion-expr-common", @@ -3094,7 +3258,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8a92b53b3193fac1916a1c5b8e3f4347c526f6822e56b71faa5fb372327a863" dependencies = [ "ahash 0.8.12", - "arrow", + "arrow 55.1.0", "datafusion-common", "datafusion-expr-common", "hashbrown 0.14.5", @@ -3107,7 +3271,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6fa0a5ac94c7cf3da97bedabd69d6bbca12aef84b9b37e6e9e8c25286511b5e2" dependencies = [ - "arrow", + "arrow 55.1.0", "datafusion-common", "datafusion-execution", "datafusion-expr", @@ -3127,9 +3291,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "690c615db468c2e5fe5085b232d8b1c088299a6c63d87fd960a354a71f7acb55" dependencies = [ "ahash 0.8.12", - "arrow", - "arrow-ord", - "arrow-schema", + "arrow 55.1.0", + "arrow-ord 55.1.0", + "arrow-schema 55.1.0", "async-trait", "chrono", "datafusion-common", @@ -3156,7 +3320,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad229a134c7406c057ece00c8743c0c34b97f4e72f78b475fe17b66c5e14fa4f" dependencies = [ - "arrow", + "arrow 55.1.0", "async-trait", "dashmap 6.1.0", "datafusion-common", @@ -3180,7 +3344,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64f6ab28b72b664c21a27b22a2ff815fd390ed224c26e89a93b5a8154a4e8607" dependencies = [ - "arrow", + "arrow 55.1.0", "bigdecimal", "datafusion-common", "datafusion-expr", @@ -4640,6 +4804,25 @@ dependencies = [ "zeroize", ] +[[package]] +name = "duckdb" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49ac283b6621e3becf8014d1efa655522794075834c72f744573debef9c9f6c8" +dependencies = [ + "arrow 54.3.1", + "cast", + "fallible-iterator 0.3.0", + "fallible-streaming-iterator", + "hashlink 0.9.1", + "libduckdb-sys", + "memchr", + "num-integer", + "rust_decimal", + "smallvec", + "strum 0.25.0", +] + [[package]] name = "dunce" version = "1.0.5" @@ -7149,6 +7332,22 @@ version = "0.2.172" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" +[[package]] +name = "libduckdb-sys" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12cac9d03484c43fefac8b2066a253c9b0b3b0cd02cbe02a9ea2312f7e382618" +dependencies = [ + "autocfg", + "cc", + "flate2", + "pkg-config", + "serde", + "serde_json", + "tar", + "vcpkg", +] + [[package]] name = "libffi" version = "3.2.0" @@ -8950,13 +9149,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be7b2d778f6b841d37083ebdf32e33a524acde1266b5884a8ca29bf00dfa1231" dependencies = [ "ahash 0.8.12", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", + "arrow-array 55.1.0", + "arrow-buffer 55.1.0", + "arrow-cast 55.1.0", + "arrow-data 55.1.0", "arrow-ipc", - "arrow-schema", - "arrow-select", + "arrow-schema 55.1.0", + "arrow-select 55.1.0", "base64 0.22.1", "brotli 8.0.1", "bytes", @@ -15056,6 +15255,7 @@ dependencies = [ "deno_web", "deno_webidl", "dotenv", + "duckdb", "dyn-iter", "flume", "futures", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 069aca4367..0273aa2c59 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -87,13 +87,14 @@ python = ["windmill-worker/python", "windmill-api/python"] rust = ["windmill-worker/rust"] mysql = ["windmill-worker/mysql"] oracledb = ["windmill-worker/oracledb"] +duckdb = ["windmill-worker/duckdb"] mssql = ["windmill-worker/mssql"] bigquery = ["windmill-worker/bigquery"] php = ["windmill-worker/php"] csharp = ["windmill-worker/csharp"] nu = ["windmill-worker/nu"] java = ["windmill-worker/java"] -all_languages = [ "python", "deno_core", "rust", "mysql", "oracledb", "mssql", "bigquery", "csharp", "nu", "php", "java"] +all_languages = [ "python", "deno_core", "rust", "mysql", "oracledb", "duckdb", "mssql", "bigquery", "csharp", "nu", "php", "java"] [patch.crates-io] @@ -238,6 +239,7 @@ json-pointer = "^0" itertools = "^0" regex = "^1" semver = "^1" +duckdb = { version = "1.2.2", features = ["bundled"] } v8 = "=130.0.7" # Exact version NOTE: Do not forget to update version and hash in flake.nix deno_fetch = "0.214.0" diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 04b305d5a2..be1e897bcd 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -6899b8151329218a1df59964dac57e0e004ae25a \ No newline at end of file +32039f675060b5996951708368bdefe14278d5cd \ No newline at end of file diff --git a/backend/migrations/20250515084520_duckdb_support.down.sql b/backend/migrations/20250515084520_duckdb_support.down.sql new file mode 100644 index 0000000000..d2f607c5b8 --- /dev/null +++ b/backend/migrations/20250515084520_duckdb_support.down.sql @@ -0,0 +1 @@ +-- Add down migration script here diff --git a/backend/migrations/20250515084520_duckdb_support.up.sql b/backend/migrations/20250515084520_duckdb_support.up.sql new file mode 100644 index 0000000000..869fcc07dc --- /dev/null +++ b/backend/migrations/20250515084520_duckdb_support.up.sql @@ -0,0 +1,3 @@ +-- Add up migration script here +ALTER TYPE SCRIPT_LANG ADD VALUE IF NOT EXISTS 'duckdb'; +UPDATE config set config = jsonb_set(config, '{worker_tags}', config->'worker_tags' || '["duckdb"]'::jsonb) where name = 'worker__default' and config @> '{"worker_tags": ["deno", "python3", "go", "bash", "powershell", "dependency", "flow", "hub", "other", "bun", "php", "rust", "ansible", "csharp", "nu", "java"]}'::jsonb AND NOT config->'worker_tags' @> '"duckdb"'::jsonb; diff --git a/backend/parsers/windmill-parser-sql/src/lib.rs b/backend/parsers/windmill-parser-sql/src/lib.rs index 21ebe45f7b..0fe72f0296 100644 --- a/backend/parsers/windmill-parser-sql/src/lib.rs +++ b/backend/parsers/windmill-parser-sql/src/lib.rs @@ -83,6 +83,21 @@ pub fn parse_bigquery_sig(code: &str) -> anyhow::Result { } } +pub fn parse_duckdb_sig(code: &str) -> anyhow::Result { + let parsed = parse_duckdb_file(&code)?; + if let Some(args) = parsed { + Ok(MainArgSignature { + star_args: false, + star_kwargs: false, + args, + no_main_func: None, + has_preprocessor: None, + }) + } else { + Err(anyhow!("Error parsing sql".to_string())) + } +} + pub fn parse_snowflake_sig(code: &str) -> anyhow::Result { let parsed = parse_snowflake_file(&code)?; if let Some(x) = parsed { @@ -212,6 +227,9 @@ lazy_static::lazy_static! { // -- @name (type) = default static ref RE_ARG_BIGQUERY: Regex = Regex::new(r#"(?m)^-- @(\w+) \((\w+(?:\[\])?)\)(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap(); + // -- $name (type) = default + static ref RE_ARG_DUCKDB: Regex = Regex::new(r#"(?m)^-- \$(\w+) \((\w+)\)(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap(); + static ref RE_ARG_SNOWFLAKE: Regex = Regex::new(r#"(?m)^-- \? (\w+) \((\w+)\)(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap(); @@ -577,6 +595,35 @@ fn parse_bigquery_file(code: &str) -> anyhow::Result>> { Ok(Some(args)) } +fn parse_duckdb_file(code: &str) -> anyhow::Result>> { + let mut args: Vec = vec![]; + + for cap in RE_ARG_DUCKDB.captures_iter(code) { + let name = cap.get(1).map(|x| x.as_str().to_string()).unwrap(); + let typ = cap + .get(2) + .map(|x| x.as_str().to_string().to_lowercase()) + .unwrap(); + let default = cap.get(3).map(|x| x.as_str().to_string()); + let has_default = default.is_some(); + let parsed_typ = parse_duckdb_typ(typ.as_str()); + + let parsed_default = default.and_then(|x| parsed_default(&parsed_typ, x)); + + args.push(Arg { + name, + typ: parsed_typ, + default: parsed_default, + otyp: Some(typ), + has_default, + oidx: None, + }); + } + + args.append(&mut parse_sql_sanitized_interpolation(code)); + Ok(Some(args)) +} + fn parse_snowflake_file(code: &str) -> anyhow::Result>> { let mut args: Vec = vec![]; @@ -729,6 +776,33 @@ pub fn parse_bigquery_typ(typ: &str) -> Typ { } } +pub fn parse_duckdb_typ(typ: &str) -> Typ { + if typ.ends_with("[]") { + let base_typ = parse_duckdb_typ(typ.strip_suffix("[]").unwrap()); + Typ::List(Box::new(base_typ)) + } else { + match typ { + "varchar" | "char" | "bpchar" | "text" | "string" => Typ::Str(None), + "blob" | "bytea" | "binary" | "varbinary" | "bitstring" => Typ::Bytes, + "boolean" | "bool" | "bit" | "logical" => Typ::Bool, + "bigint" | "int8" | "long" | "integer" | "int4" | "int" | "smallint" | "int2" + | "short" | "tinyint" | "int1" | "signed" | "ubigint" | "uhugeint" | "uinteger" + | "usmallint" | "utinyint" => Typ::Int, + "decimal" | "numeric" | "double" | "float8" | "float" | "float4" | "real" => Typ::Float, + "date" + | "time" + | "timestamp with time zone" + | "timestamptz" + | "timestamp" + | "datetime" => Typ::Datetime, + "uuid" | "json" => Typ::Str(None), + "interval" | "hugeint" => Typ::Str(None), + "s3object" => Typ::Resource("S3Object".to_string()), + _ => Typ::Str(None), + } + } +} + pub fn parse_snowflake_typ(typ: &str) -> Typ { match typ { "varchar" => Typ::Str(None), diff --git a/backend/parsers/windmill-parser-wasm/src/lib.rs b/backend/parsers/windmill-parser-wasm/src/lib.rs index cd280e7af5..4270c28c60 100644 --- a/backend/parsers/windmill-parser-wasm/src/lib.rs +++ b/backend/parsers/windmill-parser-wasm/src/lib.rs @@ -96,6 +96,12 @@ pub fn parse_oracledb(code: &str) -> String { wrap_sig(windmill_parser_sql::parse_oracledb_sig(code)) } +#[cfg(feature = "sql-parser")] +#[wasm_bindgen] +pub fn parse_duckdb(code: &str) -> String { + wrap_sig(windmill_parser_sql::parse_duckdb_sig(code)) +} + #[cfg(feature = "sql-parser")] #[wasm_bindgen] pub fn parse_bigquery(code: &str) -> String { diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index fd3e81f209..20f495568d 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -14242,7 +14242,8 @@ components: ansible, csharp, nu, - java + java, + duckdb # for related places search: ADD_NEW_LANG ] diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index c58d773f66..cd2be0f708 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -360,6 +360,7 @@ pub(crate) async fn tarball_workspace( ScriptLang::Bigquery => "bq.sql", ScriptLang::Snowflake => "sf.sql", ScriptLang::Mssql => "ms.sql", + ScriptLang::DuckDb => "duckdb.sql", ScriptLang::Graphql => "gql", ScriptLang::Nativets => "fetch.ts", ScriptLang::Bun | ScriptLang::Bunnative => { diff --git a/backend/windmill-common/src/client.rs b/backend/windmill-common/src/client.rs index 95d9f64bd2..0c5394c6e6 100644 --- a/backend/windmill-common/src/client.rs +++ b/backend/windmill-common/src/client.rs @@ -2,7 +2,11 @@ use anyhow::Context; use reqwest::{Body, Response}; use serde::de::DeserializeOwned; -use crate::utils::HTTP_CLIENT; +use crate::{ + error::{self, to_anyhow}, + s3_helpers::{DuckdbConnectionSettingsQueryV2, DuckdbConnectionSettingsResponse}, + utils::HTTP_CLIENT, +}; #[derive(Clone)] pub struct AuthedClient { @@ -197,4 +201,44 @@ impl AuthedClient { _ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default()))?, } } + + pub async fn get_duckdb_connection_settings( + &self, + s3: &DuckdbConnectionSettingsQueryV2, + ) -> error::Result { + let url = format!( + "{}/api/w/{}/job_helpers/v2/duckdb_connection_settings", + self.base_internal_url, &self.workspace + ); + let response = self + .force_client + .as_ref() + .unwrap_or(&HTTP_CLIENT) + .post(url) + .header( + reqwest::header::CONTENT_TYPE, + reqwest::header::HeaderValue::from_static("application/json"), + ) + .header( + reqwest::header::ACCEPT, + reqwest::header::HeaderValue::from_static("application/json"), + ) + .header( + reqwest::header::AUTHORIZATION, + reqwest::header::HeaderValue::from_str(&format!("Bearer {}", self.token)) + .map_err(|e| error::Error::BadConfig(e.to_string()))?, + ) + .body(serde_json::to_string(&s3).map_err(to_anyhow)?) + .send() + .await + .context(format!("Sent get_duckdb_connection_settings request",)) + .map_err(error::Error::from)?; + match response.status().as_u16() { + 200u16 => Ok(response + .json::() + .await + .context("decoding duckdb_connection_settings response as json")?), + _ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default()))?, + } + } } diff --git a/backend/windmill-common/src/s3_helpers.rs b/backend/windmill-common/src/s3_helpers.rs index 29f59cf07c..7e7c48851e 100644 --- a/backend/windmill-common/src/s3_helpers.rs +++ b/backend/windmill-common/src/s3_helpers.rs @@ -891,3 +891,20 @@ pub async fn convert_json_line_stream>( Ok(tokio_stream::wrappers::ReceiverStream::new(rx)) } + +#[derive(Deserialize, Serialize)] +pub struct DuckdbConnectionSettingsResponse { + pub connection_settings_str: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub azure_container_path: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub s3_bucket: Option, +} + +#[derive(Deserialize, Serialize)] +pub struct DuckdbConnectionSettingsQueryV2 { + #[serde(skip_serializing_if = "Option::is_none")] + pub s3_resource_path: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub storage: Option, +} diff --git a/backend/windmill-common/src/schema.rs b/backend/windmill-common/src/schema.rs index 2b24ead0d3..71ecd220d1 100644 --- a/backend/windmill-common/src/schema.rs +++ b/backend/windmill-common/src/schema.rs @@ -349,7 +349,7 @@ pub fn should_validate_schema(code: &str, lang: &ScriptLang) -> bool { let comment = match lang { Nativets | Bun | Bunnative | Deno | Php | CSharp | Java => "//", Python3 | Go | Bash | Powershell | Graphql | Ansible | Nu => "#", - Postgresql | Mysql | Bigquery | Snowflake | Mssql | OracleDB => "--", + Postgresql | Mysql | Bigquery | Snowflake | Mssql | OracleDB | DuckDb => "--", Rust => "//!", // for related places search: ADD_NEW_LANG }; diff --git a/backend/windmill-common/src/scripts.rs b/backend/windmill-common/src/scripts.rs index e0391767f8..dda9daf0f4 100644 --- a/backend/windmill-common/src/scripts.rs +++ b/backend/windmill-common/src/scripts.rs @@ -46,13 +46,13 @@ pub enum ScriptLang { Graphql, Mssql, OracleDB, + DuckDb, Php, Rust, Ansible, CSharp, Nu, - Java, - // for related places search: ADD_NEW_LANG + Java, // for related places search: ADD_NEW_LANG } impl ScriptLang { @@ -73,6 +73,7 @@ impl ScriptLang { ScriptLang::Mssql => "mssql", ScriptLang::Graphql => "graphql", ScriptLang::OracleDB => "oracledb", + ScriptLang::DuckDb => "duckdb", ScriptLang::Php => "php", ScriptLang::Rust => "rust", ScriptLang::Ansible => "ansible", diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index aa1a715758..6b11fa086f 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -79,6 +79,7 @@ lazy_static::lazy_static! { "csharp".to_string(), "nu".to_string(), "java".to_string(), + "duckdb".to_string(), // for related places search: ADD_NEW_LANG "dependency".to_string(), "flow".to_string(), diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml index 4dccc3213c..dbc8f2ab27 100644 --- a/backend/windmill-worker/Cargo.toml +++ b/backend/windmill-worker/Cargo.toml @@ -31,6 +31,7 @@ csharp = ["dep:windmill-parser-csharp"] rust = ["dep:windmill-parser-rust"] nu = ["dep:windmill-parser-nu"] java = ["dep:windmill-parser-java"] +duckdb = [] [dependencies] windmill-queue.workspace = true @@ -92,6 +93,7 @@ deno_permissions = { workspace = true, optional = true } deno_io = { workspace = true, optional = true } deno_error = { workspace = true, optional = true } async-stream.workspace = true +duckdb.workspace = true postgres-native-tls.workspace = true native-tls.workspace = true diff --git a/backend/windmill-worker/src/duckdb_executor.rs b/backend/windmill-worker/src/duckdb_executor.rs new file mode 100644 index 0000000000..8ee78292d3 --- /dev/null +++ b/backend/windmill-worker/src/duckdb_executor.rs @@ -0,0 +1,664 @@ +use std::collections::HashMap; +use std::env; + +use duckdb::types::TimeUnit; +use duckdb::{params_from_iter, Row}; +use rust_decimal::prelude::FromPrimitive; +use rust_decimal::Decimal; +use serde_json::value::RawValue; +use serde_json::{json, Value}; +use tokio::fs::remove_file; +use tokio::task; +use uuid::Uuid; +use windmill_common::error::{to_anyhow, Error, Result}; +use windmill_common::s3_helpers::{ + DuckdbConnectionSettingsQueryV2, DuckdbConnectionSettingsResponse, S3Object, +}; +use windmill_common::worker::{to_raw_value, Connection}; +use windmill_parser_sql::{parse_duckdb_sig, parse_sql_blocks}; +use windmill_queue::{CanceledBy, MiniPulledJob}; + +use crate::common::{build_args_values, OccupancyMetrics}; +use crate::handle_child::run_future_with_polling_update_job_poller; +#[cfg(feature = "mysql")] +use crate::mysql_executor::MysqlDatabase; +use crate::pg_executor::PgDatabase; +use crate::sanitized_sql_params::sanitize_and_interpolate_unsafe_sql_args; +use windmill_common::client::AuthedClient; + +fn do_duckdb_inner( + conn: &duckdb::Connection, + query: &str, + job_args: &HashMap, + skip_collect: bool, + column_order: &mut Option>, +) -> Result> { + let mut rows_vec = vec![]; + + let (query, job_args) = interpolate_named_args(query, &job_args); + + let mut stmt = conn + .prepare(&query) + .map_err(|e| Error::ExecutionErr(e.to_string()))?; + + let mut rows = stmt + .query(params_from_iter(job_args)) + .map_err(|e| Error::ExecutionErr(e.to_string()))?; + + if skip_collect { + return Ok(to_raw_value(&json!([]))); + } + + // Statement needs to be stepped at least once or stmt.column_names() will panic + let mut column_names = None; + loop { + let row = rows.next(); + match row { + Ok(Some(row)) => { + // Set column names if not already set + let stmt = row.as_ref(); + let column_names = match column_names.as_ref() { + Some(column_names) => column_names, + None => { + column_names = Some(stmt.column_names()); + column_names.as_ref().unwrap() + } + }; + + let row = row_to_value(row, &column_names.as_slice()) + .map_err(|e| Error::ExecutionErr(e.to_string()))?; + rows_vec.push(row); + } + Ok(None) => break, + Err(e) => { + return Err(Error::ExecutionErr(e.to_string())); + } + } + } + + if let (Some(column_order), Some(column_names)) = (column_order.as_mut(), column_names) { + *column_order = column_names.clone(); + } + + return Ok(to_raw_value(&rows_vec)); +} + +pub async fn do_duckdb( + job: &MiniPulledJob, + client: &AuthedClient, + query: &str, + conn: &Connection, + mem_peak: &mut i32, + canceled_by: &mut Option, + worker_name: &str, + column_order_ref: &mut Option>, + occupancy_metrics: &mut OccupancyMetrics, +) -> Result> { + let result_f = async { + let sig = parse_duckdb_sig(query)?.args; + let mut job_args = build_args_values(job, client, conn).await?; + + let (query, _) = &sanitize_and_interpolate_unsafe_sql_args(query, &sig, &job_args)?; + // Prevent interpolate_named_args from detecting argument identifiers in the signature for + // the first query block + let query = trunc_sig(query); + + let (_query_with_transformed_s3_uris, mut used_storages) = + transform_s3_uris(query, client).await?; + let query = _query_with_transformed_s3_uris.as_deref().unwrap_or(query); + + let job_args = { + let mut m: HashMap = HashMap::new(); + for sig_arg in sig.into_iter() { + let json_value = job_args + .remove(&sig_arg.name) + .or_else(|| sig_arg.default) + .unwrap_or_else(|| json!(null)); + + if matches!(&sig_arg.otyp.as_ref().map(String::as_str), Some("s3object")) { + let s3_obj = serde_json::from_value::(json_value).map_err(|e| { + Error::ExecutionErr(format!("Failed to deserialize S3Object: {}", e)) + })?; + let duckdb_conn_settings: windmill_common::s3_helpers::DuckdbConnectionSettingsResponse = client + .get_duckdb_connection_settings(&DuckdbConnectionSettingsQueryV2 { + s3_resource_path: None, + storage: s3_obj.storage.clone(), + }) + .await?; + + let uri = match ( + &duckdb_conn_settings.s3_bucket, + &duckdb_conn_settings.azure_container_path, + ) { + (Some(s3_bucket), None) => format!("s3://{}/{}", s3_bucket, &s3_obj.s3), + (None, Some(az_container)) => format!("{}/{}", az_container, &s3_obj.s3), + _ => { + return Err(Error::ExecutionErr( + "S3Object must have either s3_bucket or azure_container_path" + .to_string(), + )); + } + }; + m.insert(sig_arg.name, duckdb::types::Value::Text(uri)); + used_storages.insert(s3_obj.storage, duckdb_conn_settings); + } else { + let duckdb_value = json_value_to_duckdb_value( + &json_value, + sig_arg + .otyp + .clone() + .unwrap_or_else(|| "text".to_string()) + .as_str(), + client, + )?; + m.insert(sig_arg.name, duckdb_value); + } + } + m + }; + + let query_block_list = parse_sql_blocks(query); + + // Replace windmill resource ATTACH statements with the real instructions + let query_block_list = { + let mut v = vec![]; + for query_block in query_block_list.iter() { + match parse_attach_db_resource(query_block) { + Some(parsed) => v.extend( + transform_attach_db_resource_query(&parsed, &job.id, client).await?, + ), + None => v.push(query_block.to_string()), + }; + } + v + }; + + // duckdb::Connection is not Send so we do it in a single blocking task + let (result, column_order) = task::spawn_blocking(move || { + let conn = duckdb::Connection::open_in_memory() + .map_err(|e| Error::ConnectingToDatabase(e.to_string()))?; + + for (_, DuckdbConnectionSettingsResponse { connection_settings_str, .. }) in + used_storages.into_iter() + { + conn.execute_batch(&connection_settings_str) + .map_err(|e| Error::ExecutionErr(e.to_string()))?; + } + + let mut result: Option> = None; + let mut column_order = None; + for (query_block_index, query_block) in query_block_list.iter().enumerate() { + result = Some( + do_duckdb_inner( + &conn, + query_block.as_str(), + &job_args, + query_block_index != query_block_list.len() - 1, + &mut column_order, + ) + .map_err(|e| Error::ExecutionErr(e.to_string()))?, + ); + } + let result = result.unwrap_or_else(|| to_raw_value(&json!([]))); + Ok::<_, Error>((result, column_order)) + }) + .await + .map_err(to_anyhow)??; + + *column_order_ref = column_order; + + // BigQuery cleanup + let bq_credentials_path = make_bq_credentials_path(&job.id); + env::remove_var("GOOGLE_APPLICATION_CREDENTIALS"); + if matches!(tokio::fs::try_exists(&bq_credentials_path).await, Ok(true)) { + remove_file(&bq_credentials_path).await.map_err(to_anyhow)?; + } + Ok(result) + }; + + let result = run_future_with_polling_update_job_poller( + job.id, + job.timeout, + conn, + mem_peak, + canceled_by, + result_f, + worker_name, + &job.workspace_id, + &mut Some(occupancy_metrics), + Box::pin(futures::stream::once(async { 0 })), + ) + .await?; + + Ok(result) +} + +fn row_to_value(row: &Row<'_>, column_names: &[String]) -> Result> { + let mut obj = serde_json::Map::new(); + for (i, key) in column_names.iter().enumerate() { + let value: duckdb::types::Value = + row.get(i).map_err(|e| Error::ExecutionErr(e.to_string()))?; + let json_value = match value { + duckdb::types::Value::Null => serde_json::Value::Null, + duckdb::types::Value::Boolean(b) => serde_json::Value::Bool(b), + duckdb::types::Value::TinyInt(i) => serde_json::Value::Number(i.into()), + duckdb::types::Value::SmallInt(i) => serde_json::Value::Number(i.into()), + duckdb::types::Value::Int(i) => serde_json::Value::Number(i.into()), + duckdb::types::Value::BigInt(i) => serde_json::Value::Number(i.into()), + duckdb::types::Value::HugeInt(i) => serde_json::Value::String(i.to_string()), + duckdb::types::Value::UTinyInt(u) => serde_json::Value::Number(u.into()), + duckdb::types::Value::USmallInt(u) => serde_json::Value::Number(u.into()), + duckdb::types::Value::UInt(u) => serde_json::Value::Number(u.into()), + duckdb::types::Value::UBigInt(u) => serde_json::Value::Number(u.into()), + duckdb::types::Value::Float(f) => serde_json::Value::Number( + serde_json::Number::from_f64(f as f64) + .ok_or_else(|| Error::ExecutionErr("Could not convert to f64".to_string()))?, + ), + duckdb::types::Value::Double(f) => serde_json::Value::Number( + serde_json::Number::from_f64(f) + .ok_or_else(|| Error::ExecutionErr("Could not convert to f64".to_string()))?, + ), + duckdb::types::Value::Decimal(d) => serde_json::Value::String(d.to_string()), + duckdb::types::Value::Timestamp(_, ts) => serde_json::Value::String(ts.to_string()), + duckdb::types::Value::Text(s) => serde_json::Value::String(s), + duckdb::types::Value::Blob(b) => serde_json::Value::Array( + b.into_iter() + .map(|byte| serde_json::Value::Number(byte.into())) + .collect(), + ), + duckdb::types::Value::Date32(d) => serde_json::Value::Number(d.into()), + duckdb::types::Value::Time64(_, t) => serde_json::Value::String(t.to_string()), + duckdb::types::Value::Interval { months, days, nanos } => serde_json::json!({ + "months": months, + "days": days, + "nanos": nanos + }), + duckdb::types::Value::List(values) => serde_json::Value::Array( + values + .into_iter() + .map(|v| serde_json::Value::String(format!("{:?}", v))) + .collect(), + ), + duckdb::types::Value::Enum(e) => serde_json::Value::String(e), + duckdb::types::Value::Struct(fields) => serde_json::Value::Object( + fields + .iter() + .map(|(k, v)| (k.clone(), serde_json::Value::String(format!("{:?}", v)))) + .collect(), + ), + duckdb::types::Value::Array(values) => serde_json::Value::Array( + values + .into_iter() + .map(|v| serde_json::Value::String(format!("{:?}", v))) + .collect(), + ), + duckdb::types::Value::Map(map) => serde_json::Value::Object( + map.iter() + .map(|(k, v)| { + ( + format!("{:?}", k), + serde_json::Value::String(format!("{:?}", v)), + ) + }) + .collect(), + ), + duckdb::types::Value::Union(value) => { + serde_json::Value::String(format!("{:?}", *value)) + } + }; + obj.insert(key.clone(), json_value); + } + serde_json::value::to_raw_value(&obj).map_err(|e| e.into()) +} + +fn json_value_to_duckdb_value( + json_value: &serde_json::Value, + arg_type: &str, + client: &AuthedClient, +) -> Result { + let arg_type = arg_type.to_lowercase(); + let duckdb_value = match json_value { + serde_json::Value::Null => duckdb::types::Value::Null, + serde_json::Value::Bool(b) => duckdb::types::Value::Boolean(*b), + + serde_json::Value::String(s) + if matches!( + arg_type.as_str(), + "timestamp" | "timestamptz" | "timestamp with time zone" | "datetime" + ) => + { + string_to_duckdb_timestamp(&s)? + } + serde_json::Value::String(s) if arg_type.as_str() == "date" => string_to_duckdb_date(&s)?, + serde_json::Value::String(s) if arg_type.as_str() == "time" => string_to_duckdb_time(&s)?, + serde_json::Value::String(s) => duckdb::types::Value::Text(s.clone()), + + serde_json::Value::Number(n) if n.is_i64() => { + let v = n.as_i64().unwrap(); + match arg_type.as_str() { + "tinyint" | "int1" => duckdb::types::Value::TinyInt(v as i8), + "smallint" | "int2" | "short" => duckdb::types::Value::SmallInt(v as i16), + "integer" | "int4" | "int" | "signed" => duckdb::types::Value::Int(v as i32), + "bigint" | "int8" | "long" => duckdb::types::Value::BigInt(v), + "hugeint" => duckdb::types::Value::HugeInt(v as i128), + "float" | "float4" | "real" => duckdb::types::Value::Float(v as f32), + "double" | "float8" => duckdb::types::Value::Double(v as f64), + _ => duckdb::types::Value::BigInt(v), // default fallback + } + } + + serde_json::Value::Number(n) if n.is_u64() => { + let v = n.as_u64().unwrap(); + match arg_type.as_str() { + "utinyint" => duckdb::types::Value::UTinyInt(v as u8), + "usmallint" => duckdb::types::Value::USmallInt(v as u16), + "uinteger" => duckdb::types::Value::UInt(v as u32), + "ubigint" | "uhugeint" => duckdb::types::Value::UBigInt(v), + _ => duckdb::types::Value::UBigInt(v), // default fallback + } + } + + serde_json::Value::Number(n) if n.is_f64() => { + let v = n.as_f64().unwrap(); + match arg_type.as_str() { + "float" | "float4" | "real" => duckdb::types::Value::Float(v as f32), + "double" | "float8" => duckdb::types::Value::Double(v), + "decimal" | "numeric" => { + duckdb::types::Value::Decimal(Decimal::from_f64(v).ok_or_else(|| { + Error::ExecutionErr("Could not convert f64 to Decimal".to_string()) + })?) + } + _ => duckdb::types::Value::Double(v), // default fallback + } + } + + serde_json::Value::Array(arr) => duckdb::types::Value::Array( + arr.iter() + .map(|val| json_value_to_duckdb_value(val, arg_type.as_str(), client)) + .collect::>>()?, + ), + serde_json::Value::Object(map) => duckdb::types::Value::Struct( + map.iter() + .map(|(k, v)| { + Ok::<_, Error>(( + k.clone(), + json_value_to_duckdb_value(v, arg_type.as_str(), client)?, + )) + }) + .collect::>>()? + .into(), + ), + + value @ _ => { + return Err(Error::ExecutionErr(format!( + "Unsupported type in query: {:?} and signature {arg_type:?}", + value + ))) + } + }; + Ok(duckdb_value) +} + +fn string_to_duckdb_timestamp(s: &str) -> Result { + let ts = chrono::DateTime::parse_from_rfc3339(s) + .map_err(|e: chrono::ParseError| Error::ExecutionErr(e.to_string()))?; + Ok(duckdb::types::Value::Timestamp( + TimeUnit::Millisecond, + ts.timestamp_millis(), + )) +} + +fn string_to_duckdb_date(s: &str) -> Result { + use chrono::Datelike; + let date = chrono::NaiveDate::parse_from_str(s, "%Y-%m-%d").unwrap(); + Ok(duckdb::types::Value::Date32(date.num_days_from_ce())) +} + +fn string_to_duckdb_time(s: &str) -> Result { + use chrono::Timelike; + let time = chrono::NaiveTime::parse_from_str(s, "%H:%M:%S").unwrap(); + Ok(duckdb::types::Value::Time64( + TimeUnit::Microsecond, + time.num_seconds_from_midnight() as i64, + )) +} + +struct ParsedAttachDbResource<'a> { + resource_path: &'a str, + name: &'a str, + db_type: &'a str, + extra_args: Option<&'a str>, +} +fn parse_attach_db_resource<'a>(query: &'a str) -> Option> { + lazy_static::lazy_static! { + static ref RE: regex::Regex = regex::Regex::new(r"ATTACH '\$res:([^']+)' AS (\S+) \(TYPE (\w+)(.*)\)").unwrap(); + } + + for cap in RE.captures_iter(query) { + if let (Some(resource_path), Some(name), Some(db_type)) = + (cap.get(1), cap.get(2), cap.get(3)) + { + let extra_args = cap.get(4).map(|m| query[m.start()..m.end()].trim()); + return Some(ParsedAttachDbResource { + resource_path: query[resource_path.start()..resource_path.end()].trim(), + name: query[name.start()..name.end()].trim(), + db_type: query[db_type.start()..db_type.end()].trim(), + extra_args, + }); + } + } + None +} + +async fn transform_attach_db_resource_query( + parsed: &ParsedAttachDbResource<'_>, + job_id: &Uuid, + client: &AuthedClient, +) -> Result> { + match parsed.db_type.to_lowercase().as_str() { + "postgres" => { + let resource: PgDatabase = client + .get_resource_value_interpolated(parsed.resource_path, Some(job_id.to_string())) + .await?; + + let attach_str = format!( + "ATTACH 'dbname={} {} host={} {} {}' AS {} (TYPE postgres{});", + resource.dbname, + resource + .user + .map(|u| format!("user={}", u)) + .unwrap_or_default(), + resource.host, + resource + .password + .map(|p| format!("password={}", p)) + .unwrap_or_default(), + resource + .port + .map(|p| format!("port={}", p)) + .unwrap_or_default(), + parsed.name, + parsed.extra_args.unwrap_or("") + ); + + Ok(vec![ + "INSTALL postgres;".to_string(), + "LOAD postgres;".to_string(), + attach_str, + ]) + } + "mysql" => { + #[cfg(not(feature = "mysql"))] + return Err(Error::ExecutionErr( + "MySQL feature is not enabled".to_string(), + )); + + #[cfg(feature = "mysql")] + { + let resource: MysqlDatabase = client + .get_resource_value_interpolated(parsed.resource_path, Some(job_id.to_string())) + .await?; + + let attach_str = format!( + "ATTACH 'database={} host={} ssl_mode={} {} {} {}' AS {} (TYPE mysql{});", + resource.database, + resource.host, + resource + .ssl + .map(|ssl| if ssl { "required" } else { "disabled" }) + .unwrap_or("preferred"), + resource + .password + .map(|p| format!("password={}", p)) + .unwrap_or_default(), + resource + .port + .map(|p| format!("port={}", p)) + .unwrap_or_default(), + resource + .user + .map(|u| format!("user={}", u)) + .unwrap_or_default(), + parsed.name, + parsed.extra_args.unwrap_or("") + ); + + Ok(vec![ + "INSTALL mysql;".to_string(), + "LOAD mysql;".to_string(), + attach_str, + ]) + } + } + "bigquery" => { + let resource: Value = client + .get_resource_value_interpolated(parsed.resource_path, Some(job_id.to_string())) + .await?; + // duckdb's bigquery extension requires a json file as credentials + let bq_credentials_path = make_bq_credentials_path(job_id); + env::set_var("GOOGLE_APPLICATION_CREDENTIALS", &bq_credentials_path); + tokio::fs::write(&bq_credentials_path, resource.to_string()) + .await + .map_err(|e| { + Error::ExecutionErr(format!( + "Failed to write BigQuery credentials to {}: {}", + &bq_credentials_path, e + )) + })?; + let project_id: String = serde_json::from_value( + resource + .get("project_id") + .ok_or_else(|| { + Error::ExecutionErr("BigQuery resource must contain project_id".to_string()) + })? + .to_owned(), + ) + .map_err(|_e| Error::ExecutionErr("failed project_id deserialize".to_string()))?; + let attach_str = format!( + "ATTACH 'project={}' as {} (TYPE bigquery{});", + project_id, + parsed.name, + parsed.extra_args.unwrap_or("") + ) + .to_string(); + Ok(vec![ + "INSTALL bigquery FROM community;".to_string(), + "LOAD bigquery;".to_string(), + attach_str, + ]) + } + _ => Err(Error::ExecutionErr(format!( + "Unsupported db type in DuckDB ATTACH: {}", + parsed.db_type + ))), + } +} + +// Returns the transformed query and the set of storages used +async fn transform_s3_uris( + query: &str, + client: &AuthedClient, +) -> Result<( + Option, + HashMap, DuckdbConnectionSettingsResponse>, +)> { + let mut transformed_query = None; + lazy_static::lazy_static! { + static ref RE: regex::Regex = regex::Regex::new(r"'s3://([^'/]*)/([^']+)'").unwrap(); + } + let mut used_storages = HashMap::new(); + for cap in RE.captures_iter(query) { + if let (storage, Some(s3_path)) = (cap.get(1), cap.get(2)) { + let s3_path = s3_path.as_str(); + let storage = match storage.map(|m| m.as_str()) { + Some("") | None => None, + Some(s) => Some(s.to_string()), + }; + let original_str_lit = + format!("'s3://{}/{}'", storage.as_deref().unwrap_or(""), s3_path); + let duckdb_conn_settings = client + .get_duckdb_connection_settings(&DuckdbConnectionSettingsQueryV2 { + s3_resource_path: None, + storage: storage.clone(), + }) + .await?; + let url = match &duckdb_conn_settings { + DuckdbConnectionSettingsResponse { s3_bucket: Some(bucket), .. } => { + format!("'s3://{bucket}/{s3_path}'") + } + DuckdbConnectionSettingsResponse { azure_container_path: Some(base), .. } => { + format!("'{base}/{s3_path}'") + } + _ => { + return Err(Error::ExecutionErr( + "DuckDB connection settings response must have either s3_bucket or azure_container_path".to_string(), + ))?; + } + }; + transformed_query = Some( + transformed_query + .unwrap_or(query.to_string()) + .replace(&original_str_lit, &url), + ); + used_storages.insert(storage, duckdb_conn_settings); + } + } + Ok((transformed_query, used_storages)) +} + +// BigQuery extension requires a json file as credentials +// The file path is set as an env var by do_duckdb +// It is created by transform_attach_db_resource_query (when bigquery is detected) +// and deleted by do_duckdb after the query is executed +fn make_bq_credentials_path(job_id: &Uuid) -> String { + format!("/tmp/service-account-credentials-{}.json", job_id) +} + +// duckdb-rs does not support named parameters, +// and it raises an error when passing unused arguments. We cannot prepare batch statements +// but only single SQL statements so it doesn't work when all arguments are not used by +// every single statement. +fn interpolate_named_args<'a>( + query: &str, + args: &'a HashMap, +) -> (String, Vec<&'a duckdb::types::Value>) { + let mut query = query.to_string(); + + let mut values = vec![]; + for (arg_name, arg_value) in args { + let pat = format!("${}", arg_name); + if !query.contains(&pat) { + continue; + } + values.push(arg_value); + query = query.replace(&pat, &format!("${}", values.len())); + } + (query, values) +} + +fn trunc_sig(query: &str) -> &str { + let idx = query.rfind("-- $").unwrap_or(query.len()); + // find next \n starting from idx and return everything after it + let idx = query[idx..].find('\n').map(|i| i + idx).unwrap_or(0); + &query[idx..] +} diff --git a/backend/windmill-worker/src/lib.rs b/backend/windmill-worker/src/lib.rs index 4ea56c090e..fa6eb10960 100644 --- a/backend/windmill-worker/src/lib.rs +++ b/backend/windmill-worker/src/lib.rs @@ -20,6 +20,8 @@ mod csharp_executor; #[cfg(feature = "enterprise")] mod dedicated_worker; mod deno_executor; +#[cfg(feature = "duckdb")] +mod duckdb_executor; mod global_cache; mod go_executor; mod graphql_executor; diff --git a/backend/windmill-worker/src/mysql_executor.rs b/backend/windmill-worker/src/mysql_executor.rs index ffd84db175..04403e4e08 100644 --- a/backend/windmill-worker/src/mysql_executor.rs +++ b/backend/windmill-worker/src/mysql_executor.rs @@ -32,13 +32,13 @@ use crate::{ }; #[derive(Deserialize)] -struct MysqlDatabase { - host: String, - user: Option, - password: Option, - port: Option, - database: String, - ssl: Option, +pub struct MysqlDatabase { + pub host: String, + pub user: Option, + pub password: Option, + pub port: Option, + pub database: String, + pub ssl: Option, } fn do_mysql_inner<'a>( diff --git a/backend/windmill-worker/src/pg_executor.rs b/backend/windmill-worker/src/pg_executor.rs index 67c2d945aa..2c7fe123dc 100644 --- a/backend/windmill-worker/src/pg_executor.rs +++ b/backend/windmill-worker/src/pg_executor.rs @@ -47,14 +47,14 @@ use lazy_static::lazy_static; use urlencoding::encode; use windmill_common::client::AuthedClient; #[derive(Deserialize)] -struct PgDatabase { - host: String, - user: Option, - password: Option, - port: Option, - sslmode: Option, - dbname: String, - root_certificate_pem: Option, +pub struct PgDatabase { + pub host: String, + pub user: Option, + pub password: Option, + pub port: Option, + pub sslmode: Option, + pub dbname: String, + pub root_certificate_pem: Option, } lazy_static! { diff --git a/backend/windmill-worker/src/windmill-client.js b/backend/windmill-worker/src/windmill-client.js index 3ae15e7453..1ecf28e664 100644 --- a/backend/windmill-worker/src/windmill-client.js +++ b/backend/windmill-worker/src/windmill-client.js @@ -2998,6 +2998,8 @@ var $RawScript = { "mssql", "graphql", "nativets", + "duckdb", + // for related places search: ADD_NEW_LANG ], }, path: { diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 221650389a..e15c24ef70 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -145,6 +145,9 @@ use crate::ansible_executor::handle_ansible_job; #[cfg(feature = "mysql")] use crate::mysql_executor::do_mysql; +#[cfg(feature = "duckdb")] +use crate::duckdb_executor::do_duckdb; + #[cfg(feature = "oracledb")] use crate::oracledb_executor::do_oracledb; @@ -2614,6 +2617,30 @@ async fn handle_code_execution_job( ) .await; } + } else if language == Some(ScriptLang::DuckDb) { + #[allow(unreachable_code)] + #[cfg(not(feature = "duckdb"))] + { + return Err(Error::internal_err( + "Duck DB requires the duckdb feature to be enabled".to_string(), + )); + } + + #[cfg(feature = "duckdb")] + { + return do_duckdb( + job, + &client, + &code, + conn, + mem_peak, + canceled_by, + worker_name, + column_order, + occupancy_metrics, + ) + .await; + } } else if language == Some(ScriptLang::Graphql) { return do_graphql( job, @@ -3021,6 +3048,7 @@ fn parse_sig_of_lang( ScriptLang::Snowflake => Some(windmill_parser_sql::parse_snowflake_sig(code)?), ScriptLang::Graphql => None, ScriptLang::Mssql => Some(windmill_parser_sql::parse_mssql_sig(code)?), + ScriptLang::DuckDb => Some(windmill_parser_sql::parse_duckdb_sig(code)?), ScriptLang::OracleDB => Some(windmill_parser_sql::parse_oracledb_sig(code)?), #[cfg(feature = "php")] ScriptLang::Php => Some(windmill_parser_php::parse_php_signature( diff --git a/cli/bootstrap/script_bootstrap.ts b/cli/bootstrap/script_bootstrap.ts index 2ac67bdb83..e9e8a12db5 100644 --- a/cli/bootstrap/script_bootstrap.ts +++ b/cli/bootstrap/script_bootstrap.ts @@ -76,6 +76,7 @@ func main() (interface{}, error) { bash: `echo "Hello world" `, + duckdb: `SELECT 'Hello world' AS message`, oracledb: `SELECT 'Hello world' AS message`, powershell: `Write-Output "Hello world"`, @@ -126,5 +127,5 @@ public class Main { } } `, -// for related places search: ADD_NEW_LANG + // for related places search: ADD_NEW_LANG }; diff --git a/cli/build.sh b/cli/build.sh index bf00b99a57..e228750b51 100755 --- a/cli/build.sh +++ b/cli/build.sh @@ -1,3 +1,4 @@ #!/bin/bash +# Note for mac OS users: you need to install gnu-sed with `brew install gnu-sed` and use `gsed` instead of `sed`. ./gen_wm_client.sh deno run -A dnt.ts diff --git a/cli/metadata.ts b/cli/metadata.ts index 3a83143182..ee39d0b9da 100644 --- a/cli/metadata.ts +++ b/cli/metadata.ts @@ -528,6 +528,9 @@ export async function inferSchema( { name: "database", typ: { resource: "postgresql" } }, ...inferedSchema.args, ]; + } else if (language === "duckdb") { + const { parse_sql } = await import("./wasm/regex/windmill_parser_wasm.js"); + inferedSchema = JSON.parse(parse_sql(content)); } else if (language === "graphql") { const { parse_graphql } = await import( "./wasm/regex/windmill_parser_wasm.js" diff --git a/cli/script.ts b/cli/script.ts index 3896d19c5c..e12c15d9ba 100644 --- a/cli/script.ts +++ b/cli/script.ts @@ -547,6 +547,8 @@ export function filePathExtensionFromContentType( return ".my.sql"; } else if (language === "bigquery") { return ".bq.sql"; + } else if (language === "duckdb") { + return ".duckdb.sql"; } else if (language === "oracledb") { return ".odb.sql"; } else if (language === "snowflake") { @@ -573,7 +575,7 @@ export function filePathExtensionFromContentType( return ".nu"; } else if (language === "java") { return ".java"; - // for related places search: ADD_NEW_LANG + // for related places search: ADD_NEW_LANG } else { throw new Error("Invalid language: " + language); } @@ -593,6 +595,7 @@ export const exts = [ ".odb.sql", ".sf.sql", ".ms.sql", + ".duckdb.sql", ".sql", ".gql", ".ps1", @@ -601,9 +604,8 @@ export const exts = [ ".cs", ".nu", ".playbook.yml", - ".java" - // for related places search: ADD_NEW_LANG - + ".java", + // for related places search: ADD_NEW_LANG ]; export function removeExtensionToPath(path: string): string { diff --git a/cli/script_common.ts b/cli/script_common.ts index a41d7dee8c..39906e7bb9 100644 --- a/cli/script_common.ts +++ b/cli/script_common.ts @@ -9,6 +9,7 @@ export type ScriptLanguage = | "postgresql" | "mysql" | "bigquery" + | "duckdb" | "oracledb" | "snowflake" | "mssql" @@ -19,7 +20,7 @@ export type ScriptLanguage = | "nu" | "ansible" | "java"; - // for related places search: ADD_NEW_LANG +// for related places search: ADD_NEW_LANG export function inferContentTypeFromFilePath( contentPath: string, @@ -43,6 +44,8 @@ export function inferContentTypeFromFilePath( return "bigquery"; } else if (contentPath.endsWith(".odb.sql")) { return "oracledb"; + } else if (contentPath.endsWith(".duckdb.sql")) { + return "duckdb"; } else if (contentPath.endsWith(".sf.sql")) { return "snowflake"; } else if (contentPath.endsWith(".ms.sql")) { @@ -67,7 +70,7 @@ export function inferContentTypeFromFilePath( return "nu"; } else if (contentPath.endsWith(".java")) { return "java"; - // for related places search: ADD_NEW_LANG + // for related places search: ADD_NEW_LANG } else { throw new Error( "Invalid language: " + contentPath.substring(contentPath.lastIndexOf(".")) diff --git a/cli/sync.ts b/cli/sync.ts index 867b8c4d17..456afcadf9 100644 --- a/cli/sync.ts +++ b/cli/sync.ts @@ -344,6 +344,7 @@ export function newPathAssigner(defaultTs: "bun" | "deno"): PathAssigner { else if (language == "nu") ext = "nu"; else if (language == "ansible") ext = "playbook.yml"; else if (language == "java") ext = "java"; + else if (language == "duckdb") ext = "duckdb.sql"; // for related places search: ADD_NEW_LANG else ext = "no_ext"; diff --git a/cli/wasm/regex/windmill_parser_wasm.js b/cli/wasm/regex/windmill_parser_wasm.js index 5418fad879..7867ab4a35 100644 --- a/cli/wasm/regex/windmill_parser_wasm.js +++ b/cli/wasm/regex/windmill_parser_wasm.js @@ -1,5 +1,6 @@ + let WASM_VECTOR_LEN = 0; let cachedUint8ArrayMemory0 = null; diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 899306ebf0..8f93204d54 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -74,7 +74,7 @@ "windmill-parser-wasm-nu": "^1.474.1", "windmill-parser-wasm-php": "^1.429.0", "windmill-parser-wasm-py": "^1.477.1", - "windmill-parser-wasm-regex": "^1.481.0", + "windmill-parser-wasm-regex": "^1.492.1", "windmill-parser-wasm-rust": "^1.429.0", "windmill-parser-wasm-ts": "^1.486.1", "windmill-parser-wasm-yaml": "^1.429.0", @@ -12837,9 +12837,9 @@ "integrity": "sha512-EY3mSMWpqFPzd7fsLg2/hSfQFU8HpW9nplFwm4JHHCDbcTpBzlvzjPJoHAAGO5kMzowAxjqi5ai/mXjeUWuiSg==" }, "node_modules/windmill-parser-wasm-regex": { - "version": "1.481.0", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.481.0.tgz", - "integrity": "sha512-QY9NOaF0M55SBEbbcAkHoXppjdS2gb2NlwqhlZ1ciYWTtIrbZoL5K8xoG4FbB5vWRHxInG1aRdA2tb+JP180yA==" + "version": "1.492.1", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.492.1.tgz", + "integrity": "sha512-CBdjz3x00z2xA4BRZv8/hsuHcpZgL28UKTjYVSj5O9XMYlY9g0LNsEQ5TTl/oVyFBs62/CPyvWwGvW4oSAObzg==" }, "node_modules/windmill-parser-wasm-rust": { "version": "1.429.0", diff --git a/frontend/package.json b/frontend/package.json index f302f5c873..9d3bbe19cc 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -143,7 +143,7 @@ "windmill-parser-wasm-nu": "^1.474.1", "windmill-parser-wasm-php": "^1.429.0", "windmill-parser-wasm-py": "^1.477.1", - "windmill-parser-wasm-regex": "^1.481.0", + "windmill-parser-wasm-regex": "^1.492.1", "windmill-parser-wasm-rust": "^1.429.0", "windmill-parser-wasm-ts": "^1.486.1", "windmill-parser-wasm-yaml": "^1.429.0", diff --git a/frontend/src/lib/components/Editor.svelte b/frontend/src/lib/components/Editor.svelte index 35c4576251..02b7f4170b 100644 --- a/frontend/src/lib/components/Editor.svelte +++ b/frontend/src/lib/components/Editor.svelte @@ -133,6 +133,7 @@ import EditorTheme from './EditorTheme.svelte' import { BIGQUERY_TYPES, + DUCKDB_TYPES, MSSQL_TYPES, MYSQL_TYPES, ORACLEDB_TYPES, @@ -482,7 +483,9 @@ ? MSSQL_TYPES : scriptLang === 'oracledb' ? ORACLEDB_TYPES - : [] + : scriptLang === 'duckdb' + ? DUCKDB_TYPES + : [] ).map((t) => ({ label: t, kind: languages.CompletionItemKind.Function, diff --git a/frontend/src/lib/components/HighlightCode.svelte b/frontend/src/lib/components/HighlightCode.svelte index ac4357e5ca..bf196c7868 100644 --- a/frontend/src/lib/components/HighlightCode.svelte +++ b/frontend/src/lib/components/HighlightCode.svelte @@ -55,6 +55,8 @@ return sql case 'oracledb': return sql + case 'duckdb': + return sql case 'powershell': return powershell case 'php': @@ -66,10 +68,10 @@ case 'nu': return python case 'ansible': - return yaml; + return yaml case 'java': - return java; - // for related places search: ADD_NEW_LANG + return java + // for related places search: ADD_NEW_LANG default: return typescript } diff --git a/frontend/src/lib/components/common/languageIcons/LanguageIcon.svelte b/frontend/src/lib/components/common/languageIcons/LanguageIcon.svelte index 9dd3c9860a..6a3de43bf9 100644 --- a/frontend/src/lib/components/common/languageIcons/LanguageIcon.svelte +++ b/frontend/src/lib/components/common/languageIcons/LanguageIcon.svelte @@ -22,6 +22,7 @@ import CSharpIcon from '$lib/components/icons/CSharpIcon.svelte' import NuIcon from '$lib/components/icons/NuIcon.svelte' import JavaIcon from '$lib/components/icons/JavaIcon.svelte' + import DuckDbIcon from '$lib/components/icons/DuckDbIcon.svelte' export let lang: | SupportedLanguage @@ -49,6 +50,7 @@ postgresql: 'Postgresql', bigquery: 'BigQuery', oracledb: 'Oracle Database', + duckdb: 'DuckDB', snowflake: 'Snowflake', mysql: 'MySQL', mssql: 'MS SQL Server', @@ -59,7 +61,7 @@ csharp: 'C#', nu: 'Nu', java: 'Java' - // for related places search: ADD_NEW_LANG + // for related places search: ADD_NEW_LANG } const langToComponent: Record< @@ -91,8 +93,9 @@ ansible: AnsibleIcon, csharp: CSharpIcon, nu: NuIcon, - java: JavaIcon - // for related places search: ADD_NEW_LANG + java: JavaIcon, + duckdb: DuckDbIcon + // for related places search: ADD_NEW_LANG } let subIconScale = width === 30 ? 0.6 : 0.8 diff --git a/frontend/src/lib/components/icons/DuckDbIcon.svelte b/frontend/src/lib/components/icons/DuckDbIcon.svelte new file mode 100644 index 0000000000..7229a377a6 --- /dev/null +++ b/frontend/src/lib/components/icons/DuckDbIcon.svelte @@ -0,0 +1,19 @@ + + + + + + + diff --git a/frontend/src/lib/consts.ts b/frontend/src/lib/consts.ts index d319868795..847bbf42da 100644 --- a/frontend/src/lib/consts.ts +++ b/frontend/src/lib/consts.ts @@ -203,6 +203,56 @@ export const MSSQL_TYPES = [ 'decimal', 'bit' ] +export const DUCKDB_TYPES = [ + 'BIGINT', + 'INT8', + 'LONG', + 'BIT', + 'BITSTRING', + 'BLOB', + 'BYTEA', + 'BINARY', + 'VARBINARY', + 'BOOLEAN', + 'BOOL', + 'LOGICAL', + 'DATE', + 'DECIMAL', + 'NUMERIC', + 'DOUBLE', + 'FLOAT8', + 'FLOAT', + 'FLOAT4', + 'REAL', + 'HUGEINT', + 'INTEGER', + 'INT4', + 'INT', + 'SIGNED', + 'INTERVAL', + 'JSON', + 'SMALLINT', + 'INT2', + 'SHORT', + 'TIME', + 'TIMESTAMP WITH TIME ZONE', + 'TIMESTAMPTZ', + 'TIMESTAMP', + 'DATETIME', + 'TINYINT', + 'INT1', + 'UBIGINT', + 'UHUGEINT', + 'UINTEGER', + 'USMALLINT', + 'UTINYINT', + 'UUID', + 'VARCHAR', + 'CHAR', + 'BPCHAR', + 'TEXT', + 'STRING' +] export const DB_TYPES: Record = { bigquery: BIGQUERY_TYPES, diff --git a/frontend/src/lib/infer.ts b/frontend/src/lib/infer.ts index 5dcb6772f3..6582930715 100644 --- a/frontend/src/lib/infer.ts +++ b/frontend/src/lib/infer.ts @@ -9,6 +9,7 @@ import initRegexParsers, { parse_sql, parse_mysql, parse_oracledb, + parse_duckdb, parse_bigquery, parse_snowflake, parse_graphql, @@ -96,7 +97,9 @@ export async function inferArgs( } let inlineDBResource: string | undefined = undefined - if (['postgresql', 'mysql', 'bigquery', 'snowflake', 'mssql', 'oracledb'].includes(language ?? '')) { + if ( + ['postgresql', 'mysql', 'bigquery', 'snowflake', 'mssql', 'oracledb'].includes(language ?? '') + ) { await initWasmRegex() inlineDBResource = parse_db_resource(code) } @@ -147,6 +150,9 @@ export async function inferArgs( ...inferedSchema.args ] } + } else if (language == 'duckdb') { + await initWasmRegex() + inferedSchema = JSON.parse(parse_duckdb(code)) } else if (language == 'snowflake') { inferedSchema = JSON.parse(parse_snowflake(code)) if (inlineDBResource === undefined) { @@ -194,7 +200,7 @@ export async function inferArgs( } else if (language == 'java') { await initWasmJava() inferedSchema = JSON.parse(parse_java(code)) - // for related places search: ADD_NEW_LANG + // for related places search: ADD_NEW_LANG } else { return null } diff --git a/frontend/src/lib/inferArgSig.ts b/frontend/src/lib/inferArgSig.ts index ae4ad83794..68d3fc814e 100644 --- a/frontend/src/lib/inferArgSig.ts +++ b/frontend/src/lib/inferArgSig.ts @@ -5,23 +5,21 @@ export function argSigToJsonSchemaType( | string | { resource: string | null } | { - list: - | (string | { object: { key: string; typ: any }[] }) - | { str: any } - | { object: { key: string; typ: any }[] } - | null - } + list: + | (string | { object: { key: string; typ: any }[] }) + | { str: any } + | { object: { key: string; typ: any }[] } + | null + } | { dynselect: string } | { str: string[] | null } | { object: { key: string; typ: any }[] } | { - oneof: [ - { + oneof: { label: string properties: { key: string; typ: any }[] - } - ] - }, + }[] + }, oldS: SchemaProperty ): void { const newS: SchemaProperty = { type: '' } diff --git a/frontend/src/lib/script_helpers.ts b/frontend/src/lib/script_helpers.ts index ceb6644c42..07c05974dc 100644 --- a/frontend/src/lib/script_helpers.ts +++ b/frontend/src/lib/script_helpers.ts @@ -307,6 +307,19 @@ INSERT INTO demo VALUES (@P1, @P2); UPDATE demo SET col2 = @P3 WHERE col2 = @P2; ` +const DUCKDB_INIT_CODE = `-- $friends_csv (s3object) +-- $name (text) = Ben + +ATTACH '$res:u/demo/amazed_postgresql' AS db (TYPE postgres); +CREATE TABLE IF NOT EXISTS db.public.friends (name text); + +INSERT INTO db.public.friends + SELECT name FROM read_csv($friends_csv); + +SELECT 'Hello ' || $name || ', you have ' || + (SELECT COUNT(*) FROM read_csv($friends_csv)) || ' new friends !'; +` + const GRAPHQL_INIT_CODE = `query($name4: String, $name2: Int, $name3: [String]) { demo(name1: $name1, name2: $name2, name3: $name3) { name1, @@ -1134,6 +1147,9 @@ export const INITIAL_CODE = { mssql: { script: MSSQL_INIT_CODE }, + duckdb: { + script: DUCKDB_INIT_CODE + }, graphql: { script: GRAPHQL_INIT_CODE }, @@ -1258,6 +1274,8 @@ export function initialCode( return INITIAL_CODE.mssql.script } else if (language == 'graphql') { return INITIAL_CODE.graphql.script + } else if (language == 'duckdb') { + return INITIAL_CODE.duckdb.script } else if (language == 'php') { return INITIAL_CODE.php.script } else if (language == 'rust') { diff --git a/frontend/src/lib/scripts.ts b/frontend/src/lib/scripts.ts index bc8a44b434..961ae72488 100644 --- a/frontend/src/lib/scripts.ts +++ b/frontend/src/lib/scripts.ts @@ -38,6 +38,8 @@ export function scriptLangToEditorLang( return 'sql' } else if (lang == 'mssql') { return 'sql' + } else if (lang == 'duckdb') { + return 'sql' } else if (lang == 'python3') { return 'python' } else if (lang == 'bash') { @@ -140,7 +142,8 @@ const scriptLanguagesArray: [SupportedLanguage | 'docker' | 'bunnative', string] ['csharp', 'C#'], ['docker', 'Docker'], ['nu', 'Nu'], - ['java', 'Java'] + ['java', 'Java'], + ['duckdb', 'DuckDB'] // for related places search: ADD_NEW_LANG ] export function processLangs(selected: string | undefined, langs: string[]): string[] { @@ -150,7 +153,7 @@ export function processLangs(selected: string | undefined, langs: string[]): str let ls = langs.filter((lang) => lang !== 'nativets') //those languages are newer and may not be in the saved list - let nl = ['bunnative', 'rust', 'ansible', 'csharp', 'nu', 'java'] + let nl = ['bunnative', 'rust', 'ansible', 'csharp', 'nu', 'java', 'duckdb'] // for related places search: ADD_NEW_LANG nl.forEach((lang) => { if (!ls.includes(lang)) { diff --git a/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte b/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte index 35f23fc9e7..cab4ef6adb 100644 --- a/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte @@ -755,7 +755,7 @@ priority: {job.priority} {/if} - {#if job.tag && !['deno', 'python3', 'flow', 'other', 'go', 'postgresql', 'mysql', 'bigquery', 'snowflake', 'mssql', 'graphql', 'oracledb', 'nativets', 'bash', 'powershell', 'php', 'rust', 'other', 'ansible', 'csharp', 'nu', 'java', 'dependency'].includes(job.tag)} + {#if job.tag && !['deno', 'python3', 'flow', 'other', 'go', 'postgresql', 'mysql', 'bigquery', 'snowflake', 'mssql', 'graphql', 'oracledb', 'nativets', 'bash', 'powershell', 'php', 'rust', 'other', 'ansible', 'csharp', 'nu', 'java', 'duckdb', 'dependency'].includes(job.tag)}
Tag: {job.tag}