Compare commits

..

3 Commits

Author SHA1 Message Date
Ruben Fiszel
0cc3342e12 chore: update sqlx offline cache
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-05 06:45:57 +00:00
Ruben Fiszel
d081323250 chore: update ee-repo-ref
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-05 06:39:46 +00:00
Ruben Fiszel
ebbc259a37 feat: add volume limits info in CE volumes drawer
Show an info alert in the volumes drawer when running in Community
Edition, mentioning the 20 volumes per workspace and 50 MB per file
limits. Update ee-repo-ref for companion EE changes.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-05 06:36:24 +00:00
56 changed files with 341 additions and 1694 deletions

View File

@@ -1,28 +1,5 @@
# Changelog
## [1.651.1](https://github.com/windmill-labs/windmill/compare/v1.651.0...v1.651.1) (2026-03-05)
### Bug Fixes
* prevent slow loading toast interval from leaking on promise cancellation ([#8240](https://github.com/windmill-labs/windmill/issues/8240)) ([2e582b1](https://github.com/windmill-labs/windmill/commit/2e582b1bc1c299388a3c97cfddff9d0eb92858f2))
* suppress unused variable warnings on windows builds ([#8241](https://github.com/windmill-labs/windmill/issues/8241)) ([2d58382](https://github.com/windmill-labs/windmill/commit/2d583826dc065c05684d4cd1d1510f0d1f2d9ae9))
## [1.651.0](https://github.com/windmill-labs/windmill/compare/v1.650.0...v1.651.0) (2026-03-05)
### Features
* add sandbox annotations, volume mounts, for AI sandbox starting with claude ([#8058](https://github.com/windmill-labs/windmill/issues/8058)) ([5f0ef93](https://github.com/windmill-labs/windmill/commit/5f0ef936d1d5d07d01c8e07e26ec254feebef8fb))
* hash-based MCP tool names for long paths ([#8133](https://github.com/windmill-labs/windmill/issues/8133)) ([ce041e8](https://github.com/windmill-labs/windmill/commit/ce041e8a5e7ff105df389875d9981f3843d4ce39))
### Bug Fixes
* **python-client:** add delete_s3_object ([#8216](https://github.com/windmill-labs/windmill/issues/8216)) ([90f4c64](https://github.com/windmill-labs/windmill/commit/90f4c64ee12e1d04ce846ff88d6658f667e194e0))
* update CLI bun template to match UI template ([#8238](https://github.com/windmill-labs/windmill/issues/8238)) ([a8cbe93](https://github.com/windmill-labs/windmill/commit/a8cbe9396ffc51140dce5582d57f4dc59873304e))
* write fallback package.json for codebase mode nsjail ([#8239](https://github.com/windmill-labs/windmill/issues/8239)) ([d46913b](https://github.com/windmill-labs/windmill/commit/d46913b74a0ffd41d2323e0355cc81954f09e29d))
## [1.650.0](https://github.com/windmill-labs/windmill/compare/v1.649.0...v1.650.0) (2026-03-05)

View File

@@ -15,7 +15,7 @@
]
},
"nullable": [
false
true
]
},
"hash": "2d6607b3c38fe72b5663c32de58dacbabed4c5ae28101e3ae2694f96fd055a91"

View File

@@ -1,26 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, dedicated_workers, wm_version, vcpus, memory, job_isolation, native_mode, uses_batch_http_pull) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) ON CONFLICT (worker)\n DO UPDATE set ip = EXCLUDED.ip, custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group, dedicated_workers = EXCLUDED.dedicated_workers, native_mode = EXCLUDED.native_mode, uses_batch_http_pull = EXCLUDED.uses_batch_http_pull",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Varchar",
"TextArray",
"Varchar",
"Varchar",
"TextArray",
"Varchar",
"Int8",
"Int8",
"Text",
"Bool",
"Bool"
]
},
"nullable": []
},
"hash": "3e8afd021088a99a24f27fa6f0a1b7f3edba3e9b834c814b464305bc2eb6ba80"
}

View File

@@ -15,7 +15,7 @@
]
},
"nullable": [
true
null
]
},
"hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55"

View File

@@ -1,26 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1, custom_tags = $2,\n occupancy_rate = $3, memory_usage = $4, wm_memory_usage = $5, vcpus = COALESCE($7, vcpus),\n memory = COALESCE($8, memory), occupancy_rate_15s = $9, occupancy_rate_5m = $10, occupancy_rate_30m = $11, native_mode = $12, uses_batch_http_pull = $13 WHERE worker = $6",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Int4",
"TextArray",
"Float4",
"Int8",
"Int8",
"Text",
"Int8",
"Int8",
"Float4",
"Float4",
"Float4",
"Bool",
"Bool"
]
},
"nullable": []
},
"hash": "6cd099d458ac380d5da27b9e69da035755496ea50f2b78fb9b1cd3a2eb7e7625"
}

View File

@@ -15,7 +15,7 @@
]
},
"nullable": [
false
true
]
},
"hash": "eba16eb819e2644284fb073c891706d78a6f24cb0e614d7d81ba1b643805bf06"

143
backend/Cargo.lock generated
View File

@@ -15741,7 +15741,7 @@ dependencies = [
[[package]]
name = "windmill"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"anyhow",
"async-nats",
@@ -15773,7 +15773,6 @@ dependencies = [
"sql-builder",
"sqlx",
"strum 0.27.2",
"tar",
"tempfile",
"tikv-jemalloc-ctl",
"tikv-jemalloc-sys",
@@ -15808,7 +15807,7 @@ dependencies = [
[[package]]
name = "windmill-alerting"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"axum 0.7.9",
"chrono",
@@ -15821,7 +15820,7 @@ dependencies = [
[[package]]
name = "windmill-api"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"anyhow",
"argon2",
@@ -15960,7 +15959,7 @@ dependencies = [
[[package]]
name = "windmill-api-agent-workers"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"axum 0.7.9",
"chrono",
@@ -15983,7 +15982,7 @@ dependencies = [
[[package]]
name = "windmill-api-assets"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"axum 0.7.9",
"chrono",
@@ -15996,7 +15995,7 @@ dependencies = [
[[package]]
name = "windmill-api-auth"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"anyhow",
"axum 0.7.9",
@@ -16022,7 +16021,7 @@ dependencies = [
[[package]]
name = "windmill-api-client"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"reqwest 0.12.28",
"serde",
@@ -16032,7 +16031,7 @@ dependencies = [
[[package]]
name = "windmill-api-configs"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"axum 0.7.9",
"chrono",
@@ -16049,7 +16048,7 @@ dependencies = [
[[package]]
name = "windmill-api-debug"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"axum 0.7.9",
"base64 0.22.1",
@@ -16072,7 +16071,7 @@ dependencies = [
[[package]]
name = "windmill-api-embeddings"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"anyhow",
"axum 0.7.9",
@@ -16095,7 +16094,7 @@ dependencies = [
[[package]]
name = "windmill-api-flow-conversations"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"axum 0.7.9",
"chrono",
@@ -16111,7 +16110,7 @@ dependencies = [
[[package]]
name = "windmill-api-flows"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"axum 0.7.9",
"chrono",
@@ -16131,7 +16130,7 @@ dependencies = [
[[package]]
name = "windmill-api-groups"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"axum 0.7.9",
"chrono",
@@ -16151,7 +16150,7 @@ dependencies = [
[[package]]
name = "windmill-api-inputs"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"axum 0.7.9",
"chrono",
@@ -16165,7 +16164,7 @@ dependencies = [
[[package]]
name = "windmill-api-integration-tests"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"anyhow",
"async-nats",
@@ -16192,7 +16191,7 @@ dependencies = [
[[package]]
name = "windmill-api-jobs"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"anyhow",
"axum 0.7.9",
@@ -16217,7 +16216,7 @@ dependencies = [
[[package]]
name = "windmill-api-npm-proxy"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"axum 0.7.9",
"flate2",
@@ -16235,7 +16234,7 @@ dependencies = [
[[package]]
name = "windmill-api-openapi"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"anyhow",
"axum 0.7.9",
@@ -16256,7 +16255,7 @@ dependencies = [
[[package]]
name = "windmill-api-schedule"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"axum 0.7.9",
"chrono",
@@ -16276,7 +16275,7 @@ dependencies = [
[[package]]
name = "windmill-api-scripts"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"axum 0.7.9",
"chrono",
@@ -16306,7 +16305,7 @@ dependencies = [
[[package]]
name = "windmill-api-settings"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"anyhow",
"axum 0.7.9",
@@ -16333,7 +16332,7 @@ dependencies = [
[[package]]
name = "windmill-api-sse"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"lazy_static",
"serde",
@@ -16345,7 +16344,7 @@ dependencies = [
[[package]]
name = "windmill-api-users"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"argon2",
"axum 0.7.9",
@@ -16368,7 +16367,7 @@ dependencies = [
[[package]]
name = "windmill-api-workers"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"axum 0.7.9",
"chrono",
@@ -16382,7 +16381,7 @@ dependencies = [
[[package]]
name = "windmill-api-workspaces"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"axum 0.7.9",
"chrono",
@@ -16413,7 +16412,7 @@ dependencies = [
[[package]]
name = "windmill-audit"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"chrono",
"lazy_static",
@@ -16427,7 +16426,7 @@ dependencies = [
[[package]]
name = "windmill-autoscaling"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"anyhow",
"axum 0.7.9",
@@ -16446,7 +16445,7 @@ dependencies = [
[[package]]
name = "windmill-common"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"aes-gcm",
"anyhow",
@@ -16545,7 +16544,7 @@ dependencies = [
[[package]]
name = "windmill-dep-map"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"chrono",
"itertools 0.14.0",
@@ -16564,7 +16563,7 @@ dependencies = [
[[package]]
name = "windmill-git-sync"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"regex",
"serde",
@@ -16579,7 +16578,7 @@ dependencies = [
[[package]]
name = "windmill-indexer"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"anyhow",
"astral-tokio-tar",
@@ -16603,7 +16602,7 @@ dependencies = [
[[package]]
name = "windmill-jseval"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"anyhow",
"futures",
@@ -16620,7 +16619,7 @@ dependencies = [
[[package]]
name = "windmill-macros"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"itertools 0.14.0",
"lazy_static",
@@ -16636,7 +16635,7 @@ dependencies = [
[[package]]
name = "windmill-mcp"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16657,7 +16656,7 @@ dependencies = [
[[package]]
name = "windmill-native-triggers"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16688,7 +16687,7 @@ dependencies = [
[[package]]
name = "windmill-oauth"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"anyhow",
"async-oauth2",
@@ -16712,7 +16711,7 @@ dependencies = [
[[package]]
name = "windmill-object-store"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"anyhow",
"async-stream",
@@ -16746,7 +16745,7 @@ dependencies = [
[[package]]
name = "windmill-operator"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"anyhow",
"futures",
@@ -16764,7 +16763,7 @@ dependencies = [
[[package]]
name = "windmill-parser"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"convert_case 0.6.0",
"serde",
@@ -16773,7 +16772,7 @@ dependencies = [
[[package]]
name = "windmill-parser-bash"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -16785,7 +16784,7 @@ dependencies = [
[[package]]
name = "windmill-parser-csharp"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"anyhow",
"serde_json",
@@ -16797,7 +16796,7 @@ dependencies = [
[[package]]
name = "windmill-parser-go"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"anyhow",
"gosyn",
@@ -16809,7 +16808,7 @@ dependencies = [
[[package]]
name = "windmill-parser-graphql"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -16821,7 +16820,7 @@ dependencies = [
[[package]]
name = "windmill-parser-java"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"anyhow",
"serde_json",
@@ -16833,7 +16832,7 @@ dependencies = [
[[package]]
name = "windmill-parser-nu"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"anyhow",
"nu-parser",
@@ -16844,7 +16843,7 @@ dependencies = [
[[package]]
name = "windmill-parser-php"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -16855,7 +16854,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -16868,7 +16867,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py-imports"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"anyhow",
"async-recursion",
@@ -16892,7 +16891,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ruby"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -16906,7 +16905,7 @@ dependencies = [
[[package]]
name = "windmill-parser-rust"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"anyhow",
"convert_case 0.6.0",
@@ -16923,7 +16922,7 @@ dependencies = [
[[package]]
name = "windmill-parser-sql"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -16938,7 +16937,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ts"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -16957,7 +16956,7 @@ dependencies = [
[[package]]
name = "windmill-parser-yaml"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"anyhow",
"serde",
@@ -16968,7 +16967,7 @@ dependencies = [
[[package]]
name = "windmill-queue"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"anyhow",
"async-recursion",
@@ -17005,7 +17004,7 @@ dependencies = [
[[package]]
name = "windmill-runtime-nativets"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"anyhow",
"const_format",
@@ -17043,7 +17042,7 @@ dependencies = [
[[package]]
name = "windmill-sql-datatype-parser-wasm"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"getrandom 0.3.4",
"wasm-bindgen",
@@ -17054,7 +17053,7 @@ dependencies = [
[[package]]
name = "windmill-store"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"anyhow",
"async-recursion",
@@ -17083,7 +17082,7 @@ dependencies = [
[[package]]
name = "windmill-test-utils"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"anyhow",
"axum 0.7.9",
@@ -17106,7 +17105,7 @@ dependencies = [
[[package]]
name = "windmill-trigger"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"anyhow",
"async-trait",
@@ -17139,7 +17138,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-email"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"anyhow",
"async-trait",
@@ -17159,7 +17158,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-gcp"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"anyhow",
"async-trait",
@@ -17193,7 +17192,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-http"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"anyhow",
"async-trait",
@@ -17228,7 +17227,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-kafka"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"anyhow",
"async-trait",
@@ -17251,7 +17250,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-mqtt"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"anyhow",
"async-trait",
@@ -17275,7 +17274,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-nats"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"anyhow",
"async-nats",
@@ -17299,7 +17298,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-postgres"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"anyhow",
"async-trait",
@@ -17334,7 +17333,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-sqs"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"anyhow",
"async-trait",
@@ -17362,7 +17361,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-websocket"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"anyhow",
"async-trait",
@@ -17385,7 +17384,7 @@ dependencies = [
[[package]]
name = "windmill-types"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"anyhow",
"bitflags 2.9.4",
@@ -17403,7 +17402,7 @@ dependencies = [
[[package]]
name = "windmill-worker"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"anyhow",
"async-once-cell",
@@ -17509,7 +17508,7 @@ dependencies = [
[[package]]
name = "windmill-worker-volumes"
version = "1.651.1"
version = "1.650.0"
dependencies = [
"bytes",
"futures",

View File

@@ -1,6 +1,6 @@
[package]
name = "windmill"
version = "1.651.1"
version = "1.650.0"
authors.workspace = true
edition.workspace = true
@@ -77,7 +77,7 @@ members = [
exclude = ["./windmill-duckdb-ffi-internal"]
[workspace.package]
version = "1.651.1"
version = "1.650.0"
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
edition = "2021"
@@ -257,7 +257,6 @@ axum.workspace = true
serde.workspace = true
windmill-api-client.workspace = true
tempfile.workspace = true
tar.workspace = true
windmill-parser-ts.workspace = true
rumqttc.workspace = true
rdkafka.workspace = true

View File

@@ -1 +1 @@
c3c543f4c60a8c4dfe0d912c79a051376fb091a9
05385738e36e81f5bc51d15c0ca60bba30457c21

View File

@@ -1 +0,0 @@
ALTER TABLE worker_ping DROP COLUMN IF EXISTS uses_batch_http_pull;

View File

@@ -1 +0,0 @@
ALTER TABLE worker_ping ADD COLUMN IF NOT EXISTS uses_batch_http_pull BOOLEAN NOT NULL DEFAULT false;

View File

@@ -61,9 +61,8 @@ use windmill_common::{
MODE_AND_ADDONS,
},
worker::{
is_native_mode_from_env, reload_custom_tags_setting, Connection, HttpClient, HUB_CACHE_DIR,
HUB_RT_CACHE_DIR, NATIVE_MODE_RESOLVED, TMP_LOGS_DIR, USES_BATCH_HTTP_PULL, WINDMILL_DIR,
WORKER_GROUP,
is_native_mode_from_env, reload_custom_tags_setting, Connection, HUB_CACHE_DIR,
HUB_RT_CACHE_DIR, NATIVE_MODE_RESOLVED, TMP_LOGS_DIR, WINDMILL_DIR, WORKER_GROUP,
},
KillpillSender, DEFAULT_HUB_BASE_URL, METRICS_ENABLED,
};
@@ -921,20 +920,6 @@ Windmill Community Edition {GIT_VERSION}
default_base_internal_url.clone()
};
// BATCH_PULL_URL: explicit URL for native workers to pull jobs via HTTP.
// In standalone mode (server_mode=true), defaults to the local server.
let batch_pull_url: Option<String> = if is_native_mode_from_env() {
if let Ok(url) = std::env::var("BATCH_PULL_URL") {
Some(url)
} else if server_mode {
Some(default_base_internal_url.clone())
} else {
None
}
} else {
None
};
initial_load(
&conn,
killpill_tx.clone(),
@@ -1145,30 +1130,6 @@ Windmill Community Edition {GIT_VERSION}
)?;
let mut workers = vec![];
// For native workers, create a self-signed JWT for batch pulling via HTTP.
// Enabled when BATCH_PULL_URL is set (explicitly or auto-detected in standalone mode).
let batch_pull_client = if let Some(ref pull_url) = batch_pull_url {
match create_native_batch_pull_client(pull_url).await {
Ok(client) => {
tracing::info!(
"Native batch pull client created for HTTP pull at {}",
pull_url
);
USES_BATCH_HTTP_PULL
.store(true, std::sync::atomic::Ordering::Relaxed);
Some(client)
}
Err(e) => {
tracing::warn!(
"Failed to create native batch pull client, falling back to SQL pull: {e:#}"
);
None
}
}
} else {
None
};
for i in 0..num_workers {
let suffix = if i == 0 && first_suffix.is_some() {
first_suffix.as_ref().unwrap().clone()
@@ -1192,7 +1153,6 @@ Windmill Community Edition {GIT_VERSION}
WORKER_GROUP.as_str(),
&suffix,
),
batch_pull_client: batch_pull_client.clone(),
};
workers.push(worker_conn);
}
@@ -1806,7 +1766,6 @@ fn display_config(envs: &[&str]) {
pub struct WorkerConn {
conn: Connection,
worker_name: String,
batch_pull_client: Option<HttpClient>,
}
pub async fn run_workers(
@@ -1877,7 +1836,6 @@ pub async fn run_workers(
let wk_conf = &workers[i as usize - 1];
let conn1 = wk_conf.conn.clone();
let worker_name = wk_conf.worker_name.clone();
let batch_pull_client = wk_conf.batch_pull_client.clone();
WORKERS_NAMES.write().await.push(worker_name.clone());
let ip = ip.clone();
let rx = killpill_rxs.pop().unwrap();
@@ -1900,7 +1858,6 @@ pub async fn run_workers(
rx,
tx,
&base_internal_url,
batch_pull_client.as_ref(),
);
// #[cfg(tokio_unstable)]
@@ -1919,41 +1876,6 @@ pub async fn run_workers(
Ok(())
}
/// Create an HTTP client for native workers to pull jobs from the local server's batch buffer.
/// Self-signs a JWT with native_mode=true using the same JWT secret the server uses.
async fn create_native_batch_pull_client(base_internal_url: &str) -> anyhow::Result<HttpClient> {
use windmill_common::agent_workers::{build_agent_http_client, AGENT_JWT_PREFIX};
use windmill_common::jwt::encode_with_internal_secret;
#[derive(serde::Serialize)]
struct NativeAgentAuth {
worker_group: String,
tags: Vec<String>,
native_mode: Option<bool>,
exp: usize,
}
let worker_config = windmill_common::worker::WORKER_CONFIG.read().await;
let tags = worker_config.worker_tags.clone();
drop(worker_config);
// Token expires in 30 days — renewed on restart
let exp = (chrono::Utc::now() + chrono::Duration::days(30)).timestamp() as usize;
let claims = NativeAgentAuth {
worker_group: WORKER_GROUP.to_string(),
tags,
native_mode: Some(true),
exp,
};
let jwt = encode_with_internal_secret(claims).await?;
let token = format!("{}{}", AGENT_JWT_PREFIX, jwt);
let suffix = create_default_worker_suffix(&HOSTNAME);
Ok(build_agent_http_client(&suffix, &token, base_internal_url))
}
async fn send_delayed_killpill(tx: &KillpillSender, mut max_delay_secs: u64, context: &str) {
if max_delay_secs == 0 {
max_delay_secs = 1;

View File

@@ -174,7 +174,7 @@ websocket_trigger: path(char), url(char), script_path(char), is_flow(bool), work
windmill_migrations: name(text), created_at(ts)
worker_group_job_stats: hour(bigint), worker_group(text), script_lang(char), workspace_id(char), job_count(int), total_duration_ms(bigint)
FK: (workspace_id) -> workspace(id)
worker_ping: worker(char), worker_instance(char), ping_at(ts), started_at(ts), ip(char), jobs_executed(int), custom_tags(text[]), worker_group(char), dedicated_worker(char), wm_version(char), current_job_id(uuid), current_job_workspace_id(char), vcpus(bigint), memory(bigint), occupancy_rate(float), memory_usage(bigint), wm_memory_usage(bigint), occupancy_rate_15s(float), occupancy_rate_5m(float), occupancy_rate_30m(float), job_isolation(text), dedicated_workers(text[]), native_mode(bool), uses_batch_http_pull(bool)
worker_ping: worker(char), worker_instance(char), ping_at(ts), started_at(ts), ip(char), jobs_executed(int), custom_tags(text[]), worker_group(char), dedicated_worker(char), wm_version(char), current_job_id(uuid), current_job_workspace_id(char), vcpus(bigint), memory(bigint), occupancy_rate(float), memory_usage(bigint), wm_memory_usage(bigint), occupancy_rate_15s(float), occupancy_rate_5m(float), occupancy_rate_30m(float), job_isolation(text), dedicated_workers(text[])
workspace: id(char), name(char), owner(char), deleted(bool), premium(bool), parent_workspace_id(char)
FK: (parent_workspace_id) -> workspace(id)
workspace_dependencies: id(bigint), name(char), content(text), language(script_lang), description(text), archived(bool), workspace_id(char), created_at(ts)

View File

@@ -1,6 +1,5 @@
use sqlx::postgres::Postgres;
use sqlx::Pool;
use uuid::Uuid;
use windmill_common::jobs::{JobPayload, RawCode};
use windmill_common::scripts::ScriptLang;
use windmill_test_utils::*;
@@ -1449,240 +1448,3 @@ export function main() { return { a, b }; }
);
}
}
// ============================================================================
// Codebase Mode Tests
// ============================================================================
/// Create a TAR archive in memory containing a single `main.js` file.
fn create_codebase_tar(main_js_content: &str) -> Vec<u8> {
let mut builder = tar::Builder::new(Vec::new());
let content = main_js_content.as_bytes();
let mut header = tar::Header::new_gnu();
header.set_path("main.js").unwrap();
header.set_size(content.len() as u64);
header.set_mode(0o644);
header.set_cksum();
builder.append(&header, content).unwrap();
builder.into_inner().unwrap()
}
/// Place a TAR codebase at the expected cache path for the given job ID and hash.
fn place_codebase_in_cache(job_id: &Uuid, tar_bytes: &[u8], is_esm: bool) {
let codebase_id = if is_esm {
format!("{}.esm.tar", job_id)
} else {
format!("{}.tar", job_id)
};
let bundle_path = format!("script_bundle/test-workspace/{}", codebase_id);
let cache_path = format!(
"{}/{}.tar",
*windmill_common::worker::ROOT_CACHE_NOMOUNT_DIR,
bundle_path,
);
let parent = std::path::Path::new(&cache_path).parent().unwrap();
std::fs::create_dir_all(parent).unwrap();
std::fs::write(&cache_path, tar_bytes).unwrap();
}
#[sqlx::test(fixtures("base"))]
async fn test_cjs_codebase_tar(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
let main_js = r#"
module.exports.main = function() {
return "cjs codebase ok";
};
"#;
let inner_content = r#"export function main() { return "cjs codebase ok"; }"#;
let job_id = Uuid::new_v4();
let tar_bytes = create_codebase_tar(main_js);
place_codebase_in_cache(&job_id, &tar_bytes, false);
let job = JobPayload::Code(RawCode {
hash: Some(-43), // PREVIEW_IS_TAR_CODEBASE_HASH
content: inner_content.to_string(),
path: None,
language: ScriptLang::Bun,
lock: None,
concurrency_settings: Default::default(),
debouncing_settings: Default::default(),
cache_ttl: None,
cache_ignore_s3_path: None,
dedicated_worker: None,
});
let result = RunJob::from(job)
.job_id(job_id)
.run_until_complete(&db, false, port)
.await
.json_result()
.unwrap();
assert_eq!(result, serde_json::json!("cjs codebase ok"));
Ok(())
}
#[sqlx::test(fixtures("base"))]
async fn test_esm_codebase_tar(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
let main_js = r#"
export function main() {
return "esm codebase ok";
}
"#;
let inner_content = r#"export function main() { return "esm codebase ok"; }"#;
let job_id = Uuid::new_v4();
let tar_bytes = create_codebase_tar(main_js);
place_codebase_in_cache(&job_id, &tar_bytes, true);
let job = JobPayload::Code(RawCode {
hash: Some(-45), // PREVIEW_IS_TAR_ESM_CODEBASE_HASH
content: inner_content.to_string(),
path: None,
language: ScriptLang::Bun,
lock: None,
concurrency_settings: Default::default(),
debouncing_settings: Default::default(),
cache_ttl: None,
cache_ignore_s3_path: None,
dedicated_worker: None,
});
let result = RunJob::from(job)
.job_id(job_id)
.run_until_complete(&db, false, port)
.await
.json_result()
.unwrap();
assert_eq!(result, serde_json::json!("esm codebase ok"));
Ok(())
}
#[sqlx::test(fixtures("base"))]
async fn test_cjs_codebase_tar_nsjail(db: Pool<Postgres>) -> anyhow::Result<()> {
if std::process::Command::new("nsjail")
.arg("--help")
.output()
.is_err()
{
eprintln!("nsjail not found, skipping test");
return Ok(());
}
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
let main_js = r#"
module.exports.main = function() {
return "cjs nsjail ok";
};
"#;
let inner_content = r#"export function main() { return "cjs nsjail ok"; }"#;
let job_id = Uuid::new_v4();
let tar_bytes = create_codebase_tar(main_js);
place_codebase_in_cache(&job_id, &tar_bytes, false);
let job = JobPayload::Code(RawCode {
hash: Some(-43),
content: inner_content.to_string(),
path: None,
language: ScriptLang::Bun,
lock: None,
concurrency_settings: Default::default(),
debouncing_settings: Default::default(),
cache_ttl: None,
cache_ignore_s3_path: None,
dedicated_worker: None,
});
use std::sync::atomic::Ordering;
windmill_worker::JOB_ISOLATION.store(
windmill_worker::JobIsolationLevel::NsjailSandboxing as u8,
Ordering::Relaxed,
);
let result = RunJob::from(job)
.job_id(job_id)
.run_until_complete(&db, false, port)
.await;
windmill_worker::JOB_ISOLATION.store(
windmill_worker::JobIsolationLevel::Undefined as u8,
Ordering::Relaxed,
);
let json = result.json_result().unwrap();
assert_eq!(json, serde_json::json!("cjs nsjail ok"));
Ok(())
}
#[sqlx::test(fixtures("base"))]
async fn test_esm_codebase_tar_nsjail(db: Pool<Postgres>) -> anyhow::Result<()> {
if std::process::Command::new("nsjail")
.arg("--help")
.output()
.is_err()
{
eprintln!("nsjail not found, skipping test");
return Ok(());
}
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
let main_js = r#"
export function main() {
return "esm nsjail ok";
}
"#;
let inner_content = r#"export function main() { return "esm nsjail ok"; }"#;
let job_id = Uuid::new_v4();
let tar_bytes = create_codebase_tar(main_js);
place_codebase_in_cache(&job_id, &tar_bytes, true);
let job = JobPayload::Code(RawCode {
hash: Some(-45),
content: inner_content.to_string(),
path: None,
language: ScriptLang::Bun,
lock: None,
concurrency_settings: Default::default(),
debouncing_settings: Default::default(),
cache_ttl: None,
cache_ignore_s3_path: None,
dedicated_worker: None,
});
use std::sync::atomic::Ordering;
windmill_worker::JOB_ISOLATION.store(
windmill_worker::JobIsolationLevel::NsjailSandboxing as u8,
Ordering::Relaxed,
);
let result = RunJob::from(job)
.job_id(job_id)
.run_until_complete(&db, false, port)
.await;
windmill_worker::JOB_ISOLATION.store(
windmill_worker::JobIsolationLevel::Undefined as u8,
Ordering::Relaxed,
);
let json = result.json_result().unwrap();
assert_eq!(json, serde_json::json!("esm nsjail ok"));
Ok(())
}

View File

@@ -241,7 +241,6 @@ fn spawn_workers(
rx,
tx2,
&base_internal_url,
None,
)
.await;
};

View File

@@ -19,10 +19,7 @@ use windmill_common::DB;
use axum::Router;
#[cfg(not(feature = "private"))]
pub fn global_service(
_job_completed_tx: windmill_worker::JobCompletedSender,
_batch_buffer: Option<()>,
) -> Router {
pub fn global_service(_job_completed_tx: windmill_worker::JobCompletedSender) -> Router {
Router::new()
}
@@ -34,7 +31,6 @@ pub fn workspaced_service(
Router,
Vec<tokio::task::JoinHandle<()>>,
Option<windmill_worker::JobCompletedSender>,
Option<()>,
) {
use windmill_common::worker::Connection;
use windmill_worker::JobCompletedSender;
@@ -44,7 +40,7 @@ pub fn workspaced_service(
let router = Router::new();
(router, vec![], Some(job_completed_tx), None)
(router, vec![], Some(job_completed_tx))
}
#[cfg(not(feature = "private"))]

View File

@@ -1,7 +1,7 @@
openapi: "3.0.3"
info:
version: 1.651.1
version: 1.650.0
title: Windmill API
contact:

View File

@@ -5416,12 +5416,12 @@ async fn add_batch_jobs(
if dedicated_worker && path.is_some() {
windmill_common::worker::dedicated_worker_tag(&w_id, &path.clone().unwrap())
} else {
language.as_worker_tag(false).to_string()
format!("{}", language.as_str())
}
} else if let Some(tag) = batch_info.tag {
tag
} else {
language.as_worker_tag(false).to_string()
format!("{}", language.as_str())
};
let mut tx = user_db.begin(&authed).await?;

View File

@@ -493,16 +493,12 @@ pub async fn run_server(
};
#[cfg(feature = "agent_worker_server")]
let (
agent_workers_router,
agent_workers_bg_processor,
agent_workers_job_completed_tx,
batch_buffer,
) = if server_mode {
windmill_api_agent_workers::workspaced_service(db.clone(), _base_internal_url.clone())
} else {
(Router::new(), vec![], None, None)
};
let (agent_workers_router, agent_workers_bg_processor, agent_workers_job_completed_tx) =
if server_mode {
windmill_api_agent_workers::workspaced_service(db.clone(), _base_internal_url.clone())
} else {
(Router::new(), vec![], None)
};
#[cfg(feature = "agent_worker_server")]
let agent_cache = Arc::new(AgentCache::new());
@@ -688,7 +684,6 @@ pub async fn run_server(
{
windmill_api_agent_workers::global_service(
agent_workers_job_completed_tx,
batch_buffer.clone(),
)
.layer(Extension(agent_cache.clone()))
} else {

View File

@@ -64,10 +64,9 @@ impl McpBackend for WindmillBackend {
auth: &ApiAuthed,
workspace_id: &str,
favorites_only: bool,
path_prefix: Option<&str>,
) -> BackendResult<Vec<ScriptInfo>> {
let scope_type = if favorites_only { "favorites" } else { "all" };
get_items::<ScriptInfo>(&self.user_db, auth, workspace_id, scope_type, "script", path_prefix)
get_items::<ScriptInfo>(&self.user_db, auth, workspace_id, scope_type, "script")
.await
.map_err(|e| ErrorData::internal_error(e.message, None))
}
@@ -77,10 +76,9 @@ impl McpBackend for WindmillBackend {
auth: &ApiAuthed,
workspace_id: &str,
favorites_only: bool,
path_prefix: Option<&str>,
) -> BackendResult<Vec<FlowInfo>> {
let scope_type = if favorites_only { "favorites" } else { "all" };
get_items::<FlowInfo>(&self.user_db, auth, workspace_id, scope_type, "flow", path_prefix)
get_items::<FlowInfo>(&self.user_db, auth, workspace_id, scope_type, "flow")
.await
.map_err(|e| ErrorData::internal_error(e.message, None))
}

View File

@@ -136,7 +136,6 @@ pub async fn get_items<T: for<'a> sqlx::FromRow<'a, sqlx::postgres::PgRow> + Sen
workspace_id: &str,
scope_type: &str,
item_type: &str,
path_prefix: Option<&str>,
) -> Result<Vec<T>, ErrorData> {
let mut sqlb = SqlBuilder::select_from(&format!("{} as o", item_type));
let fields = vec!["o.path", "o.summary", "o.description", "o.schema"];
@@ -154,11 +153,6 @@ pub async fn get_items<T: for<'a> sqlx::FromRow<'a, sqlx::postgres::PgRow> + Sen
sqlb.and_where("(o.no_main_func IS NOT TRUE OR o.no_main_func IS NULL)");
}
if let Some(prefix) = path_prefix {
let escaped = prefix.replace('\\', "\\\\").replace('%', "\\%").replace('_', "\\_");
sqlb.and_where("o.path LIKE ? ESCAPE '\\'".bind(&format!("{}%", escaped)));
}
sqlb.order_by(
if item_type == "flow" {
"o.edited_at"

View File

@@ -288,10 +288,6 @@ pub fn is_native_mode_from_env() -> bool {
/// Use this for hot-path checks (e.g. per-job dispatch) to avoid read-locking WORKER_CONFIG.
pub static NATIVE_MODE_RESOLVED: AtomicBool = AtomicBool::new(false);
/// Whether this worker uses HTTP batch pull (set at startup in main.rs).
/// Reported in worker_ping so the server knows which native workers to batch-pull for.
pub static USES_BATCH_HTTP_PULL: AtomicBool = AtomicBool::new(false);
pub static MIN_VERSION_IS_LATEST: AtomicBool = AtomicBool::new(false);
#[derive(Clone)]
pub struct HttpClient {
@@ -520,62 +516,6 @@ pub fn make_pull_query(tags: &[String]) -> String {
query
}
pub fn make_batch_pull_query(tags: &[String], limit: u32) -> String {
format_batch_pull_query(format!(
"SELECT id
FROM v2_job_queue
WHERE running = false AND tag IN ({}) AND scheduled_for <= now()
ORDER BY priority DESC NULLS LAST, scheduled_for
FOR UPDATE SKIP LOCKED
LIMIT {limit}",
tags.iter().map(|x| format!("'{x}'")).join(", ")
))
}
fn format_batch_pull_query(peek: String) -> String {
// Optimizations vs single-row format_pull_query:
// 1. ANY(ARRAY(SELECT ...)) instead of IN (SELECT ...) — forces PG to materialize IDs
// into an array, enabling Bitmap Index Scan instead of Hash Semi Join / Nested Loop
// 2. r CTE chains off q (not peek) — only updates runtime for actually-locked rows,
// avoids re-scanning peek
// 3. No separate j CTE — join v2_job directly in final SELECT off q's IDs
format!(
"WITH peek AS (
{}
), q AS NOT MATERIALIZED (
UPDATE v2_job_queue SET
running = true,
started_at = coalesce(started_at, now()),
suspend_until = null,
worker = $1
WHERE id = ANY(ARRAY(SELECT id FROM peek))
RETURNING
id, started_at, scheduled_for,
canceled_by, canceled_reason, worker, cache_ignore_s3_path, runnable_settings_handle
), r AS NOT MATERIALIZED (
UPDATE v2_job_runtime SET
ping = now()
WHERE id = ANY(ARRAY(SELECT id FROM q))
) SELECT j.id, j.workspace_id, j.parent_job, j.created_by, q.started_at, q.scheduled_for,
j.runnable_id, j.runnable_path, j.args, q.canceled_by,
q.canceled_reason, j.kind, j.trigger, j.trigger_kind, j.permissioned_as,
f.flow_status, j.script_lang,
j.same_worker, j.pre_run_error, j.visible_to_owner,
j.tag, j.concurrent_limit, j.concurrency_time_window_s, j.flow_innermost_root_job, j.root_job,
j.timeout, j.flow_step_id, j.cache_ttl, q.cache_ignore_s3_path, q.runnable_settings_handle, j.priority, j.raw_code, j.raw_lock, j.raw_flow,
j.script_entrypoint_override, j.preprocessed, COALESCE(pj.runnable_path, j.args->>'_FLOW_PATH') as parent_runnable_path,
COALESCE(p.email, j.permissioned_as_email) as permissioned_as_email, p.username as permissioned_as_username, p.is_admin as permissioned_as_is_admin,
p.is_operator as permissioned_as_is_operator, p.groups as permissioned_as_groups, p.folders as permissioned_as_folders, p.end_user_email as permissioned_as_end_user_email
FROM q
JOIN v2_job j ON q.id = j.id
LEFT JOIN v2_job_status f ON f.id = q.id
LEFT JOIN job_perms p ON p.job_id = q.id
LEFT JOIN v2_job pj ON j.parent_job = pj.id
",
peek
)
}
pub async fn store_pull_query(wc: &WorkerConfig) {
let mut queries = vec![];
for tags in wc.priority_tags_sorted.iter() {
@@ -1254,8 +1194,6 @@ pub struct Ping {
pub occupancy_rate_30m: Option<f32>,
pub job_isolation: Option<String>,
pub native_mode: Option<bool>,
#[serde(default)]
pub uses_batch_http_pull: Option<bool>,
pub ping_type: PingType,
}
pub async fn update_ping_http(
@@ -1280,7 +1218,6 @@ pub async fn update_ping_http(
insert_ping.occupancy_rate_5m,
insert_ping.occupancy_rate_30m,
insert_ping.native_mode.unwrap_or(false),
insert_ping.uses_batch_http_pull.unwrap_or(false),
db,
)
.await?
@@ -1308,7 +1245,6 @@ pub async fn update_ping_http(
insert_ping.memory,
insert_ping.job_isolation,
insert_ping.native_mode.unwrap_or(false),
insert_ping.uses_batch_http_pull.unwrap_or(false),
db,
)
.await?;
@@ -1441,12 +1377,11 @@ pub async fn insert_ping_query(
memory: Option<i64>,
job_isolation: Option<String>,
native_mode: bool,
uses_batch_http_pull: bool,
db: &DB,
) -> anyhow::Result<()> {
sqlx::query!(
"INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, dedicated_workers, wm_version, vcpus, memory, job_isolation, native_mode, uses_batch_http_pull) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) ON CONFLICT (worker)
DO UPDATE set ip = EXCLUDED.ip, custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group, dedicated_workers = EXCLUDED.dedicated_workers, native_mode = EXCLUDED.native_mode, uses_batch_http_pull = EXCLUDED.uses_batch_http_pull",
"INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, dedicated_workers, wm_version, vcpus, memory, job_isolation, native_mode) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) ON CONFLICT (worker)
DO UPDATE set ip = EXCLUDED.ip, custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group, dedicated_workers = EXCLUDED.dedicated_workers, native_mode = EXCLUDED.native_mode",
worker_instance,
worker_name,
ip,
@@ -1459,7 +1394,6 @@ pub async fn insert_ping_query(
memory,
job_isolation.as_deref(),
native_mode,
uses_batch_http_pull,
)
.execute(db)
.await?;
@@ -1551,13 +1485,12 @@ pub async fn update_worker_ping_main_loop_query(
occupancy_rate_5m: Option<f32>,
occupancy_rate_30m: Option<f32>,
native_mode: bool,
uses_batch_http_pull: bool,
db: &DB,
) -> anyhow::Result<()> {
timeout(Duration::from_secs(10), sqlx::query!(
"UPDATE worker_ping SET ping_at = now(), jobs_executed = $1, custom_tags = $2,
occupancy_rate = $3, memory_usage = $4, wm_memory_usage = $5, vcpus = COALESCE($7, vcpus),
memory = COALESCE($8, memory), occupancy_rate_15s = $9, occupancy_rate_5m = $10, occupancy_rate_30m = $11, native_mode = $12, uses_batch_http_pull = $13 WHERE worker = $6",
memory = COALESCE($8, memory), occupancy_rate_15s = $9, occupancy_rate_5m = $10, occupancy_rate_30m = $11, native_mode = $12 WHERE worker = $6",
jobs_executed,
tags,
occupancy_rate,
@@ -1570,7 +1503,6 @@ pub async fn update_worker_ping_main_loop_query(
occupancy_rate_5m,
occupancy_rate_30m,
native_mode,
uses_batch_http_pull,
)
.execute(db))
.await??;

View File

@@ -11,8 +11,6 @@ pub mod types;
pub use schema::convert_schema_to_schema_type;
pub use scope::{is_resource_allowed, parse_mcp_scopes, McpScopeConfig};
pub use transform::{
apply_key_transformation, extract_hub_version_id_from_hashed,
extract_path_prefix_from_hashed, parse_tool_prefix, reverse_transform, reverse_transform_key,
transform_hub_path, transform_path,
apply_key_transformation, reverse_transform, reverse_transform_key, transform_path,
};
pub use types::*;

View File

@@ -4,15 +4,9 @@
//! to make them compatible with MCP tool naming requirements.
use super::types::SchemaType;
use windmill_common::utils::calculate_hash;
/// Max tool name length. The MCP spec allows 64 chars, but some clients
/// (e.g. Cursor) prepend the server name to the tool name, so we use 40
/// to leave room for that prefix.
const MAX_PATH_LENGTH: usize = 40;
/// Length of the SHA256 hash suffix used for hashed names
const HASH_LEN: usize = 16;
/// MCP clients do not allow names longer than 60 characters
const MAX_PATH_LENGTH: usize = 60;
/// Transform the path for workspace scripts/flows
///
@@ -20,133 +14,19 @@ const HASH_LEN: usize = 16;
/// path with the type prefix. This is used when listing, because we can't
/// have names with slashes. Because we replace slashes with underscores,
/// we also need to escape underscores.
///
/// For short names (≤40 chars): `s-{escaped_path}` or `f-{escaped_path}`
/// For long names (>40 chars): `S-{escaped[:22]}{sha256[:16]}` or `F-{escaped[:22]}{sha256[:16]}`
///
/// The uppercase prefix signals that the name is hashed.
pub fn transform_path(path: &str, type_str: &str) -> String {
let escaped_path = path.replace('_', "__").replace('/', "_");
let prefix_char = &type_str[..1];
let short_name = format!("{}-{}", prefix_char, escaped_path);
if short_name.len() <= MAX_PATH_LENGTH {
return short_name;
// first letter of type_str is used as prefix, only one letter to avoid reaching 60 char name limit
let transformed_path = format!("{}-{}", &type_str[..1], escaped_path);
if transformed_path.len() > MAX_PATH_LENGTH {
let suffix = "_TRUNC";
return format!(
"{}{}",
&transformed_path[..MAX_PATH_LENGTH - suffix.len()],
suffix
);
}
let upper_prefix = prefix_char.to_uppercase();
// Layout: "{Upper}-" (2 chars) + prefix_body (22 chars) + hash (16 chars) = 40
let prefix_body_len = MAX_PATH_LENGTH - 2 - HASH_LEN;
let hash = calculate_hash(&short_name);
let hash_suffix = &hash[..HASH_LEN];
let truncated = truncate_to_char_boundary(&escaped_path, prefix_body_len);
format!("{}-{}{}", upper_prefix, truncated, hash_suffix)
}
/// Transform the path for hub scripts
///
/// For short names (≤40 chars): `hs-{id}-{summary}`
/// For long names (>40 chars): `Hs-{id}-{summary[:N]}{sha256[:16]}`
pub fn transform_hub_path(version_id: u64, summary: &str) -> String {
let escaped_summary = summary.replace(' ', "_");
let short_name = format!("hs-{}-{}", version_id, escaped_summary);
if short_name.len() <= MAX_PATH_LENGTH {
return short_name;
}
let hash = calculate_hash(&short_name);
let hash_suffix = &hash[..HASH_LEN];
// "Hs-{id}-" prefix, then fill remaining with summary + hash
let fixed_prefix = format!("Hs-{}-", version_id);
let available = MAX_PATH_LENGTH - fixed_prefix.len() - HASH_LEN;
let truncated_summary = truncate_to_char_boundary(&escaped_summary, available);
format!("{}{}{}", fixed_prefix, truncated_summary, hash_suffix)
}
/// Parse the prefix of any tool name (both short and hashed).
/// Returns `(type_str, is_hub, is_hashed)`.
/// Hashed names use an uppercase first character as the signal.
pub fn parse_tool_prefix(name: &str) -> Result<(&str, bool, bool), String> {
let is_hashed = name.chars().next().map(|c| c.is_ascii_uppercase()).unwrap_or(false);
let lower = name.to_ascii_lowercase();
let (type_str, is_hub) = if lower.starts_with("hs-") {
("script", true)
} else if lower.starts_with("s-") {
("script", false)
} else if lower.starts_with("f-") {
("flow", false)
} else {
return Err(format!("Invalid tool name prefix: {}", name));
};
Ok((type_str, is_hub, is_hashed))
}
/// Extract the hub version_id from a hashed hub script name like `Hs-{id}-...`
pub fn extract_hub_version_id_from_hashed(name: &str) -> Result<String, String> {
let rest = name
.strip_prefix("Hs-")
.ok_or_else(|| format!("Not a hashed hub name: {}", name))?;
let id = rest
.split('-')
.next()
.ok_or_else(|| format!("No version_id in hashed hub name: {}", name))?;
if id.is_empty() {
return Err(format!("Empty version_id in hashed hub name: {}", name));
}
Ok(id.to_string())
}
/// Extract a safe original-path prefix from a hashed tool name.
///
/// Given `S-u_admin_engineering__te<hash16>`, extracts the escaped prefix between
/// the type prefix (`S-`, `F-`, or `Hs-`) and the hash, un-escapes it, and
/// returns a prefix suitable for `WHERE path LIKE '{prefix}%'`.
///
/// Returns `None` if the name is too short or has an unrecognized prefix.
pub fn extract_path_prefix_from_hashed(name: &str) -> Option<String> {
let prefix_len = if name.starts_with("Hs-") {
3
} else if name.starts_with("S-") || name.starts_with("F-") {
2
} else {
return None;
};
if name.len() <= prefix_len + HASH_LEN {
return None;
}
let escaped_prefix = &name[prefix_len..name.len() - HASH_LEN];
if escaped_prefix.is_empty() {
return None;
}
// Strip trailing underscores — they may be half of a `__` pair split by truncation
let trimmed = escaped_prefix.trim_end_matches('_');
if trimmed.is_empty() {
return None;
}
Some(unescape_path(trimmed))
}
/// Un-escape a mangled path segment: `__` → `_`, standalone `_` → `/`.
fn unescape_path(s: &str) -> String {
const TEMP_PLACEHOLDER: &str = "@@UNDERSCORE@@";
s.replace("__", TEMP_PLACEHOLDER)
.replace('_', "/")
.replace(TEMP_PLACEHOLDER, "_")
}
/// Truncate a string to at most `max_len` bytes, ensuring we don't split a UTF-8 character.
fn truncate_to_char_boundary(s: &str, max_len: usize) -> &str {
if s.len() <= max_len {
return s;
}
let mut end = max_len;
while end > 0 && !s.is_char_boundary(end) {
end -= 1;
}
&s[..end]
transformed_path
}
/// Reverse the transformation of a path
@@ -158,22 +38,25 @@ fn truncate_to_char_boundary(s: &str, max_len: usize) -> &str {
/// This is used in call_tool to get the original path, and the type of the item.
///
/// Returns: (type, original_path, is_hub)
///
/// Note: This only works for non-hashed (short) names. Hashed names must be
/// resolved via `parse_tool_prefix` + path enumeration in the runner.
pub fn reverse_transform(transformed_path: &str) -> Result<(&str, String, bool), String> {
let (type_str, is_hub, is_hashed) = parse_tool_prefix(transformed_path)?;
let is_hub = transformed_path.starts_with("h");
let transformed_path = if is_hub {
transformed_path[1..].to_string()
} else {
transformed_path.to_string()
};
let type_str = if transformed_path.starts_with("s-") {
"script"
} else if transformed_path.starts_with("f-") {
"flow"
} else {
return Err(format!(
"Invalid prefix in transformed path: {}",
transformed_path
));
};
if is_hashed {
return Err(
"Hashed names cannot be reverse-transformed directly; use path enumeration instead"
.to_string(),
);
}
// Strip the prefix: "hs-" (3 chars) for hub, "s-"/"f-" (2 chars) for others
let prefix_len = if is_hub { 3 } else { 2 };
let mangled_path = &transformed_path[prefix_len..];
let mangled_path = &transformed_path[2..];
let original_path = if is_hub {
let parts = mangled_path.split("-").collect::<Vec<&str>>();
@@ -182,7 +65,11 @@ pub fn reverse_transform(transformed_path: &str) -> Result<(&str, String, bool),
}
parts[0].to_string()
} else {
unescape_path(mangled_path)
const TEMP_PLACEHOLDER: &str = "@@UNDERSCORE@@";
mangled_path
.replace("__", TEMP_PLACEHOLDER)
.replace('_', "/")
.replace(TEMP_PLACEHOLDER, "_")
};
Ok((type_str, original_path, is_hub))
@@ -210,13 +97,16 @@ pub fn reverse_transform_key(transformed_key: &str, schema_obj: &Option<SchemaTy
let schema_obj = match schema_obj {
Some(s) => s,
None => {
// No schema available, return the key as is (best guess)
return transformed_key.to_string();
}
};
for original_key_in_schema in schema_obj.properties.keys() {
// Apply the SAME forward transformation to the schema key
let potential_transformed_key = apply_key_transformation(original_key_in_schema);
// If it matches the key we received, we found the likely original
if potential_transformed_key == transformed_key {
return original_key_in_schema.clone();
}
@@ -230,7 +120,7 @@ mod tests {
use super::*;
#[test]
fn test_transform_path_short() {
fn test_transform_path() {
assert_eq!(
transform_path("u/admin/script", "script"),
"s-u_admin_script"
@@ -240,108 +130,7 @@ mod tests {
}
#[test]
fn test_transform_path_long_is_hashed() {
let long_path = "u/engineering/team/automation/very_long_script_name_that_exceeds_limit";
let result = transform_path(long_path, "script");
assert_eq!(result.len(), MAX_PATH_LENGTH);
assert!(result.starts_with("S-"));
let (_, _, is_hashed) = parse_tool_prefix(&result).unwrap();
assert!(is_hashed);
}
#[test]
fn test_transform_path_long_flow_is_hashed() {
let long_path = "f/engineering/team/automation/very_long_flow_name_that_exceeds_limit";
let result = transform_path(long_path, "flow");
assert_eq!(result.len(), MAX_PATH_LENGTH);
assert!(result.starts_with("F-"));
let (_, _, is_hashed) = parse_tool_prefix(&result).unwrap();
assert!(is_hashed);
}
#[test]
fn test_transform_path_hashing_is_deterministic() {
let path = "u/engineering/team/automation/very_long_script_name_that_exceeds_limit";
let a = transform_path(path, "script");
let b = transform_path(path, "script");
assert_eq!(a, b);
}
#[test]
fn test_transform_path_different_long_paths_differ() {
let a = transform_path(
"u/engineering/team/automation/very_long_script_name_that_exceeds_limit_a",
"script",
);
let b = transform_path(
"u/engineering/team/automation/very_long_script_name_that_exceeds_limit_b",
"script",
);
assert_ne!(a, b);
}
#[test]
fn test_transform_hub_path_short() {
let result = transform_hub_path(12345, "Send Slack Message");
assert_eq!(result, "hs-12345-Send_Slack_Message");
let (_, _, is_hashed) = parse_tool_prefix(&result).unwrap();
assert!(!is_hashed);
}
#[test]
fn test_transform_hub_path_long_is_hashed() {
let result = transform_hub_path(
12345,
"Send Slack Message To Channel With Very Long Description That Exceeds Limit",
);
assert_eq!(result.len(), MAX_PATH_LENGTH);
assert!(result.starts_with("Hs-12345-"));
let (_, _, is_hashed) = parse_tool_prefix(&result).unwrap();
assert!(is_hashed);
}
#[test]
fn test_extract_hub_version_id_from_hashed() {
let name = "Hs-12345-Send_Slack_Message_To_Ch9e8d7c6b5a4f3e2d";
let id = extract_hub_version_id_from_hashed(name).unwrap();
assert_eq!(id, "12345");
}
#[test]
fn test_parse_tool_prefix() {
let (t, hub, hashed) = parse_tool_prefix("S-something").unwrap();
assert_eq!(t, "script");
assert!(!hub);
assert!(hashed);
let (t, hub, hashed) = parse_tool_prefix("F-something").unwrap();
assert_eq!(t, "flow");
assert!(!hub);
assert!(hashed);
let (t, hub, hashed) = parse_tool_prefix("Hs-12345-something").unwrap();
assert_eq!(t, "script");
assert!(hub);
assert!(hashed);
let (t, hub, hashed) = parse_tool_prefix("s-u_admin_script").unwrap();
assert_eq!(t, "script");
assert!(!hub);
assert!(!hashed);
let (t, hub, hashed) = parse_tool_prefix("f-f_folder_flow").unwrap();
assert_eq!(t, "flow");
assert!(!hub);
assert!(!hashed);
let (t, hub, hashed) = parse_tool_prefix("hs-12345-summary").unwrap();
assert_eq!(t, "script");
assert!(hub);
assert!(!hashed);
}
#[test]
fn test_reverse_transform_short_names() {
fn test_reverse_transform() {
let (type_str, path, is_hub) = reverse_transform("s-u_admin_script").unwrap();
assert_eq!(type_str, "script");
assert_eq!(path, "u/admin/script");
@@ -353,70 +142,6 @@ mod tests {
assert!(!is_hub);
}
#[test]
fn test_extract_path_prefix_from_hashed() {
// Generate a real hashed name and verify prefix extraction
let long_path = "u/admin/engineering/team/automation/very_long_script";
let hashed = transform_path(long_path, "script");
let (_, _, is_hashed) = parse_tool_prefix(&hashed).unwrap();
assert!(is_hashed);
let prefix = extract_path_prefix_from_hashed(&hashed).unwrap();
// The original path should start with the extracted prefix
assert!(
long_path.starts_with(&prefix),
"path '{}' should start with prefix '{}'",
long_path,
prefix
);
}
#[test]
fn test_extract_path_prefix_underscore_in_path() {
let long_path = "u/admin/my_team/automation/very_long_script_name_here";
let hashed = transform_path(long_path, "script");
let prefix = extract_path_prefix_from_hashed(&hashed).unwrap();
assert!(
long_path.starts_with(&prefix),
"path '{}' should start with prefix '{}'",
long_path,
prefix
);
}
#[test]
fn test_extract_path_prefix_rejects_invalid_prefix() {
assert!(extract_path_prefix_from_hashed("x-something").is_none());
assert!(extract_path_prefix_from_hashed("").is_none());
assert!(extract_path_prefix_from_hashed("S-").is_none());
}
#[test]
fn test_extract_path_prefix_handles_hs_prefix() {
// Hs- is 3 chars, not 2 — ensure the prefix is stripped correctly
let hashed = transform_hub_path(12345, "a]very long hub script summary that exceeds the limit");
let (_, is_hub, is_hashed) = parse_tool_prefix(&hashed).unwrap();
assert!(is_hub);
assert!(is_hashed);
let prefix = extract_path_prefix_from_hashed(&hashed);
// Should not start with 's' (leftover from Hs- if sliced at index 2)
if let Some(ref p) = prefix {
assert!(
!p.starts_with('s'),
"prefix '{}' should not start with 's' from mis-sliced Hs- prefix",
p
);
}
}
#[test]
fn test_reverse_transform_rejects_hashed_names() {
assert!(reverse_transform("S-something").is_err());
assert!(reverse_transform("F-something").is_err());
assert!(reverse_transform("Hs-12345-something").is_err());
}
#[test]
fn test_apply_key_transformation() {
assert_eq!(apply_key_transformation("my key"), "my_key");

View File

@@ -92,10 +92,8 @@ pub struct ItemSchema {
/// Trait for objects that can be converted to MCP tools
pub trait ToolableItem {
/// Get the MCP-compatible tool name (path transformed with escaping/hashing)
fn get_transformed_path(&self) -> String;
/// Get the original full path of this item (for display in tool title)
fn get_full_path(&self) -> &str;
/// Get the path or identifier for this item (transformed for MCP compatibility)
fn get_path_or_id(&self) -> String;
/// Get the summary/title of this item
fn get_summary(&self) -> &str;
/// Get the description of this item

View File

@@ -14,9 +14,9 @@ pub mod client;
// Re-export common types at crate root for convenience
pub use common::{
convert_schema_to_schema_type, is_resource_allowed, parse_mcp_scopes, transform_hub_path,
transform_path, FlowInfo, HubResponse, HubScriptInfo, ItemSchema, McpScopeConfig, ResourceInfo,
ResourceType, SchemaType, ScriptInfo, ToolableItem, WorkspaceId,
convert_schema_to_schema_type, is_resource_allowed, parse_mcp_scopes, transform_path, FlowInfo,
HubResponse, HubScriptInfo, ItemSchema, McpScopeConfig, ResourceInfo, ResourceType, SchemaType,
ScriptInfo, ToolableItem, WorkspaceId,
};
// Re-export client types at crate root for backward compatibility

View File

@@ -55,22 +55,20 @@ pub trait McpBackend: Send + Sync + Clone + 'static {
// Listing Operations
// ─────────────────────────────────────────────────────────────────
/// List scripts, optionally filtered to favorites only and/or by path prefix
/// List scripts, optionally filtered to favorites only
async fn list_scripts(
&self,
auth: &Self::Auth,
workspace_id: &str,
favorites_only: bool,
path_prefix: Option<&str>,
) -> BackendResult<Vec<ScriptInfo>>;
/// List flows, optionally filtered to favorites only and/or by path prefix
/// List flows, optionally filtered to favorites only
async fn list_flows(
&self,
auth: &Self::Auth,
workspace_id: &str,
favorites_only: bool,
path_prefix: Option<&str>,
) -> BackendResult<Vec<FlowInfo>>;
/// List resource types in workspace

View File

@@ -5,10 +5,7 @@
use crate::common::schema::extract_resource_types_from_schema;
use crate::common::scope::parse_mcp_scopes;
use crate::common::transform::{
extract_hub_version_id_from_hashed, extract_path_prefix_from_hashed, parse_tool_prefix,
reverse_transform, reverse_transform_key,
};
use crate::common::transform::{reverse_transform, reverse_transform_key};
use crate::common::types::{ResourceInfo, ToolableItem, WorkspaceId};
use crate::server::backend::{McpAuth, McpBackend};
use crate::server::endpoints::endpoint_tool_to_mcp_tool;
@@ -84,13 +81,6 @@ impl<B: McpBackend> Runner<B> {
}
}
fn find_matching_path<T: ToolableItem>(candidates: Vec<T>, request_name: &str) -> Option<String> {
candidates
.into_iter()
.find(|item| item.get_transformed_path() == request_name)
.map(|item| item.get_full_path().to_string())
}
impl<B: McpBackend> ServerHandler for Runner<B> {
fn get_info(&self) -> ServerInfo {
ServerInfo {
@@ -130,9 +120,9 @@ impl<B: McpBackend> ServerHandler for Runner<B> {
// Fetch all items concurrently
let (scripts, flows, resource_types, hub_scripts) = tokio::try_join!(
self.backend
.list_scripts(&auth, &workspace_id, favorites_only, None),
.list_scripts(&auth, &workspace_id, favorites_only),
self.backend
.list_flows(&auth, &workspace_id, favorites_only, None),
.list_flows(&auth, &workspace_id, favorites_only),
self.backend.list_resource_types(&auth, &workspace_id),
async {
if let Some(ref apps) = scope_config.hub_apps {
@@ -241,6 +231,17 @@ impl<B: McpBackend> ServerHandler for Runner<B> {
let scope_config =
parse_mcp_scopes(scopes).map_err(|e| ErrorData::internal_error(e, None))?;
// Handle truncated tool names
if request.name.ends_with("_TRUNC") {
return Ok(CallToolResult::error(vec![rmcp::model::Annotated::new(
rmcp::model::RawContent::Text(rmcp::model::RawTextContent {
text: "Tool path is too long. Consider shortening it to make it compatible with MCP.".to_string(),
meta: None,
}),
None,
)]));
}
let args = request.arguments.map(Value::Object).unwrap_or(Value::Null);
// Check if this is an endpoint tool
@@ -273,58 +274,10 @@ impl<B: McpBackend> ServerHandler for Runner<B> {
}
}
// Resolve the tool name to (type, path, is_hub)
let (type_str, is_hub, is_hashed) =
parse_tool_prefix(&request.name).map_err(|e| {
ErrorData::internal_error(format!("Failed to parse tool name: {}", e), None)
})?;
let (tool_type, path, is_hub) = if !is_hashed {
reverse_transform(&request.name).map_err(|e| {
ErrorData::internal_error(format!("Failed to parse tool name: {}", e), None)
})?
} else if is_hub {
let version_id =
extract_hub_version_id_from_hashed(&request.name).map_err(|e| {
ErrorData::internal_error(
format!("Failed to extract hub version_id: {}", e),
None,
)
})?;
(type_str, version_id, true)
} else {
let path_prefix = extract_path_prefix_from_hashed(&request.name);
let favorites_only = scope_config.favorites;
let matched_path = if type_str == "script" {
find_matching_path(
self.backend
.list_scripts(&auth, &workspace_id, favorites_only, path_prefix.as_deref())
.await
.map_err(|e| ErrorData::internal_error(e.message, None))?,
&request.name,
)
} else {
find_matching_path(
self.backend
.list_flows(&auth, &workspace_id, favorites_only, path_prefix.as_deref())
.await
.map_err(|e| ErrorData::internal_error(e.message, None))?,
&request.name,
)
};
let matched_path = matched_path.ok_or_else(|| {
ErrorData::internal_error(
format!(
"No {} found matching hashed tool name '{}'",
type_str, request.name
),
None,
)
})?;
(type_str, matched_path, false)
};
// Not an endpoint tool - parse as script/flow
let (tool_type, path, is_hub) = reverse_transform(&request.name).map_err(|e| {
ErrorData::internal_error(format!("Failed to parse tool name: {}", e), None)
})?;
// Validate script/flow scope
if !is_hub && scope_config.granular {

View File

@@ -9,7 +9,7 @@ use std::collections::HashMap;
use std::sync::Arc;
use crate::common::schema::{convert_schema_to_schema_type, make_schema_compatible};
use crate::common::transform::{transform_hub_path, transform_path};
use crate::common::transform::transform_path;
use crate::common::types::{
FlowInfo, HubScriptInfo, ResourceInfo, ResourceType, SchemaType, ScriptInfo, ToolableItem,
};
@@ -17,14 +17,10 @@ use crate::server::backend::McpBackend;
/// Implementation of ToolableItem for ScriptInfo
impl ToolableItem for ScriptInfo {
fn get_transformed_path(&self) -> String {
fn get_path_or_id(&self) -> String {
transform_path(&self.path, "script")
}
fn get_full_path(&self) -> &str {
&self.path
}
fn get_summary(&self) -> &str {
self.summary.as_deref().unwrap_or("No summary")
}
@@ -52,14 +48,10 @@ impl ToolableItem for ScriptInfo {
/// Implementation of ToolableItem for FlowInfo
impl ToolableItem for FlowInfo {
fn get_transformed_path(&self) -> String {
fn get_path_or_id(&self) -> String {
transform_path(&self.path, "flow")
}
fn get_full_path(&self) -> &str {
&self.path
}
fn get_summary(&self) -> &str {
self.summary.as_deref().unwrap_or("No summary")
}
@@ -87,13 +79,10 @@ impl ToolableItem for FlowInfo {
/// Implementation of ToolableItem for HubScriptInfo
impl ToolableItem for HubScriptInfo {
fn get_transformed_path(&self) -> String {
fn get_path_or_id(&self) -> String {
let id = self.version_id;
let summary = self.summary.as_deref().unwrap_or("No summary");
transform_hub_path(self.version_id, summary)
}
fn get_full_path(&self) -> &str {
self.summary.as_deref().unwrap_or("No summary")
format!("hs-{}-{}", id, summary.replace(" ", "_"))
}
fn get_summary(&self) -> &str {
@@ -135,7 +124,7 @@ pub fn create_tool_from_item<T: ToolableItem, B: McpBackend>(
resources_types: &[ResourceType],
) -> Tool {
let is_hub = item.is_hub();
let path = item.get_transformed_path();
let path = item.get_path_or_id();
let item_type = item.item_type();
let description = format!(
"This is a {} named `{}` with the following description: `{}`.{}",
@@ -181,24 +170,15 @@ pub fn create_tool_from_item<T: ToolableItem, B: McpBackend>(
}
};
let title = {
let summary = item.get_summary();
if summary == "No summary" {
item.get_full_path().to_string()
} else {
summary.to_string()
}
};
Tool {
name: Cow::Owned(path),
description: Some(Cow::Owned(description)),
input_schema: Arc::new(input_schema_map),
title: Some(title.clone()),
title: Some(item.get_summary().to_string()),
output_schema: None,
icons: None,
annotations: Some(ToolAnnotations {
title: Some(title),
title: Some(item.get_summary().to_string()),
read_only_hint: Some(false), // Can modify environment
destructive_hint: Some(true), // Can potentially be destructive
idempotent_hint: Some(false), // Are not guaranteed to be idempotent

View File

@@ -3434,38 +3434,6 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<'c>(
Ok(job_and_suspended)
}
/// Batch-pull up to `limit` jobs in a single query, marking them all as running.
/// The caller controls which tags are queried, so flow/dependency jobs are never
/// pulled (they use distinct tags like "flow" / "dependency").
pub async fn batch_pull(
db: &Pool<Postgres>,
worker_name: &str,
tags: &[String],
limit: u32,
) -> windmill_common::error::Result<Vec<PulledJob>> {
use windmill_common::worker::make_batch_pull_query;
if limit == 0 || tags.is_empty() {
return Ok(vec![]);
}
let query = make_batch_pull_query(tags, limit);
let jobs: Vec<PulledJob> = timeout(
Duration::from_secs(15),
sqlx::query_as::<_, PulledJob>(&query)
.bind(worker_name)
.fetch_all(db),
)
.await
.map_err(|_| {
windmill_common::error::Error::internal_err(
"batch_pull query timed out after 15s".to_string(),
)
})??;
Ok(jobs)
}
pub async fn custom_concurrency_key(
db: &Pool<Postgres>,
job_id: &Uuid,
@@ -5405,7 +5373,15 @@ async fn push_inner<'c, 'd>(
language
.as_ref()
.map(|x| {
let tag_lang = x.as_worker_tag(job_kind == JobKind::Dependencies);
let tag_lang = if x == &ScriptLang::Bunnative {
if job_kind == JobKind::Dependencies {
ScriptLang::Bun.as_str()
} else {
ScriptLang::Nativets.as_str()
}
} else {
x.as_str()
};
if per_workspace {
format!("{}-{}", tag_lang, workspace_id)
} else {

View File

@@ -151,7 +151,6 @@ pub struct RunJob {
pub args: serde_json::Map<String, serde_json::Value>,
pub scheduled_for_o: Option<chrono::DateTime<chrono::Utc>>,
pub email: String,
pub job_id: Option<Uuid>,
}
impl From<JobPayload> for RunJob {
@@ -161,7 +160,6 @@ impl From<JobPayload> for RunJob {
args: Default::default(),
scheduled_for_o: None,
email: "test@windmill.dev".to_string(),
job_id: None,
}
}
}
@@ -185,13 +183,8 @@ impl RunJob {
self
}
pub fn job_id(mut self, id: Uuid) -> Self {
self.job_id = Some(id);
self
}
pub async fn push(self, db: &Pool<Postgres>) -> Uuid {
let RunJob { payload, args, scheduled_for_o, email, job_id } = self;
let RunJob { payload, args, scheduled_for_o, email } = self;
let mut hm_args = std::collections::HashMap::new();
for (k, v) in args {
hm_args.insert(k, windmill_common::worker::to_raw_value(&v));
@@ -213,7 +206,7 @@ impl RunJob {
/* parent_job */ None,
/* root job */ None,
/* flow_innermost_root_job */ None,
/* job_id */ job_id,
/* job_id */ None,
/* is_flow_step */ false,
/* same_worker */ false,
None,
@@ -237,7 +230,7 @@ impl RunJob {
/// Push the job as a specific user (for testing permissions)
pub async fn push_as(self, db: &Pool<Postgres>, username: &str, email: &str) -> Uuid {
let RunJob { payload, args, scheduled_for_o, job_id, .. } = self;
let RunJob { payload, args, scheduled_for_o, .. } = self;
let mut hm_args = std::collections::HashMap::new();
for (k, v) in args {
hm_args.insert(k, windmill_common::worker::to_raw_value(&v));
@@ -259,7 +252,7 @@ impl RunJob {
/* parent_job */ None,
/* root job */ None,
/* flow_innermost_root_job */ None,
/* job_id */ job_id,
/* job_id */ None,
/* is_flow_step */ false,
/* same_worker */ false,
None,
@@ -421,7 +414,6 @@ pub fn spawn_test_worker(
rx,
tx2,
&base_internal_url,
None,
)
.await
};

View File

@@ -88,20 +88,6 @@ impl ScriptLang {
}
}
/// Returns the worker tag for this language.
/// Bunnative scripts run on nativets workers (not bun), except dependency jobs which use bun.
pub fn as_worker_tag(&self, is_dependency_job: bool) -> &'static str {
if *self == ScriptLang::Bunnative {
if is_dependency_job {
ScriptLang::Bun.as_str()
} else {
ScriptLang::Nativets.as_str()
}
} else {
self.as_str()
}
}
pub fn as_dependencies_filename(&self) -> Option<String> {
use ScriptLang::*;
Some(
@@ -119,15 +105,15 @@ impl ScriptLang {
pub fn is_native(&self) -> bool {
matches!(
self,
ScriptLang::Bunnative
| ScriptLang::Nativets
| ScriptLang::Postgresql
| ScriptLang::Mysql
| ScriptLang::Graphql
| ScriptLang::Snowflake
| ScriptLang::Mssql
| ScriptLang::Bigquery
| ScriptLang::OracleDB
ScriptLang::Bunnative |
ScriptLang::Nativets |
ScriptLang::Postgresql |
ScriptLang::Mysql |
ScriptLang::Graphql |
ScriptLang::Snowflake |
ScriptLang::Mssql |
ScriptLang::Bigquery |
ScriptLang::OracleDB
)
}

View File

@@ -1036,12 +1036,6 @@ pub async fn handle_bun_job(
let _ = write_file(job_dir, "main.ts", inner_content)?;
} else if !annotation.native && codebase.is_none() {
let _ = write_file(job_dir, "package.json", r#"{ "type": "module" }"#)?;
} else if codebase.is_some() {
// Write a valid fallback package.json for codebase mode. Without this,
// nsjail creates an empty 0-byte file (from the mandatory: false mount)
// which Node.js fails to parse as JSON (ERR_INVALID_PACKAGE_CONFIG).
// If the codebase TAR includes a package.json, it will overwrite this.
let _ = write_file(job_dir, "package.json", "{}")?;
};
let common_bun_proc_envs: HashMap<String, String> =

View File

@@ -563,7 +563,6 @@ pub async fn update_worker_ping_for_failed_init_script(
wm_memory_usage: None,
job_isolation: None,
native_mode: None,
uses_batch_http_pull: None,
ping_type: PingType::InitScript,
},
)

View File

@@ -1368,7 +1368,6 @@ pub async fn run_worker(
mut killpill_rx: tokio::sync::broadcast::Receiver<()>,
killpill_tx: KillpillSender,
base_internal_url: &str,
batch_pull_client: Option<&HttpClient>,
) {
#[cfg(not(feature = "enterprise"))]
if is_sandboxing_enabled() {
@@ -2069,149 +2068,135 @@ pub async fn run_worker(
continue;
}
} else {
// If batch_pull_client is set (native worker with co-located server),
// use HTTP pull from batch buffer. Otherwise use direct SQL pull.
if let Some(bpc) = batch_pull_client {
crate::agent_workers::pull_job(bpc, None, None)
.await
.map_err(|e| error::Error::InternalErr(e.to_string()))
.map(|x| x.map(|y| NextJob::Http(y)))
} else {
match &conn {
Connection::Sql(db) => {
let pull_time = Instant::now();
let likelihood_of_suspend = last_30jobs_suspended as f64 / 30.0;
match &conn {
Connection::Sql(db) => {
let pull_time = Instant::now();
let likelihood_of_suspend = last_30jobs_suspended as f64 / 30.0;
let suspend_first = suspend_first_success
|| rand::random::<f64>() < likelihood_of_suspend
|| last_suspend_first.elapsed().as_secs_f64() > 5.0;
let suspend_first = suspend_first_success
|| rand::random::<f64>() < likelihood_of_suspend
|| last_suspend_first.elapsed().as_secs_f64() > 5.0;
if suspend_first {
last_suspend_first = Instant::now();
}
let mut job = match timeout(
Duration::from_secs(30),
pull(
&db,
suspend_first,
&worker_name,
None,
#[cfg(feature = "benchmark")]
&mut bench,
)
.warn_after_seconds(2),
if suspend_first {
last_suspend_first = Instant::now();
}
let mut job = match timeout(
Duration::from_secs(30),
pull(
&db,
suspend_first,
&worker_name,
None,
#[cfg(feature = "benchmark")]
&mut bench,
)
.warn_after_seconds(2),
)
.await
{
Ok(job) => job,
Err(e) => {
tracing::error!(worker = %worker_name, hostname = %hostname, "pull timed out after 20s, sleeping for 30s: {e:?}");
tokio::time::sleep(Duration::from_secs(30)).await;
continue;
}
};
// Preprocess pulled job result
if let Ok(ref mut pulled_job_res) = job {
if let Err(e) = timeout(
// Will fail if longer than 10 seconds
core::time::Duration::from_secs(10),
pulled_job_res.maybe_apply_debouncing(db),
)
.warn_after_seconds(2)
.await
// Flatten result
.map_err(error::Error::from)
.and_then(|r| r)
{
Ok(job) => job,
Err(e) => {
tracing::error!(worker = %worker_name, hostname = %hostname, "pull timed out after 20s, sleeping for 30s: {e:?}");
tokio::time::sleep(Duration::from_secs(30)).await;
continue;
}
};
// Preprocess pulled job result
if let Ok(ref mut pulled_job_res) = job {
if let Err(e) = timeout(
// Will fail if longer than 10 seconds
core::time::Duration::from_secs(10),
pulled_job_res.maybe_apply_debouncing(db),
)
.warn_after_seconds(2)
.await
// Flatten result
.map_err(error::Error::from)
.and_then(|r| r)
{
pulled_job_res.error_while_preprocessing = Some(e.to_string());
}
pulled_job_res.error_while_preprocessing = Some(e.to_string());
}
}
add_time!(bench, "job pulled from DB");
let duration_pull_s = pull_time.elapsed().as_secs_f64();
let err_pull = job.is_ok();
// let empty = job.as_ref().is_ok_and(|x| x.is_none());
add_time!(bench, "job pulled from DB");
let duration_pull_s = pull_time.elapsed().as_secs_f64();
let err_pull = job.is_ok();
// let empty = job.as_ref().is_ok_and(|x| x.is_none());
if duration_pull_s > 0.5 {
let empty = job.as_ref().is_ok_and(|x| x.job.is_none());
tracing::warn!(worker = %worker_name, hostname = %hostname, "pull took more than 0.5s ({duration_pull_s}), this is a sign that the database is VERY undersized for this load. empty: {empty}, err: {err_pull}");
#[cfg(feature = "prometheus")]
if empty {
if let Some(wp) = worker_pull_over_500_counter_empty.as_ref() {
wp.inc();
}
} else if let Some(wp) = worker_pull_over_500_counter.as_ref() {
if duration_pull_s > 0.5 {
let empty = job.as_ref().is_ok_and(|x| x.job.is_none());
tracing::warn!(worker = %worker_name, hostname = %hostname, "pull took more than 0.5s ({duration_pull_s}), this is a sign that the database is VERY undersized for this load. empty: {empty}, err: {err_pull}");
#[cfg(feature = "prometheus")]
if empty {
if let Some(wp) = worker_pull_over_500_counter_empty.as_ref() {
wp.inc();
}
} else if duration_pull_s > 0.1 {
let empty = job.as_ref().is_ok_and(|x| x.job.is_none());
tracing::warn!(worker = %worker_name, hostname = %hostname, "pull took more than 0.1s ({duration_pull_s}) this is a sign that the database is undersized for this load. empty: {empty}, err: {err_pull}");
#[cfg(feature = "prometheus")]
if empty {
if let Some(wp) = worker_pull_over_100_counter_empty.as_ref() {
wp.inc();
}
} else if let Some(wp) = worker_pull_over_100_counter.as_ref() {
} else if let Some(wp) = worker_pull_over_500_counter.as_ref() {
wp.inc();
}
} else if duration_pull_s > 0.1 {
let empty = job.as_ref().is_ok_and(|x| x.job.is_none());
tracing::warn!(worker = %worker_name, hostname = %hostname, "pull took more than 0.1s ({duration_pull_s}) this is a sign that the database is undersized for this load. empty: {empty}, err: {err_pull}");
#[cfg(feature = "prometheus")]
if empty {
if let Some(wp) = worker_pull_over_100_counter_empty.as_ref() {
wp.inc();
}
} else if let Some(wp) = worker_pull_over_100_counter.as_ref() {
wp.inc();
}
}
if let Ok(j) = job.as_ref() {
let suspend_success = j.suspended;
if suspend_first {
if last_30jobs_suspended < 30 {
last_30jobs_suspended += 1;
}
} else {
last_30jobs_suspended -= 1;
if let Ok(j) = job.as_ref() {
let suspend_success = j.suspended;
if suspend_first {
if last_30jobs_suspended < 30 {
last_30jobs_suspended += 1;
}
suspend_first_success = suspend_first && suspend_success;
#[cfg(feature = "prometheus")]
if j.job.is_some() {
if let Some(wp) = worker_pull_duration_counter.as_ref() {
wp.inc_by(duration_pull_s);
}
if let Some(wp) = worker_pull_duration.as_ref() {
wp.observe(duration_pull_s);
}
} else {
if let Some(wp) = worker_pull_duration_counter_empty.as_ref() {
wp.inc_by(duration_pull_s);
}
if let Some(wp) = worker_pull_duration_empty.as_ref() {
wp.observe(duration_pull_s);
}
} else {
last_30jobs_suspended -= 1;
}
suspend_first_success = suspend_first && suspend_success;
#[cfg(feature = "prometheus")]
if j.job.is_some() {
if let Some(wp) = worker_pull_duration_counter.as_ref() {
wp.inc_by(duration_pull_s);
}
if let Some(wp) = worker_pull_duration.as_ref() {
wp.observe(duration_pull_s);
}
} else {
if let Some(wp) = worker_pull_duration_counter_empty.as_ref() {
wp.inc_by(duration_pull_s);
}
if let Some(wp) = worker_pull_duration_empty.as_ref() {
wp.observe(duration_pull_s);
}
}
match job {
Ok(pulled_job_result) => match pulled_job_result.to_pulled_job() {
Ok(j) => {
Ok(j.map(|job| NextJob::Sql { flow_runners: None, job }))
}
Err(PulledJobResultToJobErr::MissingConcurrencyKey(jc))
| Err(PulledJobResultToJobErr::ErrorWhilePreprocessing(jc)) => {
if let Err(err) = job_completed_tx.send_job(jc, true).await
{
tracing::error!(
}
match job {
Ok(pulled_job_result) => match pulled_job_result.to_pulled_job() {
Ok(j) => Ok(j.map(|job| NextJob::Sql { flow_runners: None, job })),
Err(PulledJobResultToJobErr::MissingConcurrencyKey(jc))
| Err(PulledJobResultToJobErr::ErrorWhilePreprocessing(jc)) => {
if let Err(err) = job_completed_tx.send_job(jc, true).await {
tracing::error!(
"An error occurred while sending job completed: {:#?}",
err
)
}
Ok(None)
}
},
Err(err) => Err(err),
}
}
Connection::Http(client) => {
crate::agent_workers::pull_job(&client, None, None)
.await
.map_err(|e| error::Error::InternalErr(e.to_string()))
.map(|x| x.map(|y| NextJob::Http(y)))
Ok(None)
}
},
Err(err) => Err(err),
}
}
Connection::Http(client) => crate::agent_workers::pull_job(&client, None, None)
.await
.map_err(|e| error::Error::InternalErr(e.to_string()))
.map(|x| x.map(|y| NextJob::Http(y))),
}
}
};
@@ -4786,7 +4771,6 @@ mount {{
// Clean up absolute-path symlinks created by setup_volume_mount_paths
if !is_sandboxing_enabled() {
#[allow(unused_variables)] // state is only used on unix
for state in &volume_setup.states {
#[cfg(unix)]
if state.mount.target.starts_with('/') {

View File

@@ -8,7 +8,7 @@ use windmill_common::{
get_memory, get_vcpus, get_windmill_memory_usage, get_worker_memory_usage,
insert_ping_query, update_job_ping_query, update_worker_ping_from_job_query,
update_worker_ping_main_loop_query, Connection, Ping, PingType, NATIVE_MODE_RESOLVED,
USES_BATCH_HTTP_PULL, WORKER_CONFIG, WORKER_GROUP,
WORKER_CONFIG, WORKER_GROUP,
},
KillpillSender, DB,
};
@@ -31,7 +31,6 @@ pub(crate) async fn update_worker_ping_full(
let tags = wc.worker_tags.clone();
let native_mode = wc.native_mode;
drop(wc);
let uses_batch_http_pull = USES_BATCH_HTTP_PULL.load(std::sync::atomic::Ordering::Relaxed);
let memory_usage = get_worker_memory_usage();
let wm_memory_usage = get_windmill_memory_usage();
@@ -65,7 +64,6 @@ pub(crate) async fn update_worker_ping_full(
occupancy_rate_5m,
occupancy_rate_30m,
native_mode,
uses_batch_http_pull,
)
})
.retry(
@@ -112,7 +110,6 @@ async fn update_worker_ping_full_inner(
occupancy_rate_5m: Option<f32>,
occupancy_rate_30m: Option<f32>,
native_mode: bool,
uses_batch_http_pull: bool,
) -> anyhow::Result<()> {
match conn {
Connection::Sql(db) => {
@@ -129,7 +126,6 @@ async fn update_worker_ping_full_inner(
occupancy_rate_5m,
occupancy_rate_30m,
native_mode,
uses_batch_http_pull,
db,
)
.await?;
@@ -159,7 +155,6 @@ async fn update_worker_ping_full_inner(
wm_memory_usage: get_windmill_memory_usage(),
job_isolation: None,
native_mode: Some(native_mode),
uses_batch_http_pull: Some(uses_batch_http_pull),
ping_type: PingType::MainLoop,
},
)
@@ -191,7 +186,6 @@ pub async fn insert_ping(
wc.native_mode,
)
};
let uses_batch_http_pull = USES_BATCH_HTTP_PULL.load(std::sync::atomic::Ordering::Relaxed);
let vcpus = get_vcpus();
let memory = get_memory();
@@ -219,7 +213,6 @@ pub async fn insert_ping(
memory,
job_isolation,
native_mode,
uses_batch_http_pull,
db,
)
.await?;
@@ -249,7 +242,6 @@ pub async fn insert_ping(
wm_memory_usage: get_windmill_memory_usage(),
job_isolation,
native_mode: Some(native_mode),
uses_batch_http_pull: Some(uses_batch_http_pull),
ping_type: PingType::Initial,
},
)
@@ -326,9 +318,6 @@ pub async fn update_worker_ping_from_job(
native_mode: Some(
NATIVE_MODE_RESOLVED.load(std::sync::atomic::Ordering::Relaxed),
),
uses_batch_http_pull: Some(
USES_BATCH_HTTP_PULL.load(std::sync::atomic::Ordering::Relaxed),
),
},
)
.await?;

View File

@@ -47,7 +47,6 @@ export async function main({
kind,
jobs,
noVerify,
skipDeploy,
}: {
host: string;
email?: string;
@@ -57,7 +56,6 @@ export async function main({
kind: string;
jobs: number;
noVerify?: boolean;
skipDeploy?: boolean;
}) {
windmill.setClient("", host);
@@ -148,8 +146,7 @@ export async function main({
}
if (
!skipDeploy &&
["deno", "python", "go", "bash", "dedicated", "bun", "nativets", "nativets_sleep", "dedicated_nativets"].includes(
["deno", "python", "go", "bash", "dedicated", "bun", "nativets", "dedicated_nativets"].includes(
kind
)
) {
@@ -168,7 +165,7 @@ export async function main({
kind: "noop",
});
} else if (
["deno", "python", "go", "bash", "dedicated", "bun", "nativets", "nativets_sleep", "dedicated_nativets"].includes(
["deno", "python", "go", "bash", "dedicated", "bun", "nativets", "dedicated_nativets"].includes(
kind
)
) {
@@ -339,7 +336,6 @@ export async function main({
!noVerify &&
kind !== "noop" &&
kind !== "nativets" &&
kind !== "nativets_sleep" &&
kind !== "dedicated_nativets" &&
!kind.startsWith("flow:") &&
!kind.startsWith("script:")
@@ -402,9 +398,6 @@ if (import.meta.main) {
.option("--no-verify", "Do not verify the output of the jobs.", {
default: false,
})
.option("--skip-deploy", "Skip script deployment (use already deployed script).", {
default: false,
})
.action(main)
.command(
"upgrade",

View File

@@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts";
import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts";
import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts";
export const VERSION = "v1.651.1";
export const VERSION = "v1.650.0";
export async function login(email: string, password: string): Promise<string> {
return await windmill.UserService.login({
@@ -95,10 +95,6 @@ export async function createBenchScript(
scriptContent =
'//native\nexport async function main(){ return (await fetch(BASE_URL + "/api/version")).text() }';
language = "bunnative";
} else if (scriptPattern === "nativets_sleep") {
scriptContent =
'//native\nexport async function main(){ const ms = 300 + Math.floor(Math.random() * 400); await new Promise(r => setTimeout(r, ms)); return { slept: ms }; }';
language = "bunnative";
} else if (scriptPattern === "dedicated_nativets") {
scriptContent = "//native\nexport function main(){ return 42; }";
language = "bunnative";

View File

@@ -1,165 +0,0 @@
/**
* Model: Batch Pull vs Direct SQL throughput
*
* Calibrated from real benchmarks (3 native workers = 24 subworkers, local PG):
* - nativets (fast): batch 291 j/s, SQL 253 j/s at N=24; batch 108, SQL 88 at N=8
* - nativets_sleep: both ~43.8 j/s at N=24 (bottlenecked by 500ms avg exec time)
*
* Per-worker job time model:
* T_pw = T_base + T_exec + T_contention(N)
* throughput = N / T_pw
*
* Batch: T_contention grows linearly with N (HTTP server load)
* T_pw_batch(N) = BASE_BATCH + T_exec + SCALE_BATCH × N
*
* SQL: T_contention grows quadratically with N (SKIP LOCKED scanning past locked rows)
* T_pw_sql(N) = BASE_SQL + T_exec + SCALE_SQL ×
*
* Parameters fitted from 2 data points each (N=8, N=24):
* Batch: BASE=69.9ms, SCALE=0.525ms/worker
* SQL: BASE=90.4ms, SCALE=0.0078ms/worker²
* (SQL quadratic overtakes batch linear around N~40)
*/
// --- Model parameters (fitted from benchmarks) ---
// Batch: per-worker time = BASE + SCALE_LINEAR * N + T_exec
const BASE_BATCH = 69.9; // ms — base overhead (worker loop, HTTP roundtrip, job completion writes)
const SCALE_BATCH = 0.525; // ms per subworker — linear growth from server load
// SQL: per-worker time = BASE + SCALE_QUAD * N² + T_exec
const BASE_SQL = 90.4; // ms — base overhead (worker loop, poll interval wait, job completion writes)
const SCALE_SQL = 0.0078; // ms per subworker² — quadratic growth from SKIP LOCKED contention
// --- Throughput functions ---
function throughputBatch(subworkers: number, execMs: number): number {
const tPerWorker = BASE_BATCH + execMs + SCALE_BATCH * subworkers;
return (subworkers / tPerWorker) * 1000; // jobs/s
}
function throughputSql(subworkers: number, execMs: number): number {
const tPerWorker = BASE_SQL + execMs + SCALE_SQL * subworkers * subworkers;
return (subworkers / tPerWorker) * 1000; // jobs/s
}
function pct(batch: number, sql: number): string {
const diff = ((batch - sql) / sql) * 100;
return `${diff >= 0 ? "+" : ""}${diff.toFixed(0)}%`;
}
// --- Validation against real data ---
console.log("=== Model Validation (vs real benchmarks) ===\n");
console.log(
" Setup | Model Batch | Real Batch | Model SQL | Real SQL",
);
console.log(
" ---------------------|-------------|------------|-----------|--------",
);
const cases = [
{ n: 8, exec: 0, label: "1W nativets", realBatch: 108, realSql: 88 },
{ n: 24, exec: 0, label: "3W nativets", realBatch: 291, realSql: 253 },
{
n: 24,
exec: 500,
label: "3W sleep(500ms)",
realBatch: 43.8,
realSql: 43.8,
},
];
for (const c of cases) {
const mb = throughputBatch(c.n, c.exec);
const ms = throughputSql(c.n, c.exec);
console.log(
` ${c.label.padEnd(21)}| ${mb.toFixed(0).padStart(7)} j/s | ${c.realBatch.toFixed(0).padStart(6)} j/s | ${ms.toFixed(0).padStart(5)} j/s | ${c.realSql.toFixed(0).padStart(4)} j/s`,
);
}
// --- Projections ---
const workerCounts = [1, 2, 3, 5, 8, 10, 15, 20]; // native workers (×8 subworkers each)
const execTimes = [
{ ms: 0, label: "~0ms (identity)" },
{ ms: 5, label: "5ms" },
{ ms: 20, label: "20ms" },
{ ms: 50, label: "50ms" },
{ ms: 200, label: "200ms" },
{ ms: 500, label: "500ms" },
];
console.log("\n\n=== Projected Throughput (jobs/s) ===\n");
for (const exec of execTimes) {
console.log(`--- Job duration: ${exec.label} ---\n`);
console.log(
" Native workers (subw) | Batch | SQL | Advantage | Batch wins?",
);
console.log(
" ----------------------|-----------|----------|-------------|------------",
);
for (const w of workerCounts) {
const n = w * 8;
const b = throughputBatch(n, exec.ms);
const s = throughputSql(n, exec.ms);
const advantage = pct(b, s);
const wins = b > s * 1.05 ? " YES" : b > s * 1.01 ? " marginal" : " no";
console.log(
` ${String(w).padStart(2)}W (${String(n).padStart(3)}) | ${b.toFixed(0).padStart(5)} j/s | ${s.toFixed(0).padStart(5)} j/s | ${advantage.padStart(8)} | ${wins}`,
);
}
console.log();
}
// --- Crossover analysis ---
console.log("=== Crossover: min workers where batch is >10% faster ===\n");
console.log(" Job duration | Min workers | Subworkers | Batch j/s | SQL j/s");
console.log(" -------------|-------------|------------|-----------|--------");
for (const exec of execTimes) {
let found = false;
for (let w = 1; w <= 50; w++) {
const n = w * 8;
const b = throughputBatch(n, exec.ms);
const s = throughputSql(n, exec.ms);
if (b > s * 1.1) {
console.log(
` ${exec.label.padEnd(13)}| ${String(w).padStart(5)}W | ${String(n).padStart(5)} | ${b.toFixed(0).padStart(5)} j/s | ${s.toFixed(0).padStart(5)} j/s`,
);
found = true;
break;
}
}
if (!found) {
console.log(
` ${exec.label.padEnd(13)}| >50W (never significant at this job duration)`,
);
}
}
console.log("\n\n=== Key Takeaways ===\n");
console.log(
"1. For fast jobs (~0ms): batch pull is always faster, advantage grows with scale",
);
console.log(" - 5 native workers (40 subworkers): ~13% faster");
console.log(" - 10 native workers (80 subworkers): ~25% faster");
console.log(" - 20 native workers (160 subworkers): ~88% faster");
console.log(
"2. For medium jobs (50ms): batch advantage meaningful from ~5 native workers",
);
console.log(
"3. For slow jobs (500ms+): only matters at 15+ native workers (120+ subworkers)",
);
console.log(
" (but still reduces DB load — fewer pull queries, less index scanning)",
);
console.log(
"4. The SQL quadratic contention (SKIP LOCKED scanning) is the dominant factor",
);
console.log(
" — SQL throughput plateaus around 15-20 native workers while batch keeps scaling",
);

View File

@@ -1,93 +0,0 @@
# Batch Pull Benchmark Results
Date: 2026-03-06
Setup: 1 server + 1 native worker (8 subworkers), standalone mode
Hardware: fedora, 1.5TB disk, ~1GB memory usage
DB: PostgreSQL local, windmill 270 MiB
## nativets — 1000 jobs
| | Batch Pull | Direct SQL |
|--|-----------|------------|
| Duration | 9.2s | 11.4s |
| **Throughput** | **108 jobs/s** | **88 jobs/s** |
| Improvement | **+23%** | baseline |
### pg_stat_statements
| Query | Batch calls | Batch ms | SQL calls | SQL ms |
|-------|------------|----------|-----------|--------|
| Native pull (FOR UPDATE SKIP LOCKED) | 2,399 (0.02ms avg) | 47 | 2,849 (0.03ms avg) | 87 |
| Default worker pull | 520 (0.04ms avg) | 19 | 445 (0.05ms avg) | 23 |
| DELETE from queue | 1,001 (0.23ms avg) | 231 | 1,001 (0.19ms avg) | 195 |
| INSERT into completed | 1,001 (0.05ms avg) | 51 | 1,001 (0.04ms avg) | 45 |
| INSERT job_logs | 2,003 (0.02ms avg) | 44 | 2,003 (0.02ms avg) | 44 |
| Agent token blacklist | 3,920 (0.00ms avg) | 19 | — | — |
| **Total** | **9,389** | **1,332** | **8,882** | **1,212** |
### pg_stat_database
| Metric | Batch Pull | Direct SQL |
|--------|-----------|------------|
| Transactions committed | 6,484 | 5,868 |
| Blocks read (disk) | 101 | 101 |
| Blocks hit (cache) | 920,257 | 699,032 |
| Tuples returned | 7,784,044 | 8,987,097 |
| Tuples fetched | 1,028,819 | 805,100 |
| Tuples inserted | 6,822 | 6,880 |
| Tuples updated | 3,377 | 3,090 |
| Tuples deleted | 2,883 | 2,654 |
---
## nativets_sleep — 1000 jobs
Each job sleeps 300-700ms (random). Theoretical max with 8 workers: ~16 jobs/s.
| | Batch Pull | Direct SQL |
|--|-----------|------------|
| Duration | 66.7s | 68.2s |
| **Throughput** | **15.0 jobs/s** | **14.7 jobs/s** |
| Improvement | ~same | baseline |
### pg_stat_statements
| Query | Batch calls | Batch ms | SQL calls | SQL ms |
|-------|------------|----------|-----------|--------|
| Native pull (FOR UPDATE SKIP LOCKED) | 2,399 (0.02ms avg) | 43 | 1,762 (0.04ms avg) | 75 |
| Default worker pull | 1,591 (0.07ms avg) | 113 | 1,392 (0.08ms avg) | 115 |
| DELETE from queue | 1,001 (0.31ms avg) | 308 | 1,001 (0.20ms avg) | 205 |
| INSERT into completed | 1,001 (0.05ms avg) | 46 | 1,001 (0.04ms avg) | 41 |
| INSERT job_logs | 2,003 (0.02ms avg) | 45 | 2,003 (0.02ms avg) | 40 |
| Job runtime ping | 1,490 (0.02ms avg) | 33 | 1,489 (0.02ms avg) | 31 |
| Worker ping (job) | 1,001 (0.03ms avg) | 32 | 1,001 (0.03ms avg) | 30 |
| **Total** | **16,523** | **5,798** | **16,364** | **5,717** |
### pg_stat_database
| Metric | Batch Pull | Direct SQL |
|--------|-----------|------------|
| Transactions committed | 13,638 | 13,388 |
| Blocks read (disk) | 394 | 850 |
| Blocks hit (cache) | 7,033,158 | 4,932,617 |
| Tuples returned | 58,424,571 | 57,204,918 |
| Tuples fetched | 8,776,186 | 6,321,947 |
| Tuples inserted | 7,500 | 7,531 |
| Tuples updated | 6,111 | 6,193 |
| Tuples deleted | 3,006 | 3,196 |
---
## Analysis
**Throughput**: +23% for fast CPU-bound jobs. Negligible difference for I/O-bound jobs.
**Pull queries**: Batch pull does MORE pull queries for nativets_sleep (2,399 vs 1,762). The refiller polls every 50ms even when all workers are busy executing jobs. With direct SQL, workers only poll when idle. This is wasted work — the refiller queries DB and gets empty results while jobs are in-flight.
**Disk I/O**: Batch pull cuts disk reads in half for nativets_sleep (394 vs 850 blocks). Likely because the batch query locks multiple rows in one pass, reducing index traversal.
**Cache hits**: Higher with batch pull (7M vs 4.9M for sleep). More buffer hits from the refiller's repeated empty polls touching the same index pages.
**Tuples fetched**: Higher with batch pull (8.7M vs 6.3M for sleep). Same cause — the refiller's empty polls scan the index.
**At scale**: With 8 subworkers the differences are small. The real benefit is with many native workers where direct SQL SKIP LOCKED contention grows O(N²).

View File

@@ -1,123 +0,0 @@
# Batch Pull Benchmark Results — 3 Workers
Date: 2026-03-06
Setup: 1 server + 3 native workers (8 subworkers each = 24 subworkers)
Hardware: fedora, 1.5TB disk, ~1GB memory usage
DB: PostgreSQL local, windmill 270 MiB
## nativets — 1000 jobs
| | 3W Batch | 3W SQL | 1W Batch | 1W SQL |
|--|---------|--------|---------|--------|
| Duration | 3.7s | 3.5s | 9.2s | 11.4s |
| **Throughput** | **272 jobs/s** | **288 jobs/s** | **108 jobs/s** | **88 jobs/s** |
| vs 1W SQL | +209% | +227% | +23% | baseline |
Note: First 3W SQL run was 33 jobs/s (outlier due to cold start or background activity). Rerun gave 288 jobs/s.
## nativets — 10,000 jobs
| | 3W Batch | 3W SQL |
|--|---------|--------|
| Duration | 34.3s | 39.5s |
| **Throughput** | **291 jobs/s** | **253 jobs/s** |
| Improvement | **+15%** | baseline |
### pg_stat_statements (1000 jobs, first run)
| Query | 3W Batch calls | 3W Batch ms | 3W SQL calls | 3W SQL ms |
|-------|---------------|-------------|-------------|----------|
| Native pull (FOR UPDATE SKIP LOCKED) | 4,801 (0.01ms) | 59 | 20,367 (0.01ms) | 231 |
| Default worker pull | 353 (0.03ms) | 10 | 880 (0.02ms) | 16 |
| DELETE from queue | 1,001 (0.44ms) | 439 | 1,001 (0.43ms) | 427 |
| INSERT into completed | 1,001 (0.06ms) | 61 | 1,001 (0.05ms) | 55 |
| INSERT job_logs | 2,003 (0.03ms) | 67 | 2,003 (0.03ms) | 64 |
| Agent token blacklist | 7,867 (0.00ms) | 33 | — | — |
| Worker ping (job) | 350 (0.04ms) | 15 | — | — |
| Outstanding wait time | 664 (0.03ms) | 21 | 742 (0.03ms) | 23 |
### pg_stat_database (1000 jobs, first run)
| Metric | 3W Batch | 3W SQL |
|--------|---------|--------|
| Transactions committed | 22,070 | 29,301 |
| Blocks read (disk) | 308 | 395 |
| Blocks hit (cache) | 280,071 | 1,276,521 |
| Tuples returned | 3,368,255 | 28,695,830 |
| Tuples fetched | 140,864 | 1,089,774 |
| Tuples inserted | 6,682 | 6,760 |
| Tuples updated | 3,521 | 3,453 |
| Tuples deleted | 3,007 | 3,007 |
---
## nativets_sleep — 1000 jobs
Each job sleeps 300-700ms (random). Theoretical max with 24 workers: ~48 jobs/s.
| | 3W Batch | 3W SQL | 1W Batch | 1W SQL |
|--|---------|--------|---------|--------|
| Duration | 22.8s | 22.8s | 66.7s | 68.2s |
| **Throughput** | **43.8 jobs/s** | **43.8 jobs/s** | **15.0 jobs/s** | **14.7 jobs/s** |
| vs 3W SQL | ~same | baseline | — | — |
### pg_stat_statements
| Query | 3W Batch calls | 3W Batch ms | 3W SQL calls | 3W SQL ms |
|-------|---------------|-------------|-------------|----------|
| Native pull (FOR UPDATE SKIP LOCKED) | 4,898 (0.01ms) | 55 | 6,440 (0.02ms) | 113 |
| Default worker pull | 696 (0.06ms) | 43 | 654 (0.06ms) | 37 |
| DELETE from queue | 1,001 (0.38ms) | 379 | 1,001 (0.29ms) | 290 |
| INSERT into completed | 1,001 (0.05ms) | 46 | 1,001 (0.04ms) | 43 |
| INSERT job_logs | 2,003 (0.02ms) | 44 | 2,003 (0.02ms) | 43 |
| Agent token blacklist | 7,295 (0.00ms) | 31 | — | — |
| Job runtime ping | 1,499 (0.02ms) | 35 | 1,444 (0.02ms) | 35 |
| Worker ping (job) | 1,001 (0.03ms) | 32 | 1,001 (0.03ms) | 31 |
| Job stats | 549 (0.05ms) | 27 | 493 (0.05ms) | 26 |
| Outstanding wait time | 928 (0.02ms) | 20 | 944 (0.02ms) | 21 |
### pg_stat_database
| Metric | 3W Batch | 3W SQL |
|--------|---------|--------|
| Transactions committed | 25,896 | 18,646 |
| Blocks read (disk) | 115 | 204 |
| Blocks hit (cache) | 1,173,696 | 1,256,397 |
| Tuples returned | 21,074,537 | 22,063,593 |
| Tuples fetched | 1,200,878 | 1,262,900 |
| Tuples inserted | 7,495 | 7,461 |
| Tuples updated | 6,204 | 6,141 |
| Tuples deleted | 3,005 | 3,011 |
---
## Analysis
### nativets (CPU-bound): +15% with 10K jobs
With 10,000 jobs, batch pull achieves **291 jobs/s vs 253 jobs/s** (+15%). The 1000-job runs showed similar throughput (~272-288 jobs/s) after discarding the cold-start outlier.
**DB load difference** (from the 1000-job first run, which captured the worst-case SQL contention):
- **20,367 pull queries** (SQL) vs 4,801 (batch) — 4x more queries
- **28.7M tuples returned** (SQL) vs 3.4M (batch) — 8.5x more index scanning
- **1.3M cache hits** (SQL) vs 280K (batch) — 4.6x more buffer activity
The batch approach consolidates all 24 subworkers into a single `LIMIT 24` query, reducing contention on the queue index.
### nativets_sleep (I/O-bound): No throughput difference
Both achieve **43.8 jobs/s** (91% of theoretical 48 jobs/s max). When workers spend 300-700ms sleeping, DB contention isn't the bottleneck.
Batch pull still shows slightly lower DB load:
- **4,898 pull queries** vs 6,440 — 24% fewer
- **115 disk reads** vs 204 — 44% fewer
### Scaling summary
| Setup | Batch jobs/s | SQL jobs/s | Batch advantage |
|-------|-------------|-----------|----------------|
| 1W × 1000 jobs | 108 | 88 | +23% |
| 3W × 1000 jobs | 272 | 288 | ~same |
| 3W × 10,000 jobs | 291 | 253 | **+15%** |
At 24 subworkers, batch pull provides a consistent ~15% throughput improvement for sustained CPU-bound workloads, with significantly lower DB load (4x fewer pull queries, 8x fewer tuples scanned). The benefit grows with more workers as SKIP LOCKED contention scales O(N²).

View File

@@ -39,10 +39,7 @@ export const scriptBootstrapCode = {
}
`,
bun: `// there are multiple modes to add as header: //nobundling //native //npm //nodejs
// https://www.windmill.dev/docs/getting_started/scripts_quickstart/typescript#modes
export async function main() {
bun: `export async function main() {
return "Hello world";
}
`,

View File

@@ -67,7 +67,7 @@ export {
workspaceAdd,
};
export const VERSION = "1.651.1";
export const VERSION = "1.650.0";
// Re-exported from constants.ts to maintain backwards compatibility
export { WM_FORK_PREFIX } from "./core/constants.ts";

View File

@@ -1,12 +1,12 @@
{
"name": "windmill-components",
"version": "1.651.1",
"version": "1.650.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "windmill-components",
"version": "1.651.1",
"version": "1.650.0",
"hasInstallScript": true,
"license": "AGPL-3.0",
"dependencies": {

View File

@@ -1,6 +1,6 @@
{
"name": "windmill-components",
"version": "1.651.1",
"version": "1.650.0",
"scripts": {
"dev": "vite dev",
"build": "vite build",

View File

@@ -133,7 +133,7 @@ export function useJobsLoader(args: () => UseJobLoaderArgs) {
])
})
}
promise = CancelablePromiseUtils.finallyDo(promise, () => {
promise = CancelablePromiseUtils.pipe(promise, () => {
if (slowStreamIntervalId) {
clearInterval(slowStreamIntervalId)
slowStreamIntervalId = undefined
@@ -161,7 +161,7 @@ export function useJobsLoader(args: () => UseJobLoaderArgs) {
)
}, 15000)
paramChangePromise = loadJobsIntern(false, size)
paramChangePromise = CancelablePromiseUtils.finallyDo(paramChangePromise, () => {
paramChangePromise = CancelablePromiseUtils.pipe(paramChangePromise, () => {
if (slowStreamIntervalId) {
clearInterval(slowStreamIntervalId)
slowStreamIntervalId = undefined
@@ -687,10 +687,6 @@ export function useJobsLoader(args: () => UseJobLoaderArgs) {
return () => {
clearTimeout(paramChangeTimeout)
paramChangePromise?.cancel()
if (slowStreamIntervalId) {
clearInterval(slowStreamIntervalId)
slowStreamIntervalId = undefined
}
}
})
$effect(() => {

View File

@@ -17,7 +17,7 @@
import { safeSelectItems } from '../select/utils.svelte'
import TokenDisplay from './TokenDisplay.svelte'
import ScopeSelector from './ScopeSelector.svelte'
import Alert from '../common/alert/Alert.svelte'
import FolderPicker from '../FolderPicker.svelte'
import TextInput from '../text_input/TextInput.svelte'
import Select from '../select/Select.svelte'
@@ -43,6 +43,8 @@
displayCreateToken = true
}: Props = $props()
// MCP clients do not allow names longer than 60 characters, here we use 55 because final tool name server side will add ~5 characters
const MAX_PATH_LENGTH = 55
let newToken = $state<string | undefined>(undefined)
let newMcpToken = $state<string | undefined>(undefined)
@@ -183,6 +185,17 @@
? `You do not have any favorite scripts or flows. You can favorite some scripts and flows to include them, or change the scope to "All scripts/flows" to include all your scripts and flows.`
: `You do not have any scripts or flows in the selected folder.`
)
const longPathRunnables = $derived(
includedRunnables.filter((path) => path.length > MAX_PATH_LENGTH)
)
const validRunnables = $derived(
includedRunnables.filter((path) => path.length <= MAX_PATH_LENGTH)
)
const longPathWarning = $derived(
longPathRunnables.length > 0
? `${longPathRunnables.length} script(s)/flow(s) have paths longer than 60 characters and will be excluded from MCP tools. Consider shortening the paths: ${longPathRunnables.slice(0, 3).join(', ')}${longPathRunnables.length > 3 ? ` and ${longPathRunnables.length - 3} more` : ''}`
: ''
)
$effect(() => {
if (mcpCreationMode) {
@@ -614,18 +627,23 @@
</div>
{:else}
<div class="flex flex-col gap-2 col-span-2 pr-4">
{#if longPathWarning}
<Alert type="warning" title="Some paths are too long" size="xs">
{longPathWarning}
</Alert>
{/if}
<span class="block text-xs">Scripts & Flows that will be available via MCP</span>
<div class="flex flex-wrap gap-1">
{#if includedRunnables.length > 0 && includedRunnables.length <= 5}
{#each includedRunnables as scriptOrFlow}
{#if validRunnables.length > 0 && validRunnables.length <= 5}
{#each validRunnables as scriptOrFlow}
<Badge rounded small color="blue">{scriptOrFlow}</Badge>
{/each}
{:else if includedRunnables.length > 0}
{#each includedRunnables.slice(0, 3) as scriptOrFlow}
{:else if validRunnables.length > 0}
{#each validRunnables.slice(0, 3) as scriptOrFlow}
<Badge rounded small color="blue">{scriptOrFlow}</Badge>
{/each}
<Badge rounded small color="dark-gray">
+{includedRunnables.length - 3} more
+{validRunnables.length - 3} more
</Badge>
{:else}
<p class="text-xs text-primary">

View File

@@ -4,7 +4,7 @@ verify_ssl = true
name = "pypi"
[packages]
wmill = ">=1.651.1"
wmill = ">=1.650.0"
sendgrid = "*"
mysql-connector-python = "*"
pymongo = "*"

View File

@@ -1,7 +1,7 @@
openapi: '3.0.3'
info:
version: 1.651.1
version: 1.650.0
title: OpenFlow Spec
contact:
name: Ruben Fiszel

View File

@@ -12,7 +12,7 @@
RootModule = 'WindmillClient.psm1'
# Version number of this module.
ModuleVersion = '1.651.1'
ModuleVersion = '1.650.0'
# Supported PSEditions
# CompatiblePSEditions = @()

View File

@@ -108,21 +108,6 @@ SET s3_secret_access_key='80yMndIMcyXwEujxVNINQbf0tBlIzRaLPyM2m1n4';
)
print(file_key)
@unittest.skip("skipping")
def test_delete_s3_object(self):
# Upload a temporary file
s3_obj = wmill.write_s3_file(
S3Object(s3="_wmill_test_delete_s3_object.txt"), b"delete_s3_object test content"
)
# Verify it exists
content = wmill.load_s3_file(s3_obj)
self.assertEqual(content, b"delete_s3_object test content")
# Delete it
wmill.delete_s3_object(s3_obj)
# Verify it's gone
with self.assertRaises(Exception):
wmill.load_s3_file(s3_obj)
if __name__ == "__main__":
unittest.main()

View File

@@ -1,6 +1,6 @@
[tool.poetry]
name = "wmill"
version = "1.651.1"
version = "1.650.0"
description = "A client library for accessing Windmill server wrapping the Windmill client API"
license = "Apache-2.0"
homepage = "https://windmill.dev"

View File

@@ -984,40 +984,6 @@ class Windmill:
raise Exception("Could not write file to S3") from e
return S3Object(s3=response["file_key"], storage=s3object.get("storage") if s3object else None)
def delete_s3_object(
self,
s3object: S3Object | str,
s3_resource_path: str | None = None,
) -> None:
"""
Permanently delete a file from the workspace S3 bucket.
'''python
from wmill import S3Object
s3_obj = S3Object(s3="/path/to/my_file.txt")
client.delete_s3_object(s3_obj)
'''
"""
s3object = parse_s3_object(s3object)
query_params: Dict[str, Any] = {"file_key": s3object["s3"]}
if s3_resource_path is not None and s3_resource_path != "":
query_params["s3_resource_path"] = s3_resource_path
if "storage" in s3object and s3object["storage"] is not None:
query_params["storage"] = s3object["storage"]
try:
resp = self.client.delete(
f"/w/{self.workspace}/job_helpers/delete_s3_file",
params=query_params,
)
resp.raise_for_status()
except httpx.HTTPStatusError as err:
error = f"{err.request.url}: {err.response.status_code}, {err.response.text}"
logger.error(error)
raise Exception(error)
except Exception as e:
raise Exception("Could not delete file from S3") from e
def sign_s3_objects(self, s3_objects: list[S3Object | str]) -> list[S3Object]:
"""Sign S3 objects for use by anonymous users in public apps.
@@ -1726,20 +1692,6 @@ def write_s3_file(
)
@init_global_client
def delete_s3_object(
s3object: S3Object | str,
s3_resource_path: str | None = None,
) -> None:
"""
Permanently delete a file from the workspace S3 bucket.
"""
return _client.delete_s3_object(
s3object,
s3_resource_path if s3_resource_path != "" else None,
)
@init_global_client
def sign_s3_objects(s3_objects: list[S3Object | str]) -> list[S3Object]:
"""
@@ -2416,4 +2368,4 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]]:
name, schema = name.split(":", 1)
if not name:
name = "main"
return name, schema
return name, schema

View File

@@ -1,6 +1,6 @@
{
"name": "@windmill/windmill",
"version": "1.651.1",
"version": "1.650.0",
"exports": "./src/index.ts",
"publish": {
"exclude": ["!src", "./s3Types.ts", "./sqlUtils.ts", "./client.ts"]

View File

@@ -1,7 +1,7 @@
{
"name": "windmill-client",
"description": "Windmill SDK client for browsers and Node.js",
"version": "1.651.1",
"version": "1.650.0",
"author": "Ruben Fiszel",
"license": "Apache 2.0",
"sideEffects": false,

View File

@@ -1 +1 @@
1.651.1
1.650.0