Compare commits
3 Commits
batch-pull
...
volume-lim
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0cc3342e12 | ||
|
|
d081323250 | ||
|
|
ebbc259a37 |
23
CHANGELOG.md
23
CHANGELOG.md
@@ -1,28 +1,5 @@
|
||||
# Changelog
|
||||
|
||||
## [1.651.1](https://github.com/windmill-labs/windmill/compare/v1.651.0...v1.651.1) (2026-03-05)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* prevent slow loading toast interval from leaking on promise cancellation ([#8240](https://github.com/windmill-labs/windmill/issues/8240)) ([2e582b1](https://github.com/windmill-labs/windmill/commit/2e582b1bc1c299388a3c97cfddff9d0eb92858f2))
|
||||
* suppress unused variable warnings on windows builds ([#8241](https://github.com/windmill-labs/windmill/issues/8241)) ([2d58382](https://github.com/windmill-labs/windmill/commit/2d583826dc065c05684d4cd1d1510f0d1f2d9ae9))
|
||||
|
||||
## [1.651.0](https://github.com/windmill-labs/windmill/compare/v1.650.0...v1.651.0) (2026-03-05)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* add sandbox annotations, volume mounts, for AI sandbox starting with claude ([#8058](https://github.com/windmill-labs/windmill/issues/8058)) ([5f0ef93](https://github.com/windmill-labs/windmill/commit/5f0ef936d1d5d07d01c8e07e26ec254feebef8fb))
|
||||
* hash-based MCP tool names for long paths ([#8133](https://github.com/windmill-labs/windmill/issues/8133)) ([ce041e8](https://github.com/windmill-labs/windmill/commit/ce041e8a5e7ff105df389875d9981f3843d4ce39))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **python-client:** add delete_s3_object ([#8216](https://github.com/windmill-labs/windmill/issues/8216)) ([90f4c64](https://github.com/windmill-labs/windmill/commit/90f4c64ee12e1d04ce846ff88d6658f667e194e0))
|
||||
* update CLI bun template to match UI template ([#8238](https://github.com/windmill-labs/windmill/issues/8238)) ([a8cbe93](https://github.com/windmill-labs/windmill/commit/a8cbe9396ffc51140dce5582d57f4dc59873304e))
|
||||
* write fallback package.json for codebase mode nsjail ([#8239](https://github.com/windmill-labs/windmill/issues/8239)) ([d46913b](https://github.com/windmill-labs/windmill/commit/d46913b74a0ffd41d2323e0355cc81954f09e29d))
|
||||
|
||||
## [1.650.0](https://github.com/windmill-labs/windmill/compare/v1.649.0...v1.650.0) (2026-03-05)
|
||||
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false
|
||||
true
|
||||
]
|
||||
},
|
||||
"hash": "2d6607b3c38fe72b5663c32de58dacbabed4c5ae28101e3ae2694f96fd055a91"
|
||||
|
||||
@@ -1,26 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, dedicated_workers, wm_version, vcpus, memory, job_isolation, native_mode, uses_batch_http_pull) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) ON CONFLICT (worker)\n DO UPDATE set ip = EXCLUDED.ip, custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group, dedicated_workers = EXCLUDED.dedicated_workers, native_mode = EXCLUDED.native_mode, uses_batch_http_pull = EXCLUDED.uses_batch_http_pull",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Varchar",
|
||||
"Varchar",
|
||||
"Varchar",
|
||||
"TextArray",
|
||||
"Varchar",
|
||||
"Varchar",
|
||||
"TextArray",
|
||||
"Varchar",
|
||||
"Int8",
|
||||
"Int8",
|
||||
"Text",
|
||||
"Bool",
|
||||
"Bool"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "3e8afd021088a99a24f27fa6f0a1b7f3edba3e9b834c814b464305bc2eb6ba80"
|
||||
}
|
||||
@@ -15,7 +15,7 @@
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
true
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55"
|
||||
|
||||
@@ -1,26 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1, custom_tags = $2,\n occupancy_rate = $3, memory_usage = $4, wm_memory_usage = $5, vcpus = COALESCE($7, vcpus),\n memory = COALESCE($8, memory), occupancy_rate_15s = $9, occupancy_rate_5m = $10, occupancy_rate_30m = $11, native_mode = $12, uses_batch_http_pull = $13 WHERE worker = $6",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Int4",
|
||||
"TextArray",
|
||||
"Float4",
|
||||
"Int8",
|
||||
"Int8",
|
||||
"Text",
|
||||
"Int8",
|
||||
"Int8",
|
||||
"Float4",
|
||||
"Float4",
|
||||
"Float4",
|
||||
"Bool",
|
||||
"Bool"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "6cd099d458ac380d5da27b9e69da035755496ea50f2b78fb9b1cd3a2eb7e7625"
|
||||
}
|
||||
@@ -15,7 +15,7 @@
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false
|
||||
true
|
||||
]
|
||||
},
|
||||
"hash": "eba16eb819e2644284fb073c891706d78a6f24cb0e614d7d81ba1b643805bf06"
|
||||
|
||||
143
backend/Cargo.lock
generated
143
backend/Cargo.lock
generated
@@ -15741,7 +15741,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-nats",
|
||||
@@ -15773,7 +15773,6 @@ dependencies = [
|
||||
"sql-builder",
|
||||
"sqlx",
|
||||
"strum 0.27.2",
|
||||
"tar",
|
||||
"tempfile",
|
||||
"tikv-jemalloc-ctl",
|
||||
"tikv-jemalloc-sys",
|
||||
@@ -15808,7 +15807,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-alerting"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"axum 0.7.9",
|
||||
"chrono",
|
||||
@@ -15821,7 +15820,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-api"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"argon2",
|
||||
@@ -15960,7 +15959,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-api-agent-workers"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"axum 0.7.9",
|
||||
"chrono",
|
||||
@@ -15983,7 +15982,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-api-assets"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"axum 0.7.9",
|
||||
"chrono",
|
||||
@@ -15996,7 +15995,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-api-auth"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"axum 0.7.9",
|
||||
@@ -16022,7 +16021,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-api-client"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"reqwest 0.12.28",
|
||||
"serde",
|
||||
@@ -16032,7 +16031,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-api-configs"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"axum 0.7.9",
|
||||
"chrono",
|
||||
@@ -16049,7 +16048,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-api-debug"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"axum 0.7.9",
|
||||
"base64 0.22.1",
|
||||
@@ -16072,7 +16071,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-api-embeddings"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"axum 0.7.9",
|
||||
@@ -16095,7 +16094,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-api-flow-conversations"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"axum 0.7.9",
|
||||
"chrono",
|
||||
@@ -16111,7 +16110,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-api-flows"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"axum 0.7.9",
|
||||
"chrono",
|
||||
@@ -16131,7 +16130,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-api-groups"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"axum 0.7.9",
|
||||
"chrono",
|
||||
@@ -16151,7 +16150,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-api-inputs"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"axum 0.7.9",
|
||||
"chrono",
|
||||
@@ -16165,7 +16164,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-api-integration-tests"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-nats",
|
||||
@@ -16192,7 +16191,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-api-jobs"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"axum 0.7.9",
|
||||
@@ -16217,7 +16216,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-api-npm-proxy"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"axum 0.7.9",
|
||||
"flate2",
|
||||
@@ -16235,7 +16234,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-api-openapi"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"axum 0.7.9",
|
||||
@@ -16256,7 +16255,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-api-schedule"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"axum 0.7.9",
|
||||
"chrono",
|
||||
@@ -16276,7 +16275,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-api-scripts"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"axum 0.7.9",
|
||||
"chrono",
|
||||
@@ -16306,7 +16305,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-api-settings"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"axum 0.7.9",
|
||||
@@ -16333,7 +16332,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-api-sse"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"lazy_static",
|
||||
"serde",
|
||||
@@ -16345,7 +16344,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-api-users"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"argon2",
|
||||
"axum 0.7.9",
|
||||
@@ -16368,7 +16367,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-api-workers"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"axum 0.7.9",
|
||||
"chrono",
|
||||
@@ -16382,7 +16381,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-api-workspaces"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"axum 0.7.9",
|
||||
"chrono",
|
||||
@@ -16413,7 +16412,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-audit"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"lazy_static",
|
||||
@@ -16427,7 +16426,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-autoscaling"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"axum 0.7.9",
|
||||
@@ -16446,7 +16445,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-common"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"anyhow",
|
||||
@@ -16545,7 +16544,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-dep-map"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"itertools 0.14.0",
|
||||
@@ -16564,7 +16563,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-git-sync"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"regex",
|
||||
"serde",
|
||||
@@ -16579,7 +16578,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-indexer"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"astral-tokio-tar",
|
||||
@@ -16603,7 +16602,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-jseval"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"futures",
|
||||
@@ -16620,7 +16619,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-macros"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"itertools 0.14.0",
|
||||
"lazy_static",
|
||||
@@ -16636,7 +16635,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-mcp"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
@@ -16657,7 +16656,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-native-triggers"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
@@ -16688,7 +16687,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-oauth"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-oauth2",
|
||||
@@ -16712,7 +16711,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-object-store"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-stream",
|
||||
@@ -16746,7 +16745,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-operator"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"futures",
|
||||
@@ -16764,7 +16763,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"convert_case 0.6.0",
|
||||
"serde",
|
||||
@@ -16773,7 +16772,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-bash"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"lazy_static",
|
||||
@@ -16785,7 +16784,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-csharp"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"serde_json",
|
||||
@@ -16797,7 +16796,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-go"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"gosyn",
|
||||
@@ -16809,7 +16808,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-graphql"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"lazy_static",
|
||||
@@ -16821,7 +16820,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-java"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"serde_json",
|
||||
@@ -16833,7 +16832,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-nu"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"nu-parser",
|
||||
@@ -16844,7 +16843,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-php"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"itertools 0.14.0",
|
||||
@@ -16855,7 +16854,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-py"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"itertools 0.14.0",
|
||||
@@ -16868,7 +16867,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-py-imports"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-recursion",
|
||||
@@ -16892,7 +16891,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-ruby"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"lazy_static",
|
||||
@@ -16906,7 +16905,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-rust"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"convert_case 0.6.0",
|
||||
@@ -16923,7 +16922,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-sql"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"lazy_static",
|
||||
@@ -16938,7 +16937,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-ts"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"lazy_static",
|
||||
@@ -16957,7 +16956,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-yaml"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"serde",
|
||||
@@ -16968,7 +16967,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-queue"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-recursion",
|
||||
@@ -17005,7 +17004,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-runtime-nativets"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"const_format",
|
||||
@@ -17043,7 +17042,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-sql-datatype-parser-wasm"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"getrandom 0.3.4",
|
||||
"wasm-bindgen",
|
||||
@@ -17054,7 +17053,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-store"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-recursion",
|
||||
@@ -17083,7 +17082,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-test-utils"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"axum 0.7.9",
|
||||
@@ -17106,7 +17105,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-trigger"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
@@ -17139,7 +17138,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-trigger-email"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
@@ -17159,7 +17158,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-trigger-gcp"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
@@ -17193,7 +17192,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-trigger-http"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
@@ -17228,7 +17227,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-trigger-kafka"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
@@ -17251,7 +17250,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-trigger-mqtt"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
@@ -17275,7 +17274,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-trigger-nats"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-nats",
|
||||
@@ -17299,7 +17298,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-trigger-postgres"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
@@ -17334,7 +17333,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-trigger-sqs"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
@@ -17362,7 +17361,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-trigger-websocket"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
@@ -17385,7 +17384,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-types"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"bitflags 2.9.4",
|
||||
@@ -17403,7 +17402,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-worker"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-once-cell",
|
||||
@@ -17509,7 +17508,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-worker-volumes"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "windmill"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
authors.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
@@ -77,7 +77,7 @@ members = [
|
||||
exclude = ["./windmill-duckdb-ffi-internal"]
|
||||
|
||||
[workspace.package]
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
|
||||
edition = "2021"
|
||||
|
||||
@@ -257,7 +257,6 @@ axum.workspace = true
|
||||
serde.workspace = true
|
||||
windmill-api-client.workspace = true
|
||||
tempfile.workspace = true
|
||||
tar.workspace = true
|
||||
windmill-parser-ts.workspace = true
|
||||
rumqttc.workspace = true
|
||||
rdkafka.workspace = true
|
||||
|
||||
@@ -1 +1 @@
|
||||
c3c543f4c60a8c4dfe0d912c79a051376fb091a9
|
||||
05385738e36e81f5bc51d15c0ca60bba30457c21
|
||||
@@ -1 +0,0 @@
|
||||
ALTER TABLE worker_ping DROP COLUMN IF EXISTS uses_batch_http_pull;
|
||||
@@ -1 +0,0 @@
|
||||
ALTER TABLE worker_ping ADD COLUMN IF NOT EXISTS uses_batch_http_pull BOOLEAN NOT NULL DEFAULT false;
|
||||
@@ -61,9 +61,8 @@ use windmill_common::{
|
||||
MODE_AND_ADDONS,
|
||||
},
|
||||
worker::{
|
||||
is_native_mode_from_env, reload_custom_tags_setting, Connection, HttpClient, HUB_CACHE_DIR,
|
||||
HUB_RT_CACHE_DIR, NATIVE_MODE_RESOLVED, TMP_LOGS_DIR, USES_BATCH_HTTP_PULL, WINDMILL_DIR,
|
||||
WORKER_GROUP,
|
||||
is_native_mode_from_env, reload_custom_tags_setting, Connection, HUB_CACHE_DIR,
|
||||
HUB_RT_CACHE_DIR, NATIVE_MODE_RESOLVED, TMP_LOGS_DIR, WINDMILL_DIR, WORKER_GROUP,
|
||||
},
|
||||
KillpillSender, DEFAULT_HUB_BASE_URL, METRICS_ENABLED,
|
||||
};
|
||||
@@ -921,20 +920,6 @@ Windmill Community Edition {GIT_VERSION}
|
||||
default_base_internal_url.clone()
|
||||
};
|
||||
|
||||
// BATCH_PULL_URL: explicit URL for native workers to pull jobs via HTTP.
|
||||
// In standalone mode (server_mode=true), defaults to the local server.
|
||||
let batch_pull_url: Option<String> = if is_native_mode_from_env() {
|
||||
if let Ok(url) = std::env::var("BATCH_PULL_URL") {
|
||||
Some(url)
|
||||
} else if server_mode {
|
||||
Some(default_base_internal_url.clone())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
initial_load(
|
||||
&conn,
|
||||
killpill_tx.clone(),
|
||||
@@ -1145,30 +1130,6 @@ Windmill Community Edition {GIT_VERSION}
|
||||
)?;
|
||||
let mut workers = vec![];
|
||||
|
||||
// For native workers, create a self-signed JWT for batch pulling via HTTP.
|
||||
// Enabled when BATCH_PULL_URL is set (explicitly or auto-detected in standalone mode).
|
||||
let batch_pull_client = if let Some(ref pull_url) = batch_pull_url {
|
||||
match create_native_batch_pull_client(pull_url).await {
|
||||
Ok(client) => {
|
||||
tracing::info!(
|
||||
"Native batch pull client created for HTTP pull at {}",
|
||||
pull_url
|
||||
);
|
||||
USES_BATCH_HTTP_PULL
|
||||
.store(true, std::sync::atomic::Ordering::Relaxed);
|
||||
Some(client)
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
"Failed to create native batch pull client, falling back to SQL pull: {e:#}"
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
for i in 0..num_workers {
|
||||
let suffix = if i == 0 && first_suffix.is_some() {
|
||||
first_suffix.as_ref().unwrap().clone()
|
||||
@@ -1192,7 +1153,6 @@ Windmill Community Edition {GIT_VERSION}
|
||||
WORKER_GROUP.as_str(),
|
||||
&suffix,
|
||||
),
|
||||
batch_pull_client: batch_pull_client.clone(),
|
||||
};
|
||||
workers.push(worker_conn);
|
||||
}
|
||||
@@ -1806,7 +1766,6 @@ fn display_config(envs: &[&str]) {
|
||||
pub struct WorkerConn {
|
||||
conn: Connection,
|
||||
worker_name: String,
|
||||
batch_pull_client: Option<HttpClient>,
|
||||
}
|
||||
|
||||
pub async fn run_workers(
|
||||
@@ -1877,7 +1836,6 @@ pub async fn run_workers(
|
||||
let wk_conf = &workers[i as usize - 1];
|
||||
let conn1 = wk_conf.conn.clone();
|
||||
let worker_name = wk_conf.worker_name.clone();
|
||||
let batch_pull_client = wk_conf.batch_pull_client.clone();
|
||||
WORKERS_NAMES.write().await.push(worker_name.clone());
|
||||
let ip = ip.clone();
|
||||
let rx = killpill_rxs.pop().unwrap();
|
||||
@@ -1900,7 +1858,6 @@ pub async fn run_workers(
|
||||
rx,
|
||||
tx,
|
||||
&base_internal_url,
|
||||
batch_pull_client.as_ref(),
|
||||
);
|
||||
|
||||
// #[cfg(tokio_unstable)]
|
||||
@@ -1919,41 +1876,6 @@ pub async fn run_workers(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Create an HTTP client for native workers to pull jobs from the local server's batch buffer.
|
||||
/// Self-signs a JWT with native_mode=true using the same JWT secret the server uses.
|
||||
async fn create_native_batch_pull_client(base_internal_url: &str) -> anyhow::Result<HttpClient> {
|
||||
use windmill_common::agent_workers::{build_agent_http_client, AGENT_JWT_PREFIX};
|
||||
use windmill_common::jwt::encode_with_internal_secret;
|
||||
|
||||
#[derive(serde::Serialize)]
|
||||
struct NativeAgentAuth {
|
||||
worker_group: String,
|
||||
tags: Vec<String>,
|
||||
native_mode: Option<bool>,
|
||||
exp: usize,
|
||||
}
|
||||
|
||||
let worker_config = windmill_common::worker::WORKER_CONFIG.read().await;
|
||||
let tags = worker_config.worker_tags.clone();
|
||||
drop(worker_config);
|
||||
|
||||
// Token expires in 30 days — renewed on restart
|
||||
let exp = (chrono::Utc::now() + chrono::Duration::days(30)).timestamp() as usize;
|
||||
|
||||
let claims = NativeAgentAuth {
|
||||
worker_group: WORKER_GROUP.to_string(),
|
||||
tags,
|
||||
native_mode: Some(true),
|
||||
exp,
|
||||
};
|
||||
|
||||
let jwt = encode_with_internal_secret(claims).await?;
|
||||
let token = format!("{}{}", AGENT_JWT_PREFIX, jwt);
|
||||
|
||||
let suffix = create_default_worker_suffix(&HOSTNAME);
|
||||
Ok(build_agent_http_client(&suffix, &token, base_internal_url))
|
||||
}
|
||||
|
||||
async fn send_delayed_killpill(tx: &KillpillSender, mut max_delay_secs: u64, context: &str) {
|
||||
if max_delay_secs == 0 {
|
||||
max_delay_secs = 1;
|
||||
|
||||
@@ -174,7 +174,7 @@ websocket_trigger: path(char), url(char), script_path(char), is_flow(bool), work
|
||||
windmill_migrations: name(text), created_at(ts)
|
||||
worker_group_job_stats: hour(bigint), worker_group(text), script_lang(char), workspace_id(char), job_count(int), total_duration_ms(bigint)
|
||||
FK: (workspace_id) -> workspace(id)
|
||||
worker_ping: worker(char), worker_instance(char), ping_at(ts), started_at(ts), ip(char), jobs_executed(int), custom_tags(text[]), worker_group(char), dedicated_worker(char), wm_version(char), current_job_id(uuid), current_job_workspace_id(char), vcpus(bigint), memory(bigint), occupancy_rate(float), memory_usage(bigint), wm_memory_usage(bigint), occupancy_rate_15s(float), occupancy_rate_5m(float), occupancy_rate_30m(float), job_isolation(text), dedicated_workers(text[]), native_mode(bool), uses_batch_http_pull(bool)
|
||||
worker_ping: worker(char), worker_instance(char), ping_at(ts), started_at(ts), ip(char), jobs_executed(int), custom_tags(text[]), worker_group(char), dedicated_worker(char), wm_version(char), current_job_id(uuid), current_job_workspace_id(char), vcpus(bigint), memory(bigint), occupancy_rate(float), memory_usage(bigint), wm_memory_usage(bigint), occupancy_rate_15s(float), occupancy_rate_5m(float), occupancy_rate_30m(float), job_isolation(text), dedicated_workers(text[])
|
||||
workspace: id(char), name(char), owner(char), deleted(bool), premium(bool), parent_workspace_id(char)
|
||||
FK: (parent_workspace_id) -> workspace(id)
|
||||
workspace_dependencies: id(bigint), name(char), content(text), language(script_lang), description(text), archived(bool), workspace_id(char), created_at(ts)
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
use sqlx::postgres::Postgres;
|
||||
use sqlx::Pool;
|
||||
use uuid::Uuid;
|
||||
use windmill_common::jobs::{JobPayload, RawCode};
|
||||
use windmill_common::scripts::ScriptLang;
|
||||
use windmill_test_utils::*;
|
||||
@@ -1449,240 +1448,3 @@ export function main() { return { a, b }; }
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Codebase Mode Tests
|
||||
// ============================================================================
|
||||
|
||||
/// Create a TAR archive in memory containing a single `main.js` file.
|
||||
fn create_codebase_tar(main_js_content: &str) -> Vec<u8> {
|
||||
let mut builder = tar::Builder::new(Vec::new());
|
||||
let content = main_js_content.as_bytes();
|
||||
let mut header = tar::Header::new_gnu();
|
||||
header.set_path("main.js").unwrap();
|
||||
header.set_size(content.len() as u64);
|
||||
header.set_mode(0o644);
|
||||
header.set_cksum();
|
||||
builder.append(&header, content).unwrap();
|
||||
builder.into_inner().unwrap()
|
||||
}
|
||||
|
||||
/// Place a TAR codebase at the expected cache path for the given job ID and hash.
|
||||
fn place_codebase_in_cache(job_id: &Uuid, tar_bytes: &[u8], is_esm: bool) {
|
||||
let codebase_id = if is_esm {
|
||||
format!("{}.esm.tar", job_id)
|
||||
} else {
|
||||
format!("{}.tar", job_id)
|
||||
};
|
||||
let bundle_path = format!("script_bundle/test-workspace/{}", codebase_id);
|
||||
let cache_path = format!(
|
||||
"{}/{}.tar",
|
||||
*windmill_common::worker::ROOT_CACHE_NOMOUNT_DIR,
|
||||
bundle_path,
|
||||
);
|
||||
let parent = std::path::Path::new(&cache_path).parent().unwrap();
|
||||
std::fs::create_dir_all(parent).unwrap();
|
||||
std::fs::write(&cache_path, tar_bytes).unwrap();
|
||||
}
|
||||
|
||||
#[sqlx::test(fixtures("base"))]
|
||||
async fn test_cjs_codebase_tar(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
initialize_tracing().await;
|
||||
let server = ApiServer::start(db.clone()).await?;
|
||||
let port = server.addr.port();
|
||||
|
||||
let main_js = r#"
|
||||
module.exports.main = function() {
|
||||
return "cjs codebase ok";
|
||||
};
|
||||
"#;
|
||||
let inner_content = r#"export function main() { return "cjs codebase ok"; }"#;
|
||||
|
||||
let job_id = Uuid::new_v4();
|
||||
let tar_bytes = create_codebase_tar(main_js);
|
||||
place_codebase_in_cache(&job_id, &tar_bytes, false);
|
||||
|
||||
let job = JobPayload::Code(RawCode {
|
||||
hash: Some(-43), // PREVIEW_IS_TAR_CODEBASE_HASH
|
||||
content: inner_content.to_string(),
|
||||
path: None,
|
||||
language: ScriptLang::Bun,
|
||||
lock: None,
|
||||
concurrency_settings: Default::default(),
|
||||
debouncing_settings: Default::default(),
|
||||
cache_ttl: None,
|
||||
cache_ignore_s3_path: None,
|
||||
dedicated_worker: None,
|
||||
});
|
||||
|
||||
let result = RunJob::from(job)
|
||||
.job_id(job_id)
|
||||
.run_until_complete(&db, false, port)
|
||||
.await
|
||||
.json_result()
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(result, serde_json::json!("cjs codebase ok"));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[sqlx::test(fixtures("base"))]
|
||||
async fn test_esm_codebase_tar(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
initialize_tracing().await;
|
||||
let server = ApiServer::start(db.clone()).await?;
|
||||
let port = server.addr.port();
|
||||
|
||||
let main_js = r#"
|
||||
export function main() {
|
||||
return "esm codebase ok";
|
||||
}
|
||||
"#;
|
||||
let inner_content = r#"export function main() { return "esm codebase ok"; }"#;
|
||||
|
||||
let job_id = Uuid::new_v4();
|
||||
let tar_bytes = create_codebase_tar(main_js);
|
||||
place_codebase_in_cache(&job_id, &tar_bytes, true);
|
||||
|
||||
let job = JobPayload::Code(RawCode {
|
||||
hash: Some(-45), // PREVIEW_IS_TAR_ESM_CODEBASE_HASH
|
||||
content: inner_content.to_string(),
|
||||
path: None,
|
||||
language: ScriptLang::Bun,
|
||||
lock: None,
|
||||
concurrency_settings: Default::default(),
|
||||
debouncing_settings: Default::default(),
|
||||
cache_ttl: None,
|
||||
cache_ignore_s3_path: None,
|
||||
dedicated_worker: None,
|
||||
});
|
||||
|
||||
let result = RunJob::from(job)
|
||||
.job_id(job_id)
|
||||
.run_until_complete(&db, false, port)
|
||||
.await
|
||||
.json_result()
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(result, serde_json::json!("esm codebase ok"));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[sqlx::test(fixtures("base"))]
|
||||
async fn test_cjs_codebase_tar_nsjail(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
if std::process::Command::new("nsjail")
|
||||
.arg("--help")
|
||||
.output()
|
||||
.is_err()
|
||||
{
|
||||
eprintln!("nsjail not found, skipping test");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
initialize_tracing().await;
|
||||
let server = ApiServer::start(db.clone()).await?;
|
||||
let port = server.addr.port();
|
||||
|
||||
let main_js = r#"
|
||||
module.exports.main = function() {
|
||||
return "cjs nsjail ok";
|
||||
};
|
||||
"#;
|
||||
let inner_content = r#"export function main() { return "cjs nsjail ok"; }"#;
|
||||
|
||||
let job_id = Uuid::new_v4();
|
||||
let tar_bytes = create_codebase_tar(main_js);
|
||||
place_codebase_in_cache(&job_id, &tar_bytes, false);
|
||||
|
||||
let job = JobPayload::Code(RawCode {
|
||||
hash: Some(-43),
|
||||
content: inner_content.to_string(),
|
||||
path: None,
|
||||
language: ScriptLang::Bun,
|
||||
lock: None,
|
||||
concurrency_settings: Default::default(),
|
||||
debouncing_settings: Default::default(),
|
||||
cache_ttl: None,
|
||||
cache_ignore_s3_path: None,
|
||||
dedicated_worker: None,
|
||||
});
|
||||
|
||||
use std::sync::atomic::Ordering;
|
||||
windmill_worker::JOB_ISOLATION.store(
|
||||
windmill_worker::JobIsolationLevel::NsjailSandboxing as u8,
|
||||
Ordering::Relaxed,
|
||||
);
|
||||
|
||||
let result = RunJob::from(job)
|
||||
.job_id(job_id)
|
||||
.run_until_complete(&db, false, port)
|
||||
.await;
|
||||
|
||||
windmill_worker::JOB_ISOLATION.store(
|
||||
windmill_worker::JobIsolationLevel::Undefined as u8,
|
||||
Ordering::Relaxed,
|
||||
);
|
||||
|
||||
let json = result.json_result().unwrap();
|
||||
assert_eq!(json, serde_json::json!("cjs nsjail ok"));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[sqlx::test(fixtures("base"))]
|
||||
async fn test_esm_codebase_tar_nsjail(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
if std::process::Command::new("nsjail")
|
||||
.arg("--help")
|
||||
.output()
|
||||
.is_err()
|
||||
{
|
||||
eprintln!("nsjail not found, skipping test");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
initialize_tracing().await;
|
||||
let server = ApiServer::start(db.clone()).await?;
|
||||
let port = server.addr.port();
|
||||
|
||||
let main_js = r#"
|
||||
export function main() {
|
||||
return "esm nsjail ok";
|
||||
}
|
||||
"#;
|
||||
let inner_content = r#"export function main() { return "esm nsjail ok"; }"#;
|
||||
|
||||
let job_id = Uuid::new_v4();
|
||||
let tar_bytes = create_codebase_tar(main_js);
|
||||
place_codebase_in_cache(&job_id, &tar_bytes, true);
|
||||
|
||||
let job = JobPayload::Code(RawCode {
|
||||
hash: Some(-45),
|
||||
content: inner_content.to_string(),
|
||||
path: None,
|
||||
language: ScriptLang::Bun,
|
||||
lock: None,
|
||||
concurrency_settings: Default::default(),
|
||||
debouncing_settings: Default::default(),
|
||||
cache_ttl: None,
|
||||
cache_ignore_s3_path: None,
|
||||
dedicated_worker: None,
|
||||
});
|
||||
|
||||
use std::sync::atomic::Ordering;
|
||||
windmill_worker::JOB_ISOLATION.store(
|
||||
windmill_worker::JobIsolationLevel::NsjailSandboxing as u8,
|
||||
Ordering::Relaxed,
|
||||
);
|
||||
|
||||
let result = RunJob::from(job)
|
||||
.job_id(job_id)
|
||||
.run_until_complete(&db, false, port)
|
||||
.await;
|
||||
|
||||
windmill_worker::JOB_ISOLATION.store(
|
||||
windmill_worker::JobIsolationLevel::Undefined as u8,
|
||||
Ordering::Relaxed,
|
||||
);
|
||||
|
||||
let json = result.json_result().unwrap();
|
||||
assert_eq!(json, serde_json::json!("esm nsjail ok"));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -241,7 +241,6 @@ fn spawn_workers(
|
||||
rx,
|
||||
tx2,
|
||||
&base_internal_url,
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
};
|
||||
|
||||
@@ -19,10 +19,7 @@ use windmill_common::DB;
|
||||
use axum::Router;
|
||||
|
||||
#[cfg(not(feature = "private"))]
|
||||
pub fn global_service(
|
||||
_job_completed_tx: windmill_worker::JobCompletedSender,
|
||||
_batch_buffer: Option<()>,
|
||||
) -> Router {
|
||||
pub fn global_service(_job_completed_tx: windmill_worker::JobCompletedSender) -> Router {
|
||||
Router::new()
|
||||
}
|
||||
|
||||
@@ -34,7 +31,6 @@ pub fn workspaced_service(
|
||||
Router,
|
||||
Vec<tokio::task::JoinHandle<()>>,
|
||||
Option<windmill_worker::JobCompletedSender>,
|
||||
Option<()>,
|
||||
) {
|
||||
use windmill_common::worker::Connection;
|
||||
use windmill_worker::JobCompletedSender;
|
||||
@@ -44,7 +40,7 @@ pub fn workspaced_service(
|
||||
|
||||
let router = Router::new();
|
||||
|
||||
(router, vec![], Some(job_completed_tx), None)
|
||||
(router, vec![], Some(job_completed_tx))
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "private"))]
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
openapi: "3.0.3"
|
||||
|
||||
info:
|
||||
version: 1.651.1
|
||||
version: 1.650.0
|
||||
title: Windmill API
|
||||
|
||||
contact:
|
||||
|
||||
@@ -5416,12 +5416,12 @@ async fn add_batch_jobs(
|
||||
if dedicated_worker && path.is_some() {
|
||||
windmill_common::worker::dedicated_worker_tag(&w_id, &path.clone().unwrap())
|
||||
} else {
|
||||
language.as_worker_tag(false).to_string()
|
||||
format!("{}", language.as_str())
|
||||
}
|
||||
} else if let Some(tag) = batch_info.tag {
|
||||
tag
|
||||
} else {
|
||||
language.as_worker_tag(false).to_string()
|
||||
format!("{}", language.as_str())
|
||||
};
|
||||
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
|
||||
@@ -493,16 +493,12 @@ pub async fn run_server(
|
||||
};
|
||||
|
||||
#[cfg(feature = "agent_worker_server")]
|
||||
let (
|
||||
agent_workers_router,
|
||||
agent_workers_bg_processor,
|
||||
agent_workers_job_completed_tx,
|
||||
batch_buffer,
|
||||
) = if server_mode {
|
||||
windmill_api_agent_workers::workspaced_service(db.clone(), _base_internal_url.clone())
|
||||
} else {
|
||||
(Router::new(), vec![], None, None)
|
||||
};
|
||||
let (agent_workers_router, agent_workers_bg_processor, agent_workers_job_completed_tx) =
|
||||
if server_mode {
|
||||
windmill_api_agent_workers::workspaced_service(db.clone(), _base_internal_url.clone())
|
||||
} else {
|
||||
(Router::new(), vec![], None)
|
||||
};
|
||||
|
||||
#[cfg(feature = "agent_worker_server")]
|
||||
let agent_cache = Arc::new(AgentCache::new());
|
||||
@@ -688,7 +684,6 @@ pub async fn run_server(
|
||||
{
|
||||
windmill_api_agent_workers::global_service(
|
||||
agent_workers_job_completed_tx,
|
||||
batch_buffer.clone(),
|
||||
)
|
||||
.layer(Extension(agent_cache.clone()))
|
||||
} else {
|
||||
|
||||
@@ -64,10 +64,9 @@ impl McpBackend for WindmillBackend {
|
||||
auth: &ApiAuthed,
|
||||
workspace_id: &str,
|
||||
favorites_only: bool,
|
||||
path_prefix: Option<&str>,
|
||||
) -> BackendResult<Vec<ScriptInfo>> {
|
||||
let scope_type = if favorites_only { "favorites" } else { "all" };
|
||||
get_items::<ScriptInfo>(&self.user_db, auth, workspace_id, scope_type, "script", path_prefix)
|
||||
get_items::<ScriptInfo>(&self.user_db, auth, workspace_id, scope_type, "script")
|
||||
.await
|
||||
.map_err(|e| ErrorData::internal_error(e.message, None))
|
||||
}
|
||||
@@ -77,10 +76,9 @@ impl McpBackend for WindmillBackend {
|
||||
auth: &ApiAuthed,
|
||||
workspace_id: &str,
|
||||
favorites_only: bool,
|
||||
path_prefix: Option<&str>,
|
||||
) -> BackendResult<Vec<FlowInfo>> {
|
||||
let scope_type = if favorites_only { "favorites" } else { "all" };
|
||||
get_items::<FlowInfo>(&self.user_db, auth, workspace_id, scope_type, "flow", path_prefix)
|
||||
get_items::<FlowInfo>(&self.user_db, auth, workspace_id, scope_type, "flow")
|
||||
.await
|
||||
.map_err(|e| ErrorData::internal_error(e.message, None))
|
||||
}
|
||||
|
||||
@@ -136,7 +136,6 @@ pub async fn get_items<T: for<'a> sqlx::FromRow<'a, sqlx::postgres::PgRow> + Sen
|
||||
workspace_id: &str,
|
||||
scope_type: &str,
|
||||
item_type: &str,
|
||||
path_prefix: Option<&str>,
|
||||
) -> Result<Vec<T>, ErrorData> {
|
||||
let mut sqlb = SqlBuilder::select_from(&format!("{} as o", item_type));
|
||||
let fields = vec!["o.path", "o.summary", "o.description", "o.schema"];
|
||||
@@ -154,11 +153,6 @@ pub async fn get_items<T: for<'a> sqlx::FromRow<'a, sqlx::postgres::PgRow> + Sen
|
||||
sqlb.and_where("(o.no_main_func IS NOT TRUE OR o.no_main_func IS NULL)");
|
||||
}
|
||||
|
||||
if let Some(prefix) = path_prefix {
|
||||
let escaped = prefix.replace('\\', "\\\\").replace('%', "\\%").replace('_', "\\_");
|
||||
sqlb.and_where("o.path LIKE ? ESCAPE '\\'".bind(&format!("{}%", escaped)));
|
||||
}
|
||||
|
||||
sqlb.order_by(
|
||||
if item_type == "flow" {
|
||||
"o.edited_at"
|
||||
|
||||
@@ -288,10 +288,6 @@ pub fn is_native_mode_from_env() -> bool {
|
||||
/// Use this for hot-path checks (e.g. per-job dispatch) to avoid read-locking WORKER_CONFIG.
|
||||
pub static NATIVE_MODE_RESOLVED: AtomicBool = AtomicBool::new(false);
|
||||
|
||||
/// Whether this worker uses HTTP batch pull (set at startup in main.rs).
|
||||
/// Reported in worker_ping so the server knows which native workers to batch-pull for.
|
||||
pub static USES_BATCH_HTTP_PULL: AtomicBool = AtomicBool::new(false);
|
||||
|
||||
pub static MIN_VERSION_IS_LATEST: AtomicBool = AtomicBool::new(false);
|
||||
#[derive(Clone)]
|
||||
pub struct HttpClient {
|
||||
@@ -520,62 +516,6 @@ pub fn make_pull_query(tags: &[String]) -> String {
|
||||
query
|
||||
}
|
||||
|
||||
pub fn make_batch_pull_query(tags: &[String], limit: u32) -> String {
|
||||
format_batch_pull_query(format!(
|
||||
"SELECT id
|
||||
FROM v2_job_queue
|
||||
WHERE running = false AND tag IN ({}) AND scheduled_for <= now()
|
||||
ORDER BY priority DESC NULLS LAST, scheduled_for
|
||||
FOR UPDATE SKIP LOCKED
|
||||
LIMIT {limit}",
|
||||
tags.iter().map(|x| format!("'{x}'")).join(", ")
|
||||
))
|
||||
}
|
||||
|
||||
fn format_batch_pull_query(peek: String) -> String {
|
||||
// Optimizations vs single-row format_pull_query:
|
||||
// 1. ANY(ARRAY(SELECT ...)) instead of IN (SELECT ...) — forces PG to materialize IDs
|
||||
// into an array, enabling Bitmap Index Scan instead of Hash Semi Join / Nested Loop
|
||||
// 2. r CTE chains off q (not peek) — only updates runtime for actually-locked rows,
|
||||
// avoids re-scanning peek
|
||||
// 3. No separate j CTE — join v2_job directly in final SELECT off q's IDs
|
||||
format!(
|
||||
"WITH peek AS (
|
||||
{}
|
||||
), q AS NOT MATERIALIZED (
|
||||
UPDATE v2_job_queue SET
|
||||
running = true,
|
||||
started_at = coalesce(started_at, now()),
|
||||
suspend_until = null,
|
||||
worker = $1
|
||||
WHERE id = ANY(ARRAY(SELECT id FROM peek))
|
||||
RETURNING
|
||||
id, started_at, scheduled_for,
|
||||
canceled_by, canceled_reason, worker, cache_ignore_s3_path, runnable_settings_handle
|
||||
), r AS NOT MATERIALIZED (
|
||||
UPDATE v2_job_runtime SET
|
||||
ping = now()
|
||||
WHERE id = ANY(ARRAY(SELECT id FROM q))
|
||||
) SELECT j.id, j.workspace_id, j.parent_job, j.created_by, q.started_at, q.scheduled_for,
|
||||
j.runnable_id, j.runnable_path, j.args, q.canceled_by,
|
||||
q.canceled_reason, j.kind, j.trigger, j.trigger_kind, j.permissioned_as,
|
||||
f.flow_status, j.script_lang,
|
||||
j.same_worker, j.pre_run_error, j.visible_to_owner,
|
||||
j.tag, j.concurrent_limit, j.concurrency_time_window_s, j.flow_innermost_root_job, j.root_job,
|
||||
j.timeout, j.flow_step_id, j.cache_ttl, q.cache_ignore_s3_path, q.runnable_settings_handle, j.priority, j.raw_code, j.raw_lock, j.raw_flow,
|
||||
j.script_entrypoint_override, j.preprocessed, COALESCE(pj.runnable_path, j.args->>'_FLOW_PATH') as parent_runnable_path,
|
||||
COALESCE(p.email, j.permissioned_as_email) as permissioned_as_email, p.username as permissioned_as_username, p.is_admin as permissioned_as_is_admin,
|
||||
p.is_operator as permissioned_as_is_operator, p.groups as permissioned_as_groups, p.folders as permissioned_as_folders, p.end_user_email as permissioned_as_end_user_email
|
||||
FROM q
|
||||
JOIN v2_job j ON q.id = j.id
|
||||
LEFT JOIN v2_job_status f ON f.id = q.id
|
||||
LEFT JOIN job_perms p ON p.job_id = q.id
|
||||
LEFT JOIN v2_job pj ON j.parent_job = pj.id
|
||||
",
|
||||
peek
|
||||
)
|
||||
}
|
||||
|
||||
pub async fn store_pull_query(wc: &WorkerConfig) {
|
||||
let mut queries = vec![];
|
||||
for tags in wc.priority_tags_sorted.iter() {
|
||||
@@ -1254,8 +1194,6 @@ pub struct Ping {
|
||||
pub occupancy_rate_30m: Option<f32>,
|
||||
pub job_isolation: Option<String>,
|
||||
pub native_mode: Option<bool>,
|
||||
#[serde(default)]
|
||||
pub uses_batch_http_pull: Option<bool>,
|
||||
pub ping_type: PingType,
|
||||
}
|
||||
pub async fn update_ping_http(
|
||||
@@ -1280,7 +1218,6 @@ pub async fn update_ping_http(
|
||||
insert_ping.occupancy_rate_5m,
|
||||
insert_ping.occupancy_rate_30m,
|
||||
insert_ping.native_mode.unwrap_or(false),
|
||||
insert_ping.uses_batch_http_pull.unwrap_or(false),
|
||||
db,
|
||||
)
|
||||
.await?
|
||||
@@ -1308,7 +1245,6 @@ pub async fn update_ping_http(
|
||||
insert_ping.memory,
|
||||
insert_ping.job_isolation,
|
||||
insert_ping.native_mode.unwrap_or(false),
|
||||
insert_ping.uses_batch_http_pull.unwrap_or(false),
|
||||
db,
|
||||
)
|
||||
.await?;
|
||||
@@ -1441,12 +1377,11 @@ pub async fn insert_ping_query(
|
||||
memory: Option<i64>,
|
||||
job_isolation: Option<String>,
|
||||
native_mode: bool,
|
||||
uses_batch_http_pull: bool,
|
||||
db: &DB,
|
||||
) -> anyhow::Result<()> {
|
||||
sqlx::query!(
|
||||
"INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, dedicated_workers, wm_version, vcpus, memory, job_isolation, native_mode, uses_batch_http_pull) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) ON CONFLICT (worker)
|
||||
DO UPDATE set ip = EXCLUDED.ip, custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group, dedicated_workers = EXCLUDED.dedicated_workers, native_mode = EXCLUDED.native_mode, uses_batch_http_pull = EXCLUDED.uses_batch_http_pull",
|
||||
"INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, dedicated_workers, wm_version, vcpus, memory, job_isolation, native_mode) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) ON CONFLICT (worker)
|
||||
DO UPDATE set ip = EXCLUDED.ip, custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group, dedicated_workers = EXCLUDED.dedicated_workers, native_mode = EXCLUDED.native_mode",
|
||||
worker_instance,
|
||||
worker_name,
|
||||
ip,
|
||||
@@ -1459,7 +1394,6 @@ pub async fn insert_ping_query(
|
||||
memory,
|
||||
job_isolation.as_deref(),
|
||||
native_mode,
|
||||
uses_batch_http_pull,
|
||||
)
|
||||
.execute(db)
|
||||
.await?;
|
||||
@@ -1551,13 +1485,12 @@ pub async fn update_worker_ping_main_loop_query(
|
||||
occupancy_rate_5m: Option<f32>,
|
||||
occupancy_rate_30m: Option<f32>,
|
||||
native_mode: bool,
|
||||
uses_batch_http_pull: bool,
|
||||
db: &DB,
|
||||
) -> anyhow::Result<()> {
|
||||
timeout(Duration::from_secs(10), sqlx::query!(
|
||||
"UPDATE worker_ping SET ping_at = now(), jobs_executed = $1, custom_tags = $2,
|
||||
occupancy_rate = $3, memory_usage = $4, wm_memory_usage = $5, vcpus = COALESCE($7, vcpus),
|
||||
memory = COALESCE($8, memory), occupancy_rate_15s = $9, occupancy_rate_5m = $10, occupancy_rate_30m = $11, native_mode = $12, uses_batch_http_pull = $13 WHERE worker = $6",
|
||||
memory = COALESCE($8, memory), occupancy_rate_15s = $9, occupancy_rate_5m = $10, occupancy_rate_30m = $11, native_mode = $12 WHERE worker = $6",
|
||||
jobs_executed,
|
||||
tags,
|
||||
occupancy_rate,
|
||||
@@ -1570,7 +1503,6 @@ pub async fn update_worker_ping_main_loop_query(
|
||||
occupancy_rate_5m,
|
||||
occupancy_rate_30m,
|
||||
native_mode,
|
||||
uses_batch_http_pull,
|
||||
)
|
||||
.execute(db))
|
||||
.await??;
|
||||
|
||||
@@ -11,8 +11,6 @@ pub mod types;
|
||||
pub use schema::convert_schema_to_schema_type;
|
||||
pub use scope::{is_resource_allowed, parse_mcp_scopes, McpScopeConfig};
|
||||
pub use transform::{
|
||||
apply_key_transformation, extract_hub_version_id_from_hashed,
|
||||
extract_path_prefix_from_hashed, parse_tool_prefix, reverse_transform, reverse_transform_key,
|
||||
transform_hub_path, transform_path,
|
||||
apply_key_transformation, reverse_transform, reverse_transform_key, transform_path,
|
||||
};
|
||||
pub use types::*;
|
||||
|
||||
@@ -4,15 +4,9 @@
|
||||
//! to make them compatible with MCP tool naming requirements.
|
||||
|
||||
use super::types::SchemaType;
|
||||
use windmill_common::utils::calculate_hash;
|
||||
|
||||
/// Max tool name length. The MCP spec allows 64 chars, but some clients
|
||||
/// (e.g. Cursor) prepend the server name to the tool name, so we use 40
|
||||
/// to leave room for that prefix.
|
||||
const MAX_PATH_LENGTH: usize = 40;
|
||||
|
||||
/// Length of the SHA256 hash suffix used for hashed names
|
||||
const HASH_LEN: usize = 16;
|
||||
/// MCP clients do not allow names longer than 60 characters
|
||||
const MAX_PATH_LENGTH: usize = 60;
|
||||
|
||||
/// Transform the path for workspace scripts/flows
|
||||
///
|
||||
@@ -20,133 +14,19 @@ const HASH_LEN: usize = 16;
|
||||
/// path with the type prefix. This is used when listing, because we can't
|
||||
/// have names with slashes. Because we replace slashes with underscores,
|
||||
/// we also need to escape underscores.
|
||||
///
|
||||
/// For short names (≤40 chars): `s-{escaped_path}` or `f-{escaped_path}`
|
||||
/// For long names (>40 chars): `S-{escaped[:22]}{sha256[:16]}` or `F-{escaped[:22]}{sha256[:16]}`
|
||||
///
|
||||
/// The uppercase prefix signals that the name is hashed.
|
||||
pub fn transform_path(path: &str, type_str: &str) -> String {
|
||||
let escaped_path = path.replace('_', "__").replace('/', "_");
|
||||
let prefix_char = &type_str[..1];
|
||||
let short_name = format!("{}-{}", prefix_char, escaped_path);
|
||||
|
||||
if short_name.len() <= MAX_PATH_LENGTH {
|
||||
return short_name;
|
||||
// first letter of type_str is used as prefix, only one letter to avoid reaching 60 char name limit
|
||||
let transformed_path = format!("{}-{}", &type_str[..1], escaped_path);
|
||||
if transformed_path.len() > MAX_PATH_LENGTH {
|
||||
let suffix = "_TRUNC";
|
||||
return format!(
|
||||
"{}{}",
|
||||
&transformed_path[..MAX_PATH_LENGTH - suffix.len()],
|
||||
suffix
|
||||
);
|
||||
}
|
||||
|
||||
let upper_prefix = prefix_char.to_uppercase();
|
||||
// Layout: "{Upper}-" (2 chars) + prefix_body (22 chars) + hash (16 chars) = 40
|
||||
let prefix_body_len = MAX_PATH_LENGTH - 2 - HASH_LEN;
|
||||
let hash = calculate_hash(&short_name);
|
||||
let hash_suffix = &hash[..HASH_LEN];
|
||||
let truncated = truncate_to_char_boundary(&escaped_path, prefix_body_len);
|
||||
format!("{}-{}{}", upper_prefix, truncated, hash_suffix)
|
||||
}
|
||||
|
||||
/// Transform the path for hub scripts
|
||||
///
|
||||
/// For short names (≤40 chars): `hs-{id}-{summary}`
|
||||
/// For long names (>40 chars): `Hs-{id}-{summary[:N]}{sha256[:16]}`
|
||||
pub fn transform_hub_path(version_id: u64, summary: &str) -> String {
|
||||
let escaped_summary = summary.replace(' ', "_");
|
||||
let short_name = format!("hs-{}-{}", version_id, escaped_summary);
|
||||
|
||||
if short_name.len() <= MAX_PATH_LENGTH {
|
||||
return short_name;
|
||||
}
|
||||
|
||||
let hash = calculate_hash(&short_name);
|
||||
let hash_suffix = &hash[..HASH_LEN];
|
||||
// "Hs-{id}-" prefix, then fill remaining with summary + hash
|
||||
let fixed_prefix = format!("Hs-{}-", version_id);
|
||||
let available = MAX_PATH_LENGTH - fixed_prefix.len() - HASH_LEN;
|
||||
let truncated_summary = truncate_to_char_boundary(&escaped_summary, available);
|
||||
format!("{}{}{}", fixed_prefix, truncated_summary, hash_suffix)
|
||||
}
|
||||
|
||||
/// Parse the prefix of any tool name (both short and hashed).
|
||||
/// Returns `(type_str, is_hub, is_hashed)`.
|
||||
/// Hashed names use an uppercase first character as the signal.
|
||||
pub fn parse_tool_prefix(name: &str) -> Result<(&str, bool, bool), String> {
|
||||
let is_hashed = name.chars().next().map(|c| c.is_ascii_uppercase()).unwrap_or(false);
|
||||
let lower = name.to_ascii_lowercase();
|
||||
let (type_str, is_hub) = if lower.starts_with("hs-") {
|
||||
("script", true)
|
||||
} else if lower.starts_with("s-") {
|
||||
("script", false)
|
||||
} else if lower.starts_with("f-") {
|
||||
("flow", false)
|
||||
} else {
|
||||
return Err(format!("Invalid tool name prefix: {}", name));
|
||||
};
|
||||
Ok((type_str, is_hub, is_hashed))
|
||||
}
|
||||
|
||||
/// Extract the hub version_id from a hashed hub script name like `Hs-{id}-...`
|
||||
pub fn extract_hub_version_id_from_hashed(name: &str) -> Result<String, String> {
|
||||
let rest = name
|
||||
.strip_prefix("Hs-")
|
||||
.ok_or_else(|| format!("Not a hashed hub name: {}", name))?;
|
||||
let id = rest
|
||||
.split('-')
|
||||
.next()
|
||||
.ok_or_else(|| format!("No version_id in hashed hub name: {}", name))?;
|
||||
if id.is_empty() {
|
||||
return Err(format!("Empty version_id in hashed hub name: {}", name));
|
||||
}
|
||||
Ok(id.to_string())
|
||||
}
|
||||
|
||||
/// Extract a safe original-path prefix from a hashed tool name.
|
||||
///
|
||||
/// Given `S-u_admin_engineering__te<hash16>`, extracts the escaped prefix between
|
||||
/// the type prefix (`S-`, `F-`, or `Hs-`) and the hash, un-escapes it, and
|
||||
/// returns a prefix suitable for `WHERE path LIKE '{prefix}%'`.
|
||||
///
|
||||
/// Returns `None` if the name is too short or has an unrecognized prefix.
|
||||
pub fn extract_path_prefix_from_hashed(name: &str) -> Option<String> {
|
||||
let prefix_len = if name.starts_with("Hs-") {
|
||||
3
|
||||
} else if name.starts_with("S-") || name.starts_with("F-") {
|
||||
2
|
||||
} else {
|
||||
return None;
|
||||
};
|
||||
if name.len() <= prefix_len + HASH_LEN {
|
||||
return None;
|
||||
}
|
||||
let escaped_prefix = &name[prefix_len..name.len() - HASH_LEN];
|
||||
if escaped_prefix.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
// Strip trailing underscores — they may be half of a `__` pair split by truncation
|
||||
let trimmed = escaped_prefix.trim_end_matches('_');
|
||||
if trimmed.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
Some(unescape_path(trimmed))
|
||||
}
|
||||
|
||||
/// Un-escape a mangled path segment: `__` → `_`, standalone `_` → `/`.
|
||||
fn unescape_path(s: &str) -> String {
|
||||
const TEMP_PLACEHOLDER: &str = "@@UNDERSCORE@@";
|
||||
s.replace("__", TEMP_PLACEHOLDER)
|
||||
.replace('_', "/")
|
||||
.replace(TEMP_PLACEHOLDER, "_")
|
||||
}
|
||||
|
||||
/// Truncate a string to at most `max_len` bytes, ensuring we don't split a UTF-8 character.
|
||||
fn truncate_to_char_boundary(s: &str, max_len: usize) -> &str {
|
||||
if s.len() <= max_len {
|
||||
return s;
|
||||
}
|
||||
let mut end = max_len;
|
||||
while end > 0 && !s.is_char_boundary(end) {
|
||||
end -= 1;
|
||||
}
|
||||
&s[..end]
|
||||
transformed_path
|
||||
}
|
||||
|
||||
/// Reverse the transformation of a path
|
||||
@@ -158,22 +38,25 @@ fn truncate_to_char_boundary(s: &str, max_len: usize) -> &str {
|
||||
/// This is used in call_tool to get the original path, and the type of the item.
|
||||
///
|
||||
/// Returns: (type, original_path, is_hub)
|
||||
///
|
||||
/// Note: This only works for non-hashed (short) names. Hashed names must be
|
||||
/// resolved via `parse_tool_prefix` + path enumeration in the runner.
|
||||
pub fn reverse_transform(transformed_path: &str) -> Result<(&str, String, bool), String> {
|
||||
let (type_str, is_hub, is_hashed) = parse_tool_prefix(transformed_path)?;
|
||||
let is_hub = transformed_path.starts_with("h");
|
||||
let transformed_path = if is_hub {
|
||||
transformed_path[1..].to_string()
|
||||
} else {
|
||||
transformed_path.to_string()
|
||||
};
|
||||
let type_str = if transformed_path.starts_with("s-") {
|
||||
"script"
|
||||
} else if transformed_path.starts_with("f-") {
|
||||
"flow"
|
||||
} else {
|
||||
return Err(format!(
|
||||
"Invalid prefix in transformed path: {}",
|
||||
transformed_path
|
||||
));
|
||||
};
|
||||
|
||||
if is_hashed {
|
||||
return Err(
|
||||
"Hashed names cannot be reverse-transformed directly; use path enumeration instead"
|
||||
.to_string(),
|
||||
);
|
||||
}
|
||||
|
||||
// Strip the prefix: "hs-" (3 chars) for hub, "s-"/"f-" (2 chars) for others
|
||||
let prefix_len = if is_hub { 3 } else { 2 };
|
||||
let mangled_path = &transformed_path[prefix_len..];
|
||||
let mangled_path = &transformed_path[2..];
|
||||
|
||||
let original_path = if is_hub {
|
||||
let parts = mangled_path.split("-").collect::<Vec<&str>>();
|
||||
@@ -182,7 +65,11 @@ pub fn reverse_transform(transformed_path: &str) -> Result<(&str, String, bool),
|
||||
}
|
||||
parts[0].to_string()
|
||||
} else {
|
||||
unescape_path(mangled_path)
|
||||
const TEMP_PLACEHOLDER: &str = "@@UNDERSCORE@@";
|
||||
mangled_path
|
||||
.replace("__", TEMP_PLACEHOLDER)
|
||||
.replace('_', "/")
|
||||
.replace(TEMP_PLACEHOLDER, "_")
|
||||
};
|
||||
|
||||
Ok((type_str, original_path, is_hub))
|
||||
@@ -210,13 +97,16 @@ pub fn reverse_transform_key(transformed_key: &str, schema_obj: &Option<SchemaTy
|
||||
let schema_obj = match schema_obj {
|
||||
Some(s) => s,
|
||||
None => {
|
||||
// No schema available, return the key as is (best guess)
|
||||
return transformed_key.to_string();
|
||||
}
|
||||
};
|
||||
|
||||
for original_key_in_schema in schema_obj.properties.keys() {
|
||||
// Apply the SAME forward transformation to the schema key
|
||||
let potential_transformed_key = apply_key_transformation(original_key_in_schema);
|
||||
|
||||
// If it matches the key we received, we found the likely original
|
||||
if potential_transformed_key == transformed_key {
|
||||
return original_key_in_schema.clone();
|
||||
}
|
||||
@@ -230,7 +120,7 @@ mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_transform_path_short() {
|
||||
fn test_transform_path() {
|
||||
assert_eq!(
|
||||
transform_path("u/admin/script", "script"),
|
||||
"s-u_admin_script"
|
||||
@@ -240,108 +130,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_transform_path_long_is_hashed() {
|
||||
let long_path = "u/engineering/team/automation/very_long_script_name_that_exceeds_limit";
|
||||
let result = transform_path(long_path, "script");
|
||||
assert_eq!(result.len(), MAX_PATH_LENGTH);
|
||||
assert!(result.starts_with("S-"));
|
||||
let (_, _, is_hashed) = parse_tool_prefix(&result).unwrap();
|
||||
assert!(is_hashed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_transform_path_long_flow_is_hashed() {
|
||||
let long_path = "f/engineering/team/automation/very_long_flow_name_that_exceeds_limit";
|
||||
let result = transform_path(long_path, "flow");
|
||||
assert_eq!(result.len(), MAX_PATH_LENGTH);
|
||||
assert!(result.starts_with("F-"));
|
||||
let (_, _, is_hashed) = parse_tool_prefix(&result).unwrap();
|
||||
assert!(is_hashed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_transform_path_hashing_is_deterministic() {
|
||||
let path = "u/engineering/team/automation/very_long_script_name_that_exceeds_limit";
|
||||
let a = transform_path(path, "script");
|
||||
let b = transform_path(path, "script");
|
||||
assert_eq!(a, b);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_transform_path_different_long_paths_differ() {
|
||||
let a = transform_path(
|
||||
"u/engineering/team/automation/very_long_script_name_that_exceeds_limit_a",
|
||||
"script",
|
||||
);
|
||||
let b = transform_path(
|
||||
"u/engineering/team/automation/very_long_script_name_that_exceeds_limit_b",
|
||||
"script",
|
||||
);
|
||||
assert_ne!(a, b);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_transform_hub_path_short() {
|
||||
let result = transform_hub_path(12345, "Send Slack Message");
|
||||
assert_eq!(result, "hs-12345-Send_Slack_Message");
|
||||
let (_, _, is_hashed) = parse_tool_prefix(&result).unwrap();
|
||||
assert!(!is_hashed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_transform_hub_path_long_is_hashed() {
|
||||
let result = transform_hub_path(
|
||||
12345,
|
||||
"Send Slack Message To Channel With Very Long Description That Exceeds Limit",
|
||||
);
|
||||
assert_eq!(result.len(), MAX_PATH_LENGTH);
|
||||
assert!(result.starts_with("Hs-12345-"));
|
||||
let (_, _, is_hashed) = parse_tool_prefix(&result).unwrap();
|
||||
assert!(is_hashed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_hub_version_id_from_hashed() {
|
||||
let name = "Hs-12345-Send_Slack_Message_To_Ch9e8d7c6b5a4f3e2d";
|
||||
let id = extract_hub_version_id_from_hashed(name).unwrap();
|
||||
assert_eq!(id, "12345");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_tool_prefix() {
|
||||
let (t, hub, hashed) = parse_tool_prefix("S-something").unwrap();
|
||||
assert_eq!(t, "script");
|
||||
assert!(!hub);
|
||||
assert!(hashed);
|
||||
|
||||
let (t, hub, hashed) = parse_tool_prefix("F-something").unwrap();
|
||||
assert_eq!(t, "flow");
|
||||
assert!(!hub);
|
||||
assert!(hashed);
|
||||
|
||||
let (t, hub, hashed) = parse_tool_prefix("Hs-12345-something").unwrap();
|
||||
assert_eq!(t, "script");
|
||||
assert!(hub);
|
||||
assert!(hashed);
|
||||
|
||||
let (t, hub, hashed) = parse_tool_prefix("s-u_admin_script").unwrap();
|
||||
assert_eq!(t, "script");
|
||||
assert!(!hub);
|
||||
assert!(!hashed);
|
||||
|
||||
let (t, hub, hashed) = parse_tool_prefix("f-f_folder_flow").unwrap();
|
||||
assert_eq!(t, "flow");
|
||||
assert!(!hub);
|
||||
assert!(!hashed);
|
||||
|
||||
let (t, hub, hashed) = parse_tool_prefix("hs-12345-summary").unwrap();
|
||||
assert_eq!(t, "script");
|
||||
assert!(hub);
|
||||
assert!(!hashed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_reverse_transform_short_names() {
|
||||
fn test_reverse_transform() {
|
||||
let (type_str, path, is_hub) = reverse_transform("s-u_admin_script").unwrap();
|
||||
assert_eq!(type_str, "script");
|
||||
assert_eq!(path, "u/admin/script");
|
||||
@@ -353,70 +142,6 @@ mod tests {
|
||||
assert!(!is_hub);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_path_prefix_from_hashed() {
|
||||
// Generate a real hashed name and verify prefix extraction
|
||||
let long_path = "u/admin/engineering/team/automation/very_long_script";
|
||||
let hashed = transform_path(long_path, "script");
|
||||
let (_, _, is_hashed) = parse_tool_prefix(&hashed).unwrap();
|
||||
assert!(is_hashed);
|
||||
|
||||
let prefix = extract_path_prefix_from_hashed(&hashed).unwrap();
|
||||
// The original path should start with the extracted prefix
|
||||
assert!(
|
||||
long_path.starts_with(&prefix),
|
||||
"path '{}' should start with prefix '{}'",
|
||||
long_path,
|
||||
prefix
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_path_prefix_underscore_in_path() {
|
||||
let long_path = "u/admin/my_team/automation/very_long_script_name_here";
|
||||
let hashed = transform_path(long_path, "script");
|
||||
let prefix = extract_path_prefix_from_hashed(&hashed).unwrap();
|
||||
assert!(
|
||||
long_path.starts_with(&prefix),
|
||||
"path '{}' should start with prefix '{}'",
|
||||
long_path,
|
||||
prefix
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_path_prefix_rejects_invalid_prefix() {
|
||||
assert!(extract_path_prefix_from_hashed("x-something").is_none());
|
||||
assert!(extract_path_prefix_from_hashed("").is_none());
|
||||
assert!(extract_path_prefix_from_hashed("S-").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_path_prefix_handles_hs_prefix() {
|
||||
// Hs- is 3 chars, not 2 — ensure the prefix is stripped correctly
|
||||
let hashed = transform_hub_path(12345, "a]very long hub script summary that exceeds the limit");
|
||||
let (_, is_hub, is_hashed) = parse_tool_prefix(&hashed).unwrap();
|
||||
assert!(is_hub);
|
||||
assert!(is_hashed);
|
||||
|
||||
let prefix = extract_path_prefix_from_hashed(&hashed);
|
||||
// Should not start with 's' (leftover from Hs- if sliced at index 2)
|
||||
if let Some(ref p) = prefix {
|
||||
assert!(
|
||||
!p.starts_with('s'),
|
||||
"prefix '{}' should not start with 's' from mis-sliced Hs- prefix",
|
||||
p
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_reverse_transform_rejects_hashed_names() {
|
||||
assert!(reverse_transform("S-something").is_err());
|
||||
assert!(reverse_transform("F-something").is_err());
|
||||
assert!(reverse_transform("Hs-12345-something").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_apply_key_transformation() {
|
||||
assert_eq!(apply_key_transformation("my key"), "my_key");
|
||||
|
||||
@@ -92,10 +92,8 @@ pub struct ItemSchema {
|
||||
|
||||
/// Trait for objects that can be converted to MCP tools
|
||||
pub trait ToolableItem {
|
||||
/// Get the MCP-compatible tool name (path transformed with escaping/hashing)
|
||||
fn get_transformed_path(&self) -> String;
|
||||
/// Get the original full path of this item (for display in tool title)
|
||||
fn get_full_path(&self) -> &str;
|
||||
/// Get the path or identifier for this item (transformed for MCP compatibility)
|
||||
fn get_path_or_id(&self) -> String;
|
||||
/// Get the summary/title of this item
|
||||
fn get_summary(&self) -> &str;
|
||||
/// Get the description of this item
|
||||
|
||||
@@ -14,9 +14,9 @@ pub mod client;
|
||||
|
||||
// Re-export common types at crate root for convenience
|
||||
pub use common::{
|
||||
convert_schema_to_schema_type, is_resource_allowed, parse_mcp_scopes, transform_hub_path,
|
||||
transform_path, FlowInfo, HubResponse, HubScriptInfo, ItemSchema, McpScopeConfig, ResourceInfo,
|
||||
ResourceType, SchemaType, ScriptInfo, ToolableItem, WorkspaceId,
|
||||
convert_schema_to_schema_type, is_resource_allowed, parse_mcp_scopes, transform_path, FlowInfo,
|
||||
HubResponse, HubScriptInfo, ItemSchema, McpScopeConfig, ResourceInfo, ResourceType, SchemaType,
|
||||
ScriptInfo, ToolableItem, WorkspaceId,
|
||||
};
|
||||
|
||||
// Re-export client types at crate root for backward compatibility
|
||||
|
||||
@@ -55,22 +55,20 @@ pub trait McpBackend: Send + Sync + Clone + 'static {
|
||||
// Listing Operations
|
||||
// ─────────────────────────────────────────────────────────────────
|
||||
|
||||
/// List scripts, optionally filtered to favorites only and/or by path prefix
|
||||
/// List scripts, optionally filtered to favorites only
|
||||
async fn list_scripts(
|
||||
&self,
|
||||
auth: &Self::Auth,
|
||||
workspace_id: &str,
|
||||
favorites_only: bool,
|
||||
path_prefix: Option<&str>,
|
||||
) -> BackendResult<Vec<ScriptInfo>>;
|
||||
|
||||
/// List flows, optionally filtered to favorites only and/or by path prefix
|
||||
/// List flows, optionally filtered to favorites only
|
||||
async fn list_flows(
|
||||
&self,
|
||||
auth: &Self::Auth,
|
||||
workspace_id: &str,
|
||||
favorites_only: bool,
|
||||
path_prefix: Option<&str>,
|
||||
) -> BackendResult<Vec<FlowInfo>>;
|
||||
|
||||
/// List resource types in workspace
|
||||
|
||||
@@ -5,10 +5,7 @@
|
||||
|
||||
use crate::common::schema::extract_resource_types_from_schema;
|
||||
use crate::common::scope::parse_mcp_scopes;
|
||||
use crate::common::transform::{
|
||||
extract_hub_version_id_from_hashed, extract_path_prefix_from_hashed, parse_tool_prefix,
|
||||
reverse_transform, reverse_transform_key,
|
||||
};
|
||||
use crate::common::transform::{reverse_transform, reverse_transform_key};
|
||||
use crate::common::types::{ResourceInfo, ToolableItem, WorkspaceId};
|
||||
use crate::server::backend::{McpAuth, McpBackend};
|
||||
use crate::server::endpoints::endpoint_tool_to_mcp_tool;
|
||||
@@ -84,13 +81,6 @@ impl<B: McpBackend> Runner<B> {
|
||||
}
|
||||
}
|
||||
|
||||
fn find_matching_path<T: ToolableItem>(candidates: Vec<T>, request_name: &str) -> Option<String> {
|
||||
candidates
|
||||
.into_iter()
|
||||
.find(|item| item.get_transformed_path() == request_name)
|
||||
.map(|item| item.get_full_path().to_string())
|
||||
}
|
||||
|
||||
impl<B: McpBackend> ServerHandler for Runner<B> {
|
||||
fn get_info(&self) -> ServerInfo {
|
||||
ServerInfo {
|
||||
@@ -130,9 +120,9 @@ impl<B: McpBackend> ServerHandler for Runner<B> {
|
||||
// Fetch all items concurrently
|
||||
let (scripts, flows, resource_types, hub_scripts) = tokio::try_join!(
|
||||
self.backend
|
||||
.list_scripts(&auth, &workspace_id, favorites_only, None),
|
||||
.list_scripts(&auth, &workspace_id, favorites_only),
|
||||
self.backend
|
||||
.list_flows(&auth, &workspace_id, favorites_only, None),
|
||||
.list_flows(&auth, &workspace_id, favorites_only),
|
||||
self.backend.list_resource_types(&auth, &workspace_id),
|
||||
async {
|
||||
if let Some(ref apps) = scope_config.hub_apps {
|
||||
@@ -241,6 +231,17 @@ impl<B: McpBackend> ServerHandler for Runner<B> {
|
||||
let scope_config =
|
||||
parse_mcp_scopes(scopes).map_err(|e| ErrorData::internal_error(e, None))?;
|
||||
|
||||
// Handle truncated tool names
|
||||
if request.name.ends_with("_TRUNC") {
|
||||
return Ok(CallToolResult::error(vec![rmcp::model::Annotated::new(
|
||||
rmcp::model::RawContent::Text(rmcp::model::RawTextContent {
|
||||
text: "Tool path is too long. Consider shortening it to make it compatible with MCP.".to_string(),
|
||||
meta: None,
|
||||
}),
|
||||
None,
|
||||
)]));
|
||||
}
|
||||
|
||||
let args = request.arguments.map(Value::Object).unwrap_or(Value::Null);
|
||||
|
||||
// Check if this is an endpoint tool
|
||||
@@ -273,58 +274,10 @@ impl<B: McpBackend> ServerHandler for Runner<B> {
|
||||
}
|
||||
}
|
||||
|
||||
// Resolve the tool name to (type, path, is_hub)
|
||||
let (type_str, is_hub, is_hashed) =
|
||||
parse_tool_prefix(&request.name).map_err(|e| {
|
||||
ErrorData::internal_error(format!("Failed to parse tool name: {}", e), None)
|
||||
})?;
|
||||
|
||||
let (tool_type, path, is_hub) = if !is_hashed {
|
||||
reverse_transform(&request.name).map_err(|e| {
|
||||
ErrorData::internal_error(format!("Failed to parse tool name: {}", e), None)
|
||||
})?
|
||||
} else if is_hub {
|
||||
let version_id =
|
||||
extract_hub_version_id_from_hashed(&request.name).map_err(|e| {
|
||||
ErrorData::internal_error(
|
||||
format!("Failed to extract hub version_id: {}", e),
|
||||
None,
|
||||
)
|
||||
})?;
|
||||
(type_str, version_id, true)
|
||||
} else {
|
||||
let path_prefix = extract_path_prefix_from_hashed(&request.name);
|
||||
let favorites_only = scope_config.favorites;
|
||||
let matched_path = if type_str == "script" {
|
||||
find_matching_path(
|
||||
self.backend
|
||||
.list_scripts(&auth, &workspace_id, favorites_only, path_prefix.as_deref())
|
||||
.await
|
||||
.map_err(|e| ErrorData::internal_error(e.message, None))?,
|
||||
&request.name,
|
||||
)
|
||||
} else {
|
||||
find_matching_path(
|
||||
self.backend
|
||||
.list_flows(&auth, &workspace_id, favorites_only, path_prefix.as_deref())
|
||||
.await
|
||||
.map_err(|e| ErrorData::internal_error(e.message, None))?,
|
||||
&request.name,
|
||||
)
|
||||
};
|
||||
|
||||
let matched_path = matched_path.ok_or_else(|| {
|
||||
ErrorData::internal_error(
|
||||
format!(
|
||||
"No {} found matching hashed tool name '{}'",
|
||||
type_str, request.name
|
||||
),
|
||||
None,
|
||||
)
|
||||
})?;
|
||||
|
||||
(type_str, matched_path, false)
|
||||
};
|
||||
// Not an endpoint tool - parse as script/flow
|
||||
let (tool_type, path, is_hub) = reverse_transform(&request.name).map_err(|e| {
|
||||
ErrorData::internal_error(format!("Failed to parse tool name: {}", e), None)
|
||||
})?;
|
||||
|
||||
// Validate script/flow scope
|
||||
if !is_hub && scope_config.granular {
|
||||
|
||||
@@ -9,7 +9,7 @@ use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::common::schema::{convert_schema_to_schema_type, make_schema_compatible};
|
||||
use crate::common::transform::{transform_hub_path, transform_path};
|
||||
use crate::common::transform::transform_path;
|
||||
use crate::common::types::{
|
||||
FlowInfo, HubScriptInfo, ResourceInfo, ResourceType, SchemaType, ScriptInfo, ToolableItem,
|
||||
};
|
||||
@@ -17,14 +17,10 @@ use crate::server::backend::McpBackend;
|
||||
|
||||
/// Implementation of ToolableItem for ScriptInfo
|
||||
impl ToolableItem for ScriptInfo {
|
||||
fn get_transformed_path(&self) -> String {
|
||||
fn get_path_or_id(&self) -> String {
|
||||
transform_path(&self.path, "script")
|
||||
}
|
||||
|
||||
fn get_full_path(&self) -> &str {
|
||||
&self.path
|
||||
}
|
||||
|
||||
fn get_summary(&self) -> &str {
|
||||
self.summary.as_deref().unwrap_or("No summary")
|
||||
}
|
||||
@@ -52,14 +48,10 @@ impl ToolableItem for ScriptInfo {
|
||||
|
||||
/// Implementation of ToolableItem for FlowInfo
|
||||
impl ToolableItem for FlowInfo {
|
||||
fn get_transformed_path(&self) -> String {
|
||||
fn get_path_or_id(&self) -> String {
|
||||
transform_path(&self.path, "flow")
|
||||
}
|
||||
|
||||
fn get_full_path(&self) -> &str {
|
||||
&self.path
|
||||
}
|
||||
|
||||
fn get_summary(&self) -> &str {
|
||||
self.summary.as_deref().unwrap_or("No summary")
|
||||
}
|
||||
@@ -87,13 +79,10 @@ impl ToolableItem for FlowInfo {
|
||||
|
||||
/// Implementation of ToolableItem for HubScriptInfo
|
||||
impl ToolableItem for HubScriptInfo {
|
||||
fn get_transformed_path(&self) -> String {
|
||||
fn get_path_or_id(&self) -> String {
|
||||
let id = self.version_id;
|
||||
let summary = self.summary.as_deref().unwrap_or("No summary");
|
||||
transform_hub_path(self.version_id, summary)
|
||||
}
|
||||
|
||||
fn get_full_path(&self) -> &str {
|
||||
self.summary.as_deref().unwrap_or("No summary")
|
||||
format!("hs-{}-{}", id, summary.replace(" ", "_"))
|
||||
}
|
||||
|
||||
fn get_summary(&self) -> &str {
|
||||
@@ -135,7 +124,7 @@ pub fn create_tool_from_item<T: ToolableItem, B: McpBackend>(
|
||||
resources_types: &[ResourceType],
|
||||
) -> Tool {
|
||||
let is_hub = item.is_hub();
|
||||
let path = item.get_transformed_path();
|
||||
let path = item.get_path_or_id();
|
||||
let item_type = item.item_type();
|
||||
let description = format!(
|
||||
"This is a {} named `{}` with the following description: `{}`.{}",
|
||||
@@ -181,24 +170,15 @@ pub fn create_tool_from_item<T: ToolableItem, B: McpBackend>(
|
||||
}
|
||||
};
|
||||
|
||||
let title = {
|
||||
let summary = item.get_summary();
|
||||
if summary == "No summary" {
|
||||
item.get_full_path().to_string()
|
||||
} else {
|
||||
summary.to_string()
|
||||
}
|
||||
};
|
||||
|
||||
Tool {
|
||||
name: Cow::Owned(path),
|
||||
description: Some(Cow::Owned(description)),
|
||||
input_schema: Arc::new(input_schema_map),
|
||||
title: Some(title.clone()),
|
||||
title: Some(item.get_summary().to_string()),
|
||||
output_schema: None,
|
||||
icons: None,
|
||||
annotations: Some(ToolAnnotations {
|
||||
title: Some(title),
|
||||
title: Some(item.get_summary().to_string()),
|
||||
read_only_hint: Some(false), // Can modify environment
|
||||
destructive_hint: Some(true), // Can potentially be destructive
|
||||
idempotent_hint: Some(false), // Are not guaranteed to be idempotent
|
||||
|
||||
@@ -3434,38 +3434,6 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<'c>(
|
||||
Ok(job_and_suspended)
|
||||
}
|
||||
|
||||
/// Batch-pull up to `limit` jobs in a single query, marking them all as running.
|
||||
/// The caller controls which tags are queried, so flow/dependency jobs are never
|
||||
/// pulled (they use distinct tags like "flow" / "dependency").
|
||||
pub async fn batch_pull(
|
||||
db: &Pool<Postgres>,
|
||||
worker_name: &str,
|
||||
tags: &[String],
|
||||
limit: u32,
|
||||
) -> windmill_common::error::Result<Vec<PulledJob>> {
|
||||
use windmill_common::worker::make_batch_pull_query;
|
||||
|
||||
if limit == 0 || tags.is_empty() {
|
||||
return Ok(vec![]);
|
||||
}
|
||||
|
||||
let query = make_batch_pull_query(tags, limit);
|
||||
let jobs: Vec<PulledJob> = timeout(
|
||||
Duration::from_secs(15),
|
||||
sqlx::query_as::<_, PulledJob>(&query)
|
||||
.bind(worker_name)
|
||||
.fetch_all(db),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
windmill_common::error::Error::internal_err(
|
||||
"batch_pull query timed out after 15s".to_string(),
|
||||
)
|
||||
})??;
|
||||
|
||||
Ok(jobs)
|
||||
}
|
||||
|
||||
pub async fn custom_concurrency_key(
|
||||
db: &Pool<Postgres>,
|
||||
job_id: &Uuid,
|
||||
@@ -5405,7 +5373,15 @@ async fn push_inner<'c, 'd>(
|
||||
language
|
||||
.as_ref()
|
||||
.map(|x| {
|
||||
let tag_lang = x.as_worker_tag(job_kind == JobKind::Dependencies);
|
||||
let tag_lang = if x == &ScriptLang::Bunnative {
|
||||
if job_kind == JobKind::Dependencies {
|
||||
ScriptLang::Bun.as_str()
|
||||
} else {
|
||||
ScriptLang::Nativets.as_str()
|
||||
}
|
||||
} else {
|
||||
x.as_str()
|
||||
};
|
||||
if per_workspace {
|
||||
format!("{}-{}", tag_lang, workspace_id)
|
||||
} else {
|
||||
|
||||
@@ -151,7 +151,6 @@ pub struct RunJob {
|
||||
pub args: serde_json::Map<String, serde_json::Value>,
|
||||
pub scheduled_for_o: Option<chrono::DateTime<chrono::Utc>>,
|
||||
pub email: String,
|
||||
pub job_id: Option<Uuid>,
|
||||
}
|
||||
|
||||
impl From<JobPayload> for RunJob {
|
||||
@@ -161,7 +160,6 @@ impl From<JobPayload> for RunJob {
|
||||
args: Default::default(),
|
||||
scheduled_for_o: None,
|
||||
email: "test@windmill.dev".to_string(),
|
||||
job_id: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -185,13 +183,8 @@ impl RunJob {
|
||||
self
|
||||
}
|
||||
|
||||
pub fn job_id(mut self, id: Uuid) -> Self {
|
||||
self.job_id = Some(id);
|
||||
self
|
||||
}
|
||||
|
||||
pub async fn push(self, db: &Pool<Postgres>) -> Uuid {
|
||||
let RunJob { payload, args, scheduled_for_o, email, job_id } = self;
|
||||
let RunJob { payload, args, scheduled_for_o, email } = self;
|
||||
let mut hm_args = std::collections::HashMap::new();
|
||||
for (k, v) in args {
|
||||
hm_args.insert(k, windmill_common::worker::to_raw_value(&v));
|
||||
@@ -213,7 +206,7 @@ impl RunJob {
|
||||
/* parent_job */ None,
|
||||
/* root job */ None,
|
||||
/* flow_innermost_root_job */ None,
|
||||
/* job_id */ job_id,
|
||||
/* job_id */ None,
|
||||
/* is_flow_step */ false,
|
||||
/* same_worker */ false,
|
||||
None,
|
||||
@@ -237,7 +230,7 @@ impl RunJob {
|
||||
|
||||
/// Push the job as a specific user (for testing permissions)
|
||||
pub async fn push_as(self, db: &Pool<Postgres>, username: &str, email: &str) -> Uuid {
|
||||
let RunJob { payload, args, scheduled_for_o, job_id, .. } = self;
|
||||
let RunJob { payload, args, scheduled_for_o, .. } = self;
|
||||
let mut hm_args = std::collections::HashMap::new();
|
||||
for (k, v) in args {
|
||||
hm_args.insert(k, windmill_common::worker::to_raw_value(&v));
|
||||
@@ -259,7 +252,7 @@ impl RunJob {
|
||||
/* parent_job */ None,
|
||||
/* root job */ None,
|
||||
/* flow_innermost_root_job */ None,
|
||||
/* job_id */ job_id,
|
||||
/* job_id */ None,
|
||||
/* is_flow_step */ false,
|
||||
/* same_worker */ false,
|
||||
None,
|
||||
@@ -421,7 +414,6 @@ pub fn spawn_test_worker(
|
||||
rx,
|
||||
tx2,
|
||||
&base_internal_url,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
};
|
||||
|
||||
@@ -88,20 +88,6 @@ impl ScriptLang {
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the worker tag for this language.
|
||||
/// Bunnative scripts run on nativets workers (not bun), except dependency jobs which use bun.
|
||||
pub fn as_worker_tag(&self, is_dependency_job: bool) -> &'static str {
|
||||
if *self == ScriptLang::Bunnative {
|
||||
if is_dependency_job {
|
||||
ScriptLang::Bun.as_str()
|
||||
} else {
|
||||
ScriptLang::Nativets.as_str()
|
||||
}
|
||||
} else {
|
||||
self.as_str()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn as_dependencies_filename(&self) -> Option<String> {
|
||||
use ScriptLang::*;
|
||||
Some(
|
||||
@@ -119,15 +105,15 @@ impl ScriptLang {
|
||||
pub fn is_native(&self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
ScriptLang::Bunnative
|
||||
| ScriptLang::Nativets
|
||||
| ScriptLang::Postgresql
|
||||
| ScriptLang::Mysql
|
||||
| ScriptLang::Graphql
|
||||
| ScriptLang::Snowflake
|
||||
| ScriptLang::Mssql
|
||||
| ScriptLang::Bigquery
|
||||
| ScriptLang::OracleDB
|
||||
ScriptLang::Bunnative |
|
||||
ScriptLang::Nativets |
|
||||
ScriptLang::Postgresql |
|
||||
ScriptLang::Mysql |
|
||||
ScriptLang::Graphql |
|
||||
ScriptLang::Snowflake |
|
||||
ScriptLang::Mssql |
|
||||
ScriptLang::Bigquery |
|
||||
ScriptLang::OracleDB
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -1036,12 +1036,6 @@ pub async fn handle_bun_job(
|
||||
let _ = write_file(job_dir, "main.ts", inner_content)?;
|
||||
} else if !annotation.native && codebase.is_none() {
|
||||
let _ = write_file(job_dir, "package.json", r#"{ "type": "module" }"#)?;
|
||||
} else if codebase.is_some() {
|
||||
// Write a valid fallback package.json for codebase mode. Without this,
|
||||
// nsjail creates an empty 0-byte file (from the mandatory: false mount)
|
||||
// which Node.js fails to parse as JSON (ERR_INVALID_PACKAGE_CONFIG).
|
||||
// If the codebase TAR includes a package.json, it will overwrite this.
|
||||
let _ = write_file(job_dir, "package.json", "{}")?;
|
||||
};
|
||||
|
||||
let common_bun_proc_envs: HashMap<String, String> =
|
||||
|
||||
@@ -563,7 +563,6 @@ pub async fn update_worker_ping_for_failed_init_script(
|
||||
wm_memory_usage: None,
|
||||
job_isolation: None,
|
||||
native_mode: None,
|
||||
uses_batch_http_pull: None,
|
||||
ping_type: PingType::InitScript,
|
||||
},
|
||||
)
|
||||
|
||||
@@ -1368,7 +1368,6 @@ pub async fn run_worker(
|
||||
mut killpill_rx: tokio::sync::broadcast::Receiver<()>,
|
||||
killpill_tx: KillpillSender,
|
||||
base_internal_url: &str,
|
||||
batch_pull_client: Option<&HttpClient>,
|
||||
) {
|
||||
#[cfg(not(feature = "enterprise"))]
|
||||
if is_sandboxing_enabled() {
|
||||
@@ -2069,149 +2068,135 @@ pub async fn run_worker(
|
||||
continue;
|
||||
}
|
||||
} else {
|
||||
// If batch_pull_client is set (native worker with co-located server),
|
||||
// use HTTP pull from batch buffer. Otherwise use direct SQL pull.
|
||||
if let Some(bpc) = batch_pull_client {
|
||||
crate::agent_workers::pull_job(bpc, None, None)
|
||||
.await
|
||||
.map_err(|e| error::Error::InternalErr(e.to_string()))
|
||||
.map(|x| x.map(|y| NextJob::Http(y)))
|
||||
} else {
|
||||
match &conn {
|
||||
Connection::Sql(db) => {
|
||||
let pull_time = Instant::now();
|
||||
let likelihood_of_suspend = last_30jobs_suspended as f64 / 30.0;
|
||||
match &conn {
|
||||
Connection::Sql(db) => {
|
||||
let pull_time = Instant::now();
|
||||
let likelihood_of_suspend = last_30jobs_suspended as f64 / 30.0;
|
||||
|
||||
let suspend_first = suspend_first_success
|
||||
|| rand::random::<f64>() < likelihood_of_suspend
|
||||
|| last_suspend_first.elapsed().as_secs_f64() > 5.0;
|
||||
let suspend_first = suspend_first_success
|
||||
|| rand::random::<f64>() < likelihood_of_suspend
|
||||
|| last_suspend_first.elapsed().as_secs_f64() > 5.0;
|
||||
|
||||
if suspend_first {
|
||||
last_suspend_first = Instant::now();
|
||||
}
|
||||
let mut job = match timeout(
|
||||
Duration::from_secs(30),
|
||||
pull(
|
||||
&db,
|
||||
suspend_first,
|
||||
&worker_name,
|
||||
None,
|
||||
#[cfg(feature = "benchmark")]
|
||||
&mut bench,
|
||||
)
|
||||
.warn_after_seconds(2),
|
||||
if suspend_first {
|
||||
last_suspend_first = Instant::now();
|
||||
}
|
||||
let mut job = match timeout(
|
||||
Duration::from_secs(30),
|
||||
pull(
|
||||
&db,
|
||||
suspend_first,
|
||||
&worker_name,
|
||||
None,
|
||||
#[cfg(feature = "benchmark")]
|
||||
&mut bench,
|
||||
)
|
||||
.warn_after_seconds(2),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(job) => job,
|
||||
Err(e) => {
|
||||
tracing::error!(worker = %worker_name, hostname = %hostname, "pull timed out after 20s, sleeping for 30s: {e:?}");
|
||||
tokio::time::sleep(Duration::from_secs(30)).await;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
// Preprocess pulled job result
|
||||
if let Ok(ref mut pulled_job_res) = job {
|
||||
if let Err(e) = timeout(
|
||||
// Will fail if longer than 10 seconds
|
||||
core::time::Duration::from_secs(10),
|
||||
pulled_job_res.maybe_apply_debouncing(db),
|
||||
)
|
||||
.warn_after_seconds(2)
|
||||
.await
|
||||
// Flatten result
|
||||
.map_err(error::Error::from)
|
||||
.and_then(|r| r)
|
||||
{
|
||||
Ok(job) => job,
|
||||
Err(e) => {
|
||||
tracing::error!(worker = %worker_name, hostname = %hostname, "pull timed out after 20s, sleeping for 30s: {e:?}");
|
||||
tokio::time::sleep(Duration::from_secs(30)).await;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
// Preprocess pulled job result
|
||||
if let Ok(ref mut pulled_job_res) = job {
|
||||
if let Err(e) = timeout(
|
||||
// Will fail if longer than 10 seconds
|
||||
core::time::Duration::from_secs(10),
|
||||
pulled_job_res.maybe_apply_debouncing(db),
|
||||
)
|
||||
.warn_after_seconds(2)
|
||||
.await
|
||||
// Flatten result
|
||||
.map_err(error::Error::from)
|
||||
.and_then(|r| r)
|
||||
{
|
||||
pulled_job_res.error_while_preprocessing = Some(e.to_string());
|
||||
}
|
||||
pulled_job_res.error_while_preprocessing = Some(e.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
add_time!(bench, "job pulled from DB");
|
||||
let duration_pull_s = pull_time.elapsed().as_secs_f64();
|
||||
let err_pull = job.is_ok();
|
||||
// let empty = job.as_ref().is_ok_and(|x| x.is_none());
|
||||
add_time!(bench, "job pulled from DB");
|
||||
let duration_pull_s = pull_time.elapsed().as_secs_f64();
|
||||
let err_pull = job.is_ok();
|
||||
// let empty = job.as_ref().is_ok_and(|x| x.is_none());
|
||||
|
||||
if duration_pull_s > 0.5 {
|
||||
let empty = job.as_ref().is_ok_and(|x| x.job.is_none());
|
||||
tracing::warn!(worker = %worker_name, hostname = %hostname, "pull took more than 0.5s ({duration_pull_s}), this is a sign that the database is VERY undersized for this load. empty: {empty}, err: {err_pull}");
|
||||
#[cfg(feature = "prometheus")]
|
||||
if empty {
|
||||
if let Some(wp) = worker_pull_over_500_counter_empty.as_ref() {
|
||||
wp.inc();
|
||||
}
|
||||
} else if let Some(wp) = worker_pull_over_500_counter.as_ref() {
|
||||
if duration_pull_s > 0.5 {
|
||||
let empty = job.as_ref().is_ok_and(|x| x.job.is_none());
|
||||
tracing::warn!(worker = %worker_name, hostname = %hostname, "pull took more than 0.5s ({duration_pull_s}), this is a sign that the database is VERY undersized for this load. empty: {empty}, err: {err_pull}");
|
||||
#[cfg(feature = "prometheus")]
|
||||
if empty {
|
||||
if let Some(wp) = worker_pull_over_500_counter_empty.as_ref() {
|
||||
wp.inc();
|
||||
}
|
||||
} else if duration_pull_s > 0.1 {
|
||||
let empty = job.as_ref().is_ok_and(|x| x.job.is_none());
|
||||
tracing::warn!(worker = %worker_name, hostname = %hostname, "pull took more than 0.1s ({duration_pull_s}) this is a sign that the database is undersized for this load. empty: {empty}, err: {err_pull}");
|
||||
#[cfg(feature = "prometheus")]
|
||||
if empty {
|
||||
if let Some(wp) = worker_pull_over_100_counter_empty.as_ref() {
|
||||
wp.inc();
|
||||
}
|
||||
} else if let Some(wp) = worker_pull_over_100_counter.as_ref() {
|
||||
} else if let Some(wp) = worker_pull_over_500_counter.as_ref() {
|
||||
wp.inc();
|
||||
}
|
||||
} else if duration_pull_s > 0.1 {
|
||||
let empty = job.as_ref().is_ok_and(|x| x.job.is_none());
|
||||
tracing::warn!(worker = %worker_name, hostname = %hostname, "pull took more than 0.1s ({duration_pull_s}) this is a sign that the database is undersized for this load. empty: {empty}, err: {err_pull}");
|
||||
#[cfg(feature = "prometheus")]
|
||||
if empty {
|
||||
if let Some(wp) = worker_pull_over_100_counter_empty.as_ref() {
|
||||
wp.inc();
|
||||
}
|
||||
} else if let Some(wp) = worker_pull_over_100_counter.as_ref() {
|
||||
wp.inc();
|
||||
}
|
||||
}
|
||||
|
||||
if let Ok(j) = job.as_ref() {
|
||||
let suspend_success = j.suspended;
|
||||
if suspend_first {
|
||||
if last_30jobs_suspended < 30 {
|
||||
last_30jobs_suspended += 1;
|
||||
}
|
||||
} else {
|
||||
last_30jobs_suspended -= 1;
|
||||
if let Ok(j) = job.as_ref() {
|
||||
let suspend_success = j.suspended;
|
||||
if suspend_first {
|
||||
if last_30jobs_suspended < 30 {
|
||||
last_30jobs_suspended += 1;
|
||||
}
|
||||
suspend_first_success = suspend_first && suspend_success;
|
||||
#[cfg(feature = "prometheus")]
|
||||
if j.job.is_some() {
|
||||
if let Some(wp) = worker_pull_duration_counter.as_ref() {
|
||||
wp.inc_by(duration_pull_s);
|
||||
}
|
||||
if let Some(wp) = worker_pull_duration.as_ref() {
|
||||
wp.observe(duration_pull_s);
|
||||
}
|
||||
} else {
|
||||
if let Some(wp) = worker_pull_duration_counter_empty.as_ref() {
|
||||
wp.inc_by(duration_pull_s);
|
||||
}
|
||||
if let Some(wp) = worker_pull_duration_empty.as_ref() {
|
||||
wp.observe(duration_pull_s);
|
||||
}
|
||||
} else {
|
||||
last_30jobs_suspended -= 1;
|
||||
}
|
||||
suspend_first_success = suspend_first && suspend_success;
|
||||
#[cfg(feature = "prometheus")]
|
||||
if j.job.is_some() {
|
||||
if let Some(wp) = worker_pull_duration_counter.as_ref() {
|
||||
wp.inc_by(duration_pull_s);
|
||||
}
|
||||
if let Some(wp) = worker_pull_duration.as_ref() {
|
||||
wp.observe(duration_pull_s);
|
||||
}
|
||||
} else {
|
||||
if let Some(wp) = worker_pull_duration_counter_empty.as_ref() {
|
||||
wp.inc_by(duration_pull_s);
|
||||
}
|
||||
if let Some(wp) = worker_pull_duration_empty.as_ref() {
|
||||
wp.observe(duration_pull_s);
|
||||
}
|
||||
}
|
||||
match job {
|
||||
Ok(pulled_job_result) => match pulled_job_result.to_pulled_job() {
|
||||
Ok(j) => {
|
||||
Ok(j.map(|job| NextJob::Sql { flow_runners: None, job }))
|
||||
}
|
||||
Err(PulledJobResultToJobErr::MissingConcurrencyKey(jc))
|
||||
| Err(PulledJobResultToJobErr::ErrorWhilePreprocessing(jc)) => {
|
||||
if let Err(err) = job_completed_tx.send_job(jc, true).await
|
||||
{
|
||||
tracing::error!(
|
||||
}
|
||||
match job {
|
||||
Ok(pulled_job_result) => match pulled_job_result.to_pulled_job() {
|
||||
Ok(j) => Ok(j.map(|job| NextJob::Sql { flow_runners: None, job })),
|
||||
Err(PulledJobResultToJobErr::MissingConcurrencyKey(jc))
|
||||
| Err(PulledJobResultToJobErr::ErrorWhilePreprocessing(jc)) => {
|
||||
if let Err(err) = job_completed_tx.send_job(jc, true).await {
|
||||
tracing::error!(
|
||||
"An error occurred while sending job completed: {:#?}",
|
||||
err
|
||||
)
|
||||
}
|
||||
Ok(None)
|
||||
}
|
||||
},
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
Connection::Http(client) => {
|
||||
crate::agent_workers::pull_job(&client, None, None)
|
||||
.await
|
||||
.map_err(|e| error::Error::InternalErr(e.to_string()))
|
||||
.map(|x| x.map(|y| NextJob::Http(y)))
|
||||
Ok(None)
|
||||
}
|
||||
},
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
Connection::Http(client) => crate::agent_workers::pull_job(&client, None, None)
|
||||
.await
|
||||
.map_err(|e| error::Error::InternalErr(e.to_string()))
|
||||
.map(|x| x.map(|y| NextJob::Http(y))),
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -4786,7 +4771,6 @@ mount {{
|
||||
|
||||
// Clean up absolute-path symlinks created by setup_volume_mount_paths
|
||||
if !is_sandboxing_enabled() {
|
||||
#[allow(unused_variables)] // state is only used on unix
|
||||
for state in &volume_setup.states {
|
||||
#[cfg(unix)]
|
||||
if state.mount.target.starts_with('/') {
|
||||
|
||||
@@ -8,7 +8,7 @@ use windmill_common::{
|
||||
get_memory, get_vcpus, get_windmill_memory_usage, get_worker_memory_usage,
|
||||
insert_ping_query, update_job_ping_query, update_worker_ping_from_job_query,
|
||||
update_worker_ping_main_loop_query, Connection, Ping, PingType, NATIVE_MODE_RESOLVED,
|
||||
USES_BATCH_HTTP_PULL, WORKER_CONFIG, WORKER_GROUP,
|
||||
WORKER_CONFIG, WORKER_GROUP,
|
||||
},
|
||||
KillpillSender, DB,
|
||||
};
|
||||
@@ -31,7 +31,6 @@ pub(crate) async fn update_worker_ping_full(
|
||||
let tags = wc.worker_tags.clone();
|
||||
let native_mode = wc.native_mode;
|
||||
drop(wc);
|
||||
let uses_batch_http_pull = USES_BATCH_HTTP_PULL.load(std::sync::atomic::Ordering::Relaxed);
|
||||
|
||||
let memory_usage = get_worker_memory_usage();
|
||||
let wm_memory_usage = get_windmill_memory_usage();
|
||||
@@ -65,7 +64,6 @@ pub(crate) async fn update_worker_ping_full(
|
||||
occupancy_rate_5m,
|
||||
occupancy_rate_30m,
|
||||
native_mode,
|
||||
uses_batch_http_pull,
|
||||
)
|
||||
})
|
||||
.retry(
|
||||
@@ -112,7 +110,6 @@ async fn update_worker_ping_full_inner(
|
||||
occupancy_rate_5m: Option<f32>,
|
||||
occupancy_rate_30m: Option<f32>,
|
||||
native_mode: bool,
|
||||
uses_batch_http_pull: bool,
|
||||
) -> anyhow::Result<()> {
|
||||
match conn {
|
||||
Connection::Sql(db) => {
|
||||
@@ -129,7 +126,6 @@ async fn update_worker_ping_full_inner(
|
||||
occupancy_rate_5m,
|
||||
occupancy_rate_30m,
|
||||
native_mode,
|
||||
uses_batch_http_pull,
|
||||
db,
|
||||
)
|
||||
.await?;
|
||||
@@ -159,7 +155,6 @@ async fn update_worker_ping_full_inner(
|
||||
wm_memory_usage: get_windmill_memory_usage(),
|
||||
job_isolation: None,
|
||||
native_mode: Some(native_mode),
|
||||
uses_batch_http_pull: Some(uses_batch_http_pull),
|
||||
ping_type: PingType::MainLoop,
|
||||
},
|
||||
)
|
||||
@@ -191,7 +186,6 @@ pub async fn insert_ping(
|
||||
wc.native_mode,
|
||||
)
|
||||
};
|
||||
let uses_batch_http_pull = USES_BATCH_HTTP_PULL.load(std::sync::atomic::Ordering::Relaxed);
|
||||
|
||||
let vcpus = get_vcpus();
|
||||
let memory = get_memory();
|
||||
@@ -219,7 +213,6 @@ pub async fn insert_ping(
|
||||
memory,
|
||||
job_isolation,
|
||||
native_mode,
|
||||
uses_batch_http_pull,
|
||||
db,
|
||||
)
|
||||
.await?;
|
||||
@@ -249,7 +242,6 @@ pub async fn insert_ping(
|
||||
wm_memory_usage: get_windmill_memory_usage(),
|
||||
job_isolation,
|
||||
native_mode: Some(native_mode),
|
||||
uses_batch_http_pull: Some(uses_batch_http_pull),
|
||||
ping_type: PingType::Initial,
|
||||
},
|
||||
)
|
||||
@@ -326,9 +318,6 @@ pub async fn update_worker_ping_from_job(
|
||||
native_mode: Some(
|
||||
NATIVE_MODE_RESOLVED.load(std::sync::atomic::Ordering::Relaxed),
|
||||
),
|
||||
uses_batch_http_pull: Some(
|
||||
USES_BATCH_HTTP_PULL.load(std::sync::atomic::Ordering::Relaxed),
|
||||
),
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
|
||||
@@ -47,7 +47,6 @@ export async function main({
|
||||
kind,
|
||||
jobs,
|
||||
noVerify,
|
||||
skipDeploy,
|
||||
}: {
|
||||
host: string;
|
||||
email?: string;
|
||||
@@ -57,7 +56,6 @@ export async function main({
|
||||
kind: string;
|
||||
jobs: number;
|
||||
noVerify?: boolean;
|
||||
skipDeploy?: boolean;
|
||||
}) {
|
||||
windmill.setClient("", host);
|
||||
|
||||
@@ -148,8 +146,7 @@ export async function main({
|
||||
}
|
||||
|
||||
if (
|
||||
!skipDeploy &&
|
||||
["deno", "python", "go", "bash", "dedicated", "bun", "nativets", "nativets_sleep", "dedicated_nativets"].includes(
|
||||
["deno", "python", "go", "bash", "dedicated", "bun", "nativets", "dedicated_nativets"].includes(
|
||||
kind
|
||||
)
|
||||
) {
|
||||
@@ -168,7 +165,7 @@ export async function main({
|
||||
kind: "noop",
|
||||
});
|
||||
} else if (
|
||||
["deno", "python", "go", "bash", "dedicated", "bun", "nativets", "nativets_sleep", "dedicated_nativets"].includes(
|
||||
["deno", "python", "go", "bash", "dedicated", "bun", "nativets", "dedicated_nativets"].includes(
|
||||
kind
|
||||
)
|
||||
) {
|
||||
@@ -339,7 +336,6 @@ export async function main({
|
||||
!noVerify &&
|
||||
kind !== "noop" &&
|
||||
kind !== "nativets" &&
|
||||
kind !== "nativets_sleep" &&
|
||||
kind !== "dedicated_nativets" &&
|
||||
!kind.startsWith("flow:") &&
|
||||
!kind.startsWith("script:")
|
||||
@@ -402,9 +398,6 @@ if (import.meta.main) {
|
||||
.option("--no-verify", "Do not verify the output of the jobs.", {
|
||||
default: false,
|
||||
})
|
||||
.option("--skip-deploy", "Skip script deployment (use already deployed script).", {
|
||||
default: false,
|
||||
})
|
||||
.action(main)
|
||||
.command(
|
||||
"upgrade",
|
||||
|
||||
@@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts";
|
||||
import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts";
|
||||
import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts";
|
||||
|
||||
export const VERSION = "v1.651.1";
|
||||
export const VERSION = "v1.650.0";
|
||||
|
||||
export async function login(email: string, password: string): Promise<string> {
|
||||
return await windmill.UserService.login({
|
||||
@@ -95,10 +95,6 @@ export async function createBenchScript(
|
||||
scriptContent =
|
||||
'//native\nexport async function main(){ return (await fetch(BASE_URL + "/api/version")).text() }';
|
||||
language = "bunnative";
|
||||
} else if (scriptPattern === "nativets_sleep") {
|
||||
scriptContent =
|
||||
'//native\nexport async function main(){ const ms = 300 + Math.floor(Math.random() * 400); await new Promise(r => setTimeout(r, ms)); return { slept: ms }; }';
|
||||
language = "bunnative";
|
||||
} else if (scriptPattern === "dedicated_nativets") {
|
||||
scriptContent = "//native\nexport function main(){ return 42; }";
|
||||
language = "bunnative";
|
||||
|
||||
@@ -1,165 +0,0 @@
|
||||
/**
|
||||
* Model: Batch Pull vs Direct SQL throughput
|
||||
*
|
||||
* Calibrated from real benchmarks (3 native workers = 24 subworkers, local PG):
|
||||
* - nativets (fast): batch 291 j/s, SQL 253 j/s at N=24; batch 108, SQL 88 at N=8
|
||||
* - nativets_sleep: both ~43.8 j/s at N=24 (bottlenecked by 500ms avg exec time)
|
||||
*
|
||||
* Per-worker job time model:
|
||||
* T_pw = T_base + T_exec + T_contention(N)
|
||||
* throughput = N / T_pw
|
||||
*
|
||||
* Batch: T_contention grows linearly with N (HTTP server load)
|
||||
* T_pw_batch(N) = BASE_BATCH + T_exec + SCALE_BATCH × N
|
||||
*
|
||||
* SQL: T_contention grows quadratically with N (SKIP LOCKED scanning past locked rows)
|
||||
* T_pw_sql(N) = BASE_SQL + T_exec + SCALE_SQL × N²
|
||||
*
|
||||
* Parameters fitted from 2 data points each (N=8, N=24):
|
||||
* Batch: BASE=69.9ms, SCALE=0.525ms/worker
|
||||
* SQL: BASE=90.4ms, SCALE=0.0078ms/worker²
|
||||
* (SQL quadratic overtakes batch linear around N~40)
|
||||
*/
|
||||
|
||||
// --- Model parameters (fitted from benchmarks) ---
|
||||
|
||||
// Batch: per-worker time = BASE + SCALE_LINEAR * N + T_exec
|
||||
const BASE_BATCH = 69.9; // ms — base overhead (worker loop, HTTP roundtrip, job completion writes)
|
||||
const SCALE_BATCH = 0.525; // ms per subworker — linear growth from server load
|
||||
|
||||
// SQL: per-worker time = BASE + SCALE_QUAD * N² + T_exec
|
||||
const BASE_SQL = 90.4; // ms — base overhead (worker loop, poll interval wait, job completion writes)
|
||||
const SCALE_SQL = 0.0078; // ms per subworker² — quadratic growth from SKIP LOCKED contention
|
||||
|
||||
// --- Throughput functions ---
|
||||
|
||||
function throughputBatch(subworkers: number, execMs: number): number {
|
||||
const tPerWorker = BASE_BATCH + execMs + SCALE_BATCH * subworkers;
|
||||
return (subworkers / tPerWorker) * 1000; // jobs/s
|
||||
}
|
||||
|
||||
function throughputSql(subworkers: number, execMs: number): number {
|
||||
const tPerWorker = BASE_SQL + execMs + SCALE_SQL * subworkers * subworkers;
|
||||
return (subworkers / tPerWorker) * 1000; // jobs/s
|
||||
}
|
||||
|
||||
function pct(batch: number, sql: number): string {
|
||||
const diff = ((batch - sql) / sql) * 100;
|
||||
return `${diff >= 0 ? "+" : ""}${diff.toFixed(0)}%`;
|
||||
}
|
||||
|
||||
// --- Validation against real data ---
|
||||
|
||||
console.log("=== Model Validation (vs real benchmarks) ===\n");
|
||||
console.log(
|
||||
" Setup | Model Batch | Real Batch | Model SQL | Real SQL",
|
||||
);
|
||||
console.log(
|
||||
" ---------------------|-------------|------------|-----------|--------",
|
||||
);
|
||||
|
||||
const cases = [
|
||||
{ n: 8, exec: 0, label: "1W nativets", realBatch: 108, realSql: 88 },
|
||||
{ n: 24, exec: 0, label: "3W nativets", realBatch: 291, realSql: 253 },
|
||||
{
|
||||
n: 24,
|
||||
exec: 500,
|
||||
label: "3W sleep(500ms)",
|
||||
realBatch: 43.8,
|
||||
realSql: 43.8,
|
||||
},
|
||||
];
|
||||
|
||||
for (const c of cases) {
|
||||
const mb = throughputBatch(c.n, c.exec);
|
||||
const ms = throughputSql(c.n, c.exec);
|
||||
console.log(
|
||||
` ${c.label.padEnd(21)}| ${mb.toFixed(0).padStart(7)} j/s | ${c.realBatch.toFixed(0).padStart(6)} j/s | ${ms.toFixed(0).padStart(5)} j/s | ${c.realSql.toFixed(0).padStart(4)} j/s`,
|
||||
);
|
||||
}
|
||||
|
||||
// --- Projections ---
|
||||
|
||||
const workerCounts = [1, 2, 3, 5, 8, 10, 15, 20]; // native workers (×8 subworkers each)
|
||||
const execTimes = [
|
||||
{ ms: 0, label: "~0ms (identity)" },
|
||||
{ ms: 5, label: "5ms" },
|
||||
{ ms: 20, label: "20ms" },
|
||||
{ ms: 50, label: "50ms" },
|
||||
{ ms: 200, label: "200ms" },
|
||||
{ ms: 500, label: "500ms" },
|
||||
];
|
||||
|
||||
console.log("\n\n=== Projected Throughput (jobs/s) ===\n");
|
||||
|
||||
for (const exec of execTimes) {
|
||||
console.log(`--- Job duration: ${exec.label} ---\n`);
|
||||
console.log(
|
||||
" Native workers (subw) | Batch | SQL | Advantage | Batch wins?",
|
||||
);
|
||||
console.log(
|
||||
" ----------------------|-----------|----------|-------------|------------",
|
||||
);
|
||||
|
||||
for (const w of workerCounts) {
|
||||
const n = w * 8;
|
||||
const b = throughputBatch(n, exec.ms);
|
||||
const s = throughputSql(n, exec.ms);
|
||||
const advantage = pct(b, s);
|
||||
const wins = b > s * 1.05 ? " YES" : b > s * 1.01 ? " marginal" : " no";
|
||||
console.log(
|
||||
` ${String(w).padStart(2)}W (${String(n).padStart(3)}) | ${b.toFixed(0).padStart(5)} j/s | ${s.toFixed(0).padStart(5)} j/s | ${advantage.padStart(8)} | ${wins}`,
|
||||
);
|
||||
}
|
||||
console.log();
|
||||
}
|
||||
|
||||
// --- Crossover analysis ---
|
||||
|
||||
console.log("=== Crossover: min workers where batch is >10% faster ===\n");
|
||||
console.log(" Job duration | Min workers | Subworkers | Batch j/s | SQL j/s");
|
||||
console.log(" -------------|-------------|------------|-----------|--------");
|
||||
|
||||
for (const exec of execTimes) {
|
||||
let found = false;
|
||||
for (let w = 1; w <= 50; w++) {
|
||||
const n = w * 8;
|
||||
const b = throughputBatch(n, exec.ms);
|
||||
const s = throughputSql(n, exec.ms);
|
||||
if (b > s * 1.1) {
|
||||
console.log(
|
||||
` ${exec.label.padEnd(13)}| ${String(w).padStart(5)}W | ${String(n).padStart(5)} | ${b.toFixed(0).padStart(5)} j/s | ${s.toFixed(0).padStart(5)} j/s`,
|
||||
);
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!found) {
|
||||
console.log(
|
||||
` ${exec.label.padEnd(13)}| >50W (never significant at this job duration)`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
console.log("\n\n=== Key Takeaways ===\n");
|
||||
console.log(
|
||||
"1. For fast jobs (~0ms): batch pull is always faster, advantage grows with scale",
|
||||
);
|
||||
console.log(" - 5 native workers (40 subworkers): ~13% faster");
|
||||
console.log(" - 10 native workers (80 subworkers): ~25% faster");
|
||||
console.log(" - 20 native workers (160 subworkers): ~88% faster");
|
||||
console.log(
|
||||
"2. For medium jobs (50ms): batch advantage meaningful from ~5 native workers",
|
||||
);
|
||||
console.log(
|
||||
"3. For slow jobs (500ms+): only matters at 15+ native workers (120+ subworkers)",
|
||||
);
|
||||
console.log(
|
||||
" (but still reduces DB load — fewer pull queries, less index scanning)",
|
||||
);
|
||||
console.log(
|
||||
"4. The SQL quadratic contention (SKIP LOCKED scanning) is the dominant factor",
|
||||
);
|
||||
console.log(
|
||||
" — SQL throughput plateaus around 15-20 native workers while batch keeps scaling",
|
||||
);
|
||||
@@ -1,93 +0,0 @@
|
||||
# Batch Pull Benchmark Results
|
||||
|
||||
Date: 2026-03-06
|
||||
Setup: 1 server + 1 native worker (8 subworkers), standalone mode
|
||||
Hardware: fedora, 1.5TB disk, ~1GB memory usage
|
||||
DB: PostgreSQL local, windmill 270 MiB
|
||||
|
||||
## nativets — 1000 jobs
|
||||
|
||||
| | Batch Pull | Direct SQL |
|
||||
|--|-----------|------------|
|
||||
| Duration | 9.2s | 11.4s |
|
||||
| **Throughput** | **108 jobs/s** | **88 jobs/s** |
|
||||
| Improvement | **+23%** | baseline |
|
||||
|
||||
### pg_stat_statements
|
||||
|
||||
| Query | Batch calls | Batch ms | SQL calls | SQL ms |
|
||||
|-------|------------|----------|-----------|--------|
|
||||
| Native pull (FOR UPDATE SKIP LOCKED) | 2,399 (0.02ms avg) | 47 | 2,849 (0.03ms avg) | 87 |
|
||||
| Default worker pull | 520 (0.04ms avg) | 19 | 445 (0.05ms avg) | 23 |
|
||||
| DELETE from queue | 1,001 (0.23ms avg) | 231 | 1,001 (0.19ms avg) | 195 |
|
||||
| INSERT into completed | 1,001 (0.05ms avg) | 51 | 1,001 (0.04ms avg) | 45 |
|
||||
| INSERT job_logs | 2,003 (0.02ms avg) | 44 | 2,003 (0.02ms avg) | 44 |
|
||||
| Agent token blacklist | 3,920 (0.00ms avg) | 19 | — | — |
|
||||
| **Total** | **9,389** | **1,332** | **8,882** | **1,212** |
|
||||
|
||||
### pg_stat_database
|
||||
|
||||
| Metric | Batch Pull | Direct SQL |
|
||||
|--------|-----------|------------|
|
||||
| Transactions committed | 6,484 | 5,868 |
|
||||
| Blocks read (disk) | 101 | 101 |
|
||||
| Blocks hit (cache) | 920,257 | 699,032 |
|
||||
| Tuples returned | 7,784,044 | 8,987,097 |
|
||||
| Tuples fetched | 1,028,819 | 805,100 |
|
||||
| Tuples inserted | 6,822 | 6,880 |
|
||||
| Tuples updated | 3,377 | 3,090 |
|
||||
| Tuples deleted | 2,883 | 2,654 |
|
||||
|
||||
---
|
||||
|
||||
## nativets_sleep — 1000 jobs
|
||||
|
||||
Each job sleeps 300-700ms (random). Theoretical max with 8 workers: ~16 jobs/s.
|
||||
|
||||
| | Batch Pull | Direct SQL |
|
||||
|--|-----------|------------|
|
||||
| Duration | 66.7s | 68.2s |
|
||||
| **Throughput** | **15.0 jobs/s** | **14.7 jobs/s** |
|
||||
| Improvement | ~same | baseline |
|
||||
|
||||
### pg_stat_statements
|
||||
|
||||
| Query | Batch calls | Batch ms | SQL calls | SQL ms |
|
||||
|-------|------------|----------|-----------|--------|
|
||||
| Native pull (FOR UPDATE SKIP LOCKED) | 2,399 (0.02ms avg) | 43 | 1,762 (0.04ms avg) | 75 |
|
||||
| Default worker pull | 1,591 (0.07ms avg) | 113 | 1,392 (0.08ms avg) | 115 |
|
||||
| DELETE from queue | 1,001 (0.31ms avg) | 308 | 1,001 (0.20ms avg) | 205 |
|
||||
| INSERT into completed | 1,001 (0.05ms avg) | 46 | 1,001 (0.04ms avg) | 41 |
|
||||
| INSERT job_logs | 2,003 (0.02ms avg) | 45 | 2,003 (0.02ms avg) | 40 |
|
||||
| Job runtime ping | 1,490 (0.02ms avg) | 33 | 1,489 (0.02ms avg) | 31 |
|
||||
| Worker ping (job) | 1,001 (0.03ms avg) | 32 | 1,001 (0.03ms avg) | 30 |
|
||||
| **Total** | **16,523** | **5,798** | **16,364** | **5,717** |
|
||||
|
||||
### pg_stat_database
|
||||
|
||||
| Metric | Batch Pull | Direct SQL |
|
||||
|--------|-----------|------------|
|
||||
| Transactions committed | 13,638 | 13,388 |
|
||||
| Blocks read (disk) | 394 | 850 |
|
||||
| Blocks hit (cache) | 7,033,158 | 4,932,617 |
|
||||
| Tuples returned | 58,424,571 | 57,204,918 |
|
||||
| Tuples fetched | 8,776,186 | 6,321,947 |
|
||||
| Tuples inserted | 7,500 | 7,531 |
|
||||
| Tuples updated | 6,111 | 6,193 |
|
||||
| Tuples deleted | 3,006 | 3,196 |
|
||||
|
||||
---
|
||||
|
||||
## Analysis
|
||||
|
||||
**Throughput**: +23% for fast CPU-bound jobs. Negligible difference for I/O-bound jobs.
|
||||
|
||||
**Pull queries**: Batch pull does MORE pull queries for nativets_sleep (2,399 vs 1,762). The refiller polls every 50ms even when all workers are busy executing jobs. With direct SQL, workers only poll when idle. This is wasted work — the refiller queries DB and gets empty results while jobs are in-flight.
|
||||
|
||||
**Disk I/O**: Batch pull cuts disk reads in half for nativets_sleep (394 vs 850 blocks). Likely because the batch query locks multiple rows in one pass, reducing index traversal.
|
||||
|
||||
**Cache hits**: Higher with batch pull (7M vs 4.9M for sleep). More buffer hits from the refiller's repeated empty polls touching the same index pages.
|
||||
|
||||
**Tuples fetched**: Higher with batch pull (8.7M vs 6.3M for sleep). Same cause — the refiller's empty polls scan the index.
|
||||
|
||||
**At scale**: With 8 subworkers the differences are small. The real benefit is with many native workers where direct SQL SKIP LOCKED contention grows O(N²).
|
||||
@@ -1,123 +0,0 @@
|
||||
# Batch Pull Benchmark Results — 3 Workers
|
||||
|
||||
Date: 2026-03-06
|
||||
Setup: 1 server + 3 native workers (8 subworkers each = 24 subworkers)
|
||||
Hardware: fedora, 1.5TB disk, ~1GB memory usage
|
||||
DB: PostgreSQL local, windmill 270 MiB
|
||||
|
||||
## nativets — 1000 jobs
|
||||
|
||||
| | 3W Batch | 3W SQL | 1W Batch | 1W SQL |
|
||||
|--|---------|--------|---------|--------|
|
||||
| Duration | 3.7s | 3.5s | 9.2s | 11.4s |
|
||||
| **Throughput** | **272 jobs/s** | **288 jobs/s** | **108 jobs/s** | **88 jobs/s** |
|
||||
| vs 1W SQL | +209% | +227% | +23% | baseline |
|
||||
|
||||
Note: First 3W SQL run was 33 jobs/s (outlier due to cold start or background activity). Rerun gave 288 jobs/s.
|
||||
|
||||
## nativets — 10,000 jobs
|
||||
|
||||
| | 3W Batch | 3W SQL |
|
||||
|--|---------|--------|
|
||||
| Duration | 34.3s | 39.5s |
|
||||
| **Throughput** | **291 jobs/s** | **253 jobs/s** |
|
||||
| Improvement | **+15%** | baseline |
|
||||
|
||||
### pg_stat_statements (1000 jobs, first run)
|
||||
|
||||
| Query | 3W Batch calls | 3W Batch ms | 3W SQL calls | 3W SQL ms |
|
||||
|-------|---------------|-------------|-------------|----------|
|
||||
| Native pull (FOR UPDATE SKIP LOCKED) | 4,801 (0.01ms) | 59 | 20,367 (0.01ms) | 231 |
|
||||
| Default worker pull | 353 (0.03ms) | 10 | 880 (0.02ms) | 16 |
|
||||
| DELETE from queue | 1,001 (0.44ms) | 439 | 1,001 (0.43ms) | 427 |
|
||||
| INSERT into completed | 1,001 (0.06ms) | 61 | 1,001 (0.05ms) | 55 |
|
||||
| INSERT job_logs | 2,003 (0.03ms) | 67 | 2,003 (0.03ms) | 64 |
|
||||
| Agent token blacklist | 7,867 (0.00ms) | 33 | — | — |
|
||||
| Worker ping (job) | 350 (0.04ms) | 15 | — | — |
|
||||
| Outstanding wait time | 664 (0.03ms) | 21 | 742 (0.03ms) | 23 |
|
||||
|
||||
### pg_stat_database (1000 jobs, first run)
|
||||
|
||||
| Metric | 3W Batch | 3W SQL |
|
||||
|--------|---------|--------|
|
||||
| Transactions committed | 22,070 | 29,301 |
|
||||
| Blocks read (disk) | 308 | 395 |
|
||||
| Blocks hit (cache) | 280,071 | 1,276,521 |
|
||||
| Tuples returned | 3,368,255 | 28,695,830 |
|
||||
| Tuples fetched | 140,864 | 1,089,774 |
|
||||
| Tuples inserted | 6,682 | 6,760 |
|
||||
| Tuples updated | 3,521 | 3,453 |
|
||||
| Tuples deleted | 3,007 | 3,007 |
|
||||
|
||||
---
|
||||
|
||||
## nativets_sleep — 1000 jobs
|
||||
|
||||
Each job sleeps 300-700ms (random). Theoretical max with 24 workers: ~48 jobs/s.
|
||||
|
||||
| | 3W Batch | 3W SQL | 1W Batch | 1W SQL |
|
||||
|--|---------|--------|---------|--------|
|
||||
| Duration | 22.8s | 22.8s | 66.7s | 68.2s |
|
||||
| **Throughput** | **43.8 jobs/s** | **43.8 jobs/s** | **15.0 jobs/s** | **14.7 jobs/s** |
|
||||
| vs 3W SQL | ~same | baseline | — | — |
|
||||
|
||||
### pg_stat_statements
|
||||
|
||||
| Query | 3W Batch calls | 3W Batch ms | 3W SQL calls | 3W SQL ms |
|
||||
|-------|---------------|-------------|-------------|----------|
|
||||
| Native pull (FOR UPDATE SKIP LOCKED) | 4,898 (0.01ms) | 55 | 6,440 (0.02ms) | 113 |
|
||||
| Default worker pull | 696 (0.06ms) | 43 | 654 (0.06ms) | 37 |
|
||||
| DELETE from queue | 1,001 (0.38ms) | 379 | 1,001 (0.29ms) | 290 |
|
||||
| INSERT into completed | 1,001 (0.05ms) | 46 | 1,001 (0.04ms) | 43 |
|
||||
| INSERT job_logs | 2,003 (0.02ms) | 44 | 2,003 (0.02ms) | 43 |
|
||||
| Agent token blacklist | 7,295 (0.00ms) | 31 | — | — |
|
||||
| Job runtime ping | 1,499 (0.02ms) | 35 | 1,444 (0.02ms) | 35 |
|
||||
| Worker ping (job) | 1,001 (0.03ms) | 32 | 1,001 (0.03ms) | 31 |
|
||||
| Job stats | 549 (0.05ms) | 27 | 493 (0.05ms) | 26 |
|
||||
| Outstanding wait time | 928 (0.02ms) | 20 | 944 (0.02ms) | 21 |
|
||||
|
||||
### pg_stat_database
|
||||
|
||||
| Metric | 3W Batch | 3W SQL |
|
||||
|--------|---------|--------|
|
||||
| Transactions committed | 25,896 | 18,646 |
|
||||
| Blocks read (disk) | 115 | 204 |
|
||||
| Blocks hit (cache) | 1,173,696 | 1,256,397 |
|
||||
| Tuples returned | 21,074,537 | 22,063,593 |
|
||||
| Tuples fetched | 1,200,878 | 1,262,900 |
|
||||
| Tuples inserted | 7,495 | 7,461 |
|
||||
| Tuples updated | 6,204 | 6,141 |
|
||||
| Tuples deleted | 3,005 | 3,011 |
|
||||
|
||||
---
|
||||
|
||||
## Analysis
|
||||
|
||||
### nativets (CPU-bound): +15% with 10K jobs
|
||||
|
||||
With 10,000 jobs, batch pull achieves **291 jobs/s vs 253 jobs/s** (+15%). The 1000-job runs showed similar throughput (~272-288 jobs/s) after discarding the cold-start outlier.
|
||||
|
||||
**DB load difference** (from the 1000-job first run, which captured the worst-case SQL contention):
|
||||
- **20,367 pull queries** (SQL) vs 4,801 (batch) — 4x more queries
|
||||
- **28.7M tuples returned** (SQL) vs 3.4M (batch) — 8.5x more index scanning
|
||||
- **1.3M cache hits** (SQL) vs 280K (batch) — 4.6x more buffer activity
|
||||
|
||||
The batch approach consolidates all 24 subworkers into a single `LIMIT 24` query, reducing contention on the queue index.
|
||||
|
||||
### nativets_sleep (I/O-bound): No throughput difference
|
||||
|
||||
Both achieve **43.8 jobs/s** (91% of theoretical 48 jobs/s max). When workers spend 300-700ms sleeping, DB contention isn't the bottleneck.
|
||||
|
||||
Batch pull still shows slightly lower DB load:
|
||||
- **4,898 pull queries** vs 6,440 — 24% fewer
|
||||
- **115 disk reads** vs 204 — 44% fewer
|
||||
|
||||
### Scaling summary
|
||||
|
||||
| Setup | Batch jobs/s | SQL jobs/s | Batch advantage |
|
||||
|-------|-------------|-----------|----------------|
|
||||
| 1W × 1000 jobs | 108 | 88 | +23% |
|
||||
| 3W × 1000 jobs | 272 | 288 | ~same |
|
||||
| 3W × 10,000 jobs | 291 | 253 | **+15%** |
|
||||
|
||||
At 24 subworkers, batch pull provides a consistent ~15% throughput improvement for sustained CPU-bound workloads, with significantly lower DB load (4x fewer pull queries, 8x fewer tuples scanned). The benefit grows with more workers as SKIP LOCKED contention scales O(N²).
|
||||
@@ -39,10 +39,7 @@ export const scriptBootstrapCode = {
|
||||
}
|
||||
`,
|
||||
|
||||
bun: `// there are multiple modes to add as header: //nobundling //native //npm //nodejs
|
||||
// https://www.windmill.dev/docs/getting_started/scripts_quickstart/typescript#modes
|
||||
|
||||
export async function main() {
|
||||
bun: `export async function main() {
|
||||
return "Hello world";
|
||||
}
|
||||
`,
|
||||
|
||||
@@ -67,7 +67,7 @@ export {
|
||||
workspaceAdd,
|
||||
};
|
||||
|
||||
export const VERSION = "1.651.1";
|
||||
export const VERSION = "1.650.0";
|
||||
|
||||
// Re-exported from constants.ts to maintain backwards compatibility
|
||||
export { WM_FORK_PREFIX } from "./core/constants.ts";
|
||||
|
||||
4
frontend/package-lock.json
generated
4
frontend/package-lock.json
generated
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "windmill-components",
|
||||
"version": "1.651.1",
|
||||
"version": "1.650.0",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "windmill-components",
|
||||
"version": "1.651.1",
|
||||
"version": "1.650.0",
|
||||
"hasInstallScript": true,
|
||||
"license": "AGPL-3.0",
|
||||
"dependencies": {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "windmill-components",
|
||||
"version": "1.651.1",
|
||||
"version": "1.650.0",
|
||||
"scripts": {
|
||||
"dev": "vite dev",
|
||||
"build": "vite build",
|
||||
|
||||
@@ -133,7 +133,7 @@ export function useJobsLoader(args: () => UseJobLoaderArgs) {
|
||||
])
|
||||
})
|
||||
}
|
||||
promise = CancelablePromiseUtils.finallyDo(promise, () => {
|
||||
promise = CancelablePromiseUtils.pipe(promise, () => {
|
||||
if (slowStreamIntervalId) {
|
||||
clearInterval(slowStreamIntervalId)
|
||||
slowStreamIntervalId = undefined
|
||||
@@ -161,7 +161,7 @@ export function useJobsLoader(args: () => UseJobLoaderArgs) {
|
||||
)
|
||||
}, 15000)
|
||||
paramChangePromise = loadJobsIntern(false, size)
|
||||
paramChangePromise = CancelablePromiseUtils.finallyDo(paramChangePromise, () => {
|
||||
paramChangePromise = CancelablePromiseUtils.pipe(paramChangePromise, () => {
|
||||
if (slowStreamIntervalId) {
|
||||
clearInterval(slowStreamIntervalId)
|
||||
slowStreamIntervalId = undefined
|
||||
@@ -687,10 +687,6 @@ export function useJobsLoader(args: () => UseJobLoaderArgs) {
|
||||
return () => {
|
||||
clearTimeout(paramChangeTimeout)
|
||||
paramChangePromise?.cancel()
|
||||
if (slowStreamIntervalId) {
|
||||
clearInterval(slowStreamIntervalId)
|
||||
slowStreamIntervalId = undefined
|
||||
}
|
||||
}
|
||||
})
|
||||
$effect(() => {
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
import { safeSelectItems } from '../select/utils.svelte'
|
||||
import TokenDisplay from './TokenDisplay.svelte'
|
||||
import ScopeSelector from './ScopeSelector.svelte'
|
||||
|
||||
import Alert from '../common/alert/Alert.svelte'
|
||||
import FolderPicker from '../FolderPicker.svelte'
|
||||
import TextInput from '../text_input/TextInput.svelte'
|
||||
import Select from '../select/Select.svelte'
|
||||
@@ -43,6 +43,8 @@
|
||||
displayCreateToken = true
|
||||
}: Props = $props()
|
||||
|
||||
// MCP clients do not allow names longer than 60 characters, here we use 55 because final tool name server side will add ~5 characters
|
||||
const MAX_PATH_LENGTH = 55
|
||||
|
||||
let newToken = $state<string | undefined>(undefined)
|
||||
let newMcpToken = $state<string | undefined>(undefined)
|
||||
@@ -183,6 +185,17 @@
|
||||
? `You do not have any favorite scripts or flows. You can favorite some scripts and flows to include them, or change the scope to "All scripts/flows" to include all your scripts and flows.`
|
||||
: `You do not have any scripts or flows in the selected folder.`
|
||||
)
|
||||
const longPathRunnables = $derived(
|
||||
includedRunnables.filter((path) => path.length > MAX_PATH_LENGTH)
|
||||
)
|
||||
const validRunnables = $derived(
|
||||
includedRunnables.filter((path) => path.length <= MAX_PATH_LENGTH)
|
||||
)
|
||||
const longPathWarning = $derived(
|
||||
longPathRunnables.length > 0
|
||||
? `${longPathRunnables.length} script(s)/flow(s) have paths longer than 60 characters and will be excluded from MCP tools. Consider shortening the paths: ${longPathRunnables.slice(0, 3).join(', ')}${longPathRunnables.length > 3 ? ` and ${longPathRunnables.length - 3} more` : ''}`
|
||||
: ''
|
||||
)
|
||||
|
||||
$effect(() => {
|
||||
if (mcpCreationMode) {
|
||||
@@ -614,18 +627,23 @@
|
||||
</div>
|
||||
{:else}
|
||||
<div class="flex flex-col gap-2 col-span-2 pr-4">
|
||||
{#if longPathWarning}
|
||||
<Alert type="warning" title="Some paths are too long" size="xs">
|
||||
{longPathWarning}
|
||||
</Alert>
|
||||
{/if}
|
||||
<span class="block text-xs">Scripts & Flows that will be available via MCP</span>
|
||||
<div class="flex flex-wrap gap-1">
|
||||
{#if includedRunnables.length > 0 && includedRunnables.length <= 5}
|
||||
{#each includedRunnables as scriptOrFlow}
|
||||
{#if validRunnables.length > 0 && validRunnables.length <= 5}
|
||||
{#each validRunnables as scriptOrFlow}
|
||||
<Badge rounded small color="blue">{scriptOrFlow}</Badge>
|
||||
{/each}
|
||||
{:else if includedRunnables.length > 0}
|
||||
{#each includedRunnables.slice(0, 3) as scriptOrFlow}
|
||||
{:else if validRunnables.length > 0}
|
||||
{#each validRunnables.slice(0, 3) as scriptOrFlow}
|
||||
<Badge rounded small color="blue">{scriptOrFlow}</Badge>
|
||||
{/each}
|
||||
<Badge rounded small color="dark-gray">
|
||||
+{includedRunnables.length - 3} more
|
||||
+{validRunnables.length - 3} more
|
||||
</Badge>
|
||||
{:else}
|
||||
<p class="text-xs text-primary">
|
||||
|
||||
@@ -4,7 +4,7 @@ verify_ssl = true
|
||||
name = "pypi"
|
||||
|
||||
[packages]
|
||||
wmill = ">=1.651.1"
|
||||
wmill = ">=1.650.0"
|
||||
sendgrid = "*"
|
||||
mysql-connector-python = "*"
|
||||
pymongo = "*"
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
openapi: '3.0.3'
|
||||
|
||||
info:
|
||||
version: 1.651.1
|
||||
version: 1.650.0
|
||||
title: OpenFlow Spec
|
||||
contact:
|
||||
name: Ruben Fiszel
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
RootModule = 'WindmillClient.psm1'
|
||||
|
||||
# Version number of this module.
|
||||
ModuleVersion = '1.651.1'
|
||||
ModuleVersion = '1.650.0'
|
||||
|
||||
# Supported PSEditions
|
||||
# CompatiblePSEditions = @()
|
||||
|
||||
@@ -108,21 +108,6 @@ SET s3_secret_access_key='80yMndIMcyXwEujxVNINQbf0tBlIzRaLPyM2m1n4';
|
||||
)
|
||||
print(file_key)
|
||||
|
||||
@unittest.skip("skipping")
|
||||
def test_delete_s3_object(self):
|
||||
# Upload a temporary file
|
||||
s3_obj = wmill.write_s3_file(
|
||||
S3Object(s3="_wmill_test_delete_s3_object.txt"), b"delete_s3_object test content"
|
||||
)
|
||||
# Verify it exists
|
||||
content = wmill.load_s3_file(s3_obj)
|
||||
self.assertEqual(content, b"delete_s3_object test content")
|
||||
# Delete it
|
||||
wmill.delete_s3_object(s3_obj)
|
||||
# Verify it's gone
|
||||
with self.assertRaises(Exception):
|
||||
wmill.load_s3_file(s3_obj)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "wmill"
|
||||
version = "1.651.1"
|
||||
version = "1.650.0"
|
||||
description = "A client library for accessing Windmill server wrapping the Windmill client API"
|
||||
license = "Apache-2.0"
|
||||
homepage = "https://windmill.dev"
|
||||
|
||||
@@ -984,40 +984,6 @@ class Windmill:
|
||||
raise Exception("Could not write file to S3") from e
|
||||
return S3Object(s3=response["file_key"], storage=s3object.get("storage") if s3object else None)
|
||||
|
||||
def delete_s3_object(
|
||||
self,
|
||||
s3object: S3Object | str,
|
||||
s3_resource_path: str | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Permanently delete a file from the workspace S3 bucket.
|
||||
|
||||
'''python
|
||||
from wmill import S3Object
|
||||
|
||||
s3_obj = S3Object(s3="/path/to/my_file.txt")
|
||||
client.delete_s3_object(s3_obj)
|
||||
'''
|
||||
"""
|
||||
s3object = parse_s3_object(s3object)
|
||||
query_params: Dict[str, Any] = {"file_key": s3object["s3"]}
|
||||
if s3_resource_path is not None and s3_resource_path != "":
|
||||
query_params["s3_resource_path"] = s3_resource_path
|
||||
if "storage" in s3object and s3object["storage"] is not None:
|
||||
query_params["storage"] = s3object["storage"]
|
||||
try:
|
||||
resp = self.client.delete(
|
||||
f"/w/{self.workspace}/job_helpers/delete_s3_file",
|
||||
params=query_params,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
except httpx.HTTPStatusError as err:
|
||||
error = f"{err.request.url}: {err.response.status_code}, {err.response.text}"
|
||||
logger.error(error)
|
||||
raise Exception(error)
|
||||
except Exception as e:
|
||||
raise Exception("Could not delete file from S3") from e
|
||||
|
||||
def sign_s3_objects(self, s3_objects: list[S3Object | str]) -> list[S3Object]:
|
||||
"""Sign S3 objects for use by anonymous users in public apps.
|
||||
|
||||
@@ -1726,20 +1692,6 @@ def write_s3_file(
|
||||
)
|
||||
|
||||
|
||||
@init_global_client
|
||||
def delete_s3_object(
|
||||
s3object: S3Object | str,
|
||||
s3_resource_path: str | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Permanently delete a file from the workspace S3 bucket.
|
||||
"""
|
||||
return _client.delete_s3_object(
|
||||
s3object,
|
||||
s3_resource_path if s3_resource_path != "" else None,
|
||||
)
|
||||
|
||||
|
||||
@init_global_client
|
||||
def sign_s3_objects(s3_objects: list[S3Object | str]) -> list[S3Object]:
|
||||
"""
|
||||
@@ -2416,4 +2368,4 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]]:
|
||||
name, schema = name.split(":", 1)
|
||||
if not name:
|
||||
name = "main"
|
||||
return name, schema
|
||||
return name, schema
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@windmill/windmill",
|
||||
"version": "1.651.1",
|
||||
"version": "1.650.0",
|
||||
"exports": "./src/index.ts",
|
||||
"publish": {
|
||||
"exclude": ["!src", "./s3Types.ts", "./sqlUtils.ts", "./client.ts"]
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "windmill-client",
|
||||
"description": "Windmill SDK client for browsers and Node.js",
|
||||
"version": "1.651.1",
|
||||
"version": "1.650.0",
|
||||
"author": "Ruben Fiszel",
|
||||
"license": "Apache 2.0",
|
||||
"sideEffects": false,
|
||||
|
||||
@@ -1 +1 @@
|
||||
1.651.1
|
||||
1.650.0
|
||||
|
||||
Reference in New Issue
Block a user