feat: batch job pulling for native workers
Reduce DB polling overhead for native workers by batch-fetching jobs server-side and serving them from an in-memory buffer via HTTP. - Add batch_pull() in windmill-queue: single SELECT...FOR UPDATE SKIP LOCKED LIMIT N - Add batch pull SQL helpers (make_batch_pull_query, format_batch_pull_query) - OSS stubs for agent-workers accept batch_buffer parameter (4-tuple return) - Native workers self-sign JWT and pull jobs via HTTP when co-located with server - Add uses_batch_http_pull column to worker_ping for server-side tracking - Worker pull loop: HTTP batch pull when client available, SQL otherwise Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
26
backend/.sqlx/query-3e8afd021088a99a24f27fa6f0a1b7f3edba3e9b834c814b464305bc2eb6ba80.json
generated
Normal file
26
backend/.sqlx/query-3e8afd021088a99a24f27fa6f0a1b7f3edba3e9b834c814b464305bc2eb6ba80.json
generated
Normal file
@@ -0,0 +1,26 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, dedicated_workers, wm_version, vcpus, memory, job_isolation, native_mode, uses_batch_http_pull) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) ON CONFLICT (worker)\n DO UPDATE set ip = EXCLUDED.ip, custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group, dedicated_workers = EXCLUDED.dedicated_workers, native_mode = EXCLUDED.native_mode, uses_batch_http_pull = EXCLUDED.uses_batch_http_pull",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Varchar",
|
||||
"Varchar",
|
||||
"Varchar",
|
||||
"TextArray",
|
||||
"Varchar",
|
||||
"Varchar",
|
||||
"TextArray",
|
||||
"Varchar",
|
||||
"Int8",
|
||||
"Int8",
|
||||
"Text",
|
||||
"Bool",
|
||||
"Bool"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "3e8afd021088a99a24f27fa6f0a1b7f3edba3e9b834c814b464305bc2eb6ba80"
|
||||
}
|
||||
26
backend/.sqlx/query-6cd099d458ac380d5da27b9e69da035755496ea50f2b78fb9b1cd3a2eb7e7625.json
generated
Normal file
26
backend/.sqlx/query-6cd099d458ac380d5da27b9e69da035755496ea50f2b78fb9b1cd3a2eb7e7625.json
generated
Normal file
@@ -0,0 +1,26 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1, custom_tags = $2,\n occupancy_rate = $3, memory_usage = $4, wm_memory_usage = $5, vcpus = COALESCE($7, vcpus),\n memory = COALESCE($8, memory), occupancy_rate_15s = $9, occupancy_rate_5m = $10, occupancy_rate_30m = $11, native_mode = $12, uses_batch_http_pull = $13 WHERE worker = $6",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Int4",
|
||||
"TextArray",
|
||||
"Float4",
|
||||
"Int8",
|
||||
"Int8",
|
||||
"Text",
|
||||
"Int8",
|
||||
"Int8",
|
||||
"Float4",
|
||||
"Float4",
|
||||
"Float4",
|
||||
"Bool",
|
||||
"Bool"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "6cd099d458ac380d5da27b9e69da035755496ea50f2b78fb9b1cd3a2eb7e7625"
|
||||
}
|
||||
Reference in New Issue
Block a user