feat: batch job pulling for native workers
Reduce DB polling overhead for native workers by batch-fetching jobs server-side and serving them from an in-memory buffer via HTTP. - Add batch_pull() in windmill-queue: single SELECT...FOR UPDATE SKIP LOCKED LIMIT N - Add batch pull SQL helpers (make_batch_pull_query, format_batch_pull_query) - OSS stubs for agent-workers accept batch_buffer parameter (4-tuple return) - Native workers self-sign JWT and pull jobs via HTTP when co-located with server - Add uses_batch_http_pull column to worker_ping for server-side tracking - Worker pull loop: HTTP batch pull when client available, SQL otherwise Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -422,12 +422,16 @@ pub async fn run_server(
|
||||
};
|
||||
|
||||
#[cfg(feature = "agent_worker_server")]
|
||||
let (agent_workers_router, agent_workers_bg_processor, agent_workers_job_completed_tx) =
|
||||
if server_mode {
|
||||
windmill_api_agent_workers::workspaced_service(db.clone(), _base_internal_url.clone())
|
||||
} else {
|
||||
(Router::new(), vec![], None)
|
||||
};
|
||||
let (
|
||||
agent_workers_router,
|
||||
agent_workers_bg_processor,
|
||||
agent_workers_job_completed_tx,
|
||||
batch_buffer,
|
||||
) = if server_mode {
|
||||
windmill_api_agent_workers::workspaced_service(db.clone(), _base_internal_url.clone())
|
||||
} else {
|
||||
(Router::new(), vec![], None, None)
|
||||
};
|
||||
|
||||
#[cfg(feature = "agent_worker_server")]
|
||||
let agent_cache = Arc::new(AgentCache::new());
|
||||
@@ -612,6 +616,7 @@ pub async fn run_server(
|
||||
{
|
||||
windmill_api_agent_workers::global_service(
|
||||
agent_workers_job_completed_tx,
|
||||
batch_buffer.clone(),
|
||||
)
|
||||
.layer(Extension(agent_cache.clone()))
|
||||
} else {
|
||||
|
||||
Reference in New Issue
Block a user