feat: batch job pulling for native workers
Reduce DB polling overhead for native workers by batch-fetching jobs server-side and serving them from an in-memory buffer via HTTP. - Add batch_pull() in windmill-queue: single SELECT...FOR UPDATE SKIP LOCKED LIMIT N - Add batch pull SQL helpers (make_batch_pull_query, format_batch_pull_query) - OSS stubs for agent-workers accept batch_buffer parameter (4-tuple return) - Native workers self-sign JWT and pull jobs via HTTP when co-located with server - Add uses_batch_http_pull column to worker_ping for server-side tracking - Worker pull loop: HTTP batch pull when client available, SQL otherwise Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -3434,6 +3434,38 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<'c>(
|
||||
Ok(job_and_suspended)
|
||||
}
|
||||
|
||||
/// Batch-pull up to `limit` jobs in a single query, marking them all as running.
|
||||
/// The caller controls which tags are queried, so flow/dependency jobs are never
|
||||
/// pulled (they use distinct tags like "flow" / "dependency").
|
||||
pub async fn batch_pull(
|
||||
db: &Pool<Postgres>,
|
||||
worker_name: &str,
|
||||
tags: &[String],
|
||||
limit: u32,
|
||||
) -> windmill_common::error::Result<Vec<PulledJob>> {
|
||||
use windmill_common::worker::make_batch_pull_query;
|
||||
|
||||
if limit == 0 || tags.is_empty() {
|
||||
return Ok(vec![]);
|
||||
}
|
||||
|
||||
let query = make_batch_pull_query(tags, limit);
|
||||
let jobs: Vec<PulledJob> = timeout(
|
||||
Duration::from_secs(15),
|
||||
sqlx::query_as::<_, PulledJob>(&query)
|
||||
.bind(worker_name)
|
||||
.fetch_all(db),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
windmill_common::error::Error::internal_err(
|
||||
"batch_pull query timed out after 15s".to_string(),
|
||||
)
|
||||
})??;
|
||||
|
||||
Ok(jobs)
|
||||
}
|
||||
|
||||
pub async fn custom_concurrency_key(
|
||||
db: &Pool<Postgres>,
|
||||
job_id: &Uuid,
|
||||
|
||||
Reference in New Issue
Block a user