From b31d8dffc323a51121df4baa02c5df1fa08fceeb Mon Sep 17 00:00:00 2001 From: Pyra <92104930+pyranota@users.noreply.github.com> Date: Wed, 7 Jan 2026 05:07:26 +0100 Subject: [PATCH] feat(git-sync): sync jobs debouncing for greatly improved perf (#7489) * feat(git-sync): sync jobs debouncing [merge-ee-first] * add compat * ee repo Signed-off-by: pyranota * nit Signed-off-by: pyranota * change is_script_meets_min_version Signed-off-by: pyranota * update version Signed-off-by: pyranota * chore: update ee-repo-ref to bc904859dd66c55ebad002e8526103c73de841cd This commit updates the EE repository reference after PR #371 was merged in windmill-ee-private. Previous ee-repo-ref: 346d38e91776e987f88866543adbf78a9c721073 New ee-repo-ref: bc904859dd66c55ebad002e8526103c73de841cd Automated by sync-ee-ref workflow. --------- Signed-off-by: pyranota Co-authored-by: windmill-internal-app[bot] Co-authored-by: Ruben Fiszel --- backend/ee-repo-ref.txt | 2 +- backend/windmill-common/src/jobs.rs | 2 +- backend/windmill-common/src/worker.rs | 3 +++ backend/windmill-common/src/workspaces.rs | 28 ++++++++++++++++++++++- backend/windmill-queue/src/jobs.rs | 3 ++- 5 files changed, 34 insertions(+), 4 deletions(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 91f000434d..2b9e540226 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -ea468d0b673257c694203517b0c5df52d9cb70dd +bc904859dd66c55ebad002e8526103c73de841cd diff --git a/backend/windmill-common/src/jobs.rs b/backend/windmill-common/src/jobs.rs index ebdb16bcdc..f2d50d2d8e 100644 --- a/backend/windmill-common/src/jobs.rs +++ b/backend/windmill-common/src/jobs.rs @@ -482,7 +482,7 @@ pub enum JobPayload { }, DeploymentCallback { path: String, - // debouncing_settings: Option, + debouncing_settings: DebouncingSettings, }, Identity, Noop, diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 0a847cefd7..f3a85b4e0a 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -265,6 +265,7 @@ lazy_static::lazy_static! { .unwrap_or(false); pub static ref MIN_VERSION: Arc> = Arc::new(RwLock::new(Version::new(0, 0, 0))); + pub static ref MIN_VERSION_SUPPORTS_SYNC_JOBS_DEBOUNCING: Arc> = Arc::new(RwLock::new(false)); pub static ref MIN_VERSION_SUPPORTS_DEBOUNCING_V2: Arc> = Arc::new(RwLock::new(false)); pub static ref MIN_VERSION_SUPPORTS_RUNNABLE_SETTINGS_V0: Arc> = Arc::new(RwLock::new(false)); /// Global flag indicating if all workers support workspace dependencies feature (>= 1.583.0) @@ -1295,6 +1296,8 @@ pub async fn update_min_version(conn: &Connection) -> bool { tracing::info!("Minimal worker version: {min_version}"); } + *MIN_VERSION_SUPPORTS_SYNC_JOBS_DEBOUNCING.write().await = + min_version >= Version::new(1, 602, 0); *MIN_VERSION_SUPPORTS_DEBOUNCING_V2.write().await = min_version >= Version::new(1, 597, 0); *MIN_VERSION_SUPPORTS_RUNNABLE_SETTINGS_V0.write().await = min_version >= *crate::runnable_settings::MIN_VERSION_RUNNABLE_SETTINGS_V0; diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index 06237536ce..3668d86978 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -6,7 +6,7 @@ use serde::{Deserialize, Serialize}; use strum::AsRefStr; use crate::{ - error::{to_anyhow, Error, Result}, + error::{self, to_anyhow, Error, Result}, get_database_url, utils::get_custom_pg_instance_password, variables::{build_crypt, decrypt}, @@ -63,6 +63,32 @@ pub struct GitRepositorySettings { pub settings: Option, } +impl GitRepositorySettings { + pub fn is_script_meets_min_version(&self, min_version: u32) -> error::Result { + // example: "hub/28102/sync-script-to-git-repo-windmill" + let current = self + .script_path + .split("/") // -> ["hub" "28102" "sync-script-to-git-repo-windmill"] + .skip(1) // omit "hub" + .next() // get numeric id + .ok_or(Error::InternalErr(format!( + "cannot get script version id from: {}", + &self.script_path + )))? + .parse() + .unwrap_or_else(|e| { + tracing::warn!( + "cannot get script version id from: {}. e: {e}", + &self.script_path + ); + + u32::MAX + }); + + Ok(current >= min_version) // this works on assumption that all scripts in hub have sequential ids + } +} + #[derive(Serialize, Deserialize, Debug)] pub struct GitSyncSettings { pub include_path: Vec, diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 9b58fc2016..cb3bdbdf92 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -4843,7 +4843,7 @@ pub async fn push<'c, 'd>( ..Default::default() } } - JobPayload::DeploymentCallback { path } => JobPayloadUntagged { + JobPayload::DeploymentCallback { path, debouncing_settings } => JobPayloadUntagged { runnable_path: Some(path.clone()), job_kind: JobKind::DeploymentCallback, concurrency_settings: ConcurrencySettings { @@ -4851,6 +4851,7 @@ pub async fn push<'c, 'd>( concurrent_limit: Some(1), concurrency_time_window_s: Some(0), }, + debouncing_settings, ..Default::default() }, JobPayload::Identity => {