feat(git-sync): sync jobs debouncing for greatly improved perf (#7489)

* feat(git-sync): sync jobs debouncing [merge-ee-first]

* add compat

* ee repo

Signed-off-by: pyranota <pyra@duck.com>

* nit

Signed-off-by: pyranota <pyra@duck.com>

* change is_script_meets_min_version

Signed-off-by: pyranota <pyra@duck.com>

* update version

Signed-off-by: pyranota <pyra@duck.com>

* chore: update ee-repo-ref to bc904859dd66c55ebad002e8526103c73de841cd

This commit updates the EE repository reference after PR #371 was merged in windmill-ee-private.

Previous ee-repo-ref: 346d38e91776e987f88866543adbf78a9c721073

New ee-repo-ref: bc904859dd66c55ebad002e8526103c73de841cd

Automated by sync-ee-ref workflow.

---------

Signed-off-by: pyranota <pyra@duck.com>
Co-authored-by: windmill-internal-app[bot] <windmill-internal-app[bot]@users.noreply.github.com>
Co-authored-by: Ruben Fiszel <ruben@windmill.dev>

backend/ee-repo-ref.txt

@@ -1 +1 @@
-ea468d0b673257c694203517b0c5df52d9cb70dd
+bc904859dd66c55ebad002e8526103c73de841cd

backend/windmill-common/src/jobs.rs

@@ -482,7 +482,7 @@ pub enum JobPayload {
     },
     DeploymentCallback {
         path: String,
-        // debouncing_settings: Option<DebouncingSettings>,
+        debouncing_settings: DebouncingSettings,
     },
     Identity,
     Noop,

backend/windmill-common/src/worker.rs

@@ -265,6 +265,7 @@ lazy_static::lazy_static! {
     .unwrap_or(false);
 
     pub static ref MIN_VERSION: Arc<RwLock<Version>> = Arc::new(RwLock::new(Version::new(0, 0, 0)));
+    pub static ref MIN_VERSION_SUPPORTS_SYNC_JOBS_DEBOUNCING: Arc<RwLock<bool>> = Arc::new(RwLock::new(false));
     pub static ref MIN_VERSION_SUPPORTS_DEBOUNCING_V2: Arc<RwLock<bool>> = Arc::new(RwLock::new(false));
     pub static ref MIN_VERSION_SUPPORTS_RUNNABLE_SETTINGS_V0: Arc<RwLock<bool>> = Arc::new(RwLock::new(false));
     /// Global flag indicating if all workers support workspace dependencies feature (>= 1.583.0)
@@ -1295,6 +1296,8 @@ pub async fn update_min_version(conn: &Connection) -> bool {
         tracing::info!("Minimal worker version: {min_version}");
     }
 
+    *MIN_VERSION_SUPPORTS_SYNC_JOBS_DEBOUNCING.write().await =
+        min_version >= Version::new(1, 602, 0);
     *MIN_VERSION_SUPPORTS_DEBOUNCING_V2.write().await = min_version >= Version::new(1, 597, 0);
     *MIN_VERSION_SUPPORTS_RUNNABLE_SETTINGS_V0.write().await =
         min_version >= *crate::runnable_settings::MIN_VERSION_RUNNABLE_SETTINGS_V0;

backend/windmill-common/src/workspaces.rs

@@ -6,7 +6,7 @@ use serde::{Deserialize, Serialize};
 use strum::AsRefStr;
 
 use crate::{
-    error::{to_anyhow, Error, Result},
+    error::{self, to_anyhow, Error, Result},
     get_database_url,
     utils::get_custom_pg_instance_password,
     variables::{build_crypt, decrypt},
@@ -63,6 +63,32 @@ pub struct GitRepositorySettings {
     pub settings: Option<GitSyncSettings>,
 }
 
+impl GitRepositorySettings {
+    pub fn is_script_meets_min_version(&self, min_version: u32) -> error::Result<bool> {
+        // example: "hub/28102/sync-script-to-git-repo-windmill"
+        let current = self
+            .script_path
+            .split("/") // -> ["hub" "28102" "sync-script-to-git-repo-windmill"]
+            .skip(1) // omit "hub"
+            .next() // get numeric id
+            .ok_or(Error::InternalErr(format!(
+                "cannot get script version id from: {}",
+                &self.script_path
+            )))?
+            .parse()
+            .unwrap_or_else(|e| {
+                tracing::warn!(
+                    "cannot get script version id from: {}. e: {e}",
+                    &self.script_path
+                );
+
+                u32::MAX
+            });
+
+        Ok(current >= min_version) // this works on assumption that all scripts in hub have sequential ids
+    }
+}
+
 #[derive(Serialize, Deserialize, Debug)]
 pub struct GitSyncSettings {
     pub include_path: Vec<String>,

backend/windmill-queue/src/jobs.rs

@@ -4843,7 +4843,7 @@ pub async fn push<'c, 'd>(
                 ..Default::default()
             }
         }
-        JobPayload::DeploymentCallback { path } => JobPayloadUntagged {
+        JobPayload::DeploymentCallback { path, debouncing_settings } => JobPayloadUntagged {
             runnable_path: Some(path.clone()),
             job_kind: JobKind::DeploymentCallback,
             concurrency_settings: ConcurrencySettings {
@@ -4851,6 +4851,7 @@ pub async fn push<'c, 'd>(
                 concurrent_limit: Some(1),
                 concurrency_time_window_s: Some(0),
             },
+            debouncing_settings,
             ..Default::default()
         },
         JobPayload::Identity => {