From cee0e9f615d7a47157d669589f7d77ddec7d2b71 Mon Sep 17 00:00:00 2001
From: Ruben Fiszel <ruben@rubenfiszel.com>
Date: Tue, 5 Nov 2024 00:17:59 +0100
Subject: [PATCH] fix(prometheus): improve queue_count when tags have no more
 jobs

---
 backend/src/monitor.rs | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs
index 0618cfd8d8..b6cbd55330 100644
--- a/backend/src/monitor.rs
+++ b/backend/src/monitor.rs
@@ -111,6 +111,9 @@ lazy_static::lazy_static! {
         "Number of jobs in the queue",
         &["tag"]
     ).unwrap();
+
+    static ref QUEUE_COUNT_TAGS: Arc<RwLock<Vec<String>>> = Arc::new(RwLock::new(Vec::new()));
+
 }
 
 pub async fn initial_load(
@@ -1089,12 +1092,23 @@ pub async fn expose_queue_metrics(db: &Pool<Postgres>) {
     if metrics_enabled || save_metrics {
         let queue_counts = windmill_common::queue::get_queue_counts(db).await;
 
+        if metrics_enabled {
+            for q in QUEUE_COUNT_TAGS.read().await.iter() {
+                if queue_counts.get(q).is_none() {
+                    (*QUEUE_COUNT).with_label_values(&[q]).set(0);
+                }
+            }
+        }
+
+        let mut tags_to_watch = vec![];
         for q in queue_counts {
             let count = q.1;
             let tag = q.0;
+
             if metrics_enabled {
                 let metric = (*QUEUE_COUNT).with_label_values(&[&tag]);
                 metric.set(count as i64);
+                tags_to_watch.push(tag.to_string());
             }
 
             // save queue_count and delay metrics per tag
@@ -1119,6 +1133,10 @@ pub async fn expose_queue_metrics(db: &Pool<Postgres>) {
                 }
             }
         }
+        if metrics_enabled {
+            let mut w = QUEUE_COUNT_TAGS.write().await;
+            *w = tags_to_watch;
+        }
     }
 
     // clean queue metrics older than 14 days