From caecbde99a0b91080ba0a13e1d296daed6333cf7 Mon Sep 17 00:00:00 2001
From: Ruben Fiszel <ruben@rubenfiszel.com>
Date: Wed, 24 Jul 2024 08:09:54 +0200
Subject: [PATCH] better split tantivy

---
 backend/Cargo.toml                  |  4 ++--
 backend/ee-repo-ref.txt             |  2 +-
 backend/src/main.rs                 | 27 +++++++++++++++++----------
 backend/windmill-api/Cargo.toml     |  3 ++-
 backend/windmill-api/src/lib.rs     | 15 +++++++++++++--
 backend/windmill-indexer/Cargo.toml |  3 +--
 6 files changed, 36 insertions(+), 18 deletions(-)

diff --git a/backend/Cargo.toml b/backend/Cargo.toml
index 6bf07a8f05..a4b522b9a2 100644
--- a/backend/Cargo.toml
+++ b/backend/Cargo.toml
@@ -53,7 +53,7 @@ flow_testing = ["windmill-worker/flow_testing"]
 openidconnect = ["windmill-api/openidconnect"]
 cloud = ["windmill-queue/cloud", "windmill-worker/cloud"]
 jemalloc = ["windmill-common/jemalloc", "dep:tikv-jemallocator", "dep:tikv-jemalloc-sys", "dep:tikv-jemalloc-ctl"]
-tantivy = ["windmill-indexer/tantivy"]
+tantivy = ["dep:windmill-indexer", "windmill-api/tantivy"]
 
 [dependencies]
 anyhow.workspace = true
@@ -64,7 +64,7 @@ windmill-common = { workspace = true, default-features = false }
 windmill-git-sync.workspace = true
 windmill-api = { workspace = true, default-features = false }
 windmill-worker.workspace = true
-windmill-indexer.workspace = true
+windmill-indexer = { workspace = true, optional = true }
 futures.workspace = true
 tracing.workspace = true
 sqlx.workspace = true
diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt
index 01edb4570b..78fa19ffa1 100644
--- a/backend/ee-repo-ref.txt
+++ b/backend/ee-repo-ref.txt
@@ -1 +1 @@
-89e36e9e73596926e9e02ce27aa7db14d840a255
+c2d059cfff5afcd4a2e87b606f899a5c752d4057
\ No newline at end of file
diff --git a/backend/src/main.rs b/backend/src/main.rs
index 57176f4c31..93ebd1d13b 100644
--- a/backend/src/main.rs
+++ b/backend/src/main.rs
@@ -367,9 +367,7 @@ Windmill Community Edition {GIT_VERSION}
         let should_index_jobs =
             mode == Mode::Indexer || (enable_standalone_indexer && mode == Mode::Standalone);
 
-        #[cfg(not(feature = "tantivy"))]
-        let should_index_jobs = false;
-
+        #[cfg(feature = "tantivy")]
         let (index_reader, index_writer) = if should_index_jobs {
             let (r, w) = windmill_indexer::indexer_ee::init_index().await?;
             (Some(r), Some(w))
@@ -377,16 +375,25 @@ Windmill Community Edition {GIT_VERSION}
             (None, None)
         };
 
-        let indexer_rx = killpill_rx.resubscribe();
-        let index_writer2 = index_writer.clone();
-        let indexer_f = async {
-            if let Some(index_writer) = index_writer2 {
-                windmill_indexer::indexer_ee::run_indexer(db.clone(), index_writer, indexer_rx)
-                    .await;
+        #[cfg(feature = "tantivy")]
+        let indexer_f = {
+            let indexer_rx = killpill_rx.resubscribe();
+            let index_writer2 = index_writer.clone();
+            async {
+                if let Some(index_writer) = index_writer2 {
+                    windmill_indexer::indexer_ee::run_indexer(db.clone(), index_writer, indexer_rx)
+                        .await;
+                }
+                Ok(())
             }
-            Ok(())
         };
 
+        #[cfg(not(feature = "tantivy"))]
+        let (index_reader, index_writer) = (None, None);
+
+        #[cfg(not(feature = "tantivy"))]
+        let indexer_f = async { Ok(()) as anyhow::Result<()> };
+
         let server_f = async {
             if !is_agent {
                 windmill_api::run_server(
diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml
index 9ea2d80f89..a443a19698 100644
--- a/backend/windmill-api/Cargo.toml
+++ b/backend/windmill-api/Cargo.toml
@@ -18,6 +18,7 @@ embedding = ["dep:tinyvector", "dep:hf-hub", "dep:tokenizers", "dep:candle-core"
 parquet = ["dep:datafusion", "dep:object_store", "dep:url", "windmill-common/parquet"]
 prometheus = ["windmill-common/prometheus", "windmill-queue/prometheus", "dep:prometheus"]
 openidconnect = ["dep:openidconnect"]
+tantivy = ["dep:windmill-indexer"]
 
 [dependencies]
 windmill-queue.workspace = true
@@ -27,7 +28,7 @@ windmill-parser.workspace = true
 windmill-parser-py-imports.workspace = true
 windmill-parser-ts.workspace = true
 windmill-git-sync.workspace = true
-windmill-indexer.workspace = true
+windmill-indexer = { workspace = true, optional = true }
 tokio.workspace = true
 anyhow.workspace = true
 argon2.workspace = true
diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs
index f0ea523fc1..1540395ed4 100644
--- a/backend/windmill-api/src/lib.rs
+++ b/backend/windmill-api/src/lib.rs
@@ -143,11 +143,22 @@ pub async fn add_webhook_allowed_origin(
     next.run(req).await
 }
 
+#[cfg(not(feature = "tantivy"))]
+type IndexReader = ();
+
+#[cfg(not(feature = "tantivy"))]
+type IndexWriter = ();
+
+#[cfg(feature = "tantivy")]
+type IndexReader = windmill_indexer::indexer_ee::IndexReader;
+#[cfg(feature = "tantivy")]
+type IndexWriter = windmill_indexer::indexer_ee::IndexWriter;
+
 pub async fn run_server(
     db: DB,
     rsmq: Option<rsmq_async::MultiplexedRsmq>,
-    index_reader: Option<windmill_indexer::indexer_ee::IndexReader>,
-    index_writer: Option<windmill_indexer::indexer_ee::IndexWriter>,
+    index_reader: Option<IndexReader>,
+    index_writer: Option<IndexWriter>,
     addr: SocketAddr,
     mut rx: tokio::sync::broadcast::Receiver<()>,
     port_tx: tokio::sync::oneshot::Sender<String>,
diff --git a/backend/windmill-indexer/Cargo.toml b/backend/windmill-indexer/Cargo.toml
index bca3efc880..fa88503511 100644
--- a/backend/windmill-indexer/Cargo.toml
+++ b/backend/windmill-indexer/Cargo.toml
@@ -10,13 +10,12 @@ path = "src/lib.rs"
 
 [features]
 default = []
-tantivy = ["dep:tantivy"]
 parquet = ["dep:object_store"]
 enterprise = []
 
 [dependencies]
 windmill-common.workspace = true
-tantivy = {workspace = true, optional = true}
+tantivy.workspace = true
 tokio.workspace = true
 sqlx.workspace = true
 anyhow.workspace = true