From edcee6d8611ff96bfbca7ba811e5e67beaa80f5a Mon Sep 17 00:00:00 2001
From: wendrul <53628737+wendrul@users.noreply.github.com>
Date: Thu, 1 Aug 2024 16:57:42 +0200
Subject: [PATCH] feat: Indexing improvements (#4167)

---
 backend/ee-repo-ref.txt |  2 +-
 docker-compose.yml      | 12 ++++++------
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt
index 270bc33db3..428f05b845 100644
--- a/backend/ee-repo-ref.txt
+++ b/backend/ee-repo-ref.txt
@@ -1 +1 @@
-a64d2990e733aba3f8cbc6691a6090bcd481c891
+1a2febc371c907789b25860366872fca888d6477
diff --git a/docker-compose.yml b/docker-compose.yml
index 59a94b8566..a867f9a292 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -124,12 +124,12 @@ services:
       - PORT=8001
       - DATABASE_URL=${DATABASE_URL}
       - MODE=indexer
-      #how often will new jobs be added to the index to be searched
-      - TANTIVY_REFRESH_INDEX_PERIOD__S=300
-      #how many documents to batch in one commit
-      - TANTIVY_DOC_COMMIT_MAX_BATCH_SIZE=100000
-      #log memory usage and progress every so many documents indexed
-      - TANTIVY_SHOW_MEMORY_EVERY=10000
+      - TANTIVY_MAX_INDEXED_JOB_LOG_SIZE__MB=1 # job logs bigger than this will be truncated before indexing
+      - TANTIVY_S3_BACKUP_PERIOD__S=3600 # how often to backup the index into object storage
+      - TANTIVY_INDEX_WRITER_MEMORY_BUDGET__MB=100 # higher budget for higher indexing throughput
+      - TANTIVY_REFRESH_INDEX_PERIOD__S=300 #how often to start indexing new jobs
+      - TANTIVY_DOC_COMMIT_MAX_BATCH_SIZE=100000 #how many documents to batch in one commit
+      - TANTIVY_SHOW_MEMORY_EVERY=10000 #log memory usage and progress every so many documents indexed
     depends_on:
       db:
         condition: service_healthy