fix: service logs: aggregation query + killpill on index pull + retention period for logs on s3 (#4795)

* Search across all hosts in index. killpill on index pull

* Add retention period for service logs

* Remove console log

* monitor log_file when older than biggest retention period (job / log)

* Retention period deletes service and job logs

* Add setting to monitor logs out of s3

* Prepare sqlx

* Update ee repo ref

* Prepare sqlx

* Unused variables

frontend/src/lib/components/ServiceLogsInner.svelte

@@ -309,21 +309,17 @@
 		debounceTimeout && clearTimeout(debounceTimeout)
 		debounceTimeout = setTimeout(async () => {
 			if (allLogs) {
-				const hostnames: string[] = []
-				Object.entries(allLogs).forEach(([mode, wgs]) => {
-					Object.entries(wgs).forEach(([wg, keys]) => {
-						Object.entries(keys).forEach(([key, _log_files]) => {
-							hostnames.push(`${mode},${wg},${key}`)
-						})
-					})
-				})
 				const countLogsResponse = await IndexSearchService.countSearchLogsIndex({
 					searchQuery: searchTerm,
-					hosts: hostnames.join(','),
 					minTs,
 					maxTs
 				})
-				countsPerHost = countLogsResponse.count_per_host
+				const buckets = (countLogsResponse.count_per_host as any)['count_per_host']['buckets']
+				countsPerHost = new Map(buckets.map(({ key, doc_count }) => [key, doc_count]));
+				countsPerHost = buckets.reduce((acc: any, { key, doc_count }) => {
+					  acc[key] = {doc_count};
+					  return acc;
+					}, {} as Record<string, number>)
 				queryParseErrors = countLogsResponse.query_parse_errors ?? []
 				loadingLogCounts = false
 			}
@@ -539,7 +535,7 @@
 												return true
 											}
 											const hostKey = `${mode},${wg},${hn}`
-											if (countsPerHost && countsPerHost[hostKey] && countsPerHost[hostKey].count === 0) {
+											if (countsPerHost && (countsPerHost[hostKey] == undefined || countsPerHost[hostKey].doc_count === 0)) {
 												return false
 											}
 											return true
@@ -570,7 +566,7 @@
 													<Loader2 size={15} class="animate-spin" />
 												{:else if countsPerHost}
 													<div class="text-tertiary text-xs">
-														{countsPerHost[hostKey]?.count} matches
+														{countsPerHost[hostKey]?.doc_count ?? 0} matches
 													</div>
 												{:else}
 													<div class="relative grow h-8 mr-2">

frontend/src/lib/components/instanceSettings.ts

@@ -7,20 +7,20 @@ export interface Setting {
 	tooltip?: string
 	key: string
 	fieldType:
-		| 'text'
-		| 'number'
-		| 'boolean'
-		| 'password'
-		| 'select'
-		| 'textarea'
-		| 'seconds'
-		| 'email'
-		| 'license_key'
-		| 'object_store_config'
-		| 'critical_error_channels'
-		| 'slack_connect'
-		| 'smtp_connect'
-		| 'indexer_rates'
+	| 'text'
+	| 'number'
+	| 'boolean'
+	| 'password'
+	| 'select'
+	| 'textarea'
+	| 'seconds'
+	| 'email'
+	| 'license_key'
+	| 'object_store_config'
+	| 'critical_error_channels'
+	| 'slack_connect'
+	| 'smtp_connect'
+	| 'indexer_rates'
 	storage: SettingStorage
 	advancedToggle?: {
 		label: string
@@ -50,9 +50,9 @@ export const settings: Record<string, Setting[]> = {
 			isValid: (value: string | undefined) =>
 				value
 					? value?.startsWith('http') &&
-					  value.includes('://') &&
-					  !value?.endsWith('/') &&
-					  !value?.endsWith(' ')
+					value.includes('://') &&
+					!value?.endsWith('/') &&
+					!value?.endsWith(' ')
 					: false
 		},
 		{
@@ -120,6 +120,14 @@ export const settings: Record<string, Setting[]> = {
 			ee_only: 'You can only adjust this setting to above 30 days in the EE version',
 			cloudonly: false
 		},
+		{
+			label: 'Delete logs from s3 periodically',
+			description: 'Job and service logs are periodically deleted from disk. When this setting is on, they will also be deleted from the object storage.',
+			key: 'monitor_logs_on_s3',
+			fieldType: 'boolean',
+			storage: 'setting',
+			ee_only: ''
+		},
 		{
 			label: 'Expose metrics',
 			description: