kalugin66/windmill
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

nits zombie job log improvement

Browse Source
This commit is contained in:
Ruben Fiszel
2024-09-01 10:33:38 +02:00
parent ebda85e94a
commit c429e09211
4 changed files with 49 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
backend/.sqlx/query-7ab02d2c060ed12531e6af98a5d3369afe534519bf48f61f88f60786db870c24.json → backend/.sqlx/query-8c15ea27be82665fe74a4be3182a5049fa937e30b4115caf0ba8d61d34f2089a.json generated
View File

@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE queue SET running = false, started_at = null, logs = logs || '\nRestarted job after not receiving job''s ping for too long the ' || now() || '\n\n' \n WHERE last_ping < now() - ($1 || ' seconds')::interval\n AND running = true AND job_kind NOT IN ('flow', 'flowpreview', 'singlescriptflow') AND same_worker = false RETURNING id, workspace_id, last_ping",
"query": "UPDATE queue SET running = false, started_at = null\n WHERE last_ping < now() - ($1 || ' seconds')::interval\n AND running = true AND job_kind NOT IN ('flow', 'flowpreview', 'singlescriptflow') AND same_worker = false RETURNING id, workspace_id, last_ping",
"describe": {
"columns": [
{
@@ -30,5 +30,5 @@
true
]
},
"hash": "7ab02d2c060ed12531e6af98a5d3369afe534519bf48f61f88f60786db870c24"
"hash": "8c15ea27be82665fe74a4be3182a5049fa937e30b4115caf0ba8d61d34f2089a"
}

14
backend/.sqlx/query-ae25fae1aca2cffc43a6054cd54639ba9deaaef419ad4cb2655e7e95f602a688.json generated Normal file
View File

@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO job_logs (job_id, logs) VALUES ($1,'Restarted job after not receiving job''s ping for too long the ' || now() || '\n\n') \n ON CONFLICT (job_id) DO UPDATE SET logs = job_logs.logs || '\nRestarted job after not receiving job''s ping for too long the ' || now() || '\n\n' WHERE job_logs.job_id = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": []
},
"hash": "ae25fae1aca2cffc43a6054cd54639ba9deaaef419ad4cb2655e7e95f602a688"
}

18
backend/src/monitor.rs
View File

@@ -1248,7 +1248,7 @@ async fn handle_zombie_jobs<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
) {
if *RESTART_ZOMBIE_JOBS {
let restarted = sqlx::query!(
"UPDATE queue SET running = false, started_at = null, logs = logs || '\nRestarted job after not receiving job''s ping for too long the ' || now() || '\n\n'
"UPDATE queue SET running = false, started_at = null
WHERE last_ping < now() - ($1 || ' seconds')::interval
AND running = true AND job_kind NOT IN ('flow', 'flowpreview', 'singlescriptflow') AND same_worker = false RETURNING id, workspace_id, last_ping",
*ZOMBIE_JOB_TIMEOUT,
@@ -1261,11 +1261,23 @@ async fn handle_zombie_jobs<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
if METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed) {
QUEUE_ZOMBIE_RESTART_COUNT.inc_by(restarted.len() as _);
}
let base_url = BASE_URL.read().await.clone();
for r in restarted {
let last_ping = if let Some(x) = r.last_ping {
format!("last ping at {x}")
} else {
"no last ping".to_string()
};
let url = format!("{}/run/{}?workspace={}", base_url, r.id, r.workspace_id,);
let error_message = format!(
"Zombie job detected, restarting it: {} {} {:?}",
r.id, r.workspace_id, r.last_ping
"Zombie job {} on {} ({}) detected, restarting it, {}",
r.id, r.workspace_id, url, last_ping
);
let _ = sqlx::query!("
INSERT INTO job_logs (job_id, logs) VALUES ($1,'Restarted job after not receiving job''s ping for too long the ' || now() || '\n\n')
ON CONFLICT (job_id) DO UPDATE SET logs = job_logs.logs || '\nRestarted job after not receiving job''s ping for too long the ' || now() || '\n\n' WHERE job_logs.job_id = $1", r.id)
.execute(db).await;
tracing::error!(error_message);
report_critical_error(error_message, db.clone()).await;
}

19
frontend/src/routes/(root)/(logged)/workers/+page.svelte
View File

@@ -18,7 +18,7 @@
import { enterpriseLicense, superadmin } from '$lib/stores'
import { sendUserToast } from '$lib/toast'
import { displayDate, groupBy, pluralize, truncate } from '$lib/utils'
import { AlertTriangle, FileJson, LineChart, Plus, Search } from 'lucide-svelte'
import { AlertTriangle, FileJson, LineChart, List, Plus, Search } from 'lucide-svelte'
import { onDestroy, onMount } from 'svelte'
import AutoComplete from 'simple-svelte-autocomplete'
@@ -237,6 +237,9 @@
return [worker_group[0], filteredWorkerGroup]
}
const openSearchWithPrefilledText: (t?: string) => void = getContext(
'openSearchWithPrefilledText'
)
</script>
{#if $superadmin}
@@ -292,6 +295,20 @@
Queue metrics
</Button>
</div>
<div>
<Button
size="xs"
color="dark"
startIcon={{
icon: List
}}
on:click={() => {
openSearchWithPrefilledText('#')
}}
>
Service logs
</Button>
</div>
</div>
{/if}
</PageHeader>