kalugin66/windmill
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: better handle same worker zombie job

Browse Source
This commit is contained in:
Ruben Fiszel
2025-10-31 07:36:38 +00:00
parent 329124a6bc
commit d86ce2e3e3
5 changed files with 3 additions and 215 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
backend/.sqlx/query-3e261911cc4c5289da49865f54350613f9e651540a279bd7d75e5e7d79f676a8.json → backend/.sqlx/query-469f8b7f691e621cce78b83994b4a4625bb7fbd0974c69745c31c7562563944f.json generated
View File

@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT worker, array_agg(v2_job_queue.id) as ids FROM v2_job_queue LEFT JOIN v2_job ON v2_job_queue.id = v2_job.id LEFT JOIN v2_job_runtime ON v2_job_queue.id = v2_job_runtime.id WHERE v2_job_queue.created_at < now() - ('60 seconds')::interval\n AND running = true AND ping IS NULL AND same_worker = true AND worker IS NOT NULL GROUP BY worker",
"query": "SELECT worker, array_agg(v2_job_queue.id) as ids FROM v2_job_queue LEFT JOIN v2_job ON v2_job_queue.id = v2_job.id LEFT JOIN v2_job_runtime ON v2_job_queue.id = v2_job_runtime.id WHERE v2_job_queue.created_at < now() - ('60 seconds')::interval\n AND running = true AND (ping IS NULL OR ping < now() - ('60 seconds')::interval) AND same_worker = true AND worker IS NOT NULL GROUP BY worker",
"describe": {
"columns": [
{
@@ -22,5 +22,5 @@
null
]
},
"hash": "3e261911cc4c5289da49865f54350613f9e651540a279bd7d75e5e7d79f676a8"
"hash": "469f8b7f691e621cce78b83994b4a4625bb7fbd0974c69745c31c7562563944f"
}

71
backend/.sqlx/query-5430f7728c1e9b539cc8aad29ca9e6733943278998d3df62a9486607827e59ec.json generated
View File

@@ -1,71 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "WITH active_users AS (SELECT distinct username as email FROM audit WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh')),\n authors as (SELECT distinct email FROM usr WHERE usr.operator IS false)\n SELECT email, email NOT IN (SELECT email FROM authors) as operator_only, login_type::text, verified, super_admin, devops, name, company, username\n FROM password\n WHERE email IN (SELECT email FROM active_users)\n ORDER BY super_admin DESC, devops DESC\n LIMIT $1 OFFSET $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "email",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "operator_only",
"type_info": "Bool"
},
{
"ordinal": 2,
"name": "login_type",
"type_info": "Text"
},
{
"ordinal": 3,
"name": "verified",
"type_info": "Bool"
},
{
"ordinal": 4,
"name": "super_admin",
"type_info": "Bool"
},
{
"ordinal": 5,
"name": "devops",
"type_info": "Bool"
},
{
"ordinal": 6,
"name": "name",
"type_info": "Varchar"
},
{
"ordinal": 7,
"name": "company",
"type_info": "Varchar"
},
{
"ordinal": 8,
"name": "username",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Int8",
"Int8"
]
},
"nullable": [
false,
null,
null,
false,
false,
false,
true,
true,
true
]
},
"hash": "5430f7728c1e9b539cc8aad29ca9e6733943278998d3df62a9486607827e59ec"
}

70
backend/.sqlx/query-df5b933f81ca7e3bbb3fb522baedf749fa3bbbf2c0e43d5d4ea148b5bc990067.json generated
View File

@@ -1,70 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT email, login_type::TEXT, super_admin, devops, verified, name, company, username, NULL::bool as operator_only FROM password WHERE email = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "email",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "login_type",
"type_info": "Text"
},
{
"ordinal": 2,
"name": "super_admin",
"type_info": "Bool"
},
{
"ordinal": 3,
"name": "devops",
"type_info": "Bool"
},
{
"ordinal": 4,
"name": "verified",
"type_info": "Bool"
},
{
"ordinal": 5,
"name": "name",
"type_info": "Varchar"
},
{
"ordinal": 6,
"name": "company",
"type_info": "Varchar"
},
{
"ordinal": 7,
"name": "username",
"type_info": "Varchar"
},
{
"ordinal": 8,
"name": "operator_only",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false,
null,
false,
false,
false,
true,
true,
true,
null
]
},
"hash": "df5b933f81ca7e3bbb3fb522baedf749fa3bbbf2c0e43d5d4ea148b5bc990067"
}

71
backend/.sqlx/query-f07a705df1a988827e099d146f5308b763293a27adf30d02df605317791d8126.json generated
View File

@@ -1,71 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT email, login_type::text, verified, super_admin, devops, name, company, username, NULL::bool as operator_only FROM password ORDER BY super_admin DESC, devops DESC, email LIMIT $1 OFFSET $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "email",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "login_type",
"type_info": "Text"
},
{
"ordinal": 2,
"name": "verified",
"type_info": "Bool"
},
{
"ordinal": 3,
"name": "super_admin",
"type_info": "Bool"
},
{
"ordinal": 4,
"name": "devops",
"type_info": "Bool"
},
{
"ordinal": 5,
"name": "name",
"type_info": "Varchar"
},
{
"ordinal": 6,
"name": "company",
"type_info": "Varchar"
},
{
"ordinal": 7,
"name": "username",
"type_info": "Varchar"
},
{
"ordinal": 8,
"name": "operator_only",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Int8",
"Int8"
]
},
"nullable": [
false,
null,
false,
false,
false,
true,
true,
true,
null
]
},
"hash": "f07a705df1a988827e099d146f5308b763293a27adf30d02df605317791d8126"
}

2
backend/src/monitor.rs
View File

@@ -2174,7 +2174,7 @@ async fn handle_zombie_jobs(db: &Pool<Postgres>, base_internal_url: &str, worker
let same_worker_timeout_jobs = {
let long_same_worker_jobs = sqlx::query!(
"SELECT worker, array_agg(v2_job_queue.id) as ids FROM v2_job_queue LEFT JOIN v2_job ON v2_job_queue.id = v2_job.id LEFT JOIN v2_job_runtime ON v2_job_queue.id = v2_job_runtime.id WHERE v2_job_queue.created_at < now() - ('60 seconds')::interval
AND running = true AND ping IS NULL AND same_worker = true AND worker IS NOT NULL GROUP BY worker",
AND running = true AND (ping IS NULL OR ping < now() - ('60 seconds')::interval) AND same_worker = true AND worker IS NOT NULL GROUP BY worker",
)
.fetch_all(db)
.await