kalugin66/windmill
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: refactor logging to improve performance by order of magnitude for longer jobs

Browse Source
This commit is contained in:
Ruben Fiszel
2024-03-16 17:55:28 +01:00
parent bab67fe3ae
commit e5366b84bb
27 changed files with 530 additions and 403 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
backend/.sqlx/query-8a3340146fe2072c7d47984c2f768c1ef632f8146766b494398421a8fc552ab3.json → backend/.sqlx/query-04d9cb2edf6933a3b8efbe274e10227a37475ff4ec351577bfdade19a15596d4.json generated
View File

@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT running, substr(logs, $1) as logs, mem_peak, \n CASE WHEN is_flow_step is true then NULL else flow_status END as flow_status \n FROM queue WHERE workspace_id = $2 AND id = $3",
"query": "SELECT running, substr(concat(coalesce(queue.logs, ''), job_logs.logs), $1) as logs, mem_peak, \n CASE WHEN is_flow_step is true then NULL else flow_status END as flow_status \n FROM queue\n LEFT JOIN job_logs ON job_logs.job_id = queue.id \n WHERE queue.workspace_id = $2 AND queue.id = $3",
"describe": {
"columns": [
{
@@ -38,5 +38,5 @@
null
]
},
"hash": "8a3340146fe2072c7d47984c2f768c1ef632f8146766b494398421a8fc552ab3"
"hash": "04d9cb2edf6933a3b8efbe274e10227a37475ff4ec351577bfdade19a15596d4"
}

15
backend/.sqlx/query-1bf2ca894246bd716875635b2d0c294a1ce2ed21916097ea165df240f7421a1e.json generated
View File

@@ -1,15 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE queue SET logs = $1 WHERE id = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Uuid"
]
},
"nullable": []
},
"hash": "1bf2ca894246bd716875635b2d0c294a1ce2ed21916097ea165df240f7421a1e"
}

5
backend/.sqlx/query-254776959f60455a00a2c29a1783ee6b27c836197df6454d3bb5c59441c34f44.json → backend/.sqlx/query-2ade671449393541fa565088b21268dad137314d250f7ded502defb9a6de0b2f.json generated
View File

@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO completed_job AS cj\n ( workspace_id\n , id\n , parent_job\n , created_by\n , created_at\n , started_at\n , duration_ms\n , success\n , script_hash\n , script_path\n , args\n , result\n , logs\n , raw_code\n , raw_lock\n , canceled\n , canceled_by\n , canceled_reason\n , job_kind\n , schedule_path\n , permissioned_as\n , flow_status\n , raw_flow\n , is_flow_step\n , is_skipped\n , language\n , email\n , visible_to_owner\n , mem_peak\n , tag\n , priority\n )\n VALUES ($1, $2, $3, $4, $5, COALESCE($6, now()), COALESCE($26, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE($6, now()))))*1000), $7, $8, $9,$10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $27, $28, $29, $30, $31)\n ON CONFLICT (id) DO UPDATE SET success = $7, result = $11, logs = concat(cj.logs, $12) RETURNING duration_ms",
"query": "INSERT INTO completed_job AS cj\n ( workspace_id\n , id\n , parent_job\n , created_by\n , created_at\n , started_at\n , duration_ms\n , success\n , script_hash\n , script_path\n , args\n , result\n , raw_code\n , raw_lock\n , canceled\n , canceled_by\n , canceled_reason\n , job_kind\n , schedule_path\n , permissioned_as\n , flow_status\n , raw_flow\n , is_flow_step\n , is_skipped\n , language\n , email\n , visible_to_owner\n , mem_peak\n , tag\n , priority\n )\n VALUES ($1, $2, $3, $4, $5, COALESCE($6, now()), COALESCE($25, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE($6, now()))))*1000), $7, $8, $9,$10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $26, $27, $28, $29, $30)\n ON CONFLICT (id) DO UPDATE SET success = $7, result = $11 RETURNING duration_ms",
"describe": {
"columns": [
{
@@ -24,7 +24,6 @@
"Jsonb",
"Text",
"Text",
"Text",
"Bool",
"Varchar",
"Text",
@@ -92,5 +91,5 @@
false
]
},
"hash": "254776959f60455a00a2c29a1783ee6b27c836197df6454d3bb5c59441c34f44"
"hash": "2ade671449393541fa565088b21268dad137314d250f7ded502defb9a6de0b2f"
}

23
backend/.sqlx/query-3dfa0bf34c38b5529f2bbad405d23bda84db69757e132859d7e04df38e503f3e.json generated Normal file
View File

@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT CONCAT(coalesce(completed_job.logs, ''), coalesce(job_logs.logs, '')) \n FROM completed_job \n LEFT JOIN job_logs ON job_logs.job_id = completed_job.id \n WHERE completed_job.id = $1 AND completed_job.workspace_id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "concat",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Uuid",
"Text"
]
},
"nullable": [
null
]
},
"hash": "3dfa0bf34c38b5529f2bbad405d23bda84db69757e132859d7e04df38e503f3e"
}

24
backend/.sqlx/query-453501fbd61efd26647baf9b6ef702ce0bc2e920914e9f08fe5f2a5f4ab03f02.json generated
View File

@@ -1,24 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT substr(logs, $1) as logs FROM completed_job WHERE workspace_id = $2 AND id = $3",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "logs",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Int4",
"Text",
"Uuid"
]
},
"nullable": [
null
]
},
"hash": "453501fbd61efd26647baf9b6ef702ce0bc2e920914e9f08fe5f2a5f4ab03f02"
}

15
backend/.sqlx/query-486f181a9ced2bdc7c8d93da22c9d3e229ef106174bff2c472dbe82622f382b6.json generated
View File

@@ -1,15 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE queue SET logs = concat(logs, $1::text) WHERE id = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Uuid"
]
},
"nullable": []
},
"hash": "486f181a9ced2bdc7c8d93da22c9d3e229ef106174bff2c472dbe82622f382b6"
}

36
backend/.sqlx/query-5cb644f89a94b6e6a1d7a84155bafc460c1749946056ded8ca712dacecbf427d.json generated Normal file
View File

@@ -0,0 +1,36 @@
{
"db_name": "PostgreSQL",
"query": "SELECT substr(concat(coalesce(completed_job.logs, ''), job_logs.logs), $1) as logs, mem_peak, \n CASE WHEN is_flow_step is true then NULL else flow_status END as flow_status \n FROM completed_job \n LEFT JOIN job_logs ON job_logs.job_id = completed_job.id \n WHERE completed_job.workspace_id = $2 AND id = $3",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "logs",
"type_info": "Text"
},
{
"ordinal": 1,
"name": "mem_peak",
"type_info": "Int4"
},
{
"ordinal": 2,
"name": "flow_status",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"Int4",
"Text",
"Uuid"
]
},
"nullable": [
null,
true,
null
]
},
"hash": "5cb644f89a94b6e6a1d7a84155bafc460c1749946056ded8ca712dacecbf427d"
}

23
backend/.sqlx/query-ab3c979c20a8ba9f6d906193c3bc7bc1941f0ecbe7ee2d8a62d2e02faeaae3f0.json generated Normal file
View File

@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT right(logs, 300) FROM job_logs WHERE job_id = $1 AND workspace_id = $2 ORDER BY created_at DESC LIMIT 1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "right",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Uuid",
"Text"
]
},
"nullable": [
null
]
},
"hash": "ab3c979c20a8ba9f6d906193c3bc7bc1941f0ecbe7ee2d8a62d2e02faeaae3f0"
}

16
backend/.sqlx/query-c7dd35561e9b1cfd86238d410139f50e4d87c762e4867d8b32c11bd8c74846eb.json generated Normal file
View File

@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO job_logs (logs, job_id, workspace_id) VALUES ($1, $2, $3) ON CONFLICT (job_id) DO UPDATE SET logs = concat(job_logs.logs, $1::text)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Uuid",
"Varchar"
]
},
"nullable": []
},
"hash": "c7dd35561e9b1cfd86238d410139f50e4d87c762e4867d8b32c11bd8c74846eb"
}

4
backend/.sqlx/query-6f246196df45be4f1860f04c0fb25b1076c9c3fa30d632f4a90cb37a8f4d419b.json → backend/.sqlx/query-d9b32f819c87250962dae44e34d13b234e765b4b90b961e7db494f4c544c08ca.json generated
View File

@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT logs FROM completed_job WHERE id = $1 AND workspace_id = $2",
"query": "SELECT logs FROM job_logs WHERE job_id = $1 AND workspace_id = $2",
"describe": {
"columns": [
{
@@ -19,5 +19,5 @@
true
]
},
"hash": "6f246196df45be4f1860f04c0fb25b1076c9c3fa30d632f4a90cb37a8f4d419b"
"hash": "d9b32f819c87250962dae44e34d13b234e765b4b90b961e7db494f4c544c08ca"
}