kalugin66/windmill
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: service logs: aggregation query + killpill on index pull + retention period for logs on s3 (#4795)

Browse Source 
* Search across all hosts in index. killpill on index pull

* Add retention period for service logs

* Remove console log

* monitor log_file when older than biggest retention period (job / log)

* Retention period deletes service and job logs

* Add setting to monitor logs out of s3

* Prepare sqlx

* Update ee repo ref

* Prepare sqlx

* Unused variables
This commit is contained in:
wendrul
2024-11-27 18:57:23 +01:00
committed by GitHub
parent c998102cf3
commit 09fe535bdc
21 changed files with 246 additions and 176 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
backend/.sqlx/query-02424907504848e983bfa89eec343061932dc5b4b17cf13d5cf8d833aedbe6d5.json generated
View File

@@ -5,7 +5,7 @@
"columns": [
{
"ordinal": 0,
"name": "?column?",
"name": "bool",
"type_info": "Bool"
}
],

22
backend/.sqlx/query-288e99211bbd45a337fc9b79c43c5139ee535e23c8b3362c52eef49998349f15.json generated Normal file
View File

@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM job_logs WHERE job_id = ANY($1) RETURNING log_file_index",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "log_file_index",
"type_info": "TextArray"
}
],
"parameters": {
"Left": [
"UuidArray"
]
},
"nullable": [
true
]
},
"hash": "288e99211bbd45a337fc9b79c43c5139ee535e23c8b3362c52eef49998349f15"
}

14
backend/.sqlx/query-467c563cdc81dd57a01735a9f535d1096bdfd6c539d8c31462f8f2bd711ab0fa.json generated
View File

@@ -1,14 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM job_logs WHERE job_id = ANY($1)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"UuidArray"
]
},
"nullable": []
},
"hash": "467c563cdc81dd57a01735a9f535d1096bdfd6c539d8c31462f8f2bd711ab0fa"
}

23
backend/.sqlx/query-4a764cdcb847b71183425a7a0e863708ef7fe2c88b28d0667988a47b9e995c0e.json generated
View File

@@ -1,23 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT flow_version.value \n FROM flow \n LEFT JOIN flow_version \n ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]\n WHERE flow.path = $1 AND flow.workspace_id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "value",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false
]
},
"hash": "4a764cdcb847b71183425a7a0e863708ef7fe2c88b28d0667988a47b9e995c0e"
}

2
backend/.sqlx/query-661f472ff3860983322162420457f5033b9c9afc344d9c3e385ba20a3ad2197a.json generated
View File

@@ -5,7 +5,7 @@
"columns": [
{
"ordinal": 0,
"name": "?column?",
"name": "bool",
"type_info": "Bool"
}
],

4
backend/.sqlx/query-6f4817fad2739a11d89b6704edf62c3c267ca336a8b6bec5b29d4409030ed561.json generated
View File

@@ -26,7 +26,9 @@
"noop",
"appdependencies",
"deploymentcallback",
"singlescriptflow"
"singlescriptflow",
"flowscript",
"flownode"
]
}
}

23
backend/.sqlx/query-721ab4b4ce441a86d43e1b89ab295fbe9c6b59e27c368eaf78b33ecdfa95140f.json generated
View File

@@ -1,23 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT coalesce(flow_version_lite.value, flow_version.value) AS \"value!: sqlx::types::Json<Box<sqlx::types::JsonRawValue>>\" \n FROM flow \n LEFT JOIN flow_version \n ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]\n LEFT JOIN flow_version_lite \n ON flow_version_lite.id = flow_version.id\n WHERE flow.path = $1 AND flow.workspace_id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "value!: sqlx::types::Json<Box<sqlx::types::JsonRawValue>>",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
null
]
},
"hash": "721ab4b4ce441a86d43e1b89ab295fbe9c6b59e27c368eaf78b33ecdfa95140f"
}

14
backend/.sqlx/query-845120388af12c2b2f57fedf95c8cce4a406b74c9a8c8590a3bae323d59d046a.json generated
View File

@@ -1,14 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM log_file WHERE log_ts <= now() - ($1::bigint::text || ' s')::interval ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Int8"
]
},
"nullable": []
},
"hash": "845120388af12c2b2f57fedf95c8cce4a406b74c9a8c8590a3bae323d59d046a"
}

5
backend/.sqlx/query-dd967c5983fa0ff05e2b320ad0e0b5a152784826cb8fb4381c1ffe228cb7feb6.json → backend/.sqlx/query-8d207cc9ed101ff116b617d25a94633c1531170ded1fdf09114718b941f5e1db.json generated
View File

@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT\n hostname,\n mode::text,\n worker_group,\n log_ts,\n file_path,\n ok_lines,\n err_lines,\n json_fmt\n FROM log_file\n ORDER BY log_ts ASC LIMIT $1",
"query": "SELECT\n hostname,\n mode::text,\n worker_group,\n log_ts,\n file_path,\n ok_lines,\n err_lines,\n json_fmt\n FROM log_file\n WHERE log_ts > NOW() - make_interval(secs => $1)\n ORDER BY log_ts ASC LIMIT $2",
"describe": {
"columns": [
{
@@ -46,6 +46,7 @@
],
"parameters": {
"Left": [
"Float8",
"Int8"
]
},
@@ -60,5 +61,5 @@
true
]
},
"hash": "dd967c5983fa0ff05e2b320ad0e0b5a152784826cb8fb4381c1ffe228cb7feb6"
"hash": "8d207cc9ed101ff116b617d25a94633c1531170ded1fdf09114718b941f5e1db"
}

28
backend/.sqlx/query-94da1e7feb4f58cc7ebe99752736f956d47810a94cb052fdcffb5cfe440f8033.json generated Normal file
View File

@@ -0,0 +1,28 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM log_file WHERE log_ts <= now() - ($1::bigint::text || ' s')::interval RETURNING file_path, hostname",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "file_path",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "hostname",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Int8"
]
},
"nullable": [
false,
false
]
},
"hash": "94da1e7feb4f58cc7ebe99752736f956d47810a94cb052fdcffb5cfe440f8033"
}

4
backend/.sqlx/query-9d3556319411a27a875bf6cf0e5eda837cc63e4d8be912c0b5bfeea4a0c8db2e.json generated
View File

@@ -26,7 +26,9 @@
"noop",
"appdependencies",
"deploymentcallback",
"singlescriptflow"
"singlescriptflow",
"flowscript",
"flownode"
]
}
}

23
backend/.sqlx/query-ae543dfa106fa6ad4e9bf45cda1110d4702a80f455c63af6fcbd7cf45bbc4f7a.json generated
View File

@@ -1,23 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT flow_version.value AS \"value: sqlx::types::Json<Box<RawValue>>\" FROM flow \n LEFT JOIN flow_version\n ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]\n WHERE flow.path = $1 AND flow.workspace_id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "value: sqlx::types::Json<Box<RawValue>>",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false
]
},
"hash": "ae543dfa106fa6ad4e9bf45cda1110d4702a80f455c63af6fcbd7cf45bbc4f7a"
}

2
backend/.sqlx/query-d6615719bf8db4b333ed55c9a3c160ad1962668fc3305d8e80ae91ef73614a80.json generated
View File

@@ -5,7 +5,7 @@
"columns": [
{
"ordinal": 0,
"name": "?column?",
"name": "bool",
"type_info": "Bool"
}
],