diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 72eaf99004..06aebf1052 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -95,7 +95,7 @@ jobs: platforms: linux/amd64,linux/arm64 push: true build-args: | - features=embedding,parquet,openidconnect,jemalloc,license,http_trigger,zip,oauth2,dind,postgres_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages + features=embedding,parquet,openidconnect,jemalloc,license,http_trigger,zip,oauth2,dind,postgres_trigger,mqtt_trigger,websocket,smtp,static_frontend,agent_worker_server,all_languages tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }} ${{ steps.meta-public.outputs.tags }} @@ -157,7 +157,7 @@ jobs: platforms: linux/amd64,linux/arm64 push: true build-args: | - features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,license,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,otel,dind,postgres_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages + features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,agent_worker_server,tantivy,license,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,otel,dind,postgres_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${{ env.DEV_SHA }} ${{ steps.meta-ee-public.outputs.tags }} diff --git a/backend/.sqlx/query-0e52a588f3edeb8fb58d6d62247b8590e51171e2811c62737bdb81fb0ac8f182.json b/backend/.sqlx/query-0e52a588f3edeb8fb58d6d62247b8590e51171e2811c62737bdb81fb0ac8f182.json deleted file mode 100644 index f23cf3f710..0000000000 --- a/backend/.sqlx/query-0e52a588f3edeb8fb58d6d62247b8590e51171e2811c62737bdb81fb0ac8f182.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE worker_ping SET \n ping_at = now(), \n jobs_executed = 1, \n current_job_id = $1, \n current_job_workspace_id = 'admins' \n WHERE worker = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid", - "Text" - ] - }, - "nullable": [] - }, - "hash": "0e52a588f3edeb8fb58d6d62247b8590e51171e2811c62737bdb81fb0ac8f182" -} diff --git a/backend/.sqlx/query-2ebb0463b790ddf7ba0ee22d8c9afc88eb57c4110a202775003fb48b2f4e317f.json b/backend/.sqlx/query-2ebb0463b790ddf7ba0ee22d8c9afc88eb57c4110a202775003fb48b2f4e317f.json new file mode 100644 index 0000000000..49e6e6c6ae --- /dev/null +++ b/backend/.sqlx/query-2ebb0463b790ddf7ba0ee22d8c9afc88eb57c4110a202775003fb48b2f4e317f.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status SET\n workflow_as_code_status = jsonb_set(\n jsonb_set(\n COALESCE(workflow_as_code_status, '{}'::jsonb),\n array[$1],\n COALESCE(workflow_as_code_status->$1, '{}'::jsonb)\n ),\n array[$1, 'started_at'],\n to_jsonb(now()::text)\n )\n WHERE id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "2ebb0463b790ddf7ba0ee22d8c9afc88eb57c4110a202775003fb48b2f4e317f" +} diff --git a/backend/.sqlx/query-6ff7a025f529c077c1b6c9632a367aa29e2f0fdac3f1984550484d5a06a6ea21.json b/backend/.sqlx/query-2faa27519624249f16cf89814ab5efe8f8daf928c1194cecacfa8223165fb9f2.json similarity index 50% rename from backend/.sqlx/query-6ff7a025f529c077c1b6c9632a367aa29e2f0fdac3f1984550484d5a06a6ea21.json rename to backend/.sqlx/query-2faa27519624249f16cf89814ab5efe8f8daf928c1194cecacfa8223165fb9f2.json index b045ed2b07..dbe3786571 100644 --- a/backend/.sqlx/query-6ff7a025f529c077c1b6c9632a367aa29e2f0fdac3f1984550484d5a06a6ea21.json +++ b/backend/.sqlx/query-2faa27519624249f16cf89814ab5efe8f8daf928c1194cecacfa8223165fb9f2.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "UPDATE v2_job_runtime r SET\n memory_peak = $1,\n ping = now()\n FROM v2_job_queue q\n WHERE r.id = $2 AND q.id = r.id\n RETURNING canceled_by, canceled_reason", + "query": "UPDATE v2_job_runtime r SET\n memory_peak = $1,\n ping = now()\n FROM v2_job_queue q\n WHERE r.id = $2 AND q.id = r.id\n RETURNING canceled_by, canceled_reason", "describe": { "columns": [ { @@ -25,5 +25,5 @@ true ] }, - "hash": "6ff7a025f529c077c1b6c9632a367aa29e2f0fdac3f1984550484d5a06a6ea21" + "hash": "2faa27519624249f16cf89814ab5efe8f8daf928c1194cecacfa8223165fb9f2" } diff --git a/backend/.sqlx/query-45d616c92ebcbe30a563e1fa7d2d0e53392e238144b039cfe042587d7fe1dea3.json b/backend/.sqlx/query-45d616c92ebcbe30a563e1fa7d2d0e53392e238144b039cfe042587d7fe1dea3.json deleted file mode 100644 index 052238874a..0000000000 --- a/backend/.sqlx/query-45d616c92ebcbe30a563e1fa7d2d0e53392e238144b039cfe042587d7fe1dea3.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_status SET\n workflow_as_code_status = jsonb_set(\n jsonb_set(\n COALESCE(workflow_as_code_status, '{}'::jsonb),\n array[$1],\n COALESCE(workflow_as_code_status->$1, '{}'::jsonb)\n ),\n array[$1, 'started_at'],\n to_jsonb(now()::text)\n )\n WHERE id = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "45d616c92ebcbe30a563e1fa7d2d0e53392e238144b039cfe042587d7fe1dea3" -} diff --git a/backend/.sqlx/query-505250098ab003ff0ca30046df283e54bf44be74305070f10a5720a04c4789f3.json b/backend/.sqlx/query-505250098ab003ff0ca30046df283e54bf44be74305070f10a5720a04c4789f3.json new file mode 100644 index 0000000000..fb3fbe4f0e --- /dev/null +++ b/backend/.sqlx/query-505250098ab003ff0ca30046df283e54bf44be74305070f10a5720a04c4789f3.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_queue\n SET canceled_by = $1\n , canceled_reason = $2\nWHERE id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "505250098ab003ff0ca30046df283e54bf44be74305070f10a5720a04c4789f3" +} diff --git a/backend/.sqlx/query-e968e879d3c52f7dd502c3cd15fc8fbd983a4a3ab25648c562497a27c74b5c8c.json b/backend/.sqlx/query-506066203c49424e9944eb3948dc1657d3d796e6233e9f0ec925879c705d4773.json similarity index 64% rename from backend/.sqlx/query-e968e879d3c52f7dd502c3cd15fc8fbd983a4a3ab25648c562497a27c74b5c8c.json rename to backend/.sqlx/query-506066203c49424e9944eb3948dc1657d3d796e6233e9f0ec925879c705d4773.json index 3bacda16d9..50d4a9595d 100644 --- a/backend/.sqlx/query-e968e879d3c52f7dd502c3cd15fc8fbd983a4a3ab25648c562497a27c74b5c8c.json +++ b/backend/.sqlx/query-506066203c49424e9944eb3948dc1657d3d796e6233e9f0ec925879c705d4773.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "UPDATE worker_ping SET ping_at = now(), current_job_id = $1, current_job_workspace_id = $2, memory_usage = $3, wm_memory_usage = $4,\n occupancy_rate = $6, occupancy_rate_15s = $7, occupancy_rate_5m = $8, occupancy_rate_30m = $9 WHERE worker = $5", + "query": "UPDATE worker_ping SET ping_at = now(), current_job_id = $1, current_job_workspace_id = $2, memory_usage = $3, wm_memory_usage = $4,\n occupancy_rate = $6, occupancy_rate_15s = $7, occupancy_rate_5m = $8, occupancy_rate_30m = $9 WHERE worker = $5", "describe": { "columns": [], "parameters": { @@ -18,5 +18,5 @@ }, "nullable": [] }, - "hash": "e968e879d3c52f7dd502c3cd15fc8fbd983a4a3ab25648c562497a27c74b5c8c" + "hash": "506066203c49424e9944eb3948dc1657d3d796e6233e9f0ec925879c705d4773" } diff --git a/backend/.sqlx/query-77701b16ee1f6dd827372835db59bbffc7254af47a8d48b7ba3cf969c2f8398c.json b/backend/.sqlx/query-77701b16ee1f6dd827372835db59bbffc7254af47a8d48b7ba3cf969c2f8398c.json new file mode 100644 index 0000000000..e6e754579f --- /dev/null +++ b/backend/.sqlx/query-77701b16ee1f6dd827372835db59bbffc7254af47a8d48b7ba3cf969c2f8398c.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_queue SET tag = $1, running = false WHERE id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "77701b16ee1f6dd827372835db59bbffc7254af47a8d48b7ba3cf969c2f8398c" +} diff --git a/backend/.sqlx/query-8efd06387ded837d7849adafe5bc93acb882ef90fc58b023650c875e0fd17047.json b/backend/.sqlx/query-7d78efab0a588f56a13a7b5251f0a72f5d341b053218e7aec83a834cf7ccc98f.json similarity index 51% rename from backend/.sqlx/query-8efd06387ded837d7849adafe5bc93acb882ef90fc58b023650c875e0fd17047.json rename to backend/.sqlx/query-7d78efab0a588f56a13a7b5251f0a72f5d341b053218e7aec83a834cf7ccc98f.json index 005e8fedc9..07c93d0d88 100644 --- a/backend/.sqlx/query-8efd06387ded837d7849adafe5bc93acb882ef90fc58b023650c875e0fd17047.json +++ b/backend/.sqlx/query-7d78efab0a588f56a13a7b5251f0a72f5d341b053218e7aec83a834cf7ccc98f.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id) \n values ($1, $2, $3, $4, $5, $6, $7, $8) \n ON CONFLICT (job_id) DO UPDATE SET email = $2, username = $3, is_admin = $4, is_operator = $5, folders = $6, groups = $7, workspace_id = $8", + "query": "INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id) \n values ($1, $2, $3, $4, $5, $6, $7, $8) \n ON CONFLICT (job_id) DO UPDATE SET email = $2, username = $3, is_admin = $4, is_operator = $5, folders = $6, groups = $7, workspace_id = $8", "describe": { "columns": [], "parameters": { @@ -17,5 +17,5 @@ }, "nullable": [] }, - "hash": "8efd06387ded837d7849adafe5bc93acb882ef90fc58b023650c875e0fd17047" + "hash": "7d78efab0a588f56a13a7b5251f0a72f5d341b053218e7aec83a834cf7ccc98f" } diff --git a/backend/.sqlx/query-a2e86f169ffbf8acee5f7c7b71db5859ac94ffbad267c9cd6c652e8ce8fc5d3c.json b/backend/.sqlx/query-a2e86f169ffbf8acee5f7c7b71db5859ac94ffbad267c9cd6c652e8ce8fc5d3c.json new file mode 100644 index 0000000000..4e0d53b0f3 --- /dev/null +++ b/backend/.sqlx/query-a2e86f169ffbf8acee5f7c7b71db5859ac94ffbad267c9cd6c652e8ce8fc5d3c.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO job_logs (logs, job_id, workspace_id) VALUES ($1, $2, $3) ON CONFLICT (job_id) DO UPDATE SET logs = concat(job_logs.logs, $1::text) RETURNING length(logs)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "length", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text", + "Uuid", + "Varchar" + ] + }, + "nullable": [ + null + ] + }, + "hash": "a2e86f169ffbf8acee5f7c7b71db5859ac94ffbad267c9cd6c652e8ce8fc5d3c" +} diff --git a/backend/.sqlx/query-03ae5b1c912b13a8a7aadf50cb4984a2ea952e782fd52eb3088454690bd13dd1.json b/backend/.sqlx/query-a9db7b2f435bb82acb8c5eeb7f800b28f3256491fdaa168591adc7b4b9f3327a.json similarity index 83% rename from backend/.sqlx/query-03ae5b1c912b13a8a7aadf50cb4984a2ea952e782fd52eb3088454690bd13dd1.json rename to backend/.sqlx/query-a9db7b2f435bb82acb8c5eeb7f800b28f3256491fdaa168591adc7b4b9f3327a.json index 71ddb140b9..8cc09349ec 100644 --- a/backend/.sqlx/query-03ae5b1c912b13a8a7aadf50cb4984a2ea952e782fd52eb3088454690bd13dd1.json +++ b/backend/.sqlx/query-a9db7b2f435bb82acb8c5eeb7f800b28f3256491fdaa168591adc7b4b9f3327a.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT content AS \"content!: String\",\n lock AS \"lock: String\", language AS \"language: Option\", envs AS \"envs: Vec\", schema AS \"schema: String\", schema_validation AS \"schema_validation: bool\", codebase LIKE '%.tar' as use_tar FROM script WHERE hash = $1 LIMIT 1", + "query": "SELECT content AS \"content!: String\",\n lock AS \"lock: String\", language AS \"language: Option\", envs AS \"envs: Vec\", schema AS \"schema: String\", schema_validation AS \"schema_validation: bool\", codebase LIKE '%.tar' as use_tar FROM script WHERE hash = $1 LIMIT 1", "describe": { "columns": [ { @@ -83,5 +83,5 @@ null ] }, - "hash": "03ae5b1c912b13a8a7aadf50cb4984a2ea952e782fd52eb3088454690bd13dd1" + "hash": "a9db7b2f435bb82acb8c5eeb7f800b28f3256491fdaa168591adc7b4b9f3327a" } diff --git a/backend/.sqlx/query-a439552f74ed0ba305e3d9cb99ae9e5d24834082ebf2fe9fd3964fdd80b69ccb.json b/backend/.sqlx/query-aa523c363186575b4bd2537b8e2430e6938e7cc35f8c9e2d1c5459a85443cbdd.json similarity index 52% rename from backend/.sqlx/query-a439552f74ed0ba305e3d9cb99ae9e5d24834082ebf2fe9fd3964fdd80b69ccb.json rename to backend/.sqlx/query-aa523c363186575b4bd2537b8e2430e6938e7cc35f8c9e2d1c5459a85443cbdd.json index 691a6d5e31..520fb2c559 100644 --- a/backend/.sqlx/query-a439552f74ed0ba305e3d9cb99ae9e5d24834082ebf2fe9fd3964fdd80b69ccb.json +++ b/backend/.sqlx/query-aa523c363186575b4bd2537b8e2430e6938e7cc35f8c9e2d1c5459a85443cbdd.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1, custom_tags = $2,\n occupancy_rate = $3, memory_usage = $4, wm_memory_usage = $5, vcpus = COALESCE($7, vcpus),\n memory = COALESCE($8, memory), occupancy_rate_15s = $9, occupancy_rate_5m = $10, occupancy_rate_30m = $11 WHERE worker = $6", + "query": "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1, custom_tags = $2,\n occupancy_rate = $3, memory_usage = $4, wm_memory_usage = $5, vcpus = COALESCE($7, vcpus),\n memory = COALESCE($8, memory), occupancy_rate_15s = $9, occupancy_rate_5m = $10, occupancy_rate_30m = $11 WHERE worker = $6", "describe": { "columns": [], "parameters": { @@ -20,5 +20,5 @@ }, "nullable": [] }, - "hash": "a439552f74ed0ba305e3d9cb99ae9e5d24834082ebf2fe9fd3964fdd80b69ccb" + "hash": "aa523c363186575b4bd2537b8e2430e6938e7cc35f8c9e2d1c5459a85443cbdd" } diff --git a/backend/.sqlx/query-c00bae0d8c9bee37cbad4de4cb02c80d00f52a3fc32bf32271ebc90f7837abda.json b/backend/.sqlx/query-c00bae0d8c9bee37cbad4de4cb02c80d00f52a3fc32bf32271ebc90f7837abda.json deleted file mode 100644 index 74aa6c4ffb..0000000000 --- a/backend/.sqlx/query-c00bae0d8c9bee37cbad4de4cb02c80d00f52a3fc32bf32271ebc90f7837abda.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_queue\n SET canceled_by = 'timeout'\n , canceled_reason = $1\n WHERE id = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "c00bae0d8c9bee37cbad4de4cb02c80d00f52a3fc32bf32271ebc90f7837abda" -} diff --git a/backend/.sqlx/query-c3025cdb6e421e1225d420e8b1efd18d1dd3bb2fac53c1f2df648b61fb7488aa.json b/backend/.sqlx/query-c3025cdb6e421e1225d420e8b1efd18d1dd3bb2fac53c1f2df648b61fb7488aa.json new file mode 100644 index 0000000000..638417aabb --- /dev/null +++ b/backend/.sqlx/query-c3025cdb6e421e1225d420e8b1efd18d1dd3bb2fac53c1f2df648b61fb7488aa.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE worker_ping SET \nping_at = now(), \njobs_executed = 1, \ncurrent_job_id = $1, \ncurrent_job_workspace_id = 'admins' \nWHERE worker = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [] + }, + "hash": "c3025cdb6e421e1225d420e8b1efd18d1dd3bb2fac53c1f2df648b61fb7488aa" +} diff --git a/backend/.sqlx/query-f367a1c8f80dd414dcbcd949374eeb5770796f00b5b3d547163bcfdaed65d8ae.json b/backend/.sqlx/query-c53e1c7133c8ae187656eef5999509fae17fb0ba43e084327accbb5b24c3dfbd.json similarity index 53% rename from backend/.sqlx/query-f367a1c8f80dd414dcbcd949374eeb5770796f00b5b3d547163bcfdaed65d8ae.json rename to backend/.sqlx/query-c53e1c7133c8ae187656eef5999509fae17fb0ba43e084327accbb5b24c3dfbd.json index 776443782e..e343b8b863 100644 --- a/backend/.sqlx/query-f367a1c8f80dd414dcbcd949374eeb5770796f00b5b3d547163bcfdaed65d8ae.json +++ b/backend/.sqlx/query-c53e1c7133c8ae187656eef5999509fae17fb0ba43e084327accbb5b24c3dfbd.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO outstanding_wait_time(job_id, self_wait_time_ms) VALUES ($1, $2)\n ON CONFLICT (job_id) DO UPDATE SET self_wait_time_ms = EXCLUDED.self_wait_time_ms", + "query": "INSERT INTO outstanding_wait_time(job_id, self_wait_time_ms) VALUES ($1, $2)\n ON CONFLICT (job_id) DO UPDATE SET self_wait_time_ms = EXCLUDED.self_wait_time_ms", "describe": { "columns": [], "parameters": { @@ -11,5 +11,5 @@ }, "nullable": [] }, - "hash": "f367a1c8f80dd414dcbcd949374eeb5770796f00b5b3d547163bcfdaed65d8ae" + "hash": "c53e1c7133c8ae187656eef5999509fae17fb0ba43e084327accbb5b24c3dfbd" } diff --git a/backend/.sqlx/query-faf2c77242e0ab39b33886edf3b742531bf1351d0be1c3631bde0adfe375497a.json b/backend/.sqlx/query-faf2c77242e0ab39b33886edf3b742531bf1351d0be1c3631bde0adfe375497a.json new file mode 100644 index 0000000000..ea798bab76 --- /dev/null +++ b/backend/.sqlx/query-faf2c77242e0ab39b33886edf3b742531bf1351d0be1c3631bde0adfe375497a.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT tag FROM v2_job WHERE id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "tag", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + false + ] + }, + "hash": "faf2c77242e0ab39b33886edf3b742531bf1351d0be1c3631bde0adfe375497a" +} diff --git a/backend/.sqlx/query-fec6d5674dc6b5a6a0ece419c40508835affcb7679a48f2a443777e829bd1e74.json b/backend/.sqlx/query-fec6d5674dc6b5a6a0ece419c40508835affcb7679a48f2a443777e829bd1e74.json index de2e819af2..649ab85650 100644 --- a/backend/.sqlx/query-fec6d5674dc6b5a6a0ece419c40508835affcb7679a48f2a443777e829bd1e74.json +++ b/backend/.sqlx/query-fec6d5674dc6b5a6a0ece419c40508835affcb7679a48f2a443777e829bd1e74.json @@ -41,11 +41,11 @@ ] }, "nullable": [ - true, - true, - true, - true, - true, + false, + false, + false, + false, + false, true ] }, diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 62a1d53025..ca8c170ad0 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -2572,7 +2572,7 @@ dependencies = [ "hashbrown 0.14.5", "lock_api", "once_cell", - "parking_lot_core", + "parking_lot_core 0.9.10", ] [[package]] @@ -2626,7 +2626,7 @@ dependencies = [ "log", "num_cpus", "object_store", - "parking_lot", + "parking_lot 0.12.3", "parquet", "paste", "pin-project-lite", @@ -2687,7 +2687,7 @@ dependencies = [ "hashbrown 0.14.5", "log", "object_store", - "parking_lot", + "parking_lot 0.12.3", "rand 0.8.5", "tempfile", "url", @@ -2867,7 +2867,7 @@ dependencies = [ "itertools 0.12.1", "log", "once_cell", - "parking_lot", + "parking_lot 0.12.3", "pin-project-lite", "rand 0.8.5", "tokio", @@ -2998,7 +2998,7 @@ dependencies = [ "indexmap 2.8.0", "log", "once_cell", - "parking_lot", + "parking_lot 0.12.3", "serde", "serde_json", "sha2 0.10.8", @@ -3080,7 +3080,7 @@ dependencies = [ "indexmap 2.8.0", "libc", "memoffset", - "parking_lot", + "parking_lot 0.12.3", "percent-encoding", "pin-project", "serde", @@ -3325,7 +3325,7 @@ dependencies = [ "log", "once_cell", "os_pipe", - "parking_lot", + "parking_lot 0.12.3", "pin-project", "rand 0.8.5", "tokio", @@ -3705,7 +3705,7 @@ dependencies = [ "log", "node_resolver", "once_cell", - "parking_lot", + "parking_lot 0.12.3", "sys_traits", "thiserror 2.0.12", "url", @@ -3862,7 +3862,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d774fd83f26b24f0805a6ab8b26834a0d06ceac0db517b769b1e4633c96a2057" dependencies = [ "futures", - "parking_lot", + "parking_lot 0.12.3", "tokio", ] @@ -4972,7 +4972,7 @@ checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" dependencies = [ "futures-core", "lock_api", - "parking_lot", + "parking_lot 0.12.3", ] [[package]] @@ -5638,7 +5638,7 @@ dependencies = [ "ipconfig", "moka", "once_cell", - "parking_lot", + "parking_lot 0.12.3", "rand 0.9.0", "resolv-conf", "serde", @@ -7230,7 +7230,7 @@ dependencies = [ "crossbeam-epoch", "crossbeam-utils", "loom", - "parking_lot", + "parking_lot 0.12.3", "portable-atomic", "rustc_version 0.4.1", "smallvec", @@ -7933,7 +7933,7 @@ dependencies = [ "hyper 1.6.0", "itertools 0.13.0", "md-5 0.10.6", - "parking_lot", + "parking_lot 0.12.3", "percent-encoding", "quick-xml 0.36.2", "rand 0.8.5", @@ -8349,6 +8349,17 @@ version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core 0.8.6", +] + [[package]] name = "parking_lot" version = "0.12.3" @@ -8356,7 +8367,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" dependencies = [ "lock_api", - "parking_lot_core", + "parking_lot_core 0.9.10", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" +dependencies = [ + "cfg-if", + "instant", + "libc", + "redox_syscall 0.2.16", + "smallvec", + "winapi", ] [[package]] @@ -8957,7 +8982,7 @@ dependencies = [ "fnv", "lazy_static", "memchr", - "parking_lot", + "parking_lot 0.12.3", "thiserror 2.0.12", ] @@ -9111,7 +9136,7 @@ dependencies = [ "ahash 0.8.11", "equivalent", "hashbrown 0.15.2", - "parking_lot", + "parking_lot 0.12.3", ] [[package]] @@ -9359,6 +9384,15 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03251193000f4bd3b042892be858ee50e8b3719f2b08e5833ac4353724632430" +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_syscall" version = "0.3.5" @@ -9568,6 +9602,43 @@ dependencies = [ "windows-registry", ] +[[package]] +name = "reqwest-middleware" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64e8975513bd9a7a43aad01030e79b3498e05db14e9d945df6483e8cf9b8c4c4" +dependencies = [ + "anyhow", + "async-trait", + "http 1.3.1", + "reqwest 0.12.15", + "serde", + "thiserror 1.0.69", + "tower-service", +] + +[[package]] +name = "reqwest-retry" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29c73e4195a6bfbcb174b790d9b3407ab90646976c55de58a6515da25d851178" +dependencies = [ + "anyhow", + "async-trait", + "futures", + "getrandom 0.2.15", + "http 1.3.1", + "hyper 1.6.0", + "parking_lot 0.11.2", + "reqwest 0.12.15", + "reqwest-middleware", + "retry-policies", + "thiserror 1.0.69", + "tokio", + "tracing", + "wasm-timer", +] + [[package]] name = "resolv-conf" version = "0.7.1" @@ -9577,6 +9648,15 @@ dependencies = [ "hostname", ] +[[package]] +name = "retry-policies" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5875471e6cab2871bc150ecb8c727db5113c9338cc3354dc5ee3425b6aa40a1c" +dependencies = [ + "rand 0.8.5", +] + [[package]] name = "rfc6979" version = "0.4.0" @@ -12264,7 +12344,7 @@ dependencies = [ "bytes", "libc", "mio 1.0.3", - "parking_lot", + "parking_lot 0.12.3", "pin-project-lite", "signal-hook-registry", "socket2", @@ -12328,7 +12408,7 @@ dependencies = [ "futures-channel", "futures-util", "log", - "parking_lot", + "parking_lot 0.12.3", "percent-encoding", "phf", "pin-project-lite", @@ -12354,7 +12434,7 @@ dependencies = [ "futures-channel", "futures-util", "log", - "parking_lot", + "parking_lot 0.12.3", "percent-encoding", "phf", "pin-project-lite", @@ -12613,7 +12693,7 @@ dependencies = [ "cookie 0.18.1", "futures-util", "http 1.3.1", - "parking_lot", + "parking_lot 0.12.3", "pin-project-lite", "tower-layer", "tower-service", @@ -13460,6 +13540,21 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wasm-timer" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be0ecb0db480561e9a7642b5d3e4187c128914e58aa84330b9493e3eb68c5e7f" +dependencies = [ + "futures", + "js-sys", + "parking_lot 0.11.2", + "pin-utils", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "wasm_dep_analyzer" version = "0.2.0" @@ -13533,7 +13628,7 @@ dependencies = [ "log", "naga", "once_cell", - "parking_lot", + "parking_lot 0.12.3", "profiling", "raw-window-handle", "ron", @@ -13575,7 +13670,7 @@ dependencies = [ "ndk-sys", "objc", "once_cell", - "parking_lot", + "parking_lot 0.12.3", "profiling", "range-alloc", "raw-window-handle", @@ -13813,8 +13908,10 @@ dependencies = [ "windmill-indexer", "windmill-parser", "windmill-parser-py", + "windmill-parser-py-imports", "windmill-parser-ts", "windmill-queue", + "windmill-worker", ] [[package]] @@ -13901,6 +13998,8 @@ dependencies = [ "rand 0.9.0", "regex", "reqwest 0.12.15", + "reqwest-middleware", + "reqwest-retry", "semver 1.0.26", "serde", "serde_json", @@ -14267,6 +14366,7 @@ dependencies = [ "rand 0.9.0", "regex", "reqwest 0.12.15", + "reqwest-middleware", "rust_decimal", "serde", "serde_json", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index ee77d6441e..561758cab2 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -49,6 +49,7 @@ lto = "thin" [features] default = [] +agent_worker_server = ["windmill-api/agent_worker_server"] enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmill-api/enterprise", "dep:windmill-autoscaling", "windmill-autoscaling/enterprise", "windmill-git-sync/enterprise", "windmill-common/prometheus", "windmill-common/enterprise"] enterprise_saml = ["windmill-api/enterprise_saml", "oauth2"] stripe = ["windmill-api/stripe"] @@ -170,6 +171,9 @@ windmill-parser-graphql = { path = "./parsers/windmill-parser-graphql" } windmill-parser-php = { path = "./parsers/windmill-parser-php" } windmill-api-client = { path = "./windmill-api-client" } +reqwest-retry = "^0" +reqwest-middleware = { version = "^0", features = ["json"] } + rustls = "0.23.0" memchr = "2.7.4" axum = { version = "^0.7", features = ["multipart"] } diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index fa6472967e..e84f5aeecd 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -619f7dcd1097bfcd2f8b85f2075782219ff76be1 \ No newline at end of file +b769ca52ebdaf2c1ad110622e258be06d00eb164 \ No newline at end of file diff --git a/backend/src/main.rs b/backend/src/main.rs index c7b988f12c..a076b4c63f 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -15,7 +15,7 @@ use monitor::{ send_logs_to_object_store, }; use rand::Rng; -use sqlx::{postgres::PgListener, Pool, Postgres}; +use sqlx::postgres::PgListener; use std::{ collections::HashMap, fs::{create_dir_all, DirBuilder}, @@ -30,6 +30,7 @@ use windmill_api::HTTP_CLIENT; use windmill_common::ee::{maybe_renew_license_key_on_start, LICENSE_KEY_ID, LICENSE_KEY_VALID}; use windmill_common::{ + agent_workers::build_agent_http_client, get_database_url, global_settings::{ BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING, @@ -48,7 +49,9 @@ use windmill_common::{ scripts::ScriptLang, stats_ee::schedule_stats, utils::{hostname, rd_string, Mode, GIT_VERSION, MODE_AND_ADDONS}, - worker::{reload_custom_tags_setting, HUB_CACHE_DIR, TMP_DIR, TMP_LOGS_DIR, WORKER_GROUP}, + worker::{ + reload_custom_tags_setting, Connection, HUB_CACHE_DIR, TMP_DIR, TMP_LOGS_DIR, WORKER_GROUP, + }, KillpillSender, METRICS_ENABLED, }; @@ -95,6 +98,31 @@ const DEFAULT_SERVER_BIND_ADDR: Ipv4Addr = Ipv4Addr::new(0, 0, 0, 0); mod ee; mod monitor; +pub fn setup_deno_runtime() -> anyhow::Result<()> { + // https://github.com/denoland/deno/blob/main/cli/main.rs#L477 + #[cfg(feature = "deno_core")] + let unrecognized_v8_flags = deno_core::v8_set_flags(vec![ + "--stack-size=1024".to_string(), + // TODO(bartlomieju): I think this can be removed as it's handled by `deno_core` + // and its settings. + // deno_ast removes TypeScript `assert` keywords, so this flag only affects JavaScript + // TODO(petamoriken): Need to check TypeScript `assert` keywords in deno_ast + "--no-harmony-import-assertions".to_string(), + ]) + .into_iter() + .skip(1) + .collect::>(); + + #[cfg(feature = "deno_core")] + if !unrecognized_v8_flags.is_empty() { + println!("Unrecognized V8 flags: {:?}", unrecognized_v8_flags); + } + + #[cfg(feature = "deno_core")] + deno_core::JsRuntime::init_platform(None, false); + Ok(()) +} + #[inline(always)] fn create_and_run_current_thread_inner(future: F) -> R where @@ -125,27 +153,7 @@ lazy_static::lazy_static! { } pub fn main() -> anyhow::Result<()> { - // https://github.com/denoland/deno/blob/main/cli/main.rs#L477 - #[cfg(feature = "deno_core")] - let unrecognized_v8_flags = deno_core::v8_set_flags(vec![ - "--stack-size=1024".to_string(), - // TODO(bartlomieju): I think this can be removed as it's handled by `deno_core` - // and its settings. - // deno_ast removes TypeScript `assert` keywords, so this flag only affects JavaScript - // TODO(petamoriken): Need to check TypeScript `assert` keywords in deno_ast - "--no-harmony-import-assertions".to_string(), - ]) - .into_iter() - .skip(1) - .collect::>(); - - #[cfg(feature = "deno_core")] - if !unrecognized_v8_flags.is_empty() { - println!("Unrecognized V8 flags: {:?}", unrecognized_v8_flags); - } - - #[cfg(feature = "deno_core")] - deno_core::JsRuntime::init_platform(None, false); + setup_deno_runtime()?; create_and_run_current_thread_inner(windmill_main()) } @@ -321,23 +329,47 @@ async fn windmill_main() -> anyhow::Result<()> { IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)) }; - println!("Connecting to database..."); - let db = windmill_common::initial_connection().await?; + let mut first_worker_suffix = None; + let mut worker_names = vec![]; - let num_version = sqlx::query_scalar!("SELECT version()").fetch_one(&db).await; + for _ in 0..num_workers { + let suffix = windmill_common::utils::worker_suffix(&hostname, &rd_string(5)); + worker_names.push(windmill_common::utils::worker_name_with_suffix( + mode == Mode::Agent, + WORKER_GROUP.as_str(), + &suffix, + )); + if first_worker_suffix.is_none() { + first_worker_suffix = Some(suffix); + } + } - tracing::info!( - "PostgreSQL version: {} (windmill require PG >= 14)", - num_version - .ok() - .flatten() - .unwrap_or_else(|| "UNKNOWN".to_string()) - ); - load_otel(&db).await; + let conn = if mode == Mode::Agent { + let worker_suffix = first_worker_suffix.unwrap_or_else(|| { + panic!("there must be at least one worker in agent mode"); + }); + Connection::Http(build_agent_http_client(&worker_suffix)) + } else { + println!("Connecting to database..."); - tracing::info!("Database connected"); + let db = windmill_common::initial_connection().await?; - let environment = load_base_url(&db) + let num_version = sqlx::query_scalar!("SELECT version()").fetch_one(&db).await; + + tracing::info!( + "PostgreSQL version: {} (windmill require PG >= 14)", + num_version + .ok() + .flatten() + .unwrap_or_else(|| "UNKNOWN".to_string()) + ); + load_otel(&db).await; + + tracing::info!("Database connected"); + Connection::Sql(db) + }; + + let environment = load_base_url(&conn) .await .unwrap_or_else(|_| "local".to_string()) .trim_start_matches("https://") @@ -357,23 +389,29 @@ async fn windmill_main() -> anyhow::Result<()> { .ok() .is_some_and(|x| x == "1" || x == "true"); - if !is_agent && !indexer_mode { - let skip_migration = std::env::var("SKIP_MIGRATION") - .map(|val| val == "true") - .unwrap_or(false); + if let Some(db) = conn.as_sql() { + if !is_agent && !indexer_mode { + let skip_migration = std::env::var("SKIP_MIGRATION") + .map(|val| val == "true") + .unwrap_or(false); - if !skip_migration { - // migration code to avoid break - migration_handle = windmill_api::migrate_db(&db).await?; - } else { - tracing::info!("SKIP_MIGRATION set, skipping db migration...") + if !skip_migration { + // migration code to avoid break + migration_handle = windmill_api::migrate_db(&db).await?; + } else { + tracing::info!("SKIP_MIGRATION set, skipping db migration...") + } } } - drop(db); let worker_mode = num_workers > 0; - let db = windmill_common::connect_db(server_mode, indexer_mode, worker_mode).await?; + let conn = if mode == Mode::Agent { + conn + } else { + let db = windmill_common::connect_db(server_mode, indexer_mode, worker_mode).await?; + Connection::Sql(db) + }; let (killpill_tx, mut killpill_rx) = KillpillSender::new(2); let mut monitor_killpill_rx = killpill_tx.subscribe(); @@ -401,12 +439,14 @@ Windmill Community Edition {GIT_VERSION} display_config(&ENV_SETTINGS); - if let Err(e) = reload_base_url_setting(&db).await { + if let Err(e) = reload_base_url_setting(&conn).await { tracing::error!("Error loading base url: {:?}", e) } - if let Err(e) = reload_critical_error_channels_setting(&db).await { - tracing::error!("Could loading critical error emails setting: {:?}", e); + if let Some(db) = conn.as_sql() { + if let Err(e) = reload_critical_error_channels_setting(&db).await { + tracing::error!("Could loading critical error emails setting: {:?}", e); + } } #[cfg(feature = "enterprise")] @@ -415,7 +455,7 @@ Windmill Community Edition {GIT_VERSION} // if not valid and not server mode just quit // if not expired and server mode then force renewal // if key still invalid and num_workers > 0, set to 0 - if let Err(err) = reload_license_key(&db).await { + if let Err(err) = reload_license_key(&conn).await { tracing::error!("Failed to reload license key: {err:#}"); } let valid_key = *LICENSE_KEY_VALID.read().await; @@ -423,17 +463,21 @@ Windmill Community Edition {GIT_VERSION} tracing::error!("Invalid license key, workers require a valid license key"); } if server_mode { - // only force renewal if invalid but not empty (= expired) - let renewed_now = maybe_renew_license_key_on_start( - &HTTP_CLIENT, - &db, - !valid_key && !LICENSE_KEY_ID.read().await.is_empty(), - ) - .await; - if renewed_now { - if let Err(err) = reload_license_key(&db).await { - tracing::error!("Failed to reload license key: {err:#}"); + if let Some(db) = conn.as_sql() { + // only force renewal if invalid but not empty (= expired) + let renewed_now = maybe_renew_license_key_on_start( + &HTTP_CLIENT, + &db, + !valid_key && !LICENSE_KEY_ID.read().await.is_empty(), + ) + .await; + if renewed_now { + if let Err(err) = reload_license_key(&conn).await { + tracing::error!("Failed to reload license key: {err:#}"); + } } + } else { + panic!("Server mode requires a database connection"); } } } @@ -461,7 +505,7 @@ Windmill Community Edition {GIT_VERSION} }; initial_load( - &db, + &conn, killpill_tx.clone(), worker_mode, server_mode, @@ -471,7 +515,7 @@ Windmill Community Edition {GIT_VERSION} .await; monitor_db( - &db, + &conn, &base_internal_url, server_mode, worker_mode, @@ -481,9 +525,11 @@ Windmill Community Edition {GIT_VERSION} .await; #[cfg(feature = "prometheus")] - crate::monitor::monitor_pool(&db).await; + if let Some(db) = conn.as_sql() { + crate::monitor::monitor_pool(&db).await; + } - send_logs_to_object_store(&db, &hostname, &mode); + send_logs_to_object_store(&conn, &hostname, &mode); #[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))] if !worker_mode { @@ -504,15 +550,18 @@ Windmill Community Edition {GIT_VERSION} #[cfg(feature = "tantivy")] if should_index_jobs { - reload_indexer_config(&db).await; + if let Some(db) = conn.as_sql() { + reload_indexer_config(&db).await; + } } #[cfg(feature = "tantivy")] let (index_reader, index_writer) = if should_index_jobs { - let mut indexer_rx = killpill_rx.resubscribe(); + if let Some(db) = conn.as_sql() { + let mut indexer_rx = killpill_rx.resubscribe(); - let (mut reader, mut writer) = (None, None); - tokio::select! { + let (mut reader, mut writer) = (None, None); + tokio::select! { _ = indexer_rx.recv() => { tracing::info!("Received killpill, aborting index initialization"); }, @@ -522,8 +571,11 @@ Windmill Community Edition {GIT_VERSION} writer = Some(res.1); } + } + (reader, writer) + } else { + (None, None) } - (reader, writer) } else { (None, None) }; @@ -533,13 +585,15 @@ Windmill Community Edition {GIT_VERSION} let indexer_rx = killpill_rx.resubscribe(); let index_writer2 = index_writer.clone(); async { - if let Some(index_writer) = index_writer2 { - windmill_indexer::completed_runs_ee::run_indexer( - db.clone(), - index_writer, - indexer_rx, - ) - .await?; + if let Some(db) = conn.as_sql() { + if let Some(index_writer) = index_writer2 { + windmill_indexer::completed_runs_ee::run_indexer( + db.clone(), + index_writer, + indexer_rx, + ) + .await?; + } } Ok(()) } @@ -547,21 +601,25 @@ Windmill Community Edition {GIT_VERSION} #[cfg(all(feature = "tantivy", feature = "parquet"))] let (log_index_reader, log_index_writer) = if should_index_jobs { - let mut indexer_rx = killpill_rx.resubscribe(); + if let Some(db) = conn.as_sql() { + let mut indexer_rx = killpill_rx.resubscribe(); + + let (mut reader, mut writer) = (None, None); + tokio::select! { + _ = indexer_rx.recv() => { + tracing::info!("Received killpill, aborting index initialization"); + }, + res = windmill_indexer::service_logs_ee::init_index(&db, killpill_tx.clone()) => { + let res = res?; + reader = Some(res.0); + writer = Some(res.1); + } - let (mut reader, mut writer) = (None, None); - tokio::select! { - _ = indexer_rx.recv() => { - tracing::info!("Received killpill, aborting index initialization"); - }, - res = windmill_indexer::service_logs_ee::init_index(&db, killpill_tx.clone()) => { - let res = res?; - reader = Some(res.0); - writer = Some(res.1); } - + (reader, writer) + } else { + (None, None) } - (reader, writer) } else { (None, None) }; @@ -571,13 +629,15 @@ Windmill Community Edition {GIT_VERSION} let log_indexer_rx = killpill_rx.resubscribe(); let log_index_writer2 = log_index_writer.clone(); async { - if let Some(log_index_writer) = log_index_writer2 { - windmill_indexer::service_logs_ee::run_indexer( - db.clone(), - log_index_writer, - log_indexer_rx, - ) - .await?; + if let Some(db) = conn.as_sql() { + if let Some(log_index_writer) = log_index_writer2 { + windmill_indexer::service_logs_ee::run_indexer( + db.clone(), + log_index_writer, + log_indexer_rx, + ) + .await?; + } } Ok(()) } @@ -597,18 +657,19 @@ Windmill Community Edition {GIT_VERSION} let server_f = async { if !is_agent { - windmill_api::run_server( - db.clone(), - index_reader, - log_index_reader, - addr, - server_killpill_rx, - base_internal_tx, - server_mode, - #[cfg(feature = "smtp")] - base_internal_url.clone(), - ) - .await?; + if let Some(db) = conn.as_sql() { + windmill_api::run_server( + db.clone(), + index_reader, + log_index_reader, + addr, + server_killpill_rx, + base_internal_tx, + server_mode, + base_internal_url.clone(), + ) + .await?; + } } else { base_internal_tx .send(base_internal_url.clone()) @@ -626,13 +687,13 @@ Windmill Community Edition {GIT_VERSION} let base_internal_url = base_internal_rx.await?; if worker_mode { run_workers( - db.clone(), + conn.clone(), rx, killpill_tx.clone(), num_workers, base_internal_url.clone(), - is_agent, hostname.clone(), + &worker_names, ) .await?; tracing::info!("All workers exited."); @@ -654,285 +715,305 @@ Windmill Community Edition {GIT_VERSION} }; let monitor_f = async { - let db = db.clone(); let tx = killpill_tx.clone(); - - let base_internal_url = base_internal_url.to_string(); - let db_url: String = get_database_url().await?; - - let h = tokio::spawn(async move { - let mut listener = retry_listen_pg(&db_url).await; - let mut last_listener_refresh = Instant::now(); - loop { - tokio::select! { - biased; - Some(_) = async { if let Some(jh) = migration_handle.take() { - tracing::info!("migration job finished"); - Some(jh.await) - } else { - None - }} => { - continue; - }, - _ = monitor_killpill_rx.recv() => { - tracing::info!("received killpill for monitor job"); - break; - }, - notification = listener.try_recv() => { - match notification { - Ok(n) => { - if n.is_none() { - tracing::error!("Could not receive notification, attempting to reconnect to pg listener"); - continue; - } - let n = n.unwrap(); - tracing::info!("Received new pg notification: {n:?}"); - match n.channel() { - "notify_config_change" => { - match n.payload() { - "server" if server_mode => { - tracing::error!("Server config change detected but server config is obsolete: {}", n.payload()); + let conn = conn.clone(); + match conn { + Connection::Sql(ref db) => { + let base_internal_url = base_internal_url.to_string(); + let db_url: String = get_database_url().await?; + let db = db.clone(); + let h = tokio::spawn(async move { + let mut listener = retry_listen_pg(&db_url).await; + let mut last_listener_refresh = Instant::now(); + loop { + let db = db.clone(); + tokio::select! { + biased; + Some(_) = async { if let Some(jh) = migration_handle.take() { + tracing::info!("migration job finished"); + Some(jh.await) + } else { + None + }} => { + continue; + }, + _ = monitor_killpill_rx.recv() => { + tracing::info!("received killpill for monitor job"); + break; + }, + notification = listener.try_recv() => { + match notification { + Ok(n) => { + if n.is_none() { + tracing::error!("Could not receive notification, attempting to reconnect to pg listener"); + continue; + } + let n = n.unwrap(); + tracing::info!("Received new pg notification: {n:?}"); + match n.channel() { + "notify_config_change" => { + match n.payload() { + "server" if server_mode => { + tracing::error!("Server config change detected but server config is obsolete: {}", n.payload()); + }, + a@ _ if worker_mode && a == format!("worker__{}", *WORKER_GROUP) => { + tracing::info!("Worker config change detected: {}", n.payload()); + reload_worker_config(&db, tx.clone(), true).await; + }, + _ => { + tracing::debug!("config changed but did not target this server/worker"); + } + } }, - a@ _ if worker_mode && a == format!("worker__{}", *WORKER_GROUP) => { - tracing::info!("Worker config change detected: {}", n.payload()); - reload_worker_config(&db, tx.clone(), true).await; + "notify_webhook_change" => { + let workspace_id = n.payload(); + tracing::info!("Webhook change detected, invalidating webhook cache: {}", workspace_id); + windmill_api::webhook_util::WEBHOOK_CACHE.remove(workspace_id); + }, + "notify_workspace_envs_change" => { + let workspace_id = n.payload(); + tracing::info!("Workspace envs change detected, invalidating workspace envs cache: {}", workspace_id); + windmill_common::variables::CUSTOM_ENVS_CACHE.remove(workspace_id); + }, + "notify_workspace_premium_change" => { + let workspace_id = n.payload(); + tracing::info!("Workspace premium change detected, invalidating workspace premium cache: {}", workspace_id); + windmill_common::workspaces::IS_PREMIUM_CACHE.remove(workspace_id); + }, + "notify_global_setting_change" => { + tracing::info!("Global setting change detected: {}", n.payload()); + match n.payload() { + BASE_URL_SETTING => { + if let Err(e) = reload_base_url_setting(&conn).await { + tracing::error!(error = %e, "Could not reload base url setting"); + } + }, + OAUTH_SETTING => { + if let Err(e) = reload_base_url_setting(&conn).await { + tracing::error!(error = %e, "Could not reload oauth setting"); + } + }, + CUSTOM_TAGS_SETTING => { + if let Err(e) = reload_custom_tags_setting(&db).await { + tracing::error!(error = %e, "Could not reload custom tags setting"); + } + }, + LICENSE_KEY_SETTING => { + if let Err(e) = reload_license_key(&db.into()).await { + tracing::error!("Failed to reload license key: {e:#}"); + } + }, + DEFAULT_TAGS_PER_WORKSPACE_SETTING => { + if let Err(e) = load_tag_per_workspace_enabled(&db).await { + tracing::error!("Error loading default tag per workspace: {e:#}"); + } + }, + DEFAULT_TAGS_WORKSPACES_SETTING => { + if let Err(e) = load_tag_per_workspace_workspaces(&db).await { + tracing::error!("Error loading default tag per workspace workspaces: {e:#}"); + } + } + SMTP_SETTING => { + reload_smtp_config(&db).await; + }, + TEAMS_SETTING => { + tracing::info!("Teams setting changed."); + }, + INDEXER_SETTING => { + reload_indexer_config(&db).await; + }, + TIMEOUT_WAIT_RESULT_SETTING => { + reload_timeout_wait_result_setting(&conn).await + }, + RETENTION_PERIOD_SECS_SETTING => { + reload_retention_period_setting(&conn).await + }, + MONITOR_LOGS_ON_OBJECT_STORE_SETTING => { + reload_delete_logs_periodically_setting(&conn).await + }, + JOB_DEFAULT_TIMEOUT_SECS_SETTING => { + reload_job_default_timeout_setting(&conn).await + }, + #[cfg(feature = "parquet")] + OBJECT_STORE_CACHE_CONFIG_SETTING => { + if !disable_s3_store { + reload_s3_cache_setting(&db).await + } + }, + SCIM_TOKEN_SETTING => { + reload_scim_token_setting(&conn).await + }, + EXTRA_PIP_INDEX_URL_SETTING => { + reload_extra_pip_index_url_setting(&conn).await + }, + PIP_INDEX_URL_SETTING => { + reload_pip_index_url_setting(&conn).await + }, + INSTANCE_PYTHON_VERSION_SETTING => { + reload_instance_python_version_setting(&conn).await + }, + NPM_CONFIG_REGISTRY_SETTING => { + reload_npm_config_registry_setting(&conn).await + }, + BUNFIG_INSTALL_SCOPES_SETTING => { + reload_bunfig_install_scopes_setting(&conn).await + }, + NUGET_CONFIG_SETTING => { + reload_nuget_config_setting(&conn).await + }, + MAVEN_REPOS_SETTING => { + reload_maven_repos_setting(&conn).await + }, + NO_DEFAULT_MAVEN_SETTING => { + reload_no_default_maven_setting(&conn).await + }, + KEEP_JOB_DIR_SETTING => { + load_keep_job_dir(&conn).await; + }, + REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING => { + load_require_preexisting_user(&db).await; + }, + EXPOSE_METRICS_SETTING => { + tracing::info!("Metrics setting changed, restarting"); + send_delayed_killpill(&tx, 40, "metrics setting change").await; + }, + EMAIL_DOMAIN_SETTING => { + tracing::info!("Email domain setting changed"); + if server_mode { + send_delayed_killpill(&tx, 4, "email domain setting change").await; + } + }, + EXPOSE_DEBUG_METRICS_SETTING => { + if let Err(e) = load_metrics_debug_enabled(&conn).await { + tracing::error!(error = %e, "Could not reload debug metrics setting"); + } + }, + OTEL_SETTING => { + tracing::info!("OTEL setting changed, restarting"); + send_delayed_killpill(&tx, 4, "OTEL setting change").await; + }, + REQUEST_SIZE_LIMIT_SETTING => { + if server_mode { + tracing::info!("Request limit size change detected, killing server expecting to be restarted"); + send_delayed_killpill(&tx, 4, "request size limit change").await; + } + }, + SAML_METADATA_SETTING => { + tracing::info!("SAML metadata change detected, killing server expecting to be restarted"); + send_delayed_killpill(&tx, 0, "SAML metadata change").await; + }, + HUB_BASE_URL_SETTING => { + if let Err(e) = reload_hub_base_url_setting(&conn, server_mode).await { + tracing::error!(error = %e, "Could not reload hub base url setting"); + } + }, + CRITICAL_ERROR_CHANNELS_SETTING => { + if let Err(e) = reload_critical_error_channels_setting(&db).await { + tracing::error!(error = %e, "Could not reload critical error emails setting"); + } + }, + JWT_SECRET_SETTING => { + if let Err(e) = reload_jwt_secret_setting(&db).await { + tracing::error!(error = %e, "Could not reload jwt secret setting"); + } + }, + CRITICAL_ALERT_MUTE_UI_SETTING => { + tracing::info!("Critical alert UI setting changed"); + if let Err(e) = reload_critical_alert_mute_ui_setting(&conn).await { + tracing::error!(error = %e, "Could not reload critical alert UI setting"); + } + }, + + a @_ => { + tracing::info!("Unrecognized Global Setting Change Payload: {:?}", a); + } + } }, _ => { - tracing::debug!("config changed but did not target this server/worker"); + tracing::warn!("Unknown notification received"); + continue; } } }, - "notify_webhook_change" => { - let workspace_id = n.payload(); - tracing::info!("Webhook change detected, invalidating webhook cache: {}", workspace_id); - windmill_api::webhook_util::WEBHOOK_CACHE.remove(workspace_id); - }, - "notify_workspace_envs_change" => { - let workspace_id = n.payload(); - tracing::info!("Workspace envs change detected, invalidating workspace envs cache: {}", workspace_id); - windmill_common::variables::CUSTOM_ENVS_CACHE.remove(workspace_id); - }, - "notify_workspace_premium_change" => { - let workspace_id = n.payload(); - tracing::info!("Workspace premium change detected, invalidating workspace premium cache: {}", workspace_id); - windmill_common::workspaces::IS_PREMIUM_CACHE.remove(workspace_id); - }, - "notify_global_setting_change" => { - tracing::info!("Global setting change detected: {}", n.payload()); - match n.payload() { - BASE_URL_SETTING => { - if let Err(e) = reload_base_url_setting(&db).await { - tracing::error!(error = %e, "Could not reload base url setting"); - } + Err(e) => { + tracing::error!(error = %e, "Could not receive notification, attempting to reconnect listener"); + tokio::select! { + biased; + _ = monitor_killpill_rx.recv() => { + tracing::info!("received killpill for monitor job"); + break; }, - OAUTH_SETTING => { - if let Err(e) = reload_base_url_setting(&db).await { - tracing::error!(error = %e, "Could not reload oauth setting"); - } - }, - CUSTOM_TAGS_SETTING => { - if let Err(e) = reload_custom_tags_setting(&db).await { - tracing::error!(error = %e, "Could not reload custom tags setting"); - } - }, - LICENSE_KEY_SETTING => { - if let Err(e) = reload_license_key(&db).await { - tracing::error!("Failed to reload license key: {e:#}"); - } - }, - DEFAULT_TAGS_PER_WORKSPACE_SETTING => { - if let Err(e) = load_tag_per_workspace_enabled(&db).await { - tracing::error!("Error loading default tag per workspace: {e:#}"); - } - }, - DEFAULT_TAGS_WORKSPACES_SETTING => { - if let Err(e) = load_tag_per_workspace_workspaces(&db).await { - tracing::error!("Error loading default tag per workspace workspaces: {e:#}"); - } - } - SMTP_SETTING => { - reload_smtp_config(&db).await; - }, - TEAMS_SETTING => { - tracing::info!("Teams setting changed."); - }, - INDEXER_SETTING => { - reload_indexer_config(&db).await; - }, - TIMEOUT_WAIT_RESULT_SETTING => { - reload_timeout_wait_result_setting(&db).await - }, - RETENTION_PERIOD_SECS_SETTING => { - reload_retention_period_setting(&db).await - }, - MONITOR_LOGS_ON_OBJECT_STORE_SETTING => { - reload_delete_logs_periodically_setting(&db).await - }, - JOB_DEFAULT_TIMEOUT_SECS_SETTING => { - reload_job_default_timeout_setting(&db).await - }, - #[cfg(feature = "parquet")] - OBJECT_STORE_CACHE_CONFIG_SETTING => { - if !disable_s3_store { - reload_s3_cache_setting(&db).await - } - }, - SCIM_TOKEN_SETTING => { - reload_scim_token_setting(&db).await - }, - EXTRA_PIP_INDEX_URL_SETTING => { - reload_extra_pip_index_url_setting(&db).await - }, - PIP_INDEX_URL_SETTING => { - reload_pip_index_url_setting(&db).await - }, - INSTANCE_PYTHON_VERSION_SETTING => { - reload_instance_python_version_setting(&db).await - }, - NPM_CONFIG_REGISTRY_SETTING => { - reload_npm_config_registry_setting(&db).await - }, - BUNFIG_INSTALL_SCOPES_SETTING => { - reload_bunfig_install_scopes_setting(&db).await - }, - NUGET_CONFIG_SETTING => { - reload_nuget_config_setting(&db).await - }, - MAVEN_REPOS_SETTING => { - reload_maven_repos_setting(&db).await - }, - NO_DEFAULT_MAVEN_SETTING => { - reload_no_default_maven_setting(&db).await - }, - KEEP_JOB_DIR_SETTING => { - load_keep_job_dir(&db).await; - }, - REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING => { - load_require_preexisting_user(&db).await; - }, - EXPOSE_METRICS_SETTING => { - tracing::info!("Metrics setting changed, restarting"); - send_delayed_killpill(&tx, 40, "metrics setting change").await; - }, - EMAIL_DOMAIN_SETTING => { - tracing::info!("Email domain setting changed"); - if server_mode { - send_delayed_killpill(&tx, 4, "email domain setting change").await; - } - }, - EXPOSE_DEBUG_METRICS_SETTING => { - if let Err(e) = load_metrics_debug_enabled(&db).await { - tracing::error!(error = %e, "Could not reload debug metrics setting"); - } - }, - OTEL_SETTING => { - tracing::info!("OTEL setting changed, restarting"); - send_delayed_killpill(&tx, 4, "OTEL setting change").await; - }, - REQUEST_SIZE_LIMIT_SETTING => { - if server_mode { - tracing::info!("Request limit size change detected, killing server expecting to be restarted"); - send_delayed_killpill(&tx, 4, "request size limit change").await; - } - }, - SAML_METADATA_SETTING => { - tracing::info!("SAML metadata change detected, killing server expecting to be restarted"); - send_delayed_killpill(&tx, 0, "SAML metadata change").await; - }, - HUB_BASE_URL_SETTING => { - if let Err(e) = reload_hub_base_url_setting(&db, server_mode).await { - tracing::error!(error = %e, "Could not reload hub base url setting"); - } - }, - CRITICAL_ERROR_CHANNELS_SETTING => { - if let Err(e) = reload_critical_error_channels_setting(&db).await { - tracing::error!(error = %e, "Could not reload critical error emails setting"); - } - }, - JWT_SECRET_SETTING => { - if let Err(e) = reload_jwt_secret_setting(&db).await { - tracing::error!(error = %e, "Could not reload jwt secret setting"); - } - }, - CRITICAL_ALERT_MUTE_UI_SETTING => { - tracing::info!("Critical alert UI setting changed"); - if let Err(e) = reload_critical_alert_mute_ui_setting(&db).await { - tracing::error!(error = %e, "Could not reload critical alert UI setting"); - } - }, - a @_ => { - tracing::info!("Unrecognized Global Setting Change Payload: {:?}", a); + new_listener = retry_listen_pg(&db_url) => { + listener = new_listener; + continue; } } - }, - _ => { - tracing::warn!("Unknown notification received"); - continue; } + }; + }, + _ = tokio::time::sleep(Duration::from_secs(30)) => { + if last_listener_refresh.elapsed() > Duration::from_secs(*PG_LISTENER_REFRESH_PERIOD_SECS) { + tracing::info!("Refreshing pg listeners, settings and license key after {}s", Duration::from_secs(*PG_LISTENER_REFRESH_PERIOD_SECS).as_secs()); + if let Err(e) = listener.unlisten_all().await { + tracing::error!(error = %e, "Could not unlisten to database"); + } + listener = retry_listen_pg(&db_url).await; + initial_load( + &conn, + tx.clone(), + worker_mode, + server_mode, + #[cfg(feature = "parquet")] + disable_s3_store, + ) + .await; + #[cfg(feature = "enterprise")] + if let Err(err) = reload_license_key(&conn).await { + tracing::error!("Failed to reload license key: {err:#}"); + } + last_listener_refresh = Instant::now(); + } + + if server_mode { + tracing::info!("monitor task started"); + } + monitor_db( + &conn, + &base_internal_url, + server_mode, + worker_mode, + false, + tx.clone(), + ) + .await; + if server_mode { + tracing::info!("monitor task finished"); } }, - Err(e) => { - tracing::error!(error = %e, "Could not receive notification, attempting to reconnect listener"); - tokio::select! { - biased; - _ = monitor_killpill_rx.recv() => { - tracing::info!("received killpill for monitor job"); - break; - }, - new_listener = retry_listen_pg(&db_url) => { - listener = new_listener; - continue; - } - } - } - }; - }, - _ = tokio::time::sleep(Duration::from_secs(30)) => { - if last_listener_refresh.elapsed() > Duration::from_secs(*PG_LISTENER_REFRESH_PERIOD_SECS) { - tracing::info!("Refreshing pg listeners, settings and license key after {}s", Duration::from_secs(*PG_LISTENER_REFRESH_PERIOD_SECS).as_secs()); - if let Err(e) = listener.unlisten_all().await { - tracing::error!(error = %e, "Could not unlisten to database"); - } - listener = retry_listen_pg(&db_url).await; - initial_load( - &db, - tx.clone(), - worker_mode, - server_mode, - #[cfg(feature = "parquet")] - disable_s3_store, - ) - .await; - #[cfg(feature = "enterprise")] - if let Err(err) = reload_license_key(&db).await { - tracing::error!("Failed to reload license key: {err:#}"); - } - last_listener_refresh = Instant::now(); } + } + }); - if server_mode { - tracing::info!("monitor task started"); - } - monitor_db( - &db, - &base_internal_url, - server_mode, - worker_mode, - false, - tx.clone(), - ) - .await; - if server_mode { - tracing::info!("monitor task finished"); - } - }, + if let Err(e) = h.await { + tracing::error!("Error waiting for monitor handle: {e:#}") } } - }); + Connection::Http(_) => loop { + tokio::select! { + _ = monitor_killpill_rx.recv() => { + tracing::info!("Received killpill, exiting"); + break; + }, + _ = tokio::time::sleep(Duration::from_secs(12 * 60 * 60)) => { + tracing::info!("Reloading config after 12 hours"); + initial_load(&conn, tx.clone(), worker_mode, server_mode, #[cfg(feature = "parquet")] disable_s3_store).await; + #[cfg(feature = "enterprise")] + ee::verify_license_key().await; + } + } + }, + }; - if let Err(e) = h.await { - tracing::error!("Error waiting for monitor handle: {e:#}") - } tracing::info!("Monitor exited"); killpill_tx.send(); Ok(()) as anyhow::Result<()> @@ -962,7 +1043,9 @@ Windmill Community Edition {GIT_VERSION} }; if server_mode { - schedule_stats(&db, &HTTP_CLIENT).await; + if let Some(db) = conn.as_sql() { + schedule_stats(&db, &HTTP_CLIENT).await; + } } futures::try_join!( @@ -977,15 +1060,17 @@ Windmill Community Edition {GIT_VERSION} } else { tracing::info!("Nothing to do, exiting."); } - send_current_log_file_to_object_store(&db, &hostname, &mode).await; + send_current_log_file_to_object_store(&conn, &hostname, &mode).await; - tracing::info!("Exiting connection pool"); - tokio::select! { - _ = db.close() => { - tracing::info!("Database connection pool closed"); - }, - _ = tokio::time::sleep(Duration::from_secs(15)) => { - tracing::warn!("Could not close database connection pool in time (15s). Exiting anyway."); + if let Some(db) = conn.as_sql() { + tracing::info!("Exiting connection pool"); + tokio::select! { + _ = db.close() => { + tracing::info!("Database connection pool closed"); + }, + _ = tokio::time::sleep(Duration::from_secs(15)) => { + tracing::warn!("Could not close database connection pool in time (15s). Exiting anyway."); + } } } Ok(()) @@ -1050,13 +1135,13 @@ fn display_config(envs: &[&str]) { } pub async fn run_workers( - db: Pool, + db: Connection, mut rx: tokio::sync::broadcast::Receiver<()>, tx: KillpillSender, num_workers: i32, base_internal_url: String, - agent_mode: bool, hostname: String, + worker_names: &[String], ) -> anyhow::Result<()> { let mut killpill_rxs = vec![]; for _ in 0..num_workers { @@ -1067,14 +1152,6 @@ pub async fn run_workers( tracing::info!("Received killpill, exiting"); return Ok(()); } - let instance_name = hostname - .clone() - .replace(" ", "") - .split("-") - .last() - .unwrap() - .to_ascii_lowercase() - .to_string(); // #[cfg(tokio_unstable)] // let monitor = tokio_metrics::TaskMonitor::new(); @@ -1126,8 +1203,7 @@ pub async fn run_workers( ); for i in 1..(num_workers + 1) { let db1 = db.clone(); - let instance_name = instance_name.clone(); - let worker_name = format!("wk-{}-{}-{}", *WORKER_GROUP, &instance_name, rd_string(5)); + let worker_name = worker_names[i as usize - 1].clone(); let ip = ip.clone(); let rx = killpill_rxs.pop().unwrap(); let tx = tx.clone(); @@ -1149,7 +1225,6 @@ pub async fn run_workers( rx, tx, &base_internal_url, - agent_mode, ); // #[cfg(tokio_unstable)] diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index d9d3f0c6eb..795e4319e2 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -13,7 +13,7 @@ use std::{ use chrono::{NaiveDateTime, Utc}; use futures::{stream::FuturesUnordered, StreamExt}; -use serde::{de::DeserializeOwned, Deserializer}; +use serde::{de::DeserializeOwned, Deserializer}; use sqlx::{Pool, Postgres}; use tokio::{ join, @@ -34,7 +34,7 @@ use windmill_common::ee::{jobs_waiting_alerts, worker_groups_alerts}; #[cfg(feature = "oauth2")] use windmill_common::global_settings::OAUTH_SETTING; use windmill_common::{ - ee::CriticalErrorChannel, error, flow_status::{FlowStatus, FlowStatusModule}, global_settings::{ + agent_workers::DECODED_AGENT_TOKEN, auth::create_token_for_owner, ee::CriticalErrorChannel, error, flow_status::{FlowStatus, FlowStatusModule}, global_settings::{ BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING, CRITICAL_ERROR_CHANNELS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, @@ -45,14 +45,12 @@ use windmill_common::{ REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, TIMEOUT_WAIT_RESULT_SETTING, }, indexer::load_indexer_config, jobs::QueuedJob, jwt::JWT_SECRET, oauth2::REQUIRE_PREEXISTING_USER_FOR_OAUTH, server::load_smtp_config, tracing_init::JSON_FMT, users::truncate_token, utils::{now_from_db, rd_string, report_critical_error, Mode}, worker::{ - load_worker_config, make_pull_query, make_suspended_pull_query, reload_custom_tags_setting, - update_min_version, DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES, INDEXER_CONFIG, - SMTP_CONFIG, TMP_DIR, WORKER_CONFIG, WORKER_GROUP, + load_worker_config, reload_custom_tags_setting, store_pull_query, store_suspended_pull_query, update_min_version, Connection, DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES, INDEXER_CONFIG, SCRIPT_TOKEN_EXPIRY, SMTP_CONFIG, TMP_DIR, WORKER_CONFIG, WORKER_GROUP }, KillpillSender, BASE_URL, CRITICAL_ALERT_MUTE_UI_ENABLED, CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, JOB_RETENTION_SECS, METRICS_DEBUG_ENABLED, METRICS_ENABLED, MONITOR_LOGS_ON_OBJECT_STORE, OTEL_LOGS_ENABLED, OTEL_METRICS_ENABLED, OTEL_TRACING_ENABLED, SERVICE_LOG_RETENTION_SECS }; -use windmill_queue::{cancel_job, MiniPulledJob}; +use windmill_queue::{cancel_job, MiniPulledJob, SameWorkerPayload}; use windmill_worker::{ - create_token_for_owner, handle_job_error, AuthedClient, SameWorkerPayload, SameWorkerSender, SendResult, BUNFIG_INSTALL_SCOPES, INSTANCE_PYTHON_VERSION, JOB_DEFAULT_TIMEOUT, KEEP_JOB_DIR, MAVEN_REPOS, NO_DEFAULT_MAVEN, NPM_CONFIG_REGISTRY, NUGET_CONFIG, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, SCRIPT_TOKEN_EXPIRY + handle_job_error, AuthedClient, JobCompletedSender, SameWorkerSender, BUNFIG_INSTALL_SCOPES, INSTANCE_PYTHON_VERSION, JOB_DEFAULT_TIMEOUT, KEEP_JOB_DIR, MAVEN_REPOS, NO_DEFAULT_MAVEN, NPM_CONFIG_REGISTRY, NUGET_CONFIG, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL }; #[cfg(feature = "parquet")] @@ -113,88 +111,109 @@ lazy_static::lazy_static! { .unwrap_or(false); + static ref QUEUE_COUNT_TAGS: Arc>> = Arc::new(RwLock::new(Vec::new())); static ref DISABLE_CONCURRENCY_LIMIT: bool = std::env::var("DISABLE_CONCURRENCY_LIMIT").is_ok_and(|s| s == "true"); } pub async fn initial_load( - db: &Pool, + conn: &Connection, tx: KillpillSender, worker_mode: bool, server_mode: bool, #[cfg(feature = "parquet")] disable_s3_store: bool, ) { - if let Err(e) = load_metrics_enabled(db).await { + if let Err(e) = load_metrics_enabled(conn).await { tracing::error!("Error loading expose metrics: {e:#}"); } - if let Err(e) = load_metrics_debug_enabled(db).await { + if let Err(e) = load_metrics_debug_enabled(conn).await { tracing::error!("Error loading expose debug metrics: {e:#}"); } - if let Err(e) = reload_critical_alert_mute_ui_setting(db).await { + if let Err(e) = reload_critical_alert_mute_ui_setting(conn).await { tracing::error!("Error loading critical alert mute ui setting: {e:#}"); } - if let Err(e) = load_tag_per_workspace_enabled(db).await { - tracing::error!("Error loading default tag per workpsace: {e:#}"); - } + if let Some(db) = conn.as_sql() { + if let Err(e) = load_tag_per_workspace_enabled(db).await { + tracing::error!("Error loading default tag per workpsace: {e:#}"); + } - if let Err(e) = load_tag_per_workspace_workspaces(db).await { - tracing::error!("Error loading default tag per workpsace workspaces: {e:#}"); + if let Err(e) = load_tag_per_workspace_workspaces(db).await { + tracing::error!("Error loading default tag per workpsace workspaces: {e:#}"); + } } if server_mode { - load_require_preexisting_user(db).await; + if let Some(db) = conn.as_sql() { + load_require_preexisting_user(db).await; + } } if worker_mode { - load_keep_job_dir(db).await; - reload_worker_config(&db, tx, false).await; + load_keep_job_dir(conn).await; + match conn { + Connection::Sql(db) => { + reload_worker_config(&db, tx, false).await; + } + Connection::Http(_) => { + // TODO: reload worker config from http + WORKER_CONFIG.write().await.worker_tags = DECODED_AGENT_TOKEN.as_ref().map(|x| x.tags.clone()).unwrap_or_default(); + } + } } + - if let Err(e) = reload_custom_tags_setting(db).await { - tracing::error!("Error reloading custom tags: {:?}", e) - } - - if let Err(e) = reload_hub_base_url_setting(db, server_mode).await { + if let Err(e) = reload_hub_base_url_setting(conn, server_mode).await { tracing::error!("Error reloading hub base url: {:?}", e) } - if let Err(e) = reload_jwt_secret_setting(&db).await { - tracing::error!("Could not reload jwt secret setting: {:?}", e); + if let Some(db) = conn.as_sql() { + if let Err(e) = reload_jwt_secret_setting(db).await { + tracing::error!("Could not reload jwt secret setting: {:?}", e); + } + + if let Err(e) = reload_custom_tags_setting(db).await { + tracing::error!("Error reloading custom tags: {:?}", e) + } + } #[cfg(feature = "parquet")] if !disable_s3_store { - reload_s3_cache_setting(&db).await; + if let Some(db) = conn.as_sql() { + reload_s3_cache_setting(db).await; + } } - reload_smtp_config(&db).await; + if let Some(db) = conn.as_sql() { + reload_smtp_config(db).await; + } if server_mode { - reload_retention_period_setting(&db).await; - reload_request_size(&db).await; - reload_saml_metadata_setting(&db).await; - reload_scim_token_setting(&db).await; + reload_retention_period_setting(&conn).await; + reload_request_size(&conn).await; + reload_saml_metadata_setting(&conn).await; + reload_scim_token_setting(&conn).await; } if worker_mode { - reload_job_default_timeout_setting(&db).await; - reload_extra_pip_index_url_setting(&db).await; - reload_pip_index_url_setting(&db).await; - reload_npm_config_registry_setting(&db).await; - reload_bunfig_install_scopes_setting(&db).await; - reload_instance_python_version_setting(&db).await; - reload_nuget_config_setting(&db).await; - reload_maven_repos_setting(&db).await; - reload_no_default_maven_setting(&db).await; + reload_job_default_timeout_setting(&conn).await; + reload_extra_pip_index_url_setting(&conn).await; + reload_pip_index_url_setting(&conn).await; + reload_npm_config_registry_setting(&conn).await; + reload_bunfig_install_scopes_setting(&conn).await; + reload_instance_python_version_setting(&conn).await; + reload_nuget_config_setting(&conn).await; + reload_maven_repos_setting(&conn).await; + reload_no_default_maven_setting(&conn).await; } } -pub async fn load_metrics_enabled(db: &DB) -> error::Result<()> { - let metrics_enabled = load_value_from_global_settings(db, EXPOSE_METRICS_SETTING).await; +pub async fn load_metrics_enabled(conn: &Connection) -> error::Result<()> { + let metrics_enabled = load_value_from_global_settings_with_conn(conn, EXPOSE_METRICS_SETTING, true).await; match metrics_enabled { Ok(Some(serde_json::Value::Bool(t))) => METRICS_ENABLED.store(t, Ordering::Relaxed), _ => (), @@ -317,9 +336,9 @@ pub async fn load_tag_per_workspace_workspaces(db: &DB) -> error::Result<()> { Ok(()) } -pub async fn reload_critical_alert_mute_ui_setting(db: &DB) -> error::Result<()> { +pub async fn reload_critical_alert_mute_ui_setting(conn: &Connection) -> error::Result<()> { if let Ok(Some(serde_json::Value::Bool(t))) = - load_value_from_global_settings(db, CRITICAL_ALERT_MUTE_UI_SETTING).await + load_value_from_global_settings_with_conn(conn, CRITICAL_ALERT_MUTE_UI_SETTING, true).await { CRITICAL_ALERT_MUTE_UI_ENABLED.store(t, Ordering::Relaxed); @@ -327,8 +346,8 @@ pub async fn reload_critical_alert_mute_ui_setting(db: &DB) -> error::Result<()> Ok(()) } -pub async fn load_metrics_debug_enabled(db: &DB) -> error::Result<()> { - let metrics_enabled = load_value_from_global_settings(db, EXPOSE_DEBUG_METRICS_SETTING).await; +pub async fn load_metrics_debug_enabled(conn: &Connection) -> error::Result<()> { + let metrics_enabled = load_value_from_global_settings_with_conn(conn, EXPOSE_DEBUG_METRICS_SETTING, true).await; match metrics_enabled { Ok(Some(serde_json::Value::Bool(t))) => { METRICS_DEBUG_ENABLED.store(t, Ordering::Relaxed); @@ -479,8 +498,8 @@ fn get_worker_group(mode: &Mode) -> Option { } } -pub fn send_logs_to_object_store(db: &DB, hostname: &str, mode: &Mode) { - let db = db.clone(); +pub fn send_logs_to_object_store(conn: &Connection, hostname: &str, mode: &Mode) { + let conn = conn.clone(); let hostname = hostname.to_string(); let mode = mode.clone(); let worker_group = get_worker_group(&mode); @@ -495,7 +514,7 @@ pub fn send_logs_to_object_store(db: &DB, hostname: &str, mode: &Mode) { &hostname, &mode, &worker_group, - &db, + &conn, snd_highest_file, false, ) @@ -504,11 +523,11 @@ pub fn send_logs_to_object_store(db: &DB, hostname: &str, mode: &Mode) { }); } -pub async fn send_current_log_file_to_object_store(db: &DB, hostname: &str, mode: &Mode) { +pub async fn send_current_log_file_to_object_store(conn: &Connection, hostname: &str, mode: &Mode) { tracing::info!("Sending current log file to object store"); let (highest_file, _) = find_two_highest_files(hostname).await; let worker_group = get_worker_group(&mode); - send_log_file_to_object_store(hostname, mode, &worker_group, db, highest_file, true).await; + send_log_file_to_object_store(hostname, mode, &worker_group, conn, highest_file, true).await; } fn get_now_and_str() -> (NaiveDateTime, String) { @@ -528,7 +547,7 @@ async fn send_log_file_to_object_store( hostname: &str, mode: &Mode, worker_group: &Option, - db: &Pool, + conn: &Connection, snd_highest_file: Option, use_now: bool, ) { @@ -588,18 +607,23 @@ async fn send_log_file_to_object_store( let (ok_lines, err_lines) = read_log_counters(ts_str); - if let Err(e) = sqlx::query!("INSERT INTO log_file (hostname, mode, worker_group, log_ts, file_path, ok_lines, err_lines, json_fmt) VALUES ($1, $2::text::LOG_MODE, $3, $4, $5, $6, $7, $8)", - hostname, mode.to_string(), worker_group.clone(), ts, highest_file, ok_lines as i64, err_lines as i64, *JSON_FMT) - .execute(db) - .await { - tracing::error!("Error inserting log file: {:?}", e); - } else { - if let Err(e) = LAST_LOG_FILE_SENT.lock().map(|mut last_log_file_sent| { - last_log_file_sent.replace(ts); - }) { - tracing::error!("Error updating last log file sent: {:?}", e); + if let Some(db) = conn.as_sql() { + if let Err(e) = sqlx::query!("INSERT INTO log_file (hostname, mode, worker_group, log_ts, file_path, ok_lines, err_lines, json_fmt) VALUES ($1, $2::text::LOG_MODE, $3, $4, $5, $6, $7, $8)", + hostname, mode.to_string(), worker_group.clone(), ts, highest_file, ok_lines as i64, err_lines as i64, *JSON_FMT) + .execute(db) + .await { + tracing::error!("Error inserting log file: {:?}", e); + } else { + if let Err(e) = LAST_LOG_FILE_SENT.lock().map(|mut last_log_file_sent| { + last_log_file_sent.replace(ts); + }) { + tracing::error!("Error updating last log file sent: {:?}", e); + } + tracing::info!("Log file sent: {}", highest_file); } - tracing::info!("Log file sent: {}", highest_file); + } else { + // tracing::warn!("Not sending log file to object store in agent mode"); + () } } } @@ -622,8 +646,8 @@ fn read_log_counters(ts_str: String) -> (usize, usize) { (ok_lines, err_lines) } -pub async fn load_keep_job_dir(db: &DB) { - let value = load_value_from_global_settings(db, KEEP_JOB_DIR_SETTING).await; +pub async fn load_keep_job_dir(conn: &Connection) { + let value = load_value_from_global_settings_with_conn(conn, KEEP_JOB_DIR_SETTING, true).await; match value { Ok(Some(serde_json::Value::Bool(t))) => KEEP_JOB_DIR.store(t, Ordering::Relaxed), Err(e) => { @@ -890,23 +914,23 @@ async fn delete_log_files_from_disk_and_store( let _: Vec<_> = delete_futures.collect().await; } -pub async fn reload_scim_token_setting(db: &DB) { - reload_option_setting_with_tracing(db, SCIM_TOKEN_SETTING, "SCIM_TOKEN", SCIM_TOKEN.clone()) +pub async fn reload_scim_token_setting(conn: &Connection) { + reload_option_setting_with_tracing(conn, SCIM_TOKEN_SETTING, "SCIM_TOKEN", SCIM_TOKEN.clone()) .await; } -pub async fn reload_timeout_wait_result_setting(db: &DB) { +pub async fn reload_timeout_wait_result_setting(conn: &Connection) { reload_option_setting_with_tracing( - db, + conn, TIMEOUT_WAIT_RESULT_SETTING, "TIMEOUT_WAIT_RESULT", TIMEOUT_WAIT_RESULT.clone(), ) .await; } -pub async fn reload_saml_metadata_setting(db: &DB) { +pub async fn reload_saml_metadata_setting(conn: &Connection) { reload_option_setting_with_tracing( - db, + conn, SAML_METADATA_SETTING, "SAML_METADATA", SAML_METADATA.clone(), @@ -914,9 +938,9 @@ pub async fn reload_saml_metadata_setting(db: &DB) { .await; } -pub async fn reload_extra_pip_index_url_setting(db: &DB) { +pub async fn reload_extra_pip_index_url_setting(conn: &Connection) { reload_option_setting_with_tracing( - db, + conn, EXTRA_PIP_INDEX_URL_SETTING, "PIP_EXTRA_INDEX_URL", PIP_EXTRA_INDEX_URL.clone(), @@ -924,9 +948,9 @@ pub async fn reload_extra_pip_index_url_setting(db: &DB) { .await; } -pub async fn reload_pip_index_url_setting(db: &DB) { +pub async fn reload_pip_index_url_setting(conn: &Connection) { reload_option_setting_with_tracing( - db, + conn, PIP_INDEX_URL_SETTING, "PIP_INDEX_URL", PIP_INDEX_URL.clone(), @@ -934,9 +958,9 @@ pub async fn reload_pip_index_url_setting(db: &DB) { .await; } -pub async fn reload_instance_python_version_setting(db: &DB) { +pub async fn reload_instance_python_version_setting(conn: &Connection) { reload_option_setting_with_tracing( - db, + conn, INSTANCE_PYTHON_VERSION_SETTING, "INSTANCE_PYTHON_VERSION", INSTANCE_PYTHON_VERSION.clone(), @@ -944,9 +968,9 @@ pub async fn reload_instance_python_version_setting(db: &DB) { .await; } -pub async fn reload_npm_config_registry_setting(db: &DB) { +pub async fn reload_npm_config_registry_setting(conn: &Connection) { reload_option_setting_with_tracing( - db, + conn, NPM_CONFIG_REGISTRY_SETTING, "NPM_CONFIG_REGISTRY", NPM_CONFIG_REGISTRY.clone(), @@ -954,9 +978,9 @@ pub async fn reload_npm_config_registry_setting(db: &DB) { .await; } -pub async fn reload_bunfig_install_scopes_setting(db: &DB) { +pub async fn reload_bunfig_install_scopes_setting(conn: &Connection) { reload_option_setting_with_tracing( - db, + conn, BUNFIG_INSTALL_SCOPES_SETTING, "BUNFIG_INSTALL_SCOPES", BUNFIG_INSTALL_SCOPES.clone(), @@ -964,21 +988,21 @@ pub async fn reload_bunfig_install_scopes_setting(db: &DB) { .await; } -pub async fn reload_nuget_config_setting(db: &DB) { +pub async fn reload_nuget_config_setting(conn: &Connection) { reload_option_setting_with_tracing( - db, + conn, NUGET_CONFIG_SETTING, "NUGET_CONFIG", NUGET_CONFIG.clone(), ) .await; } -pub async fn reload_maven_repos_setting(db: &DB) { - reload_option_setting_with_tracing(db, windmill_common::global_settings::MAVEN_REPOS_SETTING, "MAVEN_REPOS", MAVEN_REPOS.clone()) +pub async fn reload_maven_repos_setting(conn: &Connection) { + reload_option_setting_with_tracing(conn, windmill_common::global_settings::MAVEN_REPOS_SETTING, "MAVEN_REPOS", MAVEN_REPOS.clone()) .await; } -pub async fn reload_no_default_maven_setting(db: &DB) { - let value = load_value_from_global_settings(db, windmill_common::global_settings::NO_DEFAULT_MAVEN_SETTING).await; +pub async fn reload_no_default_maven_setting(conn: &Connection) { + let value = load_value_from_global_settings_with_conn(conn, windmill_common::global_settings::NO_DEFAULT_MAVEN_SETTING, true).await; match value { Ok(Some(serde_json::Value::Bool(t))) => NO_DEFAULT_MAVEN.store(t, Ordering::Relaxed), Err(e) => { @@ -988,9 +1012,9 @@ pub async fn reload_no_default_maven_setting(db: &DB) { }; } -pub async fn reload_retention_period_setting(db: &DB) { +pub async fn reload_retention_period_setting(conn: &Connection) { if let Err(e) = reload_setting( - db, + conn, RETENTION_PERIOD_SECS_SETTING, "JOB_RETENTION_SECS", 60 * 60 * 24 * 30, @@ -1002,9 +1026,9 @@ pub async fn reload_retention_period_setting(db: &DB) { tracing::error!("Error reloading retention period: {:?}", e) } } -pub async fn reload_delete_logs_periodically_setting(db: &DB) { +pub async fn reload_delete_logs_periodically_setting(conn: &Connection) { if let Err(e) = reload_setting( - db, + conn, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, "MONITOR_LOGS_ON_OBJECT_STORE", false, @@ -1072,9 +1096,9 @@ pub async fn reload_s3_cache_setting(db: &DB) { } } -pub async fn reload_job_default_timeout_setting(db: &DB) { +pub async fn reload_job_default_timeout_setting(conn: &Connection) { reload_option_setting_with_tracing( - db, + conn, JOB_DEFAULT_TIMEOUT_SECS_SETTING, "JOB_DEFAULT_TIMEOUT_SECS", JOB_DEFAULT_TIMEOUT.clone(), @@ -1082,9 +1106,9 @@ pub async fn reload_job_default_timeout_setting(db: &DB) { .await; } -pub async fn reload_request_size(db: &DB) { +pub async fn reload_request_size(conn: &Connection) { if let Err(e) = reload_setting( - db, + conn, REQUEST_SIZE_LIMIT_SETTING, "REQUEST_SIZE_LIMIT", DEFAULT_BODY_LIMIT, @@ -1097,8 +1121,8 @@ pub async fn reload_request_size(db: &DB) { } } -pub async fn reload_license_key(db: &DB) -> anyhow::Result<()> { - let q = load_value_from_global_settings(db, LICENSE_KEY_SETTING) +pub async fn reload_license_key(conn: &Connection) -> anyhow::Result<()> { + let q = load_value_from_global_settings_with_conn(conn, LICENSE_KEY_SETTING, true) .await .map_err(|err| anyhow::anyhow!("Error reloading license key: {}", err.to_string()))?; @@ -1123,12 +1147,12 @@ pub async fn reload_license_key(db: &DB) -> anyhow::Result<()> { } pub async fn reload_option_setting_with_tracing( - db: &DB, + conn: &Connection, setting_name: &str, std_env_var: &str, lock: Arc>>, ) { - if let Err(e) = reload_option_setting(db, setting_name, std_env_var, lock.clone()).await { + if let Err(e) = reload_option_setting(conn, setting_name, std_env_var, lock.clone()).await { tracing::error!("Error reloading setting {}: {:?}", setting_name, e) } } @@ -1147,8 +1171,28 @@ pub async fn load_value_from_global_settings( Ok(r) } + +pub async fn load_value_from_global_settings_with_conn( + conn: &Connection, + setting_name: &str, + load_from_http: bool, +) -> anyhow::Result> { + match conn { + Connection::Sql(db) => Ok(load_value_from_global_settings(db, setting_name).await?), + Connection::Http(client) => { + if load_from_http { + client.get::>(&format!("/api/agent_workers/get_global_setting/{}", setting_name)).await + .map_err(|e| anyhow::anyhow!("Error loading setting {}: {}", setting_name, e)) + } else { + Ok(None) + } + } + } + +} + pub async fn reload_option_setting( - db: &DB, + conn: &Connection, setting_name: &str, std_env_var: &str, lock: Arc>>, @@ -1163,7 +1207,7 @@ pub async fn reload_option_setting( return Ok(()); } - let q = load_value_from_global_settings(db, setting_name).await?; + let q = load_value_from_global_settings_with_conn(conn, setting_name, true).await?; let mut value = std::env::var(std_env_var) .ok() @@ -1190,14 +1234,14 @@ pub async fn reload_option_setting( } pub async fn reload_setting( - db: &DB, + conn: &Connection, setting_name: &str, std_env_var: &str, default: T, lock: Arc>, transformer: fn(T) -> T, ) -> error::Result<()> { - let q = load_value_from_global_settings(db, setting_name).await?; + let q = load_value_from_global_settings_with_conn(conn, setting_name, true).await?; let mut value = std::env::var(std_env_var) .ok() @@ -1255,27 +1299,32 @@ pub async fn monitor_pool(db: &DB) { } pub async fn monitor_db( - db: &Pool, + conn: &Connection, base_internal_url: &str, server_mode: bool, _worker_mode: bool, initial_load: bool, _killpill_tx: KillpillSender, ) { + tracing::info!("Starting periodic monitor task"); let zombie_jobs_f = async { if server_mode && !initial_load && !*DISABLE_ZOMBIE_JOBS_MONITORING { + if let Some(db) = conn.as_sql() { handle_zombie_jobs(db, base_internal_url, "server").await; match handle_zombie_flows(db).await { Err(err) => { tracing::error!("Error handling zombie flows: {:?}", err); - } + }, _ => {} + } } } }; let expired_items_f = async { if server_mode && !initial_load { + if let Some(db) = conn.as_sql() { delete_expired_items(&db).await; + } } }; @@ -1288,35 +1337,43 @@ pub async fn monitor_db( let expose_queue_metrics_f = async { if !initial_load && server_mode { - expose_queue_metrics(&db).await; + if let Some(db) = conn.as_sql() { + expose_queue_metrics(&db).await; + } } }; let worker_groups_alerts_f = async { #[cfg(feature = "enterprise")] if server_mode && !initial_load { - worker_groups_alerts(&db).await; + if let Some(db) = conn.as_sql() { + worker_groups_alerts(&db).await; + } } }; let jobs_waiting_alerts_f = async { #[cfg(feature = "enterprise")] if server_mode { - jobs_waiting_alerts(&db).await; + if let Some(db) = conn.as_sql() { + jobs_waiting_alerts(&db).await; + } } }; let apply_autoscaling_f = async { #[cfg(feature = "enterprise")] if server_mode && !initial_load { - if let Err(e) = windmill_autoscaling::apply_all_autoscaling(db).await { - tracing::error!("Error applying autoscaling: {:?}", e); + if let Some(db) = conn.as_sql() { + if let Err(e) = windmill_autoscaling::apply_all_autoscaling(db).await { + tracing::error!("Error applying autoscaling: {:?}", e); + } } } }; let update_min_worker_version_f = async { - update_min_version(db).await; + update_min_version(conn).await; }; join!( @@ -1329,6 +1386,7 @@ pub async fn monitor_db( apply_autoscaling_f, update_min_worker_version_f, ); + tracing::info!("Periodic monitor task completed"); } pub async fn expose_queue_metrics(db: &Pool) { @@ -1439,7 +1497,7 @@ pub async fn reload_worker_config( tx: KillpillSender, kill_if_change: bool, ) { - let config = load_worker_config(&db, tx.clone()).await; + let config = load_worker_config(db, tx.clone()).await; if let Err(e) = config { tracing::error!("Error reloading worker config: {:?}", e) } else { @@ -1473,15 +1531,15 @@ pub async fn reload_worker_config( let mut wc = WORKER_CONFIG.write().await; tracing::info!("Reloading worker config..."); - make_suspended_pull_query(&config).await; - make_pull_query(&config).await; + store_suspended_pull_query(&config).await; + store_pull_query(&config).await; *wc = config } } } -pub async fn load_base_url(db: &DB) -> error::Result { - let q_base_url = load_value_from_global_settings(db, BASE_URL_SETTING).await?; +pub async fn load_base_url(conn: &Connection) -> error::Result { + let q_base_url = load_value_from_global_settings_with_conn(conn, BASE_URL_SETTING, false).await?; let std_base_url = std::env::var("BASE_URL") .ok() @@ -1511,34 +1569,39 @@ pub async fn load_base_url(db: &DB) -> error::Result { Ok(base_url) } -pub async fn reload_base_url_setting(db: &DB) -> error::Result<()> { - #[cfg(feature = "oauth2")] - let q_oauth = load_value_from_global_settings(db, OAUTH_SETTING).await?; +pub async fn reload_base_url_setting(conn: &Connection) -> error::Result<()> { #[cfg(feature = "oauth2")] - let oauths = if let Some(q) = q_oauth { - if let Ok(v) = serde_json::from_value::< - Option>, - >(q.clone()) - { - v + let oauths = if let Some(db) = conn.as_sql() { + let q_oauth = load_value_from_global_settings (db, OAUTH_SETTING).await?; + + if let Some(q) = q_oauth { + if let Ok(v) = serde_json::from_value::< + Option>, + >(q.clone()) + { + v + } else { + tracing::error!("Could not parse oauth setting as a json, found: {:#?}", &q); + None + } } else { - tracing::error!("Could not parse oauth setting as a json, found: {:#?}", &q); None } } else { None }; - - let base_url = load_base_url(db).await?; + let base_url = load_base_url(conn).await?; let is_secure = base_url.starts_with("https://"); #[cfg(feature = "oauth2")] { - let mut l = windmill_api::OAUTH_CLIENTS.write().await; - *l = windmill_api::oauth2_ee::build_oauth_clients(&base_url, oauths, db).await - .map_err(|e| tracing::error!("Error building oauth clients (is the oauth.json mounted and in correct format? Use '{}' as minimal oauth.json): {}", "{}", e)) - .unwrap(); + if let Some(db) = conn.as_sql() { + let mut l = windmill_api::OAUTH_CLIENTS.write().await; + *l = windmill_api::oauth2_ee::build_oauth_clients(&base_url, oauths, db).await + .map_err(|e| tracing::error!("Error building oauth clients (is the oauth.json mounted and in correct format? Use '{}' as minimal oauth.json): {}", "{}", e)) + .unwrap(); + } } { @@ -1796,7 +1859,7 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, worker mpsc::channel::(1); let same_worker_tx_never_used = SameWorkerSender(same_worker_tx_never_used, Arc::new(AtomicU16::new(0))); - let (send_result_never_used, _send_result_rx_never_used) = mpsc::channel::(1); + let (send_result_never_used, _send_result_rx_never_used) = JobCompletedSender::new_never_used(); let label = if job.permissioned_as != format!("u/{}", job.created_by) && job.permissioned_as != job.created_by @@ -2006,8 +2069,8 @@ async fn cancel_zombie_flow_job( Ok(()) } -pub async fn reload_hub_base_url_setting(db: &DB, server_mode: bool) -> error::Result<()> { - let hub_base_url = load_value_from_global_settings(db, HUB_BASE_URL_SETTING).await?; +pub async fn reload_hub_base_url_setting(conn: &Connection, server_mode: bool) -> error::Result<()> { + let hub_base_url = load_value_from_global_settings_with_conn(conn, HUB_BASE_URL_SETTING, true).await?; let base_url = if let Some(q) = hub_base_url { if let Ok(v) = serde_json::from_value::(q.clone()) { @@ -2030,16 +2093,18 @@ pub async fn reload_hub_base_url_setting(db: &DB, server_mode: bool) -> error::R let mut l = HUB_BASE_URL.write().await; if server_mode { #[cfg(feature = "embedding")] - if *l != base_url { - let disable_embedding = std::env::var("DISABLE_EMBEDDING") - .ok() - .map(|x| x.parse::().unwrap_or(false)) - .unwrap_or(false); - if !disable_embedding { - let db_clone = db.clone(); - tokio::spawn(async move { - update_embeddings_db(&db_clone).await; - }); + if let Some(db) = conn.as_sql() { + if *l != base_url { + let disable_embedding = std::env::var("DISABLE_EMBEDDING") + .ok() + .map(|x| x.parse::().unwrap_or(false)) + .unwrap_or(false); + if !disable_embedding { + let db_clone = db.clone(); + tokio::spawn(async move { + update_embeddings_db(&db_clone).await; + }); + } } } } @@ -2048,9 +2113,9 @@ pub async fn reload_hub_base_url_setting(db: &DB, server_mode: bool) -> error::R Ok(()) } -pub async fn reload_critical_error_channels_setting(db: &DB) -> error::Result<()> { +pub async fn reload_critical_error_channels_setting(conn: &DB) -> error::Result<()> { let critical_error_channels = - load_value_from_global_settings(db, CRITICAL_ERROR_CHANNELS_SETTING).await?; + load_value_from_global_settings(conn, CRITICAL_ERROR_CHANNELS_SETTING).await?; let critical_error_channels = if let Some(q) = critical_error_channels { if let Ok(v) = serde_json::from_value::>(q.clone()) { diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index f942a8ee6e..c93256a450 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -137,7 +137,6 @@ impl ApiServer { rx, port_tx, false, - #[cfg(feature = "smtp")] format!("http://localhost:{}", addr.port()), )); @@ -315,7 +314,7 @@ mod suspend_resume { let second = completed.next().await.unwrap(); // print_job(second, &db).await; - let token = windmill_worker::create_token_for_owner(&db, "test-workspace", "u/test-user", "", 100, "", &Uuid::nil(), None).await.unwrap(); + let token = windmill_common::auth::create_token_for_owner(&db, "test-workspace", "u/test-user", "", 100, "", &Uuid::nil(), None).await.unwrap(); let secret = reqwest::get(format!( "http://localhost:{port}/api/w/test-workspace/jobs/job_signature/{second}/0?token={token}&approver=ruben" )) @@ -418,7 +417,7 @@ mod suspend_resume { /* ... and send a request resume it. */ let second = completed.next().await.unwrap(); - let token = windmill_worker::create_token_for_owner(&db, "test-workspace", "u/test-user", "", 100, "", &Uuid::nil(), None).await.unwrap(); + let token = windmill_common::auth::create_token_for_owner(&db, "test-workspace", "u/test-user", "", 100, "", &Uuid::nil(), None).await.unwrap(); let secret = reqwest::get(format!( "http://localhost:{port}/api/w/test-workspace/jobs/job_signature/{second}/0?token={token}" )) @@ -1033,11 +1032,11 @@ fn spawn_test_worker( priority: 0, tags: (*wc).worker_tags.clone(), }]; - windmill_common::worker::make_suspended_pull_query(&wc).await; - windmill_common::worker::make_pull_query(&wc).await; + windmill_common::worker::store_suspended_pull_query(&wc).await; + windmill_common::worker::store_pull_query(&wc).await; } windmill_worker::run_worker( - &db, + &db.into(), worker_instance, worker_name, 1, @@ -1046,7 +1045,6 @@ fn spawn_test_worker( rx, tx2, &base_internal_url, - false, ) .await }; @@ -3843,7 +3841,7 @@ async fn test_result_format(db: Pool) { let port = server.addr.port(); - let token = windmill_worker::create_token_for_owner( + let token = windmill_common::auth::create_token_for_owner( &db, "test-workspace", "u/test-user", @@ -3884,7 +3882,7 @@ async fn test_result_format(db: Pool) { assert_eq!(job_result.get(), correct_result); let response = windmill_api::jobs::run_wait_result( - &db, + &db.into(), Uuid::parse_str(ordered_result_job_id).unwrap(), "test-workspace".to_string(), None, diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index 05025e7f8e..e762e82429 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -10,13 +10,14 @@ path = "src/lib.rs" [features] default = [] -enterprise = ["windmill-queue/enterprise", "windmill-audit/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise"] +enterprise = ["windmill-queue/enterprise", "windmill-audit/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-worker/enterprise"] stripe = [] +agent_worker_server = [] enterprise_saml = ["dep:samael"] benchmark = [] embedding = ["dep:tinyvector", "dep:hf-hub", "dep:tokenizers", "dep:candle-core", "dep:candle-transformers", "dep:candle-nn", "dep:half"] -parquet = ["dep:datafusion", "dep:object_store", "dep:url", "windmill-common/parquet"] -prometheus = ["windmill-common/prometheus", "windmill-queue/prometheus", "dep:prometheus"] +parquet = ["dep:datafusion", "dep:object_store", "dep:url", "windmill-common/parquet", "windmill-worker/parquet"] +prometheus = ["windmill-common/prometheus", "windmill-queue/prometheus", "dep:prometheus", "windmill-worker/prometheus"] openidconnect = ["dep:openidconnect"] tantivy = ["dep:windmill-indexer"] kafka = ["dep:rdkafka"] @@ -40,8 +41,10 @@ windmill-audit.workspace = true windmill-parser.workspace = true windmill-parser-ts.workspace = true windmill-parser-py.workspace = true +windmill-parser-py-imports.workspace = true windmill-git-sync.workspace = true windmill-indexer = { workspace = true, optional = true } +windmill-worker.workspace = true tokio.workspace = true anyhow.workspace = true argon2.workspace = true diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index bef33a9b66..c7d905e15b 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -10498,6 +10498,41 @@ paths: items: $ref: "#/components/schemas/AutoscalingEvent" + /agent_workers/create_agent_token: + post: + summary: create agent token + operationId: createAgentToken + tags: + - agent_workers + requestBody: + description: agent token + required: true + content: + application/json: + schema: + type: object + properties: + worker_group: + type: string + tags: + type: array + items: + type: string + exp: + type: integer + required: + - worker_group + - tags + - exp + responses: + "200": + description: agent token created + content: + application/json: + schema: + type: string + + /w/{workspace}/acls/get/{kind}/{path}: get: summary: get granular acls diff --git a/backend/windmill-api/src/agent_workers_ee.rs b/backend/windmill-api/src/agent_workers_ee.rs new file mode 100644 index 0000000000..af31e61229 --- /dev/null +++ b/backend/windmill-api/src/agent_workers_ee.rs @@ -0,0 +1,52 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2042 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use crate::db::DB; + +use axum::Router; + +use serde::{Deserialize, Serialize}; + +pub fn global_service() -> Router { + Router::new() +} + +pub fn workspaced_service( + db: DB, + _base_internal_url: String, +) -> ( + Router, + Option>, + windmill_worker::JobCompletedSender, +) { + use windmill_common::worker::Connection; + use windmill_worker::JobCompletedSender; + + let (job_completed_tx, _job_completed_rx) = + JobCompletedSender::new(&Connection::Sql(db.clone()), 100); + + let router = Router::new(); + + (router, None, job_completed_tx) +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct AgentAuth { + pub worker_group: String, + pub suffix: Option, + pub tags: Vec, + pub exp: Option, +} + +pub struct AgentCache {} + +impl AgentCache { + pub fn new() -> Self { + AgentCache {} + } +} diff --git a/backend/windmill-api/src/drafts.rs b/backend/windmill-api/src/drafts.rs index 6ba1f4452f..02d3b3c4fb 100644 --- a/backend/windmill-api/src/drafts.rs +++ b/backend/windmill-api/src/drafts.rs @@ -1,6 +1,6 @@ /* * Author: Ruben Fiszel - * Copyright: Windmill Labs, Inc 2042 + * Copyright: Windmill Labs, Inc 2024 * This file and its contents are licensed under the AGPLv3 License. * Please see the included NOTICE for copyright information and * LICENSE-AGPL for a copy of the license. diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 9d35f7bc0f..71ec9c6c8f 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -26,7 +26,7 @@ use tower::ServiceBuilder; use windmill_common::error::JsonResult; use windmill_common::flow_status::{JobResult, RestartedFrom}; use windmill_common::jobs::{format_completed_job_result, format_result, ENTRYPOINT_OVERRIDE}; -use windmill_common::worker::{CLOUD_HOSTED, TMP_DIR}; +use windmill_common::worker::{Connection, CLOUD_HOSTED, TMP_DIR}; use windmill_common::scripts::PREVIEW_IS_CODEBASE_HASH; use windmill_common::variables::get_workspace_key; @@ -372,7 +372,6 @@ async fn cancel_job_api( email: "anonymous".to_string(), }, }; - let (mut tx, job_option) = tokio::time::timeout( std::time::Duration::from_secs(120), windmill_queue::cancel_job( @@ -587,7 +586,9 @@ async fn get_flow_job_debug_info( Extension(db): Extension, Path((w_id, id)): Path<(String, Uuid)>, ) -> error::Result { - let job = GetQuery::new().fetch_queued(&db, id, &w_id).await?; + let job = GetQuery::new() + .fetch_queued((&db).into(), id, &w_id) + .await?; if let Some(job) = job { let is_flow = job.is_flow(); if job.is_flow_step || !is_flow { @@ -926,8 +927,9 @@ impl<'a> GetQuery<'a> { // Try to fetch the code from the cache, fallback to the preview code. // NOTE: This could check for the job kinds instead of the `or_else` but it's not // necessary as `fetch_script` return early if the job kind is not a preview one. - cache::job::fetch_script(db, kind, hash) - .or_else(|_| cache::job::fetch_preview_script(db, &id, raw_lock, raw_code)) + let conn = Connection::from(db.clone()); + cache::job::fetch_script(db.clone(), kind, hash) + .or_else(|_| cache::job::fetch_preview_script(&conn, &id, raw_lock, raw_code)) .await .ok() .inspect(|data| { @@ -956,7 +958,7 @@ impl<'a> GetQuery<'a> { self.check_auth(job.as_ref().map(|job| job.created_by.as_str()))?; if let Some(job) = job.as_mut() { - self.resolve_raw_values(db, job.id, job.job_kind, job.script_hash, job) + self.resolve_raw_values(&db, job.id, job.job_kind, job.script_hash, job) .await; } if self.with_flow { @@ -993,12 +995,14 @@ impl<'a> GetQuery<'a> { self.resolve_raw_values(db, job.id, job.job_kind, job.script_hash, job) .await; } + if self.with_flow { cjob = resolve_maybe_value(db, workspace_id, self.with_code, cjob, |job| { job.raw_flow.as_mut() }) .await?; } + if let Some(mut cjob) = cjob { cjob.inner = format_completed_job_result(cjob.inner); return Ok(Some(cjob)); @@ -1008,7 +1012,7 @@ impl<'a> GetQuery<'a> { async fn fetch(self, db: &DB, job_id: Uuid, workspace_id: &str) -> error::Result { let cjob = self - .fetch_completed(db, job_id, workspace_id) + .fetch_completed(db.into(), job_id, workspace_id) .await? .map(Job::CompletedJob); @@ -1016,7 +1020,7 @@ impl<'a> GetQuery<'a> { Some(cjob) => Ok(cjob), None => { let job_maybe = self - .fetch_queued(db, job_id, workspace_id) + .fetch_queued(db.into(), job_id, workspace_id) .await? .map(Job::QueuedJob); // potential race condition here, if the job was in queue and completed right after the fetch completed, so we need to check one last time @@ -1024,7 +1028,7 @@ impl<'a> GetQuery<'a> { return Ok(job); } else { let cjob2 = self - .fetch_completed(db, job_id, workspace_id) + .fetch_completed(db.into(), job_id, workspace_id) .await? .map(Job::CompletedJob); not_found_if_none(cjob2, "Job", job_id.to_string()) diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index 589e9fecdb..f5e962aae3 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -25,6 +25,8 @@ use crate::{ webhook_util::WebhookShared, }; +#[cfg(feature = "agent_worker_server")] +use agent_workers_ee::AgentCache; use anyhow::Context; use argon2::Argon2; use axum::extract::DefaultBodyLimit; @@ -56,6 +58,8 @@ use windmill_common::{utils::GIT_VERSION, BASE_URL, INSTANCE_NAME}; use crate::scim_ee::has_scim_token; use windmill_common::error::AppError; +#[cfg(feature = "agent_worker_server")] +mod agent_workers_ee; mod ai; mod apps; pub mod args; @@ -74,9 +78,9 @@ mod folders; mod granular_acls; mod groups; #[cfg(feature = "http_trigger")] -mod http_triggers; -#[cfg(feature = "http_trigger")] mod http_trigger_auth; +#[cfg(feature = "http_trigger")] +mod http_triggers; mod indexer_ee; mod inputs; mod integration; @@ -209,10 +213,10 @@ pub async fn run_server( job_index_reader: Option, log_index_reader: Option, addr: SocketAddr, - mut rx: tokio::sync::broadcast::Receiver<()>, + mut killpill_rx: tokio::sync::broadcast::Receiver<()>, port_tx: tokio::sync::oneshot::Sender, server_mode: bool, - #[cfg(feature = "smtp")] base_internal_url: String, + _base_internal_url: String, ) -> anyhow::Result<()> { let user_db = UserDB::new(db.clone()); @@ -246,7 +250,10 @@ pub async fn run_server( .layer(Extension(log_index_reader)) // .layer(Extension(index_writer)) .layer(CookieManagerLayer::new()) - .layer(Extension(WebhookShared::new(rx.resubscribe(), db.clone()))) + .layer(Extension(WebhookShared::new( + killpill_rx.resubscribe(), + db.clone(), + ))) .layer(DefaultBodyLimit::max( REQUEST_SIZE_LIMIT.read().await.clone(), )); @@ -279,7 +286,7 @@ pub async fn run_server( db: db.clone(), user_db: user_db, auth_cache: auth_cache.clone(), - base_internal_url: base_internal_url.clone(), + base_internal_url: _base_internal_url.clone(), }); if let Err(err) = smtp_server.start_listener_thread(addr).await { tracing::error!("Error starting SMTP server: {err:#}"); @@ -385,41 +392,48 @@ pub async fn run_server( if !*CLOUD_HOSTED && server_mode { #[cfg(feature = "websocket")] { - let ws_killpill_rx = rx.resubscribe(); + let ws_killpill_rx = killpill_rx.resubscribe(); websocket_triggers::start_websockets(db.clone(), ws_killpill_rx); } #[cfg(all(feature = "enterprise", feature = "kafka"))] { - let kafka_killpill_rx = rx.resubscribe(); + let kafka_killpill_rx = killpill_rx.resubscribe(); kafka_triggers_ee::start_kafka_consumers(db.clone(), kafka_killpill_rx); } #[cfg(all(feature = "enterprise", feature = "nats"))] { - let nats_killpill_rx = rx.resubscribe(); + let nats_killpill_rx = killpill_rx.resubscribe(); nats_triggers_ee::start_nats_consumers(db.clone(), nats_killpill_rx); } #[cfg(feature = "postgres_trigger")] { - let db_killpill_rx = rx.resubscribe(); + let db_killpill_rx = killpill_rx.resubscribe(); postgres_triggers::start_database(db.clone(), db_killpill_rx); } #[cfg(feature = "mqtt_trigger")] { - let mqtt_killpill_rx = rx.resubscribe(); + let mqtt_killpill_rx = killpill_rx.resubscribe(); mqtt_triggers::start_mqtt_consumer(db.clone(), mqtt_killpill_rx); } #[cfg(all(feature = "enterprise", feature = "sqs_trigger"))] { - let sqs_killpill_rx = rx.resubscribe(); + let sqs_killpill_rx = killpill_rx.resubscribe(); sqs_triggers_ee::start_sqs(db.clone(), sqs_killpill_rx); } } + #[cfg(feature = "agent_worker_server")] + let (agent_workers_router, agent_workers_bg_processor, agent_workers_killpill_tx) = + agent_workers_ee::workspaced_service(db.clone(), _base_internal_url.clone()); + + #[cfg(feature = "agent_worker_server")] + let agent_cache = Arc::new(AgentCache::new()); + // build our application with a route let app = Router::new() .nest( @@ -495,6 +509,26 @@ pub async fn run_server( .nest("/ai", ai::global_service()) .route_layer(from_extractor::()) .route_layer(from_extractor::()) + .nest("/agent_workers", { + #[cfg(feature = "agent_worker_server")] + { + agent_workers_ee::global_service().layer(Extension(agent_cache.clone())) + } + #[cfg(not(feature = "agent_worker_server"))] + { + Router::new() + } + }) + .nest("/w/:workspace_id/agent_workers", { + #[cfg(feature = "agent_worker_server")] + { + agent_workers_router.layer(Extension(agent_cache.clone())) + } + #[cfg(not(feature = "agent_worker_server"))] + { + Router::new() + } + }) .nest("/jobs", jobs::global_root_service()) .nest( "/srch/w/:workspace_id/index", @@ -647,11 +681,22 @@ pub async fn run_server( .expect("Failed to send port"); let server = server.with_graceful_shutdown(async move { - rx.recv().await.ok(); + killpill_rx.recv().await.ok(); + #[cfg(feature = "agent_worker_server")] + if let Err(e) = agent_workers_killpill_tx.kill().await { + tracing::error!("Error killing agent workers: {e:#}"); + } tracing::info!("Graceful shutdown of server"); }); server.await?; + + #[cfg(feature = "agent_worker_server")] + if let Some(bg_processor) = agent_workers_bg_processor { + tracing::info!("server off. shutting down agent workers bg processor"); + bg_processor.await?; + tracing::info!("agent workers bg processor shut down"); + } Ok(()) } diff --git a/backend/windmill-api/src/resources.rs b/backend/windmill-api/src/resources.rs index e0a182107a..9b5e216047 100644 --- a/backend/windmill-api/src/resources.rs +++ b/backend/windmill-api/src/resources.rs @@ -570,7 +570,7 @@ pub async fn transform_json_value<'c>( }; let variables = variables::get_reserved_variables( - db, + &db.into(), workspace, token, &job.email, diff --git a/backend/windmill-api/src/variables.rs b/backend/windmill-api/src/variables.rs index c087915d71..e810f95750 100644 --- a/backend/windmill-api/src/variables.rs +++ b/backend/windmill-api/src/variables.rs @@ -61,7 +61,7 @@ async fn list_contextual_variables( ) -> JsonResult> { Ok(Json( get_reserved_variables( - &db, + &db.into(), &w_id, "q1A0qcPuO00yxioll7iph76N9CJDqn", &email, diff --git a/backend/windmill-common/Cargo.toml b/backend/windmill-common/Cargo.toml index fd370aedcc..140d3e2b74 100644 --- a/backend/windmill-common/Cargo.toml +++ b/backend/windmill-common/Cargo.toml @@ -34,6 +34,8 @@ serde_json.workspace = true chrono.workspace = true chrono-tz.workspace = true hex.workspace = true +reqwest-middleware = { workspace = true } +reqwest-retry = { workspace = true } rand.workspace = true sqlx = { workspace = true, features = ["postgres"] } uuid.workspace = true diff --git a/backend/windmill-common/src/agent_workers.rs b/backend/windmill-common/src/agent_workers.rs new file mode 100644 index 0000000000..7c91ec2165 --- /dev/null +++ b/backend/windmill-common/src/agent_workers.rs @@ -0,0 +1,90 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Deserialize, Serialize)] +pub struct QueueInitJob { + pub content: String, +} + +use lazy_static::lazy_static; +use std::time::Duration; + +use reqwest_middleware::ClientBuilder; +use reqwest_retry::{policies::ExponentialBackoff, RetryTransientMiddleware}; + +use crate::{jwt::decode_without_verify, worker::HttpClient}; + +lazy_static! { + pub static ref BASE_INTERNAL_URL: String = + std::env::var("BASE_INTERNAL_URL").unwrap_or("http://localhost:8080".to_string()); + pub static ref AGENT_TOKEN: String = std::env::var("AGENT_TOKEN").unwrap_or_default(); + pub static ref DECODED_AGENT_TOKEN: Option = { + if AGENT_TOKEN.is_empty() { + None + } else { + decode_without_verify::(AGENT_TOKEN.trim_start_matches(AGENT_JWT_PREFIX)) + .ok() + } + }; +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct AgentAuth { + pub worker_group: String, + pub suffix: Option, + pub tags: Vec, + pub exp: Option, +} + +pub const AGENT_JWT_PREFIX: &str = "jwt_agent_"; +pub fn build_agent_http_client(worker_suffix: &str) -> HttpClient { + let client = ClientBuilder::new( + reqwest::Client::builder() + .pool_max_idle_per_host(10) + .pool_idle_timeout(Duration::from_secs(60)) + .connect_timeout(Duration::from_secs(10)) + .timeout(Duration::from_secs(30)) + .default_headers({ + let mut headers = reqwest::header::HeaderMap::new(); + headers.insert( + "User-Agent", // Replace with your desired header name + "Windmill-Agent/1.0".parse().unwrap(), // Replace with your desired header value + ); + let token = format!( + "{}{}_{}", + AGENT_JWT_PREFIX, + worker_suffix, + AGENT_TOKEN.trim_start_matches(AGENT_JWT_PREFIX), + ); + headers.insert( + "Authorization", + format!("Bearer {}", token).parse().unwrap(), + ); + headers + }) + .build() + .expect("Failed to create HTTP client"), + ) + .with(RetryTransientMiddleware::new_with_policy( + ExponentialBackoff::builder().build_with_max_retries(5), + )) + .build(); + HttpClient(client) +} + +#[derive(Deserialize, Serialize)] +pub struct PingJobStatus { + pub mem_peak: Option, + pub current_mem: Option, +} + +#[derive(Deserialize, Serialize, Debug)] +pub struct PingJobStatusResponse { + pub canceled_by: Option, + pub canceled_reason: Option, + pub already_completed: bool, +} + +// #[derive(Serialize, Deserialize)] +// pub struct PullJobRequest { +// pub worker_name: String, +// } diff --git a/backend/windmill-common/src/auth.rs b/backend/windmill-common/src/auth.rs index d63fddc670..0266498773 100644 --- a/backend/windmill-common/src/auth.rs +++ b/backend/windmill-common/src/auth.rs @@ -1,8 +1,11 @@ +use anyhow::Context; use serde::{Deserialize, Serialize}; +use uuid::Uuid; use crate::{ db::Authed, error::{Error, Result}, + jwt, users::{SUPERADMIN_NOTIFICATION_EMAIL, SUPERADMIN_SECRET_EMAIL, SUPERADMIN_SYNC_EMAIL}, DB, }; @@ -22,7 +25,7 @@ pub struct JWTAuthClaims { pub scopes: Option>, } -#[derive(Deserialize)] +#[derive(Deserialize, Debug)] pub struct JobPerms { pub email: String, pub username: String, @@ -205,3 +208,62 @@ pub async fn get_groups_for_user( .collect(); Ok(groups) } + +#[tracing::instrument(level = "trace", skip_all)] +pub async fn create_token_for_owner( + db: &DB, + w_id: &str, + owner: &str, + label: &str, + expires_in: u64, + email: &str, + job_id: &Uuid, + perms: Option, +) -> crate::error::Result { + let job_perms = if perms.is_some() { + Ok(perms) + } else { + sqlx::query_as!( + JobPerms, + "SELECT email, username, is_admin, is_operator, groups, folders FROM job_perms WHERE job_id = $1 AND workspace_id = $2", + job_id, + w_id + ) + .fetch_optional(db) + .await + }; + let job_authed = match job_perms { + Ok(Some(jp)) => jp.into(), + _ => { + tracing::warn!("Could not get permissions for job {job_id} from job_perms table, getting permissions directly..."); + fetch_authed_from_permissioned_as(owner.to_string(), email.to_string(), w_id, db) + .await + .map_err(|e| { + Error::internal_err(format!( + "Could not get permissions directly for job {job_id}: {e:#}" + )) + })? + } + }; + + let payload = JWTAuthClaims { + email: job_authed.email, + username: job_authed.username, + is_admin: job_authed.is_admin, + is_operator: job_authed.is_operator, + groups: job_authed.groups, + folders: job_authed.folders, + label: Some(label.to_string()), + workspace_id: w_id.to_string(), + exp: (chrono::Utc::now() + chrono::Duration::seconds(expires_in as i64)).timestamp() + as usize, + job_id: Some(job_id.to_string()), + scopes: None, + }; + + let token = jwt::encode_with_internal_secret(&payload) + .await + .with_context(|| format!("Could not encode JWT token for job {job_id}"))?; + + Ok(format!("jwt_{}", token)) +} diff --git a/backend/windmill-common/src/cache.rs b/backend/windmill-common/src/cache.rs index 9374445f22..aac1457f56 100644 --- a/backend/windmill-common/src/cache.rs +++ b/backend/windmill-common/src/cache.rs @@ -7,7 +7,6 @@ //! This shall only be used for testing, e.g. [`sqlx::test`] spawn a database per test, //! and there is only one test per thread, so using thread-local cache avoid unexpected results. -use anyhow::anyhow; use crate::{ apps::AppScriptId, error, @@ -15,6 +14,7 @@ use crate::{ schema::SchemaValidator, scripts::{ScriptHash, ScriptLang}, }; +use anyhow::anyhow; #[cfg(feature = "scoped_cache")] use std::thread::ThreadId; @@ -322,6 +322,25 @@ pub struct RawScript { pub meta: Option, } +#[derive(Debug, Deserialize, Serialize)] +pub struct RawScriptApi { + pub content: String, + pub lock: Option, + pub meta: Option, +} + +impl From for RawScriptApi { + fn from(value: RawScript) -> Self { + RawScriptApi { content: value.content, lock: value.lock, meta: value.meta } + } +} + +impl From for RawScript { + fn from(value: RawScriptApi) -> Self { + RawScript { content: value.content, lock: value.lock, meta: value.meta } + } +} + #[derive(Debug)] pub struct RawFlow { pub raw_flow: Box, @@ -334,6 +353,25 @@ pub struct RawNode { pub raw_flow: Option>, } +#[derive(Debug, Deserialize, Serialize)] +pub struct RawNodeApi { + pub raw_code: Option, + pub raw_lock: Option, + pub raw_flow: Option>, +} + +impl From for RawNodeApi { + fn from(value: RawNode) -> Self { + RawNodeApi { raw_code: value.raw_code, raw_lock: value.raw_lock, raw_flow: value.raw_flow } + } +} + +impl From for RawNode { + fn from(value: RawNodeApi) -> Self { + RawNode { raw_code: value.raw_code, raw_lock: value.raw_lock, raw_flow: value.raw_flow } + } +} + #[derive(Debug, Clone)] struct Entry(Arc); @@ -343,7 +381,7 @@ struct ScriptFull { pub meta: Arc, } -fn unwrap_or_error( +pub fn unwrap_or_error( at: &'static Location, entity: &'static str, key: Key, @@ -363,6 +401,11 @@ pub fn clear() { } pub mod flow { + use crate::{ + worker::{fetch_flow_node_query, Connection}, + DB, + }; + use super::*; make_static! { @@ -388,10 +431,10 @@ pub mod flow { /// This should be preferred over fetching the database directly. #[track_caller] pub fn fetch_script<'c>( - e: impl PgExecutor<'c>, + conn: &'c Connection, node: FlowNodeId, - ) -> impl Future>> { - let fetch_node = fetch_node(e, node); + ) -> impl Future>> + 'c { + let fetch_node = fetch_node(conn, node); async move { fetch_node.await.and_then(|data| match data { RawData::Script(data) => Ok(data), @@ -409,11 +452,12 @@ pub mod flow { /// This should be preferred over fetching the database directly. #[track_caller] pub fn fetch_flow<'c>( - e: impl PgExecutor<'c>, + db: &'c DB, node: FlowNodeId, - ) -> impl Future>> { - let fetch_node = fetch_node(e, node); + ) -> impl Future>> + 'c { async move { + let conn = Connection::Sql(db.clone()); + let fetch_node = fetch_node(&conn, node); fetch_node.await.and_then(|data| match data { RawData::Flow(data) => Ok(data), RawData::Script(_) => Err(error::Error::internal_err(format!( @@ -430,31 +474,23 @@ pub mod flow { /// This should be preferred over fetching the database directly. #[track_caller] pub(super) fn fetch_node<'c>( - e: impl PgExecutor<'c>, + conn: &'c Connection, node: FlowNodeId, - ) -> impl Future> { + ) -> impl Future> + 'c { let loc = Location::caller(); // If not present, `get_or_insert_async` will lock the key until the future completes, // so only one thread will be able to fetch the data from the database and write it to // the file system and cache, hence no race on the file system. NODES.get_or_insert_async(node, async move { - sqlx::query!( - "SELECT \ - code AS \"raw_code: String\", \ - lock AS \"raw_lock: String\", \ - flow AS \"raw_flow: Json>\" \ - FROM flow_node WHERE id = $1 LIMIT 1", - node.0, - ) - .fetch_optional(e) - .await - .map_err(Into::into) - .and_then(unwrap_or_error(&loc, "Flow node", node)) - .map(|r| RawNode { - raw_code: r.raw_code, - raw_lock: r.raw_lock, - raw_flow: r.raw_flow.map(|Json(raw_flow)| raw_flow), - }) + match conn { + Connection::Sql(db) => fetch_flow_node_query(db, node.0, loc).await, + Connection::Http(client) => { + let r = client + .get::(&format!("/api/agent_workers/flow_script/{}", node.0)) + .await?; + Ok(r.into()) + } + } }) } @@ -502,6 +538,8 @@ pub mod flow { } pub mod script { + use crate::{worker::Connection, DB}; + use super::*; make_static! { @@ -520,66 +558,80 @@ pub mod script { /// it to the file system and cache. /// This should be preferred over fetching the database directly. #[track_caller] - pub fn fetch<'c>( - e: impl PgExecutor<'c>, + pub fn fetch( + conn: &Connection, hash: ScriptHash, ) -> impl Future, Arc)>> { // If not present, `get_or_insert_async` will lock the key until the future completes, // so only one thread will be able to fetch the data from the database and write it to // the file system and cache, hence no race on the file system. let loc = Location::caller(); + let conn = conn.clone(); let fut = CACHE.get_or_insert_async(hash, async move { - sqlx::query!( - "SELECT \ - content AS \"content!: String\", - lock AS \"lock: String\", \ - language AS \"language: Option\", \ - envs AS \"envs: Vec\", \ - schema AS \"schema: String\", \ - schema_validation AS \"schema_validation: bool\", \ - codebase LIKE '%.tar' as use_tar \ - FROM script WHERE hash = $1 LIMIT 1", - hash.0 - ) - .fetch_optional(e) - .await - .map_err(Into::into) - .and_then(unwrap_or_error(&loc, "Script", hash)) - .and_then(|r| { - Ok(RawScript { - content: r.content, - lock: r.lock, - meta: Some(ScriptMetadata { - language: r.language, - envs: r.envs, - codebase: if let Some(use_tar) = r.use_tar { - let sh = hash.to_string(); - if use_tar { - Some(format!("{sh}.tar")) - } else { - Some(sh) - } - } else { - None - }, - schema_validator: if r.schema_validation { - r.schema - .as_ref() - .map(|schema_str| { - SchemaValidator::from_schema(schema_str).map_err(|e| anyhow!("Couldn't create schema validator for script requiring schema validation: {e}")) - }) - .transpose()? - } else { - None - }, - schema: r.schema, - }), - }) - }) + match conn { + Connection::Sql(db) => fetch_script_from_db(&db, hash, loc).await, + Connection::Http(_) => Err(error::Error::InternalErr(format!( + "Cannot fetch script in HTTP mode" + ))), + } }); fut.map_ok(|ScriptFull { data, meta }| (data, meta)) } + async fn fetch_script_from_db( + db: &DB, + hash: ScriptHash, + loc: &'static Location<'_>, + ) -> error::Result { + sqlx::query!( + "SELECT \ + content AS \"content!: String\", + lock AS \"lock: String\", \ + language AS \"language: Option\", \ + envs AS \"envs: Vec\", \ + schema AS \"schema: String\", \ + schema_validation AS \"schema_validation: bool\", \ + codebase LIKE '%.tar' as use_tar \ + FROM script WHERE hash = $1 LIMIT 1", + hash.0 + ) + .fetch_optional(db) + .await + .map_err(Into::into) + .and_then(unwrap_or_error(&loc, "Script", hash)) + .and_then(|r| { + Ok(RawScript { + content: r.content, + lock: r.lock, + meta: Some(ScriptMetadata { + language: r.language, + envs: r.envs, + codebase: if let Some(use_tar) = r.use_tar { + let sh = hash.to_string(); + if use_tar { + Some(format!("{sh}.tar")) + } else { + Some(sh) + } + } else { + None + }, + schema_validator: if r.schema_validation { + r.schema + .as_ref() + .map(|schema_str| { + SchemaValidator::from_schema(schema_str).map_err(|e| anyhow!("Couldn't create schema validator for script requiring schema validation: {e}")) + }) + .transpose()? + } else { + None + }, + schema: r.schema, + }), + }) + }) + } + /// Invalidate the script cache for the given `hash`. pub fn invalidate(hash: ScriptHash) { let _ = CACHE.remove(&hash); @@ -587,6 +639,8 @@ pub mod script { } pub mod app { + use crate::worker::{fetch_raw_script_from_app_query, Connection}; + use super::*; make_static! { @@ -605,23 +659,23 @@ pub mod app { /// This should be preferred over fetching the database directly. #[track_caller] pub fn fetch_script<'c>( - e: impl PgExecutor<'c>, + conn: &'c Connection, id: AppScriptId, - ) -> impl Future>> { + ) -> impl Future>> + 'c { // If not present, `get_or_insert_async` will lock the key until the future completes, // so only one thread will be able to fetch the data from the database and write it to // the file system and cache, hence no race on the file system. let loc = Location::caller(); let fut = CACHE.get_or_insert_async(id, async move { - sqlx::query!( - "SELECT lock, code FROM app_script WHERE id = $1 LIMIT 1", - id.0, - ) - .fetch_optional(e) - .await - .map_err(Into::into) - .and_then(unwrap_or_error(&loc, "Application script", id)) - .map(|r| RawScript { content: r.code, lock: r.lock, meta: None }) + match conn { + Connection::Sql(db) => fetch_raw_script_from_app_query(db, id.0, loc).await, + Connection::Http(client) => { + let r = client + .get::(&format!("/api/agent_workers/app_script/{}", id.0)) + .await?; + Ok(r.into()) + } + } }); fut.map_ok(|Entry(data)| data) } @@ -629,7 +683,7 @@ pub mod app { pub mod job { use super::*; - use crate::jobs::JobKind; + use crate::{jobs::JobKind, worker::Connection, DB}; #[cfg(not(feature = "scoped_cache"))] lazy_static! { @@ -649,15 +703,18 @@ pub mod job { } #[track_caller] - pub fn fetch_preview_flow<'a, 'c>( - e: impl PgExecutor<'c> + 'a, + pub fn fetch_preview_flow<'a>( + db: &'a DB, job: &'a Uuid, - // original raw values from `queue` or `completed_job` tables: - // kept for backward compatibility. raw_flow: Option>>, ) -> impl Future>> + 'a { - let fetch_preview = fetch_preview(e, job, None, None, raw_flow); + // Create the Connection first so it lives for the entire scope + async move { + let conn = Connection::from(db); + + let fetch_preview = fetch_preview(&conn, job, None, None, raw_flow); + fetch_preview.await.and_then(|data| match data { RawData::Flow(data) => Ok(data), RawData::Script(_) => Err(error::Error::internal_err(format!( @@ -669,7 +726,7 @@ pub mod job { #[track_caller] pub fn fetch_preview_script<'a, 'c>( - e: impl PgExecutor<'c> + 'a, + e: &'a Connection, job: &'a Uuid, // original raw values from `queue` or `completed_job` tables: // kept for backward compatibility. @@ -689,7 +746,7 @@ pub mod job { #[track_caller] pub fn fetch_preview<'a, 'c>( - e: impl PgExecutor<'c> + 'a, + e: &'a Connection, job: &'a Uuid, // original raw values from `queue` or `completed_job` tables: // kept for backward compatibility. @@ -700,16 +757,21 @@ pub mod job { let loc = Location::caller(); let fetch = async move { match (raw_lock, raw_code, raw_flow) { - (None, None, None) => sqlx::query!( - "SELECT raw_code, raw_lock, raw_flow AS \"raw_flow: Json>\" \ - FROM v2_job WHERE id = $1 LIMIT 1", - job - ) - .fetch_optional(e) - .await - .map_err(Into::into) - .and_then(unwrap_or_error(&loc, "Preview", job)) - .map(|r| (r.raw_lock, r.raw_code, r.raw_flow)), + (None, None, None) => match e { + Connection::Sql(pool) => sqlx::query!( + "SELECT raw_code, raw_lock, raw_flow AS \"raw_flow: Json>\" \ + FROM v2_job WHERE id = $1 LIMIT 1", + job + ) + .fetch_optional(pool) + .await + .map_err(Into::into) + .and_then(unwrap_or_error(&loc, "Preview", job)) + .map(|r| (r.raw_lock, r.raw_code, r.raw_flow)), + Connection::Http(_) => Err(error::Error::InternalErr(format!( + "Cannot fetch preview in HTTP mode" + ))), + }, (lock, code, flow) => Ok((lock, code, flow)), } .and_then(|(lock, code, flow)| match flow { @@ -730,8 +792,8 @@ pub mod job { } #[track_caller] - pub fn fetch_script<'c>( - e: impl PgExecutor<'c>, + pub fn fetch_script( + db: DB, kind: JobKind, hash: Option, ) -> impl Future>> { @@ -739,11 +801,15 @@ pub mod job { let loc = Location::caller(); async move { match (kind, hash.map(|ScriptHash(id)| id)) { - (FlowScript, Some(id)) => flow::fetch_script(e, FlowNodeId(id)).await, - (Script | Dependencies, Some(hash)) => script::fetch(e, ScriptHash(hash)) + (FlowScript, Some(id)) => { + flow::fetch_script(&Connection::Sql(db.clone()), FlowNodeId(id)).await + } + (Script | Dependencies, Some(hash)) => script::fetch(&db.into(), ScriptHash(hash)) .await .map(|(data, _meta)| data), - (AppScript, Some(id)) => app::fetch_script(e, AppScriptId(id)).await, + (AppScript, Some(id)) => { + app::fetch_script(&Connection::Sql(db.clone()), AppScriptId(id)).await + } _ => Err(error::Error::internal_err(format!( "Isn't a script job: {:?}", kind @@ -755,19 +821,19 @@ pub mod job { #[track_caller] pub fn fetch_flow<'c>( - e: impl PgExecutor<'c> + Copy, + db: &'c DB, kind: JobKind, hash: Option, - ) -> impl Future>> { + ) -> impl Future>> + 'c { use JobKind::*; let loc = Location::caller(); async move { match (kind, hash.map(|ScriptHash(id)| id)) { - (FlowDependencies, Some(id)) => flow::fetch_version(e, id).await, - (FlowNode, Some(id)) => flow::fetch_flow(e, FlowNodeId(id)).await, - (Flow, Some(id)) => match flow::fetch_version_lite(e, id).await { + (FlowDependencies, Some(id)) => flow::fetch_version(db, id).await, + (FlowNode, Some(id)) => flow::fetch_flow(db, FlowNodeId(id)).await, + (Flow, Some(id)) => match flow::fetch_version_lite(db, id).await { Ok(raw_flow) => Ok(raw_flow), - Err(_) => flow::fetch_version(e, id).await, + Err(_) => flow::fetch_version(db, id).await, }, _ => Err(error::Error::internal_err(format!( "Isn't a flow job {:?}", diff --git a/backend/windmill-common/src/flows.rs b/backend/windmill-common/src/flows.rs index ab0e714968..1f6c265e7d 100644 --- a/backend/windmill-common/src/flows.rs +++ b/backend/windmill-common/src/flows.rs @@ -22,7 +22,7 @@ use crate::{ error::Error, more_serde::{default_empty_string, default_id, default_null, default_true, is_default}, scripts::{Schema, ScriptHash, ScriptLang}, - worker::to_raw_value, + worker::{to_raw_value, Connection}, DB, }; #[derive(Serialize, Deserialize, sqlx::FromRow)] @@ -731,7 +731,7 @@ pub async fn resolve_maybe_value( } /// Resolve modules recursively. -pub async fn resolve_value( +async fn resolve_value( e: &sqlx::PgPool, workspace_id: &str, value: &mut Box, @@ -749,7 +749,7 @@ pub async fn resolve_value( /// Resolve module value recursively. pub async fn resolve_module( - e: &sqlx::PgPool, + db: &DB, workspace_id: &str, value: &mut Box, with_code: bool, @@ -783,7 +783,7 @@ pub async fn resolve_module( let (lock, content) = if !with_code { (Some("...".to_string()), "...".to_string()) } else { - cache::flow::fetch_script(e, id) + cache::flow::fetch_script(&Connection::Sql(db.clone()), id) .await .map(|data| (data.lock.clone(), data.code.clone()))? }; @@ -801,13 +801,13 @@ pub async fn resolve_module( }; } ForloopFlow { modules, modules_node, .. } | WhileloopFlow { modules, modules_node, .. } => { - resolve_modules(e, workspace_id, modules, modules_node.take(), with_code).await?; + resolve_modules(db, workspace_id, modules, modules_node.take(), with_code).await?; } BranchOne { branches, default, default_node } => { - resolve_modules(e, workspace_id, default, default_node.take(), with_code).await?; + resolve_modules(db, workspace_id, default, default_node.take(), with_code).await?; for branch in branches { resolve_modules( - e, + db, workspace_id, &mut branch.modules, branch.modules_node.take(), @@ -819,7 +819,7 @@ pub async fn resolve_module( BranchAll { branches, .. } => { for branch in branches { resolve_modules( - e, + db, workspace_id, &mut branch.modules, branch.modules_node.take(), diff --git a/backend/windmill-common/src/jobs.rs b/backend/windmill-common/src/jobs.rs index 49de1c2415..6cad6579fb 100644 --- a/backend/windmill-common/src/jobs.rs +++ b/backend/windmill-common/src/jobs.rs @@ -10,6 +10,7 @@ use tokio::io::AsyncReadExt; use uuid::Uuid; pub const ENTRYPOINT_OVERRIDE: &str = "_ENTRYPOINT_OVERRIDE"; +pub const LARGE_LOG_THRESHOLD_SIZE: usize = 9000; use crate::{ apps::AppScriptId, @@ -24,7 +25,7 @@ use crate::{ #[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Copy, Clone)] #[sqlx(type_name = "JOB_KIND", rename_all = "lowercase")] -#[serde(rename_all(serialize = "lowercase"))] +#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))] pub enum JobKind { Script, #[allow(non_camel_case_types)] @@ -51,6 +52,13 @@ impl JobKind { JobKind::Flow | JobKind::FlowPreview | JobKind::SingleScriptFlow | JobKind::FlowNode ) } + + pub fn is_dependency(&self) -> bool { + matches!( + self, + JobKind::FlowDependencies | JobKind::AppDependencies | JobKind::Dependencies + ) + } } #[derive(sqlx::FromRow, Debug, Serialize, Clone)] diff --git a/backend/windmill-common/src/jwt.rs b/backend/windmill-common/src/jwt.rs index be93ed6614..8ebdb8dacb 100644 --- a/backend/windmill-common/src/jwt.rs +++ b/backend/windmill-common/src/jwt.rs @@ -1,6 +1,6 @@ use crate::error::{self, to_anyhow, Error}; use serde::{de::DeserializeOwned, Serialize}; -use std::sync::Arc; +use std::{collections::HashSet, sync::Arc}; use tokio::sync::RwLock; lazy_static::lazy_static! { @@ -40,3 +40,20 @@ pub async fn decode_with_internal_secret(token: &str) -> er Ok(result.claims) } + +pub fn decode_without_verify(token: &str) -> anyhow::Result { + // Create a validation that skips all checks + let mut validation = jsonwebtoken::Validation::default(); + validation.insecure_disable_signature_validation(); + validation.validate_exp = false; + validation.validate_nbf = false; + validation.required_spec_claims = HashSet::new(); + + // Use an empty key since we're not verifying + let key = jsonwebtoken::DecodingKey::from_secret(&[]); + + // Decode the token + let token_data = jsonwebtoken::decode::(token, &key, &validation)?; + + Ok(token_data.claims) +} diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index ca00da879a..fd43db2f7c 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -22,6 +22,7 @@ use error::Error; use scripts::ScriptLang; use sqlx::{Pool, Postgres}; +pub mod agent_workers; pub mod apps; pub mod auth; pub mod cache; @@ -45,6 +46,7 @@ pub mod otel_ee; pub mod queue; pub mod s3_helpers; pub mod schedule; +pub mod schema; pub mod scripts; pub mod server; pub mod stats_ee; @@ -55,7 +57,6 @@ pub mod utils; pub mod variables; pub mod worker; pub mod workspaces; -pub mod schema; pub const DEFAULT_MAX_CONNECTIONS_SERVER: u32 = 50; pub const DEFAULT_MAX_CONNECTIONS_WORKER: u32 = 5; diff --git a/backend/windmill-common/src/utils.rs b/backend/windmill-common/src/utils.rs index 539064fcf1..ee157db3f0 100644 --- a/backend/windmill-common/src/utils.rs +++ b/backend/windmill-common/src/utils.rs @@ -23,7 +23,7 @@ use rand::distr::Alphanumeric; use rand::{rng, Rng}; use reqwest::Client; use semver::Version; -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Deserializer, Serialize}; use sha2::{Digest, Sha256}; use sqlx::{Pool, Postgres}; use std::str::FromStr; @@ -75,8 +75,8 @@ lazy_static::lazy_static! { if std::env::var("BASE_INTERNAL_URL").is_err() { panic!("BASE_INTERNAL_URL is required in agent mode") } - if std::env::var("JOB_TOKEN").is_err() { - println!("JOB_TOKEN is not passed, hence workers will still need to create permissions for each job and the DATABASE_URL needs to be of a role that can INSERT into the job_perms table") + if std::env::var("AGENT_TOKEN").is_err() { + println!("AGENT_TOKEN is not passed. This is required for the agent to work and contains the JWT to authenticate with the server.") } #[cfg(not(feature = "enterprise"))] @@ -175,6 +175,28 @@ pub fn hostname() -> String { }) } +fn instance_name(hostname: &str) -> String { + hostname + .replace(" ", "") + .split("-") + .last() + .unwrap() + .to_ascii_lowercase() + .to_string() +} + +pub fn worker_suffix(hostname: &str, rd_string: &str) -> String { + format!("{}-{}", instance_name(hostname), rd_string) +} + +pub fn worker_name_with_suffix(is_agent: bool, worker_group: &str, suffix: &str) -> String { + if is_agent { + format!("ag-{}-{}", worker_group, suffix) + } else { + format!("wk-{}-{}", worker_group, suffix) + } +} + pub fn paginate(pagination: Pagination) -> (usize, usize) { let per_page = pagination .per_page @@ -396,6 +418,16 @@ pub async fn report_critical_error( } } +pub fn empty_string_as_none<'de, D>( + deserializer: D, +) -> std::result::Result, D::Error> +where + D: Deserializer<'de>, +{ + let option = as serde::Deserialize>::deserialize(deserializer)?; + Ok(option.filter(|s| !s.is_empty())) +} + pub async fn report_recovered_critical_error( message: String, _db: DB, diff --git a/backend/windmill-common/src/variables.rs b/backend/windmill-common/src/variables.rs index 90baec78a5..3fd2e0513b 100644 --- a/backend/windmill-common/src/variables.rs +++ b/backend/windmill-common/src/variables.rs @@ -7,6 +7,7 @@ */ use crate::error; +use crate::worker::Connection; use crate::{worker::WORKER_GROUP, BASE_URL, DB}; use chrono::{SecondsFormat, Utc}; use magic_crypt::{MagicCrypt256, MagicCryptError, MagicCryptTrait}; @@ -166,7 +167,7 @@ lazy_static::lazy_static! { } pub async fn get_reserved_variables( - db: &DB, + conn: &Connection, w_id: &str, token: &str, email: &str, @@ -206,7 +207,7 @@ pub async fn get_reserved_variables( } }; - let custom_envs = get_cached_workspace_envs(db, w_id).await; + let custom_envs = get_cached_workspace_envs(conn, w_id).await; let joined_schedule_path = schedule_path .clone() @@ -354,10 +355,7 @@ pub async fn get_reserved_variables( ).collect() } -async fn get_cached_workspace_envs( - db: &sqlx::Pool, - w_id: &str, -) -> Vec<(String, String)> { +async fn get_cached_workspace_envs(conn: &Connection, w_id: &str) -> Vec<(String, String)> { let cached_envs_o = CUSTOM_ENVS_CACHE.get(w_id).and_then(|(ts, envs)| { if ts > chrono::Utc::now().timestamp() - (60 * 15) { Some(envs) @@ -369,13 +367,19 @@ async fn get_cached_workspace_envs( let custom_envs = if let Some(cached_envs) = cached_envs_o { cached_envs } else { - let custom_envs = sqlx::query_as::<_, (String, String)>( - "SELECT name, value FROM workspace_env WHERE workspace_id = $1", - ) - .bind(w_id) - .fetch_all(db) - .await - .unwrap_or_default(); + let custom_envs = match conn { + Connection::Sql(db) => sqlx::query_as::<_, (String, String)>( + "SELECT name, value FROM workspace_env WHERE workspace_id = $1", + ) + .bind(w_id) + .fetch_all(db) + .await + .unwrap_or_default(), + Connection::Http(client) => client + .get(&format!("/api/w/{w_id}/agent_workers/custom_envs")) + .await + .unwrap_or_default(), + }; CUSTOM_ENVS_CACHE.insert( w_id.to_string(), (chrono::Utc::now().timestamp(), custom_envs.clone()), diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 9025194461..cdffe6f6a1 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -3,28 +3,55 @@ use bytes::Bytes; use const_format::concatcp; use itertools::Itertools; use regex::Regex; +use reqwest_middleware::ClientWithMiddleware; use semver::Version; -use serde::{Deserialize, Serialize}; +use serde::{de::DeserializeOwned, Deserialize, Serialize}; use serde_json::value::RawValue; +use sqlx::{types::Json, Pool, Postgres}; use std::{ cmp::Reverse, collections::{HashMap, HashSet}, fs::{self, File}, io::Write, + panic::Location, path::{Component, Path, PathBuf}, str::FromStr, sync::{atomic::AtomicBool, Arc}, }; use tokio::sync::RwLock; +use uuid::Uuid; use windmill_macros::annotations; use crate::{ - error, global_settings::CUSTOM_TAGS_SETTING, indexer::TantivyIndexerSettings, server::Smtp, + agent_workers::{PingJobStatusResponse, BASE_INTERNAL_URL}, + cache::{unwrap_or_error, RawNode, RawScript}, + error::{self, to_anyhow}, + global_settings::CUSTOM_TAGS_SETTING, + indexer::TantivyIndexerSettings, + server::Smtp, KillpillSender, DB, }; +pub const DEFAULT_CLOUD_TIMEOUT: u64 = 900; +pub const DEFAULT_SELFHOSTED_TIMEOUT: u64 = 604800; // 7 days + lazy_static::lazy_static! { - pub static ref WORKER_GROUP: String = std::env::var("WORKER_GROUP").unwrap_or_else(|_| "default".to_string()); + pub static ref WORKER_GROUP: String = std::env::var("WORKER_GROUP").unwrap_or_else(|_| { + #[cfg(not(feature = "enterprise"))] + { + "default".to_string() + } + + #[cfg(feature = "enterprise")] + { + if let Some(token) = crate::agent_workers::DECODED_AGENT_TOKEN.as_ref() { + token.worker_group.clone() + } else { + "default".to_string() + } + } + }); + pub static ref NO_LOGS: bool = std::env::var("NO_LOGS").ok().is_some_and(|x| x == "1" || x == "true"); pub static ref CGROUP_V2_PATH_RE: Regex = Regex::new(r#"(?m)^0::(/.*)$"#).unwrap(); @@ -61,6 +88,15 @@ lazy_static::lazy_static! { pub static ref DEFAULT_TAGS_PER_WORKSPACE: AtomicBool = AtomicBool::new(false); pub static ref DEFAULT_TAGS_WORKSPACES: Arc>>> = Arc::new(RwLock::new(None)); + pub static ref MAX_TIMEOUT: u64 = std::env::var("TIMEOUT") + .ok() + .and_then(|x| x.parse::().ok()) + .unwrap_or_else(|| if *CLOUD_HOSTED { DEFAULT_CLOUD_TIMEOUT } else { DEFAULT_SELFHOSTED_TIMEOUT }); + + pub static ref SCRIPT_TOKEN_EXPIRY: u64 = std::env::var("SCRIPT_TOKEN_EXPIRY") + .ok() + .and_then(|x| x.parse::().ok()) + .unwrap_or(*MAX_TIMEOUT); pub static ref WORKER_CONFIG: Arc> = Arc::new(RwLock::new(WorkerConfig { worker_tags: Default::default(), @@ -115,6 +151,90 @@ pub const ROOT_CACHE_NOMOUNT_DIR: &str = concatcp!(TMP_DIR, "/cache_nomount/"); pub static MIN_VERSION_IS_LATEST: AtomicBool = AtomicBool::new(false); +#[derive(Clone)] +pub struct HttpClient(pub ClientWithMiddleware); + +impl HttpClient { + pub async fn post( + &self, + url: &str, + body: &T, + ) -> anyhow::Result { + let response = self + .0 + .post(format!("{}{}", *BASE_INTERNAL_URL, url)) + .json(body) + .send() + .await + .map_err(|e| anyhow::anyhow!(e))?; + let status = response.status(); + if status.is_success() { + Ok(response.json().await?) + } else { + Err(anyhow::anyhow!(format!( + "HTTP agent request POST {} failed {}", + url, + response.status() + ))) + } + } + + pub async fn get(&self, url: &str) -> anyhow::Result { + let response = self + .0 + .get(format!("{}{}", *BASE_INTERNAL_URL, url)) + .send() + .await + .map_err(|e| anyhow::anyhow!(e))?; + let status = response.status(); + if status.is_success() { + Ok(response.json().await?) + } else { + Err(anyhow::anyhow!(format!( + "HTTP agent request GET {} failed {}", + url, + response.status() + ))) + } + } +} + +#[derive(Clone)] +pub enum Connection { + Sql(Pool), + Http(HttpClient), +} + +impl std::fmt::Debug for Connection { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Connection::Sql(_) => write!(f, "Sql"), + Connection::Http(_) => write!(f, "Http"), + } + } +} + +impl Connection { + pub fn as_sql(&self) -> Option<&Pool> { + match self { + Connection::Sql(db) => Some(db), + Connection::Http(_) => None, + } + } +} + +impl From> for Connection { + fn from(value: Pool) -> Self { + Connection::Sql(value) + } +} + +impl From<&Pool> for Connection { + fn from(value: &Pool) -> Self { + Connection::Sql(value.clone()) + } +} + fn format_pull_query(peek: String) -> String { let r = format!( "WITH peek AS ( @@ -164,40 +284,48 @@ fn format_pull_query(peek: String) -> String { r } -pub async fn make_suspended_pull_query(wc: &WorkerConfig) { - if wc.worker_tags.len() == 0 { - tracing::error!("Empty tags in worker tags, skipping"); - return; - } - let query = format_pull_query(format!( +pub fn make_suspended_pull_query(tags: &[String]) -> String { + format_pull_query(format!( "SELECT id FROM v2_job_queue WHERE suspend_until IS NOT NULL AND (suspend <= 0 OR suspend_until <= now()) AND tag IN ({}) ORDER BY priority DESC NULLS LAST, created_at FOR UPDATE SKIP LOCKED LIMIT 1", - wc.worker_tags.iter().map(|x| format!("'{x}'")).join(", ") - )); + tags.iter().map(|x| format!("'{x}'")).join(", ") + )) +} +// pub async fn make_suspended +pub async fn store_suspended_pull_query(wc: &WorkerConfig) { + if wc.worker_tags.len() == 0 { + tracing::error!("Empty tags in worker tags, skipping"); + return; + } + let query = make_suspended_pull_query(&wc.worker_tags); let mut l = WORKER_SUSPENDED_PULL_QUERY.write().await; *l = query; } -pub async fn make_pull_query(wc: &WorkerConfig) { +pub fn make_pull_query(tags: &[String]) -> String { + format_pull_query(format!( + "SELECT id + FROM v2_job_queue + WHERE running = false AND tag IN ({}) AND scheduled_for <= now() + ORDER BY priority DESC NULLS LAST, scheduled_for + FOR UPDATE SKIP LOCKED + LIMIT 1", + tags.iter().map(|x| format!("'{x}'")).join(", ") + )) +} + +pub async fn store_pull_query(wc: &WorkerConfig) { let mut queries = vec![]; for tags in wc.priority_tags_sorted.iter() { if tags.tags.len() == 0 { tracing::error!("Empty tags in priority tags, skipping"); continue; } - let query = format_pull_query(format!( - "SELECT id - FROM v2_job_queue - WHERE running = false AND tag IN ({}) AND scheduled_for <= now() - ORDER BY priority DESC NULLS LAST, scheduled_for - FOR UPDATE SKIP LOCKED - LIMIT 1", - tags.tags.iter().map(|x| format!("'{x}'")).join(", ") - )); + let query = make_pull_query(&tags.tags); queries.push(query); } let mut l = WORKER_PULL_QUERIES.write().await; @@ -758,24 +886,33 @@ pub fn get_windmill_memory_usage() -> Option { } } -pub async fn update_min_version<'c, E: sqlx::Executor<'c, Database = sqlx::Postgres>>( - executor: E, -) -> bool { +pub async fn update_min_version(conn: &Connection) -> bool { use crate::utils::{GIT_SEM_VERSION, GIT_VERSION}; - // fetch all pings with a different version than self from the last 5 minutes. - let pings = sqlx::query_scalar!( - "SELECT wm_version FROM worker_ping WHERE wm_version != $1 AND ping_at > now() - interval '5 minutes'", - GIT_VERSION - ).fetch_all(executor).await.unwrap_or_default(); - let cur_version = GIT_SEM_VERSION.clone(); - let min_version = pings - .iter() - .filter(|x| !x.is_empty()) - .filter_map(|x| semver::Version::parse(if x.starts_with('v') { &x[1..] } else { x }).ok()) - .min() - .unwrap_or_else(|| cur_version.clone()); + + let min_version = match conn { + Connection::Sql(pool) => { + // fetch all pings with a different version than self from the last 5 minutes. + let pings = sqlx::query_scalar!( + "SELECT wm_version FROM worker_ping WHERE wm_version != $1 AND ping_at > now() - interval '5 minutes'", + GIT_VERSION + ).fetch_all(pool).await.unwrap_or_default(); + + pings + .iter() + .filter(|x| !x.is_empty()) + .filter_map(|x| { + semver::Version::parse(if x.starts_with('v') { &x[1..] } else { x }).ok() + }) + .min() + .unwrap_or_else(|| cur_version.clone()) + } + Connection::Http(_) => { + // TODO: get min version from server, for now we use the current version. Min version should be of no interest for http mode workers + cur_version.clone() + } + }; if min_version != cur_version { tracing::info!("Minimal worker version: {min_version}"); @@ -790,37 +927,330 @@ pub async fn update_min_version<'c, E: sqlx::Executor<'c, Database = sqlx::Postg min_version >= cur_version } -pub async fn update_ping(worker_instance: &str, worker_name: &str, ip: &str, db: &DB) { - let (tags, dw) = { - let wc = WORKER_CONFIG.read().await.clone(); - ( - wc.worker_tags, - wc.dedicated_worker - .as_ref() - .map(|x| format!("{}:{}", x.workspace_id, x.path)), - ) - }; +#[derive(Serialize, Deserialize)] +pub enum PingType { + Initial, + MainLoop, + Job, + InitScript, +} +#[derive(Serialize, Deserialize)] +pub struct Ping { + pub last_job_executed: Option, + pub last_job_workspace_id: Option, + pub worker_instance: Option, + pub ip: Option, + pub tags: Option>, + pub dw: Option, + pub version: Option, + pub vcpus: Option, + pub memory: Option, + pub memory_usage: Option, + pub wm_memory_usage: Option, + pub jobs_executed: Option, + pub occupancy_rate: Option, + pub occupancy_rate_15s: Option, + pub occupancy_rate_5m: Option, + pub occupancy_rate_30m: Option, + pub ping_type: PingType, +} +pub async fn update_ping_http( + insert_ping: Ping, + worker_name: &str, + worker_group: &str, + db: &DB, +) -> anyhow::Result<()> { + // tracing::info!("update ping: {}", insert_ping.tags.join(",")); + match insert_ping.ping_type { + PingType::MainLoop => { + update_worker_ping_main_loop_query( + worker_name, + insert_ping.tags.unwrap_or_default().as_slice(), + insert_ping.vcpus, + insert_ping.memory, + insert_ping.jobs_executed, + insert_ping.occupancy_rate, + insert_ping.memory_usage, + insert_ping.wm_memory_usage, + insert_ping.occupancy_rate_15s, + insert_ping.occupancy_rate_5m, + insert_ping.occupancy_rate_30m, + db, + ) + .await? + } + PingType::Initial => { + if insert_ping.worker_instance.is_none() + || insert_ping.version.is_none() + || insert_ping.ip.is_none() + { + return Err(anyhow::anyhow!( + "Worker instance, version and ip are required" + )); + } - let vcpus = get_vcpus(); - let memory = get_memory(); + insert_ping_query( + &insert_ping.worker_instance.unwrap(), + &worker_name, + worker_group, + &insert_ping.ip.unwrap(), + insert_ping.tags.unwrap_or_default().as_slice(), + insert_ping.dw, + &insert_ping.version.unwrap(), + insert_ping.vcpus, + insert_ping.memory, + db, + ) + .await?; + } + PingType::Job => { + update_worker_ping_from_job_query( + &insert_ping.last_job_executed.unwrap_or_default(), + &insert_ping.last_job_workspace_id.unwrap_or_default(), + worker_name, + insert_ping.memory_usage, + insert_ping.wm_memory_usage, + insert_ping.occupancy_rate, + insert_ping.occupancy_rate_15s, + insert_ping.occupancy_rate_5m, + insert_ping.occupancy_rate_30m, + db, + ) + .await?; + } + PingType::InitScript => { + update_ping_for_failed_init_script_query( + worker_name, + insert_ping.last_job_executed.unwrap_or_default(), + db, + ) + .await? + } + } + Ok(()) +} +#[derive(Serialize, Deserialize)] +pub struct JobCancelled { + pub canceled_by: String, + pub reason: String, +} + +pub async fn set_job_cancelled_query( + job_id: Uuid, + db: &DB, + canceled_by: &str, + reason: &str, +) -> anyhow::Result<()> { + sqlx::query!( + "UPDATE v2_job_queue + SET canceled_by = $1 + , canceled_reason = $2 +WHERE id = $3", + canceled_by, + reason, + job_id + ) + .execute(db) + .await?; + Ok(()) +} + +pub async fn update_ping_for_failed_init_script_query( + worker_name: &str, + last_job_id: Uuid, + db: &DB, +) -> anyhow::Result<()> { + sqlx::query!( + "UPDATE worker_ping SET +ping_at = now(), +jobs_executed = 1, +current_job_id = $1, +current_job_workspace_id = 'admins' +WHERE worker = $2", + last_job_id, + worker_name + ) + .execute(db) + .await?; + Ok(()) +} + +pub async fn fetch_flow_node_query( + db: &DB, + id: i64, + loc: &'static Location<'_>, +) -> error::Result { + let r = sqlx::query!( + "SELECT \ + code AS \"raw_code: String\", \ + lock AS \"raw_lock: String\", \ + flow AS \"raw_flow: Json>\" \ + FROM flow_node WHERE id = $1 LIMIT 1", + id, + ) + .fetch_optional(db) + .await + .map_err(Into::into) + .and_then(unwrap_or_error(loc, "Flow node", id)) + .map(|r| RawNode { + raw_code: r.raw_code, + raw_lock: r.raw_lock, + raw_flow: r.raw_flow.map(|Json(raw_flow)| raw_flow), + })?; + Ok(r) +} + +pub async fn fetch_raw_script_from_app_query( + db: &DB, + id: i64, + loc: &'static Location<'_>, +) -> error::Result { + sqlx::query!( + "SELECT lock, code FROM app_script WHERE id = $1 LIMIT 1", + id, + ) + .fetch_optional(db) + .await + .map_err(Into::into) + .and_then(unwrap_or_error(&loc, "Application script", id)) + .map(|r| RawScript { content: r.code, lock: r.lock, meta: None }) +} + +pub async fn insert_ping_query( + worker_instance: &str, + worker_name: &str, + worker_group: &str, + ip: &str, + tags: &[String], + dw: Option, + version: &str, + vcpus: Option, + memory: Option, + db: &DB, +) -> anyhow::Result<()> { sqlx::query!( "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, wm_version, vcpus, memory) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) ON CONFLICT (worker) DO UPDATE set ip = $3, custom_tags = $4, worker_group = $5", worker_instance, worker_name, ip, - tags.as_slice(), - *WORKER_GROUP, + tags, + worker_group, dw, - crate::utils::GIT_VERSION, + version, vcpus, memory + ) + .execute(db) + .await?; + Ok(()) +} + +pub async fn update_worker_ping_from_job_query( + job_id: &Uuid, + w_id: &str, + worker_name: &str, + memory_usage: Option, + wm_memory_usage: Option, + occupancy_rate: Option, + occupancy_rate_15s: Option, + occupancy_rate_5m: Option, + occupancy_rate_30m: Option, + db: &DB, +) -> anyhow::Result<()> { + sqlx::query!( + "UPDATE worker_ping SET ping_at = now(), current_job_id = $1, current_job_workspace_id = $2, memory_usage = $3, wm_memory_usage = $4, + occupancy_rate = $6, occupancy_rate_15s = $7, occupancy_rate_5m = $8, occupancy_rate_30m = $9 WHERE worker = $5", + job_id, + w_id, + memory_usage, + wm_memory_usage, + worker_name, + occupancy_rate, + occupancy_rate_15s, + occupancy_rate_5m, + occupancy_rate_30m, ) .execute(db) - .await - .expect("insert worker_ping initial value"); + .await?; + Ok(()) } +pub async fn update_job_ping_query( + job_id: &Uuid, + db: &DB, + mem_peak: Option, +) -> anyhow::Result { + let ro = sqlx::query!( + "UPDATE v2_job_runtime r SET + memory_peak = $1, + ping = now() + FROM v2_job_queue q + WHERE r.id = $2 AND q.id = r.id + RETURNING canceled_by, canceled_reason", + mem_peak, + job_id + ) + .map(|x| PingJobStatusResponse { + canceled_by: x.canceled_by, + canceled_reason: x.canceled_reason, + already_completed: false, + }) + .fetch_optional(db) + .await; + + // TODO: add memory metrics to memory time series + + if let Ok(r) = ro { + if let Some(i) = r { + Ok(i) + } else { + Err(anyhow::anyhow!("Job not found")) + } + } else { + Err(to_anyhow(ro.unwrap_err())) + } +} + +pub async fn update_worker_ping_main_loop_query( + worker_name: &str, + tags: &[String], + vcpus: Option, + memory: Option, + jobs_executed: Option, + occupancy_rate: Option, + memory_usage: Option, + wm_memory_usage: Option, + occupancy_rate_15s: Option, + occupancy_rate_5m: Option, + occupancy_rate_30m: Option, + db: &DB, +) -> anyhow::Result<()> { + sqlx::query!( + "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1, custom_tags = $2, + occupancy_rate = $3, memory_usage = $4, wm_memory_usage = $5, vcpus = COALESCE($7, vcpus), + memory = COALESCE($8, memory), occupancy_rate_15s = $9, occupancy_rate_5m = $10, occupancy_rate_30m = $11 WHERE worker = $6", + jobs_executed, + tags, + occupancy_rate, + memory_usage, + wm_memory_usage, + worker_name, + vcpus, + memory, + occupancy_rate_15s, + occupancy_rate_5m, + occupancy_rate_30m, + ) + .execute(db) + .await?; + Ok(()) +} + +// "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1, custom_tags = $2, +// occupancy_rate = $3, memory_usage = $4, wm_memory_usage = $5, vcpus = COALESCE($7, vcpus), +// memory = COALESCE($8, memory), occupancy_rate_15s = $9, occupancy_rate_5m = $10, occupancy_rate_30m = $11 WHERE worker = $6", + pub async fn load_worker_config( db: &DB, killpill_tx: KillpillSender, @@ -1018,7 +1448,7 @@ pub struct WorkspacedPath { pub path: String, } -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize, Debug)] pub struct WorkerConfigOpt { pub worker_tags: Option>, pub priority_tags: Option>, diff --git a/backend/windmill-queue/src/flow_status.rs b/backend/windmill-queue/src/flow_status.rs new file mode 100644 index 0000000000..3bc6b3c815 --- /dev/null +++ b/backend/windmill-queue/src/flow_status.rs @@ -0,0 +1,135 @@ +use uuid::Uuid; +use windmill_common::{ + error::{self, Error}, + utils::WarnAfterExt, + DB, +}; + +#[derive(Debug, Copy, Clone)] +pub enum Step { + Step(usize), + PreprocessorStep, + FailureStep, +} + +impl Step { + pub fn from_i32_and_len(step: i32, len: usize) -> Self { + if step < 0 { + Step::PreprocessorStep + } else if (step as usize) < len { + Step::Step(step as usize) + } else { + Step::FailureStep + } + } +} + +pub async fn update_flow_status_in_progress( + db: &DB, + _w_id: &str, + flow: Uuid, + job_in_progress: Uuid, +) -> error::Result { + let step = get_step_of_flow_status(db, flow).await?; + match step { + Step::Step(step) => { + sqlx::query!( + "UPDATE v2_job_status SET + flow_status = jsonb_set( + jsonb_set(flow_status, ARRAY['modules', $3::INTEGER::TEXT, 'job'], to_jsonb($1::UUID::TEXT)), + ARRAY['modules', $3::INTEGER::TEXT, 'type'], + to_jsonb('InProgress'::text) + ) + WHERE id = $2", + job_in_progress, + flow, + step as i32 + ) + .execute(db) + .await?; + } + Step::PreprocessorStep => { + sqlx::query!( + "UPDATE v2_job_status SET + flow_status = jsonb_set( + jsonb_set(flow_status, ARRAY['preprocessor_module', 'job'], to_jsonb($1::UUID::TEXT)), + ARRAY['preprocessor_module', 'type'], + to_jsonb('InProgress'::text) + ) + WHERE id = $2", + job_in_progress, + flow + ) + .execute(db) + .await?; + } + Step::FailureStep => { + sqlx::query!( + "UPDATE v2_job_status SET + flow_status = jsonb_set( + jsonb_set(flow_status, ARRAY['failure_module', 'job'], to_jsonb($1::UUID::TEXT)), + ARRAY['failure_module', 'type'], + to_jsonb('InProgress'::text) + ) + WHERE id = $2", + job_in_progress, + flow + ) + .execute(db) + .await?; + } + } + + Ok(step) +} + +pub async fn update_workflow_as_code_status( + db: &DB, + id: &Uuid, + parent_job: &Uuid, +) -> error::Result<()> { + let _ = sqlx::query_scalar!( + "UPDATE v2_job_status SET + workflow_as_code_status = jsonb_set( + jsonb_set( + COALESCE(workflow_as_code_status, '{}'::jsonb), + array[$1], + COALESCE(workflow_as_code_status->$1, '{}'::jsonb) + ), + array[$1, 'started_at'], + to_jsonb(now()::text) + ) + WHERE id = $2", + id.to_string(), + parent_job + ) + .execute(db) + .warn_after_seconds(5) + .await + .inspect_err(|e| { + tracing::error!( + "Could not update parent job `started_at` in workflow as code status: {}", + e + ) + }); + Ok(()) +} + +// TODO: merge as a CTE +#[tracing::instrument(level = "trace", skip_all)] +async fn get_step_of_flow_status(db: &DB, id: Uuid) -> error::Result { + let r = sqlx::query!( + "SELECT (flow_status->'step')::integer as step, jsonb_array_length(flow_status->'modules') as len + FROM v2_job_status WHERE id = $1", + id + ) + .fetch_one(db) + .await + .map_err(|e| Error::internal_err(format!("fetching step flow status: {e:#}")))?; + + if let Some(step) = r.step { + Ok(Step::from_i32_and_len(step, r.len.unwrap_or(0) as usize)) + } else { + Err(Error::internal_err("step is null".to_string())) + } +} diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 6b7d04b8d7..5e744e6d7c 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -7,7 +7,7 @@ */ use std::fmt; -use std::{borrow::Borrow, collections::HashMap, sync::Arc, vec}; +use std::{collections::HashMap, sync::Arc, vec}; use anyhow::Context; use async_recursion::async_recursion; @@ -29,7 +29,9 @@ use uuid::Uuid; use windmill_audit::audit_ee::{audit_log, AuditAuthor}; use windmill_audit::ActionKind; +use windmill_common::auth::JobPerms; use windmill_common::utils::now_from_db; +use windmill_common::worker::{Connection, SCRIPT_TOKEN_EXPIRY}; use windmill_common::{ auth::{fetch_authed_from_permissioned_as, permissioned_as_to_username}, cache::{self, FlowData}, @@ -49,7 +51,7 @@ use windmill_common::{ users::{SUPERADMIN_NOTIFICATION_EMAIL, SUPERADMIN_SECRET_EMAIL}, utils::{not_found_if_none, report_critical_error, StripPath, WarnAfterExt}, worker::{ - to_raw_value, CLOUD_HOSTED, DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES, + to_raw_value, CLOUD_HOSTED, DISABLE_FLOW_SCRIPT, MIN_VERSION_IS_AT_LEAST_1_432, MIN_VERSION_IS_AT_LEAST_1_440, NO_LOGS, WORKER_PULL_QUERIES, WORKER_SUSPENDED_PULL_QUERY, }, @@ -65,8 +67,10 @@ use windmill_common::BASE_URL; #[cfg(feature = "cloud")] use windmill_common::users::SUPERADMIN_SYNC_EMAIL; +use crate::flow_status::{update_flow_status_in_progress, update_workflow_as_code_status}; use crate::jobs_ee::update_concurrency_counter; use crate::schedule::{get_schedule_opt, push_scheduled_job}; +use crate::tags::per_workspace_tag; #[cfg(feature = "prometheus")] lazy_static::lazy_static! { @@ -102,8 +106,6 @@ lazy_static::lazy_static! { .build().unwrap(); - pub static ref JOB_TOKEN: Option = std::env::var("JOB_TOKEN").ok(); - static ref JOB_ARGS_AUDIT_LOGS: bool = std::env::var("JOB_ARGS_AUDIT_LOGS") .ok() .and_then(|x| x.parse().ok()) @@ -125,12 +127,26 @@ const SCHEDULE_ERROR_HANDLER_USER_EMAIL: &str = "schedule_error_handler@windmill #[cfg(any(feature = "enterprise", feature = "cloud"))] const SCHEDULE_RECOVERY_HANDLER_USER_EMAIL: &str = "schedule_recovery_handler@windmill.dev"; -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Serialize, Deserialize)] pub struct CanceledBy { pub username: Option, pub reason: Option, } +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct JobCompleted { + pub job: Arc, + pub result: Arc>, + pub result_columns: Option>, + pub mem_peak: i32, + pub success: bool, + pub cached_res_path: Option, + pub token: String, + pub canceled_by: Option, + pub duration: Option, +} + + pub async fn cancel_single_job<'c>( username: &str, reason: Option, @@ -144,17 +160,19 @@ pub async fn cancel_single_job<'c>( let username = username.to_string(); let w_id = w_id.to_string(); let db = db.clone(); + tracing::info!("cancelling job {:?}", db); let job_running = job_running.clone(); tokio::task::spawn(async move { let reason: String = reason .clone() .unwrap_or_else(|| "unexplicited reasons".to_string()); let e = serde_json::json!({"message": format!("Job canceled: {reason} by {username}"), "name": "Canceled", "reason": reason, "canceler": username}); + append_logs( &job_running.id, w_id.to_string(), format!("canceled by {username}: (force cancel: {force_cancel})"), - &db, + &Connection::from(db.clone()), ) .await; let add_job = add_completed_job_error( @@ -335,7 +353,7 @@ pub async fn append_logs( job_id: &uuid::Uuid, workspace: impl AsRef, logs: impl AsRef, - db: impl Borrow>, + conn: &Connection, ) { if logs.as_ref().is_empty() { return; @@ -350,20 +368,81 @@ pub async fn append_logs( tracing::info!("NO LOGS [{job_id}]: {}", logs.as_ref()); return; } - if let Err(err) = sqlx::query!( - "INSERT INTO job_logs (logs, job_id, workspace_id) VALUES ($1, $2, $3) ON CONFLICT (job_id) DO UPDATE SET logs = concat(job_logs.logs, $1::text)", - logs.as_ref(), - job_id, - workspace.as_ref(), - ) - .execute(db.borrow()) - .warn_after_seconds(1) - .await - { - tracing::error!(%job_id, %err, "error updating logs for large_log job {job_id}: {err}"); + match conn { + Connection::Sql(pool) => { + if let Err(err) = sqlx::query!( + "INSERT INTO job_logs (logs, job_id, workspace_id) VALUES ($1, $2, $3) ON CONFLICT (job_id) DO UPDATE SET logs = concat(job_logs.logs, $1::text)", + logs.as_ref(), + job_id, + workspace.as_ref(), + ) + .execute(pool) + .warn_after_seconds(1) + .await + { + tracing::error!(%job_id, %err, "error updating logs for job {job_id}: {err}"); + } + } + Connection::Http(client) => { + if let Err(e) = client + .post::<_, String>( + &format!("/api/w/{}/agent_workers/push_logs/{}", workspace.as_ref(), job_id), + &logs.as_ref(), + ) + .await { + tracing::error!(%job_id, %e, "error sending logs for job {job_id}: {e}"); + }; + } } } +pub async fn push_init_job<'c>( + db: &Pool, + content: String, + worker_name: &str, +) -> error::Result { + let tx = PushIsolationLevel::IsolatedRoot(db.clone()); + let ehm = HashMap::new(); + let (uuid, inner_tx) = push( + &db, + tx, + "admins", + windmill_common::jobs::JobPayload::Code(windmill_common::jobs::RawCode { + hash: None, + content, + path: Some(format!("init_script_{worker_name}")), + language: ScriptLang::Bash, + lock: None, + custom_concurrency_key: None, + concurrent_limit: None, + concurrency_time_window_s: None, + cache_ttl: None, + dedicated_worker: None, + }), + PushArgs::from(&ehm), + worker_name, + "worker@windmill.dev", + SUPERADMIN_SECRET_EMAIL.to_string(), + None, + None, + None, + None, + None, + false, + true, + None, + true, + Some("init_script".to_string()), + None, + None, + None, + None, + ) + .await?; + inner_tx.commit().await?; + Ok(uuid) +} + pub async fn cancel_persistent_script_jobs<'c>( username: &str, reason: Option, @@ -877,11 +956,10 @@ pub async fn add_completed_job( tracing::debug!("decremented concurrency counter"); } - if JOB_TOKEN.is_none() { sqlx::query!("DELETE FROM job_perms WHERE job_id = $1", job_id) .execute(&mut *tx) .await?; - } + tx.commit().await?; @@ -1909,7 +1987,7 @@ impl fmt::Display for TriggerKind { } } -#[derive(sqlx::FromRow, Debug, Clone)] +#[derive(sqlx::FromRow, Debug, Clone, Serialize, Deserialize)] pub struct MiniPulledJob { pub workspace_id: String, pub id: Uuid, @@ -2011,6 +2089,10 @@ impl MiniPulledJob { self.kind.is_flow() } + pub fn is_dependency(&self) -> bool { + self.kind.is_dependency() + } + pub fn schedule_path(&self) -> Option { if self .trigger_kind @@ -2022,9 +2104,35 @@ impl MiniPulledJob { None } } + + + pub async fn mark_as_started_if_step(&self, db: &DB) -> Result<(), Error> { + if self.is_flow_step() { + let _ = update_flow_status_in_progress( + db, + &self.workspace_id, + self.parent_job + .ok_or_else(|| Error::internal_err(format!("expected parent job")))?, + self.id, + ) + .warn_after_seconds(5) + .await?; + } else if let Some(parent_job) = self.parent_job { + let _ = update_workflow_as_code_status( + db, + &self.id, + &parent_job, + ) + .await?; + } + Ok(()) + } + } -#[derive(sqlx::FromRow, Debug, Clone)] + + +#[derive(sqlx::FromRow, Debug, Clone, Serialize, Deserialize)] pub struct PulledJob { #[sqlx(flatten)] pub job: MiniPulledJob, @@ -2040,6 +2148,95 @@ pub struct PulledJob { pub permissioned_as_folders: Option>, } +#[derive(Serialize, Deserialize)] +pub enum PrecomputedAgentInfo { + Bun { local: String, remote: String }, + Python { py_version: Option, requirements: Option }, +} + +#[derive(Serialize, Deserialize)] +pub struct JobAndPerms { + pub job: MiniPulledJob, + pub raw_code: Option, + pub raw_flow: Option>>, + pub raw_lock: Option, + pub parent_runnable_path: Option, + pub token: String, + pub precomputed_agent_info: Option, +} +impl PulledJob { + pub async fn get_job_and_perms(self, db: &DB) -> JobAndPerms { + let job_perms = match ( + self.permissioned_as_email, + self.permissioned_as_username, + self.permissioned_as_is_admin, + self.permissioned_as_is_operator, + self.permissioned_as_groups, + self.permissioned_as_folders, + ) { + ( + Some(email), + Some(username), + Some(is_admin), + Some(is_operator), + Some(groups), + Some(folders), + ) => Some(JobPerms { + email, + username, + is_admin, + is_operator, + groups, + folders, + }), + _ => None, + }; + + let token = create_token(&db, &self.job, job_perms).await; + JobAndPerms { + job: self.job, + raw_code: self.raw_code, + raw_flow: self.raw_flow, + raw_lock: self.raw_lock, + parent_runnable_path: self.parent_runnable_path, + token, + precomputed_agent_info: None, + } + } +} + +// struct Permission +pub async fn create_token(db: &DB, job: &MiniPulledJob, perms: Option) -> String { + // skipping test runs + if job.workspace_id != "" { + let label = if job.permissioned_as != format!("u/{}", job.created_by) + && job.permissioned_as != job.created_by + { + format!("ephemeral-script-end-user-{}", job.created_by) + } else { + "ephemeral-script".to_string() + }; + windmill_common::auth::create_token_for_owner( + db, + &job.workspace_id, + &job.permissioned_as, + &label, + *SCRIPT_TOKEN_EXPIRY, + &job.permissioned_as_email, + &job.id, + perms, + ) + .warn_after_seconds(5) + .await + .expect("could not create job token") + } else { + return "".to_string(); + } +} + + + + impl std::ops::Deref for PulledJob { type Target = MiniPulledJob; fn deref(&self) -> &Self::Target { @@ -2097,23 +2294,68 @@ pub async fn get_mini_pulled_job<'c>( Ok(job) } +#[derive(Serialize, Deserialize, Debug)] +pub struct PulledJobResult { + pub job: Option, + pub suspended: bool, +} + + + pub async fn pull( db: &Pool, suspend_first: bool, worker_name: &str, -) -> windmill_common::error::Result<(Option, bool)> { + query_o: Option<(String, String)>, +) -> windmill_common::error::Result { loop { + if let Some((query_suspended, query_no_suspend)) = query_o.as_ref() { + let njob = { + let job = sqlx::query_as::<_, PulledJob>(query_suspended) + .bind(worker_name) + .fetch_optional(db) + .await?; + if let Some(job) = job { + PulledJobResult { job: Some(job), suspended: true } + } else { + let job = sqlx::query_as::<_, PulledJob>(query_no_suspend) + .bind(worker_name) + .fetch_optional(db) + .await?; + PulledJobResult { job, suspended: false } + } + }; + if let Some(job) = njob.job.as_ref() { + if job.is_flow() || job.is_dependency() { + let per_workspace = per_workspace_tag(&job.workspace_id).await; + let base_tag = if job.is_flow() { + "flow".to_string() + } else { + "dependency".to_string() + }; + let tag = if per_workspace { + format!("{}-{}", base_tag, job.workspace_id) + } else { + base_tag + }; + sqlx::query!("UPDATE v2_job_queue SET tag = $1, running = false WHERE id = $2", tag, job.id).execute(db).await?; + continue; + } + } + return Ok(njob); + }; let (job, suspended) = pull_single_job_and_mark_as_running_no_concurrency_limit( - db, - suspend_first, - worker_name, - ) - .await?; + db, + suspend_first, + worker_name, + ) + .await?; let Some(job) = job else { - return Ok((None, suspended)); + return Ok(PulledJobResult { job: None, suspended }); }; + let has_concurent_limit = job.concurrent_limit.is_some(); #[cfg(not(feature = "enterprise"))] @@ -2134,7 +2376,7 @@ pub async fn pull( if METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed) { QUEUE_PULL_COUNT.inc(); } - return Ok((Option::Some(pulled_job), suspended)); + return Ok(PulledJobResult { job: Some(pulled_job), suspended }); } let job_concurrency_key = match concurrency_key(db, &pulled_job.id).await { @@ -2184,7 +2426,7 @@ pub async fn pull( if METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed) { QUEUE_PULL_COUNT.inc(); } - return Ok((Option::Some(pulled_job), suspended)); + return Ok(PulledJobResult { job: Some(pulled_job), suspended }); } let job_script_path = pulled_job.runnable_path.clone().unwrap_or_default(); @@ -2273,7 +2515,13 @@ pub async fn pull( "\nRe-scheduled job to {estimated_next_schedule_timestamp} due to concurrency limits with key {job_concurrency_key} and limit {job_custom_concurrent_limit} in the last {job_custom_concurrency_time_window_s} seconds (min_started_at: {min_started_at_or_now}, avg script duration: {:?}, number of time windows full: {})\n", avg_script_duration, i ); - let _ = append_logs(&job_uuid, &pulled_job.workspace_id, job_log_event, db).await; + let _ = append_logs( + &job_uuid, + &pulled_job.workspace_id, + job_log_event, + &Connection::from(db.clone()), + ) + .await; sqlx::query!( " @@ -3811,14 +4059,6 @@ pub async fn push<'c, 'd>( .map(|e| (Some(e.0), e.1)) .unwrap_or_else(|| (None, None)); - let per_workspace_workspaces = DEFAULT_TAGS_WORKSPACES.read().await; - let per_workspace = DEFAULT_TAGS_PER_WORKSPACE.load(std::sync::atomic::Ordering::Relaxed) - && (per_workspace_workspaces.is_none() - || per_workspace_workspaces - .as_ref() - .unwrap() - .contains(&workspace_id.to_string())); - let tag = if dedicated_worker.is_some_and(|x| x) { format!( "{}:{}{}", @@ -3836,6 +4076,7 @@ pub async fn push<'c, 'd>( } let interpolated_tag = tag.map(|x| interpolate_args(x, &args, workspace_id)); + let per_workspace = per_workspace_tag(&workspace_id).await; let default = || { let ntag = if job_kind.is_flow() || job_kind == JobKind::Identity { @@ -4019,54 +4260,53 @@ pub async fn push<'c, 'd>( QUEUE_PUSH_COUNT.inc(); } - if JOB_TOKEN.is_none() { - let job_authed = match authed { - Some(authed) - if authed.email == email - && authed.username == permissioned_as_to_username(&permissioned_as) => - { - authed.clone() - } - _ => { - if authed.is_some() { - tracing::warn!("Authed passed to push is not the same as permissioned_as, refetching directly permissions for job {job_id}...") - } - fetch_authed_from_permissioned_as( - permissioned_as.clone(), - email.to_string(), - workspace_id, - _db, - ) - .await - .map_err(|e| { - Error::internal_err(format!( - "Could not get permissions directly for job {job_id}: {e:#}" - )) - })? - } - }; - - let folders = job_authed - .folders - .iter() - .filter_map(|x| serde_json::to_value(x).ok()) - .collect::>(); - - if let Err(err) = sqlx::query!("INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id) - values ($1, $2, $3, $4, $5, $6, $7, $8) - ON CONFLICT (job_id) DO UPDATE SET email = $2, username = $3, is_admin = $4, is_operator = $5, folders = $6, groups = $7, workspace_id = $8", - job_id, - job_authed.email, - job_authed.username, - job_authed.is_admin, - job_authed.is_operator, - folders.as_slice(), - job_authed.groups.as_slice(), - workspace_id, - ).execute(&mut *tx).await { - tracing::error!("Could not insert job_perms for job {job_id}: {err:#}"); + let job_authed = match authed { + Some(authed) + if authed.email == email + && authed.username == permissioned_as_to_username(&permissioned_as) => + { + authed.clone() } + _ => { + if authed.is_some() { + tracing::warn!("Authed passed to push is not the same as permissioned_as, refetching direclty permissions for job {job_id}...") + } + fetch_authed_from_permissioned_as( + permissioned_as.clone(), + email.to_string(), + workspace_id, + _db, + ) + .await + .map_err(|e| { + Error::internal_err(format!( + "Could not get permissions directly for job {job_id}: {e:#}" + )) + })? + } + }; + + let folders = job_authed + .folders + .iter() + .filter_map(|x| serde_json::to_value(x).ok()) + .collect::>(); + + if let Err(err) = sqlx::query!("INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id) + values ($1, $2, $3, $4, $5, $6, $7, $8) + ON CONFLICT (job_id) DO UPDATE SET email = $2, username = $3, is_admin = $4, is_operator = $5, folders = $6, groups = $7, workspace_id = $8", + job_id, + job_authed.email, + job_authed.username, + job_authed.is_admin, + job_authed.is_operator, + folders.as_slice(), + job_authed.groups.as_slice(), + workspace_id, + ).execute(&mut *tx).await { + tracing::error!("Could not insert job_perms for job {job_id}: {err:#}"); } + { let uuid_string = job_id.to_string(); @@ -4177,7 +4417,7 @@ async fn restarted_flows_resolution( })?; let flow_data = cache::job::fetch_flow(db, row.job_kind, row.script_hash) - .or_else(|_| cache::job::fetch_preview_flow(db, &completed_flow_id, row.raw_flow)) + .or_else(|_| cache::job::fetch_preview_flow(db.into(), &completed_flow_id, row.raw_flow)) .await?; let flow_value = flow_data.value(); let flow_status = row @@ -4330,3 +4570,78 @@ async fn restarted_flows_resolution( flow_status.cleanup_module, )) } + + +#[derive(Serialize, Deserialize)] +pub struct SameWorkerPayload { + pub job_id: Uuid, + pub recoverable: bool, +} + +pub async fn get_same_worker_job( + db: &DB, + same_worker_job: &SameWorkerPayload, +) -> windmill_common::error::Result> { + sqlx::query_as::<_, PulledJob>( + "WITH ping AS ( + UPDATE v2_job_runtime SET ping = NOW() WHERE id = $1 + ), + started_at AS ( + UPDATE v2_job_queue SET started_at = NOW() WHERE id = $1 + ) + SELECT + v2_job_queue.workspace_id, + v2_job_queue.id, + v2_job.args, + v2_job.parent_job, + v2_job.created_by, + v2_job_queue.started_at, + scheduled_for, + v2_job.runnable_path, + v2_job.kind, + v2_job.runnable_id, + v2_job_queue.canceled_reason, + v2_job_queue.canceled_by, + v2_job.permissioned_as, + v2_job.permissioned_as_email, + v2_job_status.flow_status, + v2_job.tag, + v2_job.script_lang, + v2_job.same_worker, + v2_job.pre_run_error, + v2_job.concurrent_limit, + v2_job.concurrency_time_window_s, + v2_job.flow_innermost_root_job, + v2_job.timeout, + v2_job.flow_step_id, + v2_job.cache_ttl, + v2_job_queue.priority, + v2_job.preprocessed, + v2_job.script_entrypoint_override, + v2_job.trigger, + v2_job.trigger_kind, + v2_job.visible_to_owner, + v2_job.raw_code, + v2_job.raw_lock, + v2_job.raw_flow, + pj.runnable_path as parent_runnable_path, + p.email as permissioned_as_email, p.username as permissioned_as_username, p.is_admin as permissioned_as_is_admin, + p.is_operator as permissioned_as_is_operator, p.groups as permissioned_as_groups, p.folders as permissioned_as_folders + FROM v2_job_queue + INNER JOIN v2_job ON v2_job.id = v2_job_queue.id + LEFT JOIN v2_job_status ON v2_job_status.id = v2_job_queue.id + LEFT JOIN job_perms p ON p.job_id = v2_job.id + LEFT JOIN v2_job pj ON v2_job.parent_job = pj.id + WHERE v2_job_queue.id = $1 +", + ) + .bind(same_worker_job.job_id) + .fetch_optional(db) + .await + .map_err(|e| { + Error::internal_err(format!( + "Impossible to fetch same_worker job {}: {}", + same_worker_job.job_id, e + )) + }) +} \ No newline at end of file diff --git a/backend/windmill-queue/src/lib.rs b/backend/windmill-queue/src/lib.rs index c2d022d16a..2496bdf818 100644 --- a/backend/windmill-queue/src/lib.rs +++ b/backend/windmill-queue/src/lib.rs @@ -10,3 +10,5 @@ mod jobs; pub mod jobs_ee; pub mod schedule; pub use jobs::*; +pub mod flow_status; +pub mod tags; diff --git a/backend/windmill-queue/src/tags.rs b/backend/windmill-queue/src/tags.rs new file mode 100644 index 0000000000..6cf6620039 --- /dev/null +++ b/backend/windmill-queue/src/tags.rs @@ -0,0 +1,11 @@ +use windmill_common::worker::{DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES}; + +pub async fn per_workspace_tag(workspace_id: &str) -> bool { + let per_workspace_workspaces = DEFAULT_TAGS_WORKSPACES.read().await; + DEFAULT_TAGS_PER_WORKSPACE.load(std::sync::atomic::Ordering::Relaxed) + && (per_workspace_workspaces.is_none() + || per_workspace_workspaces + .as_ref() + .unwrap() + .contains(&workspace_id.to_string())) +} diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml index 510a33a645..cad3ccc84a 100644 --- a/backend/windmill-worker/Cargo.toml +++ b/backend/windmill-worker/Cargo.toml @@ -103,6 +103,7 @@ urlencoding.workspace = true nix.workspace = true bytes.workspace = true reqwest.workspace = true +reqwest-middleware.workspace = true hex.workspace = true tiberius = { workspace = true, optional = true } tokio-util = { workspace = true, optional = true } diff --git a/backend/windmill-worker/src/agent_workers.rs b/backend/windmill-worker/src/agent_workers.rs new file mode 100644 index 0000000000..192ae7ae37 --- /dev/null +++ b/backend/windmill-worker/src/agent_workers.rs @@ -0,0 +1,31 @@ +use uuid::Uuid; +use windmill_common::{agent_workers::QueueInitJob, worker::HttpClient}; +use windmill_queue::{JobAndPerms, JobCompleted}; + +pub async fn queue_init_job(client: &HttpClient, content: &str) -> anyhow::Result { + client + .post( + "/api/agent_workers/queue_init_job", + &QueueInitJob { content: content.to_string() }, + ) + .await + .and_then(|x: String| Uuid::parse_str(&x).map_err(|e| anyhow::anyhow!(e))) +} + +pub async fn pull_job(client: &HttpClient) -> anyhow::Result> { + client.post("/api/agent_workers/pull_job", &()).await +} + +pub async fn send_result(client: &HttpClient, jc: JobCompleted) -> anyhow::Result { + client + .post( + &format!( + "/api/w/{}/agent_workers/send_result/{}", + jc.job.workspace_id, jc.job.id + ), + &jc, + ) + .await +} + +pub const UPDATE_PING_URL: &str = "/api/agent_workers/update_ping"; diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index 5699c44932..7e0d1bd09c 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -11,7 +11,9 @@ use tokio::process::Command; use uuid::Uuid; use windmill_common::{ error, - worker::{to_raw_value, write_file, write_file_at_user_defined_location, WORKER_CONFIG}, + worker::{ + to_raw_value, write_file, write_file_at_user_defined_location, Connection, WORKER_CONFIG, + }, }; use windmill_queue::MiniPulledJob; @@ -46,7 +48,7 @@ async fn handle_ansible_python_deps( ansible_reqs: Option<&AnsibleRequirements>, w_id: &str, job_id: &Uuid, - db: &sqlx::Pool, + conn: &Connection, worker_name: &str, worker_dir: &str, mem_peak: &mut i32, @@ -77,7 +79,7 @@ async fn handle_ansible_python_deps( mem_peak, canceled_by, job_dir, - db, + conn, worker_name, w_id, &mut Some(occupancy_metrics), @@ -103,7 +105,7 @@ async fn handle_ansible_python_deps( w_id, mem_peak, canceled_by, - db, + conn, worker_name, job_dir, worker_dir, @@ -124,7 +126,7 @@ async fn install_galaxy_collections( w_id: &str, mem_peak: &mut i32, canceled_by: &mut Option, - db: &sqlx::Pool, + conn: &Connection, occupancy_metrics: &mut OccupancyMetrics, ) -> anyhow::Result<()> { write_file(job_dir, "requirements.yml", collections_yml)?; @@ -133,7 +135,7 @@ async fn install_galaxy_collections( job_id, w_id, "\n\n--- ANSIBLE GALAXY INSTALL ---\n".to_string(), - db, + conn, ) .await; let mut galaxy_command = Command::new(ANSIBLE_GALAXY_PATH.as_str()); @@ -159,7 +161,7 @@ async fn install_galaxy_collections( let child = start_child_process(galaxy_command, ANSIBLE_GALAXY_PATH.as_str()).await?; handle_child( job_id, - db, + conn, mem_peak, canceled_by, child, @@ -185,7 +187,7 @@ pub async fn handle_ansible_job( job: &MiniPulledJob, mem_peak: &mut i32, canceled_by: &mut Option, - db: &sqlx::Pool, + conn: &Connection, client: &AuthedClient, parent_runnable_path: Option, inner_content: &String, @@ -201,7 +203,7 @@ pub async fn handle_ansible_job( )?; let (logs, reqs, playbook) = windmill_parser_yaml::parse_ansible_reqs(inner_content)?; - append_logs(&job.id, &job.workspace_id, logs, db).await; + append_logs(&job.id, &job.workspace_id, logs, conn).await; write_file(job_dir, "main.yml", &playbook)?; let additional_python_paths = handle_ansible_python_deps( @@ -210,7 +212,7 @@ pub async fn handle_ansible_job( reqs.as_ref(), &job.workspace_id, &job.id, - db, + conn, worker_name, worker_dir, mem_peak, @@ -227,7 +229,7 @@ pub async fn handle_ansible_job( args.insert(name.clone(), to_raw_value(path)); } } - if let Some(x) = transform_json(client, &job.workspace_id, &args, job, db).await? { + if let Some(x) = transform_json(client, &job.workspace_id, &args, job, conn).await? { write_file( job_dir, "args.json", @@ -267,16 +269,18 @@ pub async fn handle_ansible_job( let mut nsjail_extra_mounts = vec![]; if let Some(r) = reqs { - nsjail_extra_mounts = create_file_resources( - &job.id, - &job.workspace_id, - job_dir, - interpolated_args.as_ref(), - &r, - &client, - db, - ) - .await?; + if let Some(db) = conn.as_sql() { + nsjail_extra_mounts = create_file_resources( + &job.id, + &job.workspace_id, + job_dir, + interpolated_args.as_ref(), + &r, + &client, + db, + ) + .await?; + } if let Some(collections) = r.collections { install_galaxy_collections( @@ -287,7 +291,7 @@ pub async fn handle_ansible_job( &job.workspace_id, mem_peak, canceled_by, - db, + conn, occupancy_metrics, ) .await?; @@ -297,7 +301,7 @@ pub async fn handle_ansible_job( &job.id, &job.workspace_id, "\n\n--- ANSIBLE PLAYBOOK EXECUTION ---\n".to_string(), - db, + conn, ) .await; let ansible_cfg_content = format!( @@ -313,7 +317,7 @@ remote_tmp={job_dir}/.ansible/tmp write_file(job_dir, "ansible.cfg", &ansible_cfg_content)?; let mut reserved_variables = - get_reserved_variables(job, &client.token, db, parent_runnable_path).await?; + get_reserved_variables(job, &client.token, conn, parent_runnable_path).await?; let additional_python_paths_folders = additional_python_paths.join(":"); if !*DISABLE_NSJAIL { @@ -422,7 +426,7 @@ fi handle_child( &job.id, - db, + conn, mem_peak, canceled_by, child, @@ -564,7 +568,7 @@ async fn create_file_resources( file_res.target_path, file_res.resource_path )); } - append_logs(job_id, w_id, logs, db).await; + append_logs(job_id, w_id, logs, &Connection::Sql(db.clone())).await; Ok(nsjail_mounts) } diff --git a/backend/windmill-worker/src/bash_executor.rs b/backend/windmill-worker/src/bash_executor.rs index 85c20fd6fd..7a7a78282c 100644 --- a/backend/windmill-worker/src/bash_executor.rs +++ b/backend/windmill-worker/src/bash_executor.rs @@ -15,12 +15,9 @@ use tokio::process::Command; use uuid::Uuid; use windmill_common::{ error::Error, - worker::{to_raw_value, write_file}, + worker::{to_raw_value, write_file, Connection}, }; -#[cfg(feature = "dind")] -use windmill_common::DB; - #[cfg(feature = "dind")] use windmill_common::error::to_anyhow; @@ -63,7 +60,7 @@ pub async fn handle_bash_job( mem_peak: &mut i32, canceled_by: &mut Option, job: &MiniPulledJob, - db: &sqlx::Pool, + conn: &Connection, client: &AuthedClient, parent_runnable_path: Option, content: &str, @@ -81,7 +78,7 @@ pub async fn handle_bash_job( if annotation.docker { logs1.push_str("docker mode\n"); } - append_logs(&job.id, &job.workspace_id, logs1, db).await; + append_logs(&job.id, &job.workspace_id, logs1, &conn).await; write_file(job_dir, "main.sh", &format!("set -e\n{content}"))?; let script = format!( @@ -137,10 +134,10 @@ exit $exit_status write_file(job_dir, "wrapper.sh", &script)?; let mut reserved_variables = - get_reserved_variables(job, &client.token, db, parent_runnable_path).await?; + get_reserved_variables(job, &client.token, conn, parent_runnable_path).await?; reserved_variables.insert("RUST_LOG".to_string(), "info".to_string()); - let args = build_args_map(job, client, db).await?.map(Json); + let args = build_args_map(job, client, conn).await?.map(Json); let job_args = if args.is_some() { args.as_ref() } else { @@ -215,7 +212,7 @@ exit $exit_status }; handle_child( &job.id, - db, + conn, mem_peak, canceled_by, child, @@ -235,7 +232,7 @@ exit $exit_status return handle_docker_job( job.id, &job.workspace_id, - db, + conn, job.timeout, mem_peak, canceled_by, @@ -280,7 +277,7 @@ exit $exit_status async fn handle_docker_job( job_id: Uuid, workspace_id: &str, - db: &DB, + conn: &Connection, job_timeout: Option, mem_peak: &mut i32, canceled_by: &mut Option, @@ -315,7 +312,7 @@ async fn handle_docker_job( let ncontainer_id = container_id.to_string(); let w_id = workspace_id.to_string(); let j_id = job_id.clone(); - let db2 = db.clone(); + let conn2 = conn.clone(); let (tx, mut rx) = tokio::sync::broadcast::channel::<()>(1); let mut killpill_rx = killpill_rx.resubscribe(); @@ -337,7 +334,7 @@ async fn handle_docker_job( log = log_stream.next() => { match log { Some(Ok(log)) => { - append_logs(&j_id, w_id.clone(), log.to_string(), db2.clone()).await; + append_logs(&j_id, w_id.clone(), log.to_string(), &conn2).await; } Some(Err(e)) => { tracing::error!("Error getting logs: {:?}", e); @@ -371,7 +368,7 @@ async fn handle_docker_job( let result = run_future_with_polling_update_job_poller( job_id, job_timeout, - db, + conn, mem_peak, canceled_by, wait_f, @@ -472,7 +469,7 @@ pub async fn handle_powershell_job( mem_peak: &mut i32, canceled_by: &mut Option, job: &MiniPulledJob, - db: &sqlx::Pool, + db: &Connection, client: &AuthedClient, parent_runnable_path: Option, content: &str, @@ -484,7 +481,7 @@ pub async fn handle_powershell_job( occupancy_metrics: &mut OccupancyMetrics, ) -> Result, Error> { let pwsh_args = { - let args = build_args_map(job, client, db).await?.map(Json); + let args = build_args_map(job, client, &db).await?.map(Json); let job_args = if args.is_some() { args.as_ref() } else { diff --git a/backend/windmill-worker/src/bigquery_executor.rs b/backend/windmill-worker/src/bigquery_executor.rs index 28aa1398d3..9cc9fd6c90 100644 --- a/backend/windmill-worker/src/bigquery_executor.rs +++ b/backend/windmill-worker/src/bigquery_executor.rs @@ -5,6 +5,7 @@ use futures::{FutureExt, TryFutureExt}; use reqwest::Client; use serde_json::{json, value::RawValue, Value}; use windmill_common::error::to_anyhow; +use windmill_common::worker::Connection; use windmill_common::{error::Error, worker::to_raw_value}; use windmill_parser_sql::{ parse_bigquery_sig, parse_db_resource, parse_sql_blocks, parse_sql_statement_named_params, @@ -209,14 +210,14 @@ pub async fn do_bigquery( job: &MiniPulledJob, client: &AuthedClient, query: &str, - db: &sqlx::Pool, + conn: &Connection, mem_peak: &mut i32, canceled_by: &mut Option, worker_name: &str, column_order: &mut Option>, occupancy_metrics: &mut OccupancyMetrics, ) -> windmill_common::error::Result> { - let bigquery_args = build_args_values(job, client, db).await?; + let bigquery_args = build_args_values(job, client, conn).await?; let inline_db_res_path = parse_db_resource(&query); @@ -252,7 +253,7 @@ pub async fn do_bigquery( .map_err(|e| Error::ExecutionErr(e.to_string()))?; let (timeout_duration, _, _) = - resolve_job_timeout(&db, &job.workspace_id, job.id, job.timeout).await; + resolve_job_timeout(&conn, &job.workspace_id, job.id, job.timeout).await; let timeout_ms = timeout_duration.as_millis() as u64; let http_client = build_http_client(timeout_duration)?; @@ -366,7 +367,7 @@ pub async fn do_bigquery( let r = run_future_with_polling_update_job_poller( job.id, job.timeout, - db, + conn, mem_peak, canceled_by, result_f.map_err(to_anyhow), diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 326feb3e2b..91284c9d41 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -9,7 +9,7 @@ use serde_json::value::RawValue; use uuid::Uuid; use windmill_parser_ts::remove_pinned_imports; -use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; +use windmill_queue::{append_logs, CanceledBy, MiniPulledJob, PrecomputedAgentInfo}; #[cfg(feature = "enterprise")] use crate::common::build_envs_map; @@ -42,7 +42,7 @@ use windmill_common::{ error::{self, Result}, get_latest_hash_for_path, scripts::ScriptLang, - worker::{exists_in_cache, save_cache, write_file, DISABLE_BUNDLING}, + worker::{exists_in_cache, save_cache, write_file, Connection, DISABLE_BUNDLING}, DB, }; @@ -96,7 +96,7 @@ pub async fn gen_bun_lockfile( canceled_by: &mut Option, job_id: &Uuid, w_id: &str, - db: Option<&sqlx::Pool>, + db: Option<&Connection>, token: &str, script_path: &str, job_dir: &str, @@ -273,7 +273,7 @@ pub async fn install_bun_lockfile( canceled_by: &mut Option, job_id: &Uuid, w_id: &str, - db: Option<&sqlx::Pool>, + db: Option<&Connection>, job_dir: &str, worker_name: &str, common_bun_proc_envs: HashMap, @@ -488,7 +488,7 @@ pub async fn generate_wrapper_mjs( w_id: &str, job_id: &Uuid, worker_name: &str, - db: &sqlx::Pool, + db: &Connection, timeout: Option, mem_peak: &mut i32, canceled_by: &mut Option, @@ -538,7 +538,7 @@ pub async fn generate_bun_bundle( w_id: &str, job_id: &Uuid, worker_name: &str, - db: Option>, + db: Option<&Connection>, timeout: Option, mem_peak: &mut i32, canceled_by: &mut Option, @@ -679,21 +679,15 @@ pub async fn prebundle_bun_script( script_path: &str, job_id: &Uuid, w_id: &str, - db: Option, + db: Option<&DB>, job_dir: &str, base_internal_url: &str, worker_name: &str, token: &str, occupancy_metrics: &mut Option<&mut OccupancyMetrics>, ) -> Result<()> { - let (local_path, remote_path) = compute_bundle_local_and_remote_path( - inner_content, - lockfile, - script_path, - db.clone(), - w_id, - ) - .await; + let (local_path, remote_path) = + compute_bundle_local_and_remote_path(inner_content, lockfile, script_path, db, w_id).await; if exists_in_cache(&local_path, &remote_path).await { return Ok(()); } @@ -727,7 +721,7 @@ pub async fn prebundle_bun_script( w_id, job_id, worker_name, - db.clone(), + db.map(|x| Connection::from(x.clone())).as_ref(), None, &mut 0, &mut None, @@ -755,11 +749,11 @@ async fn get_script_import_updated_at(db: &DB, w_id: &str, script_path: &str) -> Ok(last_updated_at.to_string()) } -async fn compute_bundle_local_and_remote_path( +pub async fn compute_bundle_local_and_remote_path( inner_content: &str, requirements_o: Option<&String>, script_path: &str, - db: Option, + db: Option<&DB>, w_id: &str, ) -> (String, String) { let mut input_src = format!( @@ -827,7 +821,7 @@ pub async fn handle_bun_job( mem_peak: &mut i32, canceled_by: &mut Option, job: &MiniPulledJob, - db: &sqlx::Pool, + conn: &Connection, client: &AuthedClient, parent_runnable_path: Option, job_dir: &str, @@ -838,6 +832,7 @@ pub async fn handle_bun_job( shared_mount: &str, new_args: &mut Option>>, occupancy_metrics: &mut OccupancyMetrics, + precomputed_agent_info: Option, ) -> error::Result> { let mut annotation = windmill_common::worker::TypeScriptAnnotations::parse(inner_content); @@ -846,14 +841,29 @@ pub async fn handle_bun_job( && !*DISABLE_BUNDLING && codebase.is_none() { - let (local_path, remote_path) = compute_bundle_local_and_remote_path( - inner_content, - requirements_o, - job.runnable_path(), - Some(db.clone()), - &job.workspace_id, - ) - .await; + let (local_path, remote_path) = match conn { + Connection::Sql(db) => { + compute_bundle_local_and_remote_path( + inner_content, + requirements_o, + job.runnable_path(), + Some(db), + &job.workspace_id, + ) + .await + } + Connection::Http(_) => { + let (local_path, remote_path) = match precomputed_agent_info { + Some(PrecomputedAgentInfo::Bun { local, remote }) => (local, remote), + _ => { + return Err(error::Error::ExecutionErr( + "bun bundle is missing the precomputed agent info".to_string(), + )) + } + }; + (local_path, remote_path) + } + }; let (cache, logs) = windmill_common::worker::load_cache(&local_path, &remote_path, false).await; @@ -920,7 +930,7 @@ pub async fn handle_bun_job( canceled_by, &job.id, &job.workspace_id, - Some(db), + Some(conn), job_dir, worker_name, common_bun_proc_envs.clone(), @@ -932,13 +942,13 @@ pub async fn handle_bun_job( } else { // if !*DISABLE_NSJAIL || !empty_trusted_deps || has_custom_config_registry { let logs1 = "\n\n--- BUN INSTALL ---\n".to_string(); - append_logs(&job.id, &job.workspace_id, logs1, db).await; + append_logs(&job.id, &job.workspace_id, logs1, conn).await; let _ = gen_bun_lockfile( mem_peak, canceled_by, &job.id, &job.workspace_id, - Some(db), + Some(conn), &client.token, job.runnable_path(), job_dir, @@ -1108,12 +1118,13 @@ try {{ let reserved_variables_args_out_f = async { let args_and_out_f = async { if !annotation.native { - create_args_and_out_file(&client, job, job_dir, db).await?; + create_args_and_out_file(&client, job, job_dir, conn).await?; } Ok(()) as Result<()> }; let reserved_variables_f = async { - let vars = get_reserved_variables(job, &client.token, db, parent_runnable_path).await?; + let vars = + get_reserved_variables(job, &client.token, conn, parent_runnable_path).await?; Ok(vars) as Result> }; let (_, reserved_variables) = tokio::try_join!(args_and_out_f, reserved_variables_f)?; @@ -1176,7 +1187,7 @@ try {{ &job.workspace_id, &job.id, worker_name, - Some(db.clone()), + Some(conn), job.timeout, mem_peak, canceled_by, @@ -1225,7 +1236,7 @@ try {{ &job.workspace_id, &job.id, worker_name, - db, + conn, job.timeout, mem_peak, canceled_by, @@ -1255,7 +1266,7 @@ try {{ .join("\n")); let js_code = read_file_content(&format!("{job_dir}/main.js")).await?; let started_at = Instant::now(); - let args = crate::common::build_args_map(job, client, db) + let args = crate::common::build_args_map(job, client, conn) .await? .map(sqlx::types::Json); let job_args = if args.is_some() { @@ -1264,7 +1275,7 @@ try {{ job.args.as_ref() }; - append_logs(&job.id, &job.workspace_id, format!("{init_logs}\n"), db).await; + append_logs(&job.id, &job.workspace_id, format!("{init_logs}\n"), conn).await; let result = crate::js_eval::eval_fetch_timeout( env_code, @@ -1274,7 +1285,7 @@ try {{ job.script_entrypoint_override.clone(), job.id, job.timeout, - db, + conn, mem_peak, canceled_by, worker_name, @@ -1290,7 +1301,7 @@ try {{ return Ok(result); } } - append_logs(&job.id, &job.workspace_id, init_logs, db).await; + append_logs(&job.id, &job.workspace_id, init_logs, conn).await; //do not cache local dependencies let child = if !*DISABLE_NSJAIL { @@ -1422,7 +1433,7 @@ try {{ handle_child( &job.id, - db, + conn, mem_peak, canceled_by, child, @@ -1532,7 +1543,7 @@ pub async fn start_worker( annotation.nodejs = true; let context = variables::get_reserved_variables( - db, + &Connection::from(db.clone()), w_id, &token, "dedicated_worker@windmill.dev", @@ -1583,7 +1594,7 @@ pub async fn start_worker( &mut canceled_by, &Uuid::nil(), &w_id, - Some(db), + Some(&Connection::from(db.clone())), job_dir, worker_name, common_bun_proc_envs.clone(), @@ -1600,7 +1611,7 @@ pub async fn start_worker( &mut canceled_by, &Uuid::nil(), &w_id, - Some(db), + Some(&Connection::from(db.clone())), token, &script_path, job_dir, @@ -1701,7 +1712,7 @@ for await (const line of Readline.createInterface({{ input: process.stdin }})) { w_id, &Uuid::nil(), worker_name, - db, + &Connection::from(db.clone()), None, &mut mem_peak, &mut canceled_by, diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index 039ff4205b..582b9c3abe 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -20,7 +20,8 @@ use windmill_common::s3_helpers::{ }; use windmill_common::variables::{build_crypt_with_key_suffix, decrypt}; use windmill_common::worker::{ - to_raw_value, write_file, CLOUD_HOSTED, ROOT_CACHE_DIR, WORKER_CONFIG, + to_raw_value, update_ping_for_failed_init_script_query, write_file, Connection, Ping, PingType, + CLOUD_HOSTED, ROOT_CACHE_DIR, WORKER_CONFIG, }; use windmill_common::{ cache::{Cache, RawData}, @@ -40,6 +41,7 @@ use windmill_common::{variables, DB}; use tokio::{io::AsyncWriteExt, process::Child, time::Instant}; +use crate::agent_workers::UPDATE_PING_URL; use crate::{ AuthedClient, DISABLE_NSJAIL, JOB_DEFAULT_TIMEOUT, MAX_RESULT_SIZE, MAX_TIMEOUT_DURATION, PATH_ENV, @@ -48,10 +50,10 @@ use crate::{ pub async fn build_args_map<'a>( job: &'a MiniPulledJob, client: &AuthedClient, - db: &Pool, + conn: &Connection, ) -> error::Result>>> { if let Some(args) = &job.args { - return transform_json(client, &job.workspace_id, &args.0, &job, db).await; + return transform_json(client, &job.workspace_id, &args.0, &job, conn).await; } return Ok(None); } @@ -76,10 +78,10 @@ pub fn check_executor_binary_exists( pub async fn build_args_values( job: &MiniPulledJob, client: &AuthedClient, - db: &Pool, + conn: &Connection, ) -> error::Result> { if let Some(args) = &job.args { - transform_json_as_values(client, &job.workspace_id, &args.0, job, db).await + transform_json_as_values(client, &job.workspace_id, &args.0, job, conn).await } else { Ok(HashMap::new()) } @@ -90,10 +92,10 @@ pub async fn create_args_and_out_file( client: &AuthedClient, job: &MiniPulledJob, job_dir: &str, - db: &Pool, + conn: &Connection, ) -> Result<(), Error> { if let Some(args) = job.args.as_ref() { - if let Some(x) = transform_json(client, &job.workspace_id, &args.0, job, db).await? { + if let Some(x) = transform_json(client, &job.workspace_id, &args.0, job, conn).await? { write_file( job_dir, "args.json", @@ -131,7 +133,7 @@ pub async fn transform_json<'a>( workspace: &str, vs: &'a HashMap>, job: &MiniPulledJob, - db: &Pool, + db: &Connection, ) -> error::Result>>> { let mut has_match = false; for (_, v) in vs { @@ -168,7 +170,7 @@ pub async fn transform_json_as_values<'a>( workspace: &str, vs: &'a HashMap>, job: &MiniPulledJob, - db: &Pool, + db: &Connection, ) -> error::Result> { let mut r: HashMap = HashMap::new(); for (k, v) in vs { @@ -236,7 +238,7 @@ pub async fn transform_json_value( workspace: &str, v: Value, job: &MiniPulledJob, - db: &Pool, + conn: &Connection, ) -> error::Result { match v { Value::String(y) if y.starts_with("$var:") => { @@ -267,20 +269,32 @@ pub async fn transform_json_value( }) } Value::String(y) if y.starts_with("$encrypted:") => { - let encrypted = y.strip_prefix("$encrypted:").unwrap(); + match conn { + Connection::Sql(db) => { + let encrypted = y.strip_prefix("$encrypted:").unwrap(); - let root_job_id = - get_root_job_id(&job.flow_innermost_root_job.unwrap_or_else(|| job.id), db).await?; - let mc = build_crypt_with_key_suffix(&db, &job.workspace_id, &root_job_id.to_string()) - .await?; - decrypt(&mc, encrypted.to_string()).and_then(|x| { - serde_json::from_str(&x).map_err(|e| Error::internal_err(e.to_string())) - }) + let root_job_id = + get_root_job_id(&job.flow_innermost_root_job.unwrap_or_else(|| job.id), db) + .await?; + let mc = build_crypt_with_key_suffix( + &db, + &job.workspace_id, + &root_job_id.to_string(), + ) + .await?; + decrypt(&mc, encrypted.to_string()).and_then(|x| { + serde_json::from_str(&x).map_err(|e| Error::internal_err(e.to_string())) + }) + } + Connection::Http(_) => { + Err(Error::NotFound("Http connection not supported".to_string())) + } + } // let path = y.strip_prefix("$res:").unwrap(); } Value::String(y) if y.starts_with("$") => { - let variables = get_reserved_variables(job, &client.token, &db, None).await?; + let variables = get_reserved_variables(job, &client.token, conn, None).await?; let name = y.strip_prefix("$").unwrap(); @@ -295,7 +309,7 @@ pub async fn transform_json_value( for (a, b) in m.clone().into_iter() { m.insert( a.clone(), - transform_json_value(&a, client, workspace, b, job, &db).await?, + transform_json_value(&a, client, workspace, b, job, conn).await?, ); } Ok(Value::Object(m)) @@ -387,16 +401,21 @@ pub fn capitalize(s: &str) -> String { pub async fn get_reserved_variables( job: &MiniPulledJob, token: &str, - db: &sqlx::Pool, + db: &Connection, parent_runnable_path: Option, ) -> Result, Error> { let flow_path = if parent_runnable_path.is_some() { parent_runnable_path } else if let Some(uuid) = job.parent_job { - sqlx::query_scalar!("SELECT runnable_path FROM v2_job WHERE id = $1", uuid) - .fetch_optional(db) - .await? - .flatten() + match db { + Connection::Sql(db) => { + sqlx::query_scalar!("SELECT runnable_path FROM v2_job WHERE id = $1", uuid) + .fetch_optional(db) + .await? + .flatten() + } + Connection::Http(_) => None, + } } else { None }; @@ -457,26 +476,57 @@ pub fn sizeof_val(v: &serde_json::Value) -> usize { } pub async fn update_worker_ping_for_failed_init_script( - db: &DB, + conn: &Connection, worker_name: &str, last_job_id: Uuid, ) { - if let Err(e) = sqlx::query!( - "UPDATE worker_ping SET - ping_at = now(), - jobs_executed = 1, - current_job_id = $1, - current_job_workspace_id = 'admins' - WHERE worker = $2", - last_job_id, - worker_name - ) - .execute(db) - .await - { - tracing::error!("Error updating worker ping for failed init script: {e:?}"); + match conn { + Connection::Sql(db) => { + if let Err(e) = + update_ping_for_failed_init_script_query(worker_name, last_job_id, db).await + { + tracing::error!("Error updating worker ping for failed init script: {e:?}"); + } + } + Connection::Http(client) => { + if let Err(e) = client + .post::<_, ()>( + UPDATE_PING_URL, + &Ping { + last_job_executed: Some(last_job_id), + last_job_workspace_id: None, + worker_instance: None, + ip: None, + tags: None, + dw: None, + jobs_executed: None, + occupancy_rate: None, + occupancy_rate_15s: None, + occupancy_rate_5m: None, + occupancy_rate_30m: None, + version: None, + vcpus: None, + memory: None, + memory_usage: None, + wm_memory_usage: None, + ping_type: PingType::InitScript, + }, + ) + .await + { + tracing::error!("Error updating worker ping for failed init script: {e:?}"); + } + } } } + +pub fn error_to_value(err: Error) -> serde_json::Value { + match err { + Error::JsonErr(err) => err, + _ => json!({"message": err.to_string(), "name": "InternalErr"}), + } +} + pub struct OccupancyMetrics { pub running_job_started_at: Option, pub total_duration_of_running_jobs: f32, @@ -484,6 +534,13 @@ pub struct OccupancyMetrics { pub start_time: Instant, } +pub struct OccupancyResult { + pub occupancy_rate: f32, + pub occupancy_rate_15s: Option, + pub occupancy_rate_5m: Option, + pub occupancy_rate_30m: Option, +} + impl OccupancyMetrics { pub fn new(start_time: Instant) -> Self { OccupancyMetrics { @@ -494,7 +551,7 @@ impl OccupancyMetrics { } } - pub fn update_occupancy_metrics(&mut self) -> (f32, Option, Option, Option) { + pub fn update_occupancy_metrics(&mut self) -> OccupancyResult { let metrics = self; let current_occupied_duration = metrics .running_job_started_at @@ -545,12 +602,12 @@ impl OccupancyMetrics { .worker_occupancy_rate_history .push((total_occupation, elapsed)); - ( + OccupancyResult { occupancy_rate, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, - ) + } } } @@ -561,15 +618,19 @@ pub async fn start_child_process(mut cmd: Command, executable: &str) -> Result, + _conn: &Connection, _w_id: &str, _job_id: Uuid, custom_timeout_secs: Option, ) -> (Duration, Option, bool) { let mut warn_msg: Option = None; #[cfg(feature = "cloud")] - let cloud_premium_workspace = - *CLOUD_HOSTED && windmill_common::workspaces::is_premium_workspace(_db, _w_id).await; + let cloud_premium_workspace = *CLOUD_HOSTED + && windmill_common::workspaces::is_premium_workspace( + _conn.as_sql().expect("cloud cannot use http connection"), + _w_id, + ) + .await; #[cfg(not(feature = "cloud"))] let cloud_premium_workspace = false; @@ -1013,7 +1074,7 @@ pub async fn par_install_language_dependencies<'a>( job_id: &'a Uuid, w_id: &'a str, worker_name: &'a str, - db: &sqlx::Pool, + conn: &Connection, ) -> anyhow::Result<()> { #[cfg(not(all(feature = "enterprise", feature = "parquet")))] let _ = (platform_agnostic, language_name); @@ -1038,7 +1099,7 @@ pub async fn par_install_language_dependencies<'a>( counter_arc: Arc>, total_to_install: usize, instant: std::time::Instant, - db: Pool, + conn: &Connection, ) { #[cfg(not(all(feature = "enterprise", feature = "parquet")))] { @@ -1074,7 +1135,7 @@ pub async fn par_install_language_dependencies<'a>( if s3_push { " > (S3) " } else { "" }, instant.elapsed().as_millis(), ), - db, + conn, ) .await; // Drop lock, so next print success can fire @@ -1128,18 +1189,13 @@ pub async fn par_install_language_dependencies<'a>( job_id, w_id, format!("\n--- INSTALLATION ---\n\nTo be installed:\n\n"), - db.clone(), + conn, ) .await; to_be_installed_is_used = true; } - windmill_queue::append_logs( - job_id, - w_id, - format!("- {display_name}\n"), - db.clone(), - ) - .await; + windmill_queue::append_logs(job_id, w_id, format!("- {display_name}\n"), conn) + .await; not_installed.push(NotInstalledDependency { path, custom_name, @@ -1164,7 +1220,7 @@ pub async fn par_install_language_dependencies<'a>( job_id, w_id, format!("\nLooking for packages on S3:\n"), - db.clone(), + conn, ) .await; } @@ -1245,7 +1301,7 @@ pub async fn par_install_language_dependencies<'a>( custom_name, job_id_2, w_id_2, - db_2, + conn_2, counter_arc, language_name, installer_executable_name, @@ -1257,7 +1313,7 @@ pub async fn par_install_language_dependencies<'a>( custom_name.clone(), job_id.clone(), w_id.to_owned(), - db.clone(), + conn.clone(), counter_arc.clone(), language_name.to_owned(), installer_executable_name.to_owned(), @@ -1303,7 +1359,7 @@ pub async fn par_install_language_dependencies<'a>( counter_arc, total_to_install, start, - db_2, + &conn_2, ) .await; return; @@ -1321,7 +1377,7 @@ pub async fn par_install_language_dependencies<'a>( }; if let Err(e) = crate::handle_child::handle_child( &job_id_2, - &db_2, + &conn_2, // TODO: Return mem_peak &mut 0, // TODO: Return canceld_by_ref @@ -1342,7 +1398,7 @@ pub async fn par_install_language_dependencies<'a>( &job_id_2, &w_id_2, format!("error while installing {}: {e:?}", &display_name_2), - db_2.clone(), + &conn_2, ) .await; } else { @@ -1372,7 +1428,7 @@ pub async fn par_install_language_dependencies<'a>( counter_arc, total_to_install, start, - db_2, + &conn_2, ) .await; // TODO: Refactor @@ -1428,7 +1484,7 @@ pub async fn par_install_language_dependencies<'a>( job_id, w_id, format!("\n\nFetching {} packages...\n", not_pulled_copy.len()), - db.clone(), + &conn, ) .await; let cmd = callback(not_pulled_copy.clone())?; @@ -1439,7 +1495,7 @@ pub async fn par_install_language_dependencies<'a>( if let Err(e) = crate::handle_child::handle_child( // &job_id, &Uuid::nil(), - &db, + &conn, // TODO: Return mem_peak &mut 0, // TODO: Return canceld_by_ref @@ -1516,7 +1572,7 @@ pub async fn par_install_language_dependencies<'a>( "\nDone. Time spent on installation phase: {}ms\n", total_time ), - db, + conn, ) .await; } diff --git a/backend/windmill-worker/src/csharp_executor.rs b/backend/windmill-worker/src/csharp_executor.rs index f30d653ff5..2054e279db 100644 --- a/backend/windmill-worker/src/csharp_executor.rs +++ b/backend/windmill-worker/src/csharp_executor.rs @@ -67,7 +67,7 @@ pub async fn generate_nuget_lockfile( mem_peak: &mut i32, canceled_by: &mut Option, job_dir: &str, - db: &sqlx::Pool, + conn: &Connection, worker_name: &str, w_id: &str, occupancy_metrics: &mut OccupancyMetrics, @@ -115,7 +115,7 @@ pub async fn generate_nuget_lockfile( let gen_lockfile_process = start_child_process(gen_lockfile_cmd, DOTNET_PATH.as_str()).await?; handle_child( job_id, - db, + conn, mem_peak, canceled_by, gen_lockfile_process, @@ -150,7 +150,7 @@ pub async fn generate_nuget_lockfile( _mem_peak: &mut i32, _canceled_by: &mut Option, _job_dir: &str, - _db: &sqlx::Pool, + _conn: &Connection, _worker_name: &str, _w_id: &str, _occupancy_metrics: &mut OccupancyMetrics, @@ -311,7 +311,7 @@ async fn build_cs_proj( mem_peak: &mut i32, canceled_by: &mut Option, job_dir: &str, - db: &sqlx::Pool, + conn: &Connection, worker_name: &str, w_id: &str, base_internal_url: &str, @@ -371,7 +371,7 @@ async fn build_cs_proj( let build_cs_process = start_child_process(build_cs_cmd, DOTNET_PATH.as_str()).await?; handle_child( job_id, - db, + conn, mem_peak, canceled_by, build_cs_process, @@ -385,7 +385,7 @@ async fn build_cs_proj( None, ) .await?; - append_logs(job_id, w_id, "\n\n", db).await; + append_logs(job_id, w_id, "\n\n", conn).await; if let Err(e) = std::fs::remove_file(Path::new(job_dir).join("nuget.config")) { if e.kind() != io::ErrorKind::NotFound { Err(anyhow!("Error erasing nuget.config: {}", e))?; @@ -428,6 +428,7 @@ fn remove_lines_from_text(contents: &str, indices_to_remove: Vec) -> Stri result.join("\n") } +use windmill_common::worker::Connection; use windmill_queue::MiniPulledJob; #[cfg(not(feature = "csharp"))] @@ -435,7 +436,7 @@ pub async fn handle_csharp_job( _mem_peak: &mut i32, _canceled_by: &mut Option, _job: &MiniPulledJob, - _db: &sqlx::Pool, + _conn: &Connection, _client: &AuthedClient, _parent_runnable_path: Option, _inner_content: &str, @@ -449,13 +450,12 @@ pub async fn handle_csharp_job( ) -> Result, Error> { Err(anyhow!("C# is not available because the feature is not enabled").into()) } - #[cfg(feature = "csharp")] pub async fn handle_csharp_job( mem_peak: &mut i32, canceled_by: &mut Option, job: &MiniPulledJob, - db: &sqlx::Pool, + conn: &Connection, client: &AuthedClient, parent_runnable_path: Option, inner_content: &str, @@ -495,7 +495,7 @@ pub async fn handle_csharp_job( cache_logs } else { let logs1 = format!("{cache_logs}\n\n--- DOTNET BUILD ---\n"); - append_logs(&job.id, &job.workspace_id, logs1, db).await; + append_logs(&job.id, &job.workspace_id, logs1, conn).await; let (reqs, lines_to_remove) = parse_csharp_reqs(inner_content); for req in &reqs { @@ -507,7 +507,7 @@ pub async fn handle_csharp_job( req.0, req.1.as_ref().unwrap_or(&"".to_string()) ), - db, + conn, ) .await; } @@ -525,7 +525,7 @@ pub async fn handle_csharp_job( mem_peak, canceled_by, job_dir, - db, + conn, worker_name, &job.workspace_id, base_internal_url, @@ -535,13 +535,13 @@ pub async fn handle_csharp_job( .await? }; - create_args_and_out_file(client, job, job_dir, db).await?; + create_args_and_out_file(client, job, job_dir, conn).await?; let logs2 = format!("{cache_logs}\n\n--- C# CODE EXECUTION ---\n"); - append_logs(&job.id, &job.workspace_id, format!("{}\n", logs2), db).await; + append_logs(&job.id, &job.workspace_id, format!("{}\n", logs2), conn).await; let reserved_variables = - get_reserved_variables(job, &client.token, db, parent_runnable_path).await?; + get_reserved_variables(job, &client.token, conn, parent_runnable_path).await?; let child = if !*DISABLE_NSJAIL { write_file( @@ -627,7 +627,7 @@ pub async fn handle_csharp_job( handle_child( &job.id, - db, + conn, mem_peak, canceled_by, child, diff --git a/backend/windmill-worker/src/dedicated_worker.rs b/backend/windmill-worker/src/dedicated_worker.rs index 2b63a6d28a..33a83a1492 100644 --- a/backend/windmill-worker/src/dedicated_worker.rs +++ b/backend/windmill-worker/src/dedicated_worker.rs @@ -29,9 +29,7 @@ use windmill_queue::MiniPulledJob; use anyhow::Context; -use crate::{ - common::start_child_process, JobCompleted, JobCompletedSender, MAX_BUFFERED_DEDICATED_JOBS, -}; +use crate::{common::start_child_process, JobCompletedSender, MAX_BUFFERED_DEDICATED_JOBS}; use futures::{future, Future}; use std::{collections::HashMap, task::Poll}; @@ -78,7 +76,7 @@ pub async fn handle_dedicated_process( ) -> std::result::Result<(), error::Error> { //do not cache local dependencies - use windmill_queue::MiniPulledJob; + use windmill_queue::{JobCompleted, MiniPulledJob}; use crate::{handle_child::process_status, PROXY_ENVS}; let cmd_name = format!("dedicated {command_path}"); @@ -187,16 +185,16 @@ pub async fn handle_dedicated_process( match serde_json::from_str::>(&line.replace("wm_res[success]:", "").replace("wm_res[error]:", "")) { Ok(result) => { let result = Arc::new(result); - append_logs(&job.id, &job.workspace_id, logs.clone(), db).await; + append_logs(&job.id, &job.workspace_id, logs.clone(), &db.into()).await; if line.starts_with("wm_res[success]:") { - job_completed_tx.send(JobCompleted { job , result, result_columns: None, mem_peak: 0, canceled_by: None, success: true, cached_res_path: None, token: token.to_string(), duration: None }).await.unwrap() + job_completed_tx.send_job(JobCompleted { job , result, result_columns: None, mem_peak: 0, canceled_by: None, success: true, cached_res_path: None, token: token.to_string(), duration: None }).await.unwrap() } else { - job_completed_tx.send(JobCompleted { job , result, result_columns: None, mem_peak: 0, canceled_by: None, success: false, cached_res_path: None, token: token.to_string(), duration: None }).await.unwrap() + job_completed_tx.send_job(JobCompleted { job , result, result_columns: None, mem_peak: 0, canceled_by: None, success: false, cached_res_path: None, token: token.to_string(), duration: None }).await.unwrap() } }, Err(e) => { tracing::error!("Could not deserialize job result `{line}`: {e:?}"); - job_completed_tx.send(JobCompleted { job , result: Arc::new(to_raw_value(&serde_json::json!({"error": format!("Could not deserialize job result `{line}`: {e:?}")}))), result_columns: None, mem_peak: 0, canceled_by: None, success: false, cached_res_path: None, token: token.to_string(), duration: None }).await.unwrap(); + job_completed_tx.send_job(JobCompleted { job , result: Arc::new(to_raw_value(&serde_json::json!({"error": format!("Could not deserialize job result `{line}`: {e:?}")}))), result_columns: None, mem_peak: 0, canceled_by: None, success: false, cached_res_path: None, token: token.to_string(), duration: None }).await.unwrap(); }, }; logs = init_log.clone(); @@ -397,13 +395,16 @@ async fn spawn_dedicated_workers_for_flow( } } FlowModuleValue::FlowScript { id, language, .. } => { - let spawn = cache::flow::fetch_script(db, *id).await.map(|data| { - SpawnWorker::RawScript { - path: "".to_string(), - content: data.code.clone(), - lock: data.lock.clone(), - lang: *language, - } + let spawn = cache::flow::fetch_script( + &windmill_common::worker::Connection::Sql(db.clone()), + *id, + ) + .await + .map(|data| SpawnWorker::RawScript { + path: "".to_string(), + content: data.code.clone(), + lock: data.lock.clone(), + lang: *language, }); match spawn { Ok(spawn) => { @@ -571,7 +572,7 @@ async fn spawn_dedicated_worker( }; use windmill_queue::MiniPulledJob; - use crate::{build_envs, get_script_content_by_hash, ContentReqLangEnvs, JOB_TOKEN}; + use crate::{build_envs, get_script_content_by_hash, ContentReqLangEnvs}; #[cfg(not(feature = "enterprise"))] { @@ -586,7 +587,7 @@ async fn spawn_dedicated_worker( std::sync::Arc, >(MAX_BUFFERED_DEDICATED_JOBS); let killpill_rx = killpill_rx.resubscribe(); - let db = db.clone(); + let db2 = db.clone(); let base_internal_url = base_internal_url.to_string(); let worker_name = worker_name.to_string(); let job_completed_tx = job_completed_tx.clone(); @@ -613,11 +614,11 @@ async fn spawn_dedicated_worker( let (content, lock, language, envs, codebase) = match sw.clone() { SpawnWorker::Script { path, hash } => { let q = if let Some(hash) = hash { - get_script_content_by_hash(&hash, &w_id, &db).await.map( - |r: ContentReqLangEnvs| { + get_script_content_by_hash(&hash, &w_id, &db2.into()) + .await + .map(|r: ContentReqLangEnvs| { Some((r.content, r.lockfile, r.language, r.envs, r.codebase)) - }, - ) + }) } else { sqlx::query_as::<_, (String, Option, Option, Option>, bool, Option)>( "SELECT content, lock, language, envs, codebase IS NOT NULL, hash FROM script WHERE path = $1 AND workspace_id = $2 AND @@ -626,7 +627,7 @@ async fn spawn_dedicated_worker( ) .bind(&path) .bind(&w_id) - .fetch_optional(&db) + .fetch_optional(&db2) .await .map_err(|e| Error::internal_err(format!("expected content and lock: {e:#}"))) .map(|x| x.map(|y| (y.0, y.1, y.2, y.3, if y.4 { y.5.map(|z| z.to_string()) } else { None }))) @@ -658,10 +659,9 @@ async fn spawn_dedicated_worker( _ => return None, } + let db = db.clone(); let handle = tokio::spawn(async move { - let token = if let Some(token) = JOB_TOKEN.as_ref() { - token.clone() - } else { + let token = { let token = rd_string(32); if let Err(e) = sqlx::query_scalar!( "INSERT INTO token diff --git a/backend/windmill-worker/src/deno_executor.rs b/backend/windmill-worker/src/deno_executor.rs index 3e7f7f2bde..696957f463 100644 --- a/backend/windmill-worker/src/deno_executor.rs +++ b/backend/windmill-worker/src/deno_executor.rs @@ -15,8 +15,11 @@ use crate::{ PATH_ENV, TZ_ENV, }; use tokio::{fs::File, io::AsyncReadExt, process::Command}; -use windmill_common::error::{self}; use windmill_common::{error::Result, worker::write_file, BASE_URL}; +use windmill_common::{ + error::{self}, + worker::Connection, +}; use windmill_parser::Typ; lazy_static::lazy_static! { @@ -97,7 +100,7 @@ pub async fn generate_deno_lock( mem_peak: &mut i32, canceled_by: &mut Option, job_dir: &str, - db: Option<&sqlx::Pool>, + db: Option<&Connection>, w_id: &str, worker_name: &str, base_internal_url: &str, @@ -179,7 +182,7 @@ pub async fn handle_deno_job( mem_peak: &mut i32, canceled_by: &mut Option, job: &MiniPulledJob, - db: &sqlx::Pool, + conn: &Connection, client: &AuthedClient, parent_runnable_path: Option, job_dir: &str, @@ -192,7 +195,7 @@ pub async fn handle_deno_job( ) -> error::Result> { // let mut start = Instant::now(); let logs1 = "\n\n--- DENO CODE EXECUTION ---\n".to_string(); - append_logs(&job.id, &job.workspace_id, logs1, db).await; + append_logs(&job.id, &job.workspace_id, logs1, conn).await; let main_override = job.script_entrypoint_override.as_deref(); let apply_preprocessor = !job.is_flow_step() && job.preprocessed == Some(false); @@ -316,11 +319,12 @@ try {{ let reserved_variables_args_out_f = async { let args_and_out_f = async { - create_args_and_out_file(&client, job, job_dir, db).await?; + create_args_and_out_file(&client, job, job_dir, conn).await?; Ok(()) as Result<()> }; let reserved_variables_f = async { - let vars = get_reserved_variables(job, &client.token, db, parent_runnable_path).await?; + let vars = + get_reserved_variables(job, &client.token, conn, parent_runnable_path).await?; Ok(vars) as Result> }; let (_, reserved_variables) = tokio::try_join!(args_and_out_f, reserved_variables_f)?; @@ -404,7 +408,7 @@ try {{ // start = Instant::now(); handle_child( &job.id, - db, + conn, mem_peak, canceled_by, child, @@ -514,7 +518,7 @@ pub async fn start_worker( let common_deno_proc_envs = get_common_deno_proc_envs(&token, base_internal_url).await; let context = variables::get_reserved_variables( - db, + &db.into(), w_id, &token, "dedicated_worker@windmill.dev", diff --git a/backend/windmill-worker/src/go_executor.rs b/backend/windmill-worker/src/go_executor.rs index 959845f641..c0fc8a8085 100644 --- a/backend/windmill-worker/src/go_executor.rs +++ b/backend/windmill-worker/src/go_executor.rs @@ -8,7 +8,7 @@ use uuid::Uuid; use windmill_common::{ error::{self, Error}, utils::calculate_hash, - worker::{save_cache, write_file}, + worker::{save_cache, write_file, Connection}, }; use windmill_parser_go::{parse_go_imports, REQUIRE_PARSE}; use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; @@ -36,7 +36,7 @@ pub async fn handle_go_job( mem_peak: &mut i32, canceled_by: &mut Option, job: &MiniPulledJob, - db: &sqlx::Pool, + conn: &Connection, client: &AuthedClient, parent_runnable_path: Option, inner_content: &str, @@ -78,7 +78,7 @@ pub async fn handle_go_job( let cache_logs = if !cache { let logs1 = format!("{cache_logs}\n\n--- GO DEPENDENCIES SETUP ---\n"); - append_logs(&job.id, &job.workspace_id, logs1, db).await; + append_logs(&job.id, &job.workspace_id, logs1, conn).await; install_go_dependencies( &job.id, @@ -86,7 +86,7 @@ pub async fn handle_go_job( mem_peak, canceled_by, job_dir, - db, + conn, true, skip_go_mod, skip_tidy, @@ -96,7 +96,7 @@ pub async fn handle_go_job( ) .await?; - create_args_and_out_file(client, job, job_dir, db).await?; + create_args_and_out_file(client, job, job_dir, conn).await?; { let sig = windmill_parser_go::parse_go_sig(&inner_content)?; @@ -202,7 +202,7 @@ func Run(req Req) (interface{{}}, error){{ let build_go_process = start_child_process(build_go_cmd, GO_PATH.as_str()).await?; handle_child( &job.id, - db, + conn, mem_peak, canceled_by, build_go_process, @@ -245,15 +245,15 @@ func Run(req Req) (interface{{}}, error){{ )) })?; - create_args_and_out_file(client, job, job_dir, db).await?; + create_args_and_out_file(client, job, job_dir, conn).await?; cache_logs }; let logs2 = format!("{cache_logs}\n\n--- GO CODE EXECUTION ---\n"); - append_logs(&job.id, &job.workspace_id, logs2, db).await; + append_logs(&job.id, &job.workspace_id, logs2, conn).await; let reserved_variables = - get_reserved_variables(job, &client.token, db, parent_runnable_path).await?; + get_reserved_variables(job, &client.token, conn, parent_runnable_path).await?; let child = if !*DISABLE_NSJAIL { let _ = write_file( @@ -306,7 +306,7 @@ func Run(req Req) (interface{{}}, error){{ }; handle_child( &job.id, - db, + conn, mem_peak, canceled_by, child, @@ -351,7 +351,7 @@ pub async fn install_go_dependencies( mem_peak: &mut i32, canceled_by: &mut Option, job_dir: &str, - db: &sqlx::Pool, + conn: &Connection, non_dep_job: bool, skip_go_mod: bool, has_sum: bool, @@ -371,7 +371,7 @@ pub async fn install_go_dependencies( handle_child( job_id, - db, + conn, mem_peak, canceled_by, child_process, @@ -409,20 +409,22 @@ pub async fn install_go_dependencies( let mut skip_tidy = has_sum; if !has_sum { - if let Some(cached) = sqlx::query_scalar!( - "SELECT lockfile FROM pip_resolution_cache WHERE hash = $1", - hash - ) - .fetch_optional(db) - .await? - { - let logs1 = format!("\nfound cached resolution: {}", hash); - append_logs(&job_id, w_id, logs1, db).await; - gen_go_mod(code, job_dir, &cached).await?; - skip_tidy = true; - new_lockfile = false; - } else { - new_lockfile = true; + if let Some(db) = conn.as_sql() { + if let Some(cached) = sqlx::query_scalar!( + "SELECT lockfile FROM pip_resolution_cache WHERE hash = $1", + hash + ) + .fetch_optional(db) + .await? + { + let logs1 = format!("\nfound cached resolution: {}", hash); + append_logs(&job_id, w_id, logs1, conn).await; + gen_go_mod(code, job_dir, &cached).await?; + skip_tidy = true; + new_lockfile = false; + } else { + new_lockfile = true; + } } } @@ -438,7 +440,7 @@ pub async fn install_go_dependencies( handle_child( job_id, - db, + conn, mem_peak, canceled_by, child_process, @@ -469,11 +471,15 @@ pub async fn install_go_dependencies( } if non_dep_job { - sqlx::query!( - "INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() + ('3 days')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = $2", - hash, - req_content - ).fetch_optional(db).await?; + if let Some(db) = conn.as_sql() { + sqlx::query!( + "INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() + ('3 days')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = $2", + hash, + req_content + ) + .fetch_optional(db) + .await?; + } return Ok(String::new()); } else { diff --git a/backend/windmill-worker/src/graphql_executor.rs b/backend/windmill-worker/src/graphql_executor.rs index c755d33beb..5da8b2d193 100644 --- a/backend/windmill-worker/src/graphql_executor.rs +++ b/backend/windmill-worker/src/graphql_executor.rs @@ -4,7 +4,7 @@ use anyhow::anyhow; use futures::{stream, TryStreamExt}; use serde_json::{json, value::RawValue}; use sqlx::types::Json; -use windmill_common::worker::to_raw_value; +use windmill_common::worker::{to_raw_value, Connection}; use windmill_common::{error::Error, worker::CLOUD_HOSTED}; use windmill_parser_graphql::parse_graphql_sig; use windmill_queue::{CanceledBy, MiniPulledJob}; @@ -37,13 +37,13 @@ pub async fn do_graphql( job: &MiniPulledJob, client: &AuthedClient, query: &str, - db: &sqlx::Pool, + conn: &Connection, mem_peak: &mut i32, canceled_by: &mut Option, worker_name: &str, occupation_metrics: &mut OccupancyMetrics, ) -> windmill_common::error::Result> { - let args = build_args_map(job, client, db).await?.map(Json); + let args = build_args_map(job, client, conn).await?.map(Json); let job_args = if args.is_some() { args.as_ref() } else { @@ -81,7 +81,7 @@ pub async fn do_graphql( } } let (timeout_duration, _, _) = - resolve_job_timeout(&db, &job.workspace_id, job.id, job.timeout).await; + resolve_job_timeout(&conn, &job.workspace_id, job.id, job.timeout).await; let http_client = build_http_client(timeout_duration)?; @@ -150,7 +150,7 @@ pub async fn do_graphql( let r = run_future_with_polling_update_job_poller( job.id, job.timeout, - db, + conn, mem_peak, canceled_by, result_f, diff --git a/backend/windmill-worker/src/handle_child.rs b/backend/windmill-worker/src/handle_child.rs index 19c7d53bf7..713046207f 100644 --- a/backend/windmill-worker/src/handle_child.rs +++ b/backend/windmill-worker/src/handle_child.rs @@ -4,8 +4,9 @@ use futures::Future; use nix::sys::signal::{self, Signal}; #[cfg(any(target_os = "linux", target_os = "macos"))] use nix::unistd::Pid; +use windmill_common::agent_workers::PingJobStatusResponse; +use windmill_common::jobs::LARGE_LOG_THRESHOLD_SIZE; -use sqlx::{Pool, Postgres}; #[cfg(windows)] use std::process::Stdio; use tokio::fs::File; @@ -15,7 +16,10 @@ use windmill_common::error::to_anyhow; use windmill_common::error::{self, Error}; -use windmill_common::worker::{get_windmill_memory_usage, get_worker_memory_usage, CLOUD_HOSTED}; +use windmill_common::worker::{ + get_windmill_memory_usage, get_worker_memory_usage, set_job_cancelled_query, Connection, + JobCancelled, CLOUD_HOSTED, +}; use windmill_queue::{append_logs, CanceledBy}; @@ -29,7 +33,6 @@ use std::{io, panic, time::Duration}; use tracing::{trace_span, Instrument}; use uuid::Uuid; -use windmill_common::DB; #[cfg(feature = "enterprise")] use windmill_common::job_metrics; @@ -49,8 +52,9 @@ use futures::{ }; use crate::common::{resolve_job_timeout, OccupancyMetrics}; -use crate::job_logger::{append_job_logs, append_with_limit, LARGE_LOG_THRESHOLD_SIZE}; +use crate::job_logger::{append_job_logs, append_with_limit}; use crate::job_logger_ee::process_streaming_log_lines; +use crate::worker_utils::{ping_job_status, update_worker_ping_from_job}; use crate::{MAX_RESULT_SIZE, MAX_WAIT_FOR_SIGINT, MAX_WAIT_FOR_SIGTERM}; lazy_static::lazy_static! { @@ -92,7 +96,7 @@ async fn kill_process_tree(pid: Option) -> Result<(), String> { #[tracing::instrument(name="run_subprocess", level = "info", skip_all, fields(otel.name = %child_name))] pub async fn handle_child( job_id: &Uuid, - db: &Pool, + conn: &Connection, mem_peak: &mut i32, canceled_by_ref: &mut Option, mut child: Child, @@ -138,7 +142,7 @@ pub async fn handle_child( * waiting for the child to exit normally */ let update_job = update_job_poller( job_id, - db, + conn, mem_peak, canceled_by_ref, Box::pin(stream::unfold((), move |_| async move { @@ -184,15 +188,13 @@ pub async fn handle_child( } let (timeout_duration, timeout_warn_msg, is_job_specific) = - resolve_job_timeout(&db, w_id, job_id, custom_timeout).await; + resolve_job_timeout(&conn, w_id, job_id, custom_timeout).await; if let Some(msg) = timeout_warn_msg { - append_logs(&job_id, w_id, msg.as_str(), db).await; + append_logs(&job_id, w_id, msg.as_str(), conn).await; } /* a future that completes when the child process exits */ let wait_on_child = async { - let db = db.clone(); - let kill_reason = tokio::select! { biased; result = child.wait() => return result.map(Ok), @@ -208,18 +210,33 @@ pub async fn handle_child( let set_reason = async { if matches!(kill_reason, KillReason::Timeout { .. }) { - if let Err(err) = sqlx::query!( - "UPDATE v2_job_queue - SET canceled_by = 'timeout' - , canceled_reason = $1 - WHERE id = $2", - format!("duration > {}", timeout_duration.as_secs()), - job_id - ) - .execute(&db) - .await - { - tracing::error!(%job_id, %err, "error setting cancelation reason for job {job_id}: {err}"); + match conn { + Connection::Sql(db) => { + if let Err(err) = set_job_cancelled_query( + job_id, + db, + "timeout", + &format!("duration > {}", timeout_duration.as_secs()), + ) + .await + { + tracing::error!(%job_id, %err, "error setting cancelation reason for job {job_id}: {err}"); + } + } + Connection::Http(client) => { + if let Err(err) = client + .post::<_, ()>( + &format!("/api/agent_workers/set_job_cancelled/{}", job_id), + &JobCancelled { + canceled_by: "timeout".to_string(), + reason: format!("duration > {}", timeout_duration.as_secs()), + }, + ) + .await + { + tracing::error!(%job_id, %err, "error setting cancelation reason for job using http {job_id}: {err}"); + } + } } } }; @@ -389,11 +406,13 @@ pub async fn handle_child( let worker_name = worker.to_string(); let w_id2 = w_id.to_string(); + + if let Some(buf) = &mut pipe_stdout { buf.push_str(&joined); (do_write, write_result) = tokio::spawn(async { }).remote_handle(); } else { - (do_write, write_result) = tokio::spawn(append_job_logs(job_id, w_id2, joined, db.clone(), compact_logs, pg_log_total_size.clone(), worker_name)).remote_handle(); + (do_write, write_result) = tokio::spawn(append_job_logs(job_id, w_id2, joined, conn.clone(), compact_logs, pg_log_total_size.clone(), worker_name)).remote_handle(); } if let Err(err) = result { @@ -497,7 +516,7 @@ pub(crate) async fn get_mem_peak(pid: Option, nsjail: bool) -> i32 { pub async fn run_future_with_polling_update_job_poller( job_id: Uuid, timeout: Option, - db: &DB, + conn: &Connection, mem_peak: &mut i32, canceled_by_ref: &mut Option, result_f: Fut, @@ -514,7 +533,7 @@ where let update_job = update_job_poller( job_id, - db, + conn, mem_peak, canceled_by_ref, get_mem, @@ -525,7 +544,7 @@ where ); let timeout_ms = u64::try_from( - resolve_job_timeout(&db, &w_id, job_id, timeout) + resolve_job_timeout(&conn, &w_id, job_id, timeout) .await .0 .as_millis(), @@ -560,7 +579,7 @@ pub enum UpdateJobPollingExit { pub async fn update_job_poller( job_id: Uuid, - db: &DB, + conn: &Connection, mem_peak: &mut i32, canceled_by_ref: &mut Option, mut get_mem: S, @@ -574,8 +593,7 @@ where { let update_job_interval = Duration::from_millis(500); - let db = db.clone(); - + let conn = conn.clone(); let mut interval = interval(update_job_interval); interval.set_missed_tick_behavior(MissedTickBehavior::Skip); @@ -597,22 +615,9 @@ where tracing::info!("job {job_id} on {worker_name} in {w_id} worker memory snapshot {}kB/{}kB", memory_usage.unwrap_or_default()/1024, wm_memory_usage.unwrap_or_default()/1024); let occupancy = occupancy_metrics.as_mut().map(|x| x.update_occupancy_metrics()); if job_id != Uuid::nil() { - sqlx::query!( - "UPDATE worker_ping SET ping_at = now(), current_job_id = $1, current_job_workspace_id = $2, memory_usage = $3, wm_memory_usage = $4, - occupancy_rate = $6, occupancy_rate_15s = $7, occupancy_rate_5m = $8, occupancy_rate_30m = $9 WHERE worker = $5", - &job_id, - &w_id, - memory_usage, - wm_memory_usage, - &worker_name, - occupancy.map(|x| x.0), - occupancy.and_then(|x| x.1), - occupancy.and_then(|x| x.2), - occupancy.and_then(|x| x.3), - ) - .execute(&db) - .await - .expect("update worker ping"); + if let Err(err) = update_worker_ping_from_job(&conn, &job_id, w_id, worker_name, memory_usage, wm_memory_usage, occupancy).await { + tracing::error!("Unable to update worker ping for job {} in workspace {}. Error was: {:?}", job_id, w_id, err); + } } } let current_mem = get_mem.next().await.unwrap_or(0); @@ -627,55 +632,49 @@ where #[cfg(feature = "enterprise")] { if job_id != Uuid::nil() { - - // tracking metric starting at i >= 2 b/c first point it useless and we don't want to track metric for super fast jobs - if i == 2 { - memory_metric_id = job_metrics::register_metric_for_job( - &db, - w_id.to_string(), - job_id, - "memory_kb".to_string(), - job_metrics::MetricKind::TimeseriesInt, - Some("Job Memory Footprint (kB)".to_string()), - ) - .await; - } - if let Ok(ref metric_id) = memory_metric_id { - if let Err(err) = job_metrics::record_metric(&db, w_id.to_string(), job_id, metric_id.to_owned(), job_metrics::MetricNumericValue::Integer(current_mem)).await { - tracing::error!("Unable to save memory stat for job {} in workspace {}. Error was: {:?}", job_id, w_id, err); + if let Connection::Sql(ref db) = conn { + // tracking metric starting at i >= 2 b/c first point it useless and we don't want to track metric for super fast jobs + if i == 2 { + memory_metric_id = job_metrics::register_metric_for_job( + &db, + w_id.to_string(), + job_id, + "memory_kb".to_string(), + job_metrics::MetricKind::TimeseriesInt, + Some("Job Memory Footprint (kB)".to_string()), + ) + .await; + } + if let Ok(ref metric_id) = memory_metric_id { + if let Err(err) = job_metrics::record_metric(&db, w_id.to_string(), job_id, metric_id.to_owned(), job_metrics::MetricNumericValue::Integer(current_mem)).await { + tracing::error!("Unable to save memory stat for job {} in workspace {}. Error was: {:?}", job_id, w_id, err); + } } } } } if job_id != Uuid::nil() { - let (canceled_by, canceled_reason, already_completed) = sqlx::query!( - "UPDATE v2_job_runtime r SET - memory_peak = $1, - ping = now() - FROM v2_job_queue q - WHERE r.id = $2 AND q.id = r.id - RETURNING canceled_by, canceled_reason", - *mem_peak, - job_id - ) - .map(|x| (x.canceled_by, x.canceled_reason, false)) - .fetch_optional(&db) - .await - .unwrap_or_else(|e| { - tracing::error!(%e, "error updating job {job_id}: {e:#}"); - Some((None, None, false)) - }) - .unwrap_or_else(|| { - // if the job is not in queue, it can only be in the completed_job so it is already complete - (None, None, true) - }); - if already_completed { + if matches!(conn, Connection::Http(_)) { + if i % 4 != 0 { + // only ping every 4th time (2s) on http agent mode + continue; + } + } + let ping_job_status = ping_job_status(&conn, &job_id, Some(*mem_peak), if current_mem > 0 { Some(current_mem) } else { None }).await.unwrap_or_else(|e| { + tracing::error!("Unable to ping job status for job {job_id}. Error was: {:?}", e); + PingJobStatusResponse { + canceled_by: None, + canceled_reason: None, + already_completed: false, + } + }); + if ping_job_status.already_completed { return UpdateJobPollingExit::AlreadyCompleted } - if canceled_by.is_some() { + if ping_job_status.canceled_by.is_some() { canceled_by_ref.replace(CanceledBy { - username: canceled_by.clone(), - reason: canceled_reason.clone(), + username: ping_job_status.canceled_by.clone(), + reason: ping_job_status.canceled_reason.clone(), }); break } diff --git a/backend/windmill-worker/src/java_executor.rs b/backend/windmill-worker/src/java_executor.rs index 6f593561d2..c589252005 100644 --- a/backend/windmill-worker/src/java_executor.rs +++ b/backend/windmill-worker/src/java_executor.rs @@ -13,7 +13,7 @@ use uuid::Uuid; use windmill_common::{ error::{self, Error}, utils::calculate_hash, - worker::{copy_dir_recursively, save_cache, write_file}, + worker::{copy_dir_recursively, save_cache, write_file, Connection}, }; use windmill_parser::Arg; use windmill_parser_java::parse_java_sig_meta; @@ -44,7 +44,7 @@ pub(crate) struct JobHandlerInput<'a> { pub canceled_by: &'a mut Option, pub client: &'a AuthedClient, pub parent_runnable_path: Option, - pub db: &'a sqlx::Pool, + pub conn: &'a Connection, pub envs: HashMap, pub inner_content: &'a str, pub job: &'a MiniPulledJob, @@ -67,7 +67,7 @@ pub async fn handle_java_job<'a>(mut args: JobHandlerInput<'a>) -> Result(mut args: JobHandlerInput<'a>) -> Result( - JobHandlerInput { job, db, job_dir, client, inner_content, .. }: &mut JobHandlerInput<'a>, + JobHandlerInput { job, conn, job_dir, client, inner_content, .. }: &mut JobHandlerInput<'a>, ) -> Result<(), Error> { // Create needed files { - create_args_and_out_file(&client, job, job_dir, db).await?; + create_args_and_out_file(&client, job, job_dir, conn).await?; let app_path = format!("{}/src/main/java/net/script/", job_dir); create_dir_all(&app_path).await?; File::create(format!("{app_path}/App.java")) @@ -120,7 +120,7 @@ pub async fn resolve<'a>( job_id: &Uuid, code: &str, job_dir: &str, - db: &sqlx::Pool, + conn: &Connection, w_id: &str, ) -> Result { let deps = { @@ -152,21 +152,23 @@ pub async fn resolve<'a>( }; let req_hash = format!("java-{}", calculate_hash(&deps)); - if let Some(cached) = sqlx::query_scalar!( - "SELECT lockfile FROM pip_resolution_cache WHERE hash = $1", - req_hash - ) - .fetch_optional(db) - .await? - { - return Ok(cached); + if let Connection::Sql(db) = conn { + if let Some(cached) = sqlx::query_scalar!( + "SELECT lockfile FROM pip_resolution_cache WHERE hash = $1", + req_hash + ) + .fetch_optional(db) + .await? + { + return Ok(cached); + } } let lock = { append_logs( job_id, w_id, format!("\n--- RESOLVING LOCKFILE ---\n"), - db.clone(), + &conn, ) .await; @@ -239,18 +241,22 @@ pub async fn resolve<'a>( } }; - sqlx::query!( + if let Connection::Sql(db) = conn { + sqlx::query!( "INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() + ('3 days')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = $2", req_hash, - lock.clone(), - ).fetch_optional(db).await?; + lock.clone(), + ) + .fetch_optional(db) + .await?; + } - append_logs(job_id, w_id, format!("\n{}", &lock), db.clone()).await; + append_logs(job_id, w_id, format!("\n{}", &lock), &conn).await; Ok(lock) } async fn install<'a>( - JobHandlerInput { worker_name, job, db, job_dir, .. }: &mut JobHandlerInput<'a>, + JobHandlerInput { worker_name, job, conn, job_dir, .. }: &mut JobHandlerInput<'a>, deps: String, ) -> Result { let deps = deps @@ -411,7 +417,7 @@ async fn install<'a>( &job.id, &job.workspace_id, worker_name, - db, + conn, ) .await?; Ok(classpath) @@ -424,7 +430,7 @@ async fn compile<'a>( canceled_by, worker_name, job, - db, + conn, job_dir, client, envs, @@ -448,7 +454,7 @@ async fn compile<'a>( )) } let reserved_variables = - get_reserved_variables(job, &client.token, db, parent_runnable_path.clone()).await?; + get_reserved_variables(job, &client.token, conn, parent_runnable_path.clone()).await?; let hash = compute_hash(inner_content, *requirements_o); let bin_path = format!("{}/{hash}", JAVA_CACHE_DIR); let remote_path = format!("java_jar/{hash}"); @@ -474,7 +480,7 @@ async fn compile<'a>( &job.id, &job.workspace_id, format!("\n--- COMPILING .JAVA FILES\n"), - db.clone(), + &conn, ) .await; @@ -514,7 +520,7 @@ async fn compile<'a>( }; handle_child::handle_child( &job.id, - db, + conn, mem_peak, canceled_by, child, @@ -559,7 +565,7 @@ async fn run<'a>( canceled_by, worker_name, job, - db, + conn, job_dir, shared_mount, client, @@ -571,14 +577,14 @@ async fn run<'a>( classpath: &'a str, ) -> Result<(), Error> { let reserved_variables = - get_reserved_variables(job, &client.token, db, parent_runnable_path.clone()).await?; + get_reserved_variables(job, &client.token, conn, parent_runnable_path.clone()).await?; let child = if !cfg!(windows) && !*DISABLE_NSJAIL { append_logs( &job.id, &job.workspace_id, format!("\n--- ISOLATED JAVA CODE EXECUTION ---\n"), - db.clone(), + &conn, ) .await; @@ -640,7 +646,7 @@ async fn run<'a>( &job.id, &job.workspace_id, format!("\n--- JAVA CODE EXECUTION ---\n"), - db.clone(), + &conn, ) .await; @@ -697,7 +703,7 @@ async fn run<'a>( }; handle_child::handle_child( &job.id, - db, + conn, mem_peak, canceled_by, child, diff --git a/backend/windmill-worker/src/job_logger.rs b/backend/windmill-worker/src/job_logger.rs index fa4ac15383..97784340c5 100644 --- a/backend/windmill-worker/src/job_logger.rs +++ b/backend/windmill-worker/src/job_logger.rs @@ -1,6 +1,7 @@ use regex::Regex; -use windmill_common::worker::CLOUD_HOSTED; +pub use windmill_common::jobs::LARGE_LOG_THRESHOLD_SIZE; +use windmill_common::worker::{Connection, CLOUD_HOSTED}; use windmill_queue::append_logs; @@ -8,7 +9,6 @@ use std::sync::atomic::AtomicU32; use std::sync::Arc; use uuid::Uuid; -use windmill_common::DB; #[cfg(not(all(feature = "enterprise", feature = "parquet")))] use crate::job_logger_ee::default_disk_log_storage; @@ -25,39 +25,40 @@ pub enum CompactLogs { S3, } -pub(crate) async fn append_job_logs( +pub async fn append_job_logs( job_id: Uuid, w_id: String, logs: String, - db: DB, + conn: Connection, must_compact_logs: bool, total_size: Arc, worker_name: String, ) -> () { - if must_compact_logs { - #[cfg(all(feature = "enterprise", feature = "parquet"))] - s3_storage(job_id, &w_id, &db, logs, total_size, &worker_name).await; + match conn { + Connection::Sql(db) if must_compact_logs => { + #[cfg(all(feature = "enterprise", feature = "parquet"))] + s3_storage(job_id, &w_id, &db, logs, total_size, &worker_name).await; - #[cfg(not(all(feature = "enterprise", feature = "parquet")))] - { - default_disk_log_storage( - job_id, - &w_id, - &db, - logs, - total_size, - CompactLogs::NotEE, - &worker_name, - ) - .await; + #[cfg(not(all(feature = "enterprise", feature = "parquet")))] + { + default_disk_log_storage( + job_id, + &w_id, + &db, + logs, + total_size, + CompactLogs::NotEE, + &worker_name, + ) + .await; + } + } + _ => { + append_logs(&job_id, w_id, logs, &conn).await; } - } else { - append_logs(&job_id, w_id, logs, db).await; } } -pub const LARGE_LOG_THRESHOLD_SIZE: usize = 9000; - lazy_static::lazy_static! { static ref RE_00: Regex = Regex::new('\u{00}'.to_string().as_str()).unwrap(); pub static ref NO_LOGS_AT_ALL: bool = std::env::var("NO_LOGS_AT_ALL").ok().is_some_and(|x| x == "1" || x == "true"); diff --git a/backend/windmill-worker/src/js_eval.rs b/backend/windmill-worker/src/js_eval.rs index 822e2c7d8b..33a9b57690 100644 --- a/backend/windmill-worker/src/js_eval.rs +++ b/backend/windmill-worker/src/js_eval.rs @@ -45,7 +45,7 @@ use windmill_common::error::Error; #[cfg(feature = "deno_core")] use windmill_common::worker::{write_file, TMP_DIR}; -use windmill_common::{flow_status::JobResult, DB}; +use windmill_common::flow_status::JobResult; use windmill_queue::CanceledBy; use crate::{common::OccupancyMetrics, AuthedClient}; @@ -749,6 +749,9 @@ fn capture_proxy(s: &str) -> Option<(String, Option<(String, String)>)> { ) }) } + +use windmill_common::worker::Connection; + #[cfg(not(feature = "deno_core"))] pub async fn eval_fetch_timeout( _env_code: String, @@ -758,7 +761,7 @@ pub async fn eval_fetch_timeout( _script_entrypoint_override: Option, _job_id: Uuid, _job_timeout: Option, - _db: &DB, + _conn: &Connection, _mem_peak: &mut i32, _canceled_by: &mut Option, _worker_name: &str, @@ -779,7 +782,7 @@ pub async fn eval_fetch_timeout( script_entrypoint_override: Option, job_id: Uuid, job_timeout: Option, - db: &DB, + conn: &Connection, mem_peak: &mut i32, canceled_by: &mut Option, worker_name: &str, @@ -825,7 +828,7 @@ pub async fn eval_fetch_timeout( )); } - let db_ = db.clone(); + let conn_ = conn.clone(); let w_id_ = w_id.to_string(); let result_f = tokio::task::spawn_blocking(move || { let ops = vec![op_get_static_args(), op_log()]; @@ -921,7 +924,7 @@ pub async fn eval_fetch_timeout( "{extra_logs}{}", js_runtime.op_state().borrow().borrow::().s ), - db_, + &conn_, ) .await; @@ -936,7 +939,7 @@ pub async fn eval_fetch_timeout( let res = run_future_with_polling_update_job_poller( job_id, job_timeout, - db, + conn, mem_peak, canceled_by, async { result_f.await? }, diff --git a/backend/windmill-worker/src/lib.rs b/backend/windmill-worker/src/lib.rs index 607616117f..a9380c52e9 100644 --- a/backend/windmill-worker/src/lib.rs +++ b/backend/windmill-worker/src/lib.rs @@ -5,13 +5,11 @@ mod mssql_executor; #[cfg(feature = "enterprise")] mod snowflake_executor; +mod agent_workers; #[cfg(feature = "python")] mod ansible_executor; mod bash_executor; -#[cfg(feature = "nu")] -mod nu_executor; - #[cfg(feature = "java")] mod java_executor; @@ -28,11 +26,13 @@ mod global_cache; mod go_executor; mod graphql_executor; mod handle_child; -mod job_logger; +pub mod job_logger; mod job_logger_ee; mod js_eval; #[cfg(feature = "mysql")] mod mysql_executor; +#[cfg(feature = "nu")] +mod nu_executor; #[cfg(feature = "oracledb")] mod oracledb_executor; mod otel_ee; @@ -41,7 +41,7 @@ mod pg_executor; mod php_executor; #[cfg(feature = "python")] mod python_executor; -mod result_processor; +pub mod result_processor; #[cfg(feature = "rust")] mod rust_executor; mod sanitized_sql_params; @@ -49,12 +49,14 @@ mod schema; mod worker; mod worker_flow; mod worker_lockfiles; +mod worker_utils; pub use worker::*; pub use result_processor::handle_job_error; pub use bun_executor::{ - get_common_bun_proc_envs, install_bun_lockfile, prebundle_bun_script, prepare_job_dir, + compute_bundle_local_and_remote_path, get_common_bun_proc_envs, install_bun_lockfile, + prebundle_bun_script, prepare_job_dir, }; pub use deno_executor::generate_deno_lock; diff --git a/backend/windmill-worker/src/mssql_executor.rs b/backend/windmill-worker/src/mssql_executor.rs index 04c3642fa8..aa484753cd 100644 --- a/backend/windmill-worker/src/mssql_executor.rs +++ b/backend/windmill-worker/src/mssql_executor.rs @@ -10,7 +10,7 @@ use tokio_util::compat::TokioAsyncWriteCompatExt; use uuid::Uuid; use windmill_common::error::to_anyhow; use windmill_common::error::{self, Error}; -use windmill_common::worker::to_raw_value; +use windmill_common::worker::{to_raw_value, Connection}; use windmill_parser_sql::{parse_db_resource, parse_mssql_sig}; use windmill_queue::MiniPulledJob; use windmill_queue::{append_logs, CanceledBy}; @@ -51,14 +51,14 @@ pub async fn do_mssql( job: &MiniPulledJob, client: &AuthedClient, query: &str, - db: &sqlx::Pool, + conn: &Connection, mem_peak: &mut i32, canceled_by: &mut Option, worker_name: &str, occupancy_metrics: &mut OccupancyMetrics, job_dir: &str, ) -> error::Result> { - let mssql_args = build_args_values(job, client, db).await?; + let mssql_args = build_args_values(job, client, conn).await?; let inline_db_res_path = parse_db_resource(&query); @@ -104,7 +104,7 @@ pub async fn do_mssql( if readonly_intent { let logs = format!("\nSetting ApplicationIntent to ReadOnly"); - append_logs(&job.id, &job.workspace_id, logs, db).await; + append_logs(&job.id, &job.workspace_id, logs, conn).await; } // Handle authentication based on available credentials @@ -222,7 +222,7 @@ pub async fn do_mssql( let raw_result = run_future_with_polling_update_job_poller( job.id, job.timeout, - db, + conn, mem_peak, canceled_by, result_f, diff --git a/backend/windmill-worker/src/mysql_executor.rs b/backend/windmill-worker/src/mysql_executor.rs index 2cab8ce8b2..a1309395a7 100644 --- a/backend/windmill-worker/src/mysql_executor.rs +++ b/backend/windmill-worker/src/mysql_executor.rs @@ -11,7 +11,7 @@ use serde_json::{json, value::RawValue, Value}; use tokio::sync::Mutex; use windmill_common::{ error::{to_anyhow, Error}, - worker::to_raw_value, + worker::{to_raw_value, Connection}, }; use windmill_parser_sql::{ parse_db_resource, parse_mysql_sig, parse_sql_blocks, parse_sql_statement_named_params, @@ -106,14 +106,14 @@ pub async fn do_mysql( job: &MiniPulledJob, client: &AuthedClient, query: &str, - db: &sqlx::Pool, + conn: &Connection, mem_peak: &mut i32, canceled_by: &mut Option, worker_name: &str, column_order: &mut Option>, occupancy_metrics: &mut OccupancyMetrics, ) -> windmill_common::error::Result> { - let job_args = build_args_values(job, client, db).await?; + let job_args = build_args_values(job, client, conn).await?; let inline_db_res_path = parse_db_resource(&query); @@ -234,8 +234,8 @@ pub async fn do_mysql( } let pool = mysql_async::Pool::new(opts); - let conn = pool.get_conn().await.map_err(to_anyhow)?; - let conn_a = Arc::new(Mutex::new(conn)); + let mysql_conn = pool.get_conn().await.map_err(to_anyhow)?; + let conn_a = Arc::new(Mutex::new(mysql_conn)); let queries = parse_sql_blocks(query); @@ -281,7 +281,7 @@ pub async fn do_mysql( let result = run_future_with_polling_update_job_poller( job.id, job.timeout, - db, + conn, mem_peak, canceled_by, result_f, diff --git a/backend/windmill-worker/src/nu_executor.rs b/backend/windmill-worker/src/nu_executor.rs index 729302b7c3..b2f6590c41 100644 --- a/backend/windmill-worker/src/nu_executor.rs +++ b/backend/windmill-worker/src/nu_executor.rs @@ -3,7 +3,10 @@ use std::{collections::HashMap, process::Stdio}; use itertools::Itertools; use serde_json::value::RawValue; use tokio::{fs::File, io::AsyncWriteExt, process::Command}; -use windmill_common::{error::Error, worker::write_file}; +use windmill_common::{ + error::Error, + worker::{write_file, Connection}, +}; use windmill_parser::Arg; use windmill_parser_nu::parse_nu_signature; use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; @@ -30,7 +33,7 @@ pub(crate) struct JobHandlerInput<'a> { pub canceled_by: &'a mut Option, pub client: &'a AuthedClient, pub parent_runnable_path: Option, - pub db: &'a sqlx::Pool, + pub conn: &'a Connection, pub envs: HashMap, pub inner_content: &'a str, pub job: &'a MiniPulledJob, @@ -52,7 +55,7 @@ pub async fn handle_nu_job<'a>(mut args: JobHandlerInput<'a>) -> Result( canceled_by, worker_name, job, - db, + conn, job_dir, shared_mount, client, @@ -229,13 +232,13 @@ async fn run<'a>( // plugins: Vec<&'a str>, ) -> Result<(), Error> { let reserved_variables = - get_reserved_variables(job, &client.token, db, parent_runnable_path.clone()).await?; + get_reserved_variables(job, &client.token, conn, parent_runnable_path.clone()).await?; let child = if !cfg!(windows) && !*DISABLE_NSJAIL { append_logs( &job.id, &job.workspace_id, format!("\n\n--- ISOLATED NU CODE EXECUTION ---\n"), - db.clone(), + conn, ) .await; @@ -274,7 +277,7 @@ async fn run<'a>( &job.id, &job.workspace_id, format!("\n\n--- NU CODE EXECUTION ---\n"), - db.clone(), + &conn, ) .await; @@ -323,7 +326,7 @@ async fn run<'a>( }; handle_child::handle_child( &job.id, - db, + conn, mem_peak, canceled_by, child, diff --git a/backend/windmill-worker/src/oracledb_executor.rs b/backend/windmill-worker/src/oracledb_executor.rs index 2020e421df..7c63e02c81 100644 --- a/backend/windmill-worker/src/oracledb_executor.rs +++ b/backend/windmill-worker/src/oracledb_executor.rs @@ -10,7 +10,7 @@ use serde::{Deserialize, Serialize}; use serde_json::{json, value::RawValue, Value}; use windmill_common::{ error::{to_anyhow, Error}, - worker::to_raw_value, + worker::{to_raw_value, Connection}, }; use windmill_queue::MiniPulledJob; @@ -296,7 +296,7 @@ pub async fn do_oracledb( job: &MiniPulledJob, client: &AuthedClient, query: &str, - db: &sqlx::Pool, + conn: &Connection, mem_peak: &mut i32, canceled_by: &mut Option, worker_name: &str, @@ -309,7 +309,7 @@ pub async fn do_oracledb( "Oracle Database", )?; - let job_args = build_args_values(job, client, db).await?; + let job_args = build_args_values(job, client, conn).await?; let inline_db_res_path = parse_db_resource(&query); @@ -354,14 +354,14 @@ pub async fn do_oracledb( .init(); } - let conn = tokio::task::spawn_blocking(|| { + let oracle_conn = tokio::task::spawn_blocking(|| { oracle::Connection::connect(database.user, database.password, database.database) .map_err(|e| Error::ExecutionErr(e.to_string())) }) .await .map_err(to_anyhow)??; - let conn_a = Arc::new(std::sync::Mutex::new(conn)); + let conn_a = Arc::new(std::sync::Mutex::new(oracle_conn)); let queries = parse_sql_blocks(&query); @@ -396,7 +396,7 @@ pub async fn do_oracledb( let result = run_future_with_polling_update_job_poller( job.id, job.timeout, - db, + conn, mem_peak, canceled_by, result_f, diff --git a/backend/windmill-worker/src/pg_executor.rs b/backend/windmill-worker/src/pg_executor.rs index e9aa27eca9..79002e4d1f 100644 --- a/backend/windmill-worker/src/pg_executor.rs +++ b/backend/windmill-worker/src/pg_executor.rs @@ -27,7 +27,7 @@ use tokio_postgres::{ use uuid::Uuid; use windmill_common::error::to_anyhow; use windmill_common::error::{self, Error}; -use windmill_common::worker::{to_raw_value, CLOUD_HOSTED}; +use windmill_common::worker::{to_raw_value, Connection, CLOUD_HOSTED}; use windmill_parser::{Arg, Typ}; use windmill_parser_sql::{ parse_db_resource, parse_pg_statement_arg_indices, parse_pgsql_sig, parse_sql_blocks, @@ -161,14 +161,14 @@ pub async fn do_postgresql( job: &MiniPulledJob, client: &AuthedClient, query: &str, - db: &sqlx::Pool, + conn: &Connection, mem_peak: &mut i32, canceled_by: &mut Option, worker_name: &str, column_order: &mut Option>, occupancy_metrics: &mut OccupancyMetrics, ) -> error::Result> { - let pg_args = build_args_values(job, client, db).await?; + let pg_args = build_args_values(job, client, conn).await?; let inline_db_res_path = parse_db_resource(&query); @@ -353,7 +353,7 @@ pub async fn do_postgresql( let result = run_future_with_polling_update_job_poller( job.id, job.timeout, - db, + conn, mem_peak, canceled_by, result_f, diff --git a/backend/windmill-worker/src/php_executor.rs b/backend/windmill-worker/src/php_executor.rs index 4707c8db16..ac50beb99f 100644 --- a/backend/windmill-worker/src/php_executor.rs +++ b/backend/windmill-worker/src/php_executor.rs @@ -7,7 +7,7 @@ use tokio::{fs::File, io::AsyncReadExt, process::Command}; use uuid::Uuid; use windmill_common::{ error::{self, to_anyhow, Result}, - worker::write_file, + worker::{write_file, Connection}, }; use windmill_queue::MiniPulledJob; @@ -67,7 +67,7 @@ pub async fn composer_install( canceled_by: &mut Option, job_id: &Uuid, w_id: &str, - db: &sqlx::Pool, + conn: &Connection, job_dir: &str, worker_name: &str, requirements: String, @@ -94,7 +94,7 @@ pub async fn composer_install( handle_child( job_id, - db, + conn, mem_peak, canceled_by, child_process, @@ -139,7 +139,7 @@ pub async fn handle_php_job( mem_peak: &mut i32, canceled_by: &mut Option, job: &MiniPulledJob, - db: &sqlx::Pool, + conn: &Connection, client: &AuthedClient, parent_runnable_path: Option, job_dir: &str, @@ -167,14 +167,14 @@ pub async fn handle_php_job( let autoload_line = if let Some(composer_json) = composer_json { let logs1 = "\n\n--- COMPOSER INSTALL ---\n".to_string(); - append_logs(&job.id, &job.workspace_id, logs1, db).await; + append_logs(&job.id, &job.workspace_id, logs1, conn).await; composer_install( mem_peak, canceled_by, &job.id, &job.workspace_id, - db, + conn, job_dir, worker_name, composer_json, @@ -189,7 +189,7 @@ pub async fn handle_php_job( let init_logs = "\n\n--- PHP CODE EXECUTION ---\n".to_string(); - append_logs(&job.id, job.workspace_id.to_string(), init_logs, db).await; + append_logs(&job.id, job.workspace_id.to_string(), init_logs, conn).await; let _ = write_file(job_dir, "main.php", inner_content)?; @@ -263,12 +263,13 @@ try {{ let reserved_variables_args_out_f = async { let args_and_out_f = async { - create_args_and_out_file(&client, job, job_dir, db).await?; + create_args_and_out_file(&client, job, job_dir, conn).await?; Ok(()) as Result<()> }; let reserved_variables_f = async { - let vars = get_reserved_variables(job, &client.token, db, parent_runnable_path.clone()) - .await?; + let vars = + get_reserved_variables(job, &client.token, conn, parent_runnable_path.clone()) + .await?; Ok(vars) as Result> }; let (_, reserved_variables) = tokio::try_join!(args_and_out_f, reserved_variables_f)?; @@ -329,7 +330,7 @@ try {{ handle_child( &job.id, - db, + conn, mem_peak, canceled_by, child, diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index e3bfe6838b..f07b9abfe2 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -10,7 +10,6 @@ use anyhow::anyhow; use itertools::Itertools; use regex::Regex; use serde_json::value::RawValue; -use sqlx::{Pool, Postgres}; use tokio::{ fs::{metadata, DirBuilder, File}, io::AsyncReadExt, @@ -27,15 +26,16 @@ use windmill_common::{ Error::{self}, }, utils::calculate_hash, - worker::{copy_dir_recursively, pad_string, write_file, PythonAnnotations, WORKER_CONFIG}, - DB, + worker::{ + copy_dir_recursively, pad_string, write_file, Connection, PythonAnnotations, WORKER_CONFIG, + }, }; #[cfg(feature = "enterprise")] use windmill_common::variables::get_secret_value_as_admin; use std::env::var; -use windmill_queue::{append_logs, CanceledBy}; +use windmill_queue::{append_logs, CanceledBy, PrecomputedAgentInfo}; lazy_static::lazy_static! { static ref PYTHON_PATH: Option = var("PYTHON_PATH").ok().map(|v| { @@ -77,6 +77,7 @@ use crate::{ start_child_process, OccupancyMetrics, }, handle_child::handle_child, + worker_utils::ping_job_status, AuthedClient, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, INSTANCE_PYTHON_VERSION, NSJAIL_PATH, PATH_ENV, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, PROXY_ENVS, PY_INSTALL_DIR, TZ_ENV, UV_CACHE_DIR, }; @@ -95,7 +96,7 @@ pub enum PyVersion { } impl PyVersion { - pub async fn from_instance_version(job_id: &Uuid, w_id: &str, db: &Pool) -> Self { + pub async fn from_instance_version(job_id: &Uuid, w_id: &str, conn: &Connection) -> Self { let mut err = None; let pyv = match INSTANCE_PYTHON_VERSION.read().await.clone() { Some(v) => PyVersion::from_string_with_dots(&v).unwrap_or_else(|| { @@ -108,7 +109,7 @@ impl PyVersion { }; if let Some(msg) = err { - append_logs(job_id, w_id, &msg, db).await; + append_logs(job_id, w_id, &msg, conn).await; tracing::error!(msg); } pyv @@ -211,7 +212,7 @@ impl PyVersion { job_id: &Uuid, mem_peak: &mut i32, // canceled_by: &mut Option, - db: &Pool, + conn: &Connection, worker_name: &str, w_id: &str, occupancy_metrics: &mut Option<&mut OccupancyMetrics>, @@ -221,7 +222,7 @@ impl PyVersion { // } let res = self - .get_python_inner(job_id, mem_peak, db, worker_name, w_id, occupancy_metrics) + .get_python_inner(job_id, mem_peak, conn, worker_name, w_id, occupancy_metrics) .await; if let Err(ref e) = res { @@ -235,7 +236,7 @@ impl PyVersion { format!( "\nError while getting python from uv, falling back to system python: {e:?}" ), - db, + conn, ) .await; } @@ -246,7 +247,7 @@ impl PyVersion { job_id: &Uuid, mem_peak: &mut i32, // canceled_by: &mut Option, - db: &Pool, + conn: &Connection, worker_name: &str, w_id: &str, occupancy_metrics: &mut Option<&mut OccupancyMetrics>, @@ -257,7 +258,7 @@ impl PyVersion { if py_path.is_err() { // Install it if let Err(err) = self - .install_python(job_id, mem_peak, db, worker_name, w_id, occupancy_metrics) + .install_python(job_id, mem_peak, conn, worker_name, w_id, occupancy_metrics) .await { tracing::error!("Cannot install python: {err}"); @@ -283,13 +284,13 @@ impl PyVersion { job_id: &Uuid, mem_peak: &mut i32, // canceled_by: &mut Option, - db: &Pool, + conn: &Connection, worker_name: &str, w_id: &str, occupancy_metrics: &mut Option<&mut OccupancyMetrics>, ) -> error::Result<()> { let v = self.to_string_with_dot(); - append_logs(job_id, w_id, format!("\nINSTALLING PYTHON ({})", v), db).await; + append_logs(job_id, w_id, format!("\nINSTALLING PYTHON ({})", v), conn).await; // Create dirs for newly installed python // If we dont do this, NSJAIL will not be able to mount cache // For the default version directory created during startup (main.rs) @@ -337,10 +338,10 @@ impl PyVersion { let child_process = start_child_process(child_cmd, "uv").await?; - append_logs(&job_id, &w_id, logs, db).await; + append_logs(&job_id, &w_id, logs, conn).await; handle_child( job_id, - db, + conn, mem_peak, &mut None, child_process, @@ -459,7 +460,7 @@ pub async fn uv_pip_compile( mem_peak: &mut i32, canceled_by: &mut Option, job_dir: &str, - db: &Pool, + conn: &Connection, worker_name: &str, w_id: &str, occupancy_metrics: &mut Option<&mut OccupancyMetrics>, @@ -506,25 +507,27 @@ pub async fn uv_pip_compile( let requirements = format!("# py{}\n{}", py_version.to_string_no_dot(), requirements); #[cfg(feature = "enterprise")] - let requirements = replace_pip_secret(db, w_id, &requirements, worker_name, job_id).await?; + let requirements = replace_pip_secret(conn, w_id, &requirements, worker_name, job_id).await?; let req_hash = format!("py-{}", calculate_hash(&requirements)); if !no_cache { - if let Some(cached) = sqlx::query_scalar!( - "SELECT lockfile FROM pip_resolution_cache WHERE hash = $1", - // Python version is included in hash, - // hash will be the different for every python version - req_hash - ) - .fetch_optional(db) - .await? - { - logs.push_str(&format!( - "\nFound cached resolution: {req_hash}, on python version: {}", - py_version.to_string_with_dot() - )); - return Ok(cached); + if let Some(db) = conn.as_sql() { + if let Some(cached) = sqlx::query_scalar!( + "SELECT lockfile FROM pip_resolution_cache WHERE hash = $1", + // Python version is included in hash, + // hash will be the different for every python version + req_hash + ) + .fetch_optional(db) + .await? + { + logs.push_str(&format!( + "\nFound cached resolution: {req_hash}, on python version: {}", + py_version.to_string_with_dot() + )); + return Ok(cached); + } } } @@ -535,7 +538,7 @@ pub async fn uv_pip_compile( { // Make sure we have python runtime installed py_version - .get_python(job_id, mem_peak, db, worker_name, w_id, occupancy_metrics) + .get_python(job_id, mem_peak, conn, worker_name, w_id, occupancy_metrics) .await?; let mut args = vec![ @@ -631,10 +634,10 @@ pub async fn uv_pip_compile( } let child_process = start_child_process(child_cmd, uv_cmd).await?; - append_logs(&job_id, &w_id, logs, db).await; + append_logs(&job_id, &w_id, logs, conn).await; handle_child( job_id, - db, + conn, mem_peak, canceled_by, child_process, @@ -671,11 +674,13 @@ pub async fn uv_pip_compile( .collect::>() .join("\n") ); - sqlx::query!( + if let Some(db) = conn.as_sql() { + sqlx::query!( "INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() + ('3 days')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = $2", req_hash, lockfile ).fetch_optional(db).await?; + } Ok(lockfile) } @@ -711,7 +716,7 @@ async fn postinstall( additional_python_paths: &mut Vec, job_dir: &str, job: &MiniPulledJob, - db: &sqlx::Pool, + conn: &Connection, ) -> windmill_common::error::Result<()> { // It is guranteed that additional_python_paths only contains paths within windmill/cache/ // All other paths you would usually expect in PYTHONPATH are NOT included. These are added in downstream @@ -772,7 +777,7 @@ async fn postinstall( &job.id, &job.workspace_id, "\n\nCopying some packages from cache to job_dir...\n".to_string(), - db, + conn, ) .await; // Remove PATHs we just moved @@ -789,13 +794,20 @@ async fn get_python_path( job_id: &Uuid, w_id: &str, mem_peak: &mut i32, - db: &sqlx::Pool, + conn: &Connection, occupancy_metrics: &mut Option<&mut OccupancyMetrics>, ) -> windmill_common::error::Result { let python_path = if let Some(python_path) = PYTHON_PATH.clone() { python_path } else if let Some(python_path) = py_version - .get_python(&job_id, mem_peak, db, worker_name, w_id, occupancy_metrics) + .get_python( + &job_id, + mem_peak, + conn, + worker_name, + w_id, + occupancy_metrics, + ) .await? { python_path @@ -816,7 +828,7 @@ pub async fn handle_python_job( job: &MiniPulledJob, mem_peak: &mut i32, canceled_by: &mut Option, - db: &sqlx::Pool, + conn: &Connection, client: &AuthedClient, parent_runnable_path: Option, inner_content: &String, @@ -825,6 +837,7 @@ pub async fn handle_python_job( envs: HashMap, new_args: &mut Option>>, occupancy_metrics: &mut OccupancyMetrics, + precomputed_agent_info: Option, ) -> windmill_common::error::Result> { let script_path = crate::common::use_flow_root_path(job.runnable_path()); @@ -835,12 +848,13 @@ pub async fn handle_python_job( &job.workspace_id, &script_path, &job.id, - db, + conn, worker_name, worker_dir, mem_peak, canceled_by, &mut Some(occupancy_metrics), + precomputed_agent_info, ) .await?; @@ -852,13 +866,13 @@ pub async fn handle_python_job( &job.id, &job.workspace_id, mem_peak, - db, + conn, &mut Some(occupancy_metrics), ) .await?; if !no_postinstall { - if let Err(e) = postinstall(&mut additional_python_paths, job_dir, job, db).await { + if let Err(e) = postinstall(&mut additional_python_paths, job_dir, job, conn).await { tracing::error!("Postinstall stage has failed. Reason: {e}"); } tracing::debug!("Finished deps postinstall stage"); @@ -872,7 +886,7 @@ pub async fn handle_python_job( "\n\n--- PYTHON ({}) CODE EXECUTION ---\n", py_version.to_string_with_dot() ), - db, + conn, ) .await; } @@ -901,7 +915,7 @@ pub async fn handle_python_job( let apply_preprocessor = pre_spread.is_some(); - create_args_and_out_file(&client, job, job_dir, db).await?; + create_args_and_out_file(&client, job, job_dir, conn).await?; tracing::debug!("Finished preparing wrapper"); let preprocessor = if let Some(pre_spread) = pre_spread { @@ -1004,7 +1018,7 @@ except BaseException as e: tracing::debug!("Finished writing wrapper"); let mut reserved_variables = - get_reserved_variables(job, &client.token, db, parent_runnable_path).await?; + get_reserved_variables(job, &client.token, conn, parent_runnable_path).await?; // Add /tmp/windmill/cache/python_xyz/global-site-packages to PYTHONPATH. // Usefull if certain wheels needs to be preinstalled before execution. @@ -1129,7 +1143,7 @@ mount {{ handle_child( &job.id, - db, + conn, mem_peak, canceled_by, child, @@ -1353,43 +1367,47 @@ async fn prepare_wrapper( #[cfg(feature = "enterprise")] async fn replace_pip_secret( - db: &DB, + conn: &Connection, w_id: &str, req: &str, worker_name: &str, job_id: &Uuid, ) -> error::Result { - if PIP_SECRET_VARIABLE.is_match(req) { - let mut joined = "".to_string(); - for req in req.lines() { - let nreq = if PIP_SECRET_VARIABLE.is_match(req) { - let capture = PIP_SECRET_VARIABLE.captures(req); - let variable = capture.unwrap().get(1).unwrap().as_str(); - if !variable.contains("/PIP_SECRET_") { - return Err(error::Error::internal_err(format!( + if let Some(db) = conn.as_sql() { + if PIP_SECRET_VARIABLE.is_match(req) { + let mut joined = "".to_string(); + for req in req.lines() { + let nreq = if PIP_SECRET_VARIABLE.is_match(req) { + let capture = PIP_SECRET_VARIABLE.captures(req); + let variable = capture.unwrap().get(1).unwrap().as_str(); + if !variable.contains("/PIP_SECRET_") { + return Err(error::Error::internal_err(format!( "invalid secret variable in pip requirements, (last part of path ma): {}", req ))); - } - let secret = get_secret_value_as_admin(db, w_id, variable).await?; - tracing::info!( - worker = %worker_name, - job_id = %job_id, - workspace_id = %w_id, - "found secret variable in pip requirements: {}", - req - ); - PIP_SECRET_VARIABLE - .replace(req, secret.as_str()) - .to_string() - } else { - req.to_string() - }; - joined.push_str(&nreq); - joined.push_str("\n"); - } + } + let secret = get_secret_value_as_admin(db, w_id, variable).await?; + tracing::info!( + worker = %worker_name, + job_id = %job_id, + workspace_id = %w_id, + "found secret variable in pip requirements: {}", + req + ); + PIP_SECRET_VARIABLE + .replace(req, secret.as_str()) + .to_string() + } else { + req.to_string() + }; + joined.push_str(&nreq); + joined.push_str("\n"); + } - Ok(joined) + Ok(joined) + } else { + Ok(req.to_string()) + } } else { Ok(req.to_string()) } @@ -1402,12 +1420,13 @@ async fn handle_python_deps( w_id: &str, script_path: &str, job_id: &Uuid, - db: &DB, + conn: &Connection, worker_name: &str, worker_dir: &str, mem_peak: &mut i32, canceled_by: &mut Option, occupancy_metrics: &mut Option<&mut OccupancyMetrics>, + precomputed_agent_info: Option, ) -> error::Result<(PyVersion, Vec)> { create_dependencies_dir(job_dir).await; @@ -1423,23 +1442,32 @@ async fn handle_python_deps( let mut annotated_pyv = None; let mut annotated_pyv_numeric = None; let is_deployed = requirements_o.is_some(); - let instance_pyv = PyVersion::from_instance_version(job_id, w_id, db).await; + let instance_pyv = PyVersion::from_instance_version(job_id, w_id, conn).await; let annotations = windmill_common::worker::PythonAnnotations::parse(inner_content); let requirements = match requirements_o { Some(r) => r, None => { let mut already_visited = vec![]; - requirements = windmill_parser_py_imports::parse_python_imports( - inner_content, - w_id, - script_path, - db, - &mut already_visited, - &mut annotated_pyv_numeric, - ) - .await? - .join("\n"); + requirements = match conn { + Connection::Sql(db) => windmill_parser_py_imports::parse_python_imports( + inner_content, + w_id, + script_path, + db, + &mut already_visited, + &mut annotated_pyv_numeric, + ) + .await? + .join("\n"), + Connection::Http(_) => match precomputed_agent_info { + Some(PrecomputedAgentInfo::Python { py_version, requirements }) => { + annotated_pyv_numeric = py_version; + requirements.clone().unwrap_or_else(|| "".to_string()) + } + _ => "".to_string(), + }, + }; annotated_pyv = annotated_pyv_numeric.and_then(|v| PyVersion::from_numeric(v)); @@ -1450,7 +1478,7 @@ async fn handle_python_deps( mem_peak, canceled_by, job_dir, - db, + conn, worker_name, w_id, occupancy_metrics, @@ -1491,7 +1519,7 @@ async fn handle_python_deps( w_id, mem_peak, canceled_by, - db, + conn, worker_name, job_dir, worker_dir, @@ -1696,7 +1724,7 @@ pub async fn handle_python_reqs( w_id: &str, mem_peak: &mut i32, _canceled_by: &mut Option, - db: &sqlx::Pool, + conn: &Connection, _worker_name: &str, job_dir: &str, worker_dir: &str, @@ -1719,7 +1747,7 @@ pub async fn handle_python_reqs( counter_arc: Arc>, total_to_install: usize, instant: std::time::Instant, - db: Pool, + conn: &Connection, ) { #[cfg(not(all(feature = "enterprise", feature = "parquet", unix)))] { @@ -1748,7 +1776,7 @@ pub async fn handle_python_reqs( if s3_push { " > (S3) " } else { "" }, instant.elapsed().as_millis(), ), - db, + conn, ) .await; // Drop lock, so next print success can fire @@ -1810,7 +1838,7 @@ pub async fn handle_python_reqs( &job_id, w_id, format!("\nenv deps from local cache: {}\n", in_cache.join(", ")), - db, + conn, ) .await; } @@ -1824,7 +1852,7 @@ pub async fn handle_python_reqs( let (_done_tx, mut done_rx) = tokio::sync::mpsc::channel::<()>(1); let job_id_2 = job_id.clone(); - let db_2 = db.clone(); + let conn_2 = conn.clone(); let w_id_2 = w_id.to_string(); // Wheels to install @@ -1874,9 +1902,12 @@ pub async fn handle_python_reqs( *mem_peak_lock }; + // Notify server that we are still alive // Detect if job has been canceled - let canceled = sqlx::query_scalar!( + let canceled = match conn_2 { + Connection::Sql(ref db) => { + sqlx::query_scalar!( "UPDATE v2_job_runtime r SET memory_peak = $1, ping = now() @@ -1885,17 +1916,25 @@ pub async fn handle_python_reqs( RETURNING canceled_by IS NOT NULL AS \"canceled!\"", mem_peak_actual, job_id_2 - ) - .fetch_optional(&db_2) - .await - .unwrap_or_else(|e| { - tracing::error!(%e, "error updating job {job_id_2}: {e:#}"); - Some(false) - }) - .unwrap_or_else(|| { - // if the job is not in queue, it can only be in the completed_job so it is already complete - false - }); + ) + .fetch_optional(db) + .await + .unwrap_or_else(|e| { + tracing::error!(%e, "error updating job {job_id_2}: {e:#}"); + Some(false) + }) + .unwrap_or_else(|| { + // if the job is not in queue, it can only be in the completed_job so it is already complete + false + }) + } + Connection::Http(_) => { + if let Err(e) = ping_job_status(&conn_2, &job_id_2, Some(mem_peak_actual), None).await { + tracing::error!(%e, "error pinging job {job_id_2}: {e:#}"); + } + false + } + }; if canceled { @@ -1952,7 +1991,7 @@ pub async fn handle_python_reqs( parallel_limit )); } - append_logs(&job_id, w_id, logs, db).await; + append_logs(&job_id, w_id, logs, conn).await; } let semaphore = Arc::new(Semaphore::new(parallel_limit)); @@ -1964,7 +2003,14 @@ pub async fn handle_python_reqs( let total_time = std::time::Instant::now(); let py_path = py_version - .get_python(job_id, mem_peak, db, _worker_name, w_id, _occupancy_metrics) + .get_python( + job_id, + mem_peak, + conn, + _worker_name, + w_id, + _occupancy_metrics, + ) .await?; let has_work = req_with_penv.len() > 0; @@ -1988,7 +2034,7 @@ pub async fn handle_python_reqs( "started setup python dependencies" ); - let db = db.clone(); + let conn = conn.clone(); let job_id = job_id.clone(); let job_dir = job_dir.to_owned(); let w_id = w_id.to_owned(); @@ -2037,7 +2083,7 @@ pub async fn handle_python_reqs( counter_arc, total_to_install, start, - db + &conn ).await; pids.lock().await.get_mut(i).and_then(|e| e.take()); @@ -2076,7 +2122,7 @@ pub async fn handle_python_reqs( format!( "\nError while spawning proccess:\n{e}", ), - db, + &conn, ) .await; pids.lock().await.get_mut(i).and_then(|e| e.take()); @@ -2127,7 +2173,7 @@ pub async fn handle_python_reqs( "\nError while installing {}:\n{stderr_buf}", &req ), - db, + &conn, ) .await; pids.lock().await.get_mut(i).and_then(|e| e.take()); @@ -2164,7 +2210,7 @@ pub async fn handle_python_reqs( counter_arc, total_to_install, start, - db, // + &conn, // ) .await; @@ -2218,7 +2264,13 @@ pub async fn handle_python_reqs( if has_work { let total_time = total_time.elapsed().as_millis(); - append_logs(&job_id, w_id, format!("\nenv set in {}ms", total_time), db).await; + append_logs( + &job_id, + w_id, + format!("\nenv set in {}ms", total_time), + conn, + ) + .await; } *mem_peak = *mem_peak_thread_safe.lock().await; @@ -2285,7 +2337,7 @@ pub async fn start_worker( let mut mem_peak: i32 = 0; let mut canceled_by: Option = None; let context = variables::get_reserved_variables( - db, + &Connection::Sql(db.clone()), w_id, &token, "dedicated_worker@windmill.dev", @@ -2312,12 +2364,13 @@ pub async fn start_worker( w_id, script_path, &Uuid::nil(), - db, + &Connection::Sql(db.clone()), worker_name, job_dir, &mut mem_peak, &mut canceled_by, &mut None, + None, ) .await?; @@ -2400,7 +2453,7 @@ for line in sys.stdin: } let reserved_variables = windmill_common::variables::get_reserved_variables( - db, + &Connection::Sql(db.clone()), w_id, token, "dedicated_worker", @@ -2442,7 +2495,7 @@ for line in sys.stdin: &Uuid::nil(), w_id, &mut mem_peak, - db, + &Connection::Sql(db.clone()), &mut None, ) .await?; diff --git a/backend/windmill-worker/src/result_processor.rs b/backend/windmill-worker/src/result_processor.rs index 4c2e9fa7a0..141c886fe0 100644 --- a/backend/windmill-worker/src/result_processor.rs +++ b/backend/windmill-worker/src/result_processor.rs @@ -2,7 +2,7 @@ use opentelemetry::trace::FutureExt; use serde::Serialize; -use sqlx::{types::Json, Pool, Postgres}; +use sqlx::types::Json; use std::{ collections::HashMap, sync::{ @@ -21,35 +21,105 @@ use windmill_common::{ error::{self, Error}, jobs::JobKind, utils::WarnAfterExt, - worker::{to_raw_value, WORKER_GROUP}, + worker::{to_raw_value, Connection, WORKER_GROUP}, KillpillSender, DB, }; #[cfg(feature = "benchmark")] use crate::bench::{BenchmarkInfo, BenchmarkIter}; -use windmill_queue::{append_logs, get_queued_job, CanceledBy, MiniPulledJob, WrappedError}; +use windmill_queue::{ + append_logs, get_queued_job, CanceledBy, JobCompleted, MiniPulledJob, WrappedError, +}; use serde_json::{json, value::RawValue}; -use tokio::{ - sync::mpsc::{Receiver, Sender}, - task::JoinHandle, -}; +use tokio::{sync::mpsc::Receiver, task::JoinHandle}; use windmill_queue::{add_completed_job, add_completed_job_error}; use crate::{ bash_executor::ANSI_ESCAPE_RE, - common::{read_result, save_in_cache}, + common::{error_to_value, read_result, save_in_cache}, otel_ee::add_root_flow_job_to_otlp, worker_flow::update_flow_status_after_job_completion, - AuthedClient, JobCompleted, JobCompletedSender, SameWorkerSender, SendResult, INIT_SCRIPT_TAG, + AuthedClient, JobCompletedSender, SameWorkerSender, SendResult, INIT_SCRIPT_TAG, }; +async fn process_jc( + jc: JobCompleted, + worker_name: &str, + base_internal_url: &str, + db: &DB, + worker_dir: &str, + same_worker_tx: &SameWorkerSender, + job_completed_sender: &JobCompletedSender, + #[cfg(feature = "benchmark")] bench: &mut BenchmarkIter, +) { + let success: bool = jc.success; + + let span = tracing::span!( + tracing::Level::INFO, + "job_postprocessing", + job_id = %jc.job.id, root_job = field::Empty, workspace_id = %jc.job.workspace_id, worker = %worker_name,tag = %jc.job.tag, + // hostname = %hostname, + language = field::Empty, + script_path = field::Empty, + flow_step_id = field::Empty, + parent_job = field::Empty, + otel.name = field::Empty + ); + let rj = if let Some(root_job) = jc.job.flow_innermost_root_job { + root_job + } else { + jc.job.id + }; + windmill_common::otel_ee::set_span_parent(&span, &rj); + + if let Some(lg) = jc.job.script_lang.as_ref() { + span.record("language", lg.as_str()); + } + if let Some(step_id) = jc.job.flow_step_id.as_ref() { + span.record( + "otel.name", + format!("job_postprocessing {}", step_id).as_str(), + ); + span.record("flow_step_id", step_id.as_str()); + } else { + span.record("otel.name", "job postprocessing"); + } + if let Some(parent_job) = jc.job.parent_job.as_ref() { + span.record("parent_job", parent_job.to_string().as_str()); + } + if let Some(script_path) = jc.job.runnable_path.as_ref() { + span.record("script_path", script_path.as_str()); + } + if let Some(root_job) = jc.job.flow_innermost_root_job.as_ref() { + span.record("root_job", root_job.to_string().as_str()); + } + + let root_job = handle_receive_completed_job( + jc, + &base_internal_url, + &db, + &worker_dir, + &same_worker_tx, + &worker_name, + job_completed_sender.clone(), + #[cfg(feature = "benchmark")] + bench, + ) + .instrument(span) + .await; + + if let Some(root_job) = root_job { + add_root_flow_job_to_otlp(&root_job, success); + } +} + pub fn start_background_processor( mut job_completed_rx: Receiver, - job_completed_sender: Sender, + job_completed_sender: JobCompletedSender, same_worker_queue_size: Arc, job_completed_processor_is_done: Arc, base_internal_url: String, @@ -86,66 +156,19 @@ pub fn start_background_processor( JobKind::Dependencies | JobKind::FlowDependencies ); - let success = jc.success; - - let span = tracing::span!( - tracing::Level::INFO, - "job_postprocessing", - job_id = %jc.job.id, root_job = field::Empty, workspace_id = %jc.job.workspace_id, worker = %worker_name,tag = %jc.job.tag, - // hostname = %hostname, - language = field::Empty, - script_path = field::Empty, - flow_step_id = field::Empty, - parent_job = field::Empty, - otel.name = field::Empty - ); - let rj = if let Some(root_job) = jc.job.flow_innermost_root_job { - root_job - } else { - jc.job.id - }; - windmill_common::otel_ee::set_span_parent(&span, &rj); - - if let Some(lg) = jc.job.script_lang.as_ref() { - span.record("language", lg.as_str()); - } - if let Some(step_id) = jc.job.flow_step_id.as_ref() { - span.record( - "otel.name", - format!("job_postprocessing {}", step_id).as_str(), - ); - span.record("flow_step_id", step_id.as_str()); - } else { - span.record("otel.name", "job postprocessing"); - } - if let Some(parent_job) = jc.job.parent_job.as_ref() { - span.record("parent_job", parent_job.to_string().as_str()); - } - if let Some(script_path) = jc.job.runnable_path.as_ref() { - span.record("script_path", script_path.as_str()); - } - if let Some(root_job) = jc.job.flow_innermost_root_job.as_ref() { - span.record("root_job", root_job.to_string().as_str()); - } - - let root_job = handle_receive_completed_job( + process_jc( jc, + &worker_name, &base_internal_url, &db, &worker_dir, &same_worker_tx, - &worker_name, - job_completed_sender.clone(), + &job_completed_sender, #[cfg(feature = "benchmark")] &mut bench, ) - .instrument(span) .await; - if let Some(root_job) = root_job { - add_root_flow_job_to_otlp(&root_job, success); - } - if is_init_script_and_failure { tracing::error!("init script errored, exiting"); killpill_tx.send(); @@ -250,7 +273,7 @@ async fn send_job_completed( duration, }; job_completed_tx - .send(jc) + .send_job(jc) .with_context(windmill_common::otel_ee::otel_ctx()) .await .expect("send job completed") @@ -267,20 +290,22 @@ pub async fn process_result( token: &str, column_order: Option>, new_args: Option>>, - db: &DB, + conn: &Connection, duration: Option, ) -> error::Result { match result { Ok(r) => { // Update script args to preprocessed args - if let Some(preprocessed_args) = new_args { - sqlx::query!( - "UPDATE v2_job SET args = $1, preprocessed = TRUE WHERE id = $2", - Json(preprocessed_args) as Json>>, - job.id - ) - .execute(db) - .await?; + if let Connection::Sql(db) = conn { + if let Some(preprocessed_args) = new_args { + sqlx::query!( + "UPDATE v2_job SET args = $1, preprocessed = TRUE WHERE id = $2", + Json(preprocessed_args) as Json>>, + job.id + ) + .execute(db) + .await?; + } } send_job_completed( @@ -307,18 +332,30 @@ pub async fn process_result( if res.as_ref().is_some_and(|x| !x.get().is_empty()) { res.unwrap() } else { - let last_10_log_lines = sqlx::query_scalar!( + match conn { + Connection::Sql(db) => { + let last_10_log_lines = sqlx::query_scalar!( "SELECT right(logs, 600) FROM job_logs WHERE job_id = $1 AND workspace_id = $2 ORDER BY created_at DESC LIMIT 1", &job.id, &job.workspace_id ).fetch_one(db).await.ok().flatten().unwrap_or("".to_string()); - let log_lines = last_10_log_lines - .split("CODE EXECUTION ---") - .last() - .unwrap_or(&last_10_log_lines); + let log_lines = last_10_log_lines + .split("CODE EXECUTION ---") + .last() + .unwrap_or(&last_10_log_lines); - extract_error_value(&program, log_lines, i, job.flow_step_id.clone()) + extract_error_value( + &program, + log_lines, + i, + job.flow_step_id.clone(), + ) + } + Connection::Http(_) => { + to_raw_value(&"See logs for more details".to_string()) + } + } } } err @ _ => to_raw_value(&SerializedError { @@ -355,7 +392,7 @@ pub async fn handle_receive_completed_job( worker_dir: &str, same_worker_tx: &SameWorkerSender, worker_name: &str, - job_completed_tx: Sender, + job_completed_tx: JobCompletedSender, #[cfg(feature = "benchmark")] bench: &mut BenchmarkIter, ) -> Option> { let token = jc.token.clone(); @@ -422,7 +459,7 @@ pub async fn process_completed_job( worker_dir: &str, same_worker_tx: SameWorkerSender, worker_name: &str, - job_completed_tx: Sender, + job_completed_tx: JobCompletedSender, #[cfg(feature = "benchmark")] bench: &mut BenchmarkIter, ) -> error::Result>> { if success { @@ -543,7 +580,7 @@ pub async fn process_completed_job( #[tracing::instrument(name = "job_error", level = "info", skip_all, fields(job_id = %job.id))] pub async fn handle_job_error( - db: &Pool, + db: &DB, client: &AuthedClient, job: &MiniPulledJob, mem_peak: i32, @@ -553,20 +590,17 @@ pub async fn handle_job_error( same_worker_tx: SameWorkerSender, worker_dir: &str, worker_name: &str, - job_completed_tx: Sender, + job_completed_tx: JobCompletedSender, #[cfg(feature = "benchmark")] bench: &mut BenchmarkIter, ) { - let err = match err { - Error::JsonErr(err) => err, - _ => json!({"message": err.to_string(), "name": "InternalErr"}), - }; + let err = error_to_value(err); let update_job_future = || async { append_logs( &job.id, &job.workspace_id, format!("Unexpected error during job execution:\n{err:#?}"), - db, + &db.into(), ) .await; add_completed_job_error( @@ -626,7 +660,7 @@ pub async fn handle_job_error( &parent_job.id, &job.workspace_id, format!("Unexpected error during flow job error handling:\n{err}"), - db, + &db.into(), ) .await; let _ = add_completed_job_error( diff --git a/backend/windmill-worker/src/rust_executor.rs b/backend/windmill-worker/src/rust_executor.rs index 21ec46422d..9f0b783471 100644 --- a/backend/windmill-worker/src/rust_executor.rs +++ b/backend/windmill-worker/src/rust_executor.rs @@ -8,7 +8,7 @@ use tokio::{fs::File, io::AsyncReadExt, process::Command}; use windmill_common::{ error::{self, Error}, utils::calculate_hash, - worker::{save_cache, write_file}, + worker::{save_cache, write_file, Connection}, }; use windmill_queue::MiniPulledJob; use windmill_queue::{append_logs, CanceledBy}; @@ -127,7 +127,7 @@ pub async fn generate_cargo_lockfile( mem_peak: &mut i32, canceled_by: &mut Option, job_dir: &str, - db: &sqlx::Pool, + conn: &Connection, worker_name: &str, w_id: &str, occupancy_metrics: &mut OccupancyMetrics, @@ -153,7 +153,7 @@ pub async fn generate_cargo_lockfile( let gen_lockfile_process = start_child_process(gen_lockfile_cmd, CARGO_PATH.as_str()).await?; handle_child( job_id, - db, + conn, mem_peak, canceled_by, gen_lockfile_process, @@ -180,7 +180,7 @@ pub async fn build_rust_crate( mem_peak: &mut i32, canceled_by: &mut Option, job_dir: &str, - db: &sqlx::Pool, + conn: &Connection, worker_name: &str, w_id: &str, base_internal_url: &str, @@ -216,7 +216,7 @@ pub async fn build_rust_crate( let build_rust_process = start_child_process(build_rust_cmd, CARGO_PATH.as_str()).await?; handle_child( job_id, - db, + conn, mem_peak, canceled_by, build_rust_process, @@ -230,7 +230,7 @@ pub async fn build_rust_crate( None, ) .await?; - append_logs(job_id, w_id, "\n\n", db).await; + append_logs(job_id, w_id, "\n\n", conn).await; tokio::fs::copy( &format!("{job_dir}/target/release/main"), @@ -279,7 +279,7 @@ pub async fn handle_rust_job( mem_peak: &mut i32, canceled_by: &mut Option, job: &MiniPulledJob, - db: &sqlx::Pool, + conn: &Connection, client: &AuthedClient, parent_runnable_path: Option, inner_content: &str, @@ -314,11 +314,11 @@ pub async fn handle_rust_job( )) })?; - create_args_and_out_file(client, job, job_dir, db).await?; + create_args_and_out_file(client, job, job_dir, conn).await?; cache_logs } else { let logs1 = format!("{cache_logs}\n\n--- CARGO BUILD ---\n"); - append_logs(&job.id, &job.workspace_id, logs1, db).await; + append_logs(&job.id, &job.workspace_id, logs1, conn).await; gen_cargo_crate(inner_content, job_dir)?; @@ -328,14 +328,14 @@ pub async fn handle_rust_job( } } - create_args_and_out_file(client, job, job_dir, db).await?; + create_args_and_out_file(client, job, job_dir, conn).await?; build_rust_crate( &job.id, mem_peak, canceled_by, job_dir, - db, + conn, worker_name, &job.workspace_id, base_internal_url, @@ -346,10 +346,10 @@ pub async fn handle_rust_job( }; let logs2 = format!("{cache_logs}\n\n--- RUST CODE EXECUTION ---\n"); - append_logs(&job.id, &job.workspace_id, logs2, db).await; + append_logs(&job.id, &job.workspace_id, logs2, conn).await; let reserved_variables = - get_reserved_variables(job, &client.token, db, parent_runnable_path).await?; + get_reserved_variables(job, &client.token, conn, parent_runnable_path).await?; let child = if !*DISABLE_NSJAIL { let _ = write_file( @@ -400,7 +400,7 @@ pub async fn handle_rust_job( }; handle_child( &job.id, - db, + conn, mem_peak, canceled_by, child, diff --git a/backend/windmill-worker/src/snowflake_executor.rs b/backend/windmill-worker/src/snowflake_executor.rs index 377c553d56..a4aba420c0 100644 --- a/backend/windmill-worker/src/snowflake_executor.rs +++ b/backend/windmill-worker/src/snowflake_executor.rs @@ -9,6 +9,7 @@ use serde_json::{json, value::RawValue, Value}; use sha2::{Digest, Sha256}; use std::collections::HashMap; use windmill_common::error::to_anyhow; +use windmill_common::worker::Connection; use windmill_common::{error::Error, worker::to_raw_value}; use windmill_parser_sql::{parse_db_resource, parse_snowflake_sig, parse_sql_blocks}; @@ -249,14 +250,14 @@ pub async fn do_snowflake( job: &MiniPulledJob, client: &AuthedClient, query: &str, - db: &sqlx::Pool, + conn: &Connection, mem_peak: &mut i32, canceled_by: &mut Option, worker_name: &str, column_order: &mut Option>, occupancy_metrics: &mut OccupancyMetrics, ) -> windmill_common::error::Result> { - let snowflake_args = build_args_values(job, client, db).await?; + let snowflake_args = build_args_values(job, client, conn).await?; let inline_db_res_path = parse_db_resource(&query); @@ -362,7 +363,7 @@ pub async fn do_snowflake( json!(database.database.unwrap().to_uppercase()), ); } - let timeout = resolve_job_timeout(&db, &job.workspace_id, job.id, job.timeout) + let timeout = resolve_job_timeout(&conn, &job.workspace_id, job.id, job.timeout) .await .0 .as_secs(); @@ -371,7 +372,7 @@ pub async fn do_snowflake( let queries = parse_sql_blocks(query); let (timeout_duration, _, _) = - resolve_job_timeout(&db, &job.workspace_id, job.id, job.timeout).await; + resolve_job_timeout(&conn, &job.workspace_id, job.id, job.timeout).await; let http_client = build_http_client(timeout_duration)?; @@ -424,7 +425,7 @@ pub async fn do_snowflake( let r = run_future_with_polling_update_job_poller( job.id, job.timeout, - db, + conn, mem_peak, canceled_by, result_f.map_err(to_anyhow), diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 06ca5c79ac..d7cf3c3560 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -12,16 +12,15 @@ use anyhow::anyhow; use futures::TryFutureExt; use windmill_common::{ + agent_workers::DECODED_AGENT_TOKEN, apps::AppScriptId, - auth::{fetch_authed_from_permissioned_as, JWTAuthClaims, JobPerms}, cache::{future::FutureCachedExt, ScriptData, ScriptMetadata}, - jwt, schema::{should_validate_schema, SchemaValidator}, scripts::PREVIEW_IS_TAR_CODEBASE_HASH, utils::WarnAfterExt, worker::{ - get_memory, get_vcpus, get_windmill_memory_usage, get_worker_memory_usage, write_file, - ROOT_CACHE_DIR, ROOT_CACHE_NOMOUNT_DIR, TMP_DIR, + write_file, Connection, HttpClient, MAX_TIMEOUT, ROOT_CACHE_DIR, ROOT_CACHE_NOMOUNT_DIR, + TMP_DIR, }, KillpillSender, }; @@ -42,7 +41,7 @@ use windmill_common::METRICS_ENABLED; use reqwest::Response; use serde::{de::DeserializeOwned, Deserialize, Serialize}; -use sqlx::{types::Json, Pool, Postgres}; +use sqlx::types::Json; use std::{ collections::HashMap, fs::DirBuilder, @@ -62,15 +61,15 @@ use windmill_common::{ flows::FlowNodeId, jobs::JobKind, scripts::{get_full_hub_script_by_path, ScriptHash, ScriptLang, PREVIEW_IS_CODEBASE_HASH}, - users::SUPERADMIN_SECRET_EMAIL, utils::StripPath, - worker::{update_ping, CLOUD_HOSTED, NO_LOGS, WORKER_CONFIG, WORKER_GROUP}, + worker::{CLOUD_HOSTED, NO_LOGS, WORKER_CONFIG, WORKER_GROUP}, DB, IS_READY, }; use windmill_queue::{ - append_logs, canceled_job_to_result, empty_result, pull, push, CanceledBy, MiniPulledJob, - PulledJob, PushArgs, PushIsolationLevel, HTTP_CLIENT, + append_logs, canceled_job_to_result, empty_result, get_same_worker_job, pull, push_init_job, + CanceledBy, JobAndPerms, JobCompleted, MiniPulledJob, PrecomputedAgentInfo, PulledJob, + SameWorkerPayload, HTTP_CLIENT, }; #[cfg(feature = "prometheus")] @@ -86,7 +85,7 @@ use tokio::fs::symlink_file as symlink; use tokio::{ sync::{ - mpsc::{self, Sender}, + mpsc::{self, Receiver, Sender}, RwLock, }, task::JoinHandle, @@ -96,10 +95,11 @@ use tokio::{ use rand::Rng; use crate::{ + agent_workers::queue_init_job, bash_executor::{handle_bash_job, handle_powershell_job}, bun_executor::handle_bun_job, common::{ - build_args_map, cached_result_path, get_cached_resource_value_if_valid, + build_args_map, cached_result_path, error_to_value, get_cached_resource_value_if_valid, get_reserved_variables, update_worker_ping_for_failed_init_script, OccupancyMetrics, }, csharp_executor::handle_csharp_job, @@ -113,10 +113,11 @@ use crate::{ pg_executor::do_postgresql, result_processor::{process_result, start_background_processor}, schema::schema_validator_from_main_arg_sig, - worker_flow::{handle_flow, update_flow_status_in_progress}, + worker_flow::handle_flow, worker_lockfiles::{ handle_app_dependency_job, handle_dependency_job, handle_flow_dependency_job, }, + worker_utils::{insert_ping, queue_vacuum, update_worker_ping_full}, }; #[cfg(feature = "rust")] @@ -143,9 +144,6 @@ use crate::mysql_executor::do_mysql; #[cfg(feature = "oracledb")] use crate::oracledb_executor::do_oracledb; -use backon::ConstantBuilder; -use backon::{BackoffBuilder, Retryable}; - #[cfg(feature = "enterprise")] use crate::dedicated_worker::create_dedicated_worker_map; @@ -163,100 +161,6 @@ use crate::bench::{benchmark_init, BenchmarkInfo, BenchmarkIter}; use windmill_common::add_time; -// struct Permission -pub async fn create_token(db: &DB, job: &MiniPulledJob, perms: Option) -> String { - // skipping test runs - if job.workspace_id != "" { - let label = if job.permissioned_as != format!("u/{}", job.created_by) - && job.permissioned_as != job.created_by - { - format!("ephemeral-script-end-user-{}", job.created_by) - } else { - "ephemeral-script".to_string() - }; - create_token_for_owner( - &db, - &job.workspace_id, - &job.permissioned_as, - &label, - *SCRIPT_TOKEN_EXPIRY, - &job.permissioned_as_email, - &job.id, - perms, - ) - .warn_after_seconds(5) - .await - .expect("could not create job token") - } else { - return "".to_string(); - } -} - -#[tracing::instrument(level = "trace", skip_all)] -pub async fn create_token_for_owner( - db: &Pool, - w_id: &str, - owner: &str, - label: &str, - expires_in: u64, - email: &str, - job_id: &Uuid, - perms: Option, -) -> error::Result { - // TODO: Bad implementation. We should not have access to this DB here. - if let Some(token) = JOB_TOKEN.as_ref() { - return Ok(token.clone()); - } - - let job_perms = if perms.is_some() { - Ok(perms) - } else { - sqlx::query_as!( - JobPerms, - "SELECT email, username, is_admin, is_operator, groups, folders FROM job_perms WHERE job_id = $1 AND workspace_id = $2", - job_id, - w_id - ) - .fetch_optional(db) - .await - }; - - let job_authed = match job_perms { - Ok(Some(jp)) => jp.into(), - _ => { - tracing::warn!("Could not get permissions for job {job_id} from job_perms table, getting permissions directly..."); - fetch_authed_from_permissioned_as(owner.to_string(), email.to_string(), w_id, db) - .await - .map_err(|e| { - Error::internal_err(format!( - "Could not get permissions directly for job {job_id}: {e:#}" - )) - })? - } - }; - - let payload = JWTAuthClaims { - email: job_authed.email, - username: job_authed.username, - is_admin: job_authed.is_admin, - is_operator: job_authed.is_operator, - groups: job_authed.groups, - folders: job_authed.folders, - label: Some(label.to_string()), - workspace_id: w_id.to_string(), - exp: (chrono::Utc::now() + chrono::Duration::seconds(expires_in as i64)).timestamp() - as usize, - job_id: Some(job_id.to_string()), - scopes: None, - }; - - let token = jwt::encode_with_internal_secret(&payload) - .await - .with_context(|| format!("Could not encode JWT token for job {job_id}"))?; - - Ok(format!("jwt_{}", token)) -} - pub const PY310_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_310"); pub const PY311_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_311"); pub const PY312_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_312"); @@ -299,8 +203,6 @@ const NUM_SECS_READINGS: u64 = 60; const INCLUDE_DEPS_PY_SH_CONTENT: &str = include_str!("../nsjail/download_deps.py.sh"); -pub const DEFAULT_CLOUD_TIMEOUT: u64 = 900; -pub const DEFAULT_SELFHOSTED_TIMEOUT: u64 = 604800; // 7 days pub const DEFAULT_SLEEP_QUEUE: u64 = 50; // only 1 native job so that we don't have to worry about concurrency issues on non dedicated native jobs workers @@ -341,16 +243,20 @@ const DOTNET_DEFAULT_PATH: &str = "/usr/bin/dotnet"; lazy_static::lazy_static! { - pub static ref JOB_TOKEN: Option = std::env::var("JOB_TOKEN").ok(); - pub static ref SLEEP_QUEUE: u64 = std::env::var("SLEEP_QUEUE") .ok() - .and_then(|x| x.parse::().ok()) - .unwrap_or(DEFAULT_SLEEP_QUEUE * std::env::var("NUM_WORKERS") - .ok() - .map(|x| x.parse().ok()) - .flatten() - .unwrap_or(2) / 2); + .and_then(|x| x.parse::().ok()) + .unwrap_or_else(|| { + if std::env::var("MODE").unwrap_or_default() == "agent" { + 1000 + } else { + DEFAULT_SLEEP_QUEUE * std::env::var("NUM_WORKERS") + .ok() + .map(|x| x.parse().ok()) + .flatten() + .unwrap_or(2) / 2 + } + }); pub static ref DISABLE_NUSER: bool = std::env::var("DISABLE_NUSER") @@ -420,10 +326,7 @@ lazy_static::lazy_static! { pub static ref INSTANCE_PYTHON_VERSION: Arc>> = Arc::new(RwLock::new(None)); pub static ref JOB_DEFAULT_TIMEOUT: Arc>> = Arc::new(RwLock::new(None)); - static ref MAX_TIMEOUT: u64 = std::env::var("TIMEOUT") - .ok() - .and_then(|x| x.parse::().ok()) - .unwrap_or_else(|| if *CLOUD_HOSTED { DEFAULT_CLOUD_TIMEOUT } else { DEFAULT_SELFHOSTED_TIMEOUT }); + pub static ref MAX_WAIT_FOR_SIGINT: u64 = std::env::var("MAX_WAIT_FOR_SIGINT") .ok() @@ -437,10 +340,6 @@ lazy_static::lazy_static! { pub static ref MAX_TIMEOUT_DURATION: Duration = Duration::from_secs(*MAX_TIMEOUT); - pub static ref SCRIPT_TOKEN_EXPIRY: u64 = std::env::var("SCRIPT_TOKEN_EXPIRY") - .ok() - .and_then(|x| x.parse::().ok()) - .unwrap_or(*MAX_TIMEOUT); pub static ref GLOBAL_CACHE_INTERVAL: u64 = std::env::var("GLOBAL_CACHE_INTERVAL") .ok() @@ -621,22 +520,85 @@ impl AuthedClient { #[allow(dead_code)] #[derive(Clone)] -pub struct JobCompletedSender(Sender); +pub enum JobCompletedSender { + Sql(Sender), + Http(HttpClient), + NeverUsed, +} + +impl JobCompletedSender { + pub fn new(conn: &Connection, buffer_size: usize) -> (Self, Option>) { + match conn { + Connection::Sql(_) => { + let (sender, receiver) = mpsc::channel::(buffer_size); + (Self::Sql(sender), Some(receiver)) + } + Connection::Http(client) => (Self::Http(client.clone()), None), + } + } + pub fn new_never_used() -> (Self, Option>) { + (Self::NeverUsed, None) + } +} #[derive(Clone)] pub struct SameWorkerSender(pub Sender, pub Arc); -pub struct SameWorkerPayload { - pub job_id: Uuid, - pub recoverable: bool, -} - impl JobCompletedSender { + pub async fn send_job(&self, jc: JobCompleted) -> anyhow::Result<()> { + match self { + Self::Sql(sender) => sender + .send(SendResult::JobCompleted(jc)) + .await + .map_err(|_e| { + anyhow::anyhow!("Failed to send job completed to background processor") + }), + Self::Http(client) => { + crate::agent_workers::send_result(client, jc).await?; + Ok(()) + } + Self::NeverUsed => { + tracing::error!( + "Sending job completed to NeverUsed JobCompletedSender, this should not happen" + ); + Ok(()) + } + } + } + pub async fn send( &self, - jc: JobCompleted, + send_result: SendResult, ) -> Result<(), tokio::sync::mpsc::error::SendError> { - self.0.send(SendResult::JobCompleted(jc)).await + match self { + Self::Sql(sender) => sender.send(send_result).await, + Self::Http(_) => { + tracing::error!("Sending job completed to http client, this should not happen"); + Ok(()) + } + Self::NeverUsed => { + tracing::error!( + "Sending job completed to NeverUsed JobCompletedSender, this should not happen" + ); + Ok(()) + } + } + } + + pub async fn kill(&self) -> Result<(), tokio::sync::mpsc::error::SendError> { + match self { + Self::Sql(sender) => sender.send(SendResult::Kill).await, + Self::Http(_) => { + tracing::error!("Sending kill to http client, this should not happen"); + Ok(()) + } + Self::NeverUsed => { + tracing::error!( + "Sending kill to NeverUsed JobCompletedSender, this should not happen" + ); + Ok(()) + } + } } } @@ -679,17 +641,17 @@ const OUTSTANDING_WAIT_TIME_THRESHOLD_MS: i64 = 1000; async fn insert_wait_time( job_id: Uuid, root_job_id: Option, - db: &Pool, + db: &DB, wait_time: i64, ) -> sqlx::error::Result<()> { sqlx::query!( - "INSERT INTO outstanding_wait_time(job_id, self_wait_time_ms) VALUES ($1, $2) - ON CONFLICT (job_id) DO UPDATE SET self_wait_time_ms = EXCLUDED.self_wait_time_ms", - job_id, - wait_time - ) - .execute(db) - .await?; + "INSERT INTO outstanding_wait_time(job_id, self_wait_time_ms) VALUES ($1, $2) + ON CONFLICT (job_id) DO UPDATE SET self_wait_time_ms = EXCLUDED.self_wait_time_ms", + job_id, + wait_time + ) + .execute(db) + .await?; if let Some(root_id) = root_job_id { // TODO: queued_job.root_job is not guaranteed to be the true root job (e.g. parallel flow @@ -700,16 +662,16 @@ async fn insert_wait_time( COALESCE(outstanding_wait_time.aggregate_wait_time_ms, 0) + EXCLUDED.aggregate_wait_time_ms", root_id, wait_time - ) - .execute(db) - .await?; + ) + .execute(db) + .await?; } Ok(()) } fn add_outstanding_wait_time( + conn: &Connection, queued_job: &MiniPulledJob, - db: &Pool, waiting_threshold: i64, ) -> () { let wait_time; @@ -726,22 +688,21 @@ fn add_outstanding_wait_time( let job_id = queued_job.id; let root_job_id = queued_job.flow_innermost_root_job; - let db = db.clone(); + let conn = conn.clone(); - tokio::spawn(async move { + if let Some(db) = conn.as_sql() { + let db = db.clone(); + tokio::spawn(async move { match insert_wait_time(job_id, root_job_id, &db, wait_time).await { Ok(()) => tracing::warn!("job {job_id} waited for an executor for a significant amount of time. Recording value wait_time={}ms", wait_time), Err(e) => tracing::error!("Failed to insert outstanding wait time: {}", e), } - }.in_current_span()); + }.in_current_span()); + } } -// struct WorkerMtrics { -// job_ -// } - pub async fn run_worker( - db: &Pool, + conn: &Connection, hostname: &str, worker_name: String, i_worker: u64, @@ -750,7 +711,6 @@ pub async fn run_worker( mut killpill_rx: tokio::sync::broadcast::Receiver<()>, killpill_tx: KillpillSender, base_internal_url: &str, - agent_mode: bool, ) { #[cfg(not(feature = "enterprise"))] if !*DISABLE_NSJAIL { @@ -767,16 +727,16 @@ pub async fn run_worker( #[cfg(feature = "python")] { - let (db, worker_name, hostname, worker_dir) = ( - db.clone(), + let (conn, worker_name, hostname, worker_dir) = ( + conn.clone(), worker_name.clone(), hostname.to_owned(), worker_dir.clone(), ); tokio::spawn(async move { - if let Err(e) = PyVersion::from_instance_version(&Uuid::nil(), "", &db) + if let Err(e) = PyVersion::from_instance_version(&Uuid::nil(), "", &conn) .await - .get_python(&Uuid::nil(), &mut 0, &db, &worker_name, "", &mut None) + .get_python(&Uuid::nil(), &mut 0, &conn, &worker_name, "", &mut None) .await { tracing::error!( @@ -787,7 +747,7 @@ pub async fn run_worker( ); } if let Err(e) = PyVersion::Py311 - .get_python(&Uuid::nil(), &mut 0, &db, &worker_name, "", &mut None) + .get_python(&Uuid::nil(), &mut 0, &conn, &worker_name, "", &mut None) .await { tracing::error!( @@ -820,7 +780,9 @@ pub async fn run_worker( let mut last_ping = Instant::now() - Duration::from_secs(NUM_SECS_PING + 1); - update_ping(hostname, &worker_name, ip, db).await; + insert_ping(hostname, &worker_name, ip, conn) + .await + .expect("initial ping could be sent"); #[cfg(feature = "prometheus")] let uptime_metric = if METRICS_ENABLED.load(Ordering::Relaxed) { @@ -1029,7 +991,11 @@ pub async fn run_worker( .unwrap(); #[cfg(feature = "benchmark")] - benchmark_init(benchmark_jobs, &db).await; + { + if let Some(db) = conn.as_sql() { + benchmark_init(benchmark_jobs, db).await; + } + } #[cfg(feature = "prometheus")] if let Some(ws) = WORKER_STARTED.as_ref() { @@ -1038,27 +1004,28 @@ pub async fn run_worker( let (same_worker_tx, mut same_worker_rx) = mpsc::channel::(5); - let (job_completed_tx, job_completed_rx) = mpsc::channel::(3); - - let job_completed_tx = JobCompletedSender(job_completed_tx); + let (job_completed_tx, job_completed_rx) = JobCompletedSender::new(&conn, 3); let same_worker_queue_size = Arc::new(AtomicU16::new(0)); let same_worker_tx = SameWorkerSender(same_worker_tx, same_worker_queue_size.clone()); let job_completed_processor_is_done = Arc::new(AtomicBool::new(false)); - let send_result = start_background_processor( - job_completed_rx, - job_completed_tx.0.clone(), - same_worker_queue_size.clone(), - job_completed_processor_is_done.clone(), - base_internal_url.to_string(), - db.clone(), - worker_dir.clone(), - same_worker_tx.clone(), - worker_name.clone(), - killpill_tx.clone(), - is_dedicated_worker, - ); + let send_result = match (conn, job_completed_rx) { + (Connection::Sql(db), Some(job_completed_rx)) => Some(start_background_processor( + job_completed_rx, + job_completed_tx.clone(), + same_worker_queue_size.clone(), + job_completed_processor_is_done.clone(), + base_internal_url.to_string(), + db.clone(), + worker_dir.clone(), + same_worker_tx.clone(), + worker_name.clone(), + killpill_tx.clone(), + is_dedicated_worker, + )), + _ => None, + }; let mut last_executed_job: Option = None; @@ -1071,12 +1038,20 @@ pub async fn run_worker( let vacuum_shift = rand::rng().random_range(0..VACUUM_PERIOD); IS_READY.store(true, Ordering::Relaxed); - tracing::info!( - worker = %worker_name, hostname = %hostname, - "listening for jobs, WORKER_GROUP: {}, config: {:?}", - *WORKER_GROUP, - WORKER_CONFIG.read().await - ); + if let Some(token) = DECODED_AGENT_TOKEN.as_ref() { + tracing::info!( + worker = %worker_name, hostname = %hostname, + "listening for jobs, agent mode, tags: {:?}", + token.tags + ); + } else { + tracing::info!( + worker = %worker_name, hostname = %hostname, + "listening for jobs, WORKER_GROUP: {}, config: {:?}", + *WORKER_GROUP, + WORKER_CONFIG.read().await + ); + } // (dedi_path, dedicated_worker_tx, dedicated_worker_handle) // Option>>, @@ -1087,16 +1062,21 @@ pub async fn run_worker( HashMap>>, bool, Vec>, - ) = create_dedicated_worker_map( - &killpill_tx, - &killpill_rx, - db, - &worker_dir, - base_internal_url, - &worker_name, - &job_completed_tx, - ) - .await; + ) = match conn { + Connection::Sql(pool) => { + create_dedicated_worker_map( + &killpill_tx, + &killpill_rx, + pool, + &worker_dir, + base_internal_url, + &worker_name, + &job_completed_tx, + ) + .await + } + Connection::Http(_) => (HashMap::new(), false, vec![]), + }; #[cfg(not(feature = "enterprise"))] let (dedicated_workers, is_flow_worker, dedicated_handles): ( @@ -1106,7 +1086,7 @@ pub async fn run_worker( ) = (HashMap::new(), false, vec![]); if i_worker == 1 { - if let Err(e) = queue_init_bash_maybe(db, same_worker_tx.clone(), &worker_name).await { + if let Err(e) = queue_init_bash_maybe(conn, same_worker_tx.clone(), &worker_name).await { killpill_tx.send(); tracing::error!(worker = %worker_name, hostname = %hostname, "Error queuing init bash script for worker {worker_name}: {e:#}"); return; @@ -1143,11 +1123,12 @@ pub async fn run_worker( { if let Ok(_) = killpill_rx.try_recv() { tracing::info!(worker = %worker_name, hostname = %hostname, "killpill received on worker waiting for valid key"); - job_completed_tx - .0 - .send(SendResult::Kill) - .await - .expect("send kill to job completed tx"); + if send_result.is_some() { + job_completed_tx + .kill() + .await + .expect("send kill to job completed tx"); + } break; } let valid_key = *LICENSE_KEY_VALID.read().await; @@ -1184,85 +1165,27 @@ pub async fn run_worker( } if last_ping.elapsed().as_secs() > NUM_SECS_PING { - let tags = WORKER_CONFIG.read().await.worker_tags.clone(); - - let memory_usage = get_worker_memory_usage(); - let wm_memory_usage = get_windmill_memory_usage(); - - let (vcpus, memory) = if *REFRESH_CGROUP_READINGS - && last_reading.elapsed().as_secs() > NUM_SECS_READINGS - { - last_reading = Instant::now(); - (get_vcpus(), get_memory()) - } else { - (None, None) - }; - - let (occupancy_rate, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m) = - occupancy_metrics.update_occupancy_metrics(); - - if let Err(e) = (|| sqlx::query!( - "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1, custom_tags = $2, - occupancy_rate = $3, memory_usage = $4, wm_memory_usage = $5, vcpus = COALESCE($7, vcpus), - memory = COALESCE($8, memory), occupancy_rate_15s = $9, occupancy_rate_5m = $10, occupancy_rate_30m = $11 WHERE worker = $6", + let read_cgroups = + *REFRESH_CGROUP_READINGS && last_reading.elapsed().as_secs() > NUM_SECS_READINGS; + update_worker_ping_full( + &conn, + read_cgroups, jobs_executed, - tags.as_slice(), - occupancy_rate, - memory_usage, - wm_memory_usage, &worker_name, - vcpus, - memory, - occupancy_rate_15s, - occupancy_rate_5m, - occupancy_rate_30m - ).execute(db)).retry( - ConstantBuilder::default() - .with_delay(std::time::Duration::from_secs(2)) - .with_max_times(10) - .build(), + &hostname, + &mut occupancy_metrics, + &killpill_tx, ) - .notify(|err, dur| { - tracing::error!( - worker = %worker_name, hostname = %hostname, - "retrying updating worker ping in {dur:#?}, err: {err:#?}" - ); - }) - .sleep(tokio::time::sleep) - .await { - tracing::error!( - worker = %worker_name, hostname = %hostname, - "failed to update worker ping, exiting: {}", e); - killpill_tx.send(); - } - tracing::info!( - worker = %worker_name, hostname = %hostname, - "ping update, memory: container={}MB, windmill={}MB", - memory_usage.unwrap_or_default() / (1024 * 1024), - wm_memory_usage.unwrap_or_default() / (1024 * 1024) - ); + .await; + if read_cgroups { + last_reading = Instant::now(); + } last_ping = Instant::now(); } if (jobs_executed as u32 + vacuum_shift) % VACUUM_PERIOD == 0 { - let db2 = db.clone(); - let current_span = tracing::Span::current(); - let worker_name = worker_name.clone(); - let hostname = hostname.to_string(); - tokio::task::spawn( - (async move { - tracing::info!(worker = %worker_name, hostname = %hostname, "vacuuming queue"); - if let Err(e) = sqlx::query!("VACUUM v2_job_queue, v2_job_runtime, v2_job_status") - .execute(&db2) - .await - { - tracing::error!(worker = %worker_name, hostname = %hostname, "failed to vacuum queue: {}", e); - } - tracing::info!(worker = %worker_name, hostname = %hostname, "vacuumed queue"); - }) - .instrument(current_span), - ); + queue_vacuum(&conn, &worker_name, &hostname).await; jobs_executed += 1; } @@ -1276,14 +1199,36 @@ pub async fn run_worker( if benchmark_jobs > 0 && infos.iters == benchmark_jobs as u64 { tracing::info!("benchmark finished, exiting"); job_completed_tx - .0 - .send(SendResult::Kill) + .kill() .await .expect("send kill to job completed tx"); break; } else { tracing::info!("benchmark not finished, still pulling jobs {}", infos.iters); } + enum NextJob { + Sql(PulledJob), + Http(JobAndPerms), + } + + impl NextJob { + pub fn job(self) -> MiniPulledJob { + match self { + NextJob::Sql(job) => job.job, + NextJob::Http(job) => job.job, + } + } + } + + impl std::ops::Deref for NextJob { + type Target = MiniPulledJob; + fn deref(&self) -> &Self::Target { + match self { + NextJob::Sql(job) => &job.job, + NextJob::Http(job) => &job.job, + } + } + } let next_job = { // println!("2: {:?}", instant.elapsed()); @@ -1299,29 +1244,42 @@ pub async fn run_worker( "received {} from same worker channel", same_worker_job.job_id ); - let job = get_same_worker_job(db, &same_worker_job).await; - // tracing::error!("r: {:?}", r); - if job.is_err() && !same_worker_job.recoverable { - tracing::error!( - worker = %worker_name, hostname = %hostname, - "failed to fetch same_worker job on a non recoverable job, exiting" - ); - job_completed_tx - .0 - .send(SendResult::Kill) + + match &conn { + Connection::Sql(db) => { + let job = get_same_worker_job(db, &same_worker_job).await; + // tracing::error!("r: {:?}", r); + if job.is_err() && !same_worker_job.recoverable { + tracing::error!( + worker = %worker_name, hostname = %hostname, + "failed to fetch same_worker job on a non recoverable job, exiting" + ); + job_completed_tx + .kill() + .await + .expect("send kill to job completed tx"); + break; + } else { + job.map(|x| x.map(NextJob::Sql)) + } + } + Connection::Http(client) => client + .post( + &format!( + "/api/agent_workers/same_worker_job/{}", + same_worker_job.job_id + ), + &same_worker_job, + ) .await - .expect("send kill to job completed tx"); - break; - } else { - job + .map_err(|e| error::Error::InternalErr(e.to_string())) + .map(|x: Option| x.map(|y| NextJob::Http(y))), } } else if let Ok(_) = killpill_rx.try_recv() { if !killed_but_draining_same_worker_jobs { - tracing::info!(worker = %worker_name, hostname = %hostname, "received killpill for worker {}, jobs are not pulled anymore except same_worker jobs", i_worker); killed_but_draining_same_worker_jobs = true; job_completed_tx - .0 - .send(SendResult::Kill) + .kill() .await .expect("send kill to job completed tx"); } @@ -1336,75 +1294,84 @@ pub async fn run_worker( continue; } } else { - let pull_time = Instant::now(); - let likelihood_of_suspend = - (1.0 + last_30jobs_suspended.iter().filter(|&&x| x).count() as f64) / 31.0; - let suspend_first = suspend_first_success - || rand::random::() < likelihood_of_suspend - || last_suspend_first.elapsed().as_secs_f64() > 5.0; + match &conn { + Connection::Sql(db) => { + let pull_time = Instant::now(); + let likelihood_of_suspend = (1.0 + + last_30jobs_suspended.iter().filter(|&&x| x).count() as f64) + / 31.0; + let suspend_first = suspend_first_success + || rand::random::() < likelihood_of_suspend + || last_suspend_first.elapsed().as_secs_f64() > 5.0; - if suspend_first { - last_suspend_first = Instant::now(); + if suspend_first { + last_suspend_first = Instant::now(); + } + + let job = pull(&db, suspend_first, &worker_name, None).await; + + add_time!(bench, "job pulled from DB"); + let duration_pull_s = pull_time.elapsed().as_secs_f64(); + let err_pull = job.is_ok(); + // let empty = job.as_ref().is_ok_and(|x| x.is_none()); + + if duration_pull_s > 0.5 { + let empty = job.as_ref().is_ok_and(|x| x.job.is_none()); + tracing::warn!(worker = %worker_name, hostname = %hostname, "pull took more than 0.5s ({duration_pull_s}), this is a sign that the database is VERY undersized for this load. empty: {empty}, err: {err_pull}"); + #[cfg(feature = "prometheus")] + if empty { + if let Some(wp) = worker_pull_over_500_counter_empty.as_ref() { + wp.inc(); + } + } else if let Some(wp) = worker_pull_over_500_counter.as_ref() { + wp.inc(); + } + } else if duration_pull_s > 0.1 { + let empty = job.as_ref().is_ok_and(|x| x.job.is_none()); + tracing::warn!(worker = %worker_name, hostname = %hostname, "pull took more than 0.1s ({duration_pull_s}) this is a sign that the database is undersized for this load. empty: {empty}, err: {err_pull}"); + #[cfg(feature = "prometheus")] + if empty { + if let Some(wp) = worker_pull_over_100_counter_empty.as_ref() { + wp.inc(); + } + } else if let Some(wp) = worker_pull_over_100_counter.as_ref() { + wp.inc(); + } + } + + if let Ok(j) = job.as_ref() { + let suspend_success = j.suspended; + if suspend_first { + last_30jobs_suspended.push(suspend_success); + if last_30jobs_suspended.len() > 30 { + last_30jobs_suspended.remove(0); + } + } + suspend_first_success = suspend_first && suspend_success; + #[cfg(feature = "prometheus")] + if j.job.is_some() { + if let Some(wp) = worker_pull_duration_counter.as_ref() { + wp.inc_by(duration_pull_s); + } + if let Some(wp) = worker_pull_duration.as_ref() { + wp.observe(duration_pull_s); + } + } else { + if let Some(wp) = worker_pull_duration_counter_empty.as_ref() { + wp.inc_by(duration_pull_s); + } + if let Some(wp) = worker_pull_duration_empty.as_ref() { + wp.observe(duration_pull_s); + } + } + } + job.map(|x| x.job.map(NextJob::Sql)) + } + Connection::Http(client) => crate::agent_workers::pull_job(&client) + .await + .map_err(|e| error::Error::InternalErr(e.to_string())) + .map(|x| x.map(|y| NextJob::Http(y))), } - - let job = pull(&db, suspend_first, &worker_name).await; - - add_time!(bench, "job pulled from DB"); - let duration_pull_s = pull_time.elapsed().as_secs_f64(); - let err_pull = job.is_ok(); - // let empty = job.as_ref().is_ok_and(|x| x.is_none()); - - if !agent_mode && duration_pull_s > 0.5 { - let empty = job.as_ref().is_ok_and(|x| x.0.is_none()); - tracing::warn!(worker = %worker_name, hostname = %hostname, "pull took more than 0.5s ({duration_pull_s}), this is a sign that the database is VERY undersized for this load. empty: {empty}, err: {err_pull}"); - #[cfg(feature = "prometheus")] - if empty { - if let Some(wp) = worker_pull_over_500_counter_empty.as_ref() { - wp.inc(); - } - } else if let Some(wp) = worker_pull_over_500_counter.as_ref() { - wp.inc(); - } - } else if !agent_mode && duration_pull_s > 0.1 { - let empty = job.as_ref().is_ok_and(|x| x.0.is_none()); - tracing::warn!(worker = %worker_name, hostname = %hostname, "pull took more than 0.1s ({duration_pull_s}) this is a sign that the database is undersized for this load. empty: {empty}, err: {err_pull}"); - #[cfg(feature = "prometheus")] - if empty { - if let Some(wp) = worker_pull_over_100_counter_empty.as_ref() { - wp.inc(); - } - } else if let Some(wp) = worker_pull_over_100_counter.as_ref() { - wp.inc(); - } - } - - if let Ok(j) = job.as_ref() { - let suspend_success = j.1; - if suspend_first { - last_30jobs_suspended.push(suspend_success); - if last_30jobs_suspended.len() > 30 { - last_30jobs_suspended.remove(0); - } - } - suspend_first_success = suspend_first && suspend_success; - #[cfg(feature = "prometheus")] - if j.0.is_some() { - if let Some(wp) = worker_pull_duration_counter.as_ref() { - wp.inc_by(duration_pull_s); - } - if let Some(wp) = worker_pull_duration.as_ref() { - wp.observe(duration_pull_s); - } - } else { - if let Some(wp) = worker_pull_duration_counter_empty.as_ref() { - wp.inc_by(duration_pull_s); - } - if let Some(wp) = worker_pull_duration_empty.as_ref() { - wp.observe(duration_pull_s); - } - } - } - job.map(|x| x.0) } }; @@ -1432,7 +1399,8 @@ pub async fn run_worker( }; if let Some(key) = key_o { if let Some(dedicated_worker_tx) = dedicated_workers.get(&key) { - if let Err(e) = dedicated_worker_tx.send(Arc::new(job.job)).await { + if let Err(e) = dedicated_worker_tx.send(Arc::new(job.job())).await + { tracing::info!("failed to send jobs to dedicated workers. Likely dedicated worker has been shut down. This is normal: {e:?}"); } @@ -1450,8 +1418,8 @@ pub async fn run_worker( if matches!(job.kind, JobKind::Noop) { add_time!(bench, "send job completed START"); job_completed_tx - .send(JobCompleted { - job: Arc::new(job.job), + .send_job(JobCompleted { + job: Arc::new(job.job()), success: true, result: Arc::new(empty_result()), result_columns: None, @@ -1465,7 +1433,7 @@ pub async fn run_worker( .expect("send job completed END"); add_time!(bench, "sent job completed"); } else { - add_outstanding_wait_time(&job, db, OUTSTANDING_WAIT_TIME_THRESHOLD_MS); + add_outstanding_wait_time(&conn, &job, OUTSTANDING_WAIT_TIME_THRESHOLD_MS); #[cfg(feature = "prometheus")] register_metric( @@ -1569,46 +1537,23 @@ pub async fn run_worker( let tag = job.tag.clone(); let is_init_script: bool = job.tag.as_str() == INIT_SCRIPT_TAG; - let PulledJob { + let JobAndPerms { job, raw_code, raw_lock, raw_flow, parent_runnable_path, - permissioned_as_email, - permissioned_as_username, - permissioned_as_is_admin, - permissioned_as_is_operator, - permissioned_as_groups, - permissioned_as_folders, - } = job; - let job_perms = match ( - permissioned_as_email, - permissioned_as_username, - permissioned_as_is_admin, - permissioned_as_is_operator, - permissioned_as_groups, - permissioned_as_folders, - ) { - ( - Some(email), - Some(username), - Some(is_admin), - Some(is_operator), - Some(groups), - Some(folders), - ) => Some(JobPerms { - email, - username, - is_admin, - is_operator, - groups, - folders, - }), - _ => None, + token, + precomputed_agent_info: precomputed_bundle, + } = match (job, &conn) { + (NextJob::Sql(job), Connection::Sql(db)) => job.get_job_and_perms(db).await, + (NextJob::Sql(_), Connection::Http(_)) => { + panic!("sql job on http connection") + } + (NextJob::Http(job), _) => job, }; - let token = create_token(&db, &job, job_perms).await; + // let token = create_token(&db, &job, job_perms).await; let authed_client = AuthedClient { base_internal_url: base_internal_url.to_string(), token, @@ -1657,7 +1602,7 @@ pub async fn run_worker( raw_lock, raw_flow, parent_runnable_path, - db, + conn, &authed_client, &hostname, &worker_name, @@ -1668,6 +1613,7 @@ pub async fn run_worker( job_completed_tx.clone(), &mut occupancy_metrics, &mut killpill_rx2, + precomputed_bundle, #[cfg(feature = "benchmark")] &mut bench, ) @@ -1675,26 +1621,50 @@ pub async fn run_worker( .await { Err(err) => { - handle_job_error( - db, - &authed_client, - arc_job.as_ref(), - 0, - None, - err, - false, - same_worker_tx.clone(), - &worker_dir, - &worker_name, - (&job_completed_tx.0).clone(), - #[cfg(feature = "benchmark")] - &mut bench, - ) - .await; + match conn { + Connection::Sql(db) => { + handle_job_error( + db, + &authed_client, + arc_job.as_ref(), + 0, + None, + err, + false, + same_worker_tx.clone(), + &worker_dir, + &worker_name, + job_completed_tx.clone(), + #[cfg(feature = "benchmark")] + &mut bench, + ) + .await; + } + Connection::Http(_) => { + job_completed_tx + .send_job(JobCompleted { + job: arc_job.clone(), + result: Arc::new( + windmill_common::worker::to_raw_value( + &error_to_value(err), + ), + ), + result_columns: None, + mem_peak: 0, + canceled_by: None, + success: false, + cached_res_path: None, + token: authed_client.token.clone(), + duration: None, + }) + .await + .expect("send job completed"); + } + } if is_init_script { tracing::error!("init script job failed (in handler), exiting"); update_worker_ping_for_failed_init_script( - db, + conn, &worker_name, arc_job.id, ) @@ -1704,8 +1674,12 @@ pub async fn run_worker( } Ok(false) if is_init_script => { tracing::error!("init script job failed, exiting"); - update_worker_ping_for_failed_init_script(db, &worker_name, arc_job.id) - .await; + update_worker_ping_for_failed_init_script( + conn, + &worker_name, + arc_job.id, + ) + .await; break; } _ => {} @@ -1810,126 +1784,42 @@ pub async fn run_worker( drop(job_completed_tx); tracing::info!(worker = %worker_name, hostname = %hostname, "waiting for job_completed_processor to finish processing remaining jobs"); - if let Err(e) = send_result.await { - tracing::error!("error in awaiting send_result process: {e:?}") + if let Some(send_result) = send_result { + if let Err(e) = send_result.await { + tracing::error!("error in awaiting send_result process: {e:?}") + } } tracing::info!(worker = %worker_name, hostname = %hostname, "worker {} exited", worker_name); tracing::info!(worker = %worker_name, hostname = %hostname, "number of jobs executed: {}", jobs_executed); } -async fn get_same_worker_job( - db: &Pool, - same_worker_job: &SameWorkerPayload, -) -> windmill_common::error::Result> { - sqlx::query_as::<_, PulledJob>( - "WITH ping AS ( - UPDATE v2_job_runtime SET ping = NOW() WHERE id = $1 - ), - started_at AS ( - UPDATE v2_job_queue SET started_at = NOW() WHERE id = $1 - ) - SELECT - v2_job_queue.workspace_id, - v2_job_queue.id, - v2_job.args, - v2_job.parent_job, - v2_job.created_by, - v2_job_queue.started_at, - scheduled_for, - v2_job.runnable_path, - v2_job.kind, - v2_job.runnable_id, - v2_job_queue.canceled_reason, - v2_job_queue.canceled_by, - v2_job.permissioned_as, - v2_job.permissioned_as_email, - v2_job_status.flow_status, - v2_job.tag, - v2_job.script_lang, - v2_job.same_worker, - v2_job.pre_run_error, - v2_job.concurrent_limit, - v2_job.concurrency_time_window_s, - v2_job.flow_innermost_root_job, - v2_job.timeout, - v2_job.flow_step_id, - v2_job.cache_ttl, - v2_job_queue.priority, - v2_job.preprocessed, - v2_job.script_entrypoint_override, - v2_job.trigger, - v2_job.trigger_kind, - v2_job.visible_to_owner, - v2_job.raw_code, - v2_job.raw_lock, - v2_job.raw_flow, - pj.runnable_path as parent_runnable_path, - p.email as permissioned_as_email, p.username as permissioned_as_username, p.is_admin as permissioned_as_is_admin, - p.is_operator as permissioned_as_is_operator, p.groups as permissioned_as_groups, p.folders as permissioned_as_folders - FROM v2_job_queue - INNER JOIN v2_job ON v2_job.id = v2_job_queue.id - LEFT JOIN v2_job_status ON v2_job_status.id = v2_job_queue.id - LEFT JOIN job_perms p ON p.job_id = v2_job.id - LEFT JOIN v2_job pj ON v2_job.parent_job = pj.id - WHERE v2_job_queue.id = $1 -", - ) - .bind(same_worker_job.job_id) - .fetch_optional(db) - .await - .map_err(|e| { - Error::internal_err(format!( - "Impossible to fetch same_worker job {}: {}", - same_worker_job.job_id, e - )) - }) -} - async fn queue_init_bash_maybe<'c>( - db: &Pool, + conn: &Connection, same_worker_tx: SameWorkerSender, worker_name: &str, -) -> error::Result { - if let Some(content) = WORKER_CONFIG.read().await.init_bash.clone() { - let tx = PushIsolationLevel::IsolatedRoot(db.clone()); - let ehm = HashMap::new(); - let (uuid, inner_tx) = push( - &db, - tx, - "admins", - windmill_common::jobs::JobPayload::Code(windmill_common::jobs::RawCode { - hash: None, - content: content.clone(), - path: Some(format!("init_script_{worker_name}")), - language: ScriptLang::Bash, - lock: None, - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, - cache_ttl: None, - dedicated_worker: None, - }), - PushArgs::from(&ehm), - worker_name, - "worker@windmill.dev", - SUPERADMIN_SECRET_EMAIL.to_string(), - None, - None, - None, - None, - None, - false, - true, - None, - true, - Some("init_script".to_string()), - None, - None, - None, - None, - ) - .await?; - inner_tx.commit().await?; +) -> anyhow::Result { + let uuid_content = match conn { + Connection::Sql(db) => { + if let Some(content) = WORKER_CONFIG.read().await.init_bash.clone() { + Some(( + push_init_job(db, content.clone(), worker_name).await?, + content, + )) + } else { + None + } + } + Connection::Http(client) => { + let init_script = std::env::var("INIT_SCRIPT"); + if init_script.is_ok() { + let content = init_script.unwrap(); + Some((queue_init_job(client, &content).await?, content)) + } else { + None + } + } + }; + if let Some((uuid, content)) = uuid_content { same_worker_tx .send(SameWorkerPayload { job_id: uuid, recoverable: false }) .await @@ -1955,31 +1845,18 @@ pub enum SendResult { Kill, } -#[derive(Debug, Clone)] -pub struct JobCompleted { - pub job: Arc, - pub result: Arc>, - pub result_columns: Option>, - pub mem_peak: i32, - pub success: bool, - pub cached_res_path: Option, - pub token: String, - pub canceled_by: Option, - pub duration: Option, -} - async fn do_nativets( job: &MiniPulledJob, client: &AuthedClient, env_code: String, code: String, - db: &Pool, + conn: &Connection, mem_peak: &mut i32, canceled_by: &mut Option, worker_name: &str, occupancy_metrics: &mut OccupancyMetrics, ) -> windmill_common::error::Result> { - let args = build_args_map(job, client, db).await?.map(Json); + let args = build_args_map(job, client, conn).await?.map(Json); let job_args = if args.is_some() { args.as_ref() } else { @@ -1994,7 +1871,7 @@ async fn do_nativets( None, job.id, job.timeout, - db, + conn, mem_peak, canceled_by, worker_name, @@ -2017,7 +1894,7 @@ async fn handle_queued_job( raw_lock: Option, raw_flow: Option>>, parent_runnable_path: Option, - db: &DB, + conn: &Connection, client: &AuthedClient, hostname: &str, worker_name: &str, @@ -2028,6 +1905,7 @@ async fn handle_queued_job( job_completed_tx: JobCompletedSender, occupancy_metrics: &mut OccupancyMetrics, killpill_rx: &mut tokio::sync::broadcast::Receiver<()>, + precomputed_agent_info: Option, #[cfg(feature = "benchmark")] _bench: &mut BenchmarkIter, ) -> windmill_common::error::Result { // Extract the active span from the context @@ -2040,72 +1918,50 @@ async fn handle_queued_job( } #[cfg(any(not(feature = "enterprise"), feature = "sqlx"))] - if job.parent_job.is_none() && job.created_by.starts_with("email-") { - let daily_count = sqlx::query!( + match conn { + Connection::Sql(db) => { + if job.parent_job.is_none() && job.created_by.starts_with("email-") { + let daily_count = sqlx::query!( "SELECT value FROM metrics WHERE id = 'email_trigger_usage' AND created_at > NOW() - INTERVAL '1 day' ORDER BY created_at DESC LIMIT 1" ).fetch_optional(db) .warn_after_seconds(5) .await?.map(|x| serde_json::from_value::(x.value).unwrap_or(1)); - if let Some(count) = daily_count { - if count >= 100 { - return Err(error::Error::QuotaExceeded(format!( - "Email trigger usage limit of 100 per day has been reached." - ))); - } else { - sqlx::query!( + if let Some(count) = daily_count { + if count >= 100 { + return Err(error::Error::QuotaExceeded(format!( + "Email trigger usage limit of 100 per day has been reached." + ))); + } else { + sqlx::query!( "UPDATE metrics SET value = $1 WHERE id = 'email_trigger_usage' AND created_at > NOW() - INTERVAL '1 day'", serde_json::json!(count + 1) ) .execute(db) .warn_after_seconds(5) .await?; - } - } else { - sqlx::query!( + } + } else { + sqlx::query!( "INSERT INTO metrics (id, value) VALUES ('email_trigger_usage', to_jsonb(1))" ) - .execute(db) - .warn_after_seconds(5) - .await?; + .execute(db) + .warn_after_seconds(5) + .await?; + } + } + } + Connection::Http(_) => { + return Err(Error::internal_err(format!( + "Could not check email trigger usage for job with agent worker {}", + job.id + ))) } } - if job.is_flow_step() { - let _ = update_flow_status_in_progress( - db, - &job.workspace_id, - job.parent_job - .ok_or_else(|| Error::internal_err(format!("expected parent job")))?, - job.id, - ) - .warn_after_seconds(5) - .await?; - } else if let Some(parent_job) = job.parent_job { - let _ = sqlx::query_scalar!( - "UPDATE v2_job_status SET - workflow_as_code_status = jsonb_set( - jsonb_set( - COALESCE(workflow_as_code_status, '{}'::jsonb), - array[$1], - COALESCE(workflow_as_code_status->$1, '{}'::jsonb) - ), - array[$1, 'started_at'], - to_jsonb(now()::text) - ) - WHERE id = $2", - &job.id.to_string(), - parent_job - ) - .execute(db) - .warn_after_seconds(5) - .await - .inspect_err(|e| { - tracing::error!( - "Could not update parent job `started_at` in workflow as code status: {}", - e - ) - }); + // no need to mark job as started if http conn, it's done by the server when pulled + if let Connection::Sql(db) = conn { + job.mark_as_started_if_step(db).await?; } let started = Instant::now(); @@ -2122,7 +1978,7 @@ async fn handle_queued_job( x, ) => match x.map(|x| x.0) { None | Some(PREVIEW_IS_CODEBASE_HASH) | Some(PREVIEW_IS_TAR_CODEBASE_HASH) => { - Some(cache::job::fetch_preview(db, &job.id, raw_lock, raw_code, raw_flow).await?) + Some(cache::job::fetch_preview(conn, &job.id, raw_lock, raw_code, raw_flow).await?) } _ => None, }, @@ -2130,65 +1986,78 @@ async fn handle_queued_job( }; let cached_res_path = if job.cache_ttl.is_some() { - Some(cached_result_path(db, &client, &job, preview_data.as_ref()).await) + match conn { + Connection::Sql(db) => { + Some(cached_result_path(db, &client, &job, preview_data.as_ref()).await) + } + Connection::Http(_) => None, + } } else { None }; - if let Some(cached_res_path) = cached_res_path.as_ref() { - let cached_result_maybe = get_cached_resource_value_if_valid( - db, - &client, - &job.id, - &job.workspace_id, - &cached_res_path, - ) - .warn_after_seconds(5) - .await; - if let Some(result) = cached_result_maybe { - { - let logs = - "Job skipped because args & path found in cache and not expired".to_string(); - append_logs(&job.id, &job.workspace_id, logs, db).await; - } - job_completed_tx - .send(JobCompleted { - job, - result, - result_columns: None, - mem_peak: 0, - canceled_by: None, - success: true, - cached_res_path: None, - token: client.token.clone(), - duration: None, - }) - .await - .expect("send job completed"); + if let Some(db) = conn.as_sql() { + if let Some(cached_res_path) = cached_res_path.as_ref() { + let cached_result_maybe = get_cached_resource_value_if_valid( + db, + &client, + &job.id, + &job.workspace_id, + &cached_res_path, + ) + .warn_after_seconds(5) + .await; + if let Some(result) = cached_result_maybe { + { + let logs = "Job skipped because args & path found in cache and not expired" + .to_string(); + append_logs(&job.id, &job.workspace_id, logs, conn).await; + } + job_completed_tx + .send_job(JobCompleted { + job, + result, + result_columns: None, + mem_peak: 0, + canceled_by: None, + success: true, + cached_res_path: None, + token: client.token.clone(), + duration: None, + }) + .await + .expect("send job completed"); - return Ok(true); - } - }; - if job.is_flow() { - let flow_data = match preview_data { - Some(RawData::Flow(data)) => data, - // Not a preview: fetch from the cache or the database. - _ => cache::job::fetch_flow(db, job.kind, job.runnable_id).await?, + return Ok(true); + } }; - handle_flow( - job, - &flow_data, - db, - &client, - None, - same_worker_tx, - worker_dir, - job_completed_tx.0.clone(), - worker_name, - ) - .warn_after_seconds(10) - .await?; - Ok(true) + } + if job.is_flow() { + if let Some(db) = conn.as_sql() { + let flow_data = match preview_data { + Some(RawData::Flow(data)) => data, + // Not a preview: fetch from the cache or the database. + _ => cache::job::fetch_flow(db, job.kind, job.runnable_id).await?, + }; + handle_flow( + job, + &flow_data, + db, + &client, + None, + same_worker_tx, + worker_dir, + job_completed_tx.clone(), + worker_name, + ) + .warn_after_seconds(10) + .await?; + Ok(true) + } else { + return Err(Error::internal_err( + "Could not handle flow job with agent worker".to_string(), + )); + } } else { let mut logs = "".to_string(); let mut mem_peak: i32 = 0; @@ -2224,15 +2093,60 @@ async fn handle_queued_job( "handling job {}", job.id ); - append_logs(&job.id, &job.workspace_id, logs, db).await; + append_logs(&job.id, &job.workspace_id, logs, conn).await; let mut column_order: Option> = None; let mut new_args: Option>> = None; let result = match job.kind { - JobKind::Dependencies => { - handle_dependency_job( + JobKind::Dependencies => match conn { + Connection::Sql(db) => { + handle_dependency_job( + &job, + preview_data.as_ref(), + &mut mem_peak, + &mut canceled_by, + job_dir, + db, + worker_name, + worker_dir, + base_internal_url, + &client.token, + occupancy_metrics, + ) + .await + } + Connection::Http(_) => { + return Err(Error::internal_err( + "Could not handle dependency job with agent worker".to_string(), + )); + } + }, + JobKind::FlowDependencies => match conn { + Connection::Sql(db) => { + handle_flow_dependency_job( + &job, + preview_data.as_ref(), + &mut mem_peak, + &mut canceled_by, + job_dir, + db, + worker_name, + worker_dir, + base_internal_url, + &client.token, + occupancy_metrics, + ) + .await + } + Connection::Http(_) => { + return Err(Error::internal_err( + "Could not handle flow dependency job with agent worker".to_string(), + )); + } + }, + JobKind::AppDependencies => match conn { + Connection::Sql(db) => handle_app_dependency_job( &job, - preview_data.as_ref(), &mut mem_peak, &mut canceled_by, job_dir, @@ -2244,37 +2158,13 @@ async fn handle_queued_job( occupancy_metrics, ) .await - } - JobKind::FlowDependencies => { - handle_flow_dependency_job( - &job, - preview_data.as_ref(), - &mut mem_peak, - &mut canceled_by, - job_dir, - db, - worker_name, - worker_dir, - base_internal_url, - &client.token, - occupancy_metrics, - ) - .await - } - JobKind::AppDependencies => handle_app_dependency_job( - &job, - &mut mem_peak, - &mut canceled_by, - job_dir, - db, - worker_name, - worker_dir, - base_internal_url, - &client.token, - occupancy_metrics, - ) - .await - .map(|()| serde_json::from_str("{}").unwrap()), + .map(|()| serde_json::from_str("{}").unwrap()), + Connection::Http(_) => { + return Err(Error::internal_err( + "Could not handle app dependency job with agent worker".to_string(), + )); + } + }, JobKind::Identity => Ok(job .args .as_ref() @@ -2291,7 +2181,7 @@ async fn handle_queued_job( let r = handle_code_execution_job( job.as_ref(), preview_data, - db, + conn, client, parent_runnable_path, job_dir, @@ -2304,6 +2194,7 @@ async fn handle_queued_job( &mut new_args, occupancy_metrics, killpill_rx, + precomputed_agent_info, ) .await; occupancy_metrics.total_duration_of_running_jobs += @@ -2335,7 +2226,7 @@ async fn handle_queued_job( &client.token, column_order, new_args, - db, + conn, Some(started.elapsed().as_millis() as i64), ) .await @@ -2400,9 +2291,9 @@ pub async fn get_hub_script_content_and_requirements( pub async fn get_script_content_by_hash( script_hash: &ScriptHash, _w_id: &str, - db: &DB, + conn: &Connection, ) -> error::Result { - let (data, metadata) = cache::script::fetch(db, *script_hash).await?; + let (data, metadata) = cache::script::fetch(conn, *script_hash).await?; Ok(ContentReqLangEnvs { content: data.code.clone(), lockfile: data.lock.clone(), @@ -2419,7 +2310,7 @@ pub async fn get_script_content_by_hash( async fn try_validate_schema( job: &MiniPulledJob, - db: &Pool, + conn: &Connection, schema_validator: Option<&SchemaValidator>, code: &str, language: Option<&ScriptLang>, @@ -2480,7 +2371,7 @@ async fn try_validate_schema( &job.id, &job.workspace_id, "\n--- ARGS VALIDATION ---\nScript contains `schema_validation` annotation, running schema validation for the script arguments...\n", - db, + conn, ) .await; } @@ -2495,7 +2386,7 @@ async fn try_validate_schema( &job.id, &job.workspace_id, "Script arguments were validated!\n\n", - db, + conn, ) .await; } @@ -2509,7 +2400,7 @@ async fn try_validate_schema( async fn handle_code_execution_job( job: &MiniPulledJob, preview: Option>, - db: &sqlx::Pool, + conn: &Connection, client: &AuthedClient, parent_runnable_path: Option, job_dir: &str, @@ -2522,6 +2413,7 @@ async fn handle_code_execution_job( new_args: &mut Option>>, occupancy_metrics: &mut OccupancyMetrics, killpill_rx: &mut tokio::sync::broadcast::Receiver<()>, + precomputed_agent_info: Option, ) -> error::Result> { let script_hash = || { job.runnable_id @@ -2557,7 +2449,7 @@ async fn handle_code_execution_job( } JobKind::Script_Hub => { let ContentReqLangEnvs { content, lockfile, language, envs, codebase, schema } = - get_hub_script_content_and_requirements(job.runnable_path.as_ref(), Some(db)) + get_hub_script_content_and_requirements(job.runnable_path.as_ref(), conn.as_sql()) .await?; data = ScriptData { code: content, lock: lockfile }; @@ -2565,11 +2457,11 @@ async fn handle_code_execution_job( (&data, &metadata) } JobKind::Script => { - (arc_data, arc_metadata) = cache::script::fetch(db, script_hash()?).await?; + (arc_data, arc_metadata) = cache::script::fetch(conn, script_hash()?).await?; (arc_data.as_ref(), arc_metadata.as_ref()) } JobKind::FlowScript => { - arc_data = cache::flow::fetch_script(db, FlowNodeId(script_hash()?.0)).await?; + arc_data = cache::flow::fetch_script(conn, FlowNodeId(script_hash()?.0)).await?; metadata = ScriptMetadata { language: job.script_lang, envs: None, @@ -2580,7 +2472,7 @@ async fn handle_code_execution_job( (arc_data.as_ref(), &metadata) } JobKind::AppScript => { - arc_data = cache::app::fetch_script(db, AppScriptId(script_hash()?.0)).await?; + arc_data = cache::app::fetch_script(conn, AppScriptId(script_hash()?.0)).await?; metadata = ScriptMetadata { language: job.script_lang, envs: None, @@ -2590,33 +2482,41 @@ async fn handle_code_execution_job( }; (arc_data.as_ref(), &metadata) } - JobKind::DeploymentCallback => { - let script_path = job - .runnable_path - .as_ref() - .ok_or_else(|| Error::internal_err("expected script path".to_string()))?; - if script_path.starts_with("hub/") { - let ContentReqLangEnvs { content, lockfile, language, envs, codebase, schema } = - get_hub_script_content_and_requirements(Some(script_path), Some(db)).await?; - data = ScriptData { code: content, lock: lockfile }; - metadata = - ScriptMetadata { language, envs, codebase, schema, schema_validator: None }; - (&data, &metadata) - } else { - let hash = sqlx::query_scalar!( - "SELECT hash FROM script WHERE path = $1 AND workspace_id = $2 AND + JobKind::DeploymentCallback => match conn { + Connection::Sql(db) => { + let script_path = job + .runnable_path + .as_ref() + .ok_or_else(|| Error::internal_err("expected script path".to_string()))?; + if script_path.starts_with("hub/") { + let ContentReqLangEnvs { content, lockfile, language, envs, codebase, schema } = + get_hub_script_content_and_requirements(Some(script_path), conn.as_sql()) + .await?; + data = ScriptData { code: content, lock: lockfile }; + metadata = + ScriptMetadata { language, envs, codebase, schema, schema_validator: None }; + (&data, &metadata) + } else { + let hash = sqlx::query_scalar!( + "SELECT hash FROM script WHERE path = $1 AND workspace_id = $2 AND deleted = false AND lock IS not NULL AND lock_error_logs IS NULL", - script_path, - &job.workspace_id - ) - .fetch_optional(db) - .await? - .ok_or_else(|| Error::internal_err("expected script hash".to_string()))?; + script_path, + &job.workspace_id + ) + .fetch_optional(db) + .await? + .ok_or_else(|| Error::internal_err("expected script hash".to_string()))?; - (arc_data, arc_metadata) = cache::script::fetch(db, ScriptHash(hash)).await?; - (arc_data.as_ref(), arc_metadata.as_ref()) + (arc_data, arc_metadata) = cache::script::fetch(conn, ScriptHash(hash)).await?; + (arc_data.as_ref(), arc_metadata.as_ref()) + } } - } + Connection::Http(_) => { + return Err(Error::internal_err( + "Could not handle deployment callback with agent worker".to_string(), + )); + } + }, _ => unreachable!( "handle_code_execution_job should never be reachable with a non-code execution job" ), @@ -2624,7 +2524,7 @@ async fn handle_code_execution_job( try_validate_schema( job, - db, + conn, schema_validator.as_ref(), code, language.as_ref(), @@ -2638,7 +2538,7 @@ async fn handle_code_execution_job( job, &client, &code, - db, + conn, mem_peak, canceled_by, worker_name, @@ -2657,7 +2557,7 @@ async fn handle_code_execution_job( job, &client, &code, - db, + conn, mem_peak, canceled_by, worker_name, @@ -2687,7 +2587,7 @@ async fn handle_code_execution_job( job, &client, &code, - db, + conn, mem_peak, canceled_by, worker_name, @@ -2710,7 +2610,7 @@ async fn handle_code_execution_job( job, &client, &code, - db, + conn, mem_peak, canceled_by, worker_name, @@ -2741,7 +2641,7 @@ async fn handle_code_execution_job( job, &client, &code, - db, + conn, mem_peak, canceled_by, worker_name, @@ -2772,7 +2672,7 @@ async fn handle_code_execution_job( job, &client, &code, - db, + conn, mem_peak, canceled_by, worker_name, @@ -2786,7 +2686,7 @@ async fn handle_code_execution_job( job, &client, &code, - db, + conn, mem_peak, canceled_by, worker_name, @@ -2798,12 +2698,12 @@ async fn handle_code_execution_job( &job.id, &job.workspace_id, "\n--- FETCH TS EXECUTION ---\n", - db, + conn, ) .await; let reserved_variables = - get_reserved_variables(job, &client.token, db, parent_runnable_path).await?; + get_reserved_variables(job, &client.token, conn, parent_runnable_path).await?; let env_code = format!( "const process = {{ env: {{}} }};\nconst BASE_URL = '{base_internal_url}';\nconst BASE_INTERNAL_URL = '{base_internal_url}';\nprocess.env['BASE_URL'] = BASE_URL;process.env['BASE_INTERNAL_URL'] = BASE_INTERNAL_URL;\n{}", @@ -2818,7 +2718,7 @@ async fn handle_code_execution_job( &client, env_code, code.clone(), - db, + conn, mem_peak, canceled_by, worker_name, @@ -2886,7 +2786,7 @@ mount {{ job, mem_peak, canceled_by, - db, + conn, client, parent_runnable_path, &code, @@ -2895,6 +2795,7 @@ mount {{ envs, new_args, occupancy_metrics, + precomputed_agent_info, ) .await } @@ -2904,7 +2805,7 @@ mount {{ mem_peak, canceled_by, job, - db, + conn, client, parent_runnable_path, job_dir, @@ -2924,7 +2825,7 @@ mount {{ mem_peak, canceled_by, job, - db, + conn, client, parent_runnable_path, job_dir, @@ -2935,6 +2836,7 @@ mount {{ &shared_mount, new_args, occupancy_metrics, + precomputed_agent_info, ) .await } @@ -2943,7 +2845,7 @@ mount {{ mem_peak, canceled_by, job, - db, + conn, client, parent_runnable_path, &code, @@ -2962,7 +2864,7 @@ mount {{ mem_peak, canceled_by, job, - db, + conn, client, parent_runnable_path, &code, @@ -2981,7 +2883,7 @@ mount {{ mem_peak, canceled_by, job, - db, + conn, client, parent_runnable_path, &code, @@ -3006,7 +2908,7 @@ mount {{ mem_peak, canceled_by, job, - db, + conn, client, parent_runnable_path, job_dir, @@ -3030,7 +2932,7 @@ mount {{ mem_peak, canceled_by, job, - db, + conn, client, parent_runnable_path, &code, @@ -3059,7 +2961,7 @@ mount {{ job, mem_peak, canceled_by, - db, + conn, client, parent_runnable_path, &code, @@ -3075,7 +2977,7 @@ mount {{ mem_peak, canceled_by, job, - db, + conn, client, parent_runnable_path, &code, @@ -3100,7 +3002,7 @@ mount {{ mem_peak, canceled_by, job, - db, + conn, client, parent_runnable_path, inner_content: &code, @@ -3126,7 +3028,7 @@ mount {{ mem_peak, canceled_by, job, - db, + conn, client, parent_runnable_path, inner_content: &code, diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index d8c9a0a410..27aef9d1f9 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -16,8 +16,7 @@ use crate::bench::BenchmarkIter; use crate::common::{cached_result_path, save_in_cache}; use crate::js_eval::{eval_timeout, IdContext}; use crate::{ - AuthedClient, PreviousResult, SameWorkerPayload, SameWorkerSender, SendResult, JOB_TOKEN, - KEEP_JOB_DIR, + AuthedClient, JobCompletedSender, PreviousResult, SameWorkerSender, SendResult, KEEP_JOB_DIR, }; use anyhow::Context; use futures::TryFutureExt; @@ -27,7 +26,6 @@ use serde_json::value::RawValue; use serde_json::{json, Value}; use sqlx::types::Json; use sqlx::{FromRow, Postgres, Transaction}; -use tokio::sync::mpsc::Sender; use tracing::instrument; use uuid::Uuid; use windmill_common::add_time; @@ -54,11 +52,12 @@ use windmill_common::{ }, flows::{FlowModule, FlowModuleValue, FlowValue, InputTransform, Retry, Suspend}, }; +use windmill_queue::flow_status::Step; use windmill_queue::schedule::get_schedule_opt; use windmill_queue::{ add_completed_job, add_completed_job_error, append_logs, get_mini_pulled_job, handle_maybe_scheduled_job, CanceledBy, MiniPulledJob, PushArgs, PushIsolationLevel, - WrappedError, + SameWorkerPayload, WrappedError, }; type DB = sqlx::Pool; @@ -81,7 +80,7 @@ pub async fn update_flow_status_after_job_completion( worker_dir: &str, stop_early_override: Option, worker_name: &str, - job_completed_tx: Sender, + job_completed_tx: JobCompletedSender, #[cfg(feature = "benchmark")] bench: &mut BenchmarkIter, ) -> error::Result>> { // this is manual tailrecursion because async_recursion blows up the stack @@ -200,7 +199,7 @@ pub async fn update_flow_status_after_job_completion_internal( stop_early_override: Option, skip_error_handler: bool, worker_name: &str, - job_completed_tx: Sender, + job_completed_tx: JobCompletedSender, #[cfg(feature = "benchmark")] bench: &mut BenchmarkIter, ) -> error::Result { add_time!(bench, "update flow status internal START"); @@ -1050,7 +1049,7 @@ pub async fn update_flow_status_after_job_completion_internal( } else { "Flow job completed with error\n".to_string() }; - append_logs(&flow_job.id, w_id, logs, db).await; + append_logs(&flow_job.id, w_id, logs, &db.into()).await; } #[cfg(feature = "enterprise")] if flow_job.parent_job.is_none() { @@ -1168,7 +1167,7 @@ pub async fn update_flow_status_after_job_completion_internal( &flow_job.id, w_id, format!("Unexpected error during flow chaining:\n{:#?}", e), - db, + &db.into(), ) .await; let _ = add_completed_job_error(db, &flow_job, 0, None, e, worker_name, true, None) @@ -1388,102 +1387,6 @@ async fn compute_bool_from_expr( } } -pub async fn update_flow_status_in_progress( - db: &DB, - _w_id: &str, - flow: Uuid, - job_in_progress: Uuid, -) -> error::Result { - let step = get_step_of_flow_status(db, flow).await?; - match step { - Step::Step(step) => { - sqlx::query!( - "UPDATE v2_job_status SET - flow_status = jsonb_set( - jsonb_set(flow_status, ARRAY['modules', $3::INTEGER::TEXT, 'job'], to_jsonb($1::UUID::TEXT)), - ARRAY['modules', $3::INTEGER::TEXT, 'type'], - to_jsonb('InProgress'::text) - ) - WHERE id = $2", - job_in_progress, - flow, - step as i32 - ) - .execute(db) - .await?; - } - Step::PreprocessorStep => { - sqlx::query!( - "UPDATE v2_job_status SET - flow_status = jsonb_set( - jsonb_set(flow_status, ARRAY['preprocessor_module', 'job'], to_jsonb($1::UUID::TEXT)), - ARRAY['preprocessor_module', 'type'], - to_jsonb('InProgress'::text) - ) - WHERE id = $2", - job_in_progress, - flow - ) - .execute(db) - .await?; - } - Step::FailureStep => { - sqlx::query!( - "UPDATE v2_job_status SET - flow_status = jsonb_set( - jsonb_set(flow_status, ARRAY['failure_module', 'job'], to_jsonb($1::UUID::TEXT)), - ARRAY['failure_module', 'type'], - to_jsonb('InProgress'::text) - ) - WHERE id = $2", - job_in_progress, - flow - ) - .execute(db) - .await?; - } - } - - Ok(step) -} - -#[derive(Debug, Copy, Clone)] -pub enum Step { - Step(usize), - PreprocessorStep, - FailureStep, -} - -impl Step { - fn from_i32_and_len(step: i32, len: usize) -> Self { - if step < 0 { - Step::PreprocessorStep - } else if (step as usize) < len { - Step::Step(step as usize) - } else { - Step::FailureStep - } - } -} - -#[instrument(level = "trace", skip_all)] -pub async fn get_step_of_flow_status(db: &DB, id: Uuid) -> error::Result { - let r = sqlx::query!( - "SELECT (flow_status->'step')::integer as step, jsonb_array_length(flow_status->'modules') as len - FROM v2_job_status WHERE id = $1", - id - ) - .fetch_one(db) - .await - .map_err(|e| Error::internal_err(format!("fetching step flow status: {e:#}")))?; - - if let Some(step) = r.step { - Ok(Step::from_i32_and_len(step, r.len.unwrap_or(0) as usize)) - } else { - Err(Error::internal_err("step is null".to_string())) - } -} - /// resumes should be in order of timestamp ascending, so that more recent are at the end #[instrument(level = "trace", skip_all)] async fn transform_input( @@ -1548,7 +1451,7 @@ pub async fn handle_flow( last_result: Option>>, same_worker_tx: SameWorkerSender, worker_dir: &str, - job_completed_tx: Sender, + job_completed_tx: JobCompletedSender, worker_name: &str, ) -> anyhow::Result<()> { let flow = flow_data.value(); @@ -1669,7 +1572,7 @@ async fn push_next_flow_job( last_job_result: Option>>, same_worker_tx: SameWorkerSender, worker_dir: &str, - job_completed_tx: Sender, + job_completed_tx: JobCompletedSender, worker_name: &str, ) -> error::Result> { let job_root = flow_job @@ -2105,7 +2008,13 @@ async fn push_next_flow_job( let result: Value = json!({ "error": {"message": logs, "name": error_name}}); - append_logs(&flow_job.id, &flow_job.workspace_id, logs.clone(), db).await; + append_logs( + &flow_job.id, + &flow_job.workspace_id, + logs.clone(), + &db.into(), + ) + .await; job_completed_tx .send(SendResult::UpdateFlow { @@ -2644,7 +2553,7 @@ async fn push_next_flow_job( }; // forward root job permissions to the new job - let job_perms: Option = if JOB_TOKEN.is_none() { + let job_perms: Option = { if let Some(root_job) = &flow_job .flow_innermost_root_job .or_else(|| Some(flow_job.id)) @@ -2661,8 +2570,6 @@ async fn push_next_flow_job( } else { None } - } else { - None }; tracing::debug!(id = %flow_job.id, root_id = %job_root, "computed perms for job {i} of {len}"); diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index 8975630afb..8ba9ee501a 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -17,7 +17,7 @@ use windmill_common::jobs::JobPayload; use windmill_common::scripts::ScriptHash; #[cfg(feature = "python")] use windmill_common::worker::PythonAnnotations; -use windmill_common::worker::{to_raw_value, to_raw_value_owned, write_file}; +use windmill_common::worker::{to_raw_value, to_raw_value_owned, write_file, Connection}; use windmill_common::{ apps::AppScriptId, @@ -83,7 +83,7 @@ pub async fn update_script_dependency_map( ) .await?; tx.commit().await?; - append_logs(job_id, w_id, logs, db).await; + append_logs(job_id, w_id, logs, &db.into()).await; } Ok(()) } @@ -226,7 +226,7 @@ pub async fn handle_dependency_job( mem_peak: &mut i32, canceled_by: &mut Option, job_dir: &str, - db: &sqlx::Pool, + db: &DB, worker_name: &str, worker_dir: &str, base_internal_url: &str, @@ -265,7 +265,7 @@ pub async fn handle_dependency_job( // - A saved script `hash` in the `script_hash` column. // - Preview raw lock and code in the `queue` or `job` table. let script_data = &match job.runnable_id { - Some(hash) => match cache::script::fetch(db, hash).await { + Some(hash) => match cache::script::fetch(&Connection::from(db.clone()), hash).await { Ok(d) => Cow::Owned(d.0), Err(e) => { let logs2 = sqlx::query_scalar!( @@ -1121,7 +1121,7 @@ async fn lock_modules<'c>( Some(e.id.clone()), ) .await?; - append_logs(&job.id, &job.workspace_id, logs, db).await; + append_logs(&job.id, &job.workspace_id, logs, &db.into()).await; } if language == ScriptLang::Bun || language == ScriptLang::Bunnative { @@ -1513,7 +1513,7 @@ async fn lock_modules_app( .await; match new_lock { Ok(new_lock) => { - append_logs(&job.id, &job.workspace_id, logs, db).await; + append_logs(&job.id, &job.workspace_id, logs, &db.into()).await; let anns = windmill_common::worker::TypeScriptAnnotations::parse( &content, @@ -1761,7 +1761,7 @@ async fn python_dep( let final_version = annotated_pyv_numeric .and_then(|pyv| PyVersion::from_numeric(pyv)) - .unwrap_or(PyVersion::from_instance_version(job_id, w_id, db).await); + .unwrap_or(PyVersion::from_instance_version(job_id, w_id, &db.into()).await); let req: std::result::Result = uv_pip_compile( job_id, @@ -1769,7 +1769,7 @@ async fn python_dep( mem_peak, canceled_by, job_dir, - db, + &db.into(), worker_name, w_id, occupancy_metrics, @@ -1785,7 +1785,7 @@ async fn python_dep( w_id, mem_peak, canceled_by, - db, + &Connection::Sql(db.clone()), worker_name, job_dir, worker_dir, @@ -1918,7 +1918,7 @@ async fn capture_dependency_job( mem_peak, canceled_by, job_dir, - db, + &db.into(), false, false, false, @@ -1940,7 +1940,7 @@ async fn capture_dependency_job( mem_peak, canceled_by, job_dir, - Some(db), + Some(&db.into()), w_id, worker_name, base_internal_url, @@ -1960,7 +1960,7 @@ async fn capture_dependency_job( canceled_by, job_id, w_id, - Some(db), + Some(&db.into()), token, script_path, job_dir, @@ -1983,7 +1983,7 @@ async fn capture_dependency_job( script_path, job_id, w_id, - Some(db.clone()), + Some(&db), &job_dir, base_internal_url, worker_name, @@ -2020,7 +2020,7 @@ async fn capture_dependency_job( canceled_by, job_id, w_id, - db, + &Connection::Sql(db.clone()), job_dir, worker_name, reqs, @@ -2049,7 +2049,7 @@ async fn capture_dependency_job( mem_peak, canceled_by, job_dir, - db, + &Connection::Sql(db.clone()), worker_name, w_id, occupancy_metrics, @@ -2072,7 +2072,7 @@ async fn capture_dependency_job( mem_peak, canceled_by, job_dir, - db, + &Connection::Sql(db.clone()), worker_name, w_id, occupancy_metrics, @@ -2087,7 +2087,14 @@ async fn capture_dependency_job( )); } - resolve(job_id, job_raw_code, job_dir, db, w_id).await + resolve( + job_id, + job_raw_code, + job_dir, + &Connection::Sql(db.clone()), + w_id, + ) + .await } // for related places search: ADD_NEW_LANG _ => Ok("".to_owned()), diff --git a/backend/windmill-worker/src/worker_utils.rs b/backend/windmill-worker/src/worker_utils.rs new file mode 100644 index 0000000000..4c83448236 --- /dev/null +++ b/backend/windmill-worker/src/worker_utils.rs @@ -0,0 +1,322 @@ +use backon::{BackoffBuilder, ConstantBuilder, Retryable}; +use tracing::Instrument; +use uuid::Uuid; +use windmill_common::{ + agent_workers::{PingJobStatus, PingJobStatusResponse}, + worker::{ + get_memory, get_vcpus, get_windmill_memory_usage, get_worker_memory_usage, + insert_ping_query, update_job_ping_query, update_worker_ping_from_job_query, + update_worker_ping_main_loop_query, Connection, Ping, PingType, WORKER_CONFIG, + WORKER_GROUP, + }, + KillpillSender, +}; + +use crate::{ + agent_workers::UPDATE_PING_URL, + common::{OccupancyMetrics, OccupancyResult}, +}; + +pub(crate) async fn update_worker_ping_full( + conn: &Connection, + read_cgroups: bool, + jobs_executed: i32, + worker_name: &str, + hostname: &str, + occupancy_metrics: &mut OccupancyMetrics, + killpill_tx: &KillpillSender, +) { + let tags = WORKER_CONFIG.read().await.worker_tags.clone(); + + let memory_usage = get_worker_memory_usage(); + let wm_memory_usage = get_windmill_memory_usage(); + + let (vcpus, memory) = if read_cgroups { + (get_vcpus(), get_memory()) + } else { + (None, None) + }; + + let OccupancyResult { + occupancy_rate, + occupancy_rate_15s, + occupancy_rate_5m, + occupancy_rate_30m, + } = occupancy_metrics.update_occupancy_metrics(); + + if let Err(e) = (|| { + update_worker_ping_full_inner( + conn, + jobs_executed, + &worker_name, + &tags, + memory_usage, + wm_memory_usage, + vcpus, + memory, + occupancy_rate, + occupancy_rate_15s, + occupancy_rate_5m, + occupancy_rate_30m, + ) + }) + .retry( + ConstantBuilder::default() + .with_delay(std::time::Duration::from_secs(2)) + .with_max_times(10) + .build(), + ) + .notify(|err, dur| { + tracing::error!( + worker = %worker_name, hostname = %hostname, + "retrying updating worker ping in {dur:#?}, err: {err:#?}" + ); + }) + .sleep(tokio::time::sleep) + .await + { + tracing::error!( + worker = %worker_name, hostname = %hostname, + "failed to update worker ping, exiting: {}", e); + killpill_tx.send(); + } + tracing::info!( + worker = %worker_name, hostname = %hostname, + "ping update, memory: container={}MB, windmill={}MB", + memory_usage.unwrap_or_default() / (1024 * 1024), + wm_memory_usage.unwrap_or_default() / (1024 * 1024) + ); +} + +async fn update_worker_ping_full_inner( + conn: &Connection, + jobs_executed: i32, + worker_name: &str, + tags: &[String], + memory_usage: Option, + wm_memory_usage: Option, + vcpus: Option, + memory: Option, + occupancy_rate: f32, + occupancy_rate_15s: Option, + occupancy_rate_5m: Option, + occupancy_rate_30m: Option, +) -> anyhow::Result<()> { + match conn { + Connection::Sql(db) => { + update_worker_ping_main_loop_query( + worker_name, + tags, + vcpus, + memory, + Some(jobs_executed), + Some(occupancy_rate), + memory_usage, + wm_memory_usage, + occupancy_rate_15s, + occupancy_rate_5m, + occupancy_rate_30m, + db, + ) + .await?; + } + Connection::Http(client) => { + client + .post::<_, ()>( + UPDATE_PING_URL, + &Ping { + last_job_executed: None, + last_job_workspace_id: None, + worker_instance: None, + ip: None, + tags: Some(tags.to_vec()), + dw: None, + jobs_executed: Some(jobs_executed), + occupancy_rate: Some(occupancy_rate), + occupancy_rate_15s: Some(occupancy_rate_15s.unwrap_or(0.0)), + occupancy_rate_5m: Some(occupancy_rate_5m.unwrap_or(0.0)), + occupancy_rate_30m: Some(occupancy_rate_30m.unwrap_or(0.0)), + version: None, + vcpus: vcpus, + memory: memory, + memory_usage: get_worker_memory_usage(), + wm_memory_usage: get_windmill_memory_usage(), + ping_type: PingType::MainLoop, + }, + ) + .await?; + } + } + Ok(()) +} + +pub async fn insert_ping( + worker_instance: &str, + worker_name: &str, + ip: &str, + db: &Connection, +) -> anyhow::Result<()> { + let (tags, dw) = { + let wc = WORKER_CONFIG.read().await.clone(); + ( + wc.worker_tags, + wc.dedicated_worker + .as_ref() + .map(|x| format!("{}:{}", x.workspace_id, x.path)), + ) + }; + + let vcpus = get_vcpus(); + let memory = get_memory(); + + match db { + Connection::Sql(db) => { + insert_ping_query( + worker_instance, + worker_name, + WORKER_GROUP.as_str(), + ip, + tags.as_slice(), + dw, + windmill_common::utils::GIT_VERSION, + vcpus, + memory, + db, + ) + .await?; + } + Connection::Http(client) => { + client + .post::<_, ()>( + UPDATE_PING_URL, + &Ping { + last_job_executed: None, + last_job_workspace_id: None, + worker_instance: Some(worker_instance.to_string()), + ip: Some(ip.to_string()), + tags: Some(tags.to_vec()), + dw: dw, + jobs_executed: None, + occupancy_rate: None, + occupancy_rate_15s: None, + occupancy_rate_5m: None, + occupancy_rate_30m: None, + version: Some(windmill_common::utils::GIT_VERSION.to_string()), + vcpus: vcpus, + memory: memory, + memory_usage: get_worker_memory_usage(), + wm_memory_usage: get_windmill_memory_usage(), + ping_type: PingType::Initial, + }, + ) + .await?; + } + } + Ok(()) +} + +pub async fn update_worker_ping_from_job( + conn: &Connection, + job_id: &Uuid, + w_id: &str, + worker_name: &str, + memory_usage: Option, + wm_memory_usage: Option, + occupancy: Option, +) -> anyhow::Result<()> { + let occupancy_rate = occupancy.as_ref().map(|x| x.occupancy_rate); + let occupancy_rate_15s = occupancy.as_ref().and_then(|x| x.occupancy_rate_15s); + let occupancy_rate_5m = occupancy.as_ref().and_then(|x| x.occupancy_rate_5m); + let occupancy_rate_30m = occupancy.as_ref().and_then(|x| x.occupancy_rate_30m); + match conn.clone() { + Connection::Sql(ref db) => { + update_worker_ping_from_job_query( + job_id, + w_id, + worker_name, + memory_usage, + wm_memory_usage, + occupancy_rate, + occupancy_rate_15s, + occupancy_rate_5m, + occupancy_rate_30m, + db, + ) + .await?; + } + Connection::Http(client) => { + client + .post::( + UPDATE_PING_URL, + &Ping { + last_job_executed: Some(job_id.clone()), + last_job_workspace_id: Some(w_id.to_string()), + ping_type: PingType::Job, + worker_instance: None, + ip: None, + tags: None, + dw: None, + version: None, + vcpus: None, + memory: None, + memory_usage: memory_usage, + wm_memory_usage: wm_memory_usage, + jobs_executed: None, + occupancy_rate: occupancy_rate, + occupancy_rate_15s: occupancy_rate_15s, + occupancy_rate_5m: occupancy_rate_5m, + occupancy_rate_30m: occupancy_rate_30m, + }, + ) + .await?; + } + } + Ok(()) +} + +pub async fn ping_job_status( + conn: &Connection, + job_id: &Uuid, + mem_peak: Option, + current_mem: Option, +) -> anyhow::Result { + match conn { + Connection::Sql(ref db) => update_job_ping_query(job_id, db, mem_peak).await, + Connection::Http(client) => { + client + .post( + &format!("/api/agent_workers/ping_job_status/{}", job_id), + &PingJobStatus { mem_peak, current_mem }, + ) + .await + } + } +} + +pub(crate) async fn queue_vacuum(conn: &Connection, worker_name: &str, hostname: &str) { + match conn { + Connection::Sql(db) => { + let db2 = db.clone(); + let current_span = tracing::Span::current(); + let worker_name = worker_name.to_string(); + let hostname = hostname.to_string(); + tokio::task::spawn( + (async move { + tracing::info!(worker = %worker_name, hostname = %hostname, "vacuuming queue"); + if let Err(e) = sqlx::query!("VACUUM v2_job_queue, v2_job_runtime, v2_job_status") + .execute(&db2) + .await + { + tracing::error!(worker = %worker_name, hostname = %hostname, "failed to vacuum queue: {}", e); + } + tracing::info!(worker = %worker_name, hostname = %hostname, "vacuumed queue"); + }) + .instrument(current_span), + ); + } + Connection::Http(_) => { + // do nothing in http mode + () + } + } +} diff --git a/frontend/src/lib/components/HttpAgentWorkerDrawer.svelte b/frontend/src/lib/components/HttpAgentWorkerDrawer.svelte new file mode 100644 index 0000000000..7250a906c8 --- /dev/null +++ b/frontend/src/lib/components/HttpAgentWorkerDrawer.svelte @@ -0,0 +1,118 @@ + + +
+ Use HTTP agent workers only when the workers need to be deployed remotely OR with only HTTP + connectivity OR in untrusted environments. HTTP agent workers have more latency and less + capabilities than normal workers. +
+ +
+
+ {#if !$enterpriseLicense} +
+ Agent workers are only available in the enterprise edition. For evaluation purposes, you can + only use the tag `agent_test` tag and it is limited to 100 jobs. +
+ {/if} + +
+
+
+ { + navigator.clipboard.writeText(token) + sendUserToast('Copied to clipboard') + }} + placeholder="Select tags to generate a jwt token" + type="text" + disabled + value={token} + class="pr-8 text-sm text-secondary" + /> + +
+ +
+ Pass the env variables: +
    +
  • MODE=agent
    • +
  • AGENT_TOKEN={'""'}
    • +
  • BASE_INTERNAL_URL={'""'}
    • +
+ to a worker to have it act as an HTTP agent worker. INIT_SCRIPT, if needed, must be passed as an + env variable. + +

+ Remember to have at least one normal worker that listens to the tags `flow` and `dependency` + (or `flow-$workspace` and `dependency-$workspace` if using workspace specific default tags) + to have flow and dependency job being runnable as agent workers can't run dependency jobs + nor can run the flow state machine (but can run the subjobs within them). +

+
+ + +
+ Use the following api endpoint with a superadmin bearer token: + POST /api/agent_workers/create_agent_token + with body: + 
+			
+{`{
+	"worker_group": "agent",
+	"tags": ["tag1", "tag2"],
+	"exp": 1717334400 // 3 years from now
+}`}
+					
+
+ JSON response will be the JWT token. +
+ +
+
diff --git a/frontend/src/lib/components/TagsToListenTo.svelte b/frontend/src/lib/components/TagsToListenTo.svelte new file mode 100644 index 0000000000..ce56a575e8 --- /dev/null +++ b/frontend/src/lib/components/TagsToListenTo.svelte @@ -0,0 +1,86 @@ + + +
+ {#if worker_tags?.length == 0} +
No tags selected
+ {/if} + {#each worker_tags as tag} +
{tag}
+ {#if $superadmin && !disabled} + + {/if}
+ {/each} +
+{#if $superadmin} +
+ !worker_tags?.includes(x) + )} + {disabled} + bind:selectedItem={newTag} + hideArrow={true} + inputClassName={'flex !font-gray-600 !font-primary !bg-surface-primary"'} + dropdownClassName="!text-sm !py-2 !rounded-sm !border-gray-200 !border !shadow-md" + className="w-full !font-gray-600 !font-primary !bg-surface-primary" + onFocus={() => { + dispatch('focus') + }} + create + onCreate={(c) => { + createdTags.push(c) + createdTags = [...createdTags] + return c + }} + createText="Press enter to use this tag" + /> + +
+
+ +
+
+{/if} diff --git a/frontend/src/lib/components/WorkerGroup.svelte b/frontend/src/lib/components/WorkerGroup.svelte index 978b58b878..00230b7255 100644 --- a/frontend/src/lib/components/WorkerGroup.svelte +++ b/frontend/src/lib/components/WorkerGroup.svelte @@ -18,8 +18,9 @@ import AutoComplete from 'simple-svelte-autocomplete' import YAML from 'yaml' import Toggle from './Toggle.svelte' - import type { AutoscalingConfig } from './worker_group' + import { defaultTags, nativeTags, type AutoscalingConfig } from './worker_group' import AutoscalingConfigEditor from './AutoscalingConfigEditor.svelte' + import TagsToListenTo from './TagsToListenTo.svelte' export let name: string export let config: @@ -77,10 +78,10 @@ ? config : { worker_tags: [] - } + } : { worker_tags: [] - } + } if (nconfig.priority_tags === undefined) { nconfig.priority_tags = new Map() } @@ -109,35 +110,6 @@ value: string | undefined }[] = [] - const defaultTags = [ - 'deno', - 'python3', - 'go', - 'bash', - 'powershell', - 'dependency', - 'flow', - 'other', - 'bun', - 'php', - 'rust', - 'ansible', - 'csharp', - 'nu', - 'java' - // for related places search: ADD_NEW_LANG - ] - const nativeTags = [ - 'nativets', - 'postgresql', - 'mysql', - 'graphql', - 'snowflake', - 'mssql', - 'bigquery', - 'oracledb' - ] - const aws_env_vars_preset = [ 'AWS_REGION', 'AWS_DEFAULT_REGION', @@ -153,7 +125,6 @@ 'SSL_CERT_DIR' ] - let newTag: string = '' $: selected = nconfig?.dedicated_worker != undefined ? 'dedicated' : 'normal' $: { selectedPriorityTags = [] @@ -185,8 +156,6 @@ let openClean = false let drawer: Drawer - - let createdTags: string[] = [] -
+
{/if} {#if selected == 'normal'}
- {#if config?.worker_tags != undefined} -
- {#each config.worker_tags as tag} -
{tag}
- {#if $superadmin} - - {/if}
- {/each} -
- {#if $superadmin} -
- !nconfig?.worker_tags?.includes(x))} - bind:selectedItem={newTag} - hideArrow={true} - inputClassName={'flex !font-gray-600 !font-primary !bg-surface-primary"'} - dropdownClassName="!text-sm !py-2 !rounded-sm !border-gray-200 !border !shadow-md" - className="w-full !font-gray-600 !font-primary !bg-surface-primary" - onFocus={() => { - dispatch('focus') - }} - create - onCreate={(c) => { - createdTags.push(c) - createdTags = [...createdTags] - return c - }} - createText="Press enter to use this non-predefined value" - /> + {#if nconfig?.worker_tags != undefined} + { + dirty = true + }} + on:deletePriorityTag={(e) => { + const tag = e.detail + if (nconfig.priority_tags) { + delete nconfig.priority_tags[tag] + } + selectedPriorityTags = selectedPriorityTags.filter((t) => t != tag) ?? [] + }} + bind:worker_tags={nconfig.worker_tags} + {customTags} + /> -
-
- + + + + {#if defaultTagPerWorkspace} + w.id)} + inputClassName={'flex !font-gray-600 !font-primary !bg-surface-primary'} + dropdownClassName="!text-sm !py-2 !rounded-sm !border-gray-200 !border !shadow-md" + className="!font-gray-600 !font-primary !bg-surface-primary" + create + onCreate={(c) => c} + placeholder="Workspace ID" + /> + {/if} +
+
+ {#if nconfig?.worker_tags !== undefined && nconfig?.worker_tags.length > 0} +
-
-
- - - - - {#if defaultTagPerWorkspace} - w.id)} - inputClassName={'flex !font-gray-600 !font-primary !bg-surface-primary'} - dropdownClassName="!text-sm !py-2 !rounded-sm !border-gray-200 !border !shadow-md" - className="!font-gray-600 !font-primary !bg-surface-primary" - create - onCreate={(c) => c} - placeholder="Workspace ID" + options={nconfig?.worker_tags} + selectedOptionsDraggable={false} + ulOptionsClass={'!bg-surface-secondary'} + placeholder="High priority tags" /> - {/if} -
-
- {#if nconfig?.worker_tags !== undefined && nconfig?.worker_tags.length > 0} - - {/if} -
- {/if} + + {/if} +
{/if} {#if nconfig !== undefined} -
+
+
-
+
{/if}
-
+
@@ -823,7 +731,7 @@ bind:config={nconfig.autoscaling} />
-
+
| undefined = undefined @@ -256,6 +257,8 @@ return Math.ceil(occupancy_rate * 100) + '%' } + + let newHttpAgentWorkerDrawer: Drawer | undefined = undefined {#if $superadmin} @@ -279,6 +282,15 @@ + + newHttpAgentWorkerDrawer?.toggleDrawer?.()} + > + + + + {#if $userStore?.operator && $workspaceStore && !$userWorkspaces.find((_) => _.id === $workspaceStore)?.operator_settings?.workers}