diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 14735dc246..f95dee00da 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -14,9 +14,13 @@ jobs: env: POSTGRES_DB: windmill POSTGRES_PASSWORD: changeme + POSTGRES_INITDB_ARGS: "-c shared_buffers=2GB -c work_mem=32MB -c effective_cache_size=4GB" options: >- --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + --shm-size=2g + + windmill: image: ghcr.io/windmill-labs/windmill-ee:main env: @@ -55,6 +59,7 @@ jobs: env: POSTGRES_DB: windmill POSTGRES_PASSWORD: changeme + POSTGRES_INITDB_ARGS: "-c shared_buffers=2GB -c work_mem=32MB -c effective_cache_size=4GB" options: >- --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 @@ -96,6 +101,7 @@ jobs: env: POSTGRES_DB: windmill POSTGRES_PASSWORD: changeme + POSTGRES_INITDB_ARGS: "-c shared_buffers=2GB -c work_mem=32MB -c effective_cache_size=4GB" options: >- --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 @@ -171,6 +177,7 @@ jobs: env: POSTGRES_DB: windmill POSTGRES_PASSWORD: changeme + POSTGRES_INITDB_ARGS: "-c shared_buffers=2GB -c work_mem=32MB -c effective_cache_size=4GB" options: >- --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 diff --git a/.gitignore b/.gitignore index 27885185f1..3e6c2f3a75 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,4 @@ CaddyfileRemoteMalo *.swp **/.idea/ .direnv -.vscode \ No newline at end of file +.vscode diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f664e6685..8dec2071f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,84 @@ # Changelog +## [1.459.0](https://github.com/windmill-labs/windmill/compare/v1.458.4...v1.459.0) (2025-02-10) + + +### Features + +* triggers cli sync ([#5243](https://github.com/windmill-labs/windmill/issues/5243)) ([df62925](https://github.com/windmill-labs/windmill/commit/df6292589479766acfe642d757f3736dfc369e33)) + + +### Bug Fixes + +* if user is authed, no need to use anonymous path for display result in apps ([deb1861](https://github.com/windmill-labs/windmill/commit/deb18615c20c4650e1bf765350f7abf4d2320a0a)) + +## [1.458.4](https://github.com/windmill-labs/windmill/compare/v1.458.3...v1.458.4) (2025-02-10) + + +### Bug Fixes + +* fix concurrent limit jobs non restarting ([4828a77](https://github.com/windmill-labs/windmill/commit/4828a77f21fe62f36632490f811fb01b39977662)) + +## [1.458.3](https://github.com/windmill-labs/windmill/compare/v1.458.2...v1.458.3) (2025-02-10) + + +### Bug Fixes + +* Support authentication with auth0 ([#5249](https://github.com/windmill-labs/windmill/issues/5249)) ([3d8dee9](https://github.com/windmill-labs/windmill/commit/3d8dee9e6ac10a59caf8e1ef9eff077fd78d2e20)) + +## [1.458.2](https://github.com/windmill-labs/windmill/compare/v1.458.1...v1.458.2) (2025-02-09) + + +### Bug Fixes + +* **frontend:** accordion list header on eval / background function ([#5244](https://github.com/windmill-labs/windmill/issues/5244)) ([32298e5](https://github.com/windmill-labs/windmill/commit/32298e5bfcd9ad1ca2954642d789e0f5d03b1680)) +* worker name in job + better timeout handling for same_worker jobs ([#5248](https://github.com/windmill-labs/windmill/issues/5248)) ([403826f](https://github.com/windmill-labs/windmill/commit/403826fca994535e59cc3c042f41bb47448dd951)) +* workflow as code status ([#5246](https://github.com/windmill-labs/windmill/issues/5246)) ([61ac7e9](https://github.com/windmill-labs/windmill/commit/61ac7e91de7da54bd405d721fe6e47ed8e5a5e9e)) + +## [1.458.1](https://github.com/windmill-labs/windmill/compare/v1.458.0...v1.458.1) (2025-02-07) + + +### Bug Fixes + +* **cli:** ai resource type typo ([#5240](https://github.com/windmill-labs/windmill/issues/5240)) ([b09eb04](https://github.com/windmill-labs/windmill/commit/b09eb04c24743aa43f80bc65e6158cc3b2b557f2)) +* get update endpoint ([#5245](https://github.com/windmill-labs/windmill/issues/5245)) ([fcf8f31](https://github.com/windmill-labs/windmill/commit/fcf8f312bc4a27d8da02542964d5d0122ef71a12)) + +## [1.458.0](https://github.com/windmill-labs/windmill/compare/v1.457.1...v1.458.0) (2025-02-07) + + +### Features + +* serve static websites ([#5218](https://github.com/windmill-labs/windmill/issues/5218)) ([48e4ae6](https://github.com/windmill-labs/windmill/commit/48e4ae6561bfac725bc9ddee2697b44af48ad3a8)) + + +### Bug Fixes + +* remove unnecessary rw on cache for powershell in nsjail ([#5236](https://github.com/windmill-labs/windmill/issues/5236)) ([3bb19dd](https://github.com/windmill-labs/windmill/commit/3bb19dd87cbdb0e36148b8b4e19c4312804fb38e)) + +## [1.457.1](https://github.com/windmill-labs/windmill/compare/v1.457.0...v1.457.1) (2025-02-05) + + +### Bug Fixes + +* fix autoscaling inc increase by customized parameter ([a457c01](https://github.com/windmill-labs/windmill/commit/a457c0137cd60901619817cc2a3906def64667ac)) +* preprocessor args python ([#5210](https://github.com/windmill-labs/windmill/issues/5210)) ([8a446a6](https://github.com/windmill-labs/windmill/commit/8a446a658a1b8e82fd8c3a051dd7274df8c54c71)) + +## [1.457.0](https://github.com/windmill-labs/windmill/compare/v1.456.0...v1.457.0) (2025-02-04) + + +### Features + +* more AI models ([#5207](https://github.com/windmill-labs/windmill/issues/5207)) ([245c871](https://github.com/windmill-labs/windmill/commit/245c8719fc4bf9779be6c653f057d9ffb2724886)) +* **python:** make S3 cache arch specific ([#5196](https://github.com/windmill-labs/windmill/issues/5196)) ([0e80775](https://github.com/windmill-labs/windmill/commit/0e80775d6d2bd1931fed6d5a1d63fc2e70980d55)) + + +### Bug Fixes + +* hide values of WHITELIST_ENVS ([62bfec0](https://github.com/windmill-labs/windmill/commit/62bfec029c00fd067b9906f546b3c9597a54c319)) +* **python:** clear env before installing/finding python ([#5209](https://github.com/windmill-labs/windmill/issues/5209)) ([97c1134](https://github.com/windmill-labs/windmill/commit/97c11340c3175ba946ac6fc77481899fed508af5)) +* support specialization of list of strings to list of enums ([76afbc3](https://github.com/windmill-labs/windmill/commit/76afbc3df33ec87aff47b0db8c57e64209f8f613)) +* timeout on list_user_usage after 300s ([fd0cd58](https://github.com/windmill-labs/windmill/commit/fd0cd587bbe8f453893ae600ae553ee74eb1ba04)) + ## [1.456.0](https://github.com/windmill-labs/windmill/compare/v1.455.2...v1.456.0) (2025-02-01) diff --git a/backend/.sqlx/query-e03b8e0360ed7c282742b9b8657abdeecb3fec75bf10773544339bd025fc45bb.json b/backend/.sqlx/query-00b6340396f5121aff49323d12a7e3244ebb2cb864828637e7ff7476a4a4939b.json similarity index 58% rename from backend/.sqlx/query-e03b8e0360ed7c282742b9b8657abdeecb3fec75bf10773544339bd025fc45bb.json rename to backend/.sqlx/query-00b6340396f5121aff49323d12a7e3244ebb2cb864828637e7ff7476a4a4939b.json index e17fd3f203..389390524e 100644 --- a/backend/.sqlx/query-e03b8e0360ed7c282742b9b8657abdeecb3fec75bf10773544339bd025fc45bb.json +++ b/backend/.sqlx/query-00b6340396f5121aff49323d12a7e3244ebb2cb864828637e7ff7476a4a4939b.json @@ -1,11 +1,11 @@ { "db_name": "PostgreSQL", - "query": "SELECT script_path FROM completed_job WHERE id = $1", + "query": "SELECT runnable_path FROM v2_job WHERE id = $1", "describe": { "columns": [ { "ordinal": 0, - "name": "script_path", + "name": "runnable_path", "type_info": "Varchar" } ], @@ -18,5 +18,5 @@ true ] }, - "hash": "e03b8e0360ed7c282742b9b8657abdeecb3fec75bf10773544339bd025fc45bb" + "hash": "00b6340396f5121aff49323d12a7e3244ebb2cb864828637e7ff7476a4a4939b" } diff --git a/backend/.sqlx/query-00e63eab76d26e148b77e932848de74e8b0943d30481465da453942e299a128f.json b/backend/.sqlx/query-00e63eab76d26e148b77e932848de74e8b0943d30481465da453942e299a128f.json new file mode 100644 index 0000000000..9c1f8d00a7 --- /dev/null +++ b/backend/.sqlx/query-00e63eab76d26e148b77e932848de74e8b0943d30481465da453942e299a128f.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO metrics (id, value)\n VALUES ($1, to_jsonb((\n SELECT EXTRACT(EPOCH FROM now() - scheduled_for)\n FROM v2_job_queue\n WHERE tag = $2 AND running = false AND scheduled_for <= now() - ('3 seconds')::interval\n ORDER BY priority DESC NULLS LAST, scheduled_for LIMIT 1\n )))", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "00e63eab76d26e148b77e932848de74e8b0943d30481465da453942e299a128f" +} diff --git a/backend/.sqlx/query-029ed3dcba207c58aa6936e44bd825b2166f1846b1bb684522607d5ca31a0df3.json b/backend/.sqlx/query-029ed3dcba207c58aa6936e44bd825b2166f1846b1bb684522607d5ca31a0df3.json deleted file mode 100644 index d9cf2bd091..0000000000 --- a/backend/.sqlx/query-029ed3dcba207c58aa6936e44bd825b2166f1846b1bb684522607d5ca31a0df3.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE queue SET flow_status = jsonb_set(COALESCE(flow_status, '{}'::jsonb), array[$1], jsonb_set(jsonb_set('{}'::jsonb, '{scheduled_for}', to_jsonb(now()::text)), '{name}', to_jsonb($4::text))) WHERE id = $2 AND workspace_id = $3", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Uuid", - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "029ed3dcba207c58aa6936e44bd825b2166f1846b1bb684522607d5ca31a0df3" -} diff --git a/backend/.sqlx/query-036af7b1cf6d731647fd718458944b9a9759bdb034e73f3065cde6a2f88c8dce.json b/backend/.sqlx/query-036af7b1cf6d731647fd718458944b9a9759bdb034e73f3065cde6a2f88c8dce.json deleted file mode 100644 index 6f17ee0e9d..0000000000 --- a/backend/.sqlx/query-036af7b1cf6d731647fd718458944b9a9759bdb034e73f3065cde6a2f88c8dce.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE queue SET permissioned_as = ('u/' || $1) WHERE permissioned_as = ('u/' || $2) AND workspace_id = $3", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "036af7b1cf6d731647fd718458944b9a9759bdb034e73f3065cde6a2f88c8dce" -} diff --git a/backend/.sqlx/query-036c84bb9ce72748956bc9c18fbe276444fab025a281dc4784596b0e31c1cb9d.json b/backend/.sqlx/query-036c84bb9ce72748956bc9c18fbe276444fab025a281dc4784596b0e31c1cb9d.json new file mode 100644 index 0000000000..dedd3fd272 --- /dev/null +++ b/backend/.sqlx/query-036c84bb9ce72748956bc9c18fbe276444fab025a281dc4784596b0e31c1cb9d.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "create index concurrently if not exists ix_job_workspace_id_created_at_new_9 ON v2_job (workspace_id, created_at DESC) where kind in ('dependencies', 'flowdependencies', 'appdependencies') AND parent_job IS NULL", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "036c84bb9ce72748956bc9c18fbe276444fab025a281dc4784596b0e31c1cb9d" +} diff --git a/backend/.sqlx/query-7f717130a398c8d52f814a968c2b0bc4dbb9cd654307f5167d8dbe794f17a1cf.json b/backend/.sqlx/query-05d6405b2cc6aabf564a10f05402878e9f2a13e0ce0dad42723f95ac7fb15d4b.json similarity index 50% rename from backend/.sqlx/query-7f717130a398c8d52f814a968c2b0bc4dbb9cd654307f5167d8dbe794f17a1cf.json rename to backend/.sqlx/query-05d6405b2cc6aabf564a10f05402878e9f2a13e0ce0dad42723f95ac7fb15d4b.json index 1df4da3a93..195c9bfe92 100644 --- a/backend/.sqlx/query-7f717130a398c8d52f814a968c2b0bc4dbb9cd654307f5167d8dbe794f17a1cf.json +++ b/backend/.sqlx/query-05d6405b2cc6aabf564a10f05402878e9f2a13e0ce0dad42723f95ac7fb15d4b.json @@ -1,11 +1,11 @@ { "db_name": "PostgreSQL", - "query": "SELECT success FROM completed_job WHERE id = $1 AND workspace_id = $2", + "query": "SELECT success AS \"success!\"\n FROM v2_as_completed_job WHERE id = $1 AND workspace_id = $2", "describe": { "columns": [ { "ordinal": 0, - "name": "success", + "name": "success!", "type_info": "Bool" } ], @@ -16,8 +16,8 @@ ] }, "nullable": [ - false + true ] }, - "hash": "7f717130a398c8d52f814a968c2b0bc4dbb9cd654307f5167d8dbe794f17a1cf" + "hash": "05d6405b2cc6aabf564a10f05402878e9f2a13e0ce0dad42723f95ac7fb15d4b" } diff --git a/backend/.sqlx/query-06731936fb073169b3a1a8a9817f1e669b60edccd260625a95094b7244f5fb83.json b/backend/.sqlx/query-06731936fb073169b3a1a8a9817f1e669b60edccd260625a95094b7244f5fb83.json new file mode 100644 index 0000000000..c509986552 --- /dev/null +++ b/backend/.sqlx/query-06731936fb073169b3a1a8a9817f1e669b60edccd260625a95094b7244f5fb83.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status\n SET flow_status = JSONB_SET(flow_status, ARRAY['retry'], $1)\n WHERE id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "06731936fb073169b3a1a8a9817f1e669b60edccd260625a95094b7244f5fb83" +} diff --git a/backend/.sqlx/query-0721acae4f627df4687bb43b830a47faeee5c0a152cda8d62794c14dd200fac1.json b/backend/.sqlx/query-0721acae4f627df4687bb43b830a47faeee5c0a152cda8d62794c14dd200fac1.json deleted file mode 100644 index fde6a8d881..0000000000 --- a/backend/.sqlx/query-0721acae4f627df4687bb43b830a47faeee5c0a152cda8d62794c14dd200fac1.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE completed_job SET created_by = $1 WHERE created_by = $2 AND workspace_id = $3", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "0721acae4f627df4687bb43b830a47faeee5c0a152cda8d62794c14dd200fac1" -} diff --git a/backend/.sqlx/query-4422b7183ede17a9cbde4afae41be4da5447020e39398deedbcca9121492834a.json b/backend/.sqlx/query-099894523449a70eb301ecd1d744210d39f405016ffec374e4b15a2528baccb5.json similarity index 89% rename from backend/.sqlx/query-4422b7183ede17a9cbde4afae41be4da5447020e39398deedbcca9121492834a.json rename to backend/.sqlx/query-099894523449a70eb301ecd1d744210d39f405016ffec374e4b15a2528baccb5.json index e16088e45a..8a4a9b9c20 100644 --- a/backend/.sqlx/query-4422b7183ede17a9cbde4afae41be4da5447020e39398deedbcca9121492834a.json +++ b/backend/.sqlx/query-099894523449a70eb301ecd1d744210d39f405016ffec374e4b15a2528baccb5.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT schedule.*, t.jobs FROM schedule, LATERAL ( SELECT ARRAY (SELECT json_build_object('id', id, 'success', success, 'duration_ms', duration_ms) FROM completed_job WHERE\n completed_job.schedule_path = schedule.path AND completed_job.workspace_id = $1 AND parent_job IS NULL AND is_skipped = False ORDER BY started_at DESC LIMIT 20) AS jobs ) t\n WHERE schedule.workspace_id = $1 ORDER BY schedule.edited_at desc LIMIT $2 OFFSET $3", + "query": "SELECT schedule.*, t.jobs FROM schedule, LATERAL ( SELECT ARRAY (SELECT json_build_object('id', id, 'success', success, 'duration_ms', duration_ms) FROM v2_as_completed_job WHERE\n v2_as_completed_job.schedule_path = schedule.path AND v2_as_completed_job.workspace_id = $1 AND parent_job IS NULL AND is_skipped = False ORDER BY started_at DESC LIMIT 20) AS jobs ) t\n WHERE schedule.workspace_id = $1 ORDER BY schedule.edited_at desc LIMIT $2 OFFSET $3", "describe": { "columns": [ { @@ -194,5 +194,5 @@ null ] }, - "hash": "4422b7183ede17a9cbde4afae41be4da5447020e39398deedbcca9121492834a" + "hash": "099894523449a70eb301ecd1d744210d39f405016ffec374e4b15a2528baccb5" } diff --git a/backend/.sqlx/query-099e7c7a66968575f896e0c11ecd9cfe9a2ec315d6589e940be157a0563f81af.json b/backend/.sqlx/query-099e7c7a66968575f896e0c11ecd9cfe9a2ec315d6589e940be157a0563f81af.json deleted file mode 100644 index 59fb4a5dba..0000000000 --- a/backend/.sqlx/query-099e7c7a66968575f896e0c11ecd9cfe9a2ec315d6589e940be157a0563f81af.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE queue SET last_ping = now() WHERE id = $1 AND workspace_id = $2 AND canceled = false", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid", - "Text" - ] - }, - "nullable": [] - }, - "hash": "099e7c7a66968575f896e0c11ecd9cfe9a2ec315d6589e940be157a0563f81af" -} diff --git a/backend/.sqlx/query-0a7fb25aa9404c2e6eabfd5d912ed0c1018f249c7ec2a5564787175aa667a456.json b/backend/.sqlx/query-0a7fb25aa9404c2e6eabfd5d912ed0c1018f249c7ec2a5564787175aa667a456.json new file mode 100644 index 0000000000..d340c0e2bc --- /dev/null +++ b/backend/.sqlx/query-0a7fb25aa9404c2e6eabfd5d912ed0c1018f249c7ec2a5564787175aa667a456.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job SET labels = (\n SELECT array_agg(DISTINCT all_labels)\n FROM unnest(coalesce(labels, ARRAY[]::TEXT[]) || $2) all_labels\n ) WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "0a7fb25aa9404c2e6eabfd5d912ed0c1018f249c7ec2a5564787175aa667a456" +} diff --git a/backend/.sqlx/query-286fa00c088df146c08c1934556f06a4243c66787ab8759a8045e60effd2fb77.json b/backend/.sqlx/query-0a9dd1addaf48eeb46eed59abb6daf9819d07b08cf7ca442ea7ec78a9b2b63b2.json similarity index 59% rename from backend/.sqlx/query-286fa00c088df146c08c1934556f06a4243c66787ab8759a8045e60effd2fb77.json rename to backend/.sqlx/query-0a9dd1addaf48eeb46eed59abb6daf9819d07b08cf7ca442ea7ec78a9b2b63b2.json index 790948f378..a0f52168b9 100644 --- a/backend/.sqlx/query-286fa00c088df146c08c1934556f06a4243c66787ab8759a8045e60effd2fb77.json +++ b/backend/.sqlx/query-0a9dd1addaf48eeb46eed59abb6daf9819d07b08cf7ca442ea7ec78a9b2b63b2.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT value\n FROM resource\n WHERE path = $1 AND workspace_id = $2", + "query": "SELECT value\n FROM resource\n WHERE path = $1 AND workspace_id = $2", "describe": { "columns": [ { @@ -19,5 +19,5 @@ true ] }, - "hash": "286fa00c088df146c08c1934556f06a4243c66787ab8759a8045e60effd2fb77" + "hash": "0a9dd1addaf48eeb46eed59abb6daf9819d07b08cf7ca442ea7ec78a9b2b63b2" } diff --git a/backend/.sqlx/query-0aa8f50fe377a23e2ae3821fd6eda4cbe80b0150827c6e90f6e4f9e512587ba1.json b/backend/.sqlx/query-0aa8f50fe377a23e2ae3821fd6eda4cbe80b0150827c6e90f6e4f9e512587ba1.json new file mode 100644 index 0000000000..d48b865a92 --- /dev/null +++ b/backend/.sqlx/query-0aa8f50fe377a23e2ae3821fd6eda4cbe80b0150827c6e90f6e4f9e512587ba1.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_queue SET started_at = NOW() WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "0aa8f50fe377a23e2ae3821fd6eda4cbe80b0150827c6e90f6e4f9e512587ba1" +} diff --git a/backend/.sqlx/query-0bc1c617786bb2fdc71b85442b1d52dbd4c922436edacced18b9620c70e0cc8b.json b/backend/.sqlx/query-0bc1c617786bb2fdc71b85442b1d52dbd4c922436edacced18b9620c70e0cc8b.json new file mode 100644 index 0000000000..104cedde3b --- /dev/null +++ b/backend/.sqlx/query-0bc1c617786bb2fdc71b85442b1d52dbd4c922436edacced18b9620c70e0cc8b.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT result #> $3 AS \"result: Json>\"\n FROM v2_job_completed WHERE id = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "result: Json>", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "TextArray" + ] + }, + "nullable": [ + null + ] + }, + "hash": "0bc1c617786bb2fdc71b85442b1d52dbd4c922436edacced18b9620c70e0cc8b" +} diff --git a/backend/.sqlx/query-0c0b5d5d1e6ab2fed7532f94b50be3210e3845b61551691bbef81c2b6fb01121.json b/backend/.sqlx/query-0c0b5d5d1e6ab2fed7532f94b50be3210e3845b61551691bbef81c2b6fb01121.json deleted file mode 100644 index 9b37241475..0000000000 --- a/backend/.sqlx/query-0c0b5d5d1e6ab2fed7532f94b50be3210e3845b61551691bbef81c2b6fb01121.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE queue\n SET flow_status = JSONB_SET(flow_status, ARRAY['failure_module'], $1)\n WHERE id = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Jsonb", - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "0c0b5d5d1e6ab2fed7532f94b50be3210e3845b61551691bbef81c2b6fb01121" -} diff --git a/backend/.sqlx/query-f9e0e35b4789a4da89f7bb21fa6b250fd64ee0e0b299471979db5ff8ee906929.json b/backend/.sqlx/query-0c9b5b01f7599ad2590a2be01e35da5f5e318582d0bb8ebca6bff5a87c587111.json similarity index 72% rename from backend/.sqlx/query-f9e0e35b4789a4da89f7bb21fa6b250fd64ee0e0b299471979db5ff8ee906929.json rename to backend/.sqlx/query-0c9b5b01f7599ad2590a2be01e35da5f5e318582d0bb8ebca6bff5a87c587111.json index 39c96b8989..c87479e8ab 100644 --- a/backend/.sqlx/query-f9e0e35b4789a4da89f7bb21fa6b250fd64ee0e0b299471979db5ff8ee906929.json +++ b/backend/.sqlx/query-0c9b5b01f7599ad2590a2be01e35da5f5e318582d0bb8ebca6bff5a87c587111.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT (flow_status->'step')::integer as step, jsonb_array_length(flow_status->'modules') as len FROM queue WHERE id = $1", + "query": "SELECT (flow_status->'step')::integer as step, jsonb_array_length(flow_status->'modules') as len\n FROM v2_job_status WHERE id = $1", "describe": { "columns": [ { @@ -24,5 +24,5 @@ null ] }, - "hash": "f9e0e35b4789a4da89f7bb21fa6b250fd64ee0e0b299471979db5ff8ee906929" + "hash": "0c9b5b01f7599ad2590a2be01e35da5f5e318582d0bb8ebca6bff5a87c587111" } diff --git a/backend/.sqlx/query-0d86a31d7d53e52d24df76fa745d968cda48e036139cdaecf4e87d948f8c365e.json b/backend/.sqlx/query-0d86a31d7d53e52d24df76fa745d968cda48e036139cdaecf4e87d948f8c365e.json new file mode 100644 index 0000000000..67c10a27dc --- /dev/null +++ b/backend/.sqlx/query-0d86a31d7d53e52d24df76fa745d968cda48e036139cdaecf4e87d948f8c365e.json @@ -0,0 +1,25 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE v2_job_status f SET flow_status = JSONB_SET(flow_status, ARRAY['user_states'], JSONB_SET(COALESCE(flow_status->'user_states', '{}'::jsonb), ARRAY[$1], $2))\n FROM v2_job j\n WHERE f.id = $3 AND f.id = j.id AND j.workspace_id = $4 AND kind IN ('flow', 'flowpreview', 'flownode') RETURNING 1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text", + "Jsonb", + "Uuid", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "0d86a31d7d53e52d24df76fa745d968cda48e036139cdaecf4e87d948f8c365e" +} diff --git a/backend/.sqlx/query-f4849960aff7387cb6b130d2bc62dbfce45209fbbccf3b4b9b13019e0cd55ddb.json b/backend/.sqlx/query-0dd236e9bfc27ce6cc3ae5a4995003407d523eb0835f59fa2060b7da88ccf601.json similarity index 74% rename from backend/.sqlx/query-f4849960aff7387cb6b130d2bc62dbfce45209fbbccf3b4b9b13019e0cd55ddb.json rename to backend/.sqlx/query-0dd236e9bfc27ce6cc3ae5a4995003407d523eb0835f59fa2060b7da88ccf601.json index f97b5d1da4..88f28ca27b 100644 --- a/backend/.sqlx/query-f4849960aff7387cb6b130d2bc62dbfce45209fbbccf3b4b9b13019e0cd55ddb.json +++ b/backend/.sqlx/query-0dd236e9bfc27ce6cc3ae5a4995003407d523eb0835f59fa2060b7da88ccf601.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT flow_status->'failure_module'->>'parent_module' FROM queue WHERE id = $1", + "query": "SELECT flow_status->'failure_module'->>'parent_module' FROM v2_job_status WHERE id = $1", "describe": { "columns": [ { @@ -18,5 +18,5 @@ null ] }, - "hash": "f4849960aff7387cb6b130d2bc62dbfce45209fbbccf3b4b9b13019e0cd55ddb" + "hash": "0dd236e9bfc27ce6cc3ae5a4995003407d523eb0835f59fa2060b7da88ccf601" } diff --git a/backend/.sqlx/query-cbf5a2b315a7e89689ee86acf452f40fd82605c456aff479890462f4d0202316.json b/backend/.sqlx/query-0ea5ba568ec0f62b808fe938a41174646b6bdd658b8461db1bb90a871d076718.json similarity index 52% rename from backend/.sqlx/query-cbf5a2b315a7e89689ee86acf452f40fd82605c456aff479890462f4d0202316.json rename to backend/.sqlx/query-0ea5ba568ec0f62b808fe938a41174646b6bdd658b8461db1bb90a871d076718.json index 889f2433d7..19fe394067 100644 --- a/backend/.sqlx/query-cbf5a2b315a7e89689ee86acf452f40fd82605c456aff479890462f4d0202316.json +++ b/backend/.sqlx/query-0ea5ba568ec0f62b808fe938a41174646b6bdd658b8461db1bb90a871d076718.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "UPDATE queue SET workspace_id = $1 WHERE workspace_id = $2", + "query": "UPDATE v2_job_queue SET workspace_id = $1 WHERE workspace_id = $2", "describe": { "columns": [], "parameters": { @@ -11,5 +11,5 @@ }, "nullable": [] }, - "hash": "cbf5a2b315a7e89689ee86acf452f40fd82605c456aff479890462f4d0202316" + "hash": "0ea5ba568ec0f62b808fe938a41174646b6bdd658b8461db1bb90a871d076718" } diff --git a/backend/.sqlx/query-8d655c34a00510699d2ad7044f7e526ba5082e5d1945c76a98404fe5d92e32ee.json b/backend/.sqlx/query-0f23535bdfe222eee5a27d52fe4fdce4ad1a487afcd62735ef25386586cfc036.json similarity index 52% rename from backend/.sqlx/query-8d655c34a00510699d2ad7044f7e526ba5082e5d1945c76a98404fe5d92e32ee.json rename to backend/.sqlx/query-0f23535bdfe222eee5a27d52fe4fdce4ad1a487afcd62735ef25386586cfc036.json index 028e35f5dc..c30c505a7b 100644 --- a/backend/.sqlx/query-8d655c34a00510699d2ad7044f7e526ba5082e5d1945c76a98404fe5d92e32ee.json +++ b/backend/.sqlx/query-0f23535bdfe222eee5a27d52fe4fdce4ad1a487afcd62735ef25386586cfc036.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "DELETE FROM queue WHERE id = any($1) AND workspace_id = $2", + "query": "DELETE FROM v2_job_queue WHERE id = any($1) AND workspace_id = $2", "describe": { "columns": [], "parameters": { @@ -11,5 +11,5 @@ }, "nullable": [] }, - "hash": "8d655c34a00510699d2ad7044f7e526ba5082e5d1945c76a98404fe5d92e32ee" + "hash": "0f23535bdfe222eee5a27d52fe4fdce4ad1a487afcd62735ef25386586cfc036" } diff --git a/backend/.sqlx/query-0230bd64d2b6719c3536a106e18a68c7b43b3a9a9efa92b5517132e9c0d8a25b.json b/backend/.sqlx/query-103ef3cf5cf4d25d780e4aefd5b290d810a5e8ea6458d9f9fd484ced549ea82e.json similarity index 50% rename from backend/.sqlx/query-0230bd64d2b6719c3536a106e18a68c7b43b3a9a9efa92b5517132e9c0d8a25b.json rename to backend/.sqlx/query-103ef3cf5cf4d25d780e4aefd5b290d810a5e8ea6458d9f9fd484ced549ea82e.json index e38ddcb726..15d772ab16 100644 --- a/backend/.sqlx/query-0230bd64d2b6719c3536a106e18a68c7b43b3a9a9efa92b5517132e9c0d8a25b.json +++ b/backend/.sqlx/query-103ef3cf5cf4d25d780e4aefd5b290d810a5e8ea6458d9f9fd484ced549ea82e.json @@ -1,21 +1,22 @@ { "db_name": "PostgreSQL", - "query": "SELECT ai_resource, code_completion_enabled FROM workspace_settings WHERE workspace_id = $1", + "query": "SELECT value, resource_type FROM resource WHERE path = $1 AND workspace_id = $2", "describe": { "columns": [ { "ordinal": 0, - "name": "ai_resource", + "name": "value", "type_info": "Jsonb" }, { "ordinal": 1, - "name": "code_completion_enabled", - "type_info": "Bool" + "name": "resource_type", + "type_info": "Varchar" } ], "parameters": { "Left": [ + "Text", "Text" ] }, @@ -24,5 +25,5 @@ false ] }, - "hash": "0230bd64d2b6719c3536a106e18a68c7b43b3a9a9efa92b5517132e9c0d8a25b" + "hash": "103ef3cf5cf4d25d780e4aefd5b290d810a5e8ea6458d9f9fd484ced549ea82e" } diff --git a/backend/.sqlx/query-b053117536c067095e2fb2864ce5af33f84b22c24c92fcb870f37f0501f8ea9a.json b/backend/.sqlx/query-11db65c493990f6935103033b2fbb0c08ae6d91b05b2f3f7c89a990d1d5a5f8a.json similarity index 62% rename from backend/.sqlx/query-b053117536c067095e2fb2864ce5af33f84b22c24c92fcb870f37f0501f8ea9a.json rename to backend/.sqlx/query-11db65c493990f6935103033b2fbb0c08ae6d91b05b2f3f7c89a990d1d5a5f8a.json index 316f6d7eda..1afb036030 100644 --- a/backend/.sqlx/query-b053117536c067095e2fb2864ce5af33f84b22c24c92fcb870f37f0501f8ea9a.json +++ b/backend/.sqlx/query-11db65c493990f6935103033b2fbb0c08ae6d91b05b2f3f7c89a990d1d5a5f8a.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT COUNT(id) FROM queue WHERE running = true AND email = $1", + "query": "SELECT COUNT(id) FROM v2_as_queue WHERE running = true AND email = $1", "describe": { "columns": [ { @@ -18,5 +18,5 @@ null ] }, - "hash": "b053117536c067095e2fb2864ce5af33f84b22c24c92fcb870f37f0501f8ea9a" + "hash": "11db65c493990f6935103033b2fbb0c08ae6d91b05b2f3f7c89a990d1d5a5f8a" } diff --git a/backend/.sqlx/query-1252ef3a652ffb99529c2ce84928197fa15efb9c78d68e3a191c01a04efe153f.json b/backend/.sqlx/query-1252ef3a652ffb99529c2ce84928197fa15efb9c78d68e3a191c01a04efe153f.json new file mode 100644 index 0000000000..d3d6dd84cf --- /dev/null +++ b/backend/.sqlx/query-1252ef3a652ffb99529c2ce84928197fa15efb9c78d68e3a191c01a04efe153f.json @@ -0,0 +1,25 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status SET\n flow_status = JSONB_SET(\n JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4),\n ARRAY['modules', $1::TEXT, 'iterator', 'index'],\n ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb\n )\n WHERE id = $2\n RETURNING (flow_status->'modules'->$1::int->'iterator'->>'index')::int", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "int4", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Int4", + "Uuid", + "Text", + "Jsonb" + ] + }, + "nullable": [ + null + ] + }, + "hash": "1252ef3a652ffb99529c2ce84928197fa15efb9c78d68e3a191c01a04efe153f" +} diff --git a/backend/.sqlx/query-126be8832776644e0d2c5d004acb26f6d4820a6a1fc8e028c3550c438248a82b.json b/backend/.sqlx/query-126be8832776644e0d2c5d004acb26f6d4820a6a1fc8e028c3550c438248a82b.json deleted file mode 100644 index ecc75e957a..0000000000 --- a/backend/.sqlx/query-126be8832776644e0d2c5d004acb26f6d4820a6a1fc8e028c3550c438248a82b.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE queue\n SET running = false\n , started_at = null\n , scheduled_for = $1\n , last_ping = null\n WHERE id = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Timestamptz", - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "126be8832776644e0d2c5d004acb26f6d4820a6a1fc8e028c3550c438248a82b" -} diff --git a/backend/.sqlx/query-45950064cce9f53f73a01ddcd6911ec677297009b71041d39019c4700a571c0f.json b/backend/.sqlx/query-12828c9b2964f2b484a68de1e01b65cdcd277257192ee0a6d18a00f41bce49d4.json similarity index 54% rename from backend/.sqlx/query-45950064cce9f53f73a01ddcd6911ec677297009b71041d39019c4700a571c0f.json rename to backend/.sqlx/query-12828c9b2964f2b484a68de1e01b65cdcd277257192ee0a6d18a00f41bce49d4.json index ecba5ab5c0..84c0804c95 100644 --- a/backend/.sqlx/query-45950064cce9f53f73a01ddcd6911ec677297009b71041d39019c4700a571c0f.json +++ b/backend/.sqlx/query-12828c9b2964f2b484a68de1e01b65cdcd277257192ee0a6d18a00f41bce49d4.json @@ -1,11 +1,11 @@ { "db_name": "PostgreSQL", - "query": "SELECT raw_flow->'failure_module' != 'null'::jsonb FROM completed_job WHERE id = $1", + "query": "SELECT suspend > 0 AS \"r!\" FROM v2_job_queue WHERE id = $1", "describe": { "columns": [ { "ordinal": 0, - "name": "?column?", + "name": "r!", "type_info": "Bool" } ], @@ -18,5 +18,5 @@ null ] }, - "hash": "45950064cce9f53f73a01ddcd6911ec677297009b71041d39019c4700a571c0f" + "hash": "12828c9b2964f2b484a68de1e01b65cdcd277257192ee0a6d18a00f41bce49d4" } diff --git a/backend/.sqlx/query-170f620fbd99269d194d14d56f6a3863d9db5fe736a0a34325b824d9cec9b1a0.json b/backend/.sqlx/query-170f620fbd99269d194d14d56f6a3863d9db5fe736a0a34325b824d9cec9b1a0.json deleted file mode 100644 index 61fd7ed3ef..0000000000 --- a/backend/.sqlx/query-170f620fbd99269d194d14d56f6a3863d9db5fe736a0a34325b824d9cec9b1a0.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO completed_job AS cj\n ( workspace_id\n , id\n , parent_job\n , created_by\n , created_at\n , started_at\n , duration_ms\n , success\n , script_hash\n , script_path\n , args\n , result\n , raw_code\n , raw_lock\n , canceled\n , canceled_by\n , canceled_reason\n , job_kind\n , schedule_path\n , permissioned_as\n , flow_status\n , raw_flow\n , is_flow_step\n , is_skipped\n , language\n , email\n , visible_to_owner\n , mem_peak\n , tag\n , priority\n )\n SELECT workspace_id\n , id\n , parent_job\n , created_by\n , created_at\n , now()\n , 0\n , false\n , script_hash\n , script_path\n , args\n , $4\n , raw_code\n , raw_lock\n , true\n , $1\n , canceled_reason\n , job_kind\n , schedule_path\n , permissioned_as\n , flow_status\n , raw_flow\n , is_flow_step\n , false\n , language\n , email\n , visible_to_owner\n , mem_peak\n , tag\n , priority FROM queue \n WHERE id = any($2) AND running = false AND parent_job IS NULL AND workspace_id = $3 AND schedule_path IS NULL FOR UPDATE SKIP LOCKED\n ON CONFLICT (id) DO NOTHING RETURNING id", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Uuid" - } - ], - "parameters": { - "Left": [ - "Varchar", - "UuidArray", - "Text", - "Jsonb" - ] - }, - "nullable": [ - false - ] - }, - "hash": "170f620fbd99269d194d14d56f6a3863d9db5fe736a0a34325b824d9cec9b1a0" -} diff --git a/backend/.sqlx/query-1730f39fd1793d45fbb41b21389c61296a3ff7489ae12f52a19f9543173ac597.json b/backend/.sqlx/query-1730f39fd1793d45fbb41b21389c61296a3ff7489ae12f52a19f9543173ac597.json index b6fee2c5ff..03da2cee85 100644 --- a/backend/.sqlx/query-1730f39fd1793d45fbb41b21389c61296a3ff7489ae12f52a19f9543173ac597.json +++ b/backend/.sqlx/query-1730f39fd1793d45fbb41b21389c61296a3ff7489ae12f52a19f9543173ac597.json @@ -70,68 +70,73 @@ }, { "ordinal": 13, - "name": "code_completion_enabled", - "type_info": "Bool" - }, - { - "ordinal": 14, "name": "error_handler_extra_args", "type_info": "Json" }, { - "ordinal": 15, + "ordinal": 14, "name": "error_handler_muted_on_cancel", "type_info": "Bool" }, { - "ordinal": 16, + "ordinal": 15, "name": "large_file_storage", "type_info": "Jsonb" }, { - "ordinal": 17, + "ordinal": 16, "name": "git_sync", "type_info": "Jsonb" }, { - "ordinal": 18, + "ordinal": 17, "name": "default_app", "type_info": "Varchar" }, { - "ordinal": 19, + "ordinal": 18, "name": "auto_add", "type_info": "Bool" }, { - "ordinal": 20, + "ordinal": 19, "name": "automatic_billing", "type_info": "Bool" }, { - "ordinal": 21, + "ordinal": 20, "name": "default_scripts", "type_info": "Jsonb" }, { - "ordinal": 22, + "ordinal": 21, "name": "deploy_ui", "type_info": "Jsonb" }, { - "ordinal": 23, + "ordinal": 22, "name": "mute_critical_alerts", "type_info": "Bool" }, { - "ordinal": 24, + "ordinal": 23, "name": "color", "type_info": "Varchar" }, { - "ordinal": 25, + "ordinal": 24, "name": "operator_settings", "type_info": "Jsonb" + }, + { + "ordinal": 25, + "name": "ai_models", + "type_info": "VarcharArray" + }, + { + "ordinal": 26, + "name": "code_completion_model", + "type_info": "Varchar" } ], "parameters": { @@ -153,7 +158,6 @@ true, true, true, - false, true, false, true, @@ -165,6 +169,8 @@ true, true, true, + true, + false, true ] }, diff --git a/backend/.sqlx/query-173fbfd3ca2344fd08f73af75524c917d27fdb6273a35a563292b1f0701dc6ed.json b/backend/.sqlx/query-173fbfd3ca2344fd08f73af75524c917d27fdb6273a35a563292b1f0701dc6ed.json new file mode 100644 index 0000000000..214bf50c6e --- /dev/null +++ b/backend/.sqlx/query-173fbfd3ca2344fd08f73af75524c917d27fdb6273a35a563292b1f0701dc6ed.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status SET\n flow_status = JSONB_SET(\n JSONB_SET(flow_status, ARRAY['failure_module'], $1),\n ARRAY['step'],\n $2\n )\n WHERE id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Jsonb", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "173fbfd3ca2344fd08f73af75524c917d27fdb6273a35a563292b1f0701dc6ed" +} diff --git a/backend/.sqlx/query-17851a0710b80ffd6bebe42012a354665dff01554549ea7bbbb9953c68231296.json b/backend/.sqlx/query-17851a0710b80ffd6bebe42012a354665dff01554549ea7bbbb9953c68231296.json new file mode 100644 index 0000000000..eefc009b5e --- /dev/null +++ b/backend/.sqlx/query-17851a0710b80ffd6bebe42012a354665dff01554549ea7bbbb9953c68231296.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status SET\n flow_status = JSONB_SET(\n flow_status,\n ARRAY['modules', $1::TEXT, 'iterator', 'index'],\n ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb\n )\n WHERE id = $2\n RETURNING (flow_status->'modules'->$1::int->'iterator'->>'index')::int", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "int4", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Int4", + "Uuid" + ] + }, + "nullable": [ + null + ] + }, + "hash": "17851a0710b80ffd6bebe42012a354665dff01554549ea7bbbb9953c68231296" +} diff --git a/backend/.sqlx/query-17e3e2a3232865c33fa535b5d99455942e30f932c5b97f1e5b508128f39a288f.json b/backend/.sqlx/query-17e3e2a3232865c33fa535b5d99455942e30f932c5b97f1e5b508128f39a288f.json deleted file mode 100644 index 4000bbb962..0000000000 --- a/backend/.sqlx/query-17e3e2a3232865c33fa535b5d99455942e30f932c5b97f1e5b508128f39a288f.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE queue\n SET flow_status = JSONB_SET(\n JSONB_SET(flow_status, ARRAY['preprocessor_module'], $1), ARRAY['step'], $2)\n WHERE id = $3", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Jsonb", - "Jsonb", - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "17e3e2a3232865c33fa535b5d99455942e30f932c5b97f1e5b508128f39a288f" -} diff --git a/backend/.sqlx/query-1850552883e67da181d68ff5c4e1babaa2fe072900b57e78e461590a6dafb682.json b/backend/.sqlx/query-1850552883e67da181d68ff5c4e1babaa2fe072900b57e78e461590a6dafb682.json new file mode 100644 index 0000000000..037e63ae49 --- /dev/null +++ b/backend/.sqlx/query-1850552883e67da181d68ff5c4e1babaa2fe072900b57e78e461590a6dafb682.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status\n SET flow_status = JSONB_SET(flow_status, ARRAY['preprocessor_module'], $1)\n WHERE id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "1850552883e67da181d68ff5c4e1babaa2fe072900b57e78e461590a6dafb682" +} diff --git a/backend/.sqlx/query-a76be7f4e9e8b8c81afe50fcbf1f3d393ed0bb73314e5c430f32541c43bcac52.json b/backend/.sqlx/query-1961d15ae075072bd5f677c95f9b4dac7f747585d98aa8bcf60dcfb4c8124028.json similarity index 84% rename from backend/.sqlx/query-a76be7f4e9e8b8c81afe50fcbf1f3d393ed0bb73314e5c430f32541c43bcac52.json rename to backend/.sqlx/query-1961d15ae075072bd5f677c95f9b4dac7f747585d98aa8bcf60dcfb4c8124028.json index fb1cea135a..418be57757 100644 --- a/backend/.sqlx/query-a76be7f4e9e8b8c81afe50fcbf1f3d393ed0bb73314e5c430f32541c43bcac52.json +++ b/backend/.sqlx/query-1961d15ae075072bd5f677c95f9b4dac7f747585d98aa8bcf60dcfb4c8124028.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT path, script_path, is_flow, route_path, workspace_id, is_async, requires_auth, edited_by, email, static_asset_config as \"static_asset_config: _\" FROM http_trigger WHERE workspace_id = $1 AND http_method = $2", + "query": "SELECT path, script_path, is_flow, route_path, workspace_id, is_async, requires_auth, edited_by, email, static_asset_config as \"static_asset_config: _\", is_static_website FROM http_trigger WHERE http_method = $1", "describe": { "columns": [ { @@ -52,11 +52,15 @@ "ordinal": 9, "name": "static_asset_config: _", "type_info": "Jsonb" + }, + { + "ordinal": 10, + "name": "is_static_website", + "type_info": "Bool" } ], "parameters": { "Left": [ - "Text", { "Custom": { "name": "http_method", @@ -83,8 +87,9 @@ false, false, false, - true + true, + false ] }, - "hash": "a76be7f4e9e8b8c81afe50fcbf1f3d393ed0bb73314e5c430f32541c43bcac52" + "hash": "1961d15ae075072bd5f677c95f9b4dac7f747585d98aa8bcf60dcfb4c8124028" } diff --git a/backend/.sqlx/query-f904702536c106b0e5da8facae119c6af887c49a29ae44b3a95350ff27fb1ccf.json b/backend/.sqlx/query-197321abfe4667256761884970334f58c1b3edfcc1e863ec4316c9742a1ac7c8.json similarity index 86% rename from backend/.sqlx/query-f904702536c106b0e5da8facae119c6af887c49a29ae44b3a95350ff27fb1ccf.json rename to backend/.sqlx/query-197321abfe4667256761884970334f58c1b3edfcc1e863ec4316c9742a1ac7c8.json index 08592a3f04..bfc823c911 100644 --- a/backend/.sqlx/query-f904702536c106b0e5da8facae119c6af887c49a29ae44b3a95350ff27fb1ccf.json +++ b/backend/.sqlx/query-197321abfe4667256761884970334f58c1b3edfcc1e863ec4316c9742a1ac7c8.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT workspace_id, path, route_path, route_path_key, script_path, is_flow, http_method as \"http_method: _\", edited_by, email, edited_at, extra_perms, is_async, requires_auth, static_asset_config as \"static_asset_config: _\"\n FROM http_trigger\n WHERE workspace_id = $1 AND path = $2", + "query": "SELECT workspace_id, path, route_path, route_path_key, script_path, is_flow, http_method as \"http_method: _\", edited_by, email, edited_at, extra_perms, is_async, requires_auth, static_asset_config as \"static_asset_config: _\", is_static_website\n FROM http_trigger\n WHERE workspace_id = $1 AND path = $2", "describe": { "columns": [ { @@ -85,6 +85,11 @@ "ordinal": 13, "name": "static_asset_config: _", "type_info": "Jsonb" + }, + { + "ordinal": 14, + "name": "is_static_website", + "type_info": "Bool" } ], "parameters": { @@ -107,8 +112,9 @@ false, false, false, - true + true, + false ] }, - "hash": "f904702536c106b0e5da8facae119c6af887c49a29ae44b3a95350ff27fb1ccf" + "hash": "197321abfe4667256761884970334f58c1b3edfcc1e863ec4316c9742a1ac7c8" } diff --git a/backend/.sqlx/query-28a878c59b6d52f42d315eabb34c96133b69542b01232295c26cc9e093c372f9.json b/backend/.sqlx/query-19cc8499f682ec34d54bc4f694cb281a9bd7f5431c646c6268513751fff95395.json similarity index 72% rename from backend/.sqlx/query-28a878c59b6d52f42d315eabb34c96133b69542b01232295c26cc9e093c372f9.json rename to backend/.sqlx/query-19cc8499f682ec34d54bc4f694cb281a9bd7f5431c646c6268513751fff95395.json index fcdf2ac38d..0c379a7bdb 100644 --- a/backend/.sqlx/query-28a878c59b6d52f42d315eabb34c96133b69542b01232295c26cc9e093c372f9.json +++ b/backend/.sqlx/query-19cc8499f682ec34d54bc4f694cb281a9bd7f5431c646c6268513751fff95395.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT coalesce(COUNT(*) FILTER(WHERE suspend = 0 AND running = false), 0) as \"database_length!\", coalesce(COUNT(*) FILTER(WHERE suspend > 0), 0) as \"suspended!\" FROM queue WHERE (workspace_id = $1 OR $2) AND scheduled_for <= now()", + "query": "SELECT coalesce(COUNT(*) FILTER(WHERE suspend = 0 AND running = false), 0) as \"database_length!\", coalesce(COUNT(*) FILTER(WHERE suspend > 0), 0) as \"suspended!\" FROM v2_as_queue WHERE (workspace_id = $1 OR $2) AND scheduled_for <= now()", "describe": { "columns": [ { @@ -25,5 +25,5 @@ null ] }, - "hash": "28a878c59b6d52f42d315eabb34c96133b69542b01232295c26cc9e093c372f9" + "hash": "19cc8499f682ec34d54bc4f694cb281a9bd7f5431c646c6268513751fff95395" } diff --git a/backend/.sqlx/query-e6f85cdbe681ace495fde31e67339dc58460b6914440473fe94de7a7bc292af4.json b/backend/.sqlx/query-1a85ca0a6d0ba5ab3462907e35037136fd123f80389abe6ebbcf12084da45868.json similarity index 52% rename from backend/.sqlx/query-e6f85cdbe681ace495fde31e67339dc58460b6914440473fe94de7a7bc292af4.json rename to backend/.sqlx/query-1a85ca0a6d0ba5ab3462907e35037136fd123f80389abe6ebbcf12084da45868.json index b35f375b8c..1d83c36fd8 100644 --- a/backend/.sqlx/query-e6f85cdbe681ace495fde31e67339dc58460b6914440473fe94de7a7bc292af4.json +++ b/backend/.sqlx/query-1a85ca0a6d0ba5ab3462907e35037136fd123f80389abe6ebbcf12084da45868.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "DELETE FROM completed_job WHERE workspace_id = $1", + "query": "DELETE FROM v2_job_queue WHERE workspace_id = $1", "describe": { "columns": [], "parameters": { @@ -10,5 +10,5 @@ }, "nullable": [] }, - "hash": "e6f85cdbe681ace495fde31e67339dc58460b6914440473fe94de7a7bc292af4" + "hash": "1a85ca0a6d0ba5ab3462907e35037136fd123f80389abe6ebbcf12084da45868" } diff --git a/backend/.sqlx/query-12a0fd7d8d99fb73b01bc24774fe9a8da57b5204bb6b1207aed47143c17a20bc.json b/backend/.sqlx/query-1a9984fa378634f7f12356407831807b5e3415c28aacbf09cf7bac58bb1d8470.json similarity index 61% rename from backend/.sqlx/query-12a0fd7d8d99fb73b01bc24774fe9a8da57b5204bb6b1207aed47143c17a20bc.json rename to backend/.sqlx/query-1a9984fa378634f7f12356407831807b5e3415c28aacbf09cf7bac58bb1d8470.json index e60e67da63..3f9c7a61a4 100644 --- a/backend/.sqlx/query-12a0fd7d8d99fb73b01bc24774fe9a8da57b5204bb6b1207aed47143c17a20bc.json +++ b/backend/.sqlx/query-1a9984fa378634f7f12356407831807b5e3415c28aacbf09cf7bac58bb1d8470.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT COUNT(id) FROM queue WHERE running = true AND workspace_id = $1", + "query": "SELECT COUNT(id) FROM v2_job_queue WHERE running = true AND workspace_id = $1", "describe": { "columns": [ { @@ -18,5 +18,5 @@ null ] }, - "hash": "12a0fd7d8d99fb73b01bc24774fe9a8da57b5204bb6b1207aed47143c17a20bc" + "hash": "1a9984fa378634f7f12356407831807b5e3415c28aacbf09cf7bac58bb1d8470" } diff --git a/backend/.sqlx/query-1b58b90c184ca21d777ea4e264c79aecc2361134a4817c2b9580f2680425352d.json b/backend/.sqlx/query-1b58b90c184ca21d777ea4e264c79aecc2361134a4817c2b9580f2680425352d.json new file mode 100644 index 0000000000..d0b582d17b --- /dev/null +++ b/backend/.sqlx/query-1b58b90c184ca21d777ea4e264c79aecc2361134a4817c2b9580f2680425352d.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT args AS \"args: Json>>\"\n FROM v2_job WHERE id = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "args: Json>>", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "1b58b90c184ca21d777ea4e264c79aecc2361134a4817c2b9580f2680425352d" +} diff --git a/backend/.sqlx/query-6940ddc5ee8d2a1048213d46f52d964b199604ba66dedbe9f89cea727d726a2d.json b/backend/.sqlx/query-1bbef6baa5b8e2522d685df2979bb1e4b9022f5e841afd9eeb08a81688f6c0c8.json similarity index 57% rename from backend/.sqlx/query-6940ddc5ee8d2a1048213d46f52d964b199604ba66dedbe9f89cea727d726a2d.json rename to backend/.sqlx/query-1bbef6baa5b8e2522d685df2979bb1e4b9022f5e841afd9eeb08a81688f6c0c8.json index 4891854d48..4fcd1f0969 100644 --- a/backend/.sqlx/query-6940ddc5ee8d2a1048213d46f52d964b199604ba66dedbe9f89cea727d726a2d.json +++ b/backend/.sqlx/query-1bbef6baa5b8e2522d685df2979bb1e4b9022f5e841afd9eeb08a81688f6c0c8.json @@ -1,15 +1,15 @@ { "db_name": "PostgreSQL", - "query": "UPDATE workspace_settings SET ai_resource = NULL, code_completion_enabled = $1 WHERE workspace_id = $2", + "query": "UPDATE workspace_settings SET ai_resource = NULL, code_completion_model = $1, ai_models = '{}' WHERE workspace_id = $2", "describe": { "columns": [], "parameters": { "Left": [ - "Bool", + "Varchar", "Text" ] }, "nullable": [] }, - "hash": "6940ddc5ee8d2a1048213d46f52d964b199604ba66dedbe9f89cea727d726a2d" + "hash": "1bbef6baa5b8e2522d685df2979bb1e4b9022f5e841afd9eeb08a81688f6c0c8" } diff --git a/backend/.sqlx/query-1c2254c15696d3dbc091488311676641ba3c6f1f1b5e006fc75427c9b231d323.json b/backend/.sqlx/query-1c2254c15696d3dbc091488311676641ba3c6f1f1b5e006fc75427c9b231d323.json deleted file mode 100644 index 110713fd0e..0000000000 --- a/backend/.sqlx/query-1c2254c15696d3dbc091488311676641ba3c6f1f1b5e006fc75427c9b231d323.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE completed_job SET script_path = REGEXP_REPLACE(script_path,'u/' || $2 || '/(.*)','u/' || $1 || '/\\1') WHERE script_path LIKE ('u/' || $2 || '/%') AND workspace_id = $3", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "1c2254c15696d3dbc091488311676641ba3c6f1f1b5e006fc75427c9b231d323" -} diff --git a/backend/.sqlx/query-1d819b829cd92995c39d29540df8cffbcc3334bada244a331a0bd8db06029d42.json b/backend/.sqlx/query-1d819b829cd92995c39d29540df8cffbcc3334bada244a331a0bd8db06029d42.json new file mode 100644 index 0000000000..9f40c5293d --- /dev/null +++ b/backend/.sqlx/query-1d819b829cd92995c39d29540df8cffbcc3334bada244a331a0bd8db06029d42.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM v2_job_completed c\n USING v2_job j\n WHERE\n created_at <= now() - ($1::bigint::text || ' s')::interval\n AND completed_at + ($1::bigint::text || ' s')::interval <= now()\n AND c.id = j.id\n RETURNING c.id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + false + ] + }, + "hash": "1d819b829cd92995c39d29540df8cffbcc3334bada244a331a0bd8db06029d42" +} diff --git a/backend/.sqlx/query-1d87f41fd1abb9361d795a899120e6b77e24bf5a9044fdc5284d0d7f1e14eafa.json b/backend/.sqlx/query-1d87f41fd1abb9361d795a899120e6b77e24bf5a9044fdc5284d0d7f1e14eafa.json new file mode 100644 index 0000000000..6a79013c0b --- /dev/null +++ b/backend/.sqlx/query-1d87f41fd1abb9361d795a899120e6b77e24bf5a9044fdc5284d0d7f1e14eafa.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO usage (id, is_workspace, month_, usage) \n VALUES ($1, TRUE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), $2) \n ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Int4" + ] + }, + "nullable": [] + }, + "hash": "1d87f41fd1abb9361d795a899120e6b77e24bf5a9044fdc5284d0d7f1e14eafa" +} diff --git a/backend/.sqlx/query-1e188d8e427cab25dbe18aa900260e26e644a9d939e74a8317c4a09335f110fe.json b/backend/.sqlx/query-1e188d8e427cab25dbe18aa900260e26e644a9d939e74a8317c4a09335f110fe.json deleted file mode 100644 index 01fb19b1c2..0000000000 --- a/backend/.sqlx/query-1e188d8e427cab25dbe18aa900260e26e644a9d939e74a8317c4a09335f110fe.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE queue\n SET flow_status = JSONB_SET(\n JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4),\n ARRAY['modules', $1::TEXT, 'branchall', 'branch'], ((flow_status->'modules'->$1::int->'branchall'->>'branch')::int + 1)::text::jsonb),\n last_ping = NULL\n WHERE id = $2\n RETURNING (flow_status->'modules'->$1::int->'branchall'->>'branch')::int", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "int4", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Int4", - "Uuid", - "Text", - "Jsonb" - ] - }, - "nullable": [ - null - ] - }, - "hash": "1e188d8e427cab25dbe18aa900260e26e644a9d939e74a8317c4a09335f110fe" -} diff --git a/backend/.sqlx/query-1e43e6040ac95b586d4d73999025f4a3e79c87fc1c5b43e787fd8e19c555b44b.json b/backend/.sqlx/query-1e43e6040ac95b586d4d73999025f4a3e79c87fc1c5b43e787fd8e19c555b44b.json deleted file mode 100644 index b64ace475a..0000000000 --- a/backend/.sqlx/query-1e43e6040ac95b586d4d73999025f4a3e79c87fc1c5b43e787fd8e19c555b44b.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE queue\n SET last_ping = null\n WHERE id = $1", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "1e43e6040ac95b586d4d73999025f4a3e79c87fc1c5b43e787fd8e19c555b44b" -} diff --git a/backend/.sqlx/query-202e580eacc7907bb34b6464a66fe96ff91951fc8486d1f0f5733b8e63f043bc.json b/backend/.sqlx/query-202e580eacc7907bb34b6464a66fe96ff91951fc8486d1f0f5733b8e63f043bc.json new file mode 100644 index 0000000000..23d9bb4cc4 --- /dev/null +++ b/backend/.sqlx/query-202e580eacc7907bb34b6464a66fe96ff91951fc8486d1f0f5733b8e63f043bc.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_runtime SET ping = null WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "202e580eacc7907bb34b6464a66fe96ff91951fc8486d1f0f5733b8e63f043bc" +} diff --git a/backend/.sqlx/query-205e4faf72177cae685c7f0727659d0e223b98f3ca84fdff21b1bc57c2ca3512.json b/backend/.sqlx/query-205e4faf72177cae685c7f0727659d0e223b98f3ca84fdff21b1bc57c2ca3512.json new file mode 100644 index 0000000000..980cc6787b --- /dev/null +++ b/backend/.sqlx/query-205e4faf72177cae685c7f0727659d0e223b98f3ca84fdff21b1bc57c2ca3512.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE job_logs SET logs = '##DELETED##' WHERE job_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "205e4faf72177cae685c7f0727659d0e223b98f3ca84fdff21b1bc57c2ca3512" +} diff --git a/backend/.sqlx/query-215e0d320a304c8cb9ef12e7ea98a4eafb2456c123f9b6b96bb4ba2409166e5a.json b/backend/.sqlx/query-215e0d320a304c8cb9ef12e7ea98a4eafb2456c123f9b6b96bb4ba2409166e5a.json deleted file mode 100644 index 0889a223e6..0000000000 --- a/backend/.sqlx/query-215e0d320a304c8cb9ef12e7ea98a4eafb2456c123f9b6b96bb4ba2409166e5a.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE queue SET running = false, started_at = null WHERE id = $1 AND canceled = false", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "215e0d320a304c8cb9ef12e7ea98a4eafb2456c123f9b6b96bb4ba2409166e5a" -} diff --git a/backend/.sqlx/query-217a5291438d23597b2c7f05d2c481f406364d56a129089a268f6423c548bca6.json b/backend/.sqlx/query-217a5291438d23597b2c7f05d2c481f406364d56a129089a268f6423c548bca6.json deleted file mode 100644 index 9dac2a2451..0000000000 --- a/backend/.sqlx/query-217a5291438d23597b2c7f05d2c481f406364d56a129089a268f6423c548bca6.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT raw_code, raw_lock, raw_flow AS \"raw_flow: Json>\"\n FROM queue WHERE id = $1 AND workspace_id = $2 LIMIT 1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "raw_code", - "type_info": "Text" - }, - { - "ordinal": 1, - "name": "raw_lock", - "type_info": "Text" - }, - { - "ordinal": 2, - "name": "raw_flow: Json>", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [ - "Uuid", - "Text" - ] - }, - "nullable": [ - true, - true, - true - ] - }, - "hash": "217a5291438d23597b2c7f05d2c481f406364d56a129089a268f6423c548bca6" -} diff --git a/backend/.sqlx/query-241270e20c751806dece12fbc2de360e389da8c7f653ef8e6bc0d30c823aea51.json b/backend/.sqlx/query-241270e20c751806dece12fbc2de360e389da8c7f653ef8e6bc0d30c823aea51.json new file mode 100644 index 0000000000..eefc3dd8e8 --- /dev/null +++ b/backend/.sqlx/query-241270e20c751806dece12fbc2de360e389da8c7f653ef8e6bc0d30c823aea51.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT \n flow_status->>'step' = '0' \n AND (\n jsonb_array_length(flow_status->'modules') = 0 \n OR flow_status->'modules'->0->>'type' = 'WaitingForPriorSteps' \n OR (\n flow_status->'modules'->0->>'type' = 'Failure' \n AND flow_status->'modules'->0->>'job' = $1\n )\n )\n FROM v2_job_completed WHERE id = $2 AND workspace_id = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Uuid", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "241270e20c751806dece12fbc2de360e389da8c7f653ef8e6bc0d30c823aea51" +} diff --git a/backend/.sqlx/query-2456fc71fc7a0758a4c1fbe77d72fbac2fead0e1bff4e909fd7fb1a41bc35d8f.json b/backend/.sqlx/query-2456fc71fc7a0758a4c1fbe77d72fbac2fead0e1bff4e909fd7fb1a41bc35d8f.json new file mode 100644 index 0000000000..07ae8f17c5 --- /dev/null +++ b/backend/.sqlx/query-2456fc71fc7a0758a4c1fbe77d72fbac2fead0e1bff4e909fd7fb1a41bc35d8f.json @@ -0,0 +1,41 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n script_path, args AS \"args: sqlx::types::Json>>\",\n tag AS \"tag!\", priority\n FROM v2_as_completed_job\n WHERE id = $1 and workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "script_path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "args: sqlx::types::Json>>", + "type_info": "Jsonb" + }, + { + "ordinal": 2, + "name": "tag!", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "priority", + "type_info": "Int2" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + true, + true, + true, + true + ] + }, + "hash": "2456fc71fc7a0758a4c1fbe77d72fbac2fead0e1bff4e909fd7fb1a41bc35d8f" +} diff --git a/backend/.sqlx/query-26106be4d94c159cc8d9eb37a3b94927c5d3d43c18cdb5e98e979e761ed6ed0e.json b/backend/.sqlx/query-26106be4d94c159cc8d9eb37a3b94927c5d3d43c18cdb5e98e979e761ed6ed0e.json deleted file mode 100644 index fa5e4bff4c..0000000000 --- a/backend/.sqlx/query-26106be4d94c159cc8d9eb37a3b94927c5d3d43c18cdb5e98e979e761ed6ed0e.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE queue\n SET flow_status = JSONB_SET(flow_status, ARRAY['step'], $1)\n WHERE id = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Jsonb", - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "26106be4d94c159cc8d9eb37a3b94927c5d3d43c18cdb5e98e979e761ed6ed0e" -} diff --git a/backend/.sqlx/query-262c7b21e77a8d2943fefb9cabe1e60c7c3b4e3ce7ed6b2b3eb78dd99b7d8fcf.json b/backend/.sqlx/query-262c7b21e77a8d2943fefb9cabe1e60c7c3b4e3ce7ed6b2b3eb78dd99b7d8fcf.json deleted file mode 100644 index 0025d31dc8..0000000000 --- a/backend/.sqlx/query-262c7b21e77a8d2943fefb9cabe1e60c7c3b4e3ce7ed6b2b3eb78dd99b7d8fcf.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE queue\n SET last_ping = now()\n WHERE id = $1 AND last_ping < now()", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "262c7b21e77a8d2943fefb9cabe1e60c7c3b4e3ce7ed6b2b3eb78dd99b7d8fcf" -} diff --git a/backend/.sqlx/query-04effcc6050250a02661323c880d493982dd1bfb63ca7373e035a98c268428e2.json b/backend/.sqlx/query-280a361076d1c6317610765960f543252891c53351bdc98da66cc30ffc895866.json similarity index 64% rename from backend/.sqlx/query-04effcc6050250a02661323c880d493982dd1bfb63ca7373e035a98c268428e2.json rename to backend/.sqlx/query-280a361076d1c6317610765960f543252891c53351bdc98da66cc30ffc895866.json index caf2c0b4c8..e9705c23a6 100644 --- a/backend/.sqlx/query-04effcc6050250a02661323c880d493982dd1bfb63ca7373e035a98c268428e2.json +++ b/backend/.sqlx/query-280a361076d1c6317610765960f543252891c53351bdc98da66cc30ffc895866.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT script_path FROM queue WHERE id = $1", + "query": "SELECT script_path FROM v2_as_completed_job WHERE id = $1", "describe": { "columns": [ { @@ -18,5 +18,5 @@ true ] }, - "hash": "04effcc6050250a02661323c880d493982dd1bfb63ca7373e035a98c268428e2" + "hash": "280a361076d1c6317610765960f543252891c53351bdc98da66cc30ffc895866" } diff --git a/backend/.sqlx/query-f04fa0262091c5a4abf7dddafeec1b34ed806dda70592b0030067aed46a104d9.json b/backend/.sqlx/query-282afbff89d3186d47ef5dbd0b65026ad37fb31b485fc44b6ec257dd77825428.json similarity index 59% rename from backend/.sqlx/query-f04fa0262091c5a4abf7dddafeec1b34ed806dda70592b0030067aed46a104d9.json rename to backend/.sqlx/query-282afbff89d3186d47ef5dbd0b65026ad37fb31b485fc44b6ec257dd77825428.json index 58fec1a0d0..200bd3155d 100644 --- a/backend/.sqlx/query-f04fa0262091c5a4abf7dddafeec1b34ed806dda70592b0030067aed46a104d9.json +++ b/backend/.sqlx/query-282afbff89d3186d47ef5dbd0b65026ad37fb31b485fc44b6ec257dd77825428.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT created_by, CONCAT(coalesce(completed_job.logs, ''), coalesce(job_logs.logs, '')) as logs, job_logs.log_offset, job_logs.log_file_index\n FROM completed_job\n LEFT JOIN job_logs ON job_logs.job_id = completed_job.id\n WHERE completed_job.id = $1 AND completed_job.workspace_id = $2", + "query": "SELECT created_by, coalesce(job_logs.logs, '') as logs, job_logs.log_offset, job_logs.log_file_index\n FROM v2_as_completed_job\n LEFT JOIN job_logs ON job_logs.job_id = v2_as_completed_job.id\n WHERE v2_as_completed_job.id = $1 AND v2_as_completed_job.workspace_id = $2", "describe": { "columns": [ { @@ -31,11 +31,11 @@ ] }, "nullable": [ - false, + true, null, false, true ] }, - "hash": "f04fa0262091c5a4abf7dddafeec1b34ed806dda70592b0030067aed46a104d9" + "hash": "282afbff89d3186d47ef5dbd0b65026ad37fb31b485fc44b6ec257dd77825428" } diff --git a/backend/.sqlx/query-28b42ab9c3ce0c2f05cf385e81f3b72fa7c4b3c458d52a5891a61f9c53a49c6d.json b/backend/.sqlx/query-28b42ab9c3ce0c2f05cf385e81f3b72fa7c4b3c458d52a5891a61f9c53a49c6d.json new file mode 100644 index 0000000000..817b993d8f --- /dev/null +++ b/backend/.sqlx/query-28b42ab9c3ce0c2f05cf385e81f3b72fa7c4b3c458d52a5891a61f9c53a49c6d.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status SET\n flow_status = JSONB_SET(\n flow_status,\n ARRAY['modules', $1::TEXT, 'branchall', 'branch'],\n ((flow_status->'modules'->$1::int->'branchall'->>'branch')::int + 1)::text::jsonb\n )\n WHERE id = $2\n RETURNING (flow_status->'modules'->$1::int->'branchall'->>'branch')::int", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "int4", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Int4", + "Uuid" + ] + }, + "nullable": [ + null + ] + }, + "hash": "28b42ab9c3ce0c2f05cf385e81f3b72fa7c4b3c458d52a5891a61f9c53a49c6d" +} diff --git a/backend/.sqlx/query-bc01d5ca0138527b796a373f76404b8b6e4c6d711d0f6a557f929d831b8cfd3e.json b/backend/.sqlx/query-28c40cfbe796c6fb942cc9f72e00f9d3de633cdc4f9e23c6d9bd27eba07b0cb2.json similarity index 64% rename from backend/.sqlx/query-bc01d5ca0138527b796a373f76404b8b6e4c6d711d0f6a557f929d831b8cfd3e.json rename to backend/.sqlx/query-28c40cfbe796c6fb942cc9f72e00f9d3de633cdc4f9e23c6d9bd27eba07b0cb2.json index 6fcbd0a266..b39b9ef58a 100644 --- a/backend/.sqlx/query-bc01d5ca0138527b796a373f76404b8b6e4c6d711d0f6a557f929d831b8cfd3e.json +++ b/backend/.sqlx/query-28c40cfbe796c6fb942cc9f72e00f9d3de633cdc4f9e23c6d9bd27eba07b0cb2.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT null FROM queue WHERE id = $1 FOR UPDATE", + "query": "SELECT null FROM v2_job_queue WHERE id = $1 FOR UPDATE", "describe": { "columns": [ { @@ -18,5 +18,5 @@ null ] }, - "hash": "bc01d5ca0138527b796a373f76404b8b6e4c6d711d0f6a557f929d831b8cfd3e" + "hash": "28c40cfbe796c6fb942cc9f72e00f9d3de633cdc4f9e23c6d9bd27eba07b0cb2" } diff --git a/backend/.sqlx/query-0ad36c1598ff4ece0c325eaeb9a9177a87e1accd192402e21db5ae09c3498ab0.json b/backend/.sqlx/query-29624682d687790dd199c4af759132d79fdb2982de111cb5fd43e3d9ecd0f15e.json similarity index 65% rename from backend/.sqlx/query-0ad36c1598ff4ece0c325eaeb9a9177a87e1accd192402e21db5ae09c3498ab0.json rename to backend/.sqlx/query-29624682d687790dd199c4af759132d79fdb2982de111cb5fd43e3d9ecd0f15e.json index 2d06bac0c9..7623243f07 100644 --- a/backend/.sqlx/query-0ad36c1598ff4ece0c325eaeb9a9177a87e1accd192402e21db5ae09c3498ab0.json +++ b/backend/.sqlx/query-29624682d687790dd199c4af759132d79fdb2982de111cb5fd43e3d9ecd0f15e.json @@ -1,27 +1,21 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO queue\n (workspace_id, id, running, parent_job, created_by, permissioned_as, scheduled_for, \n script_hash, script_path, raw_code, raw_lock, args, job_kind, schedule_path, raw_flow, flow_status, is_flow_step, language, started_at, same_worker, pre_run_error, email, visible_to_owner, root_job, tag, concurrent_limit, concurrency_time_window_s, timeout, flow_step_id, cache_ttl, priority, last_ping)\n VALUES ($1, $2, $3, $4, $5, $6, COALESCE($7, now()), $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, CASE WHEN $3 THEN now() END, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, NULL) RETURNING id", + "query": "INSERT INTO v2_job (id, workspace_id, raw_code, raw_lock, raw_flow, tag, parent_job,\n created_by, permissioned_as, runnable_id, runnable_path, args, kind, trigger,\n script_lang, same_worker, pre_run_error, permissioned_as_email, visible_to_owner,\n flow_innermost_root_job, concurrent_limit, concurrency_time_window_s, timeout, flow_step_id,\n cache_ttl, priority, trigger_kind, script_entrypoint_override, preprocessed)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18,\n $19, $20, $21, $22, $23, $24, $25, $26,\n CASE WHEN $14::VARCHAR IS NOT NULL THEN 'schedule'::job_trigger_kind END,\n ($12::JSONB)->>'_ENTRYPOINT_OVERRIDE', $27)", "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Uuid" - } - ], + "columns": [], "parameters": { "Left": [ - "Varchar", "Uuid", - "Bool", + "Varchar", + "Text", + "Text", + "Jsonb", + "Varchar", "Uuid", "Varchar", "Varchar", - "Timestamptz", "Int8", "Varchar", - "Text", - "Text", "Jsonb", { "Custom": { @@ -51,9 +45,6 @@ } }, "Varchar", - "Jsonb", - "Jsonb", - "Bool", { "Custom": { "name": "script_lang", @@ -87,18 +78,16 @@ "Varchar", "Bool", "Uuid", - "Varchar", "Int4", "Int4", "Int4", "Varchar", "Int4", - "Int2" + "Int2", + "Bool" ] }, - "nullable": [ - false - ] + "nullable": [] }, - "hash": "0ad36c1598ff4ece0c325eaeb9a9177a87e1accd192402e21db5ae09c3498ab0" + "hash": "29624682d687790dd199c4af759132d79fdb2982de111cb5fd43e3d9ecd0f15e" } diff --git a/backend/.sqlx/query-2bfb918104568288bb57e64d1cf914c14cd493f96005017c299a805c48aef092.json b/backend/.sqlx/query-2bfb918104568288bb57e64d1cf914c14cd493f96005017c299a805c48aef092.json deleted file mode 100644 index 42d0fefc9f..0000000000 --- a/backend/.sqlx/query-2bfb918104568288bb57e64d1cf914c14cd493f96005017c299a805c48aef092.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT CASE WHEN pg_column_size(args) < 40000 OR $3 THEN args ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as args FROM completed_job WHERE id = $1 AND workspace_id = $2 UNION ALL SELECT CASE WHEN pg_column_size(args) < 40000 OR $3 THEN args ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as args FROM input WHERE id = $1 AND workspace_id = $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "args", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [ - "Uuid", - "Text", - "Bool" - ] - }, - "nullable": [ - null - ] - }, - "hash": "2bfb918104568288bb57e64d1cf914c14cd493f96005017c299a805c48aef092" -} diff --git a/backend/.sqlx/query-afd0d1b0511f32ba1981bc8d09b729639a6c0b468aa3f9108071d160d2dee250.json b/backend/.sqlx/query-2c5a1b1ebd872ce8a03a1b7e5246c876750c85075cfda816c025c805d4c3cd4c.json similarity index 51% rename from backend/.sqlx/query-afd0d1b0511f32ba1981bc8d09b729639a6c0b468aa3f9108071d160d2dee250.json rename to backend/.sqlx/query-2c5a1b1ebd872ce8a03a1b7e5246c876750c85075cfda816c025c805d4c3cd4c.json index 5ccb236cc6..6d916281eb 100644 --- a/backend/.sqlx/query-afd0d1b0511f32ba1981bc8d09b729639a6c0b468aa3f9108071d160d2dee250.json +++ b/backend/.sqlx/query-2c5a1b1ebd872ce8a03a1b7e5246c876750c85075cfda816c025c805d4c3cd4c.json @@ -1,11 +1,11 @@ { "db_name": "PostgreSQL", - "query": "SELECT created_by, CONCAT(coalesce(completed_job.logs, ''), coalesce(job_logs.logs, '')) as logs, job_logs.log_offset, job_logs.log_file_index\n FROM completed_job \n LEFT JOIN job_logs ON job_logs.job_id = completed_job.id \n WHERE completed_job.id = $1 AND completed_job.workspace_id = $2 AND ($3::text[] IS NULL OR completed_job.tag = ANY($3))", + "query": "SELECT created_by AS \"created_by!\", CONCAT(coalesce(v2_as_completed_job.logs, ''), coalesce(job_logs.logs, '')) as logs, job_logs.log_offset, job_logs.log_file_index\n FROM v2_as_completed_job \n LEFT JOIN job_logs ON job_logs.job_id = v2_as_completed_job.id \n WHERE v2_as_completed_job.id = $1 AND v2_as_completed_job.workspace_id = $2 AND ($3::text[] IS NULL OR v2_as_completed_job.tag = ANY($3))", "describe": { "columns": [ { "ordinal": 0, - "name": "created_by", + "name": "created_by!", "type_info": "Varchar" }, { @@ -32,11 +32,11 @@ ] }, "nullable": [ - false, + true, null, false, true ] }, - "hash": "afd0d1b0511f32ba1981bc8d09b729639a6c0b468aa3f9108071d160d2dee250" + "hash": "2c5a1b1ebd872ce8a03a1b7e5246c876750c85075cfda816c025c805d4c3cd4c" } diff --git a/backend/.sqlx/query-2e6935811a6d818bc523f076674f794f8be6c6bad3d06e74586e8ab668d91861.json b/backend/.sqlx/query-2e6935811a6d818bc523f076674f794f8be6c6bad3d06e74586e8ab668d91861.json new file mode 100644 index 0000000000..2ebfb49b8e --- /dev/null +++ b/backend/.sqlx/query-2e6935811a6d818bc523f076674f794f8be6c6bad3d06e74586e8ab668d91861.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO v2_job_status (id, workflow_as_code_status)\n VALUES ($1, JSONB_SET('{}'::JSONB, array[$2], $3))\n ON CONFLICT (id) DO UPDATE SET\n workflow_as_code_status = JSONB_SET(\n COALESCE(v2_job_status.workflow_as_code_status, '{}'::JSONB), \n array[$2],\n $3\n )", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "2e6935811a6d818bc523f076674f794f8be6c6bad3d06e74586e8ab668d91861" +} diff --git a/backend/.sqlx/query-2f27fd4d7aed67c987d9f0cca0a2426f3fbd22c1f0759be943625ad5c921f91b.json b/backend/.sqlx/query-2f27fd4d7aed67c987d9f0cca0a2426f3fbd22c1f0759be943625ad5c921f91b.json new file mode 100644 index 0000000000..eef5b530e7 --- /dev/null +++ b/backend/.sqlx/query-2f27fd4d7aed67c987d9f0cca0a2426f3fbd22c1f0759be943625ad5c921f91b.json @@ -0,0 +1,42 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n result AS \"result: sqlx::types::Json>\",\n result_columns,\n status = 'success' AS \"success!\",\n created_by AS \"created_by!\"\n FROM v2_job_completed c\n JOIN v2_job j USING (id)\n WHERE c.id = $1 AND c.workspace_id = $2 AND ($3::text[] IS NULL OR tag = ANY($3))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "result: sqlx::types::Json>", + "type_info": "Jsonb" + }, + { + "ordinal": 1, + "name": "result_columns", + "type_info": "TextArray" + }, + { + "ordinal": 2, + "name": "success!", + "type_info": "Bool" + }, + { + "ordinal": 3, + "name": "created_by!", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "TextArray" + ] + }, + "nullable": [ + true, + true, + null, + false + ] + }, + "hash": "2f27fd4d7aed67c987d9f0cca0a2426f3fbd22c1f0759be943625ad5c921f91b" +} diff --git a/backend/.sqlx/query-2f2ef9b1ccff527c48fa01cf1b78cd0e58c8d534ac22ec0356d82a854b31d087.json b/backend/.sqlx/query-2f2ef9b1ccff527c48fa01cf1b78cd0e58c8d534ac22ec0356d82a854b31d087.json new file mode 100644 index 0000000000..77e16162bd --- /dev/null +++ b/backend/.sqlx/query-2f2ef9b1ccff527c48fa01cf1b78cd0e58c8d534ac22ec0356d82a854b31d087.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_runtime SET ping = NULL\n WHERE id = $1 AND ping = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Timestamptz" + ] + }, + "nullable": [] + }, + "hash": "2f2ef9b1ccff527c48fa01cf1b78cd0e58c8d534ac22ec0356d82a854b31d087" +} diff --git a/backend/.sqlx/query-31df83e9eb6078e93ec5fe4168306caccb849db9e0f71d86da655b01c6a3e8d0.json b/backend/.sqlx/query-31df83e9eb6078e93ec5fe4168306caccb849db9e0f71d86da655b01c6a3e8d0.json new file mode 100644 index 0000000000..b96c05d674 --- /dev/null +++ b/backend/.sqlx/query-31df83e9eb6078e93ec5fe4168306caccb849db9e0f71d86da655b01c6a3e8d0.json @@ -0,0 +1,27 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO v2_job_queue\n (workspace_id, id, running, scheduled_for, started_at, tag, priority)\n VALUES ($1, $2, $3, COALESCE($4, now()), CASE WHEN $3 THEN now() END, $5, $6) RETURNING id AS \"id!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id!", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Uuid", + "Bool", + "Timestamptz", + "Varchar", + "Int2" + ] + }, + "nullable": [ + false + ] + }, + "hash": "31df83e9eb6078e93ec5fe4168306caccb849db9e0f71d86da655b01c6a3e8d0" +} diff --git a/backend/.sqlx/query-bc8ac03254669951654cda4bcfa12491341e745aef5e0e5090c2f4e4a4dc54fb.json b/backend/.sqlx/query-31f6f0712f1f53362628a487e8cd35d6d711b05d6eefffdc8e6a1b9c9bc7085d.json similarity index 80% rename from backend/.sqlx/query-bc8ac03254669951654cda4bcfa12491341e745aef5e0e5090c2f4e4a4dc54fb.json rename to backend/.sqlx/query-31f6f0712f1f53362628a487e8cd35d6d711b05d6eefffdc8e6a1b9c9bc7085d.json index fc27de30d0..cff7935a88 100644 --- a/backend/.sqlx/query-bc8ac03254669951654cda4bcfa12491341e745aef5e0e5090c2f4e4a4dc54fb.json +++ b/backend/.sqlx/query-31f6f0712f1f53362628a487e8cd35d6d711b05d6eefffdc8e6a1b9c9bc7085d.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT raw_code, raw_lock, raw_flow AS \"raw_flow: Json>\" FROM job WHERE id = $1 LIMIT 1", + "query": "SELECT raw_code, raw_lock, raw_flow AS \"raw_flow: Json>\" FROM v2_job WHERE id = $1 LIMIT 1", "describe": { "columns": [ { @@ -30,5 +30,5 @@ true ] }, - "hash": "bc8ac03254669951654cda4bcfa12491341e745aef5e0e5090c2f4e4a4dc54fb" + "hash": "31f6f0712f1f53362628a487e8cd35d6d711b05d6eefffdc8e6a1b9c9bc7085d" } diff --git a/backend/.sqlx/query-e58cf2e3deb9aa1e9f37313a33e5d44fcfdb40a4764e1dc34896bd85b3d007f9.json b/backend/.sqlx/query-33351de09c72ccc0a39eb977d26f867595813bfa1ae0b26bc4181780801294bf.json similarity index 53% rename from backend/.sqlx/query-e58cf2e3deb9aa1e9f37313a33e5d44fcfdb40a4764e1dc34896bd85b3d007f9.json rename to backend/.sqlx/query-33351de09c72ccc0a39eb977d26f867595813bfa1ae0b26bc4181780801294bf.json index ab1bd399ea..a3d39e0b59 100644 --- a/backend/.sqlx/query-e58cf2e3deb9aa1e9f37313a33e5d44fcfdb40a4764e1dc34896bd85b3d007f9.json +++ b/backend/.sqlx/query-33351de09c72ccc0a39eb977d26f867595813bfa1ae0b26bc4181780801294bf.json @@ -1,11 +1,11 @@ { "db_name": "PostgreSQL", - "query": "SELECT success FROM completed_job WHERE id = ANY($1)", + "query": "SELECT success AS \"success!\" FROM v2_as_completed_job WHERE id = ANY($1)", "describe": { "columns": [ { "ordinal": 0, - "name": "success", + "name": "success!", "type_info": "Bool" } ], @@ -15,8 +15,8 @@ ] }, "nullable": [ - false + true ] }, - "hash": "e58cf2e3deb9aa1e9f37313a33e5d44fcfdb40a4764e1dc34896bd85b3d007f9" + "hash": "33351de09c72ccc0a39eb977d26f867595813bfa1ae0b26bc4181780801294bf" } diff --git a/backend/.sqlx/query-364248db86a9228bba6ff522e811d68c5c902ee0a07ea69bff772ff10d0dc5aa.json b/backend/.sqlx/query-364248db86a9228bba6ff522e811d68c5c902ee0a07ea69bff772ff10d0dc5aa.json deleted file mode 100644 index 7ead72a26d..0000000000 --- a/backend/.sqlx/query-364248db86a9228bba6ff522e811d68c5c902ee0a07ea69bff772ff10d0dc5aa.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE queue SET flow_status = JSONB_SET(flow_status, ARRAY['user_states'], JSONB_SET(COALESCE(flow_status->'user_states', '{}'::jsonb), ARRAY[$1], $2))\n WHERE id = $3 AND workspace_id = $4 AND job_kind IN ('flow', 'flowpreview', 'flownode') RETURNING 1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Text", - "Jsonb", - "Uuid", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "364248db86a9228bba6ff522e811d68c5c902ee0a07ea69bff772ff10d0dc5aa" -} diff --git a/backend/.sqlx/query-36c4e57afcab22f4b6825ccebe47767b8a8fe0a638250f7c7777e5a9f7530e5c.json b/backend/.sqlx/query-36c4e57afcab22f4b6825ccebe47767b8a8fe0a638250f7c7777e5a9f7530e5c.json new file mode 100644 index 0000000000..e810fc4754 --- /dev/null +++ b/backend/.sqlx/query-36c4e57afcab22f4b6825ccebe47767b8a8fe0a638250f7c7777e5a9f7530e5c.json @@ -0,0 +1,31 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO v2_job_completed AS cj\n ( workspace_id\n , id\n , started_at\n , duration_ms\n , result\n , result_columns\n , canceled_by\n , canceled_reason\n , flow_status\n , workflow_as_code_status\n , memory_peak\n , status\n , worker\n )\n SELECT q.workspace_id, q.id, started_at, COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(started_at, now()))))*1000), $3, $10, $5, $6,\n flow_status, workflow_as_code_status,\n $8, CASE WHEN $4::BOOL THEN 'canceled'::job_status\n WHEN $7::BOOL THEN 'skipped'::job_status\n WHEN $2::BOOL THEN 'success'::job_status\n ELSE 'failure'::job_status END AS status,\n q.worker\n FROM v2_job_queue q LEFT JOIN v2_job_status USING (id) WHERE q.id = $1\n ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = $3 RETURNING duration_ms AS \"duration_ms!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "duration_ms!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Bool", + "Jsonb", + "Bool", + "Varchar", + "Text", + "Bool", + "Int4", + "Int8", + "TextArray" + ] + }, + "nullable": [ + false + ] + }, + "hash": "36c4e57afcab22f4b6825ccebe47767b8a8fe0a638250f7c7777e5a9f7530e5c" +} diff --git a/backend/.sqlx/query-38846b12201990f8e776b256ac419b24ffec46fa02d710544a3074745be9455f.json b/backend/.sqlx/query-38846b12201990f8e776b256ac419b24ffec46fa02d710544a3074745be9455f.json deleted file mode 100644 index f94429e4f9..0000000000 --- a/backend/.sqlx/query-38846b12201990f8e776b256ac419b24ffec46fa02d710544a3074745be9455f.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE queue SET suspend = 0 WHERE parent_job = $1 AND suspend = $2 AND (flow_status->'step')::int = 0", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid", - "Int4" - ] - }, - "nullable": [] - }, - "hash": "38846b12201990f8e776b256ac419b24ffec46fa02d710544a3074745be9455f" -} diff --git a/backend/.sqlx/query-3a5edf3dd884b5a8862bb112f6520967ed4a218782192c6c6fc1498f45d753a6.json b/backend/.sqlx/query-3a5edf3dd884b5a8862bb112f6520967ed4a218782192c6c6fc1498f45d753a6.json new file mode 100644 index 0000000000..26d0b63b5c --- /dev/null +++ b/backend/.sqlx/query-3a5edf3dd884b5a8862bb112f6520967ed4a218782192c6c6fc1498f45d753a6.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "create index concurrently if not exists ix_job_created_at ON v2_job (created_at DESC)", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "3a5edf3dd884b5a8862bb112f6520967ed4a218782192c6c6fc1498f45d753a6" +} diff --git a/backend/.sqlx/query-3b4b62161a5197f37850c8c4197ea026d8e94a3cb9cdcfa5fde19343acf81ecc.json b/backend/.sqlx/query-3b4b62161a5197f37850c8c4197ea026d8e94a3cb9cdcfa5fde19343acf81ecc.json deleted file mode 100644 index 367aae4a31..0000000000 --- a/backend/.sqlx/query-3b4b62161a5197f37850c8c4197ea026d8e94a3cb9cdcfa5fde19343acf81ecc.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE queue SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4) WHERE id = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Uuid", - "Text", - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "3b4b62161a5197f37850c8c4197ea026d8e94a3cb9cdcfa5fde19343acf81ecc" -} diff --git a/backend/.sqlx/query-3bacf9cd9aa63f4bec5f983f4a0c3030216b5a4ed669f77962509d1c2c6cb780.json b/backend/.sqlx/query-3bacf9cd9aa63f4bec5f983f4a0c3030216b5a4ed669f77962509d1c2c6cb780.json new file mode 100644 index 0000000000..200a5bf47f --- /dev/null +++ b/backend/.sqlx/query-3bacf9cd9aa63f4bec5f983f4a0c3030216b5a4ed669f77962509d1c2c6cb780.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "create index concurrently if not exists root_job_index_by_path_2 ON v2_job (workspace_id, runnable_path, created_at desc) WHERE parent_job IS NULL", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "3bacf9cd9aa63f4bec5f983f4a0c3030216b5a4ed669f77962509d1c2c6cb780" +} diff --git a/backend/.sqlx/query-3c0b2a840102b12864c5d721b8e0142602ab37f3e1a95d39b3c7cbd7ff34d0b2.json b/backend/.sqlx/query-3c0b2a840102b12864c5d721b8e0142602ab37f3e1a95d39b3c7cbd7ff34d0b2.json new file mode 100644 index 0000000000..dbb99a4ec1 --- /dev/null +++ b/backend/.sqlx/query-3c0b2a840102b12864c5d721b8e0142602ab37f3e1a95d39b3c7cbd7ff34d0b2.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n id As \"id!\",\n flow_status->'restarted_from'->'flow_job_id' AS \"restarted_from: Json\"\n FROM v2_as_queue\n WHERE COALESCE((SELECT flow_innermost_root_job FROM v2_job WHERE id = $1), $1) = id AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id!", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "restarted_from: Json", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + true, + null + ] + }, + "hash": "3c0b2a840102b12864c5d721b8e0142602ab37f3e1a95d39b3c7cbd7ff34d0b2" +} diff --git a/backend/.sqlx/query-3d0f036a3176dcc787bef3f10a336c93f43dec5df570f869580056702b7d1e09.json b/backend/.sqlx/query-3d0f036a3176dcc787bef3f10a336c93f43dec5df570f869580056702b7d1e09.json new file mode 100644 index 0000000000..4356a016c0 --- /dev/null +++ b/backend/.sqlx/query-3d0f036a3176dcc787bef3f10a336c93f43dec5df570f869580056702b7d1e09.json @@ -0,0 +1,118 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT workspace_id, path, url, script_path, is_flow, edited_by, email, edited_at, server_id, last_server_ping, extra_perms, error, enabled, filters as \"filters: _\", initial_messages as \"initial_messages: _\", url_runnable_args as \"url_runnable_args: _\", can_return_message FROM websocket_trigger\n WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "url", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "script_path", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "is_flow", + "type_info": "Bool" + }, + { + "ordinal": 5, + "name": "edited_by", + "type_info": "Varchar" + }, + { + "ordinal": 6, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "edited_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 8, + "name": "server_id", + "type_info": "Varchar" + }, + { + "ordinal": 9, + "name": "last_server_ping", + "type_info": "Timestamptz" + }, + { + "ordinal": 10, + "name": "extra_perms", + "type_info": "Jsonb" + }, + { + "ordinal": 11, + "name": "error", + "type_info": "Text" + }, + { + "ordinal": 12, + "name": "enabled", + "type_info": "Bool" + }, + { + "ordinal": 13, + "name": "filters: _", + "type_info": "JsonbArray" + }, + { + "ordinal": 14, + "name": "initial_messages: _", + "type_info": "JsonbArray" + }, + { + "ordinal": 15, + "name": "url_runnable_args: _", + "type_info": "Jsonb" + }, + { + "ordinal": 16, + "name": "can_return_message", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + true, + true, + false, + true, + false, + false, + true, + true, + false + ] + }, + "hash": "3d0f036a3176dcc787bef3f10a336c93f43dec5df570f869580056702b7d1e09" +} diff --git a/backend/.sqlx/query-3d58b5861c62f0b092b6b95c17ba1dac2cbcf00db116624bd2fe27a4d0dfb436.json b/backend/.sqlx/query-3d58b5861c62f0b092b6b95c17ba1dac2cbcf00db116624bd2fe27a4d0dfb436.json new file mode 100644 index 0000000000..4f1fe255f9 --- /dev/null +++ b/backend/.sqlx/query-3d58b5861c62f0b092b6b95c17ba1dac2cbcf00db116624bd2fe27a4d0dfb436.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status\n SET flow_status = JSONB_SET(flow_status, ARRAY['approval_conditions'], $1)\n WHERE id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "3d58b5861c62f0b092b6b95c17ba1dac2cbcf00db116624bd2fe27a4d0dfb436" +} diff --git a/backend/.sqlx/query-3da0cf7edc975cb365c36d167df34d9d30d3e86f231bb9d1d3cc3bad6a50dfff.json b/backend/.sqlx/query-3da0cf7edc975cb365c36d167df34d9d30d3e86f231bb9d1d3cc3bad6a50dfff.json deleted file mode 100644 index df93a90efc..0000000000 --- a/backend/.sqlx/query-3da0cf7edc975cb365c36d167df34d9d30d3e86f231bb9d1d3cc3bad6a50dfff.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT parent_job\n FROM queue\n WHERE id = $1 AND workspace_id = $2\n UNION ALL\n SELECT parent_job\n FROM completed_job\n WHERE id = $1 AND workspace_id = $2\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "parent_job", - "type_info": "Uuid" - } - ], - "parameters": { - "Left": [ - "Uuid", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "3da0cf7edc975cb365c36d167df34d9d30d3e86f231bb9d1d3cc3bad6a50dfff" -} diff --git a/backend/.sqlx/query-3de870d80a3617e7b31209d922164bf957b8bac3b5c04626ec750b18f705cbfd.json b/backend/.sqlx/query-3de870d80a3617e7b31209d922164bf957b8bac3b5c04626ec750b18f705cbfd.json new file mode 100644 index 0000000000..7b0eb81646 --- /dev/null +++ b/backend/.sqlx/query-3de870d80a3617e7b31209d922164bf957b8bac3b5c04626ec750b18f705cbfd.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "CREATE INDEX CONCURRENTLY ix_v2_job_labels ON v2_job\n USING GIN (labels)\n WHERE labels IS NOT NULL", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "3de870d80a3617e7b31209d922164bf957b8bac3b5c04626ec750b18f705cbfd" +} diff --git a/backend/.sqlx/query-3e0cdd500dffc3bd1d8374ca3cc8fd60ce778c2fece27637d9985d4650778653.json b/backend/.sqlx/query-3e0cdd500dffc3bd1d8374ca3cc8fd60ce778c2fece27637d9985d4650778653.json new file mode 100644 index 0000000000..9f5cb1d19d --- /dev/null +++ b/backend/.sqlx/query-3e0cdd500dffc3bd1d8374ca3cc8fd60ce778c2fece27637d9985d4650778653.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status\n SET flow_status = JSONB_SET(flow_status, ARRAY['cleanup_module', 'flow_jobs_to_clean'], COALESCE(flow_status->'cleanup_module'->'flow_jobs_to_clean', '[]'::jsonb) || $1)\n WHERE id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "3e0cdd500dffc3bd1d8374ca3cc8fd60ce778c2fece27637d9985d4650778653" +} diff --git a/backend/.sqlx/query-3e55d027327bd3c76810fbe22d3ccb1bbbf83c8cff69d8f5907d1417a2522e69.json b/backend/.sqlx/query-3e55d027327bd3c76810fbe22d3ccb1bbbf83c8cff69d8f5907d1417a2522e69.json new file mode 100644 index 0000000000..2603a95877 --- /dev/null +++ b/backend/.sqlx/query-3e55d027327bd3c76810fbe22d3ccb1bbbf83c8cff69d8f5907d1417a2522e69.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_queue SET running = false, started_at = null\n WHERE id = $1 AND canceled_by IS NULL", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "3e55d027327bd3c76810fbe22d3ccb1bbbf83c8cff69d8f5907d1417a2522e69" +} diff --git a/backend/.sqlx/query-404d54032aa7814370a8cecb7795246e9f289e647351adb24ced507ff8a4d835.json b/backend/.sqlx/query-404d54032aa7814370a8cecb7795246e9f289e647351adb24ced507ff8a4d835.json new file mode 100644 index 0000000000..bd082c24f3 --- /dev/null +++ b/backend/.sqlx/query-404d54032aa7814370a8cecb7795246e9f289e647351adb24ced507ff8a4d835.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status SET\n flow_status = jsonb_set(\n jsonb_set(flow_status, ARRAY['failure_module', 'job'], to_jsonb($1::UUID::TEXT)),\n ARRAY['failure_module', 'type'],\n to_jsonb('InProgress'::text)\n )\n WHERE id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "404d54032aa7814370a8cecb7795246e9f289e647351adb24ced507ff8a4d835" +} diff --git a/backend/.sqlx/query-41f68f4ce5bed783cf69e42da115e9ad2c9fcbd75f55817b2114c04207f66e4a.json b/backend/.sqlx/query-41f68f4ce5bed783cf69e42da115e9ad2c9fcbd75f55817b2114c04207f66e4a.json deleted file mode 100644 index 9397f52b83..0000000000 --- a/backend/.sqlx/query-41f68f4ce5bed783cf69e42da115e9ad2c9fcbd75f55817b2114c04207f66e4a.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO metrics (id, value)\n VALUES ($1, to_jsonb((SELECT EXTRACT(EPOCH FROM now() - scheduled_for)\n FROM queue WHERE tag = $2 AND running = false AND scheduled_for <= now() - ('3 seconds')::interval\n ORDER BY priority DESC NULLS LAST, scheduled_for LIMIT 1)))", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Text" - ] - }, - "nullable": [] - }, - "hash": "41f68f4ce5bed783cf69e42da115e9ad2c9fcbd75f55817b2114c04207f66e4a" -} diff --git a/backend/.sqlx/query-43fcdf5243e17bfbdcd21f09feee6e104b40f4b937914f56f01c299cddfc17e9.json b/backend/.sqlx/query-4244640e62fffb0f6978f8f7d78291b3294a6a7d1549d752f14acf5972552ba5.json similarity index 53% rename from backend/.sqlx/query-43fcdf5243e17bfbdcd21f09feee6e104b40f4b937914f56f01c299cddfc17e9.json rename to backend/.sqlx/query-4244640e62fffb0f6978f8f7d78291b3294a6a7d1549d752f14acf5972552ba5.json index edbade4821..841d549e20 100644 --- a/backend/.sqlx/query-43fcdf5243e17bfbdcd21f09feee6e104b40f4b937914f56f01c299cddfc17e9.json +++ b/backend/.sqlx/query-4244640e62fffb0f6978f8f7d78291b3294a6a7d1549d752f14acf5972552ba5.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "UPDATE job SET workspace_id = $1 WHERE workspace_id = $2", + "query": "UPDATE v2_job SET workspace_id = $1 WHERE workspace_id = $2", "describe": { "columns": [], "parameters": { @@ -11,5 +11,5 @@ }, "nullable": [] }, - "hash": "43fcdf5243e17bfbdcd21f09feee6e104b40f4b937914f56f01c299cddfc17e9" + "hash": "4244640e62fffb0f6978f8f7d78291b3294a6a7d1549d752f14acf5972552ba5" } diff --git a/backend/.sqlx/query-42543450a5c1b988b258fb1e0c00cd0a53a0395bc60e35817ff149c4f27a19fb.json b/backend/.sqlx/query-42543450a5c1b988b258fb1e0c00cd0a53a0395bc60e35817ff149c4f27a19fb.json deleted file mode 100644 index 89287ed48d..0000000000 --- a/backend/.sqlx/query-42543450a5c1b988b258fb1e0c00cd0a53a0395bc60e35817ff149c4f27a19fb.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE completed_job\n SET logs = '##DELETED##', args = '{}'::jsonb, result = '{}'::jsonb\n WHERE id = ANY($1)", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "UuidArray" - ] - }, - "nullable": [] - }, - "hash": "42543450a5c1b988b258fb1e0c00cd0a53a0395bc60e35817ff149c4f27a19fb" -} diff --git a/backend/.sqlx/query-429aef2c320a152b16fe20c1ab84aab41142897db108dc8cbc2c51abb2e30c7c.json b/backend/.sqlx/query-429aef2c320a152b16fe20c1ab84aab41142897db108dc8cbc2c51abb2e30c7c.json new file mode 100644 index 0000000000..c40a18e4c9 --- /dev/null +++ b/backend/.sqlx/query-429aef2c320a152b16fe20c1ab84aab41142897db108dc8cbc2c51abb2e30c7c.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status\n SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT], $2)\n WHERE id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Jsonb", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "429aef2c320a152b16fe20c1ab84aab41142897db108dc8cbc2c51abb2e30c7c" +} diff --git a/backend/.sqlx/query-4331bb1a3559f56c1ee91916b7f15251f0b32afa2445401170d931e0fe1febb8.json b/backend/.sqlx/query-4331bb1a3559f56c1ee91916b7f15251f0b32afa2445401170d931e0fe1febb8.json new file mode 100644 index 0000000000..d2648d4230 --- /dev/null +++ b/backend/.sqlx/query-4331bb1a3559f56c1ee91916b7f15251f0b32afa2445401170d931e0fe1febb8.json @@ -0,0 +1,119 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT workspace_id, path, route_path, route_path_key, script_path, is_flow, edited_by, edited_at, email, extra_perms, is_async, requires_auth, http_method as \"http_method: _\", static_asset_config as \"static_asset_config: _\", is_static_website FROM http_trigger\n WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "route_path", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "route_path_key", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "script_path", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "is_flow", + "type_info": "Bool" + }, + { + "ordinal": 6, + "name": "edited_by", + "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "edited_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 8, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 9, + "name": "extra_perms", + "type_info": "Jsonb" + }, + { + "ordinal": 10, + "name": "is_async", + "type_info": "Bool" + }, + { + "ordinal": 11, + "name": "requires_auth", + "type_info": "Bool" + }, + { + "ordinal": 12, + "name": "http_method: _", + "type_info": { + "Custom": { + "name": "http_method", + "kind": { + "Enum": [ + "get", + "post", + "put", + "delete", + "patch" + ] + } + } + } + }, + { + "ordinal": 13, + "name": "static_asset_config: _", + "type_info": "Jsonb" + }, + { + "ordinal": 14, + "name": "is_static_website", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + true, + false + ] + }, + "hash": "4331bb1a3559f56c1ee91916b7f15251f0b32afa2445401170d931e0fe1febb8" +} diff --git a/backend/.sqlx/query-43aa468aac174529a74e6108af55a383f8a20e98b8c502929f4dc5041a55e72f.json b/backend/.sqlx/query-43aa468aac174529a74e6108af55a383f8a20e98b8c502929f4dc5041a55e72f.json new file mode 100644 index 0000000000..e33b0dc0b8 --- /dev/null +++ b/backend/.sqlx/query-43aa468aac174529a74e6108af55a383f8a20e98b8c502929f4dc5041a55e72f.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS (\n SELECT 1 FROM v2_as_completed_job \n WHERE workspace_id = $2 \n AND (job_kind = 'appscript' OR job_kind = 'preview')\n AND created_by = 'anonymous' \n AND started_at > now() - interval '3 hours'\n AND script_path LIKE $3 || '/%' \n AND result @> ('{\"s3\":\"' || $1 || '\"}')::jsonb \n )", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "43aa468aac174529a74e6108af55a383f8a20e98b8c502929f4dc5041a55e72f" +} diff --git a/backend/.sqlx/query-0a6a89e6ab3037f02c3c4c84ee02138d5fded1e6360bb992046fe9711b5ea213.json b/backend/.sqlx/query-4535c8effd1bae49894d13293a37e1ee949cf9108239032cb3addbf350fb33de.json similarity index 73% rename from backend/.sqlx/query-0a6a89e6ab3037f02c3c4c84ee02138d5fded1e6360bb992046fe9711b5ea213.json rename to backend/.sqlx/query-4535c8effd1bae49894d13293a37e1ee949cf9108239032cb3addbf350fb33de.json index 460ce2bf8d..a500b04785 100644 --- a/backend/.sqlx/query-0a6a89e6ab3037f02c3c4c84ee02138d5fded1e6360bb992046fe9711b5ea213.json +++ b/backend/.sqlx/query-4535c8effd1bae49894d13293a37e1ee949cf9108239032cb3addbf350fb33de.json @@ -1,11 +1,11 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n job_kind AS \"job_kind: JobKind\",\n script_hash AS \"script_hash: ScriptHash\",\n flow_status AS \"flow_status!: Json>\",\n raw_flow AS \"raw_flow: Json>\"\n FROM queue WHERE id = $1 AND workspace_id = $2 LIMIT 1", + "query": "SELECT\n job_kind AS \"job_kind!: JobKind\",\n script_hash AS \"script_hash: ScriptHash\",\n flow_status AS \"flow_status!: Json>\",\n raw_flow AS \"raw_flow: Json>\"\n FROM v2_as_queue WHERE id = $1 AND workspace_id = $2 LIMIT 1", "describe": { "columns": [ { "ordinal": 0, - "name": "job_kind: JobKind", + "name": "job_kind!: JobKind", "type_info": { "Custom": { "name": "job_kind", @@ -57,11 +57,11 @@ ] }, "nullable": [ - false, + true, true, true, true ] }, - "hash": "0a6a89e6ab3037f02c3c4c84ee02138d5fded1e6360bb992046fe9711b5ea213" + "hash": "4535c8effd1bae49894d13293a37e1ee949cf9108239032cb3addbf350fb33de" } diff --git a/backend/.sqlx/query-45d616c92ebcbe30a563e1fa7d2d0e53392e238144b039cfe042587d7fe1dea3.json b/backend/.sqlx/query-45d616c92ebcbe30a563e1fa7d2d0e53392e238144b039cfe042587d7fe1dea3.json new file mode 100644 index 0000000000..052238874a --- /dev/null +++ b/backend/.sqlx/query-45d616c92ebcbe30a563e1fa7d2d0e53392e238144b039cfe042587d7fe1dea3.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status SET\n workflow_as_code_status = jsonb_set(\n jsonb_set(\n COALESCE(workflow_as_code_status, '{}'::jsonb),\n array[$1],\n COALESCE(workflow_as_code_status->$1, '{}'::jsonb)\n ),\n array[$1, 'started_at'],\n to_jsonb(now()::text)\n )\n WHERE id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "45d616c92ebcbe30a563e1fa7d2d0e53392e238144b039cfe042587d7fe1dea3" +} diff --git a/backend/.sqlx/query-46046b1435d1f01e6f7cafe054367d88ddbdfe81237b72d5efbc630278e44bb5.json b/backend/.sqlx/query-46046b1435d1f01e6f7cafe054367d88ddbdfe81237b72d5efbc630278e44bb5.json new file mode 100644 index 0000000000..d56b85cd62 --- /dev/null +++ b/backend/.sqlx/query-46046b1435d1f01e6f7cafe054367d88ddbdfe81237b72d5efbc630278e44bb5.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_runtime r SET\n memory_peak = $1,\n ping = now()\n FROM v2_job_queue q\n WHERE r.id = $2 AND q.id = r.id\n RETURNING canceled_by IS NOT NULL AS \"canceled!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "canceled!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Int4", + "Uuid" + ] + }, + "nullable": [ + null + ] + }, + "hash": "46046b1435d1f01e6f7cafe054367d88ddbdfe81237b72d5efbc630278e44bb5" +} diff --git a/backend/.sqlx/query-464c51a8ea8c06232d33c45b8e59008b382c445476117a2311030734bc6d6d53.json b/backend/.sqlx/query-464c51a8ea8c06232d33c45b8e59008b382c445476117a2311030734bc6d6d53.json new file mode 100644 index 0000000000..df664b5b8b --- /dev/null +++ b/backend/.sqlx/query-464c51a8ea8c06232d33c45b8e59008b382c445476117a2311030734bc6d6d53.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_runtime SET ping = now() WHERE id = $1 AND ping < now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "464c51a8ea8c06232d33c45b8e59008b382c445476117a2311030734bc6d6d53" +} diff --git a/backend/.sqlx/query-47455b0ebaf999ab58b2cba3d74cb4bdd64075939a4aa2c117e18372511ea7e0.json b/backend/.sqlx/query-47455b0ebaf999ab58b2cba3d74cb4bdd64075939a4aa2c117e18372511ea7e0.json new file mode 100644 index 0000000000..6ccaeece1c --- /dev/null +++ b/backend/.sqlx/query-47455b0ebaf999ab58b2cba3d74cb4bdd64075939a4aa2c117e18372511ea7e0.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "VACUUM (skip_locked) v2_job_queue, v2_job_runtime, v2_job_status", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "47455b0ebaf999ab58b2cba3d74cb4bdd64075939a4aa2c117e18372511ea7e0" +} diff --git a/backend/.sqlx/query-47e6b25cc092ec8718a6581c76aca10b275653e10ea4aa17a8ef5091ca09294a.json b/backend/.sqlx/query-47e6b25cc092ec8718a6581c76aca10b275653e10ea4aa17a8ef5091ca09294a.json new file mode 100644 index 0000000000..3206f22bc5 --- /dev/null +++ b/backend/.sqlx/query-47e6b25cc092ec8718a6581c76aca10b275653e10ea4aa17a8ef5091ca09294a.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT result, id\n FROM v2_job_completed\n WHERE id = ANY($1) AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "result", + "type_info": "Jsonb" + }, + { + "ordinal": 1, + "name": "id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "UuidArray", + "Text" + ] + }, + "nullable": [ + true, + false + ] + }, + "hash": "47e6b25cc092ec8718a6581c76aca10b275653e10ea4aa17a8ef5091ca09294a" +} diff --git a/backend/.sqlx/query-47fb4bceddd36fa60ccbe84f6341436fb133f0b3c97abb1267a9c12326dd1a33.json b/backend/.sqlx/query-47fb4bceddd36fa60ccbe84f6341436fb133f0b3c97abb1267a9c12326dd1a33.json new file mode 100644 index 0000000000..9e1d80c2f6 --- /dev/null +++ b/backend/.sqlx/query-47fb4bceddd36fa60ccbe84f6341436fb133f0b3c97abb1267a9c12326dd1a33.json @@ -0,0 +1,30 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT created_by AS \"created_by!\", args as \"args: sqlx::types::Json>\"\n FROM v2_as_completed_job \n WHERE id = $1 AND workspace_id = $2 AND ($3::text[] IS NULL OR tag = ANY($3))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "created_by!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "args: sqlx::types::Json>", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "TextArray" + ] + }, + "nullable": [ + true, + true + ] + }, + "hash": "47fb4bceddd36fa60ccbe84f6341436fb133f0b3c97abb1267a9c12326dd1a33" +} diff --git a/backend/.sqlx/query-337f31c2172194cd594042c561998a03f751b246c40daf056fced0fd91f6dd73.json b/backend/.sqlx/query-488dd591096b2b47787afdc3a1d73917ed13269f2ee20b86df79fca2c8efe672.json similarity index 69% rename from backend/.sqlx/query-337f31c2172194cd594042c561998a03f751b246c40daf056fced0fd91f6dd73.json rename to backend/.sqlx/query-488dd591096b2b47787afdc3a1d73917ed13269f2ee20b86df79fca2c8efe672.json index 075a898e29..8622eb9125 100644 --- a/backend/.sqlx/query-337f31c2172194cd594042c561998a03f751b246c40daf056fced0fd91f6dd73.json +++ b/backend/.sqlx/query-488dd591096b2b47787afdc3a1d73917ed13269f2ee20b86df79fca2c8efe672.json @@ -1,16 +1,21 @@ { "db_name": "PostgreSQL", - "query": "WITH uuid_table as (\n select unnest($11::uuid[]) as uuid\n )\n INSERT INTO queue \n (id, script_hash, script_path, job_kind, language, args, tag, created_by, permissioned_as, email, scheduled_for, workspace_id, concurrent_limit, concurrency_time_window_s, timeout, flow_status)\n (SELECT uuid, $1, $2, $3, $4, ('{ \"uuid\": \"' || uuid || '\" }')::jsonb, $5, $6, $7, $8, $9, $10, $12, $13, $14, $15 FROM uuid_table) \n RETURNING id", + "query": "WITH uuid_table as (\n select gen_random_uuid() as uuid from generate_series(1, $16)\n )\n INSERT INTO v2_job\n (id, workspace_id, raw_code, raw_lock, raw_flow, tag, runnable_id, runnable_path, kind,\n script_lang, created_by, permissioned_as, permissioned_as_email, concurrent_limit,\n concurrency_time_window_s, timeout, args)\n (SELECT uuid, $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15,\n ('{ \"uuid\": \"' || uuid || '\" }')::jsonb FROM uuid_table)\n RETURNING id AS \"id!\"", "describe": { "columns": [ { "ordinal": 0, - "name": "id", + "name": "id!", "type_info": "Uuid" } ], "parameters": { "Left": [ + "Varchar", + "Text", + "Text", + "Jsonb", + "Varchar", "Int8", "Varchar", { @@ -71,19 +76,15 @@ "Varchar", "Varchar", "Varchar", - "Varchar", - "Timestamptz", - "Varchar", - "UuidArray", "Int4", "Int4", "Int4", - "Jsonb" + "Int4" ] }, "nullable": [ false ] }, - "hash": "337f31c2172194cd594042c561998a03f751b246c40daf056fced0fd91f6dd73" + "hash": "488dd591096b2b47787afdc3a1d73917ed13269f2ee20b86df79fca2c8efe672" } diff --git a/backend/.sqlx/query-4914b74a7956ab41e1a1c812460cc84bbd5e4045acc529f61de892a4e3aceef9.json b/backend/.sqlx/query-4914b74a7956ab41e1a1c812460cc84bbd5e4045acc529f61de892a4e3aceef9.json new file mode 100644 index 0000000000..26af54b1fa --- /dev/null +++ b/backend/.sqlx/query-4914b74a7956ab41e1a1c812460cc84bbd5e4045acc529f61de892a4e3aceef9.json @@ -0,0 +1,35 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT username, is_admin, operator FROM usr WHERE\n email = $1 AND workspace_id = $2 AND disabled = false", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "username", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "is_admin", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "operator", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "4914b74a7956ab41e1a1c812460cc84bbd5e4045acc529f61de892a4e3aceef9" +} diff --git a/backend/.sqlx/query-4b923c94f6adcc7a76e8073de5e46b116dba3211487c8408ce2777aafdf94a44.json b/backend/.sqlx/query-4b923c94f6adcc7a76e8073de5e46b116dba3211487c8408ce2777aafdf94a44.json deleted file mode 100644 index 9d14bc62cc..0000000000 --- a/backend/.sqlx/query-4b923c94f6adcc7a76e8073de5e46b116dba3211487c8408ce2777aafdf94a44.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO job (id, workspace_id, raw_code, raw_lock, raw_flow, tag)\n VALUES ($1, $2, $3, $4, $5, $6)", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid", - "Varchar", - "Text", - "Text", - "Jsonb", - "Varchar" - ] - }, - "nullable": [] - }, - "hash": "4b923c94f6adcc7a76e8073de5e46b116dba3211487c8408ce2777aafdf94a44" -} diff --git a/backend/.sqlx/query-4c5502cdf49a59ce6d8bdce90e1cba3a2bc575dc69f6aa01fc6df1b5d095ab41.json b/backend/.sqlx/query-4c5502cdf49a59ce6d8bdce90e1cba3a2bc575dc69f6aa01fc6df1b5d095ab41.json new file mode 100644 index 0000000000..6651f3d2d9 --- /dev/null +++ b/backend/.sqlx/query-4c5502cdf49a59ce6d8bdce90e1cba3a2bc575dc69f6aa01fc6df1b5d095ab41.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_completed SET result = '{}'::jsonb WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "4c5502cdf49a59ce6d8bdce90e1cba3a2bc575dc69f6aa01fc6df1b5d095ab41" +} diff --git a/backend/.sqlx/query-4c97fcc93b31c4b3262419d6ee183773a95de3b6694a398cb80d288bef4f130f.json b/backend/.sqlx/query-4c97fcc93b31c4b3262419d6ee183773a95de3b6694a398cb80d288bef4f130f.json deleted file mode 100644 index 53a59847e7..0000000000 --- a/backend/.sqlx/query-4c97fcc93b31c4b3262419d6ee183773a95de3b6694a398cb80d288bef4f130f.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE completed_job SET args = '{\"reason\":\"PREPROCESSOR_ARGS_ARE_DISCARDED\"}'::jsonb WHERE id = $1", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "4c97fcc93b31c4b3262419d6ee183773a95de3b6694a398cb80d288bef4f130f" -} diff --git a/backend/.sqlx/query-4c9cf8c3176abc2b8b9a1c3f671949e16830671a458d5a73762cd8545d26172d.json b/backend/.sqlx/query-4c9cf8c3176abc2b8b9a1c3f671949e16830671a458d5a73762cd8545d26172d.json new file mode 100644 index 0000000000..0d108538dc --- /dev/null +++ b/backend/.sqlx/query-4c9cf8c3176abc2b8b9a1c3f671949e16830671a458d5a73762cd8545d26172d.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status SET\n flow_status = JSONB_SET(\n JSONB_SET(flow_status, ARRAY['modules', $1::TEXT], $2),\n ARRAY['step'],\n $3\n )\n WHERE id = $4", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Jsonb", + "Jsonb", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "4c9cf8c3176abc2b8b9a1c3f671949e16830671a458d5a73762cd8545d26172d" +} diff --git a/backend/.sqlx/query-4d4cc1ed4857edb473dc9397ca90de3797d671a7c0395d16eec8a2f84c1fe3d1.json b/backend/.sqlx/query-4d4cc1ed4857edb473dc9397ca90de3797d671a7c0395d16eec8a2f84c1fe3d1.json new file mode 100644 index 0000000000..1b237fbb7b --- /dev/null +++ b/backend/.sqlx/query-4d4cc1ed4857edb473dc9397ca90de3797d671a7c0395d16eec8a2f84c1fe3d1.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job SET args = '{}'::jsonb WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "4d4cc1ed4857edb473dc9397ca90de3797d671a7c0395d16eec8a2f84c1fe3d1" +} diff --git a/backend/.sqlx/query-bf04a017bc55f6e1f5f25f697b3c4828611591c76dd32541735095c010d6cdf2.json b/backend/.sqlx/query-4eb44e8ea9313646749bb73dfa6237081246a2985eec1d0d933dfedf634a7191.json similarity index 74% rename from backend/.sqlx/query-bf04a017bc55f6e1f5f25f697b3c4828611591c76dd32541735095c010d6cdf2.json rename to backend/.sqlx/query-4eb44e8ea9313646749bb73dfa6237081246a2985eec1d0d933dfedf634a7191.json index aad7404583..ffb6d3eee2 100644 --- a/backend/.sqlx/query-bf04a017bc55f6e1f5f25f697b3c4828611591c76dd32541735095c010d6cdf2.json +++ b/backend/.sqlx/query-4eb44e8ea9313646749bb73dfa6237081246a2985eec1d0d933dfedf634a7191.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT coalesce(COUNT(*), 0) as \"database_length!\", null::bigint as suspended FROM completed_job WHERE workspace_id = $1", + "query": "SELECT coalesce(COUNT(*), 0) as \"database_length!\", null::bigint as suspended FROM v2_job_completed WHERE workspace_id = $1", "describe": { "columns": [ { @@ -24,5 +24,5 @@ null ] }, - "hash": "bf04a017bc55f6e1f5f25f697b3c4828611591c76dd32541735095c010d6cdf2" + "hash": "4eb44e8ea9313646749bb73dfa6237081246a2985eec1d0d933dfedf634a7191" } diff --git a/backend/.sqlx/query-1610c79b8d238e3a35d795f34a2a0100b933b7f199d41cc2f554d57c811d55f3.json b/backend/.sqlx/query-4edf05cbf35325d444de0e74ee070aafd27ef61c940daba186e7e66f668c31ed.json similarity index 56% rename from backend/.sqlx/query-1610c79b8d238e3a35d795f34a2a0100b933b7f199d41cc2f554d57c811d55f3.json rename to backend/.sqlx/query-4edf05cbf35325d444de0e74ee070aafd27ef61c940daba186e7e66f668c31ed.json index 9b94af8432..4eae8c22fb 100644 --- a/backend/.sqlx/query-1610c79b8d238e3a35d795f34a2a0100b933b7f199d41cc2f554d57c811d55f3.json +++ b/backend/.sqlx/query-4edf05cbf35325d444de0e74ee070aafd27ef61c940daba186e7e66f668c31ed.json @@ -1,16 +1,17 @@ { "db_name": "PostgreSQL", - "query": "UPDATE workspace_settings SET ai_resource = $1, code_completion_enabled = $2 WHERE workspace_id = $3", + "query": "UPDATE workspace_settings SET ai_resource = $1, code_completion_model = $2, ai_models = $3 WHERE workspace_id = $4", "describe": { "columns": [], "parameters": { "Left": [ "Jsonb", - "Bool", + "Varchar", + "VarcharArray", "Text" ] }, "nullable": [] }, - "hash": "1610c79b8d238e3a35d795f34a2a0100b933b7f199d41cc2f554d57c811d55f3" + "hash": "4edf05cbf35325d444de0e74ee070aafd27ef61c940daba186e7e66f668c31ed" } diff --git a/backend/.sqlx/query-519f4f76649947f036a2129c11e92ef0ad30e39eec59c27bae8cb0622062c8fb.json b/backend/.sqlx/query-519f4f76649947f036a2129c11e92ef0ad30e39eec59c27bae8cb0622062c8fb.json new file mode 100644 index 0000000000..b2c0b4702c --- /dev/null +++ b/backend/.sqlx/query-519f4f76649947f036a2129c11e92ef0ad30e39eec59c27bae8cb0622062c8fb.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*) FROM v2_as_queue WHERE canceled = false AND (scheduled_for <= now()\n OR (suspend_until IS NOT NULL\n AND ( suspend <= 0\n OR suspend_until <= now())))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "519f4f76649947f036a2129c11e92ef0ad30e39eec59c27bae8cb0622062c8fb" +} diff --git a/backend/.sqlx/query-51a19a7d611807c8b0957babf821e58e9aa1ce345f1cd5965298fda6f9e158a9.json b/backend/.sqlx/query-51a19a7d611807c8b0957babf821e58e9aa1ce345f1cd5965298fda6f9e158a9.json new file mode 100644 index 0000000000..c50b3531b0 --- /dev/null +++ b/backend/.sqlx/query-51a19a7d611807c8b0957babf821e58e9aa1ce345f1cd5965298fda6f9e158a9.json @@ -0,0 +1,25 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO v2_job_completed AS cj\n ( workspace_id\n , id\n , duration_ms\n , result\n , canceled_by\n , canceled_reason\n , flow_status\n , status\n , worker\n )\n SELECT q.workspace_id\n , q.id\n , 0\n , $4\n , $1\n , 'cancel all'\n , (SELECT flow_status FROM v2_job_status WHERE id = q.id)\n , 'canceled'::job_status\n , worker\n FROM v2_job_queue q\n JOIN v2_job USING (id)\n WHERE q.id = any($2) AND running = false AND parent_job IS NULL AND q.workspace_id = $3 AND trigger IS NULL\n FOR UPDATE SKIP LOCKED\n ON CONFLICT (id) DO NOTHING RETURNING id AS \"id!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id!", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Varchar", + "UuidArray", + "Text", + "Jsonb" + ] + }, + "nullable": [ + false + ] + }, + "hash": "51a19a7d611807c8b0957babf821e58e9aa1ce345f1cd5965298fda6f9e158a9" +} diff --git a/backend/.sqlx/query-53ff0e14c35a3e84585a699e55093546db5c5f0ad0c5f92f34aaf2cdd125d130.json b/backend/.sqlx/query-53ff0e14c35a3e84585a699e55093546db5c5f0ad0c5f92f34aaf2cdd125d130.json deleted file mode 100644 index ba39c47062..0000000000 --- a/backend/.sqlx/query-53ff0e14c35a3e84585a699e55093546db5c5f0ad0c5f92f34aaf2cdd125d130.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE queue\n SET suspend = $1, suspend_until = now() + interval '14 day', running = true\n WHERE id = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int4", - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "53ff0e14c35a3e84585a699e55093546db5c5f0ad0c5f92f34aaf2cdd125d130" -} diff --git a/backend/.sqlx/query-5401c521b5e63b7d9e7bc51c19d116599f6bcedbe70f3bf346b482fe79501958.json b/backend/.sqlx/query-5401c521b5e63b7d9e7bc51c19d116599f6bcedbe70f3bf346b482fe79501958.json new file mode 100644 index 0000000000..9a1b65ff52 --- /dev/null +++ b/backend/.sqlx/query-5401c521b5e63b7d9e7bc51c19d116599f6bcedbe70f3bf346b482fe79501958.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "create index concurrently if not exists ix_job_workspace_id_created_at_new_8 ON v2_job (workspace_id, created_at DESC) where kind in ('deploymentcallback') AND parent_job IS NULL", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "5401c521b5e63b7d9e7bc51c19d116599f6bcedbe70f3bf346b482fe79501958" +} diff --git a/backend/.sqlx/query-55002cccf17e32af5b077a17707122cfeeeebf5a9392c6798a486274d8d233d5.json b/backend/.sqlx/query-55002cccf17e32af5b077a17707122cfeeeebf5a9392c6798a486274d8d233d5.json new file mode 100644 index 0000000000..f9d007d4c1 --- /dev/null +++ b/backend/.sqlx/query-55002cccf17e32af5b077a17707122cfeeeebf5a9392c6798a486274d8d233d5.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT CASE WHEN pg_column_size(args) < 40000 OR $3 THEN args ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as args FROM v2_job WHERE id = $1 AND workspace_id = $2\n UNION ALL\n SELECT CASE WHEN pg_column_size(args) < 40000 OR $3 THEN args ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as args FROM input WHERE id = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "args", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Bool" + ] + }, + "nullable": [ + null + ] + }, + "hash": "55002cccf17e32af5b077a17707122cfeeeebf5a9392c6798a486274d8d233d5" +} diff --git a/backend/.sqlx/query-55cb03040bc2a8c53dd7fbb42bbdcc40f463cbc52d94ed9315cf9a547d4c89f2.json b/backend/.sqlx/query-55cb03040bc2a8c53dd7fbb42bbdcc40f463cbc52d94ed9315cf9a547d4c89f2.json index baf7cb42f5..920176991b 100644 --- a/backend/.sqlx/query-55cb03040bc2a8c53dd7fbb42bbdcc40f463cbc52d94ed9315cf9a547d4c89f2.json +++ b/backend/.sqlx/query-55cb03040bc2a8c53dd7fbb42bbdcc40f463cbc52d94ed9315cf9a547d4c89f2.json @@ -70,68 +70,73 @@ }, { "ordinal": 13, - "name": "code_completion_enabled", - "type_info": "Bool" - }, - { - "ordinal": 14, "name": "error_handler_extra_args", "type_info": "Json" }, { - "ordinal": 15, + "ordinal": 14, "name": "error_handler_muted_on_cancel", "type_info": "Bool" }, { - "ordinal": 16, + "ordinal": 15, "name": "large_file_storage", "type_info": "Jsonb" }, { - "ordinal": 17, + "ordinal": 16, "name": "git_sync", "type_info": "Jsonb" }, { - "ordinal": 18, + "ordinal": 17, "name": "default_app", "type_info": "Varchar" }, { - "ordinal": 19, + "ordinal": 18, "name": "auto_add", "type_info": "Bool" }, { - "ordinal": 20, + "ordinal": 19, "name": "automatic_billing", "type_info": "Bool" }, { - "ordinal": 21, + "ordinal": 20, "name": "default_scripts", "type_info": "Jsonb" }, { - "ordinal": 22, + "ordinal": 21, "name": "deploy_ui", "type_info": "Jsonb" }, { - "ordinal": 23, + "ordinal": 22, "name": "mute_critical_alerts", "type_info": "Bool" }, { - "ordinal": 24, + "ordinal": 23, "name": "color", "type_info": "Varchar" }, { - "ordinal": 25, + "ordinal": 24, "name": "operator_settings", "type_info": "Jsonb" + }, + { + "ordinal": 25, + "name": "ai_models", + "type_info": "VarcharArray" + }, + { + "ordinal": 26, + "name": "code_completion_model", + "type_info": "Varchar" } ], "parameters": { @@ -153,7 +158,6 @@ true, true, true, - false, true, false, true, @@ -165,6 +169,8 @@ true, true, true, + true, + false, true ] }, diff --git a/backend/.sqlx/query-56eafd6d3c72f7114e3d6764184825e2671283fc70abba9a036e88699430af0f.json b/backend/.sqlx/query-56eafd6d3c72f7114e3d6764184825e2671283fc70abba9a036e88699430af0f.json deleted file mode 100644 index 9f22b4df19..0000000000 --- a/backend/.sqlx/query-56eafd6d3c72f7114e3d6764184825e2671283fc70abba9a036e88699430af0f.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "CREATE INDEX CONCURRENTLY labeled_jobs_on_jobs ON completed_job USING GIN ((result -> 'wm_labels')) WHERE result ? 'wm_labels'", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "56eafd6d3c72f7114e3d6764184825e2671283fc70abba9a036e88699430af0f" -} diff --git a/backend/.sqlx/query-56f7325e3b0316866714e76d94b50d9d258c288883b2b5b0ab286f5cb50850b5.json b/backend/.sqlx/query-56f7325e3b0316866714e76d94b50d9d258c288883b2b5b0ab286f5cb50850b5.json new file mode 100644 index 0000000000..6b47103c3a --- /dev/null +++ b/backend/.sqlx/query-56f7325e3b0316866714e76d94b50d9d258c288883b2b5b0ab286f5cb50850b5.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status SET\n workflow_as_code_status = jsonb_set(\n jsonb_set(\n COALESCE(workflow_as_code_status, '{}'::jsonb),\n array[$1],\n COALESCE(workflow_as_code_status->$1, '{}'::jsonb)\n ),\n array[$1, 'duration_ms'],\n to_jsonb($2::bigint)\n )\n WHERE id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Int8", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "56f7325e3b0316866714e76d94b50d9d258c288883b2b5b0ab286f5cb50850b5" +} diff --git a/backend/.sqlx/query-9f16a61d6a9a42f3fd3e30a1e7776503cee1b45eba150c3082eb246ea3f98d47.json b/backend/.sqlx/query-572492b357bc4e2576d77ed135682f4623c74e61759dda257bd1f633f1a2b725.json similarity index 62% rename from backend/.sqlx/query-9f16a61d6a9a42f3fd3e30a1e7776503cee1b45eba150c3082eb246ea3f98d47.json rename to backend/.sqlx/query-572492b357bc4e2576d77ed135682f4623c74e61759dda257bd1f633f1a2b725.json index aa64996dc8..d825f28c4d 100644 --- a/backend/.sqlx/query-9f16a61d6a9a42f3fd3e30a1e7776503cee1b45eba150c3082eb246ea3f98d47.json +++ b/backend/.sqlx/query-572492b357bc4e2576d77ed135682f4623c74e61759dda257bd1f633f1a2b725.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT EXISTS(SELECT 1 FROM completed_job WHERE id = $1 AND workspace_id = $2)", + "query": "SELECT EXISTS(SELECT 1 FROM v2_job_completed WHERE id = $1 AND workspace_id = $2)", "describe": { "columns": [ { @@ -19,5 +19,5 @@ null ] }, - "hash": "9f16a61d6a9a42f3fd3e30a1e7776503cee1b45eba150c3082eb246ea3f98d47" + "hash": "572492b357bc4e2576d77ed135682f4623c74e61759dda257bd1f633f1a2b725" } diff --git a/backend/.sqlx/query-a405e637f5f3b3203de6d65dfcb0ba1be406ee5167f7b8aa90213ef52c97441f.json b/backend/.sqlx/query-58c0eb36b630d5eba9d12edca672ebd56e13193395701e26328a243055bee6b8.json similarity index 55% rename from backend/.sqlx/query-a405e637f5f3b3203de6d65dfcb0ba1be406ee5167f7b8aa90213ef52c97441f.json rename to backend/.sqlx/query-58c0eb36b630d5eba9d12edca672ebd56e13193395701e26328a243055bee6b8.json index 21dacfcc9b..b3a4714404 100644 --- a/backend/.sqlx/query-a405e637f5f3b3203de6d65dfcb0ba1be406ee5167f7b8aa90213ef52c97441f.json +++ b/backend/.sqlx/query-58c0eb36b630d5eba9d12edca672ebd56e13193395701e26328a243055bee6b8.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT usage.usage + 1 FROM usage \n WHERE is_workspace IS TRUE AND\n month_ = EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date)\n AND id = $1", + "query": "SELECT usage.usage + 1 FROM usage \n WHERE is_workspace IS TRUE AND\n month_ = EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date)\n AND id = $1", "describe": { "columns": [ { @@ -18,5 +18,5 @@ null ] }, - "hash": "a405e637f5f3b3203de6d65dfcb0ba1be406ee5167f7b8aa90213ef52c97441f" + "hash": "58c0eb36b630d5eba9d12edca672ebd56e13193395701e26328a243055bee6b8" } diff --git a/backend/.sqlx/query-59368ac2d4e0918c7ee4275a0b1e3935e9d57be68a1811ffdee904a5f56e7023.json b/backend/.sqlx/query-59368ac2d4e0918c7ee4275a0b1e3935e9d57be68a1811ffdee904a5f56e7023.json new file mode 100644 index 0000000000..d1e0d20596 --- /dev/null +++ b/backend/.sqlx/query-59368ac2d4e0918c7ee4275a0b1e3935e9d57be68a1811ffdee904a5f56e7023.json @@ -0,0 +1,118 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT * FROM nats_trigger\n WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "nats_resource_path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "subjects", + "type_info": "VarcharArray" + }, + { + "ordinal": 3, + "name": "stream_name", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "consumer_name", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "use_jetstream", + "type_info": "Bool" + }, + { + "ordinal": 6, + "name": "script_path", + "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "is_flow", + "type_info": "Bool" + }, + { + "ordinal": 8, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 9, + "name": "edited_by", + "type_info": "Varchar" + }, + { + "ordinal": 10, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 11, + "name": "edited_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 12, + "name": "extra_perms", + "type_info": "Jsonb" + }, + { + "ordinal": 13, + "name": "server_id", + "type_info": "Varchar" + }, + { + "ordinal": 14, + "name": "last_server_ping", + "type_info": "Timestamptz" + }, + { + "ordinal": 15, + "name": "error", + "type_info": "Text" + }, + { + "ordinal": 16, + "name": "enabled", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + false, + true, + true, + false, + false, + false, + false, + false, + false, + false, + false, + true, + true, + true, + false + ] + }, + "hash": "59368ac2d4e0918c7ee4275a0b1e3935e9d57be68a1811ffdee904a5f56e7023" +} diff --git a/backend/.sqlx/query-597b148ff09a1e0f369bb04781ee4e429ebce64a4d5c16b0f136142ad213cdb1.json b/backend/.sqlx/query-597b148ff09a1e0f369bb04781ee4e429ebce64a4d5c16b0f136142ad213cdb1.json new file mode 100644 index 0000000000..4a704e58d8 --- /dev/null +++ b/backend/.sqlx/query-597b148ff09a1e0f369bb04781ee4e429ebce64a4d5c16b0f136142ad213cdb1.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n args AS \"args: Json>>\"\n FROM v2_job\n WHERE id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "args: Json>>", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + true + ] + }, + "hash": "597b148ff09a1e0f369bb04781ee4e429ebce64a4d5c16b0f136142ad213cdb1" +} diff --git a/backend/.sqlx/query-5c141db5cd6806cc27696a38b7f56a190dbea3ff59e2961658dd1677365e0fdf.json b/backend/.sqlx/query-5c141db5cd6806cc27696a38b7f56a190dbea3ff59e2961658dd1677365e0fdf.json new file mode 100644 index 0000000000..3d61a47eb1 --- /dev/null +++ b/backend/.sqlx/query-5c141db5cd6806cc27696a38b7f56a190dbea3ff59e2961658dd1677365e0fdf.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status SET\n flow_status = jsonb_set(\n jsonb_set(flow_status, ARRAY['modules', $3::INTEGER::TEXT, 'job'], to_jsonb($1::UUID::TEXT)),\n ARRAY['modules', $3::INTEGER::TEXT, 'type'],\n to_jsonb('InProgress'::text)\n )\n WHERE id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Uuid", + "Int4" + ] + }, + "nullable": [] + }, + "hash": "5c141db5cd6806cc27696a38b7f56a190dbea3ff59e2961658dd1677365e0fdf" +} diff --git a/backend/.sqlx/query-5d79c4817696d0ba0d2062eef27ff3856fc4a732adf5796e9b06c826406584dc.json b/backend/.sqlx/query-5d79c4817696d0ba0d2062eef27ff3856fc4a732adf5796e9b06c826406584dc.json deleted file mode 100644 index 9bdc6fac55..0000000000 --- a/backend/.sqlx/query-5d79c4817696d0ba0d2062eef27ff3856fc4a732adf5796e9b06c826406584dc.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE completed_job SET permissioned_as = ('u/' || $1) WHERE permissioned_as = ('u/' || $2) AND workspace_id = $3", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "5d79c4817696d0ba0d2062eef27ff3856fc4a732adf5796e9b06c826406584dc" -} diff --git a/backend/.sqlx/query-61ecf606dec2978f3e63b2dec92465202213b7ada2198ea1d9b4d4d1f5ed658f.json b/backend/.sqlx/query-61ecf606dec2978f3e63b2dec92465202213b7ada2198ea1d9b4d4d1f5ed658f.json new file mode 100644 index 0000000000..cb0a2c30c9 --- /dev/null +++ b/backend/.sqlx/query-61ecf606dec2978f3e63b2dec92465202213b7ada2198ea1d9b4d4d1f5ed658f.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_queue SET worker = $2 WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "61ecf606dec2978f3e63b2dec92465202213b7ada2198ea1d9b4d4d1f5ed658f" +} diff --git a/backend/.sqlx/query-6239e15d2389e24e290d86bb96e3ade1656cd403000a34a19a942993b60ff612.json b/backend/.sqlx/query-6239e15d2389e24e290d86bb96e3ade1656cd403000a34a19a942993b60ff612.json deleted file mode 100644 index 19ca4025d8..0000000000 --- a/backend/.sqlx/query-6239e15d2389e24e290d86bb96e3ade1656cd403000a34a19a942993b60ff612.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE queue SET schedule_path = REGEXP_REPLACE(schedule_path,'u/' || $2 || '/(.*)','u/' || $1 || '/\\1') WHERE schedule_path LIKE ('u/' || $2 || '/%') AND workspace_id = $3", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "6239e15d2389e24e290d86bb96e3ade1656cd403000a34a19a942993b60ff612" -} diff --git a/backend/.sqlx/query-631d4637e4137a0680ffa56e4639c009214eeaee8fd27a0d3050998b354c45ff.json b/backend/.sqlx/query-631d4637e4137a0680ffa56e4639c009214eeaee8fd27a0d3050998b354c45ff.json deleted file mode 100644 index d38bf22973..0000000000 --- a/backend/.sqlx/query-631d4637e4137a0680ffa56e4639c009214eeaee8fd27a0d3050998b354c45ff.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE queue\n SET last_ping = null\n WHERE id = $1 AND last_ping = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid", - "Timestamptz" - ] - }, - "nullable": [] - }, - "hash": "631d4637e4137a0680ffa56e4639c009214eeaee8fd27a0d3050998b354c45ff" -} diff --git a/backend/.sqlx/query-867e0f8299e22a6b8e02d21adeb0281b1a10593715db823a5adeeac3ab1058cd.json b/backend/.sqlx/query-63b5f03741be97d0e8763dd070649ebb6ec02aa083d7e175c1a02a38935a4024.json similarity index 77% rename from backend/.sqlx/query-867e0f8299e22a6b8e02d21adeb0281b1a10593715db823a5adeeac3ab1058cd.json rename to backend/.sqlx/query-63b5f03741be97d0e8763dd070649ebb6ec02aa083d7e175c1a02a38935a4024.json index ec6143fe39..4d914355db 100644 --- a/backend/.sqlx/query-867e0f8299e22a6b8e02d21adeb0281b1a10593715db823a5adeeac3ab1058cd.json +++ b/backend/.sqlx/query-63b5f03741be97d0e8763dd070649ebb6ec02aa083d7e175c1a02a38935a4024.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT EXISTS(SELECT 1 FROM http_trigger WHERE route_path_key = $1 AND workspace_id = $2 AND http_method = $3)", + "query": "SELECT EXISTS(SELECT 1 FROM http_trigger WHERE route_path_key = $1 AND http_method = $2 AND ($3::TEXT IS NULL OR path != $3))", "describe": { "columns": [ { @@ -11,7 +11,6 @@ ], "parameters": { "Left": [ - "Text", "Text", { "Custom": { @@ -26,12 +25,13 @@ ] } } - } + }, + "Text" ] }, "nullable": [ null ] }, - "hash": "867e0f8299e22a6b8e02d21adeb0281b1a10593715db823a5adeeac3ab1058cd" + "hash": "63b5f03741be97d0e8763dd070649ebb6ec02aa083d7e175c1a02a38935a4024" } diff --git a/backend/.sqlx/query-63e54fe57ec439b68eead00a02209f81076c5317d590e1441b557555b4d7ad96.json b/backend/.sqlx/query-63e54fe57ec439b68eead00a02209f81076c5317d590e1441b557555b4d7ad96.json deleted file mode 100644 index a30a87991e..0000000000 --- a/backend/.sqlx/query-63e54fe57ec439b68eead00a02209f81076c5317d590e1441b557555b4d7ad96.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH uuid_table as (\n select gen_random_uuid() as uuid from generate_series(1, $5)\n )\n INSERT INTO job\n (id, workspace_id, raw_code, raw_lock, raw_flow)\n (SELECT uuid, $1, $2, $3, $4 FROM uuid_table)\n RETURNING id", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Uuid" - } - ], - "parameters": { - "Left": [ - "Varchar", - "Text", - "Text", - "Jsonb", - "Int4" - ] - }, - "nullable": [ - false - ] - }, - "hash": "63e54fe57ec439b68eead00a02209f81076c5317d590e1441b557555b4d7ad96" -} diff --git a/backend/.sqlx/query-653574b381a31548d82c1f6f3f44ec826597c42310ab78f9aacd7d9448206c6a.json b/backend/.sqlx/query-653574b381a31548d82c1f6f3f44ec826597c42310ab78f9aacd7d9448206c6a.json new file mode 100644 index 0000000000..3d3309cb5c --- /dev/null +++ b/backend/.sqlx/query-653574b381a31548d82c1f6f3f44ec826597c42310ab78f9aacd7d9448206c6a.json @@ -0,0 +1,34 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH zombie_jobs AS (\n UPDATE v2_job_queue q SET running = false, started_at = null\n FROM v2_job j, v2_job_runtime r\n WHERE j.id = q.id AND j.id = r.id\n AND ping < now() - ($1 || ' seconds')::interval\n AND running = true\n AND kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlescriptflow')\n AND same_worker = false\n RETURNING q.id, q.workspace_id, ping\n ),\n update_concurrency AS (\n UPDATE concurrency_counter cc\n SET job_uuids = job_uuids - zj.id::text\n FROM zombie_jobs zj\n INNER JOIN concurrency_key ck ON ck.job_id = zj.id\n WHERE cc.concurrency_id = ck.key\n )\n SELECT id, workspace_id, ping FROM zombie_jobs", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "ping", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + true + ] + }, + "hash": "653574b381a31548d82c1f6f3f44ec826597c42310ab78f9aacd7d9448206c6a" +} diff --git a/backend/.sqlx/query-6664be80f0d72ea7b8b184c5348063db3bbfea67f4a056d6e3be1fc4255cfc06.json b/backend/.sqlx/query-6664be80f0d72ea7b8b184c5348063db3bbfea67f4a056d6e3be1fc4255cfc06.json new file mode 100644 index 0000000000..0e7ae566fd --- /dev/null +++ b/backend/.sqlx/query-6664be80f0d72ea7b8b184c5348063db3bbfea67f4a056d6e3be1fc4255cfc06.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status\n SET flow_leaf_jobs = JSONB_SET(coalesce(flow_leaf_jobs, '{}'::jsonb), ARRAY[$1::TEXT], $2)\n WHERE COALESCE((SELECT flow_innermost_root_job FROM v2_job WHERE id = $3), $3) = id", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Jsonb", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "6664be80f0d72ea7b8b184c5348063db3bbfea67f4a056d6e3be1fc4255cfc06" +} diff --git a/backend/.sqlx/query-67afe352fc26dda9107c90e50e954642d877178ce2c0e73b72c3824135ef86f4.json b/backend/.sqlx/query-67afe352fc26dda9107c90e50e954642d877178ce2c0e73b72c3824135ef86f4.json new file mode 100644 index 0000000000..a09be9741e --- /dev/null +++ b/backend/.sqlx/query-67afe352fc26dda9107c90e50e954642d877178ce2c0e73b72c3824135ef86f4.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO job_logs (job_id, logs)\n VALUES ($1, 'Restarted job after not receiving job''s ping for too long the ' || now() || '\n\n')\n ON CONFLICT (job_id) DO UPDATE SET logs = job_logs.logs || '\n' || EXCLUDED.logs\n WHERE job_logs.job_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "67afe352fc26dda9107c90e50e954642d877178ce2c0e73b72c3824135ef86f4" +} diff --git a/backend/.sqlx/query-67f671d3feb0c7beedd2e99740fec9c0f01b077026c0be08e87b9ba65fde5608.json b/backend/.sqlx/query-67f671d3feb0c7beedd2e99740fec9c0f01b077026c0be08e87b9ba65fde5608.json new file mode 100644 index 0000000000..eb0c2c8f3c --- /dev/null +++ b/backend/.sqlx/query-67f671d3feb0c7beedd2e99740fec9c0f01b077026c0be08e87b9ba65fde5608.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM _sqlx_migrations WHERE version=20250201145632", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "67f671d3feb0c7beedd2e99740fec9c0f01b077026c0be08e87b9ba65fde5608" +} diff --git a/backend/.sqlx/query-6818cae88492f83baf55f54bf5dd5397e04dbe771445818f88903ee5677b3631.json b/backend/.sqlx/query-6818cae88492f83baf55f54bf5dd5397e04dbe771445818f88903ee5677b3631.json deleted file mode 100644 index 19ac6b90d3..0000000000 --- a/backend/.sqlx/query-6818cae88492f83baf55f54bf5dd5397e04dbe771445818f88903ee5677b3631.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE queue SET script_path = REGEXP_REPLACE(script_path,'u/' || $2 || '/(.*)','u/' || $1 || '/\\1') WHERE script_path LIKE ('u/' || $2 || '/%') AND workspace_id = $3", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "6818cae88492f83baf55f54bf5dd5397e04dbe771445818f88903ee5677b3631" -} diff --git a/backend/.sqlx/query-689b9b39c2e51440b428b521a60eca212d91bb79365a117d754f1d47d8f1ba23.json b/backend/.sqlx/query-689b9b39c2e51440b428b521a60eca212d91bb79365a117d754f1d47d8f1ba23.json new file mode 100644 index 0000000000..c64d090bd1 --- /dev/null +++ b/backend/.sqlx/query-689b9b39c2e51440b428b521a60eca212d91bb79365a117d754f1d47d8f1ba23.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_completed SET result = '{}'::jsonb WHERE id = ANY($1)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "UuidArray" + ] + }, + "nullable": [] + }, + "hash": "689b9b39c2e51440b428b521a60eca212d91bb79365a117d754f1d47d8f1ba23" +} diff --git a/backend/.sqlx/query-695a02db2ee43d18fdf139b52eb546cffa44845cd634f188d6619cf39462ca93.json b/backend/.sqlx/query-695a02db2ee43d18fdf139b52eb546cffa44845cd634f188d6619cf39462ca93.json new file mode 100644 index 0000000000..546f3e8026 --- /dev/null +++ b/backend/.sqlx/query-695a02db2ee43d18fdf139b52eb546cffa44845cd634f188d6619cf39462ca93.json @@ -0,0 +1,37 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n result #> $3 AS \"result: sqlx::types::Json>\",\n result_columns,\n created_by AS \"created_by!\"\n FROM v2_job_completed c\n JOIN v2_job USING (id)\n WHERE c.id = $1 AND c.workspace_id = $2 AND ($4::text[] IS NULL OR tag = ANY($4))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "result: sqlx::types::Json>", + "type_info": "Jsonb" + }, + { + "ordinal": 1, + "name": "result_columns", + "type_info": "TextArray" + }, + { + "ordinal": 2, + "name": "created_by!", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "TextArray", + "TextArray" + ] + }, + "nullable": [ + null, + true, + false + ] + }, + "hash": "695a02db2ee43d18fdf139b52eb546cffa44845cd634f188d6619cf39462ca93" +} diff --git a/backend/.sqlx/query-69ff806066a24c60188d7547a7776c160ee65eaa01b259c996f3f96ed44fcaaf.json b/backend/.sqlx/query-69ff806066a24c60188d7547a7776c160ee65eaa01b259c996f3f96ed44fcaaf.json new file mode 100644 index 0000000000..75ef2996a0 --- /dev/null +++ b/backend/.sqlx/query-69ff806066a24c60188d7547a7776c160ee65eaa01b259c996f3f96ed44fcaaf.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status\n SET flow_status = JSONB_SET(flow_status, ARRAY['step'], $1)\n WHERE id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "69ff806066a24c60188d7547a7776c160ee65eaa01b259c996f3f96ed44fcaaf" +} diff --git a/backend/.sqlx/query-5ba4b87528ad49f17d72b53c3db30f5ca4b3b0b0afbd5d9721c8b5d692af601b.json b/backend/.sqlx/query-6ab112fa42a9ae332bfa30427b70fa742351c5c180ac3de106df54f7badb494c.json similarity index 65% rename from backend/.sqlx/query-5ba4b87528ad49f17d72b53c3db30f5ca4b3b0b0afbd5d9721c8b5d692af601b.json rename to backend/.sqlx/query-6ab112fa42a9ae332bfa30427b70fa742351c5c180ac3de106df54f7badb494c.json index f8b5b54892..ef26a8453e 100644 --- a/backend/.sqlx/query-5ba4b87528ad49f17d72b53c3db30f5ca4b3b0b0afbd5d9721c8b5d692af601b.json +++ b/backend/.sqlx/query-6ab112fa42a9ae332bfa30427b70fa742351c5c180ac3de106df54f7badb494c.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT COUNT(id) FROM queue WHERE email = $1", + "query": "SELECT COUNT(id) FROM v2_as_queue WHERE email = $1", "describe": { "columns": [ { @@ -18,5 +18,5 @@ null ] }, - "hash": "5ba4b87528ad49f17d72b53c3db30f5ca4b3b0b0afbd5d9721c8b5d692af601b" + "hash": "6ab112fa42a9ae332bfa30427b70fa742351c5c180ac3de106df54f7badb494c" } diff --git a/backend/.sqlx/query-6681048ee83236e9eb33b407b5d3cf89f563e57d5d3e7981d58cecf147b9bf1e.json b/backend/.sqlx/query-6bdb3fcfe16fc40222dc7010a11026d4d4e0d381b31fe02da7d2667c0cdc1a85.json similarity index 52% rename from backend/.sqlx/query-6681048ee83236e9eb33b407b5d3cf89f563e57d5d3e7981d58cecf147b9bf1e.json rename to backend/.sqlx/query-6bdb3fcfe16fc40222dc7010a11026d4d4e0d381b31fe02da7d2667c0cdc1a85.json index 0ca0331f8f..c498b1790a 100644 --- a/backend/.sqlx/query-6681048ee83236e9eb33b407b5d3cf89f563e57d5d3e7981d58cecf147b9bf1e.json +++ b/backend/.sqlx/query-6bdb3fcfe16fc40222dc7010a11026d4d4e0d381b31fe02da7d2667c0cdc1a85.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "UPDATE completed_job SET workspace_id = $1 WHERE workspace_id = $2", + "query": "UPDATE v2_job_completed SET workspace_id = $1 WHERE workspace_id = $2", "describe": { "columns": [], "parameters": { @@ -11,5 +11,5 @@ }, "nullable": [] }, - "hash": "6681048ee83236e9eb33b407b5d3cf89f563e57d5d3e7981d58cecf147b9bf1e" + "hash": "6bdb3fcfe16fc40222dc7010a11026d4d4e0d381b31fe02da7d2667c0cdc1a85" } diff --git a/backend/.sqlx/query-02bb4ea17e83c79f870e2655d6d9c035af6d763b7ee9577280785ccf0220a123.json b/backend/.sqlx/query-6daf2aca3e272a1efd7b26b4c80d6b490628f502ed9f9de75cc78d0fe196f260.json similarity index 56% rename from backend/.sqlx/query-02bb4ea17e83c79f870e2655d6d9c035af6d763b7ee9577280785ccf0220a123.json rename to backend/.sqlx/query-6daf2aca3e272a1efd7b26b4c80d6b490628f502ed9f9de75cc78d0fe196f260.json index e8df1339ff..d94865aafa 100644 --- a/backend/.sqlx/query-02bb4ea17e83c79f870e2655d6d9c035af6d763b7ee9577280785ccf0220a123.json +++ b/backend/.sqlx/query-6daf2aca3e272a1efd7b26b4c80d6b490628f502ed9f9de75cc78d0fe196f260.json @@ -1,11 +1,11 @@ { "db_name": "PostgreSQL", - "query": "SELECT running FROM queue WHERE id = $1 AND workspace_id = $2", + "query": "SELECT running AS \"running!\" FROM v2_job_queue WHERE id = $1 AND workspace_id = $2", "describe": { "columns": [ { "ordinal": 0, - "name": "running", + "name": "running!", "type_info": "Bool" } ], @@ -19,5 +19,5 @@ false ] }, - "hash": "02bb4ea17e83c79f870e2655d6d9c035af6d763b7ee9577280785ccf0220a123" + "hash": "6daf2aca3e272a1efd7b26b4c80d6b490628f502ed9f9de75cc78d0fe196f260" } diff --git a/backend/.sqlx/query-6e7f234267fbb4720b29f288fba82c1df21ba601ac0989e175f834c569962d46.json b/backend/.sqlx/query-6e7f234267fbb4720b29f288fba82c1df21ba601ac0989e175f834c569962d46.json deleted file mode 100644 index e8db78adbd..0000000000 --- a/backend/.sqlx/query-6e7f234267fbb4720b29f288fba82c1df21ba601ac0989e175f834c569962d46.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE queue\n SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'iterator', 'index'], ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb),\n last_ping = NULL\n WHERE id = $2\n RETURNING (flow_status->'modules'->$1::int->'iterator'->>'index')::int", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "int4", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Int4", - "Uuid" - ] - }, - "nullable": [ - null - ] - }, - "hash": "6e7f234267fbb4720b29f288fba82c1df21ba601ac0989e175f834c569962d46" -} diff --git a/backend/.sqlx/query-6ff7a025f529c077c1b6c9632a367aa29e2f0fdac3f1984550484d5a06a6ea21.json b/backend/.sqlx/query-6ff7a025f529c077c1b6c9632a367aa29e2f0fdac3f1984550484d5a06a6ea21.json new file mode 100644 index 0000000000..b045ed2b07 --- /dev/null +++ b/backend/.sqlx/query-6ff7a025f529c077c1b6c9632a367aa29e2f0fdac3f1984550484d5a06a6ea21.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_runtime r SET\n memory_peak = $1,\n ping = now()\n FROM v2_job_queue q\n WHERE r.id = $2 AND q.id = r.id\n RETURNING canceled_by, canceled_reason", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "canceled_by", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "canceled_reason", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Int4", + "Uuid" + ] + }, + "nullable": [ + true, + true + ] + }, + "hash": "6ff7a025f529c077c1b6c9632a367aa29e2f0fdac3f1984550484d5a06a6ea21" +} diff --git a/backend/.sqlx/query-70e740465e648d84d32a506916ed437d1e32d4b5bdc7f7fae22e804428660836.json b/backend/.sqlx/query-70e740465e648d84d32a506916ed437d1e32d4b5bdc7f7fae22e804428660836.json new file mode 100644 index 0000000000..9278984d51 --- /dev/null +++ b/backend/.sqlx/query-70e740465e648d84d32a506916ed437d1e32d4b5bdc7f7fae22e804428660836.json @@ -0,0 +1,25 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH uuid_table as (\n select unnest($4::uuid[]) as uuid\n )\n INSERT INTO v2_job_queue\n (id, workspace_id, scheduled_for, tag)\n (SELECT uuid, $1, $2, $3 FROM uuid_table) \n RETURNING id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Timestamptz", + "Varchar", + "UuidArray" + ] + }, + "nullable": [ + false + ] + }, + "hash": "70e740465e648d84d32a506916ed437d1e32d4b5bdc7f7fae22e804428660836" +} diff --git a/backend/.sqlx/query-7145a1a6cf0ccfae5874c882e63ff0d370cf1ab4d41f68494a940951eaa52bc3.json b/backend/.sqlx/query-7145a1a6cf0ccfae5874c882e63ff0d370cf1ab4d41f68494a940951eaa52bc3.json new file mode 100644 index 0000000000..edec8c5053 --- /dev/null +++ b/backend/.sqlx/query-7145a1a6cf0ccfae5874c882e63ff0d370cf1ab4d41f68494a940951eaa52bc3.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH ping AS (UPDATE v2_job_runtime SET ping = NULL WHERE id = $2 RETURNING id)\n UPDATE v2_job_queue SET\n running = false,\n started_at = null,\n scheduled_for = $1\n WHERE id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Timestamptz", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "7145a1a6cf0ccfae5874c882e63ff0d370cf1ab4d41f68494a940951eaa52bc3" +} diff --git a/backend/.sqlx/query-e207bbcdd478758562e3082330836d6efe1dd6c4f719bc415222eba78d5e63fc.json b/backend/.sqlx/query-729c5c6430431f044148aeeec18ad3056ee4fdf8691ba5503937424d75f3815e.json similarity index 67% rename from backend/.sqlx/query-e207bbcdd478758562e3082330836d6efe1dd6c4f719bc415222eba78d5e63fc.json rename to backend/.sqlx/query-729c5c6430431f044148aeeec18ad3056ee4fdf8691ba5503937424d75f3815e.json index ac91e4f1d1..b31bd17bd3 100644 --- a/backend/.sqlx/query-e207bbcdd478758562e3082330836d6efe1dd6c4f719bc415222eba78d5e63fc.json +++ b/backend/.sqlx/query-729c5c6430431f044148aeeec18ad3056ee4fdf8691ba5503937424d75f3815e.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT id FROM queue", + "query": "SELECT id FROM v2_job_queue", "describe": { "columns": [ { @@ -16,5 +16,5 @@ false ] }, - "hash": "e207bbcdd478758562e3082330836d6efe1dd6c4f719bc415222eba78d5e63fc" + "hash": "729c5c6430431f044148aeeec18ad3056ee4fdf8691ba5503937424d75f3815e" } diff --git a/backend/.sqlx/query-7315c588aac4ff3a1aa6972a7553ac47452a4ed4e8dff0e94f22e6225f4eebc7.json b/backend/.sqlx/query-7315c588aac4ff3a1aa6972a7553ac47452a4ed4e8dff0e94f22e6225f4eebc7.json new file mode 100644 index 0000000000..ebbd00889d --- /dev/null +++ b/backend/.sqlx/query-7315c588aac4ff3a1aa6972a7553ac47452a4ed4e8dff0e94f22e6225f4eebc7.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO v2_job_runtime (id, ping) SELECT unnest($1::uuid[]), null", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "UuidArray" + ] + }, + "nullable": [] + }, + "hash": "7315c588aac4ff3a1aa6972a7553ac47452a4ed4e8dff0e94f22e6225f4eebc7" +} diff --git a/backend/.sqlx/query-7d3180d119da0f6215a571118439469569c31e0234bb073af33f2cd4d6ca71d4.json b/backend/.sqlx/query-7463c1eb3f690239c9277b62d935af4f372c07263c6f86bc89b0d25cb573fa80.json similarity index 53% rename from backend/.sqlx/query-7d3180d119da0f6215a571118439469569c31e0234bb073af33f2cd4d6ca71d4.json rename to backend/.sqlx/query-7463c1eb3f690239c9277b62d935af4f372c07263c6f86bc89b0d25cb573fa80.json index b332ab2871..77ab2c28bd 100644 --- a/backend/.sqlx/query-7d3180d119da0f6215a571118439469569c31e0234bb073af33f2cd4d6ca71d4.json +++ b/backend/.sqlx/query-7463c1eb3f690239c9277b62d935af4f372c07263c6f86bc89b0d25cb573fa80.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "DELETE FROM queue WHERE workspace_id = $1", + "query": "DELETE FROM v2_job WHERE workspace_id = $1", "describe": { "columns": [], "parameters": { @@ -10,5 +10,5 @@ }, "nullable": [] }, - "hash": "7d3180d119da0f6215a571118439469569c31e0234bb073af33f2cd4d6ca71d4" + "hash": "7463c1eb3f690239c9277b62d935af4f372c07263c6f86bc89b0d25cb573fa80" } diff --git a/backend/.sqlx/query-74dbd5a09255c30991078492ba3850e02ffbef25fbfd29cbedc041b0e439e580.json b/backend/.sqlx/query-74dbd5a09255c30991078492ba3850e02ffbef25fbfd29cbedc041b0e439e580.json new file mode 100644 index 0000000000..77515a838f --- /dev/null +++ b/backend/.sqlx/query-74dbd5a09255c30991078492ba3850e02ffbef25fbfd29cbedc041b0e439e580.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id AS \"id!\", flow_status AS \"flow_status!: Json\"\n FROM v2_as_completed_job\n WHERE parent_job = $1 AND workspace_id = $2 AND flow_status IS NOT NULL", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id!", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "flow_status!: Json", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + true, + true + ] + }, + "hash": "74dbd5a09255c30991078492ba3850e02ffbef25fbfd29cbedc041b0e439e580" +} diff --git a/backend/.sqlx/query-c75761c9aa900391251596771782039809bc01e4a0aa05701d6607b06769caa5.json b/backend/.sqlx/query-76690ea29969da28840c90016ec78e5ea7f326df9f19e9d2f0445e662c921fea.json similarity index 61% rename from backend/.sqlx/query-c75761c9aa900391251596771782039809bc01e4a0aa05701d6607b06769caa5.json rename to backend/.sqlx/query-76690ea29969da28840c90016ec78e5ea7f326df9f19e9d2f0445e662c921fea.json index abe61cf520..67ad1da697 100644 --- a/backend/.sqlx/query-c75761c9aa900391251596771782039809bc01e4a0aa05701d6607b06769caa5.json +++ b/backend/.sqlx/query-76690ea29969da28840c90016ec78e5ea7f326df9f19e9d2f0445e662c921fea.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT id, flow_status, suspend, script_path\n FROM queue\n WHERE id = ( SELECT parent_job FROM queue WHERE id = $1 UNION ALL SELECT parent_job FROM completed_job WHERE id = $1)\n FOR UPDATE\n ", + "query": "\n SELECT q.id, f.flow_status, q.suspend, j.runnable_path AS script_path\n FROM v2_job_queue q\n JOIN v2_job j USING (id)\n JOIN v2_job_status f USING (id)\n WHERE id = ( SELECT parent_job FROM v2_job WHERE id = $1 )\n FOR UPDATE\n ", "describe": { "columns": [ { @@ -36,5 +36,5 @@ true ] }, - "hash": "c75761c9aa900391251596771782039809bc01e4a0aa05701d6607b06769caa5" + "hash": "76690ea29969da28840c90016ec78e5ea7f326df9f19e9d2f0445e662c921fea" } diff --git a/backend/.sqlx/query-76ca60e456022cf3d1931245b7daf22783c81bc757d735a4b247cc693dfed719.json b/backend/.sqlx/query-76ca60e456022cf3d1931245b7daf22783c81bc757d735a4b247cc693dfed719.json deleted file mode 100644 index 530638af1f..0000000000 --- a/backend/.sqlx/query-76ca60e456022cf3d1931245b7daf22783c81bc757d735a4b247cc693dfed719.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE queue\n SET flow_status = JSONB_SET(flow_status, ARRAY['modules', flow_status->>'step', 'progress'], $1)\n WHERE id = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Jsonb", - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "76ca60e456022cf3d1931245b7daf22783c81bc757d735a4b247cc693dfed719" -} diff --git a/backend/.sqlx/query-777190559e27c8c8fb6718b0a0c1d7db9b956abd88b94db3948f2c579c3826d0.json b/backend/.sqlx/query-777190559e27c8c8fb6718b0a0c1d7db9b956abd88b94db3948f2c579c3826d0.json deleted file mode 100644 index 0c1d65bb6b..0000000000 --- a/backend/.sqlx/query-777190559e27c8c8fb6718b0a0c1d7db9b956abd88b94db3948f2c579c3826d0.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE queue\n SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'branchall', 'branch'], ((flow_status->'modules'->$1::int->'branchall'->>'branch')::int + 1)::text::jsonb),\n last_ping = NULL\n WHERE id = $2\n RETURNING (flow_status->'modules'->$1::int->'branchall'->>'branch')::int", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "int4", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Int4", - "Uuid" - ] - }, - "nullable": [ - null - ] - }, - "hash": "777190559e27c8c8fb6718b0a0c1d7db9b956abd88b94db3948f2c579c3826d0" -} diff --git a/backend/.sqlx/query-c732dcb4df5877560a3e75e0032179140cbe442f0b80ce1172d80903dd8a14fa.json b/backend/.sqlx/query-798a6bdc8f2fb9421808e084fbddf7de61fa9953998e3dc66af0e5f7491a65f4.json similarity index 52% rename from backend/.sqlx/query-c732dcb4df5877560a3e75e0032179140cbe442f0b80ce1172d80903dd8a14fa.json rename to backend/.sqlx/query-798a6bdc8f2fb9421808e084fbddf7de61fa9953998e3dc66af0e5f7491a65f4.json index d19fc9d699..15d39aa539 100644 --- a/backend/.sqlx/query-c732dcb4df5877560a3e75e0032179140cbe442f0b80ce1172d80903dd8a14fa.json +++ b/backend/.sqlx/query-798a6bdc8f2fb9421808e084fbddf7de61fa9953998e3dc66af0e5f7491a65f4.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "UPDATE queue SET canceled = true WHERE id = $1", + "query": "UPDATE v2_job_queue SET suspend = 0 WHERE id = $1", "describe": { "columns": [], "parameters": { @@ -10,5 +10,5 @@ }, "nullable": [] }, - "hash": "c732dcb4df5877560a3e75e0032179140cbe442f0b80ce1172d80903dd8a14fa" + "hash": "798a6bdc8f2fb9421808e084fbddf7de61fa9953998e3dc66af0e5f7491a65f4" } diff --git a/backend/.sqlx/query-3df03ec2345c905f03450e1e3f0c3ce7f41a22e72c5180ef8cc910305e4d0fce.json b/backend/.sqlx/query-7af1cf089022fc1c3597b270b69aa669a153ab0c0bb2807cd4f7fd405afa6f69.json similarity index 78% rename from backend/.sqlx/query-3df03ec2345c905f03450e1e3f0c3ce7f41a22e72c5180ef8cc910305e4d0fce.json rename to backend/.sqlx/query-7af1cf089022fc1c3597b270b69aa669a153ab0c0bb2807cd4f7fd405afa6f69.json index 3bfcb0a9f3..d2e54f45c8 100644 --- a/backend/.sqlx/query-3df03ec2345c905f03450e1e3f0c3ce7f41a22e72c5180ef8cc910305e4d0fce.json +++ b/backend/.sqlx/query-7af1cf089022fc1c3597b270b69aa669a153ab0c0bb2807cd4f7fd405afa6f69.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n script_path, script_hash AS \"script_hash: ScriptHash\",\n job_kind AS \"job_kind: JobKind\",\n flow_status AS \"flow_status: Json>\",\n raw_flow AS \"raw_flow: Json>\"\n FROM completed_job WHERE id = $1 and workspace_id = $2", + "query": "SELECT\n script_path, script_hash AS \"script_hash: ScriptHash\",\n job_kind AS \"job_kind!: JobKind\",\n flow_status AS \"flow_status: Json>\",\n raw_flow AS \"raw_flow: Json>\"\n FROM v2_as_completed_job WHERE id = $1 and workspace_id = $2", "describe": { "columns": [ { @@ -15,7 +15,7 @@ }, { "ordinal": 2, - "name": "job_kind: JobKind", + "name": "job_kind!: JobKind", "type_info": { "Custom": { "name": "job_kind", @@ -64,10 +64,10 @@ "nullable": [ true, true, - false, + true, true, true ] }, - "hash": "3df03ec2345c905f03450e1e3f0c3ce7f41a22e72c5180ef8cc910305e4d0fce" + "hash": "7af1cf089022fc1c3597b270b69aa669a153ab0c0bb2807cd4f7fd405afa6f69" } diff --git a/backend/.sqlx/query-7b084617bf2de5ed9a657d2bd2cbc9de1246427bf0302c032f8bd26ff93313f7.json b/backend/.sqlx/query-7b084617bf2de5ed9a657d2bd2cbc9de1246427bf0302c032f8bd26ff93313f7.json new file mode 100644 index 0000000000..3f72b7b760 --- /dev/null +++ b/backend/.sqlx/query-7b084617bf2de5ed9a657d2bd2cbc9de1246427bf0302c032f8bd26ff93313f7.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status\n SET flow_status = JSONB_SET(flow_status, ARRAY['failure_module'], $1)\n WHERE id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "7b084617bf2de5ed9a657d2bd2cbc9de1246427bf0302c032f8bd26ff93313f7" +} diff --git a/backend/.sqlx/query-c845c06aa46b52a1e3672fd379dbcec167f4044e45e932aeeefd3d9237e5042c.json b/backend/.sqlx/query-7b1e6b67a20ae1128118d5f5cc0db4007fb9dc6fd20582a46ebb951fca3a7abd.json similarity index 58% rename from backend/.sqlx/query-c845c06aa46b52a1e3672fd379dbcec167f4044e45e932aeeefd3d9237e5042c.json rename to backend/.sqlx/query-7b1e6b67a20ae1128118d5f5cc0db4007fb9dc6fd20582a46ebb951fca3a7abd.json index e698dac427..a094b8580f 100644 --- a/backend/.sqlx/query-c845c06aa46b52a1e3672fd379dbcec167f4044e45e932aeeefd3d9237e5042c.json +++ b/backend/.sqlx/query-7b1e6b67a20ae1128118d5f5cc0db4007fb9dc6fd20582a46ebb951fca3a7abd.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT CAST(ROUND(AVG(duration_ms), 0) AS BIGINT) AS avg_duration_s FROM\n (SELECT duration_ms FROM concurrency_key LEFT JOIN completed_job ON completed_job.id = concurrency_key.job_id WHERE key = $1 AND ended_at IS NOT NULL\n ORDER BY ended_at\n DESC LIMIT 10) AS t", + "query": "SELECT CAST(ROUND(AVG(duration_ms), 0) AS BIGINT) AS avg_duration_s FROM\n (SELECT duration_ms FROM concurrency_key LEFT JOIN v2_as_completed_job ON v2_as_completed_job.id = concurrency_key.job_id WHERE key = $1 AND ended_at IS NOT NULL\n ORDER BY ended_at\n DESC LIMIT 10) AS t", "describe": { "columns": [ { @@ -18,5 +18,5 @@ null ] }, - "hash": "c845c06aa46b52a1e3672fd379dbcec167f4044e45e932aeeefd3d9237e5042c" + "hash": "7b1e6b67a20ae1128118d5f5cc0db4007fb9dc6fd20582a46ebb951fca3a7abd" } diff --git a/backend/.sqlx/query-7bd7505b008954aae6152554c1f9bb0611d0ec451b48aa1a4de309dadeb53315.json b/backend/.sqlx/query-7bd7505b008954aae6152554c1f9bb0611d0ec451b48aa1a4de309dadeb53315.json new file mode 100644 index 0000000000..c7f3146594 --- /dev/null +++ b/backend/.sqlx/query-7bd7505b008954aae6152554c1f9bb0611d0ec451b48aa1a4de309dadeb53315.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status\n SET flow_status = flow_status - 'approval_conditions'\n WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "7bd7505b008954aae6152554c1f9bb0611d0ec451b48aa1a4de309dadeb53315" +} diff --git a/backend/.sqlx/query-e9d9edc030061c5ccf2fb4294acbbae160e97faede6934d90f4882d806c14813.json b/backend/.sqlx/query-7c7234d8234fbaff4d42eb47d1e1ea1747ffe28986055c01a4661628e47c0e51.json similarity index 51% rename from backend/.sqlx/query-e9d9edc030061c5ccf2fb4294acbbae160e97faede6934d90f4882d806c14813.json rename to backend/.sqlx/query-7c7234d8234fbaff4d42eb47d1e1ea1747ffe28986055c01a4661628e47c0e51.json index 8b79600c50..b1ace4f315 100644 --- a/backend/.sqlx/query-e9d9edc030061c5ccf2fb4294acbbae160e97faede6934d90f4882d806c14813.json +++ b/backend/.sqlx/query-7c7234d8234fbaff4d42eb47d1e1ea1747ffe28986055c01a4661628e47c0e51.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "UPDATE completed_job SET canceled_by = $1 WHERE canceled_by = $2 AND workspace_id = $3", + "query": "UPDATE v2_job_completed SET canceled_by = $1 WHERE canceled_by = $2 AND workspace_id = $3", "describe": { "columns": [], "parameters": { @@ -12,5 +12,5 @@ }, "nullable": [] }, - "hash": "e9d9edc030061c5ccf2fb4294acbbae160e97faede6934d90f4882d806c14813" + "hash": "7c7234d8234fbaff4d42eb47d1e1ea1747ffe28986055c01a4661628e47c0e51" } diff --git a/backend/.sqlx/query-5dfa6932d7c6d5006fe352da3041680b2ee1ebe9258355a0db24ce4fd26f23de.json b/backend/.sqlx/query-7cb0b49f5898b1adbf6579922aa02be4b9550b043240b2076e105976fedb789c.json similarity index 51% rename from backend/.sqlx/query-5dfa6932d7c6d5006fe352da3041680b2ee1ebe9258355a0db24ce4fd26f23de.json rename to backend/.sqlx/query-7cb0b49f5898b1adbf6579922aa02be4b9550b043240b2076e105976fedb789c.json index 0416e38fb1..3e507fe815 100644 --- a/backend/.sqlx/query-5dfa6932d7c6d5006fe352da3041680b2ee1ebe9258355a0db24ce4fd26f23de.json +++ b/backend/.sqlx/query-7cb0b49f5898b1adbf6579922aa02be4b9550b043240b2076e105976fedb789c.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "UPDATE queue SET canceled = true, canceled_by = $1, canceled_reason = $2, scheduled_for = now(), suspend = 0 WHERE id = $3 AND workspace_id = $4 AND (canceled = false OR canceled_reason != $2) RETURNING id", + "query": "UPDATE v2_job_queue SET canceled_by = $1, canceled_reason = $2, scheduled_for = now(), suspend = 0 WHERE id = $3 AND workspace_id = $4 AND (canceled_by IS NULL OR canceled_reason != $2) RETURNING id", "describe": { "columns": [ { @@ -21,5 +21,5 @@ false ] }, - "hash": "5dfa6932d7c6d5006fe352da3041680b2ee1ebe9258355a0db24ce4fd26f23de" + "hash": "7cb0b49f5898b1adbf6579922aa02be4b9550b043240b2076e105976fedb789c" } diff --git a/backend/.sqlx/query-7d07a717533bfcaf581f6655bc387095542490fbb4aae30ec7fa75c2dae98ec8.json b/backend/.sqlx/query-7d07a717533bfcaf581f6655bc387095542490fbb4aae30ec7fa75c2dae98ec8.json new file mode 100644 index 0000000000..4e90a2a0ec --- /dev/null +++ b/backend/.sqlx/query-7d07a717533bfcaf581f6655bc387095542490fbb4aae30ec7fa75c2dae98ec8.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_runtime r SET\n ping = now()\n FROM v2_job_queue q\n WHERE r.id = $1 AND q.id = r.id\n AND q.workspace_id = $2\n AND canceled_by IS NULL", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [] + }, + "hash": "7d07a717533bfcaf581f6655bc387095542490fbb4aae30ec7fa75c2dae98ec8" +} diff --git a/backend/.sqlx/query-7f6649b177f4ec948e396e179ea044812f8103498080a24246063e06b0ebfccf.json b/backend/.sqlx/query-7f6649b177f4ec948e396e179ea044812f8103498080a24246063e06b0ebfccf.json new file mode 100644 index 0000000000..6c4a2a910f --- /dev/null +++ b/backend/.sqlx/query-7f6649b177f4ec948e396e179ea044812f8103498080a24246063e06b0ebfccf.json @@ -0,0 +1,106 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT * FROM postgres_trigger\n WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "script_path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "is_flow", + "type_info": "Bool" + }, + { + "ordinal": 3, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "edited_by", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 6, + "name": "edited_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 7, + "name": "extra_perms", + "type_info": "Jsonb" + }, + { + "ordinal": 8, + "name": "postgres_resource_path", + "type_info": "Varchar" + }, + { + "ordinal": 9, + "name": "error", + "type_info": "Text" + }, + { + "ordinal": 10, + "name": "server_id", + "type_info": "Varchar" + }, + { + "ordinal": 11, + "name": "last_server_ping", + "type_info": "Timestamptz" + }, + { + "ordinal": 12, + "name": "replication_slot_name", + "type_info": "Varchar" + }, + { + "ordinal": 13, + "name": "publication_name", + "type_info": "Varchar" + }, + { + "ordinal": 14, + "name": "enabled", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + true, + false, + true, + true, + true, + false, + false, + false + ] + }, + "hash": "7f6649b177f4ec948e396e179ea044812f8103498080a24246063e06b0ebfccf" +} diff --git a/backend/.sqlx/query-8123ba05f6e7b9bd395175ee4ec0c36c3726b2da9ff3592589ac7e83df1c537c.json b/backend/.sqlx/query-8123ba05f6e7b9bd395175ee4ec0c36c3726b2da9ff3592589ac7e83df1c537c.json new file mode 100644 index 0000000000..1126431b46 --- /dev/null +++ b/backend/.sqlx/query-8123ba05f6e7b9bd395175ee4ec0c36c3726b2da9ff3592589ac7e83df1c537c.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, flow_status AS \"flow_status!: Json\"\n FROM v2_job_completed WHERE id = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "flow_status!: Json", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + false, + true + ] + }, + "hash": "8123ba05f6e7b9bd395175ee4ec0c36c3726b2da9ff3592589ac7e83df1c537c" +} diff --git a/backend/.sqlx/query-830297547ea33969f96a5c4c2b82a540f1d147948e1b1a7523b21151ffa22305.json b/backend/.sqlx/query-830297547ea33969f96a5c4c2b82a540f1d147948e1b1a7523b21151ffa22305.json new file mode 100644 index 0000000000..2e67b21966 --- /dev/null +++ b/backend/.sqlx/query-830297547ea33969f96a5c4c2b82a540f1d147948e1b1a7523b21151ffa22305.json @@ -0,0 +1,38 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n success AS \"success!\",\n result AS \"result: Json>\",\n started_at AS \"started_at!\"FROM v2_as_completed_job WHERE workspace_id = $1 AND schedule_path = $2 AND script_path = $3 AND id != $4\n ORDER BY created_at DESC\n LIMIT $5", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "success!", + "type_info": "Bool" + }, + { + "ordinal": 1, + "name": "result: Json>", + "type_info": "Jsonb" + }, + { + "ordinal": 2, + "name": "started_at!", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Uuid", + "Int8" + ] + }, + "nullable": [ + true, + true, + true + ] + }, + "hash": "830297547ea33969f96a5c4c2b82a540f1d147948e1b1a7523b21151ffa22305" +} diff --git a/backend/.sqlx/query-6fa6fa8eb511119c6adb18bd4f9f174bbb48eef8952f91911b8d9b45357372d9.json b/backend/.sqlx/query-8381cee8bec06f8eff847b22d25574fc2cbcb32fef9f44632a8bc4c5c487fc4f.json similarity index 65% rename from backend/.sqlx/query-6fa6fa8eb511119c6adb18bd4f9f174bbb48eef8952f91911b8d9b45357372d9.json rename to backend/.sqlx/query-8381cee8bec06f8eff847b22d25574fc2cbcb32fef9f44632a8bc4c5c487fc4f.json index 9623039711..0a5c7ae2c8 100644 --- a/backend/.sqlx/query-6fa6fa8eb511119c6adb18bd4f9f174bbb48eef8952f91911b8d9b45357372d9.json +++ b/backend/.sqlx/query-8381cee8bec06f8eff847b22d25574fc2cbcb32fef9f44632a8bc4c5c487fc4f.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT CASE WHEN pg_column_size(args) < 40000 OR $3 THEN args ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as args FROM completed_job WHERE id = $1 AND workspace_id = $2", + "query": "SELECT CASE WHEN pg_column_size(args) < 40000 OR $3 THEN args ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as args FROM v2_job WHERE id = $1 AND workspace_id = $2", "describe": { "columns": [ { @@ -20,5 +20,5 @@ null ] }, - "hash": "6fa6fa8eb511119c6adb18bd4f9f174bbb48eef8952f91911b8d9b45357372d9" + "hash": "8381cee8bec06f8eff847b22d25574fc2cbcb32fef9f44632a8bc4c5c487fc4f" } diff --git a/backend/.sqlx/query-8440081df15fa7874dded86b8af572d971d5892b7dd4c8e824b953113bd6c4a9.json b/backend/.sqlx/query-8440081df15fa7874dded86b8af572d971d5892b7dd4c8e824b953113bd6c4a9.json new file mode 100644 index 0000000000..4a7ebbc0f7 --- /dev/null +++ b/backend/.sqlx/query-8440081df15fa7874dded86b8af572d971d5892b7dd4c8e824b953113bd6c4a9.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT parent_job\n FROM v2_job\n WHERE id = $1 AND workspace_id = $2\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "parent_job", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "8440081df15fa7874dded86b8af572d971d5892b7dd4c8e824b953113bd6c4a9" +} diff --git a/backend/.sqlx/query-1f93b533fa6fee0db4340445da3fac8e6773bc1db1f88cd60fd3c1e8c9781eb0.json b/backend/.sqlx/query-84ad6f24aa9ccdc6f6e705b7de1bb707637a9bb1475711f9f9f3f2a6e28cb169.json similarity index 54% rename from backend/.sqlx/query-1f93b533fa6fee0db4340445da3fac8e6773bc1db1f88cd60fd3c1e8c9781eb0.json rename to backend/.sqlx/query-84ad6f24aa9ccdc6f6e705b7de1bb707637a9bb1475711f9f9f3f2a6e28cb169.json index abb52bb3c0..510e47e50c 100644 --- a/backend/.sqlx/query-1f93b533fa6fee0db4340445da3fac8e6773bc1db1f88cd60fd3c1e8c9781eb0.json +++ b/backend/.sqlx/query-84ad6f24aa9ccdc6f6e705b7de1bb707637a9bb1475711f9f9f3f2a6e28cb169.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "UPDATE queue SET suspend = $1 WHERE id = $2", + "query": "UPDATE v2_job_queue SET suspend = $1 WHERE id = $2", "describe": { "columns": [], "parameters": { @@ -11,5 +11,5 @@ }, "nullable": [] }, - "hash": "1f93b533fa6fee0db4340445da3fac8e6773bc1db1f88cd60fd3c1e8c9781eb0" + "hash": "84ad6f24aa9ccdc6f6e705b7de1bb707637a9bb1475711f9f9f3f2a6e28cb169" } diff --git a/backend/.sqlx/query-8512298656d6872a2123c2801d842119537374a815c1292fac4c5b75bdd58898.json b/backend/.sqlx/query-8512298656d6872a2123c2801d842119537374a815c1292fac4c5b75bdd58898.json new file mode 100644 index 0000000000..8390708116 --- /dev/null +++ b/backend/.sqlx/query-8512298656d6872a2123c2801d842119537374a815c1292fac4c5b75bdd58898.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job SET runnable_path = REGEXP_REPLACE(runnable_path,'u/' || $2 || '/(.*)','u/' || $1 || '/\\1') WHERE runnable_path LIKE ('u/' || $2 || '/%') AND workspace_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "8512298656d6872a2123c2801d842119537374a815c1292fac4c5b75bdd58898" +} diff --git a/backend/.sqlx/query-85705fc3d7f8ba5f1b12d5fb222c38fc64deb1226aab9dc3bc4465324fce37d1.json b/backend/.sqlx/query-85705fc3d7f8ba5f1b12d5fb222c38fc64deb1226aab9dc3bc4465324fce37d1.json new file mode 100644 index 0000000000..bce7324fb6 --- /dev/null +++ b/backend/.sqlx/query-85705fc3d7f8ba5f1b12d5fb222c38fc64deb1226aab9dc3bc4465324fce37d1.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO v2_job_queue (id, workspace_id, scheduled_for, tag) SELECT unnest($1::uuid[]), $2, now(), $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "UuidArray", + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "85705fc3d7f8ba5f1b12d5fb222c38fc64deb1226aab9dc3bc4465324fce37d1" +} diff --git a/backend/.sqlx/query-867d5c75ddc6c5d20136880c7294844b4c1a38701190795a801fa43c74a0beeb.json b/backend/.sqlx/query-867d5c75ddc6c5d20136880c7294844b4c1a38701190795a801fa43c74a0beeb.json new file mode 100644 index 0000000000..9232baf39c --- /dev/null +++ b/backend/.sqlx/query-867d5c75ddc6c5d20136880c7294844b4c1a38701190795a801fa43c74a0beeb.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT workflow_as_code_status FROM v2_job_completed WHERE id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workflow_as_code_status", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + true + ] + }, + "hash": "867d5c75ddc6c5d20136880c7294844b4c1a38701190795a801fa43c74a0beeb" +} diff --git a/backend/.sqlx/query-86cc1e3c18e936a700d8842a51a6f8377ec190669e2e22f8d511871d6fbe07b8.json b/backend/.sqlx/query-86cc1e3c18e936a700d8842a51a6f8377ec190669e2e22f8d511871d6fbe07b8.json new file mode 100644 index 0000000000..47b30f2895 --- /dev/null +++ b/backend/.sqlx/query-86cc1e3c18e936a700d8842a51a6f8377ec190669e2e22f8d511871d6fbe07b8.json @@ -0,0 +1,38 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n success AS \"success!\",\n result AS \"result: Json>\",\n started_at AS \"started_at!\"\n FROM v2_as_completed_job\n WHERE workspace_id = $1 AND schedule_path = $2 AND script_path = $3 AND id != $4\n ORDER BY created_at DESC\n LIMIT $5", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "success!", + "type_info": "Bool" + }, + { + "ordinal": 1, + "name": "result: Json>", + "type_info": "Jsonb" + }, + { + "ordinal": 2, + "name": "started_at!", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Uuid", + "Int8" + ] + }, + "nullable": [ + true, + true, + true + ] + }, + "hash": "86cc1e3c18e936a700d8842a51a6f8377ec190669e2e22f8d511871d6fbe07b8" +} diff --git a/backend/.sqlx/query-870d7feec169f70d140b7477562315ed2fd4662975bf0d01cfbffac86a959368.json b/backend/.sqlx/query-870d7feec169f70d140b7477562315ed2fd4662975bf0d01cfbffac86a959368.json deleted file mode 100644 index f5c36ed27f..0000000000 --- a/backend/.sqlx/query-870d7feec169f70d140b7477562315ed2fd4662975bf0d01cfbffac86a959368.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE queue\n SET flow_status = JSONB_SET(\n JSONB_SET(flow_status, ARRAY['modules', $1::TEXT], $2), ARRAY['step'], $3)\n WHERE id = $4", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Jsonb", - "Jsonb", - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "870d7feec169f70d140b7477562315ed2fd4662975bf0d01cfbffac86a959368" -} diff --git a/backend/.sqlx/query-87cd382066f3d8a89da5deaa592c1d1bb1978ff51c3e061674fe6db32b45eaf1.json b/backend/.sqlx/query-87cd382066f3d8a89da5deaa592c1d1bb1978ff51c3e061674fe6db32b45eaf1.json new file mode 100644 index 0000000000..7af7170f7e --- /dev/null +++ b/backend/.sqlx/query-87cd382066f3d8a89da5deaa592c1d1bb1978ff51c3e061674fe6db32b45eaf1.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO v2_job_status (id, flow_status) VALUES ($1, $2)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "87cd382066f3d8a89da5deaa592c1d1bb1978ff51c3e061674fe6db32b45eaf1" +} diff --git a/backend/.sqlx/query-8929150cb9262623eb12c908cb96c4d7e8cac594900ab0956c098e1cfa75e2f0.json b/backend/.sqlx/query-8929150cb9262623eb12c908cb96c4d7e8cac594900ab0956c098e1cfa75e2f0.json deleted file mode 100644 index 9e66b2ae2c..0000000000 --- a/backend/.sqlx/query-8929150cb9262623eb12c908cb96c4d7e8cac594900ab0956c098e1cfa75e2f0.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE queue\n SET leaf_jobs = JSONB_SET(coalesce(leaf_jobs, '{}'::jsonb), ARRAY[$1::TEXT], $2)\n WHERE COALESCE((SELECT root_job FROM queue WHERE id = $3), $3) = id", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Jsonb", - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "8929150cb9262623eb12c908cb96c4d7e8cac594900ab0956c098e1cfa75e2f0" -} diff --git a/backend/.sqlx/query-7113d7cc72e44e4b7e01b69cc18cbe7b0399cf8ec0e9e6d2b05ceef589c432df.json b/backend/.sqlx/query-89c08575afb31b70984f6b2b7dd4297af93b5b83ffdfb3ec91eba5df7ad3fd95.json similarity index 80% rename from backend/.sqlx/query-7113d7cc72e44e4b7e01b69cc18cbe7b0399cf8ec0e9e6d2b05ceef589c432df.json rename to backend/.sqlx/query-89c08575afb31b70984f6b2b7dd4297af93b5b83ffdfb3ec91eba5df7ad3fd95.json index da3503cf35..a463eafdb7 100644 --- a/backend/.sqlx/query-7113d7cc72e44e4b7e01b69cc18cbe7b0399cf8ec0e9e6d2b05ceef589c432df.json +++ b/backend/.sqlx/query-89c08575afb31b70984f6b2b7dd4297af93b5b83ffdfb3ec91eba5df7ad3fd95.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "UPDATE http_trigger SET script_path = $1, path = $2, is_flow = $3, http_method = $4, static_asset_config = $5, edited_by = $6, email = $7, is_async = $8, requires_auth = $9, edited_at = now() \n WHERE workspace_id = $10 AND path = $11", + "query": "UPDATE http_trigger SET script_path = $1, path = $2, is_flow = $3, http_method = $4, static_asset_config = $5, edited_by = $6, email = $7, is_async = $8, requires_auth = $9, edited_at = now(), is_static_website = $10\n WHERE workspace_id = $11 AND path = $12", "describe": { "columns": [], "parameters": { @@ -27,11 +27,12 @@ "Varchar", "Bool", "Bool", + "Bool", "Text", "Text" ] }, "nullable": [] }, - "hash": "7113d7cc72e44e4b7e01b69cc18cbe7b0399cf8ec0e9e6d2b05ceef589c432df" + "hash": "89c08575afb31b70984f6b2b7dd4297af93b5b83ffdfb3ec91eba5df7ad3fd95" } diff --git a/backend/.sqlx/query-8a1c9119f6f4763f64597684dc7b312404e637212de1897e97999755a2e492fd.json b/backend/.sqlx/query-8a1c9119f6f4763f64597684dc7b312404e637212de1897e97999755a2e492fd.json new file mode 100644 index 0000000000..8a92be2af8 --- /dev/null +++ b/backend/.sqlx/query-8a1c9119f6f4763f64597684dc7b312404e637212de1897e97999755a2e492fd.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job SET\n args = (SELECT result FROM v2_job_completed WHERE id = $1),\n preprocessed = TRUE\n WHERE id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "8a1c9119f6f4763f64597684dc7b312404e637212de1897e97999755a2e492fd" +} diff --git a/backend/.sqlx/query-8a44ca0cfe1e154138cbbf5ebddebbba2e1149d2e329d1d89eecc1c008c93a31.json b/backend/.sqlx/query-8a44ca0cfe1e154138cbbf5ebddebbba2e1149d2e329d1d89eecc1c008c93a31.json deleted file mode 100644 index fdfaa0c029..0000000000 --- a/backend/.sqlx/query-8a44ca0cfe1e154138cbbf5ebddebbba2e1149d2e329d1d89eecc1c008c93a31.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE completed_job\n SET logs = '##DELETED##', args = '{}'::jsonb, result = '{}'::jsonb\n WHERE id = $1", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "8a44ca0cfe1e154138cbbf5ebddebbba2e1149d2e329d1d89eecc1c008c93a31" -} diff --git a/backend/.sqlx/query-8be1ddb20ffd8c375b7d1ecb14bdb3a7c2f0c8f9308946b9262e14b8c584dd99.json b/backend/.sqlx/query-8be1ddb20ffd8c375b7d1ecb14bdb3a7c2f0c8f9308946b9262e14b8c584dd99.json new file mode 100644 index 0000000000..60ca066b36 --- /dev/null +++ b/backend/.sqlx/query-8be1ddb20ffd8c375b7d1ecb14bdb3a7c2f0c8f9308946b9262e14b8c584dd99.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status\n SET flow_status = flow_status - 'retry'\n WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "8be1ddb20ffd8c375b7d1ecb14bdb3a7c2f0c8f9308946b9262e14b8c584dd99" +} diff --git a/backend/.sqlx/query-8be277b89102a26dda506202a3ef7eb05342cfb3aa9b4f5d80c70fbc50d437ba.json b/backend/.sqlx/query-8be277b89102a26dda506202a3ef7eb05342cfb3aa9b4f5d80c70fbc50d437ba.json new file mode 100644 index 0000000000..e5819f5c6a --- /dev/null +++ b/backend/.sqlx/query-8be277b89102a26dda506202a3ef7eb05342cfb3aa9b4f5d80c70fbc50d437ba.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "create index concurrently if not exists ix_job_workspace_id_created_at_new_6 ON v2_job (workspace_id, created_at DESC) where kind in ('script', 'flow') AND parent_job IS NULL", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "8be277b89102a26dda506202a3ef7eb05342cfb3aa9b4f5d80c70fbc50d437ba" +} diff --git a/backend/.sqlx/query-8cb755510f2cfb23bdd0d1cf66b69949549a44855529f77a530f681a6e714646.json b/backend/.sqlx/query-8cb755510f2cfb23bdd0d1cf66b69949549a44855529f77a530f681a6e714646.json new file mode 100644 index 0000000000..82b70ebe92 --- /dev/null +++ b/backend/.sqlx/query-8cb755510f2cfb23bdd0d1cf66b69949549a44855529f77a530f681a6e714646.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status SET\n flow_status = JSONB_SET(\n flow_status,\n ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT],\n $4\n )\n WHERE id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Uuid", + "Text", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "8cb755510f2cfb23bdd0d1cf66b69949549a44855529f77a530f681a6e714646" +} diff --git a/backend/.sqlx/query-8d4235984f27d8b939ffd5c660d5b57dc38dd3b2643361ed3b7cdcd1534d2e21.json b/backend/.sqlx/query-8d4235984f27d8b939ffd5c660d5b57dc38dd3b2643361ed3b7cdcd1534d2e21.json deleted file mode 100644 index 824deba7fa..0000000000 --- a/backend/.sqlx/query-8d4235984f27d8b939ffd5c660d5b57dc38dd3b2643361ed3b7cdcd1534d2e21.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM queue WHERE schedule_path = $1 AND running = false AND workspace_id = $2 AND is_flow_step = false", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "8d4235984f27d8b939ffd5c660d5b57dc38dd3b2643361ed3b7cdcd1534d2e21" -} diff --git a/backend/.sqlx/query-8f3ed45a0290cd9989f40f34775de5e8c3762597e6f55f8b9575a54ccc31e085.json b/backend/.sqlx/query-8f3ed45a0290cd9989f40f34775de5e8c3762597e6f55f8b9575a54ccc31e085.json new file mode 100644 index 0000000000..4fd9cd8a4f --- /dev/null +++ b/backend/.sqlx/query-8f3ed45a0290cd9989f40f34775de5e8c3762597e6f55f8b9575a54ccc31e085.json @@ -0,0 +1,25 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status SET\n flow_status = JSONB_SET(\n JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4),\n ARRAY['modules', $1::TEXT, 'branchall', 'branch'],\n ((flow_status->'modules'->$1::int->'branchall'->>'branch')::int + 1)::text::jsonb\n )\n WHERE id = $2\n RETURNING (flow_status->'modules'->$1::int->'branchall'->>'branch')::int", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "int4", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Int4", + "Uuid", + "Text", + "Jsonb" + ] + }, + "nullable": [ + null + ] + }, + "hash": "8f3ed45a0290cd9989f40f34775de5e8c3762597e6f55f8b9575a54ccc31e085" +} diff --git a/backend/.sqlx/query-28a389a93a3d2472b13d956ec55eb357c6147e186e00e371f7374d166903ef64.json b/backend/.sqlx/query-8ff25d890d3f7019c5c6f2b47f29f4c7b2c91094036d9b5037ef9a4ac986ee53.json similarity index 82% rename from backend/.sqlx/query-28a389a93a3d2472b13d956ec55eb357c6147e186e00e371f7374d166903ef64.json rename to backend/.sqlx/query-8ff25d890d3f7019c5c6f2b47f29f4c7b2c91094036d9b5037ef9a4ac986ee53.json index a8a68c9237..de933d99e1 100644 --- a/backend/.sqlx/query-28a389a93a3d2472b13d956ec55eb357c6147e186e00e371f7374d166903ef64.json +++ b/backend/.sqlx/query-8ff25d890d3f7019c5c6f2b47f29f4c7b2c91094036d9b5037ef9a4ac986ee53.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT path, script_path, is_flow, route_path, workspace_id, is_async, requires_auth, edited_by, email, static_asset_config as \"static_asset_config: _\" FROM http_trigger WHERE http_method = $1", + "query": "SELECT path, script_path, is_flow, route_path, workspace_id, is_async, requires_auth, edited_by, email, static_asset_config as \"static_asset_config: _\", is_static_website FROM http_trigger WHERE workspace_id = $1 AND http_method = $2", "describe": { "columns": [ { @@ -52,10 +52,16 @@ "ordinal": 9, "name": "static_asset_config: _", "type_info": "Jsonb" + }, + { + "ordinal": 10, + "name": "is_static_website", + "type_info": "Bool" } ], "parameters": { "Left": [ + "Text", { "Custom": { "name": "http_method", @@ -82,8 +88,9 @@ false, false, false, - true + true, + false ] }, - "hash": "28a389a93a3d2472b13d956ec55eb357c6147e186e00e371f7374d166903ef64" + "hash": "8ff25d890d3f7019c5c6f2b47f29f4c7b2c91094036d9b5037ef9a4ac986ee53" } diff --git a/backend/.sqlx/query-bf282f7ed77ed09bb44b5e253725a15acefac87bedb2725827938c54a3c0e8f8.json b/backend/.sqlx/query-902a16d9e7ac34e7f1a0ad633bae754a0d64bc934a2b6434b9cb67f28d3ef950.json similarity index 51% rename from backend/.sqlx/query-bf282f7ed77ed09bb44b5e253725a15acefac87bedb2725827938c54a3c0e8f8.json rename to backend/.sqlx/query-902a16d9e7ac34e7f1a0ad633bae754a0d64bc934a2b6434b9cb67f28d3ef950.json index 356aa78547..7a6d47ba81 100644 --- a/backend/.sqlx/query-bf282f7ed77ed09bb44b5e253725a15acefac87bedb2725827938c54a3c0e8f8.json +++ b/backend/.sqlx/query-902a16d9e7ac34e7f1a0ad633bae754a0d64bc934a2b6434b9cb67f28d3ef950.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "UPDATE queue SET canceled_by = $1 WHERE canceled_by = $2 AND workspace_id = $3", + "query": "UPDATE v2_job_queue SET canceled_by = $1 WHERE canceled_by = $2 AND workspace_id = $3", "describe": { "columns": [], "parameters": { @@ -12,5 +12,5 @@ }, "nullable": [] }, - "hash": "bf282f7ed77ed09bb44b5e253725a15acefac87bedb2725827938c54a3c0e8f8" + "hash": "902a16d9e7ac34e7f1a0ad633bae754a0d64bc934a2b6434b9cb67f28d3ef950" } diff --git a/backend/.sqlx/query-90635149190c59396ca557bf1670554a1e40d0ce9cc686ad09adca0904324cd8.json b/backend/.sqlx/query-90635149190c59396ca557bf1670554a1e40d0ce9cc686ad09adca0904324cd8.json new file mode 100644 index 0000000000..9dce5f08b4 --- /dev/null +++ b/backend/.sqlx/query-90635149190c59396ca557bf1670554a1e40d0ce9cc686ad09adca0904324cd8.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_queue q SET suspend = 0\n FROM v2_job j, v2_job_status f\n WHERE parent_job = $1\n AND f.id = j.id AND q.id = j.id\n AND suspend = $2 AND (f.flow_status->'step')::int = 0", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Int4" + ] + }, + "nullable": [] + }, + "hash": "90635149190c59396ca557bf1670554a1e40d0ce9cc686ad09adca0904324cd8" +} diff --git a/backend/.sqlx/query-c3b5abbf2c9079d597a55f7c63bc83b8b4da98bda204a40f045a62172cfb4ebb.json b/backend/.sqlx/query-90fbb9430ab03ce3aadd95cc263e5a3d1a91ea02de7608676575e1c03023ed71.json similarity index 59% rename from backend/.sqlx/query-c3b5abbf2c9079d597a55f7c63bc83b8b4da98bda204a40f045a62172cfb4ebb.json rename to backend/.sqlx/query-90fbb9430ab03ce3aadd95cc263e5a3d1a91ea02de7608676575e1c03023ed71.json index b545c48599..61598c0b2d 100644 --- a/backend/.sqlx/query-c3b5abbf2c9079d597a55f7c63bc83b8b4da98bda204a40f045a62172cfb4ebb.json +++ b/backend/.sqlx/query-90fbb9430ab03ce3aadd95cc263e5a3d1a91ea02de7608676575e1c03023ed71.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT tag as \"tag!\", COUNT(*) as \"count!\"\n FROM completed_job\n WHERE started_at > NOW() - make_interval(secs => $1) AND ($2::text IS NULL OR workspace_id = $2)\n GROUP BY tag\n ORDER BY \"count!\" DESC\n ", + "query": "\n SELECT tag as \"tag!\", COUNT(*) as \"count!\"\n FROM v2_as_completed_job\n WHERE started_at > NOW() - make_interval(secs => $1) AND ($2::text IS NULL OR workspace_id = $2)\n GROUP BY tag\n ORDER BY \"count!\" DESC\n ", "describe": { "columns": [ { @@ -21,9 +21,9 @@ ] }, "nullable": [ - false, + true, null ] }, - "hash": "c3b5abbf2c9079d597a55f7c63bc83b8b4da98bda204a40f045a62172cfb4ebb" + "hash": "90fbb9430ab03ce3aadd95cc263e5a3d1a91ea02de7608676575e1c03023ed71" } diff --git a/backend/.sqlx/query-0331a81262e2d3c1bcfaeb64617b11eb68ab4599d64e5e5af639a9ac5d791fd0.json b/backend/.sqlx/query-91878f06c6e27d864bd50d8cd4adde4395ab6ef1181237a886fa398dcfa0b589.json similarity index 67% rename from backend/.sqlx/query-0331a81262e2d3c1bcfaeb64617b11eb68ab4599d64e5e5af639a9ac5d791fd0.json rename to backend/.sqlx/query-91878f06c6e27d864bd50d8cd4adde4395ab6ef1181237a886fa398dcfa0b589.json index 5d0a70183e..91492c037b 100644 --- a/backend/.sqlx/query-0331a81262e2d3c1bcfaeb64617b11eb68ab4599d64e5e5af639a9ac5d791fd0.json +++ b/backend/.sqlx/query-91878f06c6e27d864bd50d8cd4adde4395ab6ef1181237a886fa398dcfa0b589.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n -- slack_team_id, \n -- slack_name, \n -- slack_command_script, \n -- CASE WHEN slack_email = 'missing@email.xyz' THEN NULL ELSE slack_email END AS slack_email,\n auto_invite_domain IS NOT NULL AS \"auto_invite_enabled!\",\n CASE WHEN auto_invite_operator IS TRUE THEN 'operator' ELSE 'developer' END AS \"auto_invite_as!\", \n CASE WHEN auto_add IS TRUE THEN 'add' ELSE 'invite' END AS \"auto_invite_mode!\", \n webhook, \n deploy_to, \n error_handler, \n ai_resource, \n code_completion_enabled, \n error_handler_extra_args, \n error_handler_muted_on_cancel, \n large_file_storage, \n git_sync,\n default_app,\n default_scripts,\n workspace.name\n FROM workspace_settings\n LEFT JOIN workspace ON workspace.id = workspace_settings.workspace_id\n WHERE workspace_id = $1", + "query": "SELECT\n -- slack_team_id, \n -- slack_name, \n -- slack_command_script, \n -- CASE WHEN slack_email = 'missing@email.xyz' THEN NULL ELSE slack_email END AS slack_email,\n auto_invite_domain IS NOT NULL AS \"auto_invite_enabled!\",\n CASE WHEN auto_invite_operator IS TRUE THEN 'operator' ELSE 'developer' END AS \"auto_invite_as!\", \n CASE WHEN auto_add IS TRUE THEN 'add' ELSE 'invite' END AS \"auto_invite_mode!\", \n webhook, \n deploy_to, \n error_handler, \n ai_resource, \n ai_models,\n code_completion_model,\n error_handler_extra_args, \n error_handler_muted_on_cancel, \n large_file_storage, \n git_sync,\n default_app,\n default_scripts,\n workspace.name,\n mute_critical_alerts,\n color,\n operator_settings\n FROM workspace_settings\n LEFT JOIN workspace ON workspace.id = workspace_settings.workspace_id\n WHERE workspace_id = $1", "describe": { "columns": [ { @@ -40,43 +40,63 @@ }, { "ordinal": 7, - "name": "code_completion_enabled", - "type_info": "Bool" + "name": "ai_models", + "type_info": "VarcharArray" }, { "ordinal": 8, + "name": "code_completion_model", + "type_info": "Varchar" + }, + { + "ordinal": 9, "name": "error_handler_extra_args", "type_info": "Json" }, { - "ordinal": 9, + "ordinal": 10, "name": "error_handler_muted_on_cancel", "type_info": "Bool" }, { - "ordinal": 10, + "ordinal": 11, "name": "large_file_storage", "type_info": "Jsonb" }, { - "ordinal": 11, + "ordinal": 12, "name": "git_sync", "type_info": "Jsonb" }, { - "ordinal": 12, + "ordinal": 13, "name": "default_app", "type_info": "Varchar" }, { - "ordinal": 13, + "ordinal": 14, "name": "default_scripts", "type_info": "Jsonb" }, { - "ordinal": 14, + "ordinal": 15, "name": "name", "type_info": "Varchar" + }, + { + "ordinal": 16, + "name": "mute_critical_alerts", + "type_info": "Bool" + }, + { + "ordinal": 17, + "name": "color", + "type_info": "Varchar" + }, + { + "ordinal": 18, + "name": "operator_settings", + "type_info": "Jsonb" } ], "parameters": { @@ -94,13 +114,17 @@ true, false, true, + true, false, true, true, true, true, - false + false, + true, + true, + true ] }, - "hash": "0331a81262e2d3c1bcfaeb64617b11eb68ab4599d64e5e5af639a9ac5d791fd0" + "hash": "91878f06c6e27d864bd50d8cd4adde4395ab6ef1181237a886fa398dcfa0b589" } diff --git a/backend/.sqlx/query-931703a98d2ee5fb58d3380896baaee032e731db1e6bd49d991a54f49ab8fa46.json b/backend/.sqlx/query-931703a98d2ee5fb58d3380896baaee032e731db1e6bd49d991a54f49ab8fa46.json new file mode 100644 index 0000000000..5f86a45011 --- /dev/null +++ b/backend/.sqlx/query-931703a98d2ee5fb58d3380896baaee032e731db1e6bd49d991a54f49ab8fa46.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "create index concurrently if not exists ix_completed_job_workspace_id_started_at_new_2 ON v2_job_completed (workspace_id, started_at DESC)", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "931703a98d2ee5fb58d3380896baaee032e731db1e6bd49d991a54f49ab8fa46" +} diff --git a/backend/.sqlx/query-9422431d79de41518f651ef24e86819d7b6a2f5531740a7deea1b51775335977.json b/backend/.sqlx/query-9422431d79de41518f651ef24e86819d7b6a2f5531740a7deea1b51775335977.json deleted file mode 100644 index 7f2936f195..0000000000 --- a/backend/.sqlx/query-9422431d79de41518f651ef24e86819d7b6a2f5531740a7deea1b51775335977.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT scalar_int FROM job_stats WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "scalar_int", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Text", - "Uuid", - "Text" - ] - }, - "nullable": [ - true - ] - }, - "hash": "9422431d79de41518f651ef24e86819d7b6a2f5531740a7deea1b51775335977" -} diff --git a/backend/.sqlx/query-6682bf34caf7efa95b60c747b45dcdd41de7f8e197b163f7865810391471db5b.json b/backend/.sqlx/query-95ae90094ec0e2c22660cc2e3788b22231dab9c558723cc54894597ce4cd3d5a.json similarity index 52% rename from backend/.sqlx/query-6682bf34caf7efa95b60c747b45dcdd41de7f8e197b163f7865810391471db5b.json rename to backend/.sqlx/query-95ae90094ec0e2c22660cc2e3788b22231dab9c558723cc54894597ce4cd3d5a.json index 86968663f2..d451727f46 100644 --- a/backend/.sqlx/query-6682bf34caf7efa95b60c747b45dcdd41de7f8e197b163f7865810391471db5b.json +++ b/backend/.sqlx/query-95ae90094ec0e2c22660cc2e3788b22231dab9c558723cc54894597ce4cd3d5a.json @@ -1,11 +1,11 @@ { "db_name": "PostgreSQL", - "query": "SELECT created_by, CONCAT(coalesce(queue.logs, ''), coalesce(job_logs.logs, '')) as logs, coalesce(job_logs.log_offset, 0) as log_offset, job_logs.log_file_index\n FROM queue \n LEFT JOIN job_logs ON job_logs.job_id = queue.id \n WHERE queue.id = $1 AND queue.workspace_id = $2 AND ($3::text[] IS NULL OR queue.tag = ANY($3))", + "query": "SELECT created_by AS \"created_by!\", CONCAT(coalesce(v2_as_queue.logs, ''), coalesce(job_logs.logs, '')) as logs, coalesce(job_logs.log_offset, 0) as log_offset, job_logs.log_file_index\n FROM v2_as_queue \n LEFT JOIN job_logs ON job_logs.job_id = v2_as_queue.id \n WHERE v2_as_queue.id = $1 AND v2_as_queue.workspace_id = $2 AND ($3::text[] IS NULL OR v2_as_queue.tag = ANY($3))", "describe": { "columns": [ { "ordinal": 0, - "name": "created_by", + "name": "created_by!", "type_info": "Varchar" }, { @@ -32,11 +32,11 @@ ] }, "nullable": [ - false, + true, null, null, true ] }, - "hash": "6682bf34caf7efa95b60c747b45dcdd41de7f8e197b163f7865810391471db5b" + "hash": "95ae90094ec0e2c22660cc2e3788b22231dab9c558723cc54894597ce4cd3d5a" } diff --git a/backend/.sqlx/query-976295cc04007f3cf5d8ba8d3fea665692a2192ee664216bdab04e6d2547422f.json b/backend/.sqlx/query-976295cc04007f3cf5d8ba8d3fea665692a2192ee664216bdab04e6d2547422f.json new file mode 100644 index 0000000000..b32af76e07 --- /dev/null +++ b/backend/.sqlx/query-976295cc04007f3cf5d8ba8d3fea665692a2192ee664216bdab04e6d2547422f.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM _sqlx_migrations WHERE version=20250201145630 OR version=20250201145631", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "976295cc04007f3cf5d8ba8d3fea665692a2192ee664216bdab04e6d2547422f" +} diff --git a/backend/.sqlx/query-971175f6169857c3e1cdc08ac8aeed57300b7792e1797a9cdd73c9b3967cd7b9.json b/backend/.sqlx/query-99f74bf675120daf965e063e5eaff808ba646f4f99d0c8837097e747b481f03a.json similarity index 57% rename from backend/.sqlx/query-971175f6169857c3e1cdc08ac8aeed57300b7792e1797a9cdd73c9b3967cd7b9.json rename to backend/.sqlx/query-99f74bf675120daf965e063e5eaff808ba646f4f99d0c8837097e747b481f03a.json index f389cb6432..68982a2183 100644 --- a/backend/.sqlx/query-971175f6169857c3e1cdc08ac8aeed57300b7792e1797a9cdd73c9b3967cd7b9.json +++ b/backend/.sqlx/query-99f74bf675120daf965e063e5eaff808ba646f4f99d0c8837097e747b481f03a.json @@ -1,11 +1,11 @@ { "db_name": "PostgreSQL", - "query": "SELECT root_job FROM queue WHERE id = $1", + "query": "SELECT id FROM v2_job WHERE parent_job = $1", "describe": { "columns": [ { "ordinal": 0, - "name": "root_job", + "name": "id", "type_info": "Uuid" } ], @@ -15,8 +15,8 @@ ] }, "nullable": [ - true + false ] }, - "hash": "971175f6169857c3e1cdc08ac8aeed57300b7792e1797a9cdd73c9b3967cd7b9" + "hash": "99f74bf675120daf965e063e5eaff808ba646f4f99d0c8837097e747b481f03a" } diff --git a/backend/.sqlx/query-9a9b639611459659ae355a43f219c6da2c3d1e04d49306adaaaaf06e54ee8357.json b/backend/.sqlx/query-9a9b639611459659ae355a43f219c6da2c3d1e04d49306adaaaaf06e54ee8357.json deleted file mode 100644 index 8ae3d80ca6..0000000000 --- a/backend/.sqlx/query-9a9b639611459659ae355a43f219c6da2c3d1e04d49306adaaaaf06e54ee8357.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH zombie_jobs AS (\n UPDATE queue SET running = false, started_at = null\n WHERE last_ping < now() - ($1 || ' seconds')::interval\n AND running = true AND job_kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlescriptflow') AND same_worker = false \n RETURNING id, workspace_id, last_ping\n ),\n update_concurrency AS (\n UPDATE concurrency_counter cc\n SET job_uuids = job_uuids - zj.id::text\n FROM zombie_jobs zj\n INNER JOIN concurrency_key ck ON ck.job_id = zj.id\n WHERE cc.concurrency_id = ck.key\n )\n SELECT id, workspace_id, last_ping FROM zombie_jobs", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Uuid" - }, - { - "ordinal": 1, - "name": "workspace_id", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "last_ping", - "type_info": "Timestamptz" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - false, - false, - true - ] - }, - "hash": "9a9b639611459659ae355a43f219c6da2c3d1e04d49306adaaaaf06e54ee8357" -} diff --git a/backend/.sqlx/query-9b2c42327378963b16eec5037537b8d9ae95c194b201bd1ce1d8f90925d7dba8.json b/backend/.sqlx/query-9b2c42327378963b16eec5037537b8d9ae95c194b201bd1ce1d8f90925d7dba8.json new file mode 100644 index 0000000000..e73b49d4b0 --- /dev/null +++ b/backend/.sqlx/query-9b2c42327378963b16eec5037537b8d9ae95c194b201bd1ce1d8f90925d7dba8.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job SET permissioned_as = ('u/' || $1) WHERE permissioned_as = ('u/' || $2) AND workspace_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "9b2c42327378963b16eec5037537b8d9ae95c194b201bd1ce1d8f90925d7dba8" +} diff --git a/backend/.sqlx/query-9b716307fdbd479879224ba60b77eeb7e8487b02b13710c68be99057e2c32cb9.json b/backend/.sqlx/query-9b716307fdbd479879224ba60b77eeb7e8487b02b13710c68be99057e2c32cb9.json deleted file mode 100644 index 49663a5013..0000000000 --- a/backend/.sqlx/query-9b716307fdbd479879224ba60b77eeb7e8487b02b13710c68be99057e2c32cb9.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE queue SET flow_status = jsonb_set(jsonb_set(COALESCE(flow_status, '{}'::jsonb), array[$1], COALESCE(flow_status->$1, '{}'::jsonb)), array[$1, 'started_at'], to_jsonb(now()::text)) WHERE id = $2 AND workspace_id = $3", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Uuid", - "Text" - ] - }, - "nullable": [] - }, - "hash": "9b716307fdbd479879224ba60b77eeb7e8487b02b13710c68be99057e2c32cb9" -} diff --git a/backend/.sqlx/query-9c17ddca92e0a93051b36bb688ad6c1c24108e45c94be52595ca6cb82135f4eb.json b/backend/.sqlx/query-9c17ddca92e0a93051b36bb688ad6c1c24108e45c94be52595ca6cb82135f4eb.json deleted file mode 100644 index 5de165462e..0000000000 --- a/backend/.sqlx/query-9c17ddca92e0a93051b36bb688ad6c1c24108e45c94be52595ca6cb82135f4eb.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE queue SET args = (select result FROM completed_job WHERE id = $1) WHERE id = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid", - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "9c17ddca92e0a93051b36bb688ad6c1c24108e45c94be52595ca6cb82135f4eb" -} diff --git a/backend/.sqlx/query-293054a4d6a2b00228a5029a81c026e8891576c7e87cdab41d7fcc3afc32d583.json b/backend/.sqlx/query-9c19ad9ab14325587d662539c04e18e8dfbdb0bf1dd4c0dc07a55f4eeb4eb5f8.json similarity index 50% rename from backend/.sqlx/query-293054a4d6a2b00228a5029a81c026e8891576c7e87cdab41d7fcc3afc32d583.json rename to backend/.sqlx/query-9c19ad9ab14325587d662539c04e18e8dfbdb0bf1dd4c0dc07a55f4eeb4eb5f8.json index 291b3f3601..89368ea0c8 100644 --- a/backend/.sqlx/query-293054a4d6a2b00228a5029a81c026e8891576c7e87cdab41d7fcc3afc32d583.json +++ b/backend/.sqlx/query-9c19ad9ab14325587d662539c04e18e8dfbdb0bf1dd4c0dc07a55f4eeb4eb5f8.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT COUNT(*) FROM queue LEFT JOIN concurrency_key ON concurrency_key.job_id = queue.id\n WHERE key = $1 AND running = false AND canceled = false AND scheduled_for >= $2 AND scheduled_for < $3", + "query": "SELECT COUNT(*) FROM v2_as_queue LEFT JOIN concurrency_key ON concurrency_key.job_id = v2_as_queue.id\n WHERE key = $1 AND running = false AND canceled = false AND scheduled_for >= $2 AND scheduled_for < $3", "describe": { "columns": [ { @@ -20,5 +20,5 @@ null ] }, - "hash": "293054a4d6a2b00228a5029a81c026e8891576c7e87cdab41d7fcc3afc32d583" + "hash": "9c19ad9ab14325587d662539c04e18e8dfbdb0bf1dd4c0dc07a55f4eeb4eb5f8" } diff --git a/backend/.sqlx/query-9c7b2d9708a65764d91bf7f8a2cb9e59aba53d8e17f1d8a887128b7863c3924d.json b/backend/.sqlx/query-9c7b2d9708a65764d91bf7f8a2cb9e59aba53d8e17f1d8a887128b7863c3924d.json new file mode 100644 index 0000000000..b9429cbab7 --- /dev/null +++ b/backend/.sqlx/query-9c7b2d9708a65764d91bf7f8a2cb9e59aba53d8e17f1d8a887128b7863c3924d.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job SET args = '{}'::jsonb WHERE id = ANY($1)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "UuidArray" + ] + }, + "nullable": [] + }, + "hash": "9c7b2d9708a65764d91bf7f8a2cb9e59aba53d8e17f1d8a887128b7863c3924d" +} diff --git a/backend/.sqlx/query-16be0560028361d46bf3b842a5fa07472994d8942c684f5b75339fe71ea23cdd.json b/backend/.sqlx/query-9d518842a9ad90ff9c28dc39690deb0ee6b62cf1d8ae1a02b28c23255d377b3d.json similarity index 51% rename from backend/.sqlx/query-16be0560028361d46bf3b842a5fa07472994d8942c684f5b75339fe71ea23cdd.json rename to backend/.sqlx/query-9d518842a9ad90ff9c28dc39690deb0ee6b62cf1d8ae1a02b28c23255d377b3d.json index d273132682..79bd8d0666 100644 --- a/backend/.sqlx/query-16be0560028361d46bf3b842a5fa07472994d8942c684f5b75339fe71ea23cdd.json +++ b/backend/.sqlx/query-9d518842a9ad90ff9c28dc39690deb0ee6b62cf1d8ae1a02b28c23255d377b3d.json @@ -1,12 +1,12 @@ { "db_name": "PostgreSQL", - "query": "SELECT script_path FROM completed_job WHERE id = $1", + "query": "SELECT labels FROM v2_job WHERE id = $1", "describe": { "columns": [ { "ordinal": 0, - "name": "script_path", - "type_info": "Varchar" + "name": "labels", + "type_info": "TextArray" } ], "parameters": { @@ -18,5 +18,5 @@ true ] }, - "hash": "16be0560028361d46bf3b842a5fa07472994d8942c684f5b75339fe71ea23cdd" + "hash": "9d518842a9ad90ff9c28dc39690deb0ee6b62cf1d8ae1a02b28c23255d377b3d" } diff --git a/backend/.sqlx/query-3ecb25b05d6c14b499f9b00af42ae74134728899f6b59c68b246979bc5143e30.json b/backend/.sqlx/query-9e2810312302b36d3b4d761481c00296ee84c9536228496e19b4ec5df1781bc5.json similarity index 67% rename from backend/.sqlx/query-3ecb25b05d6c14b499f9b00af42ae74134728899f6b59c68b246979bc5143e30.json rename to backend/.sqlx/query-9e2810312302b36d3b4d761481c00296ee84c9536228496e19b4ec5df1781bc5.json index a652b56bad..70df593afe 100644 --- a/backend/.sqlx/query-3ecb25b05d6c14b499f9b00af42ae74134728899f6b59c68b246979bc5143e30.json +++ b/backend/.sqlx/query-9e2810312302b36d3b4d761481c00296ee84c9536228496e19b4ec5df1781bc5.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT COUNT(*) as count, \n MIN(scheduled_for) as oldest_job\n FROM queue \n WHERE tag = $1 \n AND scheduled_for <= NOW() - $2::interval \n AND running = false\n ", + "query": "\n SELECT COUNT(*) as count, \n MIN(scheduled_for) as oldest_job\n FROM v2_job_queue \n WHERE tag = $1 \n AND scheduled_for <= NOW() - $2::interval \n AND running = false\n ", "describe": { "columns": [ { @@ -25,5 +25,5 @@ null ] }, - "hash": "3ecb25b05d6c14b499f9b00af42ae74134728899f6b59c68b246979bc5143e30" + "hash": "9e2810312302b36d3b4d761481c00296ee84c9536228496e19b4ec5df1781bc5" } diff --git a/backend/.sqlx/query-a241c56415759105ccbcbf7fff77287fa4ec2cc096c0060d14db421115d63e2d.json b/backend/.sqlx/query-9e7e6fe1dfba032e586f64531e12529d57faf5e77f6792d5bda608ff9658d7c9.json similarity index 60% rename from backend/.sqlx/query-a241c56415759105ccbcbf7fff77287fa4ec2cc096c0060d14db421115d63e2d.json rename to backend/.sqlx/query-9e7e6fe1dfba032e586f64531e12529d57faf5e77f6792d5bda608ff9658d7c9.json index 226f49f2fe..2ee7a1acef 100644 --- a/backend/.sqlx/query-a241c56415759105ccbcbf7fff77287fa4ec2cc096c0060d14db421115d63e2d.json +++ b/backend/.sqlx/query-9e7e6fe1dfba032e586f64531e12529d57faf5e77f6792d5bda608ff9658d7c9.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT EXISTS (SELECT 1 FROM queue WHERE workspace_id = $1 AND schedule_path = $2 AND scheduled_for = $3)", + "query": "SELECT EXISTS (SELECT 1 FROM v2_as_queue WHERE workspace_id = $1 AND schedule_path = $2 AND scheduled_for = $3)", "describe": { "columns": [ { @@ -20,5 +20,5 @@ null ] }, - "hash": "a241c56415759105ccbcbf7fff77287fa4ec2cc096c0060d14db421115d63e2d" + "hash": "9e7e6fe1dfba032e586f64531e12529d57faf5e77f6792d5bda608ff9658d7c9" } diff --git a/backend/.sqlx/query-a0b3e10e077d30c1da135dff9feca3761d400391f1f46a8294da3e6c9af63887.json b/backend/.sqlx/query-a0b3e10e077d30c1da135dff9feca3761d400391f1f46a8294da3e6c9af63887.json new file mode 100644 index 0000000000..1afc61978e --- /dev/null +++ b/backend/.sqlx/query-a0b3e10e077d30c1da135dff9feca3761d400391f1f46a8294da3e6c9af63887.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO v2_job_status (id, flow_status) SELECT unnest($1::uuid[]), $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "UuidArray", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "a0b3e10e077d30c1da135dff9feca3761d400391f1f46a8294da3e6c9af63887" +} diff --git a/backend/.sqlx/query-a0c35cb515a842067b294343c90f1bfbe4e2db85da9a478a07460733999e9beb.json b/backend/.sqlx/query-a0c35cb515a842067b294343c90f1bfbe4e2db85da9a478a07460733999e9beb.json new file mode 100644 index 0000000000..298a66c89c --- /dev/null +++ b/backend/.sqlx/query-a0c35cb515a842067b294343c90f1bfbe4e2db85da9a478a07460733999e9beb.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT count(*) AS \"count!\" FROM resume_job", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "a0c35cb515a842067b294343c90f1bfbe4e2db85da9a478a07460733999e9beb" +} diff --git a/backend/.sqlx/query-611e3cd49d38a37db8912397c4eddd7cd50a4782101e971175d0c5c798593a40.json b/backend/.sqlx/query-a21d501d3bda8b7cd04b9a75515222ee33c2cd863c8e3fde0527cee9be4e09e2.json similarity index 62% rename from backend/.sqlx/query-611e3cd49d38a37db8912397c4eddd7cd50a4782101e971175d0c5c798593a40.json rename to backend/.sqlx/query-a21d501d3bda8b7cd04b9a75515222ee33c2cd863c8e3fde0527cee9be4e09e2.json index 679b05d29d..7a65d688b5 100644 --- a/backend/.sqlx/query-611e3cd49d38a37db8912397c4eddd7cd50a4782101e971175d0c5c798593a40.json +++ b/backend/.sqlx/query-a21d501d3bda8b7cd04b9a75515222ee33c2cd863c8e3fde0527cee9be4e09e2.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT id FROM queue WHERE workspace_id = $1 and root_job = $2", + "query": "SELECT id FROM v2_job WHERE workspace_id = $1 and flow_innermost_root_job = $2", "describe": { "columns": [ { @@ -19,5 +19,5 @@ false ] }, - "hash": "611e3cd49d38a37db8912397c4eddd7cd50a4782101e971175d0c5c798593a40" + "hash": "a21d501d3bda8b7cd04b9a75515222ee33c2cd863c8e3fde0527cee9be4e09e2" } diff --git a/backend/.sqlx/query-a2b7d364705468bd9dfb8f10e0bcc151604feae83d4671dfb9ca77eb8fd8d4f1.json b/backend/.sqlx/query-a2b7d364705468bd9dfb8f10e0bcc151604feae83d4671dfb9ca77eb8fd8d4f1.json new file mode 100644 index 0000000000..fffd61d360 --- /dev/null +++ b/backend/.sqlx/query-a2b7d364705468bd9dfb8f10e0bcc151604feae83d4671dfb9ca77eb8fd8d4f1.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM v2_job_completed WHERE workspace_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "a2b7d364705468bd9dfb8f10e0bcc151604feae83d4671dfb9ca77eb8fd8d4f1" +} diff --git a/backend/.sqlx/query-a56eef5f5ecbe1a8d309ff65d9a8c456a3c165f7f2a107cf7fa6a4cdd30d55c0.json b/backend/.sqlx/query-a56eef5f5ecbe1a8d309ff65d9a8c456a3c165f7f2a107cf7fa6a4cdd30d55c0.json new file mode 100644 index 0000000000..eb852c75ee --- /dev/null +++ b/backend/.sqlx/query-a56eef5f5ecbe1a8d309ff65d9a8c456a3c165f7f2a107cf7fa6a4cdd30d55c0.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT preprocessed, script_entrypoint_override FROM v2_job WHERE id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "preprocessed", + "type_info": "Bool" + }, + { + "ordinal": 1, + "name": "script_entrypoint_override", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + true, + true + ] + }, + "hash": "a56eef5f5ecbe1a8d309ff65d9a8c456a3c165f7f2a107cf7fa6a4cdd30d55c0" +} diff --git a/backend/.sqlx/query-a79b1d0884c02f92fd40b23c6181717ffa5e01250020cfb24e4bad276397dee2.json b/backend/.sqlx/query-a79b1d0884c02f92fd40b23c6181717ffa5e01250020cfb24e4bad276397dee2.json new file mode 100644 index 0000000000..659dedc8cf --- /dev/null +++ b/backend/.sqlx/query-a79b1d0884c02f92fd40b23c6181717ffa5e01250020cfb24e4bad276397dee2.json @@ -0,0 +1,106 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT * FROM kafka_trigger\n WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "kafka_resource_path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "topics", + "type_info": "VarcharArray" + }, + { + "ordinal": 3, + "name": "group_id", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "script_path", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "is_flow", + "type_info": "Bool" + }, + { + "ordinal": 6, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "edited_by", + "type_info": "Varchar" + }, + { + "ordinal": 8, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 9, + "name": "edited_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 10, + "name": "extra_perms", + "type_info": "Jsonb" + }, + { + "ordinal": 11, + "name": "server_id", + "type_info": "Varchar" + }, + { + "ordinal": 12, + "name": "last_server_ping", + "type_info": "Timestamptz" + }, + { + "ordinal": 13, + "name": "error", + "type_info": "Text" + }, + { + "ordinal": 14, + "name": "enabled", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + true, + true, + true, + false + ] + }, + "hash": "a79b1d0884c02f92fd40b23c6181717ffa5e01250020cfb24e4bad276397dee2" +} diff --git a/backend/.sqlx/query-a810e7912b65e6d4b361c9d34aa4c34d17c8444699f7bf29cc6907cdac6106f3.json b/backend/.sqlx/query-a810e7912b65e6d4b361c9d34aa4c34d17c8444699f7bf29cc6907cdac6106f3.json new file mode 100644 index 0000000000..52c3039fa5 --- /dev/null +++ b/backend/.sqlx/query-a810e7912b65e6d4b361c9d34aa4c34d17c8444699f7bf29cc6907cdac6106f3.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO v2_job_status (id, flow_status)\n SELECT unnest($1::uuid[]), $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "UuidArray", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "a810e7912b65e6d4b361c9d34aa4c34d17c8444699f7bf29cc6907cdac6106f3" +} diff --git a/backend/.sqlx/query-a8ccf0f70cb8286e9dcafda5cbd542af647ca392cf6e307dc9358ed8b2ba640f.json b/backend/.sqlx/query-a8ccf0f70cb8286e9dcafda5cbd542af647ca392cf6e307dc9358ed8b2ba640f.json new file mode 100644 index 0000000000..b4ffae3d05 --- /dev/null +++ b/backend/.sqlx/query-a8ccf0f70cb8286e9dcafda5cbd542af647ca392cf6e307dc9358ed8b2ba640f.json @@ -0,0 +1,36 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n result AS \"result: sqlx::types::Json>\",\n result_columns,\n created_by AS \"created_by!\"\n FROM v2_job_completed c\n JOIN v2_job USING (id)\n WHERE c.id = $1 AND c.workspace_id = $2 AND ($3::text[] IS NULL OR tag = ANY($3))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "result: sqlx::types::Json>", + "type_info": "Jsonb" + }, + { + "ordinal": 1, + "name": "result_columns", + "type_info": "TextArray" + }, + { + "ordinal": 2, + "name": "created_by!", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "TextArray" + ] + }, + "nullable": [ + true, + true, + false + ] + }, + "hash": "a8ccf0f70cb8286e9dcafda5cbd542af647ca392cf6e307dc9358ed8b2ba640f" +} diff --git a/backend/.sqlx/query-16c4fb122ceb30722f54c92db97427b0ddd36f495d549f8cd23d659d2c866a63.json b/backend/.sqlx/query-a96ff22bc78b74d7234550a12d9fb5c555c1187b276f1353dae1b2ac0670a92a.json similarity index 73% rename from backend/.sqlx/query-16c4fb122ceb30722f54c92db97427b0ddd36f495d549f8cd23d659d2c866a63.json rename to backend/.sqlx/query-a96ff22bc78b74d7234550a12d9fb5c555c1187b276f1353dae1b2ac0670a92a.json index 00361e6e95..060d87a941 100644 --- a/backend/.sqlx/query-16c4fb122ceb30722f54c92db97427b0ddd36f495d549f8cd23d659d2c866a63.json +++ b/backend/.sqlx/query-a96ff22bc78b74d7234550a12d9fb5c555c1187b276f1353dae1b2ac0670a92a.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT EXISTS(SELECT 1 FROM http_trigger WHERE route_path_key = $1 AND http_method = $2)", + "query": "SELECT EXISTS(SELECT 1 FROM http_trigger WHERE route_path_key = $1 AND workspace_id = $2 AND http_method = $3 AND ($4::TEXT IS NULL OR path != $4))", "describe": { "columns": [ { @@ -11,6 +11,7 @@ ], "parameters": { "Left": [ + "Text", "Text", { "Custom": { @@ -25,12 +26,13 @@ ] } } - } + }, + "Text" ] }, "nullable": [ null ] }, - "hash": "16c4fb122ceb30722f54c92db97427b0ddd36f495d549f8cd23d659d2c866a63" + "hash": "a96ff22bc78b74d7234550a12d9fb5c555c1187b276f1353dae1b2ac0670a92a" } diff --git a/backend/.sqlx/query-a6feede7d9c3060b1e4c94bb4fa700d38a0d1b9d25429021c08fc94d57f952d2.json b/backend/.sqlx/query-aaacf51f3a43617b60e4253549ec4cc6a3648992b590c6e0121e4d7434da516b.json similarity index 53% rename from backend/.sqlx/query-a6feede7d9c3060b1e4c94bb4fa700d38a0d1b9d25429021c08fc94d57f952d2.json rename to backend/.sqlx/query-aaacf51f3a43617b60e4253549ec4cc6a3648992b590c6e0121e4d7434da516b.json index c3658e02a2..5ce9a0ea5b 100644 --- a/backend/.sqlx/query-a6feede7d9c3060b1e4c94bb4fa700d38a0d1b9d25429021c08fc94d57f952d2.json +++ b/backend/.sqlx/query-aaacf51f3a43617b60e4253549ec4cc6a3648992b590c6e0121e4d7434da516b.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "UPDATE queue SET created_by = $1 WHERE created_by = $2 AND workspace_id = $3", + "query": "UPDATE v2_job SET created_by = $1 WHERE created_by = $2 AND workspace_id = $3", "describe": { "columns": [], "parameters": { @@ -12,5 +12,5 @@ }, "nullable": [] }, - "hash": "a6feede7d9c3060b1e4c94bb4fa700d38a0d1b9d25429021c08fc94d57f952d2" + "hash": "aaacf51f3a43617b60e4253549ec4cc6a3648992b590c6e0121e4d7434da516b" } diff --git a/backend/.sqlx/query-6f4817fad2739a11d89b6704edf62c3c267ca336a8b6bec5b29d4409030ed561.json b/backend/.sqlx/query-ab04cda71f8e2be9acbecabe1ee5ef756b8e5c1955fbe111df9ee171dc262338.json similarity index 76% rename from backend/.sqlx/query-6f4817fad2739a11d89b6704edf62c3c267ca336a8b6bec5b29d4409030ed561.json rename to backend/.sqlx/query-ab04cda71f8e2be9acbecabe1ee5ef756b8e5c1955fbe111df9ee171dc262338.json index 3e2098f170..3284fb1846 100644 --- a/backend/.sqlx/query-6f4817fad2739a11d89b6704edf62c3c267ca336a8b6bec5b29d4409030ed561.json +++ b/backend/.sqlx/query-ab04cda71f8e2be9acbecabe1ee5ef756b8e5c1955fbe111df9ee171dc262338.json @@ -1,8 +1,14 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO queue (id, script_hash, script_path, job_kind, language, tag, created_by, permissioned_as, email, scheduled_for, workspace_id, raw_flow, flow_status) (SELECT gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12 FROM generate_series(1, 1))", + "query": "INSERT INTO v2_job (id, runnable_id, runnable_path, kind, script_lang, tag, created_by, permissioned_as, permissioned_as_email, workspace_id, raw_flow) (SELECT gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8, $9, $10 FROM generate_series(1, 1)) RETURNING id", "describe": { - "columns": [], + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + } + ], "parameters": { "Left": [ "Int8", @@ -66,13 +72,13 @@ "Varchar", "Varchar", "Varchar", - "Timestamptz", "Varchar", - "Jsonb", "Jsonb" ] }, - "nullable": [] + "nullable": [ + false + ] }, - "hash": "6f4817fad2739a11d89b6704edf62c3c267ca336a8b6bec5b29d4409030ed561" + "hash": "ab04cda71f8e2be9acbecabe1ee5ef756b8e5c1955fbe111df9ee171dc262338" } diff --git a/backend/.sqlx/query-f3d20b0fa17836538ec93b84b08ffc6b555371d6eeeadfc54f5bf0bcbe93d8b4.json b/backend/.sqlx/query-ab9e47e5b510e7df5a41db12896675393a6bb27f8e14245410751961218a7df5.json similarity index 61% rename from backend/.sqlx/query-f3d20b0fa17836538ec93b84b08ffc6b555371d6eeeadfc54f5bf0bcbe93d8b4.json rename to backend/.sqlx/query-ab9e47e5b510e7df5a41db12896675393a6bb27f8e14245410751961218a7df5.json index 2cfac3a744..868541a175 100644 --- a/backend/.sqlx/query-f3d20b0fa17836538ec93b84b08ffc6b555371d6eeeadfc54f5bf0bcbe93d8b4.json +++ b/backend/.sqlx/query-ab9e47e5b510e7df5a41db12896675393a6bb27f8e14245410751961218a7df5.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT COALESCE((SELECT MIN(started_at) as min_started_at\n FROM queue\n WHERE script_path = $1 AND job_kind != 'dependencies' AND running = true AND workspace_id = $2 AND canceled = false AND concurrent_limit > 0), $3) as min_started_at, now() AS now", + "query": "SELECT COALESCE((SELECT MIN(started_at) as min_started_at\n FROM v2_as_queue\n WHERE script_path = $1 AND job_kind != 'dependencies' AND running = true AND workspace_id = $2 AND canceled = false AND concurrent_limit > 0), $3) as min_started_at, now() AS now", "describe": { "columns": [ { @@ -26,5 +26,5 @@ null ] }, - "hash": "f3d20b0fa17836538ec93b84b08ffc6b555371d6eeeadfc54f5bf0bcbe93d8b4" + "hash": "ab9e47e5b510e7df5a41db12896675393a6bb27f8e14245410751961218a7df5" } diff --git a/backend/.sqlx/query-55541316c690e4f2e1b7a41071ef0a297a2e65c8e25a17d5c43715481e7633a0.json b/backend/.sqlx/query-acc0b67c8e768b524b5cfb309e4307daeb0c095e07063c57b26ab94211bf6359.json similarity index 50% rename from backend/.sqlx/query-55541316c690e4f2e1b7a41071ef0a297a2e65c8e25a17d5c43715481e7633a0.json rename to backend/.sqlx/query-acc0b67c8e768b524b5cfb309e4307daeb0c095e07063c57b26ab94211bf6359.json index bbcba9903c..58f693df33 100644 --- a/backend/.sqlx/query-55541316c690e4f2e1b7a41071ef0a297a2e65c8e25a17d5c43715481e7633a0.json +++ b/backend/.sqlx/query-acc0b67c8e768b524b5cfb309e4307daeb0c095e07063c57b26ab94211bf6359.json @@ -1,11 +1,11 @@ { "db_name": "PostgreSQL", - "query": "SELECT id FROM queue WHERE id = ANY($1) AND schedule_path IS NULL AND ($2::text[] IS NULL OR tag = ANY($2))", + "query": "SELECT id AS \"id!\" FROM v2_as_queue WHERE id = ANY($1) AND schedule_path IS NULL AND ($2::text[] IS NULL OR tag = ANY($2))", "describe": { "columns": [ { "ordinal": 0, - "name": "id", + "name": "id!", "type_info": "Uuid" } ], @@ -16,8 +16,8 @@ ] }, "nullable": [ - false + true ] }, - "hash": "55541316c690e4f2e1b7a41071ef0a297a2e65c8e25a17d5c43715481e7633a0" + "hash": "acc0b67c8e768b524b5cfb309e4307daeb0c095e07063c57b26ab94211bf6359" } diff --git a/backend/.sqlx/query-ad8f9e0b06f288051cbec3c91877430a1d61353b9122fd42777d92e4cbc9f4fa.json b/backend/.sqlx/query-ad8f9e0b06f288051cbec3c91877430a1d61353b9122fd42777d92e4cbc9f4fa.json deleted file mode 100644 index 00453be45b..0000000000 --- a/backend/.sqlx/query-ad8f9e0b06f288051cbec3c91877430a1d61353b9122fd42777d92e4cbc9f4fa.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE queue\n SET flow_status = JSONB_SET(\n JSONB_SET(flow_status, ARRAY['failure_module'], $1), ARRAY['step'], $2)\n WHERE id = $3", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Jsonb", - "Jsonb", - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "ad8f9e0b06f288051cbec3c91877430a1d61353b9122fd42777d92e4cbc9f4fa" -} diff --git a/backend/.sqlx/query-ae25fae1aca2cffc43a6054cd54639ba9deaaef419ad4cb2655e7e95f602a688.json b/backend/.sqlx/query-ae25fae1aca2cffc43a6054cd54639ba9deaaef419ad4cb2655e7e95f602a688.json deleted file mode 100644 index 5634e60233..0000000000 --- a/backend/.sqlx/query-ae25fae1aca2cffc43a6054cd54639ba9deaaef419ad4cb2655e7e95f602a688.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO job_logs (job_id, logs) VALUES ($1,'Restarted job after not receiving job''s ping for too long the ' || now() || '\n\n') \n ON CONFLICT (job_id) DO UPDATE SET logs = job_logs.logs || '\nRestarted job after not receiving job''s ping for too long the ' || now() || '\n\n' WHERE job_logs.job_id = $1", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "ae25fae1aca2cffc43a6054cd54639ba9deaaef419ad4cb2655e7e95f602a688" -} diff --git a/backend/.sqlx/query-ae8dfecd46425d5f86003eea9a578e9831fc0e700cc76ab9627afe9040a4efe0.json b/backend/.sqlx/query-ae8dfecd46425d5f86003eea9a578e9831fc0e700cc76ab9627afe9040a4efe0.json new file mode 100644 index 0000000000..c0703bc518 --- /dev/null +++ b/backend/.sqlx/query-ae8dfecd46425d5f86003eea9a578e9831fc0e700cc76ab9627afe9040a4efe0.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "create index concurrently if not exists ix_job_workspace_id_created_at_new_7 ON v2_job (workspace_id, created_at DESC) where kind in ('script', 'flow') AND parent_job IS NULL", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "ae8dfecd46425d5f86003eea9a578e9831fc0e700cc76ab9627afe9040a4efe0" +} diff --git a/backend/.sqlx/query-af925931f3217bbd32313678989ad1a66bbd8dacd12dea36608cc20197df358f.json b/backend/.sqlx/query-af925931f3217bbd32313678989ad1a66bbd8dacd12dea36608cc20197df358f.json new file mode 100644 index 0000000000..b90300787b --- /dev/null +++ b/backend/.sqlx/query-af925931f3217bbd32313678989ad1a66bbd8dacd12dea36608cc20197df358f.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status SET\n flow_status = JSONB_SET(\n JSONB_SET(flow_status, ARRAY['preprocessor_module'], $1),\n ARRAY['step'],\n $2\n )\n WHERE id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Jsonb", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "af925931f3217bbd32313678989ad1a66bbd8dacd12dea36608cc20197df358f" +} diff --git a/backend/.sqlx/query-b075c77caad37cdd75faab1d934b7a63521c1ad788033c564ffd7a2944454378.json b/backend/.sqlx/query-b075c77caad37cdd75faab1d934b7a63521c1ad788033c564ffd7a2944454378.json new file mode 100644 index 0000000000..b495061906 --- /dev/null +++ b/backend/.sqlx/query-b075c77caad37cdd75faab1d934b7a63521c1ad788033c564ffd7a2944454378.json @@ -0,0 +1,68 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n c.id IS NOT NULL AS completed,\n CASE \n WHEN q.id IS NOT NULL THEN (CASE WHEN NOT $5 AND q.running THEN true ELSE null END)\n ELSE false\n END AS running,\n SUBSTR(logs, GREATEST($1 - log_offset, 0)) AS logs,\n COALESCE(r.memory_peak, c.memory_peak) AS mem_peak,\n CASE\n -- flow step:\n WHEN flow_step_id IS NOT NULL THEN NULL\n -- completed:\n WHEN c.id IS NOT NULL THEN COALESCE(\n c.workflow_as_code_status || c.flow_status,\n c.workflow_as_code_status,\n c.flow_status\n )\n -- not completed:\n ELSE COALESCE(\n f.workflow_as_code_status || f.flow_status,\n f.workflow_as_code_status,\n f.flow_status\n )\n END AS \"flow_status: sqlx::types::Json>\",\n job_logs.log_offset + CHAR_LENGTH(job_logs.logs) + 1 AS log_offset,\n created_by AS \"created_by!\",\n CASE WHEN $4::BOOLEAN THEN (\n SELECT scalar_int FROM job_stats WHERE job_id = $3 AND metric_id = 'progress_perc'\n ) END AS progress\n FROM v2_job j\n LEFT JOIN v2_job_queue q USING (id)\n LEFT JOIN v2_job_runtime r USING (id)\n LEFT JOIN v2_job_status f USING (id)\n LEFT JOIN v2_job_completed c USING (id)\n LEFT JOIN job_logs ON job_logs.job_id = $3\n WHERE j.workspace_id = $2 AND j.id = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "completed", + "type_info": "Bool" + }, + { + "ordinal": 1, + "name": "running", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "logs", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "mem_peak", + "type_info": "Int4" + }, + { + "ordinal": 4, + "name": "flow_status: sqlx::types::Json>", + "type_info": "Jsonb" + }, + { + "ordinal": 5, + "name": "log_offset", + "type_info": "Int4" + }, + { + "ordinal": 6, + "name": "created_by!", + "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "progress", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Int4", + "Text", + "Uuid", + "Bool", + "Bool" + ] + }, + "nullable": [ + null, + null, + null, + null, + null, + null, + false, + null + ] + }, + "hash": "b075c77caad37cdd75faab1d934b7a63521c1ad788033c564ffd7a2944454378" +} diff --git a/backend/.sqlx/query-b08bf73ca7da6af302eeb5a5f443d71e03478e642a7999157a631a7ff0b7c63e.json b/backend/.sqlx/query-b08bf73ca7da6af302eeb5a5f443d71e03478e642a7999157a631a7ff0b7c63e.json deleted file mode 100644 index 53fc18f0fc..0000000000 --- a/backend/.sqlx/query-b08bf73ca7da6af302eeb5a5f443d71e03478e642a7999157a631a7ff0b7c63e.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE completed_job SET flow_status = q.flow_status FROM queue q WHERE completed_job.id = $1 AND q.id = $1 AND q.workspace_id = $2 AND completed_job.workspace_id = $2 AND q.flow_status IS NOT NULL", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid", - "Text" - ] - }, - "nullable": [] - }, - "hash": "b08bf73ca7da6af302eeb5a5f443d71e03478e642a7999157a631a7ff0b7c63e" -} diff --git a/backend/.sqlx/query-b186efe51e7bb1924efbd7e9b36085502a1e77e30e59c7310c78976f77a810a3.json b/backend/.sqlx/query-b186efe51e7bb1924efbd7e9b36085502a1e77e30e59c7310c78976f77a810a3.json new file mode 100644 index 0000000000..d856ab0109 --- /dev/null +++ b/backend/.sqlx/query-b186efe51e7bb1924efbd7e9b36085502a1e77e30e59c7310c78976f77a810a3.json @@ -0,0 +1,34 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT ai_resource, code_completion_model, ai_models FROM workspace_settings WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "ai_resource", + "type_info": "Jsonb" + }, + { + "ordinal": 1, + "name": "code_completion_model", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "ai_models", + "type_info": "VarcharArray" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + true, + true, + false + ] + }, + "hash": "b186efe51e7bb1924efbd7e9b36085502a1e77e30e59c7310c78976f77a810a3" +} diff --git a/backend/.sqlx/query-b2e4dfaaee713604d0700ea4675ed2d4534e08471a60005bff85b952874c54c2.json b/backend/.sqlx/query-b2e4dfaaee713604d0700ea4675ed2d4534e08471a60005bff85b952874c54c2.json new file mode 100644 index 0000000000..fabcec9b2a --- /dev/null +++ b/backend/.sqlx/query-b2e4dfaaee713604d0700ea4675ed2d4534e08471a60005bff85b952874c54c2.json @@ -0,0 +1,71 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n email AS \"email!\",\n created_by AS \"created_by!\",\n parent_job, permissioned_as AS \"permissioned_as!\",\n script_path, schedule_path, flow_step_id, root_job,\n scheduled_for AS \"scheduled_for!: chrono::DateTime\"\n FROM v2_as_queue WHERE id = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "email!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "created_by!", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "parent_job", + "type_info": "Uuid" + }, + { + "ordinal": 3, + "name": "permissioned_as!", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "script_path", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "schedule_path", + "type_info": "Varchar" + }, + { + "ordinal": 6, + "name": "flow_step_id", + "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "root_job", + "type_info": "Uuid" + }, + { + "ordinal": 8, + "name": "scheduled_for!: chrono::DateTime", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + true, + true, + true, + true, + true, + true, + true, + true, + true + ] + }, + "hash": "b2e4dfaaee713604d0700ea4675ed2d4534e08471a60005bff85b952874c54c2" +} diff --git a/backend/.sqlx/query-b3fd0be8a7ef6d8dd8943aef45f1555d6e088cb981b230d1cbbf8e8b832cad43.json b/backend/.sqlx/query-b3fd0be8a7ef6d8dd8943aef45f1555d6e088cb981b230d1cbbf8e8b832cad43.json new file mode 100644 index 0000000000..4c668f9b25 --- /dev/null +++ b/backend/.sqlx/query-b3fd0be8a7ef6d8dd8943aef45f1555d6e088cb981b230d1cbbf8e8b832cad43.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_queue SET\n canceled_by = $2,\n canceled_reason = 're-deployment'\n WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "b3fd0be8a7ef6d8dd8943aef45f1555d6e088cb981b230d1cbbf8e8b832cad43" +} diff --git a/backend/.sqlx/query-333b484ffa030dee08e7b1161fcbc48af411377d2d9f58f92fc9d5eacdf0fba1.json b/backend/.sqlx/query-b41cef713e822bbd89b49b1f35cc662539d5e4af1dd0b5923d8ee17b772c5677.json similarity index 79% rename from backend/.sqlx/query-333b484ffa030dee08e7b1161fcbc48af411377d2d9f58f92fc9d5eacdf0fba1.json rename to backend/.sqlx/query-b41cef713e822bbd89b49b1f35cc662539d5e4af1dd0b5923d8ee17b772c5677.json index bb21c2a9db..8475175c69 100644 --- a/backend/.sqlx/query-333b484ffa030dee08e7b1161fcbc48af411377d2d9f58f92fc9d5eacdf0fba1.json +++ b/backend/.sqlx/query-b41cef713e822bbd89b49b1f35cc662539d5e4af1dd0b5923d8ee17b772c5677.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO http_trigger (workspace_id, path, route_path, route_path_key, script_path, is_flow, is_async, requires_auth, http_method, static_asset_config, edited_by, email, edited_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, now())", + "query": "INSERT INTO http_trigger (workspace_id, path, route_path, route_path_key, script_path, is_flow, is_async, requires_auth, http_method, static_asset_config, edited_by, email, edited_at, is_static_website) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, now(), $13)", "describe": { "columns": [], "parameters": { @@ -29,10 +29,11 @@ }, "Jsonb", "Varchar", - "Varchar" + "Varchar", + "Bool" ] }, "nullable": [] }, - "hash": "333b484ffa030dee08e7b1161fcbc48af411377d2d9f58f92fc9d5eacdf0fba1" + "hash": "b41cef713e822bbd89b49b1f35cc662539d5e4af1dd0b5923d8ee17b772c5677" } diff --git a/backend/.sqlx/query-b4a9abcb38997587b28655b0f4a212a5bd4039b57fab20b163617e33a4c9dd46.json b/backend/.sqlx/query-b4a9abcb38997587b28655b0f4a212a5bd4039b57fab20b163617e33a4c9dd46.json new file mode 100644 index 0000000000..a49baeefaf --- /dev/null +++ b/backend/.sqlx/query-b4a9abcb38997587b28655b0f4a212a5bd4039b57fab20b163617e33a4c9dd46.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO v2_job_runtime (id) SELECT unnest($1::uuid[])", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "UuidArray" + ] + }, + "nullable": [] + }, + "hash": "b4a9abcb38997587b28655b0f4a212a5bd4039b57fab20b163617e33a4c9dd46" +} diff --git a/backend/.sqlx/query-b50b8d3367bd1f74986e6f923fe8497bcbfffea2570a3f3d1f8100207bc1557c.json b/backend/.sqlx/query-b50b8d3367bd1f74986e6f923fe8497bcbfffea2570a3f3d1f8100207bc1557c.json deleted file mode 100644 index dd2a7b9ede..0000000000 --- a/backend/.sqlx/query-b50b8d3367bd1f74986e6f923fe8497bcbfffea2570a3f3d1f8100207bc1557c.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT COUNT(*) FROM queue WHERE canceled = false AND (scheduled_for <= now()\n OR (suspend_until IS NOT NULL\n AND ( suspend <= 0\n OR suspend_until <= now())))", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "count", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - null - ] - }, - "hash": "b50b8d3367bd1f74986e6f923fe8497bcbfffea2570a3f3d1f8100207bc1557c" -} diff --git a/backend/.sqlx/query-bb0ee03198bcad69a1777447cf5c42fc22700e3cbf8cd44621a8380536d77552.json b/backend/.sqlx/query-bb0ee03198bcad69a1777447cf5c42fc22700e3cbf8cd44621a8380536d77552.json deleted file mode 100644 index 91a88794f4..0000000000 --- a/backend/.sqlx/query-bb0ee03198bcad69a1777447cf5c42fc22700e3cbf8cd44621a8380536d77552.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM completed_job WHERE created_at <= now() - ($1::bigint::text || ' s')::interval AND started_at + ((duration_ms/1000 + $1::bigint) || ' s')::interval <= now() RETURNING id", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Uuid" - } - ], - "parameters": { - "Left": [ - "Int8" - ] - }, - "nullable": [ - false - ] - }, - "hash": "bb0ee03198bcad69a1777447cf5c42fc22700e3cbf8cd44621a8380536d77552" -} diff --git a/backend/.sqlx/query-bcc79dc05e227625bc2ba70350a41a254e00e84030aefff91a7b420fe7513b0d.json b/backend/.sqlx/query-bcc79dc05e227625bc2ba70350a41a254e00e84030aefff91a7b420fe7513b0d.json new file mode 100644 index 0000000000..25db03d1b5 --- /dev/null +++ b/backend/.sqlx/query-bcc79dc05e227625bc2ba70350a41a254e00e84030aefff91a7b420fe7513b0d.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT result #> $3 AS \"result: Json>\"\n FROM v2_job_completed WHERE id = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "result: Json>", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "TextArray" + ] + }, + "nullable": [ + null + ] + }, + "hash": "bcc79dc05e227625bc2ba70350a41a254e00e84030aefff91a7b420fe7513b0d" +} diff --git a/backend/.sqlx/query-bcd7e2c47baf27de2b0919f415dc8cc78fdc1527020e3c73f82eb01f64670923.json b/backend/.sqlx/query-bcd7e2c47baf27de2b0919f415dc8cc78fdc1527020e3c73f82eb01f64670923.json new file mode 100644 index 0000000000..1060bafebd --- /dev/null +++ b/backend/.sqlx/query-bcd7e2c47baf27de2b0919f415dc8cc78fdc1527020e3c73f82eb01f64670923.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job SET\n args = '{\"reason\":\"PREPROCESSOR_ARGS_ARE_DISCARDED\"}'::jsonb,\n preprocessed = TRUE\n WHERE id = $1 AND preprocessed = FALSE", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "bcd7e2c47baf27de2b0919f415dc8cc78fdc1527020e3c73f82eb01f64670923" +} diff --git a/backend/.sqlx/query-bd0ca94343399fa2fa06d9304da4ed90a18a3c952783c419707ffbb38e232ce0.json b/backend/.sqlx/query-bd0ca94343399fa2fa06d9304da4ed90a18a3c952783c419707ffbb38e232ce0.json new file mode 100644 index 0000000000..2e06f25b98 --- /dev/null +++ b/backend/.sqlx/query-bd0ca94343399fa2fa06d9304da4ed90a18a3c952783c419707ffbb38e232ce0.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT tag AS \"tag!\", count(*) AS \"count!\" FROM v2_job_queue WHERE\n scheduled_for <= now() - ('3 seconds')::interval AND running = false\n GROUP BY tag", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "tag!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + null + ] + }, + "hash": "bd0ca94343399fa2fa06d9304da4ed90a18a3c952783c419707ffbb38e232ce0" +} diff --git a/backend/.sqlx/query-bd5a0c06e2f2361c9fc670eb0b975b58d65ca93d68b29124d04bd526239b9df2.json b/backend/.sqlx/query-bd5a0c06e2f2361c9fc670eb0b975b58d65ca93d68b29124d04bd526239b9df2.json new file mode 100644 index 0000000000..81a97984ee --- /dev/null +++ b/backend/.sqlx/query-bd5a0c06e2f2361c9fc670eb0b975b58d65ca93d68b29124d04bd526239b9df2.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job SET labels = $2 WHERE id = $1 AND $2::TEXT[] IS NOT NULL", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "bd5a0c06e2f2361c9fc670eb0b975b58d65ca93d68b29124d04bd526239b9df2" +} diff --git a/backend/.sqlx/query-bfcf13b04f07f7d29c66ef5ba270febc663a8dc5b596acfe8b6771a35bf6e6cc.json b/backend/.sqlx/query-bfcf13b04f07f7d29c66ef5ba270febc663a8dc5b596acfe8b6771a35bf6e6cc.json new file mode 100644 index 0000000000..9899208666 --- /dev/null +++ b/backend/.sqlx/query-bfcf13b04f07f7d29c66ef5ba270febc663a8dc5b596acfe8b6771a35bf6e6cc.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT flow_innermost_root_job FROM v2_job WHERE id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "flow_innermost_root_job", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + true + ] + }, + "hash": "bfcf13b04f07f7d29c66ef5ba270febc663a8dc5b596acfe8b6771a35bf6e6cc" +} diff --git a/backend/.sqlx/query-f6a275ad8bc7dfec7f9a6b60c669f6f2ff93ce57b5afadabd618e1fb52951fef.json b/backend/.sqlx/query-bff39cc57aba0729ddef1d53f3806c6736556f0a14b489d6708f9879393f9ea3.json similarity index 59% rename from backend/.sqlx/query-f6a275ad8bc7dfec7f9a6b60c669f6f2ff93ce57b5afadabd618e1fb52951fef.json rename to backend/.sqlx/query-bff39cc57aba0729ddef1d53f3806c6736556f0a14b489d6708f9879393f9ea3.json index fc5dce48f3..11e498f38d 100644 --- a/backend/.sqlx/query-f6a275ad8bc7dfec7f9a6b60c669f6f2ff93ce57b5afadabd618e1fb52951fef.json +++ b/backend/.sqlx/query-bff39cc57aba0729ddef1d53f3806c6736556f0a14b489d6708f9879393f9ea3.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT usr.email, usage.executions\n FROM usr\n , LATERAL (\n SELECT COALESCE(SUM(duration_ms + 1000)/1000 , 0)::BIGINT executions\n FROM completed_job\n WHERE workspace_id = $1\n AND job_kind NOT IN ('flow', 'flowpreview', 'flownode')\n AND email = usr.email\n AND now() - '1 week'::interval < created_at \n ) usage\n WHERE workspace_id = $1\n ", + "query": "\n SELECT usr.email, usage.executions\n FROM usr\n , LATERAL (\n SELECT COALESCE(SUM(duration_ms + 1000)/1000 , 0)::BIGINT executions\n FROM v2_as_completed_job\n WHERE workspace_id = $1\n AND job_kind NOT IN ('flow', 'flowpreview', 'flownode')\n AND email = usr.email\n AND now() - '1 week'::interval < created_at \n ) usage\n WHERE workspace_id = $1\n ", "describe": { "columns": [ { @@ -24,5 +24,5 @@ null ] }, - "hash": "f6a275ad8bc7dfec7f9a6b60c669f6f2ff93ce57b5afadabd618e1fb52951fef" + "hash": "bff39cc57aba0729ddef1d53f3806c6736556f0a14b489d6708f9879393f9ea3" } diff --git a/backend/.sqlx/query-c00bae0d8c9bee37cbad4de4cb02c80d00f52a3fc32bf32271ebc90f7837abda.json b/backend/.sqlx/query-c00bae0d8c9bee37cbad4de4cb02c80d00f52a3fc32bf32271ebc90f7837abda.json new file mode 100644 index 0000000000..74aa6c4ffb --- /dev/null +++ b/backend/.sqlx/query-c00bae0d8c9bee37cbad4de4cb02c80d00f52a3fc32bf32271ebc90f7837abda.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_queue\n SET canceled_by = 'timeout'\n , canceled_reason = $1\n WHERE id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "c00bae0d8c9bee37cbad4de4cb02c80d00f52a3fc32bf32271ebc90f7837abda" +} diff --git a/backend/.sqlx/query-cb0eecb1130617f2132b8ec74a401ac3fb932bbeafe3aecc0b14465cce7e192d.json b/backend/.sqlx/query-c03b4c5878bd0a0c545cb66b146d0b6ceae9de456d3ddb6008707589dbe30747.json similarity index 60% rename from backend/.sqlx/query-cb0eecb1130617f2132b8ec74a401ac3fb932bbeafe3aecc0b14465cce7e192d.json rename to backend/.sqlx/query-c03b4c5878bd0a0c545cb66b146d0b6ceae9de456d3ddb6008707589dbe30747.json index 128863559f..5cb05f49a0 100644 --- a/backend/.sqlx/query-cb0eecb1130617f2132b8ec74a401ac3fb932bbeafe3aecc0b14465cce7e192d.json +++ b/backend/.sqlx/query-c03b4c5878bd0a0c545cb66b146d0b6ceae9de456d3ddb6008707589dbe30747.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT 1 FROM queue WHERE id = $1 UNION ALL select 1 FROM completed_job WHERE id = $1", + "query": "SELECT 1 FROM v2_job WHERE id = $1", "describe": { "columns": [ { @@ -18,5 +18,5 @@ null ] }, - "hash": "cb0eecb1130617f2132b8ec74a401ac3fb932bbeafe3aecc0b14465cce7e192d" + "hash": "c03b4c5878bd0a0c545cb66b146d0b6ceae9de456d3ddb6008707589dbe30747" } diff --git a/backend/.sqlx/query-4fb3a4712d88afed40082d8d8bd63b5dedad61caa68e0e470252083d80df605f.json b/backend/.sqlx/query-c03decd33061ab912c9afed841092b05105c261976cce78c77d7f85abf9ad40c.json similarity index 53% rename from backend/.sqlx/query-4fb3a4712d88afed40082d8d8bd63b5dedad61caa68e0e470252083d80df605f.json rename to backend/.sqlx/query-c03decd33061ab912c9afed841092b05105c261976cce78c77d7f85abf9ad40c.json index 90bdd1c9b8..17532bfc77 100644 --- a/backend/.sqlx/query-4fb3a4712d88afed40082d8d8bd63b5dedad61caa68e0e470252083d80df605f.json +++ b/backend/.sqlx/query-c03decd33061ab912c9afed841092b05105c261976cce78c77d7f85abf9ad40c.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "UPDATE queue SET suspend = 0 WHERE id = $1", + "query": "UPDATE v2_job SET args = NULL WHERE id = $1", "describe": { "columns": [], "parameters": { @@ -10,5 +10,5 @@ }, "nullable": [] }, - "hash": "4fb3a4712d88afed40082d8d8bd63b5dedad61caa68e0e470252083d80df605f" + "hash": "c03decd33061ab912c9afed841092b05105c261976cce78c77d7f85abf9ad40c" } diff --git a/backend/.sqlx/query-c14009d133956710f4435a5984c18b9517e256978512f014e1bf5c270f499772.json b/backend/.sqlx/query-c14009d133956710f4435a5984c18b9517e256978512f014e1bf5c270f499772.json deleted file mode 100644 index 7e9e9b84aa..0000000000 --- a/backend/.sqlx/query-c14009d133956710f4435a5984c18b9517e256978512f014e1bf5c270f499772.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE queue\n SET flow_status = JSONB_SET(flow_status, ARRAY['preprocessor_module'], $1)\n WHERE id = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Jsonb", - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "c14009d133956710f4435a5984c18b9517e256978512f014e1bf5c270f499772" -} diff --git a/backend/.sqlx/query-c202f6fbae6a727f88f3ac692985c70e6ebc68e4a16d02e4e36b79f3cfb1c661.json b/backend/.sqlx/query-c202f6fbae6a727f88f3ac692985c70e6ebc68e4a16d02e4e36b79f3cfb1c661.json new file mode 100644 index 0000000000..e1094f38ef --- /dev/null +++ b/backend/.sqlx/query-c202f6fbae6a727f88f3ac692985c70e6ebc68e4a16d02e4e36b79f3cfb1c661.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH suspend AS (\n UPDATE v2_job_queue SET suspend = $2, suspend_until = now() + $3\n WHERE id = $4\n RETURNING id\n ) UPDATE v2_job_status SET flow_status = JSONB_SET(\n flow_status,\n ARRAY['modules', flow_status->>'step'::TEXT],\n $1\n ) WHERE id = (SELECT id FROM suspend)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Int4", + "Interval", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "c202f6fbae6a727f88f3ac692985c70e6ebc68e4a16d02e4e36b79f3cfb1c661" +} diff --git a/backend/.sqlx/query-c24d63fb137b805f1e674261681d15e6405de27d40b799b0a65b31fd41bcc625.json b/backend/.sqlx/query-c24d63fb137b805f1e674261681d15e6405de27d40b799b0a65b31fd41bcc625.json deleted file mode 100644 index d30c48c5ca..0000000000 --- a/backend/.sqlx/query-c24d63fb137b805f1e674261681d15e6405de27d40b799b0a65b31fd41bcc625.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "VACUUM (skip_locked) queue", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "c24d63fb137b805f1e674261681d15e6405de27d40b799b0a65b31fd41bcc625" -} diff --git a/backend/.sqlx/query-c4bdbf1c6bc7d93db4cf2633105b088b781354cb7c02628d9f8ff7f9ea0e7ed9.json b/backend/.sqlx/query-c4bdbf1c6bc7d93db4cf2633105b088b781354cb7c02628d9f8ff7f9ea0e7ed9.json new file mode 100644 index 0000000000..4d138db49c --- /dev/null +++ b/backend/.sqlx/query-c4bdbf1c6bc7d93db4cf2633105b088b781354cb7c02628d9f8ff7f9ea0e7ed9.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "CREATE INDEX CONCURRENTLY labeled_jobs_on_jobs ON v2_job_completed USING GIN ((result -> 'wm_labels')) WHERE result ? 'wm_labels'", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "c4bdbf1c6bc7d93db4cf2633105b088b781354cb7c02628d9f8ff7f9ea0e7ed9" +} diff --git a/backend/.sqlx/query-4cdb9b9d562f3c692e5597598db937511bc8431c3652746684ee803172053885.json b/backend/.sqlx/query-c5259e37703c3e48104438bad6e1f3615f4439c090a75e6fde03702a21589b25.json similarity index 57% rename from backend/.sqlx/query-4cdb9b9d562f3c692e5597598db937511bc8431c3652746684ee803172053885.json rename to backend/.sqlx/query-c5259e37703c3e48104438bad6e1f3615f4439c090a75e6fde03702a21589b25.json index 48a2a5451d..ee362d8845 100644 --- a/backend/.sqlx/query-4cdb9b9d562f3c692e5597598db937511bc8431c3652746684ee803172053885.json +++ b/backend/.sqlx/query-c5259e37703c3e48104438bad6e1f3615f4439c090a75e6fde03702a21589b25.json @@ -1,11 +1,11 @@ { "db_name": "PostgreSQL", - "query": "SELECT id FROM queue WHERE parent_job = $1 AND workspace_id = $2", + "query": "SELECT id AS \"id!\" FROM v2_job WHERE parent_job = $1 AND workspace_id = $2", "describe": { "columns": [ { "ordinal": 0, - "name": "id", + "name": "id!", "type_info": "Uuid" } ], @@ -19,5 +19,5 @@ false ] }, - "hash": "4cdb9b9d562f3c692e5597598db937511bc8431c3652746684ee803172053885" + "hash": "c5259e37703c3e48104438bad6e1f3615f4439c090a75e6fde03702a21589b25" } diff --git a/backend/.sqlx/query-c71e12ec9d0054dd5605a4ea2ef77fa54ce686da9c93790420a91a0735589ac7.json b/backend/.sqlx/query-c71e12ec9d0054dd5605a4ea2ef77fa54ce686da9c93790420a91a0735589ac7.json new file mode 100644 index 0000000000..ecc62b1682 --- /dev/null +++ b/backend/.sqlx/query-c71e12ec9d0054dd5605a4ea2ef77fa54ce686da9c93790420a91a0735589ac7.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status\n SET flow_status = JSONB_SET(JSONB_SET(flow_status, ARRAY['retry'], $1), ARRAY['modules', $3::TEXT, 'failed_retries'], $4)\n WHERE id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Uuid", + "Text", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "c71e12ec9d0054dd5605a4ea2ef77fa54ce686da9c93790420a91a0735589ac7" +} diff --git a/backend/.sqlx/query-c7be5fa2eaf66147c1213046e615f5e9fd168ef1e3aba8af64b15341055d6007.json b/backend/.sqlx/query-c7be5fa2eaf66147c1213046e615f5e9fd168ef1e3aba8af64b15341055d6007.json new file mode 100644 index 0000000000..d79a7f5465 --- /dev/null +++ b/backend/.sqlx/query-c7be5fa2eaf66147c1213046e615f5e9fd168ef1e3aba8af64b15341055d6007.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "create index concurrently if not exists ix_job_workspace_id_created_at_new_5 ON v2_job (workspace_id, created_at DESC) where kind in ('preview', 'flowpreview') AND parent_job IS NULL", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "c7be5fa2eaf66147c1213046e615f5e9fd168ef1e3aba8af64b15341055d6007" +} diff --git a/backend/.sqlx/query-c7d595d2a12228c49359440ca3a9622f1de5f5ee4bbe5d2b23f6fdb6379cebf3.json b/backend/.sqlx/query-c7d595d2a12228c49359440ca3a9622f1de5f5ee4bbe5d2b23f6fdb6379cebf3.json new file mode 100644 index 0000000000..a3d8c2502f --- /dev/null +++ b/backend/.sqlx/query-c7d595d2a12228c49359440ca3a9622f1de5f5ee4bbe5d2b23f6fdb6379cebf3.json @@ -0,0 +1,47 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE token SET last_used_at = now() WHERE\n token = $1\n AND (expiration > NOW() OR expiration IS NULL)\n AND (workspace_id IS NULL OR workspace_id = $2)\n RETURNING owner, email, super_admin, scopes, label", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "owner", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "super_admin", + "type_info": "Bool" + }, + { + "ordinal": 3, + "name": "scopes", + "type_info": "TextArray" + }, + { + "ordinal": 4, + "name": "label", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + true, + true, + false, + true, + true + ] + }, + "hash": "c7d595d2a12228c49359440ca3a9622f1de5f5ee4bbe5d2b23f6fdb6379cebf3" +} diff --git a/backend/.sqlx/query-ca3ba808e020c8c7a35eaef842b20cfeee64fd47ded72fce55cc75e0bbb291a8.json b/backend/.sqlx/query-ca3ba808e020c8c7a35eaef842b20cfeee64fd47ded72fce55cc75e0bbb291a8.json deleted file mode 100644 index 6d44fb2840..0000000000 --- a/backend/.sqlx/query-ca3ba808e020c8c7a35eaef842b20cfeee64fd47ded72fce55cc75e0bbb291a8.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO usage (id, is_workspace, month_, usage) \n VALUES ($1, TRUE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), $2) \n ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Int4" - ] - }, - "nullable": [] - }, - "hash": "ca3ba808e020c8c7a35eaef842b20cfeee64fd47ded72fce55cc75e0bbb291a8" -} diff --git a/backend/.sqlx/query-ca8bf3dda133556ee7051d5436a5d76ae2223d59e9f321a8b6d0c27adc09f741.json b/backend/.sqlx/query-ca8bf3dda133556ee7051d5436a5d76ae2223d59e9f321a8b6d0c27adc09f741.json new file mode 100644 index 0000000000..b7a302213a --- /dev/null +++ b/backend/.sqlx/query-ca8bf3dda133556ee7051d5436a5d76ae2223d59e9f321a8b6d0c27adc09f741.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO v2_job_runtime (id, ping) VALUES ($1, null)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "ca8bf3dda133556ee7051d5436a5d76ae2223d59e9f321a8b6d0c27adc09f741" +} diff --git a/backend/.sqlx/query-487d377e2df67fc3ea39d183ba9f99d45828d7c8e0ff10c5d74c454472e0493c.json b/backend/.sqlx/query-ccb7aea162fa8781675d547fb82f2996dede4de49c5bd9cca2928209dc40b8f1.json similarity index 82% rename from backend/.sqlx/query-487d377e2df67fc3ea39d183ba9f99d45828d7c8e0ff10c5d74c454472e0493c.json rename to backend/.sqlx/query-ccb7aea162fa8781675d547fb82f2996dede4de49c5bd9cca2928209dc40b8f1.json index 7cd37b4c50..f4bc3f2174 100644 --- a/backend/.sqlx/query-487d377e2df67fc3ea39d183ba9f99d45828d7c8e0ff10c5d74c454472e0493c.json +++ b/backend/.sqlx/query-ccb7aea162fa8781675d547fb82f2996dede4de49c5bd9cca2928209dc40b8f1.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "UPDATE http_trigger \n SET route_path = $1, route_path_key = $2, script_path = $3, path = $4, is_flow = $5, http_method = $6, static_asset_config = $7, edited_by = $8, email = $9, is_async = $10, requires_auth = $11, edited_at = now() \n WHERE workspace_id = $12 AND path = $13", + "query": "UPDATE http_trigger \n SET route_path = $1, route_path_key = $2, script_path = $3, path = $4, is_flow = $5, http_method = $6, static_asset_config = $7, edited_by = $8, email = $9, is_async = $10, requires_auth = $11, edited_at = now(), is_static_website = $12\n WHERE workspace_id = $13 AND path = $14", "describe": { "columns": [], "parameters": { @@ -29,11 +29,12 @@ "Varchar", "Bool", "Bool", + "Bool", "Text", "Text" ] }, "nullable": [] }, - "hash": "487d377e2df67fc3ea39d183ba9f99d45828d7c8e0ff10c5d74c454472e0493c" + "hash": "ccb7aea162fa8781675d547fb82f2996dede4de49c5bd9cca2928209dc40b8f1" } diff --git a/backend/.sqlx/query-230d58732a08164268ca10d248a93cced646632a76864b693ed2325d85b36c45.json b/backend/.sqlx/query-cd5f02cf10cbf92dd1df53a54f2110efa11a7731ad0f0e5509f55efabdf535cd.json similarity index 56% rename from backend/.sqlx/query-230d58732a08164268ca10d248a93cced646632a76864b693ed2325d85b36c45.json rename to backend/.sqlx/query-cd5f02cf10cbf92dd1df53a54f2110efa11a7731ad0f0e5509f55efabdf535cd.json index 37d4e7d370..f5eb13f1fd 100644 --- a/backend/.sqlx/query-230d58732a08164268ca10d248a93cced646632a76864b693ed2325d85b36c45.json +++ b/backend/.sqlx/query-cd5f02cf10cbf92dd1df53a54f2110efa11a7731ad0f0e5509f55efabdf535cd.json @@ -1,11 +1,11 @@ { "db_name": "PostgreSQL", - "query": "SELECT canceled FROM queue WHERE id = $1", + "query": "SELECT preprocessed FROM v2_job WHERE id = $1", "describe": { "columns": [ { "ordinal": 0, - "name": "canceled", + "name": "preprocessed", "type_info": "Bool" } ], @@ -15,8 +15,8 @@ ] }, "nullable": [ - false + true ] }, - "hash": "230d58732a08164268ca10d248a93cced646632a76864b693ed2325d85b36c45" + "hash": "cd5f02cf10cbf92dd1df53a54f2110efa11a7731ad0f0e5509f55efabdf535cd" } diff --git a/backend/.sqlx/query-cd6a2559d76e7c6462bd18c8f0bc07c31ca9bdd96f8f92ba19d90e6060721354.json b/backend/.sqlx/query-cd6a2559d76e7c6462bd18c8f0bc07c31ca9bdd96f8f92ba19d90e6060721354.json new file mode 100644 index 0000000000..f11e9daf54 --- /dev/null +++ b/backend/.sqlx/query-cd6a2559d76e7c6462bd18c8f0bc07c31ca9bdd96f8f92ba19d90e6060721354.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status\n SET flow_status = JSONB_SET(flow_status, ARRAY['modules', flow_status->>'step', 'progress'], $1)\n WHERE id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "cd6a2559d76e7c6462bd18c8f0bc07c31ca9bdd96f8f92ba19d90e6060721354" +} diff --git a/backend/.sqlx/query-cf12a70e7b75ae471a0944de34502384be156cf25129f9c52bda34b240cf469a.json b/backend/.sqlx/query-cf12a70e7b75ae471a0944de34502384be156cf25129f9c52bda34b240cf469a.json new file mode 100644 index 0000000000..962572fa3d --- /dev/null +++ b/backend/.sqlx/query-cf12a70e7b75ae471a0944de34502384be156cf25129f9c52bda34b240cf469a.json @@ -0,0 +1,30 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT leaf_jobs->$1::text AS \"leaf_jobs: Json>\", parent_job\n FROM v2_as_queue\n WHERE COALESCE((SELECT flow_innermost_root_job FROM v2_job WHERE id = $2), $2) = id AND workspace_id = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "leaf_jobs: Json>", + "type_info": "Jsonb" + }, + { + "ordinal": 1, + "name": "parent_job", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Text", + "Uuid", + "Text" + ] + }, + "nullable": [ + null, + true + ] + }, + "hash": "cf12a70e7b75ae471a0944de34502384be156cf25129f9c52bda34b240cf469a" +} diff --git a/backend/.sqlx/query-e7a1c2b5d79e72f557181782419a9d8d1a502796842f185374d2d0f69043086b.json b/backend/.sqlx/query-cf85496774cd3ce58543dd0d52b5eadedad1a29049f8023e90deb57b19a41c72.json similarity index 64% rename from backend/.sqlx/query-e7a1c2b5d79e72f557181782419a9d8d1a502796842f185374d2d0f69043086b.json rename to backend/.sqlx/query-cf85496774cd3ce58543dd0d52b5eadedad1a29049f8023e90deb57b19a41c72.json index 829d4f6768..047b758fdf 100644 --- a/backend/.sqlx/query-e7a1c2b5d79e72f557181782419a9d8d1a502796842f185374d2d0f69043086b.json +++ b/backend/.sqlx/query-cf85496774cd3ce58543dd0d52b5eadedad1a29049f8023e90deb57b19a41c72.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT COUNT(id) FROM queue WHERE workspace_id = $1", + "query": "SELECT COUNT(id) FROM v2_job_queue WHERE workspace_id = $1", "describe": { "columns": [ { @@ -18,5 +18,5 @@ null ] }, - "hash": "e7a1c2b5d79e72f557181782419a9d8d1a502796842f185374d2d0f69043086b" + "hash": "cf85496774cd3ce58543dd0d52b5eadedad1a29049f8023e90deb57b19a41c72" } diff --git a/backend/.sqlx/query-d067bf59ed65562f1efbfbc9c264cbcb90e5a8d94d628cc78d1f2271c43e56f1.json b/backend/.sqlx/query-d067bf59ed65562f1efbfbc9c264cbcb90e5a8d94d628cc78d1f2271c43e56f1.json new file mode 100644 index 0000000000..53490d5b60 --- /dev/null +++ b/backend/.sqlx/query-d067bf59ed65562f1efbfbc9c264cbcb90e5a8d94d628cc78d1f2271c43e56f1.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO v2_job_status (id, workflow_as_code_status) VALUES ($1, '{}'::JSONB)\n ON CONFLICT (id) DO NOTHING", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "d067bf59ed65562f1efbfbc9c264cbcb90e5a8d94d628cc78d1f2271c43e56f1" +} diff --git a/backend/.sqlx/query-0355b53b1d45955ca56b2829372ce9c656d7f0ad7b8d0709161047f0d8cdc4f4.json b/backend/.sqlx/query-d25c58d2722ad3dcd91101ce6f66e1d802dd5d82e1cd5f5ed3a15cbc75eb6745.json similarity index 63% rename from backend/.sqlx/query-0355b53b1d45955ca56b2829372ce9c656d7f0ad7b8d0709161047f0d8cdc4f4.json rename to backend/.sqlx/query-d25c58d2722ad3dcd91101ce6f66e1d802dd5d82e1cd5f5ed3a15cbc75eb6745.json index 2f2cb27400..2872c1655b 100644 --- a/backend/.sqlx/query-0355b53b1d45955ca56b2829372ce9c656d7f0ad7b8d0709161047f0d8cdc4f4.json +++ b/backend/.sqlx/query-d25c58d2722ad3dcd91101ce6f66e1d802dd5d82e1cd5f5ed3a15cbc75eb6745.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "DELETE FROM queue WHERE workspace_id = $1 AND id = $2 RETURNING 1", + "query": "DELETE FROM v2_job_queue WHERE workspace_id = $1 AND id = $2 RETURNING 1", "describe": { "columns": [ { @@ -19,5 +19,5 @@ null ] }, - "hash": "0355b53b1d45955ca56b2829372ce9c656d7f0ad7b8d0709161047f0d8cdc4f4" + "hash": "d25c58d2722ad3dcd91101ce6f66e1d802dd5d82e1cd5f5ed3a15cbc75eb6745" } diff --git a/backend/.sqlx/query-d2a9e6a31bab0551d32093b1afe4e5d414cf439e4db3325b5bf6bbeb86c5bd2a.json b/backend/.sqlx/query-d2a9e6a31bab0551d32093b1afe4e5d414cf439e4db3325b5bf6bbeb86c5bd2a.json new file mode 100644 index 0000000000..4b96e7005b --- /dev/null +++ b/backend/.sqlx/query-d2a9e6a31bab0551d32093b1afe4e5d414cf439e4db3325b5bf6bbeb86c5bd2a.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH to_delete AS (\n SELECT id FROM v2_job_queue\n JOIN v2_job j USING (id)\n WHERE trigger_kind = 'schedule'\n AND trigger = $1\n AND j.workspace_id = $2\n AND flow_step_id IS NULL\n AND running = false\n FOR UPDATE\n ), deleted AS (\n DELETE FROM v2_job_queue\n WHERE id IN (SELECT id FROM to_delete)\n RETURNING id\n ) DELETE FROM v2_job WHERE id IN (SELECT id FROM deleted)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "d2a9e6a31bab0551d32093b1afe4e5d414cf439e4db3325b5bf6bbeb86c5bd2a" +} diff --git a/backend/.sqlx/query-d37a7c2c19beb6963794670312b4331ea22d3e06b487b8037c20cdaec392b311.json b/backend/.sqlx/query-d37a7c2c19beb6963794670312b4331ea22d3e06b487b8037c20cdaec392b311.json new file mode 100644 index 0000000000..896f7d92c7 --- /dev/null +++ b/backend/.sqlx/query-d37a7c2c19beb6963794670312b4331ea22d3e06b487b8037c20cdaec392b311.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status SET\n flow_status = jsonb_set(\n jsonb_set(flow_status, ARRAY['preprocessor_module', 'job'], to_jsonb($1::UUID::TEXT)),\n ARRAY['preprocessor_module', 'type'],\n to_jsonb('InProgress'::text)\n )\n WHERE id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "d37a7c2c19beb6963794670312b4331ea22d3e06b487b8037c20cdaec392b311" +} diff --git a/backend/.sqlx/query-1c28baaadd7d0c86a92bf9880a4ea33457bf8cff669e983431f1fd26ff275f83.json b/backend/.sqlx/query-d4d83d8177144c91aa489b5a42a45c83f8b069a52f681f14afb4931ac77baf45.json similarity index 62% rename from backend/.sqlx/query-1c28baaadd7d0c86a92bf9880a4ea33457bf8cff669e983431f1fd26ff275f83.json rename to backend/.sqlx/query-d4d83d8177144c91aa489b5a42a45c83f8b069a52f681f14afb4931ac77baf45.json index 0cb39dcb9b..3422605ef1 100644 --- a/backend/.sqlx/query-1c28baaadd7d0c86a92bf9880a4ea33457bf8cff669e983431f1fd26ff275f83.json +++ b/backend/.sqlx/query-d4d83d8177144c91aa489b5a42a45c83f8b069a52f681f14afb4931ac77baf45.json @@ -1,11 +1,11 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT id, flow_status, suspend, script_path\n FROM queue\n WHERE id = $1\n ", + "query": "\n SELECT id AS \"id!\", flow_status, suspend AS \"suspend!\", script_path\n FROM v2_as_queue\n WHERE id = $1\n ", "describe": { "columns": [ { "ordinal": 0, - "name": "id", + "name": "id!", "type_info": "Uuid" }, { @@ -15,7 +15,7 @@ }, { "ordinal": 2, - "name": "suspend", + "name": "suspend!", "type_info": "Int4" }, { @@ -30,11 +30,11 @@ ] }, "nullable": [ - false, true, - false, + true, + true, true ] }, - "hash": "1c28baaadd7d0c86a92bf9880a4ea33457bf8cff669e983431f1fd26ff275f83" + "hash": "d4d83d8177144c91aa489b5a42a45c83f8b069a52f681f14afb4931ac77baf45" } diff --git a/backend/.sqlx/query-1af5ccc82048df95a791949e7b141861dbfd5c08daea615dde081e29f7459b9d.json b/backend/.sqlx/query-d6c8f4e49cf7b6db5c979c88e02bd41c7b823186ac056a0a676da85dc5d9a027.json similarity index 52% rename from backend/.sqlx/query-1af5ccc82048df95a791949e7b141861dbfd5c08daea615dde081e29f7459b9d.json rename to backend/.sqlx/query-d6c8f4e49cf7b6db5c979c88e02bd41c7b823186ac056a0a676da85dc5d9a027.json index 9ab47f361b..d090c25a37 100644 --- a/backend/.sqlx/query-1af5ccc82048df95a791949e7b141861dbfd5c08daea615dde081e29f7459b9d.json +++ b/backend/.sqlx/query-d6c8f4e49cf7b6db5c979c88e02bd41c7b823186ac056a0a676da85dc5d9a027.json @@ -1,11 +1,11 @@ { "db_name": "PostgreSQL", - "query": "SELECT id FROM queue WHERE schedule_path = $1 AND workspace_id = $2 AND id != $3 AND running = true", + "query": "SELECT id AS \"id!\" FROM v2_as_queue WHERE schedule_path = $1 AND workspace_id = $2 AND id != $3 AND running = true", "describe": { "columns": [ { "ordinal": 0, - "name": "id", + "name": "id!", "type_info": "Uuid" } ], @@ -17,8 +17,8 @@ ] }, "nullable": [ - false + true ] }, - "hash": "1af5ccc82048df95a791949e7b141861dbfd5c08daea615dde081e29f7459b9d" + "hash": "d6c8f4e49cf7b6db5c979c88e02bd41c7b823186ac056a0a676da85dc5d9a027" } diff --git a/backend/.sqlx/query-d6f62e25faf271876874fc09ee460313159bb6ad91227f5dec37cd28006e2add.json b/backend/.sqlx/query-d6f62e25faf271876874fc09ee460313159bb6ad91227f5dec37cd28006e2add.json new file mode 100644 index 0000000000..c4c138a886 --- /dev/null +++ b/backend/.sqlx/query-d6f62e25faf271876874fc09ee460313159bb6ad91227f5dec37cd28006e2add.json @@ -0,0 +1,35 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n result AS \"result: sqlx::types::Json>\",\n result_columns,\n status = 'success' AS \"success!\"\n FROM v2_job_completed\n WHERE id = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "result: sqlx::types::Json>", + "type_info": "Jsonb" + }, + { + "ordinal": 1, + "name": "result_columns", + "type_info": "TextArray" + }, + { + "ordinal": 2, + "name": "success!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + true, + true, + null + ] + }, + "hash": "d6f62e25faf271876874fc09ee460313159bb6ad91227f5dec37cd28006e2add" +} diff --git a/backend/.sqlx/query-829130d74c107e4e1a86f3e772657ae030463c139c39072777e453bfb7e9c0c3.json b/backend/.sqlx/query-d70ad8045589d746fc224ca950dcf967033942e489831cb5911723af1ec5161e.json similarity index 63% rename from backend/.sqlx/query-829130d74c107e4e1a86f3e772657ae030463c139c39072777e453bfb7e9c0c3.json rename to backend/.sqlx/query-d70ad8045589d746fc224ca950dcf967033942e489831cb5911723af1ec5161e.json index e00aba3aab..0873f799d6 100644 --- a/backend/.sqlx/query-829130d74c107e4e1a86f3e772657ae030463c139c39072777e453bfb7e9c0c3.json +++ b/backend/.sqlx/query-d70ad8045589d746fc224ca950dcf967033942e489831cb5911723af1ec5161e.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT job_kind = 'identity' FROM completed_job WHERE id = $1", + "query": "SELECT kind = 'identity' FROM v2_job WHERE id = $1", "describe": { "columns": [ { @@ -18,5 +18,5 @@ null ] }, - "hash": "829130d74c107e4e1a86f3e772657ae030463c139c39072777e453bfb7e9c0c3" + "hash": "d70ad8045589d746fc224ca950dcf967033942e489831cb5911723af1ec5161e" } diff --git a/backend/.sqlx/query-d988e91087695742d75946100cf2b7593cb8eed2a97411697819849958c022b3.json b/backend/.sqlx/query-d988e91087695742d75946100cf2b7593cb8eed2a97411697819849958c022b3.json new file mode 100644 index 0000000000..15625b26c2 --- /dev/null +++ b/backend/.sqlx/query-d988e91087695742d75946100cf2b7593cb8eed2a97411697819849958c022b3.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT result AS \"result!: Json>\"\n FROM v2_job_completed WHERE id = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "result!: Json>", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "d988e91087695742d75946100cf2b7593cb8eed2a97411697819849958c022b3" +} diff --git a/backend/.sqlx/query-e1923bc755bd6b8cc871ae9381a97d3c63a0fce3dd57c93f48a2136f7395fa3a.json b/backend/.sqlx/query-d99ae453289ae967ce92a221375aeca87ed9764bfa683b9d49294c24495ad392.json similarity index 76% rename from backend/.sqlx/query-e1923bc755bd6b8cc871ae9381a97d3c63a0fce3dd57c93f48a2136f7395fa3a.json rename to backend/.sqlx/query-d99ae453289ae967ce92a221375aeca87ed9764bfa683b9d49294c24495ad392.json index a477ebf141..739ec47346 100644 --- a/backend/.sqlx/query-e1923bc755bd6b8cc871ae9381a97d3c63a0fce3dd57c93f48a2136f7395fa3a.json +++ b/backend/.sqlx/query-d99ae453289ae967ce92a221375aeca87ed9764bfa683b9d49294c24495ad392.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT raw_flow->'failure_module' != 'null'::jsonb FROM job WHERE id = $1", + "query": "SELECT raw_flow->'failure_module' != 'null'::jsonb FROM v2_job WHERE id = $1", "describe": { "columns": [ { @@ -18,5 +18,5 @@ null ] }, - "hash": "e1923bc755bd6b8cc871ae9381a97d3c63a0fce3dd57c93f48a2136f7395fa3a" + "hash": "d99ae453289ae967ce92a221375aeca87ed9764bfa683b9d49294c24495ad392" } diff --git a/backend/.sqlx/query-61656548991bf6d44c839373cb3e29d3ca1170a2ac0d3dce0b5df0e8677a4874.json b/backend/.sqlx/query-db91141ae55b96a3237e05e3f127386339cb3c7a6f88bb2102498ec5d34f9537.json similarity index 51% rename from backend/.sqlx/query-61656548991bf6d44c839373cb3e29d3ca1170a2ac0d3dce0b5df0e8677a4874.json rename to backend/.sqlx/query-db91141ae55b96a3237e05e3f127386339cb3c7a6f88bb2102498ec5d34f9537.json index 1cb035f456..cc6474870c 100644 --- a/backend/.sqlx/query-61656548991bf6d44c839373cb3e29d3ca1170a2ac0d3dce0b5df0e8677a4874.json +++ b/backend/.sqlx/query-db91141ae55b96a3237e05e3f127386339cb3c7a6f88bb2102498ec5d34f9537.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "UPDATE queue\n SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT], $2)\n WHERE id = $3", + "query": "UPDATE v2_job_status\n SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT], $2)\n WHERE id = $3", "describe": { "columns": [], "parameters": { @@ -12,5 +12,5 @@ }, "nullable": [] }, - "hash": "61656548991bf6d44c839373cb3e29d3ca1170a2ac0d3dce0b5df0e8677a4874" + "hash": "db91141ae55b96a3237e05e3f127386339cb3c7a6f88bb2102498ec5d34f9537" } diff --git a/backend/.sqlx/query-dd5432830c1555ecbd6a0e45988fff455f3191f152e0900e09b71abb90d074ab.json b/backend/.sqlx/query-dd5432830c1555ecbd6a0e45988fff455f3191f152e0900e09b71abb90d074ab.json new file mode 100644 index 0000000000..f7690bb33a --- /dev/null +++ b/backend/.sqlx/query-dd5432830c1555ecbd6a0e45988fff455f3191f152e0900e09b71abb90d074ab.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, result AS \"result: Json>\"\n FROM v2_job_completed WHERE id = ANY($1) AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "result: Json>", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "UuidArray", + "Text" + ] + }, + "nullable": [ + false, + true + ] + }, + "hash": "dd5432830c1555ecbd6a0e45988fff455f3191f152e0900e09b71abb90d074ab" +} diff --git a/backend/.sqlx/query-ddf2eccb78a310ed00c7d8b9c3f05d394a7cbcf0038c72a78add5c7b02ef5927.json b/backend/.sqlx/query-ddf2eccb78a310ed00c7d8b9c3f05d394a7cbcf0038c72a78add5c7b02ef5927.json new file mode 100644 index 0000000000..c2dfed73a2 --- /dev/null +++ b/backend/.sqlx/query-ddf2eccb78a310ed00c7d8b9c3f05d394a7cbcf0038c72a78add5c7b02ef5927.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH worker_ids AS (SELECT unnest($1::text[]) as worker) \n SELECT worker_ids.worker FROM worker_ids \n LEFT JOIN worker_ping ON worker_ids.worker = worker_ping.worker \n WHERE worker_ping.worker IS NULL OR worker_ping.ping_at < now() - ('60 seconds')::interval", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "worker", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "TextArray" + ] + }, + "nullable": [ + null + ] + }, + "hash": "ddf2eccb78a310ed00c7d8b9c3f05d394a7cbcf0038c72a78add5c7b02ef5927" +} diff --git a/backend/.sqlx/query-defd99dd2427cdc54bb662d1ba3a1aea7f410ef204ec3465f4fb6c9acd256c95.json b/backend/.sqlx/query-defd99dd2427cdc54bb662d1ba3a1aea7f410ef204ec3465f4fb6c9acd256c95.json new file mode 100644 index 0000000000..6aab7bc049 --- /dev/null +++ b/backend/.sqlx/query-defd99dd2427cdc54bb662d1ba3a1aea7f410ef204ec3465f4fb6c9acd256c95.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_queue SET\n suspend = $1,\n suspend_until = now() + interval '14 day',\n running = true\n WHERE id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "defd99dd2427cdc54bb662d1ba3a1aea7f410ef204ec3465f4fb6c9acd256c95" +} diff --git a/backend/.sqlx/query-df0454f75e819d7d3d03fef0a7d5606940a089f6179e5443090afa9aba7c5b24.json b/backend/.sqlx/query-df0454f75e819d7d3d03fef0a7d5606940a089f6179e5443090afa9aba7c5b24.json deleted file mode 100644 index 166dc4a5d8..0000000000 --- a/backend/.sqlx/query-df0454f75e819d7d3d03fef0a7d5606940a089f6179e5443090afa9aba7c5b24.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT EXISTS (\n SELECT 1 FROM completed_job \n WHERE workspace_id = $2 \n AND (job_kind = 'appscript' OR job_kind = 'preview')\n AND created_by = 'anonymous' \n AND started_at > now() - interval '3 hours'\n AND script_path LIKE $3 || '/%' \n AND result @> ('{\"s3\":\"' || $1 || '\"}')::jsonb \n )", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "exists", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Text", - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "df0454f75e819d7d3d03fef0a7d5606940a089f6179e5443090afa9aba7c5b24" -} diff --git a/backend/.sqlx/query-e01c6ed633560647cfdf24f79298c0213ae4f66fdcc3a486120bb5116b1ce086.json b/backend/.sqlx/query-e01c6ed633560647cfdf24f79298c0213ae4f66fdcc3a486120bb5116b1ce086.json new file mode 100644 index 0000000000..ed8554404e --- /dev/null +++ b/backend/.sqlx/query-e01c6ed633560647cfdf24f79298c0213ae4f66fdcc3a486120bb5116b1ce086.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_runtime SET ping = NULL WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "e01c6ed633560647cfdf24f79298c0213ae4f66fdcc3a486120bb5116b1ce086" +} diff --git a/backend/.sqlx/query-e3ee812acd5bb9d5af39ca7dc61481ddd56a641e8e6f6c7a145f2cd5f3dc4602.json b/backend/.sqlx/query-e3ee812acd5bb9d5af39ca7dc61481ddd56a641e8e6f6c7a145f2cd5f3dc4602.json new file mode 100644 index 0000000000..2acb3c771e --- /dev/null +++ b/backend/.sqlx/query-e3ee812acd5bb9d5af39ca7dc61481ddd56a641e8e6f6c7a145f2cd5f3dc4602.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job SET args = $1, preprocessed = TRUE WHERE id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "e3ee812acd5bb9d5af39ca7dc61481ddd56a641e8e6f6c7a145f2cd5f3dc4602" +} diff --git a/backend/.sqlx/query-e4e87539ae18f7e5c6bd9a28d16b7527ececad87d218182ff723e6a6c43ecd50.json b/backend/.sqlx/query-e4e87539ae18f7e5c6bd9a28d16b7527ececad87d218182ff723e6a6c43ecd50.json deleted file mode 100644 index 3148f59bb4..0000000000 --- a/backend/.sqlx/query-e4e87539ae18f7e5c6bd9a28d16b7527ececad87d218182ff723e6a6c43ecd50.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT \n flow_status->>'step' = '0' \n AND (\n jsonb_array_length(flow_status->'modules') = 0 \n OR flow_status->'modules'->0->>'type' = 'WaitingForPriorSteps' \n OR (\n flow_status->'modules'->0->>'type' = 'Failure' \n AND flow_status->'modules'->0->>'job' = $1\n )\n )\n FROM completed_job WHERE id = $2 AND workspace_id = $3", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Text", - "Uuid", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "e4e87539ae18f7e5c6bd9a28d16b7527ececad87d218182ff723e6a6c43ecd50" -} diff --git a/backend/.sqlx/query-e653d36b607a16c0dfc0324690942ab25883b53a81ebb581fe019af2ec5eb567.json b/backend/.sqlx/query-e653d36b607a16c0dfc0324690942ab25883b53a81ebb581fe019af2ec5eb567.json new file mode 100644 index 0000000000..284cf3338f --- /dev/null +++ b/backend/.sqlx/query-e653d36b607a16c0dfc0324690942ab25883b53a81ebb581fe019af2ec5eb567.json @@ -0,0 +1,58 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n id AS \"id!\", workspace_id AS \"workspace_id!\", parent_job, is_flow_step,\n flow_status AS \"flow_status: Box\", last_ping, same_worker\n FROM v2_as_queue\n WHERE running = true AND suspend = 0 AND suspend_until IS null AND scheduled_for <= now()\n AND (job_kind = 'flow' OR job_kind = 'flowpreview' OR job_kind = 'flownode')\n AND last_ping IS NOT NULL AND last_ping < NOW() - ($1 || ' seconds')::interval\n AND canceled = false\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id!", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "workspace_id!", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "parent_job", + "type_info": "Uuid" + }, + { + "ordinal": 3, + "name": "is_flow_step", + "type_info": "Bool" + }, + { + "ordinal": 4, + "name": "flow_status: Box", + "type_info": "Jsonb" + }, + { + "ordinal": 5, + "name": "last_ping", + "type_info": "Timestamptz" + }, + { + "ordinal": 6, + "name": "same_worker", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + true, + true, + true, + true, + true, + true, + true + ] + }, + "hash": "e653d36b607a16c0dfc0324690942ab25883b53a81ebb581fe019af2ec5eb567" +} diff --git a/backend/.sqlx/query-e67509f23f769854c8ee74677b4634b90631242e3e3991659a785ff1256d5f4d.json b/backend/.sqlx/query-e67509f23f769854c8ee74677b4634b90631242e3e3991659a785ff1256d5f4d.json new file mode 100644 index 0000000000..a7662bec95 --- /dev/null +++ b/backend/.sqlx/query-e67509f23f769854c8ee74677b4634b90631242e3e3991659a785ff1256d5f4d.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT canceled_by IS NOT NULL AS \"canceled!\" FROM v2_job_queue WHERE id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "canceled!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + null + ] + }, + "hash": "e67509f23f769854c8ee74677b4634b90631242e3e3991659a785ff1256d5f4d" +} diff --git a/backend/.sqlx/query-ebd25047d14bec8457465ba6ed9b6b15b8f2303157b4c6e80ac9e3d84a57d8b1.json b/backend/.sqlx/query-ebd25047d14bec8457465ba6ed9b6b15b8f2303157b4c6e80ac9e3d84a57d8b1.json new file mode 100644 index 0000000000..6c92e12f6a --- /dev/null +++ b/backend/.sqlx/query-ebd25047d14bec8457465ba6ed9b6b15b8f2303157b4c6e80ac9e3d84a57d8b1.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT worker, array_agg(v2_job_queue.id) as ids FROM v2_job_queue LEFT JOIN v2_job ON v2_job_queue.id = v2_job.id LEFT JOIN v2_job_runtime ON v2_job_queue.id = v2_job_runtime.id WHERE v2_job_queue.created_at < now() - ('60 seconds')::interval \n AND running = true AND ping IS NULL AND same_worker = true AND worker IS NOT NULL GROUP BY worker", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "worker", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "ids", + "type_info": "UuidArray" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + true, + null + ] + }, + "hash": "ebd25047d14bec8457465ba6ed9b6b15b8f2303157b4c6e80ac9e3d84a57d8b1" +} diff --git a/backend/.sqlx/query-803b9c1373541cf52f416cde9e9e99ab79072e21dfaafb498aac25a059bd2f30.json b/backend/.sqlx/query-ecd62c48fe2fba2fc2582e9e7ae5590d5dea8c67f6ae7b14743ac4f265dd89a3.json similarity index 55% rename from backend/.sqlx/query-803b9c1373541cf52f416cde9e9e99ab79072e21dfaafb498aac25a059bd2f30.json rename to backend/.sqlx/query-ecd62c48fe2fba2fc2582e9e7ae5590d5dea8c67f6ae7b14743ac4f265dd89a3.json index b416f93d3b..7431f3e1fb 100644 --- a/backend/.sqlx/query-803b9c1373541cf52f416cde9e9e99ab79072e21dfaafb498aac25a059bd2f30.json +++ b/backend/.sqlx/query-ecd62c48fe2fba2fc2582e9e7ae5590d5dea8c67f6ae7b14743ac4f265dd89a3.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "DELETE FROM job WHERE id = ANY($1)", + "query": "DELETE FROM v2_job WHERE id = ANY($1)", "describe": { "columns": [], "parameters": { @@ -10,5 +10,5 @@ }, "nullable": [] }, - "hash": "803b9c1373541cf52f416cde9e9e99ab79072e21dfaafb498aac25a059bd2f30" + "hash": "ecd62c48fe2fba2fc2582e9e7ae5590d5dea8c67f6ae7b14743ac4f265dd89a3" } diff --git a/backend/.sqlx/query-ef8413620c6860c3bf200894c5917c9209817a9cd8eb3bcd05a74f55423054ae.json b/backend/.sqlx/query-ef8413620c6860c3bf200894c5917c9209817a9cd8eb3bcd05a74f55423054ae.json new file mode 100644 index 0000000000..58485a7cfb --- /dev/null +++ b/backend/.sqlx/query-ef8413620c6860c3bf200894c5917c9209817a9cd8eb3bcd05a74f55423054ae.json @@ -0,0 +1,30 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT created_by AS \"created_by!\", args as \"args: sqlx::types::Json>\"\n FROM v2_job\n WHERE id = $1 AND workspace_id = $2 AND ($3::text[] IS NULL OR tag = ANY($3))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "created_by!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "args: sqlx::types::Json>", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "TextArray" + ] + }, + "nullable": [ + false, + true + ] + }, + "hash": "ef8413620c6860c3bf200894c5917c9209817a9cd8eb3bcd05a74f55423054ae" +} diff --git a/backend/.sqlx/query-f33dd637181439ba7fa441dbd7d7430c1ccc0f410c377cde71c70c9211f9c1df.json b/backend/.sqlx/query-f0d3c2641924b1f1026d4dfa19290850047653549ee20870e6272cb7d4cfb9aa.json similarity index 64% rename from backend/.sqlx/query-f33dd637181439ba7fa441dbd7d7430c1ccc0f410c377cde71c70c9211f9c1df.json rename to backend/.sqlx/query-f0d3c2641924b1f1026d4dfa19290850047653549ee20870e6272cb7d4cfb9aa.json index 536a8116f4..5fa15ac703 100644 --- a/backend/.sqlx/query-f33dd637181439ba7fa441dbd7d7430c1ccc0f410c377cde71c70c9211f9c1df.json +++ b/backend/.sqlx/query-f0d3c2641924b1f1026d4dfa19290850047653549ee20870e6272cb7d4cfb9aa.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT parent_job FROM queue WHERE id = $1 AND workspace_id = $2", + "query": "SELECT parent_job FROM v2_job WHERE id = $1 AND workspace_id = $2", "describe": { "columns": [ { @@ -19,5 +19,5 @@ true ] }, - "hash": "f33dd637181439ba7fa441dbd7d7430c1ccc0f410c377cde71c70c9211f9c1df" + "hash": "f0d3c2641924b1f1026d4dfa19290850047653549ee20870e6272cb7d4cfb9aa" } diff --git a/backend/.sqlx/query-da9114fc6689ebc78422b3572de1ed44050bd28212540db50fe235463c15a900.json b/backend/.sqlx/query-f0fdeb7aea3e71099e7db0f4343bbd7ec86610ddc8589bf5b606fab0947c8b75.json similarity index 63% rename from backend/.sqlx/query-da9114fc6689ebc78422b3572de1ed44050bd28212540db50fe235463c15a900.json rename to backend/.sqlx/query-f0fdeb7aea3e71099e7db0f4343bbd7ec86610ddc8589bf5b606fab0947c8b75.json index 0e259af63c..26111685ba 100644 --- a/backend/.sqlx/query-da9114fc6689ebc78422b3572de1ed44050bd28212540db50fe235463c15a900.json +++ b/backend/.sqlx/query-f0fdeb7aea3e71099e7db0f4343bbd7ec86610ddc8589bf5b606fab0947c8b75.json @@ -1,11 +1,11 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n queue.job_kind AS \"job_kind: JobKind\",\n queue.script_hash AS \"script_hash: ScriptHash\",\n queue.raw_flow AS \"raw_flow: sqlx::types::Json>\",\n completed_job.parent_job AS \"parent_job: Uuid\",\n completed_job.created_at AS \"created_at: chrono::NaiveDateTime\",\n completed_job.created_by AS \"created_by!\",\n queue.script_path,\n queue.args AS \"args: sqlx::types::Json>\"\n FROM queue\n JOIN completed_job ON completed_job.parent_job = queue.id\n WHERE completed_job.id = $1 AND completed_job.workspace_id = $2\n LIMIT 1", + "query": "SELECT\n v2_as_queue.job_kind AS \"job_kind!: JobKind\",\n v2_as_queue.script_hash AS \"script_hash: ScriptHash\",\n v2_as_queue.raw_flow AS \"raw_flow: sqlx::types::Json>\",\n v2_as_completed_job.parent_job AS \"parent_job: Uuid\",\n v2_as_completed_job.created_at AS \"created_at!: chrono::NaiveDateTime\",\n v2_as_completed_job.created_by AS \"created_by!\",\n v2_as_queue.script_path,\n v2_as_queue.args AS \"args: sqlx::types::Json>\"\n FROM v2_as_queue\n JOIN v2_as_completed_job ON v2_as_completed_job.parent_job = v2_as_queue.id\n WHERE v2_as_completed_job.id = $1 AND v2_as_completed_job.workspace_id = $2\n LIMIT 1", "describe": { "columns": [ { "ordinal": 0, - "name": "job_kind: JobKind", + "name": "job_kind!: JobKind", "type_info": { "Custom": { "name": "job_kind", @@ -51,7 +51,7 @@ }, { "ordinal": 4, - "name": "created_at: chrono::NaiveDateTime", + "name": "created_at!: chrono::NaiveDateTime", "type_info": "Timestamptz" }, { @@ -77,15 +77,15 @@ ] }, "nullable": [ - false, true, true, true, - false, - false, + true, + true, + true, true, true ] }, - "hash": "da9114fc6689ebc78422b3572de1ed44050bd28212540db50fe235463c15a900" + "hash": "f0fdeb7aea3e71099e7db0f4343bbd7ec86610ddc8589bf5b606fab0947c8b75" } diff --git a/backend/.sqlx/query-1438e8dc5738fc69bc6601eb11729610f671b7df0ab25da058e16c6654279d61.json b/backend/.sqlx/query-f3571e1d2b57011e5f6a38725eb42d909456d28a98563923cca43e760862e5e0.json similarity index 70% rename from backend/.sqlx/query-1438e8dc5738fc69bc6601eb11729610f671b7df0ab25da058e16c6654279d61.json rename to backend/.sqlx/query-f3571e1d2b57011e5f6a38725eb42d909456d28a98563923cca43e760862e5e0.json index 452510a97f..804f35777e 100644 --- a/backend/.sqlx/query-1438e8dc5738fc69bc6601eb11729610f671b7df0ab25da058e16c6654279d61.json +++ b/backend/.sqlx/query-f3571e1d2b57011e5f6a38725eb42d909456d28a98563923cca43e760862e5e0.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT flow_status->'user_states'->$1\n FROM queue\n WHERE id = $2 AND workspace_id = $3\n ", + "query": "\n SELECT flow_status->'user_states'->$1\n FROM v2_as_queue\n WHERE id = $2 AND workspace_id = $3\n ", "describe": { "columns": [ { @@ -20,5 +20,5 @@ null ] }, - "hash": "1438e8dc5738fc69bc6601eb11729610f671b7df0ab25da058e16c6654279d61" + "hash": "f3571e1d2b57011e5f6a38725eb42d909456d28a98563923cca43e760862e5e0" } diff --git a/backend/.sqlx/query-f3c78cb67379f9407f1f32ce3387184d7f4fcb04c07a8e2f07c5effc10f8fd5c.json b/backend/.sqlx/query-f3c78cb67379f9407f1f32ce3387184d7f4fcb04c07a8e2f07c5effc10f8fd5c.json new file mode 100644 index 0000000000..0cf878b15c --- /dev/null +++ b/backend/.sqlx/query-f3c78cb67379f9407f1f32ce3387184d7f4fcb04c07a8e2f07c5effc10f8fd5c.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status\n SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'approvers'], $2)\n WHERE id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Jsonb", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "f3c78cb67379f9407f1f32ce3387184d7f4fcb04c07a8e2f07c5effc10f8fd5c" +} diff --git a/backend/.sqlx/query-f3f1b312bff773fe04a5dbfc2c03963042fa19d5eb15cfb3e0c291a29482aa5b.json b/backend/.sqlx/query-f3f1b312bff773fe04a5dbfc2c03963042fa19d5eb15cfb3e0c291a29482aa5b.json deleted file mode 100644 index 99d4d7075c..0000000000 --- a/backend/.sqlx/query-f3f1b312bff773fe04a5dbfc2c03963042fa19d5eb15cfb3e0c291a29482aa5b.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE job_logs\n SET logs = '##DELETED##'\n WHERE job_id = $1", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "f3f1b312bff773fe04a5dbfc2c03963042fa19d5eb15cfb3e0c291a29482aa5b" -} diff --git a/backend/.sqlx/query-f4a792eca82e9974d13d0731e7862e3c75dbee326e0856944539b2c5574cb6d3.json b/backend/.sqlx/query-f4a792eca82e9974d13d0731e7862e3c75dbee326e0856944539b2c5574cb6d3.json deleted file mode 100644 index a0ec3da44b..0000000000 --- a/backend/.sqlx/query-f4a792eca82e9974d13d0731e7862e3c75dbee326e0856944539b2c5574cb6d3.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE queue SET flow_status = jsonb_set(jsonb_set(COALESCE(flow_status, '{}'::jsonb), array[$1], COALESCE(flow_status->$1, '{}'::jsonb)), array[$1, 'duration_ms'], to_jsonb($2::bigint)) WHERE id = $3 AND workspace_id = $4", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Int8", - "Uuid", - "Text" - ] - }, - "nullable": [] - }, - "hash": "f4a792eca82e9974d13d0731e7862e3c75dbee326e0856944539b2c5574cb6d3" -} diff --git a/backend/.sqlx/query-f64ae18811e211dbf0cb98b43d3b018b0dcc0abc7e4a1f0b45885cfe18efd9b2.json b/backend/.sqlx/query-f64ae18811e211dbf0cb98b43d3b018b0dcc0abc7e4a1f0b45885cfe18efd9b2.json new file mode 100644 index 0000000000..99e04c9bff --- /dev/null +++ b/backend/.sqlx/query-f64ae18811e211dbf0cb98b43d3b018b0dcc0abc7e4a1f0b45885cfe18efd9b2.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "create index concurrently if not exists ix_job_workspace_id_created_at_new_3 ON v2_job (workspace_id, created_at DESC)", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "f64ae18811e211dbf0cb98b43d3b018b0dcc0abc7e4a1f0b45885cfe18efd9b2" +} diff --git a/backend/.sqlx/query-f746c5d2ffa24dd75124b5f8d04f587c9581ef407417ec979b8a3ab07816cc69.json b/backend/.sqlx/query-f746c5d2ffa24dd75124b5f8d04f587c9581ef407417ec979b8a3ab07816cc69.json new file mode 100644 index 0000000000..c65cfb397a --- /dev/null +++ b/backend/.sqlx/query-f746c5d2ffa24dd75124b5f8d04f587c9581ef407417ec979b8a3ab07816cc69.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job SET trigger = REGEXP_REPLACE(trigger,'u/' || $2 || '/(.*)','u/' || $1 || '/\\1') WHERE trigger LIKE ('u/' || $2 || '/%') AND workspace_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "f746c5d2ffa24dd75124b5f8d04f587c9581ef407417ec979b8a3ab07816cc69" +} diff --git a/backend/.sqlx/query-f790a016c4a1333e3d4d1ce468a1679ba5e61ea88a5c65be9696312d2f455508.json b/backend/.sqlx/query-f790a016c4a1333e3d4d1ce468a1679ba5e61ea88a5c65be9696312d2f455508.json deleted file mode 100644 index d64348ef46..0000000000 --- a/backend/.sqlx/query-f790a016c4a1333e3d4d1ce468a1679ba5e61ea88a5c65be9696312d2f455508.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n DELETE\n FROM parallel_monitor_lock\n WHERE last_ping IS NOT NULL AND last_ping < NOW() - ($1 || ' seconds')::interval \n RETURNING parent_flow_id, job_id, last_ping\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "parent_flow_id", - "type_info": "Uuid" - }, - { - "ordinal": 1, - "name": "job_id", - "type_info": "Uuid" - }, - { - "ordinal": 2, - "name": "last_ping", - "type_info": "Timestamptz" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - false, - false, - true - ] - }, - "hash": "f790a016c4a1333e3d4d1ce468a1679ba5e61ea88a5c65be9696312d2f455508" -} diff --git a/backend/.sqlx/query-f822df86bf511fe0d3044b69525a6ff2524167877929886adca9b0fa3d3fee45.json b/backend/.sqlx/query-f822df86bf511fe0d3044b69525a6ff2524167877929886adca9b0fa3d3fee45.json new file mode 100644 index 0000000000..3adc1f8cd5 --- /dev/null +++ b/backend/.sqlx/query-f822df86bf511fe0d3044b69525a6ff2524167877929886adca9b0fa3d3fee45.json @@ -0,0 +1,40 @@ +{ + "db_name": "PostgreSQL", + "query": "\n DELETE\n FROM parallel_monitor_lock\n WHERE last_ping IS NOT NULL AND last_ping < NOW() - ($1 || ' seconds')::interval \n RETURNING parent_flow_id, job_id, last_ping, (SELECT workspace_id FROM v2_job_queue q\n WHERE q.id = parent_flow_id AND q.running = true AND q.canceled_by IS NULL\n ) AS workspace_id\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "parent_flow_id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "job_id", + "type_info": "Uuid" + }, + { + "ordinal": 2, + "name": "last_ping", + "type_info": "Timestamptz" + }, + { + "ordinal": 3, + "name": "workspace_id", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + true, + null + ] + }, + "hash": "f822df86bf511fe0d3044b69525a6ff2524167877929886adca9b0fa3d3fee45" +} diff --git a/backend/.sqlx/query-f916ec232837ece9323675e5f5142e7285f4266a1471e5ffdefadf421a67e44b.json b/backend/.sqlx/query-f916ec232837ece9323675e5f5142e7285f4266a1471e5ffdefadf421a67e44b.json deleted file mode 100644 index 2cf1fb31e9..0000000000 --- a/backend/.sqlx/query-f916ec232837ece9323675e5f5142e7285f4266a1471e5ffdefadf421a67e44b.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE queue\n SET flow_status = JSONB_SET(\n JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4),\n ARRAY['modules', $1::TEXT, 'iterator', 'index'],\n ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb\n ),\n last_ping = NULL\n WHERE id = $2\n RETURNING (flow_status->'modules'->$1::int->'iterator'->>'index')::int", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "int4", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Int4", - "Uuid", - "Text", - "Jsonb" - ] - }, - "nullable": [ - null - ] - }, - "hash": "f916ec232837ece9323675e5f5142e7285f4266a1471e5ffdefadf421a67e44b" -} diff --git a/backend/.sqlx/query-f9fc0084fe086ef80005bb64a8bb6b493e53583017c18e2ab44f44125c52d548.json b/backend/.sqlx/query-f9fc0084fe086ef80005bb64a8bb6b493e53583017c18e2ab44f44125c52d548.json deleted file mode 100644 index bad609fb21..0000000000 --- a/backend/.sqlx/query-f9fc0084fe086ef80005bb64a8bb6b493e53583017c18e2ab44f44125c52d548.json +++ /dev/null @@ -1,104 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO completed_job AS cj\n ( workspace_id\n , id\n , parent_job\n , created_by\n , created_at\n , started_at\n , duration_ms\n , success\n , script_hash\n , script_path\n , args\n , result\n , raw_code\n , raw_lock\n , canceled\n , canceled_by\n , canceled_reason\n , job_kind\n , schedule_path\n , permissioned_as\n , flow_status\n , raw_flow\n , is_flow_step\n , is_skipped\n , language\n , email\n , visible_to_owner\n , mem_peak\n , tag\n , priority\n )\n VALUES ($1, $2, $3, $4, $5, COALESCE($6, now()), COALESCE($30::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE($6, now()))))*1000), $7, $8, $9,$10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29)\n ON CONFLICT (id) DO UPDATE SET success = $7, result = $11 RETURNING duration_ms", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "duration_ms", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Varchar", - "Uuid", - "Uuid", - "Varchar", - "Timestamptz", - "Timestamptz", - "Bool", - "Int8", - "Varchar", - "Jsonb", - "Jsonb", - "Text", - "Text", - "Bool", - "Varchar", - "Text", - { - "Custom": { - "name": "job_kind", - "kind": { - "Enum": [ - "script", - "preview", - "flow", - "dependencies", - "flowpreview", - "script_hub", - "identity", - "flowdependencies", - "http", - "graphql", - "postgresql", - "noop", - "appdependencies", - "deploymentcallback", - "singlescriptflow", - "flowscript", - "flownode", - "appscript" - ] - } - } - }, - "Varchar", - "Varchar", - "Jsonb", - "Jsonb", - "Bool", - "Bool", - { - "Custom": { - "name": "script_lang", - "kind": { - "Enum": [ - "python3", - "deno", - "go", - "bash", - "postgresql", - "nativets", - "bun", - "mysql", - "bigquery", - "snowflake", - "graphql", - "powershell", - "mssql", - "php", - "bunnative", - "rust", - "ansible", - "csharp", - "oracledb" - ] - } - } - }, - "Varchar", - "Bool", - "Int4", - "Varchar", - "Int2", - "Int8" - ] - }, - "nullable": [ - false - ] - }, - "hash": "f9fc0084fe086ef80005bb64a8bb6b493e53583017c18e2ab44f44125c52d548" -} diff --git a/backend/.sqlx/query-fb1a32318b35ec5c8129eb3660b79eb5e6d1e01fcf01cc05d3c7ddf47295c2f5.json b/backend/.sqlx/query-fb1a32318b35ec5c8129eb3660b79eb5e6d1e01fcf01cc05d3c7ddf47295c2f5.json deleted file mode 100644 index 6997a5e2f4..0000000000 --- a/backend/.sqlx/query-fb1a32318b35ec5c8129eb3660b79eb5e6d1e01fcf01cc05d3c7ddf47295c2f5.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT raw_code, raw_lock, raw_flow AS \"raw_flow: Json>\"\n FROM job WHERE id = $1 AND workspace_id = $2 LIMIT 1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "raw_code", - "type_info": "Text" - }, - { - "ordinal": 1, - "name": "raw_lock", - "type_info": "Text" - }, - { - "ordinal": 2, - "name": "raw_flow: Json>", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [ - "Uuid", - "Text" - ] - }, - "nullable": [ - true, - true, - true - ] - }, - "hash": "fb1a32318b35ec5c8129eb3660b79eb5e6d1e01fcf01cc05d3c7ddf47295c2f5" -} diff --git a/backend/.sqlx/query-fdedd3909a97db5d43d9c46ff77b800b8efd647121b538deb023f96dbaac3715.json b/backend/.sqlx/query-fdedd3909a97db5d43d9c46ff77b800b8efd647121b538deb023f96dbaac3715.json new file mode 100644 index 0000000000..905707255f --- /dev/null +++ b/backend/.sqlx/query-fdedd3909a97db5d43d9c46ff77b800b8efd647121b538deb023f96dbaac3715.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT args AS \"args: Json>>\"\n FROM v2_job WHERE id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "args: Json>>", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + true + ] + }, + "hash": "fdedd3909a97db5d43d9c46ff77b800b8efd647121b538deb023f96dbaac3715" +} diff --git a/backend/.sqlx/query-fe5e941310ffdbbe7cc5f3e2beb213999ff64bbe4f8e2532a7aa48184cb008e7.json b/backend/.sqlx/query-fe5e941310ffdbbe7cc5f3e2beb213999ff64bbe4f8e2532a7aa48184cb008e7.json deleted file mode 100644 index ca0ef918e9..0000000000 --- a/backend/.sqlx/query-fe5e941310ffdbbe7cc5f3e2beb213999ff64bbe4f8e2532a7aa48184cb008e7.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE completed_job SET schedule_path = REGEXP_REPLACE(schedule_path,'u/' || $2 || '/(.*)','u/' || $1 || '/\\1') WHERE schedule_path LIKE ('u/' || $2 || '/%') AND workspace_id = $3", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "fe5e941310ffdbbe7cc5f3e2beb213999ff64bbe4f8e2532a7aa48184cb008e7" -} diff --git a/backend/.sqlx/query-9d3556319411a27a875bf6cf0e5eda837cc63e4d8be912c0b5bfeea4a0c8db2e.json b/backend/.sqlx/query-ff0403790674cdb07022af71c2377afbd8b3a660b3be27514b517c077c63c238.json similarity index 76% rename from backend/.sqlx/query-9d3556319411a27a875bf6cf0e5eda837cc63e4d8be912c0b5bfeea4a0c8db2e.json rename to backend/.sqlx/query-ff0403790674cdb07022af71c2377afbd8b3a660b3be27514b517c077c63c238.json index 2abc565def..6a4a3b3d75 100644 --- a/backend/.sqlx/query-9d3556319411a27a875bf6cf0e5eda837cc63e4d8be912c0b5bfeea4a0c8db2e.json +++ b/backend/.sqlx/query-ff0403790674cdb07022af71c2377afbd8b3a660b3be27514b517c077c63c238.json @@ -1,8 +1,14 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO queue (id, script_hash, script_path, job_kind, language, tag, created_by, permissioned_as, email, scheduled_for, workspace_id) (SELECT gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8, $9, $10 FROM generate_series(1, $11))", + "query": "INSERT INTO v2_job (id, runnable_id, runnable_path, kind, script_lang, tag, created_by, permissioned_as, permissioned_as_email, workspace_id) (SELECT gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8, $9 FROM generate_series(1, $10)) RETURNING id", "describe": { - "columns": [], + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + } + ], "parameters": { "Left": [ "Int8", @@ -66,12 +72,13 @@ "Varchar", "Varchar", "Varchar", - "Timestamptz", "Varchar", "Int4" ] }, - "nullable": [] + "nullable": [ + false + ] }, - "hash": "9d3556319411a27a875bf6cf0e5eda837cc63e4d8be912c0b5bfeea4a0c8db2e" + "hash": "ff0403790674cdb07022af71c2377afbd8b3a660b3be27514b517c077c63c238" } diff --git a/backend/Cargo.lock b/backend/Cargo.lock index a50ea3f309..335f1a7fa5 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -442,7 +442,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -538,7 +538,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -560,7 +560,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -596,7 +596,7 @@ checksum = "644dd749086bf3771a2fbc5f256fdb982d53f011c7d5d560304eafeecebce79d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -651,9 +651,9 @@ checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "aws-config" -version = "1.5.15" +version = "1.5.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc47e70fc35d054c8fcd296d47a61711f043ac80534a10b4f741904f81e73a90" +checksum = "50236e4d60fe8458de90a71c0922c761e41755adf091b1b03de1cef537179915" dependencies = [ "aws-credential-types", "aws-runtime", @@ -693,9 +693,9 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.5.4" +version = "1.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bee7643696e7fdd74c10f9eb42848a87fe469d35eae9c3323f80aa98f350baac" +checksum = "76dd04d39cc12844c0994f2c9c5a6f5184c22e9188ec1ff723de41910a21dcad" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -713,14 +713,14 @@ dependencies = [ "percent-encoding", "pin-project-lite", "tracing", - "uuid 1.12.1", + "uuid 1.13.1", ] [[package]] name = "aws-sdk-sso" -version = "1.57.0" +version = "1.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c54bab121fe1881a74c338c5f723d1592bf3b53167f80268a1274f404e1acc38" +checksum = "16ff718c9ee45cc1ebd4774a0e086bb80a6ab752b4902edf1c9f56b86ee1f770" dependencies = [ "aws-credential-types", "aws-runtime", @@ -740,9 +740,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.58.0" +version = "1.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c8234fd024f7ac61c4e44ea008029bde934250f371efe7d4a39708397b1080c" +checksum = "5183e088715cc135d8d396fdd3bc02f018f0da4c511f53cb8d795b6a31c55809" dependencies = [ "aws-credential-types", "aws-runtime", @@ -762,9 +762,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.58.0" +version = "1.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba60e1d519d6f23a9df712c04fdeadd7872ac911c84b2f62a8bda92e129b7962" +checksum = "c9f944ef032717596639cea4a2118a3a457268ef51bbb5fde9637e54c465da00" dependencies = [ "aws-credential-types", "aws-runtime", @@ -785,9 +785,9 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.2.7" +version = "1.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "690118821e46967b3c4501d67d7d52dd75106a9c54cf36cefa1985cedbe94e05" +checksum = "0bc5bbd1e4a2648fd8c5982af03935972c24a2f9846b396de661d351ee3ce837" dependencies = [ "aws-credential-types", "aws-smithy-http", @@ -858,9 +858,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.7.7" +version = "1.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "865f7050bbc7107a6c98a397a9fcd9413690c27fa718446967cf03b2d3ac517e" +checksum = "d526a12d9ed61fadefda24abe2e682892ba288c2018bcb38b1b4c111d13f6d92" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -902,9 +902,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.2.12" +version = "1.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a28f6feb647fb5e0d5b50f0472c19a7db9462b74e2fec01bb0b44eedcc834e97" +checksum = "c7b8a53819e42f10d0821f56da995e1470b199686a1809168db6ca485665f042" dependencies = [ "base64-simd 0.8.0", "bytes", @@ -937,9 +937,9 @@ dependencies = [ [[package]] name = "aws-types" -version = "1.3.4" +version = "1.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0df5a18c4f951c645300d365fec53a61418bcf4650f604f85fe2a665bfaa0c2" +checksum = "dfbd0a668309ec1f66c0f6bda4840dd6d4796ae26d699ebc266d7cc95c6d040f" dependencies = [ "aws-credential-types", "aws-smithy-async", @@ -1136,7 +1136,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.96", + "syn 2.0.98", "which 4.4.2", ] @@ -1157,7 +1157,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -1332,7 +1332,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -1434,7 +1434,7 @@ checksum = "3fa76293b4f7bb636ab88fd78228235b5248b4d05cc589aed610f954af5d7c7a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -1445,9 +1445,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b" +checksum = "f61dac84819c6588b558454b194026eb1f09c293b9036ae9b159e74e73ab6cf9" dependencies = [ "serde", ] @@ -1552,9 +1552,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.11" +version = "1.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4730490333d58093109dc02c23174c3f4d490998c3fed3cc8e82d57afedb9cf" +checksum = "c7777341816418c02e033934a09f20dc0ccaf65a5201ef8a450ae0105a573fda" dependencies = [ "jobserver", "libc", @@ -1668,9 +1668,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.27" +version = "4.5.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "769b0145982b4b48713e01ec42d61614425f27b7058bda7180a3a41f30104796" +checksum = "3e77c3243bd94243c03672cb5154667347c457ca271254724f9f393aee1c05ff" dependencies = [ "clap_builder", "clap_derive", @@ -1690,14 +1690,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.24" +version = "4.5.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54b755194d6389280185988721fffba69495eed5ee9feeee9a599b53db80318c" +checksum = "bf4ced95c6f4a675af3da73304b9ac4ed991640c36374e4b46795c49e17cf1ed" dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -1708,9 +1708,9 @@ checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" [[package]] name = "cmake" -version = "0.1.53" +version = "0.1.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e24a03c8b52922d68a1589ad61032f2c1aa5a8158d2aa0d93c6e9534944bbad6" +checksum = "e7caa3f9de89ddbe2c607f4101924c5abec803763ae9534e4f4d7d8f84aa81f0" dependencies = [ "cc", ] @@ -1723,12 +1723,11 @@ checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" [[package]] name = "comfy-table" -version = "7.1.3" +version = "7.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24f165e7b643266ea80cb858aed492ad9280e3e05ce24d4a99d7d7b889b6a4d9" +checksum = "4a65ebfec4fb190b6f90e944a817d60499ee0744e582530e2c9900a22e591d9a" dependencies = [ - "strum 0.26.3", - "strum_macros 0.26.4", + "unicode-segmentation", "unicode-width 0.2.0", ] @@ -2082,7 +2081,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -2154,7 +2153,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -2187,7 +2186,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core 0.20.10", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -2205,9 +2204,9 @@ dependencies = [ [[package]] name = "data-encoding" -version = "2.7.0" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e60eed09d8c01d3cee5b7d30acb059b76614c918fa0f992e0dd6eeb10daad6f" +checksum = "575f75dfd25738df5b91b8e43e14d44bda14637a58fae779fd2b064f8bf3e010" [[package]] name = "data-url" @@ -2264,7 +2263,7 @@ dependencies = [ "tokio", "tokio-util", "url", - "uuid 1.12.1", + "uuid 1.13.1", "xz2", "zstd", ] @@ -2364,7 +2363,7 @@ dependencies = [ "regex", "sha2 0.10.8", "unicode-segmentation", - "uuid 1.12.1", + "uuid 1.13.1", ] [[package]] @@ -2531,7 +2530,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d" dependencies = [ "serde", - "uuid 1.12.1", + "uuid 1.13.1", ] [[package]] @@ -2714,7 +2713,7 @@ dependencies = [ "stringcase", "strum 0.25.0", "strum_macros 0.25.3", - "syn 2.0.96", + "syn 2.0.98", "thiserror 1.0.69", ] @@ -2826,7 +2825,7 @@ dependencies = [ "serde", "thiserror 1.0.69", "tokio", - "uuid 1.12.1", + "uuid 1.13.1", ] [[package]] @@ -2892,15 +2891,15 @@ dependencies = [ [[package]] name = "derive_more" -version = "0.99.18" +version = "0.99.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f33878137e4dafd7fa914ad4e259e18a4e8e532b9617a2d0150262bf53abfce" +checksum = "3da29a38df43d6f156149c9b43ded5e018ddff2a855cf2cfd62e8cd7d079c69f" dependencies = [ "convert_case 0.4.0", "proc-macro2", "quote", "rustc_version 0.4.1", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -3005,7 +3004,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -3028,7 +3027,7 @@ checksum = "f2b99bf03862d7f545ebc28ddd33a665b50865f4dfd84031a393823879bd4c54" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -3185,7 +3184,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -3205,7 +3204,7 @@ checksum = "fc4caf64a58d7a6d65ab00639b046ff54399a39f5f2554728895ace4b297cd79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -3410,7 +3409,7 @@ checksum = "32016f1242eb82af5474752d00fd8ebcd9004bd69b462b1c91de833972d08ed4" dependencies = [ "proc-macro2", "swc_macros_common", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -3534,7 +3533,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -3799,7 +3798,7 @@ checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -3964,7 +3963,7 @@ checksum = "f208758247e68e239acaa059e72e4ce1f30f2a4b6523f19c1b923d25b7e9cceb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -4019,9 +4018,9 @@ dependencies = [ [[package]] name = "hickory-proto" -version = "0.24.2" +version = "0.24.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "447afdcdb8afb9d0a852af6dc65d9b285ce720ed7a59e42a8bf2e931c67bc1b5" +checksum = "2ad3d6d98c648ed628df039541a5577bee1a7c83e9e16fe3dbedeea4cdfeb971" dependencies = [ "async-trait", "cfg-if", @@ -4044,9 +4043,9 @@ dependencies = [ [[package]] name = "hickory-resolver" -version = "0.24.2" +version = "0.24.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a2e2aba9c389ce5267d31cf1e4dace82390ae276b0b364ea55630b1fa1b44b4" +checksum = "dcf287bde7b776e85d7188e6e5db7cf410a2f9531fe82817eb87feed034c8d14" dependencies = [ "cfg-if", "futures-util", @@ -4527,7 +4526,7 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -4649,7 +4648,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -5099,9 +5098,9 @@ dependencies = [ [[package]] name = "malachite-base" -version = "0.4.21" +version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc7d99f6446827fb878a005ea5cf7db1d78d236391733012216a9b8eb5dbd824" +checksum = "5ea0ed76adf7defc1a92240b5c36d5368cfe9251640dcce5bd2d0b7c1fd87aeb" dependencies = [ "hashbrown 0.14.5", "itertools 0.11.0", @@ -5124,9 +5123,9 @@ dependencies = [ [[package]] name = "malachite-nz" -version = "0.4.21" +version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f36157fa6b969ca652feefe159e202ebd85683e57463074492a7ec3eaa468c97" +checksum = "34a79feebb2bc9aa7762047c8e5495269a367da6b5a90a99882a0aeeac1841f7" dependencies = [ "itertools 0.11.0", "libm", @@ -5135,9 +5134,9 @@ dependencies = [ [[package]] name = "malachite-q" -version = "0.4.21" +version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dde29a4fcf7025f482745247951f7c5d0459671f59aedf09035b2e2c3746fdf" +checksum = "50f235d5747b1256b47620f5640c2a17a88c7569eebdf27cd9cb130e1a619191" dependencies = [ "itertools 0.11.0", "malachite-base", @@ -5302,7 +5301,7 @@ checksum = "a7ce64b975ed4f123575d11afd9491f2e37bbd5813fbfbc0f09ae1fbddea74e0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -5341,7 +5340,7 @@ dependencies = [ "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", "termcolor", "thiserror 1.0.69", ] @@ -5406,7 +5405,7 @@ dependencies = [ "sha2 0.10.8", "subprocess", "thiserror 1.0.69", - "uuid 1.12.1", + "uuid 1.13.1", "zstd", ] @@ -5616,7 +5615,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -5695,15 +5694,15 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.20.2" +version = "1.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" +checksum = "945462a4b81e43c4e3ba96bd7b49d834c6f61198356aa858733bc4acf3cbe62e" [[package]] name = "oneshot" -version = "0.1.8" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e296cf87e61c9cfc1a61c3c63a0f7f286ed4554e0e22be84e8a38e1d264a2a29" +checksum = "79d72a7c0f743d2ebb0a2ad1d219db75fdc799092ed3a884c9144c42a31225bd" [[package]] name = "onig" @@ -5777,9 +5776,9 @@ dependencies = [ [[package]] name = "openssl" -version = "0.10.69" +version = "0.10.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5e534d133a060a3c19daec1eb3e98ec6f4685978834f2dbadfe2ec215bab64e" +checksum = "61cfb4e166a8bb8c9b55c500bc2308550148ece889be90f609377e58140f42c6" dependencies = [ "bitflags 2.8.0", "cfg-if", @@ -5798,7 +5797,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -5818,9 +5817,9 @@ dependencies = [ [[package]] name = "openssl-sys" -version = "0.9.104" +version = "0.9.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45abf306cbf99debc8195b66b7346498d7b10c210de50418b5ccd7ceba08c741" +checksum = "8b22d5b84be05a8d6947c7cb71f7c849aa0f112acd4bf51c2a7c1c988ac0a9dc" dependencies = [ "cc", "libc", @@ -5996,9 +5995,9 @@ dependencies = [ [[package]] name = "p384" -version = "0.13.0" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70786f51bcc69f6a4c0360e063a4cac5419ef7c5cd5b3c99ad70f3be5ba79209" +checksum = "fe42f1670a52a47d448f14b6a5c61dd78fce51856e68edaa38f7ae3a46b8d6b6" dependencies = [ "ecdsa", "elliptic-curve", @@ -6196,7 +6195,7 @@ dependencies = [ "phf_shared", "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -6222,22 +6221,22 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.8" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e2ec53ad785f4d35dac0adea7f7dc6f1bb277ad84a680c7afefeae05d1f5916" +checksum = "dfe2e71e1471fe07709406bf725f710b02927c9c54b2b5b2ec0e8087d97c327d" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.8" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d56a66c0c55993aa927429d0f8a0abfd74f084e4d9c192cffed01e418d83eefb" +checksum = "f6e859e6e5bd50440ab63c47e3ebabc90f26251f7c73c3d3e837b74a1cc3fa67" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -6285,19 +6284,6 @@ version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "280dc24453071f1b63954171985a0b0d30058d287960968b9b2aca264c8d4ee6" -[[package]] -name = "postgres-native-tls" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d442770e2b1e244bb5eb03b31c79b65bb2568f413b899eaba850fa945a65954" -dependencies = [ - "futures", - "native-tls", - "tokio", - "tokio-native-tls", - "tokio-postgres 0.7.12", -] - [[package]] name = "postgres-native-tls" version = "0.5.0" @@ -6309,11 +6295,22 @@ dependencies = [ "tokio-postgres 0.7.11", ] +[[package]] +name = "postgres-native-tls" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1f39498473c92f7b6820ae970382c1d83178a3454c618161cb772e8598d9f6f" +dependencies = [ + "native-tls", + "tokio", + "tokio-native-tls", + "tokio-postgres 0.7.13", +] + [[package]] name = "postgres-protocol" version = "0.6.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acda0ebdebc28befa84bee35e651e4c5f09073d668c7aed4cf7e23c3cda84b23" +source = "git+https://github.com/imor/rust-postgres?rev=20265ef38e32a06f76b6f9b678e2077fc2211f6b#20265ef38e32a06f76b6f9b678e2077fc2211f6b" dependencies = [ "base64 0.22.1", "byteorder", @@ -6329,8 +6326,9 @@ dependencies = [ [[package]] name = "postgres-protocol" -version = "0.6.7" -source = "git+https://github.com/imor/rust-postgres?rev=20265ef38e32a06f76b6f9b678e2077fc2211f6b#20265ef38e32a06f76b6f9b678e2077fc2211f6b" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76ff0abab4a9b844b93ef7b81f1efc0a366062aaef2cd702c76256b5dc075c54" dependencies = [ "base64 0.22.1", "byteorder", @@ -6339,7 +6337,7 @@ dependencies = [ "hmac", "md-5 0.10.6", "memchr", - "rand 0.8.5", + "rand 0.9.0", "sha2 0.10.8", "stringprep", ] @@ -6351,24 +6349,24 @@ source = "git+https://github.com/imor/rust-postgres?rev=20265ef38e32a06f76b6f9b6 dependencies = [ "bytes", "fallible-iterator", - "postgres-protocol 0.6.7 (git+https://github.com/imor/rust-postgres?rev=20265ef38e32a06f76b6f9b678e2077fc2211f6b)", + "postgres-protocol 0.6.7", ] [[package]] name = "postgres-types" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f66ea23a2d0e5734297357705193335e0a957696f34bed2f2faefacb2fec336f" +checksum = "613283563cd90e1dfc3518d548caee47e0e725455ed619881f5cf21f36de4b48" dependencies = [ "array-init", "bit-vec", "bytes", "chrono", "fallible-iterator", - "postgres-protocol 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)", + "postgres-protocol 0.6.8", "serde", "serde_json", - "uuid 1.12.1", + "uuid 1.13.1", ] [[package]] @@ -6409,7 +6407,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6924ced06e1f7dfe3fa48d57b9f74f55d8915f5036121bef647ef4b204895fac" dependencies = [ "proc-macro2", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -6449,7 +6447,7 @@ dependencies = [ "proc-macro-error-attr2", "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -6460,7 +6458,7 @@ checksum = "07c277e4e643ef00c1233393c673f655e3672cf7eb3ba08a00bdd0ea59139b5f" dependencies = [ "proc-macro-rules-macros", "proc-macro2", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -6472,7 +6470,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -6533,7 +6531,7 @@ dependencies = [ "schemars", "serde", "serde_json", - "syn 2.0.96", + "syn 2.0.98", "thiserror 1.0.69", "typify", "unicode-ident", @@ -6553,7 +6551,7 @@ dependencies = [ "serde_json", "serde_tokenstream", "serde_yaml", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -6596,7 +6594,7 @@ dependencies = [ "itertools 0.13.0", "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -6689,9 +6687,9 @@ dependencies = [ [[package]] name = "quick_cache" -version = "0.6.9" +version = "0.6.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d7c94f8935a9df96bb6380e8592c70edf497a643f94bd23b2f76b399385dbf4" +checksum = "3f67cfc9c723c39f3615eb0840b00c4cb9e2b068d2fa761a30d845ec91730a59" dependencies = [ "ahash 0.8.11", "equivalent", @@ -6709,7 +6707,7 @@ dependencies = [ "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash 2.1.0", + "rustc-hash 2.1.1", "rustls 0.23.22", "socket2", "thiserror 2.0.11", @@ -6727,7 +6725,7 @@ dependencies = [ "getrandom 0.2.15", "rand 0.8.5", "ring 0.17.8", - "rustc-hash 2.1.0", + "rustc-hash 2.1.1", "rustls 0.23.22", "rustls-pki-types", "slab", @@ -6798,7 +6796,7 @@ checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.0", - "zerocopy 0.8.14", + "zerocopy 0.8.17", ] [[package]] @@ -6856,7 +6854,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b08f3c9802962f7e1b25113931d94f43ed9725bebc59db9d0c3e9a23b67e15ff" dependencies = [ "getrandom 0.3.1", - "zerocopy 0.8.14", + "zerocopy 0.8.17", ] [[package]] @@ -7225,7 +7223,7 @@ dependencies = [ "rkyv_derive", "seahash", "tinyvec", - "uuid 1.12.1", + "uuid 1.13.1", ] [[package]] @@ -7280,7 +7278,7 @@ dependencies = [ "quote", "rust-embed-utils", "shellexpand", - "syn 2.0.96", + "syn 2.0.98", "walkdir", ] @@ -7314,7 +7312,7 @@ dependencies = [ "borsh", "bytes", "num-traits", - "postgres-types 0.2.8", + "postgres-types 0.2.9", "rand 0.8.5", "rkyv", "serde", @@ -7335,9 +7333,9 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustc-hash" -version = "2.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7fb8039b3032c191086b10f11f319a6e99e1e82889c5cc6046f515c9db1d497" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" [[package]] name = "rustc_version" @@ -7366,7 +7364,7 @@ dependencies = [ "serde", "tempfile", "thiserror 1.0.69", - "toml 0.8.19", + "toml 0.8.20", "toolchain_find", ] @@ -7616,7 +7614,7 @@ dependencies = [ "serde", "thiserror 1.0.69", "url", - "uuid 1.12.1", + "uuid 1.13.1", ] [[package]] @@ -7654,7 +7652,7 @@ dependencies = [ "schemars_derive", "serde", "serde_json", - "uuid 1.12.1", + "uuid 1.13.1", ] [[package]] @@ -7666,7 +7664,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -7826,7 +7824,7 @@ checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -7837,7 +7835,7 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -7911,7 +7909,7 @@ checksum = "6c64451ba24fc7a6a2d60fc75dd9c83c90903b19028d4eff35e88fc1e86564e9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -7932,7 +7930,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -7987,7 +7985,7 @@ dependencies = [ "darling 0.20.10", "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -8337,7 +8335,7 @@ checksum = "01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -8388,7 +8386,7 @@ dependencies = [ "tokio-stream", "tracing", "url", - "uuid 1.12.1", + "uuid 1.13.1", "webpki-roots", ] @@ -8402,7 +8400,7 @@ dependencies = [ "quote", "sqlx-core", "sqlx-macros-core", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -8425,7 +8423,7 @@ dependencies = [ "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", - "syn 2.0.96", + "syn 2.0.98", "tempfile", "tokio", "url", @@ -8472,7 +8470,7 @@ dependencies = [ "stringprep", "thiserror 2.0.11", "tracing", - "uuid 1.12.1", + "uuid 1.13.1", "whoami", ] @@ -8513,7 +8511,7 @@ dependencies = [ "stringprep", "thiserror 2.0.11", "tracing", - "uuid 1.12.1", + "uuid 1.13.1", "whoami", ] @@ -8539,7 +8537,7 @@ dependencies = [ "sqlx-core", "tracing", "url", - "uuid 1.12.1", + "uuid 1.13.1", ] [[package]] @@ -8550,9 +8548,9 @@ checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" [[package]] name = "stacker" -version = "0.1.17" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "799c883d55abdb5e98af1a7b3f23b9b6de8ecada0ecac058672d7635eb48ca7b" +checksum = "1d08feb8f695b465baed819b03c128dc23f57a694510ab1f06c77f763975685e" dependencies = [ "cc", "cfg-if", @@ -8576,7 +8574,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -8636,7 +8634,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -8649,7 +8647,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -8757,7 +8755,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -8806,7 +8804,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -8891,7 +8889,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -8998,7 +8996,7 @@ checksum = "63db0adcff29d220c3d151c5b25c0eabe7e32dd936212b84cdaa1392e3130497" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -9009,7 +9007,7 @@ checksum = "f486687bfb7b5c560868f69ed2d458b880cebc9babebcb67e49f31b55c5bf847" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -9032,7 +9030,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -9048,9 +9046,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.96" +version = "2.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5d0adab1ae378d7f53bdebc67a39f1f151407ef230f0ce2883572f5d8985c80" +checksum = "36147f1a48ae0ec2b5b3bc5b537d267457555a10dc06f3dbc8cb11ba3006d3b1" dependencies = [ "proc-macro2", "quote", @@ -9080,7 +9078,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -9186,7 +9184,7 @@ dependencies = [ "tempfile", "thiserror 1.0.69", "time", - "uuid 1.12.1", + "uuid 1.13.1", "winapi", ] @@ -9355,7 +9353,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -9366,7 +9364,7 @@ checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -9416,7 +9414,7 @@ dependencies = [ "tokio-rustls 0.24.1", "tokio-util", "tracing", - "uuid 1.12.1", + "uuid 1.13.1", ] [[package]] @@ -9601,7 +9599,7 @@ checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -9630,7 +9628,7 @@ dependencies = [ "percent-encoding", "phf", "pin-project-lite", - "postgres-protocol 0.6.7 (git+https://github.com/imor/rust-postgres?rev=20265ef38e32a06f76b6f9b678e2077fc2211f6b)", + "postgres-protocol 0.6.7", "postgres-types 0.2.7", "rand 0.8.5", "socket2", @@ -9641,9 +9639,9 @@ dependencies = [ [[package]] name = "tokio-postgres" -version = "0.7.12" +version = "0.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b5d3742945bc7d7f210693b0c58ae542c6fd47b17adbbda0885f3dcb34a6bdb" +checksum = "6c95d533c83082bb6490e0189acaa0bbeef9084e60471b696ca6988cd0541fb0" dependencies = [ "async-trait", "byteorder", @@ -9656,9 +9654,9 @@ dependencies = [ "percent-encoding", "phf", "pin-project-lite", - "postgres-protocol 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)", - "postgres-types 0.2.8", - "rand 0.8.5", + "postgres-protocol 0.6.8", + "postgres-types 0.2.9", + "rand 0.9.0", "socket2", "tokio", "tokio-util", @@ -9786,9 +9784,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.19" +version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e" +checksum = "cd87a5cdd6ffab733b2f74bc4fd7ee5fff6634124999ac278c35fc78c6120148" dependencies = [ "serde", "serde_spanned", @@ -9828,7 +9826,7 @@ dependencies = [ "serde", "serde_spanned", "toml_datetime", - "winnow 0.7.0", + "winnow 0.7.1", ] [[package]] @@ -9995,7 +9993,7 @@ checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -10224,7 +10222,7 @@ dependencies = [ "regress", "schemars", "serde_json", - "syn 2.0.96", + "syn 2.0.98", "thiserror 1.0.69", "unicode-ident", ] @@ -10241,18 +10239,18 @@ dependencies = [ "serde", "serde_json", "serde_tokenstream", - "syn 2.0.96", + "syn 2.0.98", "typify-impl", ] [[package]] name = "ulid" -version = "1.1.4" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f294bff79170ed1c5633812aff1e565c35d993a36e757f9bc0accf5eec4e6045" +checksum = "ab82fc73182c29b02e2926a6df32f2241dbadb5cfc111fd595515b3598f46bb3" dependencies = [ - "rand 0.8.5", - "uuid 1.12.1", + "rand 0.9.0", + "uuid 1.13.1", "web-time", ] @@ -10534,11 +10532,11 @@ dependencies = [ [[package]] name = "uuid" -version = "1.12.1" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3758f5e68192bb96cc8f9b7e2c2cfdabb435499a28499a42f8f984092adad4b" +checksum = "ced87ca4be083373936a67f8de945faa23b6b42384bd5b64434850802c6dccd0" dependencies = [ - "getrandom 0.2.15", + "getrandom 0.3.1", "serde", ] @@ -10656,7 +10654,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", "wasm-bindgen-shared", ] @@ -10690,7 +10688,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -10723,7 +10721,7 @@ checksum = "b7f89739351a2e03cb94beb799d47fb2cac01759b40ec441f7de39b00cbf7ef0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -10860,7 +10858,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill" -version = "1.456.0" +version = "1.458.4" dependencies = [ "anyhow", "axum", @@ -10889,7 +10887,7 @@ dependencies = [ "tokio", "tracing", "url", - "uuid 1.12.1", + "uuid 1.13.1", "v8", "windmill-api", "windmill-api-client", @@ -10903,7 +10901,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.456.0" +version = "1.458.4" dependencies = [ "anyhow", "argon2", @@ -10948,7 +10946,7 @@ dependencies = [ "openssl", "pg_escape", "pin-project", - "postgres-native-tls 0.5.0 (git+https://github.com/imor/rust-postgres?rev=20265ef38e32a06f76b6f9b678e2077fc2211f6b)", + "postgres-native-tls 0.5.0", "prometheus", "quick_cache", "rand 0.9.0", @@ -10984,7 +10982,7 @@ dependencies = [ "ulid", "url", "urlencoding", - "uuid 1.12.1", + "uuid 1.13.1", "windmill-audit", "windmill-common", "windmill-git-sync", @@ -10996,7 +10994,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.456.0" +version = "1.458.4" dependencies = [ "base64 0.22.1", "chrono", @@ -11009,12 +11007,12 @@ dependencies = [ "serde", "serde_json", "syn 1.0.109", - "uuid 1.12.1", + "uuid 1.13.1", ] [[package]] name = "windmill-audit" -version = "1.456.0" +version = "1.458.4" dependencies = [ "chrono", "serde", @@ -11027,21 +11025,21 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.456.0" +version = "1.458.4" dependencies = [ "anyhow", "serde", "serde_json", "sqlx", "tracing", - "uuid 1.12.1", + "uuid 1.13.1", "windmill-common", "windmill-queue", ] [[package]] name = "windmill-common" -version = "1.456.0" +version = "1.458.4" dependencies = [ "anyhow", "async-stream", @@ -11094,27 +11092,27 @@ dependencies = [ "tracing-loki", "tracing-opentelemetry", "tracing-subscriber", - "uuid 1.12.1", + "uuid 1.13.1", "windmill-macros", ] [[package]] name = "windmill-git-sync" -version = "1.456.0" +version = "1.458.4" dependencies = [ "regex", "serde", "serde_json", "sqlx", "tracing", - "uuid 1.12.1", + "uuid 1.13.1", "windmill-common", "windmill-queue", ] [[package]] name = "windmill-indexer" -version = "1.456.0" +version = "1.458.4" dependencies = [ "anyhow", "bytes", @@ -11131,25 +11129,25 @@ dependencies = [ "tokio", "tokio-tar", "tracing", - "uuid 1.12.1", + "uuid 1.13.1", "windmill-common", ] [[package]] name = "windmill-macros" -version = "1.456.0" +version = "1.458.4" dependencies = [ "itertools 0.14.0", "lazy_static", "proc-macro2", "quote", "regex", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] name = "windmill-parser" -version = "1.456.0" +version = "1.458.4" dependencies = [ "convert_case 0.6.0", "serde", @@ -11158,7 +11156,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.456.0" +version = "1.458.4" dependencies = [ "anyhow", "lazy_static", @@ -11170,7 +11168,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.456.0" +version = "1.458.4" dependencies = [ "anyhow", "serde_json", @@ -11182,7 +11180,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.456.0" +version = "1.458.4" dependencies = [ "anyhow", "gosyn", @@ -11194,7 +11192,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.456.0" +version = "1.458.4" dependencies = [ "anyhow", "lazy_static", @@ -11206,7 +11204,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.456.0" +version = "1.458.4" dependencies = [ "anyhow", "itertools 0.14.0", @@ -11217,7 +11215,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.456.0" +version = "1.458.4" dependencies = [ "anyhow", "itertools 0.14.0", @@ -11228,7 +11226,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.456.0" +version = "1.458.4" dependencies = [ "anyhow", "async-recursion", @@ -11248,7 +11246,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.456.0" +version = "1.458.4" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -11258,14 +11256,14 @@ dependencies = [ "quote", "regex", "serde_json", - "syn 2.0.96", + "syn 2.0.98", "toml 0.7.8", "windmill-parser", ] [[package]] name = "windmill-parser-sql" -version = "1.456.0" +version = "1.458.4" dependencies = [ "anyhow", "lazy_static", @@ -11277,7 +11275,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.456.0" +version = "1.458.4" dependencies = [ "anyhow", "lazy_static", @@ -11295,7 +11293,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.456.0" +version = "1.458.4" dependencies = [ "anyhow", "getrandom 0.2.15", @@ -11317,7 +11315,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.456.0" +version = "1.458.4" dependencies = [ "anyhow", "serde_json", @@ -11327,7 +11325,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.456.0" +version = "1.458.4" dependencies = [ "anyhow", "async-recursion", @@ -11353,14 +11351,14 @@ dependencies = [ "tokio", "tracing", "ulid", - "uuid 1.12.1", + "uuid 1.13.1", "windmill-audit", "windmill-common", ] [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.456.0" +version = "1.458.4" dependencies = [ "wasm-bindgen", "wasm-bindgen-test", @@ -11370,7 +11368,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.456.0" +version = "1.458.4" dependencies = [ "anyhow", "async-recursion", @@ -11410,7 +11408,7 @@ dependencies = [ "opentelemetry", "oracle", "pem 3.0.4", - "postgres-native-tls 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", + "postgres-native-tls 0.5.1", "prometheus", "rand 0.9.0", "regex", @@ -11423,11 +11421,11 @@ dependencies = [ "tar", "tiberius", "tokio", - "tokio-postgres 0.7.12", + "tokio-postgres 0.7.13", "tokio-util", "tracing", "urlencoding", - "uuid 1.12.1", + "uuid 1.13.1", "windmill-audit", "windmill-common", "windmill-git-sync", @@ -11654,9 +11652,9 @@ dependencies = [ [[package]] name = "winnow" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e49d2d35d3fad69b39b94139037ecfb4f359f08958b9c11e7315ce770462419" +checksum = "86e376c75f4f43f44db463cf729e0d3acbf954d13e22c51e26e4c264b4ab545f" dependencies = [ "memchr", ] @@ -11768,7 +11766,7 @@ checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", "synstructure", ] @@ -11784,11 +11782,11 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.14" +version = "0.8.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a367f292d93d4eab890745e75a778da40909cab4d6ff8173693812f79c4a2468" +checksum = "aa91407dacce3a68c56de03abe2760159582b846c6a4acd2f456618087f12713" dependencies = [ - "zerocopy-derive 0.8.14", + "zerocopy-derive 0.8.17", ] [[package]] @@ -11799,18 +11797,18 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] name = "zerocopy-derive" -version = "0.8.14" +version = "0.8.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3931cb58c62c13adec22e38686b559c86a30565e16ad6e8510a337cedc611e1" +checksum = "06718a168365cad3d5ff0bb133aad346959a2074bd4a85c121255a11304a8626" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -11830,7 +11828,7 @@ checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", "synstructure", ] @@ -11859,7 +11857,7 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] diff --git a/backend/Cargo.toml b/backend/Cargo.toml index c174739a84..fe9c780d14 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.456.0" +version = "1.458.4" authors.workspace = true edition.workspace = true @@ -30,7 +30,7 @@ members = [ ] [workspace.package] -version = "1.456.0" +version = "1.458.4" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index cdeb8927e5..b3154d15ef 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -15ddca63acd769e4e7355269a239639c50989def \ No newline at end of file +8dab3198496461e40610145e4c818fce2345e20a \ No newline at end of file diff --git a/backend/migrations/20250131153251_ai_models.down.sql b/backend/migrations/20250131153251_ai_models.down.sql new file mode 100644 index 0000000000..76eaf5ba6b --- /dev/null +++ b/backend/migrations/20250131153251_ai_models.down.sql @@ -0,0 +1,5 @@ +ALTER TABLE workspace_settings ADD COLUMN code_completion_enabled BOOLEAN DEFAULT FALSE NOT NULL; + +UPDATE workspace_settings SET code_completion_enabled = TRUE WHERE code_completion_model IS NOT NULL; + +ALTER TABLE workspace_settings DROP COLUMN ai_models, DROP COLUMN code_completion_model; \ No newline at end of file diff --git a/backend/migrations/20250131153251_ai_models.up.sql b/backend/migrations/20250131153251_ai_models.up.sql new file mode 100644 index 0000000000..7145e0bb98 --- /dev/null +++ b/backend/migrations/20250131153251_ai_models.up.sql @@ -0,0 +1,24 @@ +ALTER TABLE workspace_settings + ADD COLUMN ai_models varchar(255)[] DEFAULT '{}' NOT NULL, + ADD COLUMN code_completion_model varchar(255); + +UPDATE workspace_settings +SET ai_models = CASE + WHEN ai_resource->>'provider' = 'openai' THEN ARRAY['gpt-4o'] + WHEN ai_resource->>'provider' = 'anthropic' THEN ARRAY['claude-3-5-sonnet-latest'] + WHEN ai_resource->>'provider' = 'mistral' THEN ARRAY['codestral-latest'] + ELSE ai_models +END +WHERE ai_resource->>'path' IS NOT NULL; + +UPDATE workspace_settings +SET code_completion_model = CASE + WHEN ai_resource->>'provider' = 'openai' THEN 'gpt-4o' + WHEN ai_resource->>'provider' = 'anthropic' THEN 'claude-3-5-sonnet-latest' + WHEN ai_resource->>'provider' = 'mistral' THEN 'codestral-latest' + ELSE code_completion_model +END +WHERE code_completion_enabled IS TRUE; + + +ALTER TABLE workspace_settings DROP COLUMN code_completion_enabled; \ No newline at end of file diff --git a/backend/migrations/20250201124344_v2_job_runtime.down.sql b/backend/migrations/20250201124344_v2_job_runtime.down.sql new file mode 100644 index 0000000000..45b34f0f7a --- /dev/null +++ b/backend/migrations/20250201124344_v2_job_runtime.down.sql @@ -0,0 +1,4 @@ +-- Add down migration script here +-- Lock `queue` in access exclusive to prevent deadlocks when dropping the foreign key to `queue`. +LOCK TABLE queue IN ACCESS EXCLUSIVE MODE; +DROP TABLE v2_job_runtime CASCADE; diff --git a/backend/migrations/20250201124344_v2_job_runtime.up.sql b/backend/migrations/20250201124344_v2_job_runtime.up.sql new file mode 100644 index 0000000000..60b3a99384 --- /dev/null +++ b/backend/migrations/20250201124344_v2_job_runtime.up.sql @@ -0,0 +1,14 @@ +-- Add up migration script here +CREATE TABLE IF NOT EXISTS v2_job_runtime ( + id UUID REFERENCES queue (id) ON DELETE CASCADE PRIMARY KEY NOT NULL, + -- Metrics fields: + ping TIMESTAMP WITH TIME ZONE DEFAULT now(), + memory_peak INTEGER +); + +CREATE POLICY admin_policy ON v2_job_runtime + AS PERMISSIVE + FOR ALL + TO windmill_admin; + +GRANT ALL ON v2_job_runtime TO windmill_user, windmill_admin; diff --git a/backend/migrations/20250201124345_v2_job_status.down.sql b/backend/migrations/20250201124345_v2_job_status.down.sql new file mode 100644 index 0000000000..fa8e14f3ff --- /dev/null +++ b/backend/migrations/20250201124345_v2_job_status.down.sql @@ -0,0 +1,4 @@ +-- Add down migration script here +-- Lock `queue` in access exclusive to prevent deadlocks when dropping the foreign key to `queue`. +LOCK TABLE queue IN ACCESS EXCLUSIVE MODE; +DROP TABLE v2_job_status CASCADE; diff --git a/backend/migrations/20250201124345_v2_job_status.up.sql b/backend/migrations/20250201124345_v2_job_status.up.sql new file mode 100644 index 0000000000..c43ab1f78a --- /dev/null +++ b/backend/migrations/20250201124345_v2_job_status.up.sql @@ -0,0 +1,16 @@ +-- Add up migration script here +CREATE TABLE IF NOT EXISTS v2_job_status ( + id UUID REFERENCES queue (id) ON DELETE CASCADE PRIMARY KEY NOT NULL, + -- Flow status fields: + flow_status JSONB, + flow_leaf_jobs JSONB, + -- Workflow as code fields: + workflow_as_code_status JSONB +); + +CREATE POLICY admin_policy ON v2_job_status + AS PERMISSIVE + FOR ALL + TO windmill_admin; + +GRANT ALL ON v2_job_status TO windmill_user, windmill_admin; diff --git a/backend/migrations/20250201124409_v2_job_queue_compatiblity_view.down.sql b/backend/migrations/20250201124409_v2_job_queue_compatiblity_view.down.sql new file mode 100644 index 0000000000..43f51a18ce --- /dev/null +++ b/backend/migrations/20250201124409_v2_job_queue_compatiblity_view.down.sql @@ -0,0 +1,3 @@ +-- Add down migration script here +DROP VIEW queue; +ALTER TABLE v2_job_queue RENAME TO queue; diff --git a/backend/migrations/20250201124409_v2_job_queue_compatiblity_view.up.sql b/backend/migrations/20250201124409_v2_job_queue_compatiblity_view.up.sql new file mode 100644 index 0000000000..6a4f1b9c74 --- /dev/null +++ b/backend/migrations/20250201124409_v2_job_queue_compatiblity_view.up.sql @@ -0,0 +1,48 @@ +-- Add down migration script here +ALTER TABLE queue RENAME TO v2_job_queue; +CREATE OR REPLACE VIEW queue AS ( + SELECT + id, + workspace_id, + parent_job AS parent_job, + created_by AS created_by, + created_at, + started_at, + scheduled_for, + running, + script_hash AS script_hash, + script_path AS script_path, + args AS args, + logs AS logs, + raw_code AS raw_code, + canceled AS canceled, + canceled_by, + canceled_reason, + last_ping AS last_ping, + job_kind AS job_kind, + env_id AS env_id, + schedule_path AS schedule_path, + permissioned_as AS permissioned_as, + flow_status AS flow_status, + raw_flow AS raw_flow, + is_flow_step AS is_flow_step, + language AS language, + suspend, + suspend_until, + same_worker AS same_worker, + raw_lock AS raw_lock, + pre_run_error AS pre_run_error, + email AS email, + visible_to_owner AS visible_to_owner, + mem_peak AS mem_peak, + root_job AS root_job, + leaf_jobs AS leaf_jobs, + tag, + concurrent_limit AS concurrent_limit, + concurrency_time_window_s AS concurrency_time_window_s, + timeout AS timeout, + flow_step_id AS flow_step_id, + cache_ttl AS cache_ttl, + priority + FROM v2_job_queue +); diff --git a/backend/migrations/20250201124410_v2_job_queue.down.sql b/backend/migrations/20250201124410_v2_job_queue.down.sql new file mode 100644 index 0000000000..7da7e1fa51 --- /dev/null +++ b/backend/migrations/20250201124410_v2_job_queue.down.sql @@ -0,0 +1,34 @@ +-- Add down migration script here +ALTER TABLE v2_job_queue + DROP COLUMN worker CASCADE, + DROP COLUMN extras CASCADE; +ALTER TABLE v2_job_queue RENAME COLUMN __parent_job TO parent_job; +ALTER TABLE v2_job_queue RENAME COLUMN __created_by TO created_by; +ALTER TABLE v2_job_queue RENAME COLUMN __script_hash TO script_hash; +ALTER TABLE v2_job_queue RENAME COLUMN __script_path TO script_path; +ALTER TABLE v2_job_queue RENAME COLUMN __args TO args; +ALTER TABLE v2_job_queue RENAME COLUMN __logs TO logs; +ALTER TABLE v2_job_queue RENAME COLUMN __raw_code TO raw_code; +ALTER TABLE v2_job_queue RENAME COLUMN __canceled TO canceled; +ALTER TABLE v2_job_queue RENAME COLUMN __last_ping TO last_ping; +ALTER TABLE v2_job_queue RENAME COLUMN __job_kind TO job_kind; +ALTER TABLE v2_job_queue RENAME COLUMN __env_id TO env_id; +ALTER TABLE v2_job_queue RENAME COLUMN __schedule_path TO schedule_path; +ALTER TABLE v2_job_queue RENAME COLUMN __permissioned_as TO permissioned_as; +ALTER TABLE v2_job_queue RENAME COLUMN __flow_status TO flow_status; +ALTER TABLE v2_job_queue RENAME COLUMN __raw_flow TO raw_flow; +ALTER TABLE v2_job_queue RENAME COLUMN __is_flow_step TO is_flow_step; +ALTER TABLE v2_job_queue RENAME COLUMN __language TO language; +ALTER TABLE v2_job_queue RENAME COLUMN __same_worker TO same_worker; +ALTER TABLE v2_job_queue RENAME COLUMN __raw_lock TO raw_lock; +ALTER TABLE v2_job_queue RENAME COLUMN __pre_run_error TO pre_run_error; +ALTER TABLE v2_job_queue RENAME COLUMN __email TO email; +ALTER TABLE v2_job_queue RENAME COLUMN __visible_to_owner TO visible_to_owner; +ALTER TABLE v2_job_queue RENAME COLUMN __mem_peak TO mem_peak; +ALTER TABLE v2_job_queue RENAME COLUMN __root_job TO root_job; +ALTER TABLE v2_job_queue RENAME COLUMN __leaf_jobs TO leaf_jobs; +ALTER TABLE v2_job_queue RENAME COLUMN __concurrent_limit TO concurrent_limit; +ALTER TABLE v2_job_queue RENAME COLUMN __concurrency_time_window_s TO concurrency_time_window_s; +ALTER TABLE v2_job_queue RENAME COLUMN __timeout TO timeout; +ALTER TABLE v2_job_queue RENAME COLUMN __flow_step_id TO flow_step_id; +ALTER TABLE v2_job_queue RENAME COLUMN __cache_ttl TO cache_ttl; diff --git a/backend/migrations/20250201124410_v2_job_queue.up.sql b/backend/migrations/20250201124410_v2_job_queue.up.sql new file mode 100644 index 0000000000..dfd0611fca --- /dev/null +++ b/backend/migrations/20250201124410_v2_job_queue.up.sql @@ -0,0 +1,34 @@ +-- Add up migration script here +ALTER TABLE v2_job_queue + ADD COLUMN IF NOT EXISTS worker VARCHAR(255), + ADD COLUMN IF NOT EXISTS extras JSONB; +ALTER TABLE v2_job_queue RENAME COLUMN parent_job TO __parent_job; +ALTER TABLE v2_job_queue RENAME COLUMN created_by TO __created_by; +ALTER TABLE v2_job_queue RENAME COLUMN script_hash TO __script_hash; +ALTER TABLE v2_job_queue RENAME COLUMN script_path TO __script_path; +ALTER TABLE v2_job_queue RENAME COLUMN args TO __args; +ALTER TABLE v2_job_queue RENAME COLUMN logs TO __logs; +ALTER TABLE v2_job_queue RENAME COLUMN raw_code TO __raw_code; +ALTER TABLE v2_job_queue RENAME COLUMN canceled TO __canceled; +ALTER TABLE v2_job_queue RENAME COLUMN last_ping TO __last_ping; +ALTER TABLE v2_job_queue RENAME COLUMN job_kind TO __job_kind; +ALTER TABLE v2_job_queue RENAME COLUMN env_id TO __env_id; +ALTER TABLE v2_job_queue RENAME COLUMN schedule_path TO __schedule_path; +ALTER TABLE v2_job_queue RENAME COLUMN permissioned_as TO __permissioned_as; +ALTER TABLE v2_job_queue RENAME COLUMN flow_status TO __flow_status; +ALTER TABLE v2_job_queue RENAME COLUMN raw_flow TO __raw_flow; +ALTER TABLE v2_job_queue RENAME COLUMN is_flow_step TO __is_flow_step; +ALTER TABLE v2_job_queue RENAME COLUMN language TO __language; +ALTER TABLE v2_job_queue RENAME COLUMN same_worker TO __same_worker; +ALTER TABLE v2_job_queue RENAME COLUMN raw_lock TO __raw_lock; +ALTER TABLE v2_job_queue RENAME COLUMN pre_run_error TO __pre_run_error; +ALTER TABLE v2_job_queue RENAME COLUMN email TO __email; +ALTER TABLE v2_job_queue RENAME COLUMN visible_to_owner TO __visible_to_owner; +ALTER TABLE v2_job_queue RENAME COLUMN mem_peak TO __mem_peak; +ALTER TABLE v2_job_queue RENAME COLUMN root_job TO __root_job; +ALTER TABLE v2_job_queue RENAME COLUMN leaf_jobs TO __leaf_jobs; +ALTER TABLE v2_job_queue RENAME COLUMN concurrent_limit TO __concurrent_limit; +ALTER TABLE v2_job_queue RENAME COLUMN concurrency_time_window_s TO __concurrency_time_window_s; +ALTER TABLE v2_job_queue RENAME COLUMN timeout TO __timeout; +ALTER TABLE v2_job_queue RENAME COLUMN flow_step_id TO __flow_step_id; +ALTER TABLE v2_job_queue RENAME COLUMN cache_ttl TO __cache_ttl; diff --git a/backend/migrations/20250201124422_v2_job_completed_compatibility_view.down.sql b/backend/migrations/20250201124422_v2_job_completed_compatibility_view.down.sql new file mode 100644 index 0000000000..5eb87e01e0 --- /dev/null +++ b/backend/migrations/20250201124422_v2_job_completed_compatibility_view.down.sql @@ -0,0 +1,3 @@ +-- Add down migration script here +DROP VIEW IF EXISTS completed_job; +ALTER TABLE IF EXISTS v2_job_completed RENAME TO completed_job; diff --git a/backend/migrations/20250201124422_v2_job_completed_compatibility_view.up.sql b/backend/migrations/20250201124422_v2_job_completed_compatibility_view.up.sql new file mode 100644 index 0000000000..7519a1c585 --- /dev/null +++ b/backend/migrations/20250201124422_v2_job_completed_compatibility_view.up.sql @@ -0,0 +1,39 @@ +-- Add down migration script here +ALTER TABLE completed_job RENAME TO v2_job_completed; +CREATE OR REPLACE VIEW completed_job AS ( + SELECT + id, + workspace_id, + parent_job AS parent_job, + created_by AS created_by, + created_at AS created_at, + duration_ms, + success AS success, + script_hash AS script_hash, + script_path AS script_path, + args AS args, + result, + logs AS logs, + deleted, + raw_code AS raw_code, + canceled AS canceled, + canceled_by, + canceled_reason, + job_kind AS job_kind, + env_id AS env_id, + schedule_path AS schedule_path, + permissioned_as AS permissioned_as, + flow_status, + raw_flow AS raw_flow, + is_flow_step AS is_flow_step, + language AS language, + started_at, + is_skipped AS is_skipped, + raw_lock AS raw_lock, + email AS email, + visible_to_owner AS visible_to_owner, + mem_peak AS mem_peak, + tag AS tag, + priority AS priority + FROM v2_job_completed +); diff --git a/backend/migrations/20250201124423_v2_job_completed.down.sql b/backend/migrations/20250201124423_v2_job_completed.down.sql new file mode 100644 index 0000000000..02391a6152 --- /dev/null +++ b/backend/migrations/20250201124423_v2_job_completed.down.sql @@ -0,0 +1,34 @@ +-- Add down migration script here +ALTER TABLE v2_job_completed + DROP COLUMN status CASCADE, + DROP COLUMN completed_at CASCADE, + DROP COLUMN worker CASCADE, + DROP COLUMN workflow_as_code_status CASCADE, + DROP COLUMN result_columns CASCADE, + DROP COLUMN retries CASCADE, + DROP COLUMN extras CASCADE; +ALTER TABLE v2_job_completed RENAME COLUMN memory_peak TO mem_peak; +ALTER TABLE v2_job_completed RENAME COLUMN __parent_job TO parent_job; +ALTER TABLE v2_job_completed RENAME COLUMN __created_by TO created_by; +ALTER TABLE v2_job_completed RENAME COLUMN __created_at TO created_at; +ALTER TABLE v2_job_completed RENAME COLUMN __success TO success; +ALTER TABLE v2_job_completed RENAME COLUMN __script_hash TO script_hash; +ALTER TABLE v2_job_completed RENAME COLUMN __script_path TO script_path; +ALTER TABLE v2_job_completed RENAME COLUMN __args TO args; +ALTER TABLE v2_job_completed RENAME COLUMN __logs TO logs; +ALTER TABLE v2_job_completed RENAME COLUMN __raw_code TO raw_code; +ALTER TABLE v2_job_completed RENAME COLUMN __canceled TO canceled; +ALTER TABLE v2_job_completed RENAME COLUMN __job_kind TO job_kind; +ALTER TABLE v2_job_completed RENAME COLUMN __env_id TO env_id; +ALTER TABLE v2_job_completed RENAME COLUMN __schedule_path TO schedule_path; +ALTER TABLE v2_job_completed RENAME COLUMN __permissioned_as TO permissioned_as; +ALTER TABLE v2_job_completed RENAME COLUMN __raw_flow TO raw_flow; +ALTER TABLE v2_job_completed RENAME COLUMN __is_flow_step TO is_flow_step; +ALTER TABLE v2_job_completed RENAME COLUMN __language TO language; +ALTER TABLE v2_job_completed RENAME COLUMN __is_skipped TO is_skipped; +ALTER TABLE v2_job_completed RENAME COLUMN __raw_lock TO raw_lock; +ALTER TABLE v2_job_completed RENAME COLUMN __email TO email; +ALTER TABLE v2_job_completed RENAME COLUMN __visible_to_owner TO visible_to_owner; +ALTER TABLE v2_job_completed RENAME COLUMN __tag TO tag; +ALTER TABLE v2_job_completed RENAME COLUMN __priority TO priority; +DROP TYPE IF EXISTS job_status CASCADE; diff --git a/backend/migrations/20250201124423_v2_job_completed.up.sql b/backend/migrations/20250201124423_v2_job_completed.up.sql new file mode 100644 index 0000000000..1aebff0b58 --- /dev/null +++ b/backend/migrations/20250201124423_v2_job_completed.up.sql @@ -0,0 +1,34 @@ +-- Add up migration script here +CREATE TYPE job_status AS ENUM ('success', 'failure', 'canceled', 'skipped'); +ALTER TABLE v2_job_completed + ADD COLUMN IF NOT EXISTS status job_status, + ADD COLUMN IF NOT EXISTS completed_at TIMESTAMP WITH TIME ZONE, + ADD COLUMN IF NOT EXISTS worker VARCHAR(255), + ADD COLUMN IF NOT EXISTS workflow_as_code_status JSONB, + ADD COLUMN IF NOT EXISTS result_columns TEXT[], + ADD COLUMN IF NOT EXISTS retries UUID[], + ADD COLUMN IF NOT EXISTS extras JSONB; +ALTER TABLE v2_job_completed RENAME COLUMN mem_peak TO memory_peak; +ALTER TABLE v2_job_completed RENAME COLUMN parent_job TO __parent_job; +ALTER TABLE v2_job_completed RENAME COLUMN created_by TO __created_by; +ALTER TABLE v2_job_completed RENAME COLUMN created_at TO __created_at; +ALTER TABLE v2_job_completed RENAME COLUMN success TO __success; +ALTER TABLE v2_job_completed RENAME COLUMN script_hash TO __script_hash; +ALTER TABLE v2_job_completed RENAME COLUMN script_path TO __script_path; +ALTER TABLE v2_job_completed RENAME COLUMN args TO __args; +ALTER TABLE v2_job_completed RENAME COLUMN logs TO __logs; +ALTER TABLE v2_job_completed RENAME COLUMN raw_code TO __raw_code; +ALTER TABLE v2_job_completed RENAME COLUMN canceled TO __canceled; +ALTER TABLE v2_job_completed RENAME COLUMN job_kind TO __job_kind; +ALTER TABLE v2_job_completed RENAME COLUMN env_id TO __env_id; +ALTER TABLE v2_job_completed RENAME COLUMN schedule_path TO __schedule_path; +ALTER TABLE v2_job_completed RENAME COLUMN permissioned_as TO __permissioned_as; +ALTER TABLE v2_job_completed RENAME COLUMN raw_flow TO __raw_flow; +ALTER TABLE v2_job_completed RENAME COLUMN is_flow_step TO __is_flow_step; +ALTER TABLE v2_job_completed RENAME COLUMN language TO __language; +ALTER TABLE v2_job_completed RENAME COLUMN is_skipped TO __is_skipped; +ALTER TABLE v2_job_completed RENAME COLUMN raw_lock TO __raw_lock; +ALTER TABLE v2_job_completed RENAME COLUMN email TO __email; +ALTER TABLE v2_job_completed RENAME COLUMN visible_to_owner TO __visible_to_owner; +ALTER TABLE v2_job_completed RENAME COLUMN tag TO __tag; +ALTER TABLE v2_job_completed RENAME COLUMN priority TO __priority; diff --git a/backend/migrations/20250201124430_v2_job_compatibility_view.down.sql b/backend/migrations/20250201124430_v2_job_compatibility_view.down.sql new file mode 100644 index 0000000000..45dd1e24a3 --- /dev/null +++ b/backend/migrations/20250201124430_v2_job_compatibility_view.down.sql @@ -0,0 +1,3 @@ +-- Add down migration script here +DROP VIEW IF EXISTS job; +ALTER TABLE IF EXISTS v2_job RENAME TO job; diff --git a/backend/migrations/20250201124430_v2_job_compatibility_view.up.sql b/backend/migrations/20250201124430_v2_job_compatibility_view.up.sql new file mode 100644 index 0000000000..246d46e2b6 --- /dev/null +++ b/backend/migrations/20250201124430_v2_job_compatibility_view.up.sql @@ -0,0 +1,6 @@ +-- Add down migration script here +ALTER TABLE job RENAME TO v2_job; +CREATE OR REPLACE VIEW job AS ( + SELECT id, raw_code, raw_lock, raw_flow, tag, workspace_id + FROM v2_job +); diff --git a/backend/migrations/20250201124431_v2_job.down.sql b/backend/migrations/20250201124431_v2_job.down.sql new file mode 100644 index 0000000000..663ecb106b --- /dev/null +++ b/backend/migrations/20250201124431_v2_job.down.sql @@ -0,0 +1,36 @@ +-- Add down migration script here +DROP POLICY see_folder_extra_perms_user ON v2_job; +DROP POLICY see_own_path ON v2_job; +DROP POLICY see_member_path ON v2_job; +DROP POLICY see_own ON v2_job; +DROP POLICY see_member ON v2_job; +DROP POLICY admin_policy ON v2_job; +ALTER TABLE v2_job + DROP COLUMN created_at CASCADE, + DROP COLUMN created_by CASCADE, + DROP COLUMN permissioned_as CASCADE, + DROP COLUMN permissioned_as_email CASCADE, + DROP COLUMN kind CASCADE, + DROP COLUMN runnable_id CASCADE, + DROP COLUMN runnable_path CASCADE, + DROP COLUMN parent_job CASCADE, + DROP COLUMN root_job CASCADE, + DROP COLUMN script_lang CASCADE, + DROP COLUMN script_entrypoint_override CASCADE, + DROP COLUMN flow_step CASCADE, + DROP COLUMN flow_step_id CASCADE, + DROP COLUMN flow_innermost_root_job CASCADE, + DROP COLUMN trigger CASCADE, + DROP COLUMN trigger_kind CASCADE, + DROP COLUMN same_worker CASCADE, + DROP COLUMN visible_to_owner CASCADE, + DROP COLUMN concurrent_limit CASCADE, + DROP COLUMN concurrency_time_window_s CASCADE, + DROP COLUMN cache_ttl CASCADE, + DROP COLUMN timeout CASCADE, + DROP COLUMN priority CASCADE, + DROP COLUMN preprocessed CASCADE, + DROP COLUMN args CASCADE, + DROP COLUMN labels CASCADE, + DROP COLUMN pre_run_error CASCADE; +DROP TYPE job_trigger_kind; diff --git a/backend/migrations/20250201124431_v2_job.up.sql b/backend/migrations/20250201124431_v2_job.up.sql new file mode 100644 index 0000000000..ab5f0769d6 --- /dev/null +++ b/backend/migrations/20250201124431_v2_job.up.sql @@ -0,0 +1,78 @@ +-- Add up migration script here +CREATE TYPE job_trigger_kind AS ENUM ( + 'webhook', 'http', 'websocket', 'kafka', 'email', 'nats', 'schedule', 'app', 'ui', 'postgres' +); + +ALTER TABLE v2_job + ADD COLUMN IF NOT EXISTS created_at TIMESTAMP WITH TIME ZONE DEFAULT now() NOT NULL, + ADD COLUMN IF NOT EXISTS created_by VARCHAR(255) DEFAULT 'missing' NOT NULL, + ADD COLUMN IF NOT EXISTS permissioned_as VARCHAR(55) DEFAULT 'g/all' NOT NULL, + ADD COLUMN IF NOT EXISTS permissioned_as_email VARCHAR(255) DEFAULT 'missing@email.xyz' NOT NULL, + ADD COLUMN IF NOT EXISTS kind job_kind DEFAULT 'script'::job_kind NOT NULL, + ADD COLUMN IF NOT EXISTS runnable_id BIGINT, + ADD COLUMN IF NOT EXISTS runnable_path VARCHAR(255), + ADD COLUMN IF NOT EXISTS parent_job UUID, + ADD COLUMN IF NOT EXISTS root_job UUID, + ADD COLUMN IF NOT EXISTS script_lang script_lang DEFAULT 'python3'::script_lang, + ADD COLUMN IF NOT EXISTS script_entrypoint_override VARCHAR(255), + ADD COLUMN IF NOT EXISTS flow_step INTEGER, + ADD COLUMN IF NOT EXISTS flow_step_id VARCHAR(255), + ADD COLUMN IF NOT EXISTS flow_innermost_root_job UUID, + ADD COLUMN IF NOT EXISTS trigger VARCHAR(255), + ADD COLUMN IF NOT EXISTS trigger_kind job_trigger_kind, + ADD COLUMN IF NOT EXISTS same_worker BOOLEAN DEFAULT FALSE NOT NULL, + ADD COLUMN IF NOT EXISTS visible_to_owner BOOLEAN DEFAULT TRUE NOT NULL, + ADD COLUMN IF NOT EXISTS concurrent_limit INTEGER, + ADD COLUMN IF NOT EXISTS concurrency_time_window_s INTEGER, + ADD COLUMN IF NOT EXISTS cache_ttl INTEGER, + ADD COLUMN IF NOT EXISTS timeout INTEGER, + ADD COLUMN IF NOT EXISTS priority SMALLINT, + ADD COLUMN IF NOT EXISTS preprocessed BOOLEAN, + ADD COLUMN IF NOT EXISTS args JSONB, + ADD COLUMN IF NOT EXISTS labels TEXT[], + ADD COLUMN IF NOT EXISTS pre_run_error TEXT; + +CREATE POLICY see_folder_extra_perms_user ON v2_job + AS PERMISSIVE + FOR ALL + TO windmill_user + USING ((visible_to_owner IS TRUE) AND (SPLIT_PART((runnable_path)::TEXT, '/'::TEXT, 1) = 'f'::TEXT) AND + (SPLIT_PART((runnable_path)::TEXT, '/'::TEXT, 2) = ANY ( + REGEXP_SPLIT_TO_ARRAY(CURRENT_SETTING('session.folders_read'::TEXT), ','::TEXT)))); + +CREATE POLICY see_own_path ON v2_job + AS PERMISSIVE + FOR ALL + TO windmill_user + USING ((visible_to_owner IS TRUE) AND (SPLIT_PART((runnable_path)::TEXT, '/'::TEXT, 1) = 'u'::TEXT) AND + (SPLIT_PART((runnable_path)::TEXT, '/'::TEXT, 2) = CURRENT_SETTING('session.user'::TEXT))); + +CREATE POLICY see_member_path ON v2_job + AS PERMISSIVE + FOR ALL + TO windmill_user + USING ((visible_to_owner IS TRUE) AND (SPLIT_PART((runnable_path)::TEXT, '/'::TEXT, 1) = 'g'::TEXT) AND + (SPLIT_PART((runnable_path)::TEXT, '/'::TEXT, 2) = ANY + (REGEXP_SPLIT_TO_ARRAY(CURRENT_SETTING('session.groups'::TEXT), ','::TEXT)))); + +CREATE POLICY see_own ON v2_job + AS PERMISSIVE + FOR ALL + TO windmill_user + USING ((SPLIT_PART((permissioned_as)::TEXT, '/'::TEXT, 1) = 'f'::TEXT) AND + (SPLIT_PART((permissioned_as)::TEXT, '/'::TEXT, 2) = CURRENT_SETTING('session.user'::TEXT))); + +CREATE POLICY see_member ON v2_job + AS PERMISSIVE + FOR ALL + TO windmill_user + USING ((SPLIT_PART((permissioned_as)::TEXT, '/'::TEXT, 1) = 'g'::TEXT) AND + (SPLIT_PART((permissioned_as)::TEXT, '/'::TEXT, 2) = ANY + (REGEXP_SPLIT_TO_ARRAY(CURRENT_SETTING('session.groups'::TEXT), ','::TEXT)))); + +CREATE POLICY admin_policy ON v2_job + AS PERMISSIVE + FOR ALL + TO windmill_admin; + +GRANT ALL ON v2_job TO windmill_user, windmill_admin; diff --git a/backend/migrations/20250201124743_v2_job_queue_sync.down.sql b/backend/migrations/20250201124743_v2_job_queue_sync.down.sql new file mode 100644 index 0000000000..b64ab629bf --- /dev/null +++ b/backend/migrations/20250201124743_v2_job_queue_sync.down.sql @@ -0,0 +1,4 @@ +-- Add down migration script here +DROP FUNCTION v2_job_queue_before_insert() CASCADE; +DROP FUNCTION v2_job_queue_after_insert() CASCADE; +DROP FUNCTION v2_job_queue_before_update() CASCADE; diff --git a/backend/migrations/20250201124743_v2_job_queue_sync.up.sql b/backend/migrations/20250201124743_v2_job_queue_sync.up.sql new file mode 100644 index 0000000000..2ac8a37ca2 --- /dev/null +++ b/backend/migrations/20250201124743_v2_job_queue_sync.up.sql @@ -0,0 +1,220 @@ +-- Add up migration script here +-- This trigger will be removed once all server(s)/worker(s) are updated to use `v2_*` tables +CREATE OR REPLACE FUNCTION v2_job_queue_before_insert() RETURNS TRIGGER AS $$ +DECLARE job v2_job; +BEGIN + IF NEW.__created_by IS NOT NULL THEN + -- v1 -> v2 + IF NEW.__logs IS NOT NULL THEN + INSERT INTO job_logs (job_id, workspace_id, logs) + VALUES (NEW.id, NEW.workspace_id, NEW.__logs) + ON CONFLICT (job_id) DO UPDATE SET + logs = CONCAT(job_logs.logs, EXCLUDED.logs) + ; + -- Need to be "before" to remove logs: + NEW.__logs := NULL; + END IF; + RETURN NEW; + END IF; + -- v2 -> v1 + -- When inserting to `v2_job_queue` from `v2` code, set `v1` columns: + SELECT * INTO job FROM v2_job WHERE id = NEW.id; + NEW.__parent_job := job.parent_job; + NEW.__created_by := job.created_by; + NEW.__script_hash := job.runnable_id; + NEW.__script_path := job.runnable_path; + NEW.__args := job.args; + -- __logs + NEW.__raw_code := job.raw_code; + NEW.__canceled := NEW.canceled_by IS NOT NULL; + -- __last_ping + NEW.__job_kind := job.kind; + NEW.__env_id := 123456789; -- Magic used bellow. + NEW.__schedule_path := CASE WHEN job.trigger_kind = 'schedule'::job_trigger_kind THEN job.trigger END; + NEW.__permissioned_as := job.permissioned_as; + -- __flow_status + NEW.__raw_flow := job.raw_flow; + NEW.__is_flow_step := job.flow_step_id IS NOT NULL; + NEW.__language := job.script_lang; + NEW.__same_worker := job.same_worker; + NEW.__raw_lock := job.raw_lock; + NEW.__pre_run_error := job.pre_run_error; + NEW.__email := job.permissioned_as_email; + NEW.__visible_to_owner := job.visible_to_owner; + -- __mem_peak + NEW.__root_job := job.flow_innermost_root_job; + -- __leaf_jobs + NEW.__concurrent_limit := job.concurrent_limit; + NEW.__concurrency_time_window_s := job.concurrency_time_window_s; + NEW.__timeout := job.timeout; + NEW.__flow_step_id := job.flow_step_id; + NEW.__cache_ttl := job.cache_ttl; + IF job.script_entrypoint_override IS NOT NULL THEN + NEW.__args = jsonb_set( + coalesce(NEW.__args, '{}'::JSONB), + '{_ENTRYPOINT_OVERRIDE}', + to_jsonb(job.script_entrypoint_override) + ); + END IF; + RETURN NEW; +END $$ LANGUAGE plpgsql; + +CREATE OR REPLACE TRIGGER v2_job_queue_before_insert_trigger + BEFORE INSERT ON v2_job_queue + FOR EACH ROW + WHEN (pg_trigger_depth() < 1) -- Prevent infinite loop v1 <-> v2 +EXECUTE FUNCTION v2_job_queue_before_insert(); + +-- v1 -> v2 +-- On every insert to `v2_job_queue`, insert to `v2_job`, `v2_job_runtime` and `v2_job_status` as well +-- This trigger will be removed once all server(s)/worker(s) are updated to use `v2_*` tables +CREATE OR REPLACE FUNCTION v2_job_queue_after_insert() RETURNS TRIGGER AS $$ BEGIN + INSERT INTO v2_job ( + id, workspace_id, created_at, created_by, permissioned_as, permissioned_as_email, + kind, runnable_id, runnable_path, parent_job, + script_lang, script_entrypoint_override, + flow_step, flow_step_id, flow_innermost_root_job, + trigger, trigger_kind, + tag, same_worker, visible_to_owner, concurrent_limit, concurrency_time_window_s, cache_ttl, timeout, priority, + preprocessed, args, pre_run_error, + raw_code, raw_lock, raw_flow + ) VALUES ( + NEW.id, NEW.workspace_id, NEW.created_at, NEW.__created_by, NEW.__permissioned_as, NEW.__email, + NEW.__job_kind, NEW.__script_hash, NEW.__script_path, NEW.__parent_job, + NEW.__language, NULLIF(NEW.__args->>'_ENTRYPOINT_OVERRIDE', '__WM_PREPROCESSOR'), + NULL, NEW.__flow_step_id, NEW.__root_job, + NEW.__schedule_path, CASE WHEN NEW.__schedule_path IS NOT NULL THEN 'schedule'::job_trigger_kind END, + NEW.tag, NEW.__same_worker, NEW.__visible_to_owner, NEW.__concurrent_limit, NEW.__concurrency_time_window_s, + NEW.__cache_ttl, NEW.__timeout, NEW.priority, + CASE WHEN ( + NEW.__args->>'_ENTRYPOINT_OVERRIDE' IN ('__WM_PREPROCESSOR', 'preprocessor') + OR NEW.__flow_status->'preprocessor_module' IS NOT NULL + ) THEN FALSE END, NEW.__args, NEW.__pre_run_error, + NEW.__raw_code, NEW.__raw_lock, NEW.__raw_flow + ) ON CONFLICT (id) DO UPDATE SET + workspace_id = EXCLUDED.workspace_id, + created_at = EXCLUDED.created_at, + created_by = EXCLUDED.created_by, + permissioned_as = EXCLUDED.permissioned_as, + permissioned_as_email = EXCLUDED.permissioned_as_email, + kind = EXCLUDED.kind, + runnable_id = EXCLUDED.runnable_id, + runnable_path = EXCLUDED.runnable_path, + parent_job = EXCLUDED.parent_job, + script_lang = EXCLUDED.script_lang, + script_entrypoint_override = EXCLUDED.script_entrypoint_override, + flow_step = EXCLUDED.flow_step, + flow_step_id = EXCLUDED.flow_step_id, + flow_innermost_root_job = EXCLUDED.flow_innermost_root_job, + trigger = EXCLUDED.trigger, + trigger_kind = EXCLUDED.trigger_kind, + tag = EXCLUDED.tag, + same_worker = EXCLUDED.same_worker, + visible_to_owner = EXCLUDED.visible_to_owner, + concurrent_limit = EXCLUDED.concurrent_limit, + concurrency_time_window_s = EXCLUDED.concurrency_time_window_s, + cache_ttl = EXCLUDED.cache_ttl, + timeout = EXCLUDED.timeout, + priority = EXCLUDED.priority, + preprocessed = EXCLUDED.preprocessed, + args = EXCLUDED.args, + pre_run_error = EXCLUDED.pre_run_error, + raw_code = COALESCE(v2_job.raw_code, EXCLUDED.raw_code), + raw_lock = COALESCE(v2_job.raw_lock, EXCLUDED.raw_lock), + raw_flow = COALESCE(v2_job.raw_flow, EXCLUDED.raw_flow) + ; + INSERT INTO v2_job_runtime (id, ping, memory_peak) + VALUES (NEW.id, NEW.__last_ping, NEW.__mem_peak) + ON CONFLICT (id) DO UPDATE SET + ping = COALESCE(v2_job_runtime.ping, EXCLUDED.ping), + memory_peak = COALESCE(v2_job_runtime.memory_peak, EXCLUDED.memory_peak) + ; + IF NEW.__flow_status IS NOT NULL OR NEW.__leaf_jobs IS NOT NULL THEN + INSERT INTO v2_job_status (id, flow_status, flow_leaf_jobs) + VALUES (NEW.id, NEW.__flow_status, NEW.__leaf_jobs) + ON CONFLICT (id) DO UPDATE SET + flow_status = COALESCE(v2_job_status.flow_status, EXCLUDED.flow_status), + flow_leaf_jobs = COALESCE(v2_job_status.flow_leaf_jobs, EXCLUDED.flow_leaf_jobs) + ; + END IF; + RETURN NEW; +END $$ LANGUAGE plpgsql; + +CREATE OR REPLACE TRIGGER v2_job_queue_after_insert_trigger + AFTER INSERT ON v2_job_queue + FOR EACH ROW + -- Prevent infinite loop v1 <-> v2 + WHEN (pg_trigger_depth() < 1 AND NEW.__created_by IS NOT NULL AND NEW.__env_id IS DISTINCT FROM 123456789) +EXECUTE FUNCTION v2_job_queue_after_insert(); + +-- On every update to `v2_job_queue`, update `v2_job`, `v2_job_runtime` and `v2_job_status` as well +-- This trigger will be removed once all server(s)/worker(s) are updated to use `v2_*` tables +CREATE OR REPLACE FUNCTION v2_job_queue_before_update() RETURNS TRIGGER AS $$ BEGIN + IF NEW.canceled_by IS NOT NULL THEN + NEW.__canceled := TRUE; + END IF; + -- `v2_job`: Only `args` are updated + IF NEW.__args::TEXT IS DISTINCT FROM OLD.__args::TEXT THEN + UPDATE v2_job SET + args = NEW.__args, + preprocessed = CASE WHEN preprocessed = FALSE THEN TRUE ELSE preprocessed END + WHERE id = NEW.id; + END IF; + -- `v2_job_runtime`: + IF NEW.__last_ping IS DISTINCT FROM OLD.__last_ping OR NEW.__mem_peak IS DISTINCT FROM OLD.__mem_peak THEN + INSERT INTO v2_job_runtime (id, ping, memory_peak) + VALUES (NEW.id, NEW.__last_ping, NEW.__mem_peak) + ON CONFLICT (id) DO UPDATE SET + ping = EXCLUDED.ping, + memory_peak = EXCLUDED.memory_peak + ; + END IF; + -- `v2_job_status`: + IF NEW.__flow_status::TEXT IS DISTINCT FROM OLD.__flow_status::TEXT OR + NEW.__leaf_jobs::TEXT IS DISTINCT FROM OLD.__leaf_jobs::TEXT THEN + IF NEW.__is_flow_step = false AND NEW.__parent_job IS NOT NULL THEN + -- `workflow_as_code`: + INSERT INTO v2_job_status (id, workflow_as_code_status) + VALUES (NEW.id, NEW.__flow_status) + ON CONFLICT (id) DO UPDATE SET + workflow_as_code_status = EXCLUDED.workflow_as_code_status + ; + ELSE + INSERT INTO v2_job_status (id, flow_status, flow_leaf_jobs) + VALUES (NEW.id, NEW.__flow_status, NEW.__leaf_jobs) + ON CONFLICT (id) DO UPDATE SET + flow_status = EXCLUDED.flow_status, + flow_leaf_jobs = EXCLUDED.flow_leaf_jobs + ; + END IF; + END IF; + -- `job_logs`: + IF NEW.__logs IS DISTINCT FROM OLD.__logs THEN + INSERT INTO job_logs (job_id, workspace_id, logs) + VALUES (NEW.id, NEW.workspace_id, NEW.__logs) + ON CONFLICT (job_id) DO UPDATE SET + logs = CONCAT(job_logs.logs, EXCLUDED.logs) + ; + NEW.__logs := NULL; + END IF; + RETURN NEW; +END $$ LANGUAGE plpgsql; + +CREATE OR REPLACE TRIGGER v2_job_queue_before_update_trigger + BEFORE UPDATE ON v2_job_queue + FOR EACH ROW + WHEN (pg_trigger_depth() < 1) -- Prevent infinite loop v1 <-> v2 +EXECUTE FUNCTION v2_job_queue_before_update(); + +-- v2 -> v1: update preprocessed args +CREATE OR REPLACE FUNCTION v2_job_after_update() RETURNS TRIGGER AS $$ BEGIN + UPDATE v2_job_queue SET __args = NEW.args WHERE id = NEW.id; + UPDATE v2_job_completed SET __args = NEW.args WHERE id = NEW.id; + RETURN NEW; +END $$ LANGUAGE PLPGSQL; + +CREATE OR REPLACE TRIGGER v2_job_after_update_trigger + AFTER UPDATE ON v2_job + FOR EACH ROW + WHEN (pg_trigger_depth() < 1 AND NEW.args::TEXT IS DISTINCT FROM OLD.args::TEXT) -- Prevent infinite loop v1 <-> v2 +EXECUTE FUNCTION v2_job_after_update(); diff --git a/backend/migrations/20250201124744_v2_job_completed_sync.down.sql b/backend/migrations/20250201124744_v2_job_completed_sync.down.sql new file mode 100644 index 0000000000..4875cd34ff --- /dev/null +++ b/backend/migrations/20250201124744_v2_job_completed_sync.down.sql @@ -0,0 +1,3 @@ +-- Add down migration script here +DROP FUNCTION v2_job_completed_before_insert() CASCADE; +DROP FUNCTION v2_job_completed_before_update() CASCADE; diff --git a/backend/migrations/20250201124744_v2_job_completed_sync.up.sql b/backend/migrations/20250201124744_v2_job_completed_sync.up.sql new file mode 100644 index 0000000000..d1017ca7fd --- /dev/null +++ b/backend/migrations/20250201124744_v2_job_completed_sync.up.sql @@ -0,0 +1,153 @@ +-- Add up migration script here + +-- v1 -> v2 +-- On every insert to `v2_job_completed`, insert to `v2_job` as well +-- This trigger will be removed once all server(s)/worker(s) are updated to use `v2_*` tables +CREATE OR REPLACE FUNCTION v2_job_completed_before_insert() RETURNS TRIGGER AS $$ +DECLARE job v2_job; +DECLARE final_labels TEXT[]; +BEGIN + -- New columns synchronization: + -- 1. `result_columns` <-> `flow_status._metadata.column_order` + -- 2. `v2_job.labels` <-> `result.wm_labels` + -- 3. v2 <-> flow_status._metadata.preprocessed_args` + IF NEW.__created_by IS NULL THEN + -- v2 -> v1 + -- When inserting to `v2_job_completed` from `v2` code, set `v1` columns: + SELECT * INTO job FROM v2_job WHERE id = NEW.id; + NEW.__parent_job := job.parent_job; + NEW.__created_by := job.created_by; + NEW.__created_at := job.created_at; + NEW.__success := NEW.status = 'success'::job_status; + NEW.__script_hash := job.runnable_id; + NEW.__script_path := job.runnable_path; + NEW.__args := job.args; + -- __logs + NEW.__raw_code := job.raw_code; + NEW.__canceled := NEW.status = 'canceled'::job_status; + NEW.__job_kind := job.kind; + -- __env_id + NEW.__schedule_path := CASE WHEN job.trigger_kind = 'schedule'::job_trigger_kind THEN job.trigger END; + NEW.__permissioned_as := job.permissioned_as; + NEW.__raw_flow := job.raw_flow; + NEW.__is_flow_step := job.flow_step_id IS NOT NULL; + NEW.__language := job.script_lang; + NEW.__is_skipped := NEW.status = 'skipped'::job_status; + NEW.__raw_lock := job.raw_lock; + NEW.__email := job.permissioned_as_email; + NEW.__visible_to_owner := job.visible_to_owner; + NEW.__tag := job.tag; + NEW.__priority := job.priority; + -- 1. `result_columns` -> `flow_status._metadata.column_order` + IF NEW.result_columns IS NOT NULL AND (NEW.flow_status IS NULL OR jsonb_typeof(NEW.flow_status) = 'object') THEN + NEW.flow_status := jsonb_set( + coalesce(NEW.flow_status, '{}'::JSONB), + '{_metadata}', + jsonb_set( + coalesce(NEW.flow_status->'_metadata', '{}'::JSONB), + '{column_order}', + to_jsonb(NEW.result_columns) + ) + ); + END IF; + -- 2. `v2_job.labels` -> `result.wm_labels` + IF job.labels IS NOT NULL AND (NEW.result IS NULL OR jsonb_typeof(NEW.result) = 'object') THEN + IF jsonb_typeof(NEW.result->'wm_labels') = 'array' AND ( + SELECT bool_and(jsonb_typeof(elem) = 'string') + FROM jsonb_array_elements(NEW.result->'wm_labels') AS elem + ) THEN + SELECT array_agg(DISTINCT all_labels) INTO final_labels + FROM unnest( + coalesce(job.labels, ARRAY[]::TEXT[]) || translate(NEW.result->>'wm_labels', '[]', '{}')::TEXT[] + ) all_labels; + ELSE + final_labels := job.labels; + END IF; + -- Update `v2_job.labels` if needed + IF job.labels IS DISTINCT FROM final_labels THEN + UPDATE v2_job SET labels = final_labels WHERE id = NEW.id; + END IF; + NEW.result := jsonb_set( + coalesce(NEW.result, '{}'::JSONB), + '{wm_labels}', + to_jsonb(final_labels) + ); + END IF; + -- 3. v2 -> flow_status._metadata.preprocessed_args` + IF job.kind = 'script' AND job.preprocessed = TRUE + AND (NEW.flow_status IS NULL OR jsonb_typeof(NEW.flow_status) = 'object') + THEN + NEW.flow_status := jsonb_set( + coalesce(NEW.flow_status, '{}'::JSONB), + '{_metadata}', + jsonb_set( + coalesce(NEW.flow_status->'_metadata', '{}'::JSONB), + '{preprocessed_args}', + 'true'::JSONB + ) + ); + END IF; + ELSE + -- v1 -> v2 + NEW.completed_at := now(); + NEW.status := CASE + WHEN NEW.__is_skipped THEN 'skipped'::job_status + WHEN NEW.__canceled THEN 'canceled'::job_status + WHEN NEW.__success THEN 'success'::job_status + ELSE 'failure'::job_status + END; + -- 1. `result_columns` <- `flow_status._metadata.column_order` + IF jsonb_typeof(NEW.flow_status->'_metadata'->'column_order') = 'array' AND ( + SELECT bool_and(jsonb_typeof(elem) = 'string') + FROM jsonb_array_elements(NEW.flow_status->'_metadata'->'column_order') AS elem + ) THEN + NEW.result_columns := translate(NEW.flow_status->'_metadata'->>'column_order', '[]', '{}')::TEXT[]; + END IF; + -- 2. `v2_job.labels` <- `result.wm_labels` + IF jsonb_typeof(NEW.result->'wm_labels') = 'array' AND ( + SELECT bool_and(jsonb_typeof(elem) = 'string') + FROM jsonb_array_elements(NEW.result->'wm_labels') AS elem + ) THEN + UPDATE v2_job SET + labels = ( + SELECT array_agg(DISTINCT all_labels) + FROM unnest( + coalesce(labels, ARRAY[]::TEXT[]) + || translate(NEW.result->>'wm_labels', '[]', '{}')::TEXT[] + ) all_labels + ) + WHERE id = NEW.id; + END IF; + -- 3. v2 <- flow_status._metadata.preprocessed_args` + IF NEW.flow_status->'_metadata'->'preprocessed_args' = 'true'::JSONB THEN + UPDATE v2_job SET + args = NEW.__args, + preprocessed = TRUE + WHERE id = NEW.id AND preprocessed = FALSE; + END IF; + END IF; + RETURN NEW; +END $$ LANGUAGE plpgsql; + +CREATE OR REPLACE TRIGGER v2_job_completed_before_insert_trigger + BEFORE INSERT ON v2_job_completed + FOR EACH ROW + WHEN (pg_trigger_depth() < 1) -- Prevent infinite loop v1 <-> v2 +EXECUTE FUNCTION v2_job_completed_before_insert(); + +CREATE OR REPLACE FUNCTION v2_job_completed_before_update() RETURNS TRIGGER AS $$ BEGIN + -- `v2_job`: Only `args` are updated + IF NEW.__args::TEXT IS DISTINCT FROM OLD.__args::TEXT THEN + UPDATE v2_job SET + args = NEW.__args, + preprocessed = CASE WHEN preprocessed = FALSE THEN TRUE ELSE preprocessed END + WHERE id = NEW.id; + END IF; + RETURN NEW; +END $$ LANGUAGE plpgsql; + +CREATE OR REPLACE TRIGGER v2_job_completed_before_update_trigger + BEFORE UPDATE ON v2_job_completed + FOR EACH ROW + WHEN (pg_trigger_depth() < 1) -- Prevent infinite loop v1 <-> v2 +EXECUTE FUNCTION v2_job_completed_before_update(); diff --git a/backend/migrations/20250201124748_v2_migrate_from_v1.down.sql b/backend/migrations/20250201124748_v2_migrate_from_v1.down.sql new file mode 100644 index 0000000000..6bc376fc3d --- /dev/null +++ b/backend/migrations/20250201124748_v2_migrate_from_v1.down.sql @@ -0,0 +1,2 @@ +-- Add down migration script here +-- Nothing to do here diff --git a/backend/migrations/20250201124748_v2_migrate_from_v1.up.sql b/backend/migrations/20250201124748_v2_migrate_from_v1.up.sql new file mode 100644 index 0000000000..d3095e0e3d --- /dev/null +++ b/backend/migrations/20250201124748_v2_migrate_from_v1.up.sql @@ -0,0 +1,194 @@ +-- Add up migration script here + +-- Set new columns in `v2_job_completed`: +UPDATE v2_job_completed +SET completed_at = started_at + (interval '1 millisecond' * duration_ms), + status = CASE + WHEN __is_skipped THEN 'skipped'::job_status + WHEN __canceled THEN 'canceled'::job_status + WHEN __success THEN 'success'::job_status + ELSE 'failure'::job_status + END, + result_columns = CASE + WHEN jsonb_typeof(flow_status->'_metadata'->'column_order') = 'array' AND ( + SELECT bool_and(jsonb_typeof(elem) = 'string') + FROM jsonb_array_elements(flow_status->'_metadata'->'column_order') AS elem + ) THEN translate(flow_status->'_metadata'->>'column_order', '[]', '{}')::TEXT[] + END +WHERE status IS NULL; + +-- Insert missing `v2_job` rows from `v2_job_queue`: +INSERT INTO v2_job ( + id, workspace_id, created_at, created_by, permissioned_as, permissioned_as_email, + kind, runnable_id, runnable_path, parent_job, + script_lang, script_entrypoint_override, + flow_step_id, flow_innermost_root_job, + trigger, trigger_kind, + tag, same_worker, visible_to_owner, concurrent_limit, concurrency_time_window_s, cache_ttl, timeout, priority, + preprocessed, args, pre_run_error, + raw_code, raw_lock, raw_flow +) SELECT + id, workspace_id, created_at, __created_by, __permissioned_as, __email, + __job_kind, __script_hash, __script_path, __parent_job, + __language, NULLIF(__args->>'_ENTRYPOINT_OVERRIDE', '__WM_PREPROCESSOR'), + __flow_step_id, __root_job, + __schedule_path, CASE WHEN __schedule_path IS NOT NULL THEN 'schedule'::job_trigger_kind END, + tag, __same_worker, __visible_to_owner, __concurrent_limit, __concurrency_time_window_s, + __cache_ttl, __timeout, priority, + CASE + WHEN __args->>'_ENTRYPOINT_OVERRIDE' IN ('__WM_PREPROCESSOR', 'preprocessor') THEN FALSE + WHEN __flow_status->'preprocessor_module' IS NOT NULL THEN __args->'wm_trigger' IS NULL + END, __args, __pre_run_error, + __raw_code, __raw_lock, __raw_flow +FROM v2_job_queue +WHERE NOT EXISTS (SELECT 1 FROM v2_job WHERE v2_job.id = v2_job_queue.id); + +-- Insert missing `v2_job` rows from `v2_job_completed`: +INSERT INTO v2_job ( + id, workspace_id, created_at, created_by, permissioned_as, permissioned_as_email, + kind, runnable_id, runnable_path, parent_job, + script_lang, script_entrypoint_override, + trigger, trigger_kind, + tag, visible_to_owner, priority, + preprocessed, args, + raw_code, raw_lock, raw_flow, + labels +) SELECT + id, workspace_id, __created_at, __created_by, __permissioned_as, __email, + __job_kind, __script_hash, __script_path, __parent_job, + __language, NULLIF(__args->>'_ENTRYPOINT_OVERRIDE', '__WM_PREPROCESSOR'), + __schedule_path, CASE WHEN __schedule_path IS NOT NULL THEN 'schedule'::job_trigger_kind END, + __tag, __visible_to_owner, __priority, + CASE WHEN v2_job_completed.flow_status->'_metadata'->'preprocessed_args' = 'true'::JSONB THEN TRUE END, __args, + __raw_code, __raw_lock, __raw_flow, + CASE + WHEN jsonb_typeof(result->'wm_labels') = 'array' AND ( + SELECT bool_and(jsonb_typeof(elem) = 'string') + FROM jsonb_array_elements(result->'wm_labels') AS elem + ) THEN ( + SELECT array_agg(DISTINCT all_labels) + FROM unnest( + translate(result->>'wm_labels', '[]', '{}')::TEXT[] + ) all_labels + ) + END +FROM v2_job_completed +WHERE NOT EXISTS (SELECT 1 FROM v2_job WHERE v2_job.id = v2_job_completed.id); + +-- Set existing `v2_job` rows from `v2_job_queue`: +UPDATE v2_job SET + created_at = v2_job_queue.created_at, + created_by = v2_job_queue.__created_by, + permissioned_as = v2_job_queue.__permissioned_as, + permissioned_as_email = v2_job_queue.__email, + kind = v2_job_queue.__job_kind, + runnable_id = v2_job_queue.__script_hash, + runnable_path = v2_job_queue.__script_path, + parent_job = v2_job_queue.__parent_job, + script_lang = v2_job_queue.__language, + script_entrypoint_override = NULLIF(v2_job_queue.__args->>'_ENTRYPOINT_OVERRIDE', '__WM_PREPROCESSOR'), + flow_step_id = v2_job_queue.__flow_step_id, + flow_innermost_root_job = v2_job_queue.__root_job, + trigger = v2_job_queue.__schedule_path, + trigger_kind = CASE WHEN v2_job_queue.__schedule_path IS NOT NULL THEN 'schedule'::job_trigger_kind END, + tag = v2_job_queue.tag, + same_worker = v2_job_queue.__same_worker, + visible_to_owner = v2_job_queue.__visible_to_owner, + concurrent_limit = v2_job_queue.__concurrent_limit, + concurrency_time_window_s = v2_job_queue.__concurrency_time_window_s, + cache_ttl = v2_job_queue.__cache_ttl, + timeout = v2_job_queue.__timeout, + priority = v2_job_queue.priority, + preprocessed = CASE + WHEN __args->>'_ENTRYPOINT_OVERRIDE' IN ('__WM_PREPROCESSOR', 'preprocessor') THEN FALSE + WHEN __flow_status->'preprocessor_module' IS NOT NULL THEN __args->'wm_trigger' IS NULL + END, + args = v2_job_queue.__args, + pre_run_error = v2_job_queue.__pre_run_error, + raw_code = COALESCE(v2_job.raw_code, v2_job_queue.__raw_code), + raw_lock = COALESCE(v2_job.raw_lock, v2_job_queue.__raw_lock), + raw_flow = COALESCE(v2_job.raw_flow, v2_job_queue.__raw_flow) +FROM v2_job_queue +WHERE v2_job.id = v2_job_queue.id AND v2_job.created_by = 'missing'; + +-- Set existing `v2_job` rows from `v2_job_completed`: +UPDATE v2_job SET + created_at = v2_job_completed.__created_at, + created_by = v2_job_completed.__created_by, + permissioned_as = v2_job_completed.__permissioned_as, + permissioned_as_email = v2_job_completed.__email, + kind = v2_job_completed.__job_kind, + runnable_id = v2_job_completed.__script_hash, + runnable_path = v2_job_completed.__script_path, + parent_job = v2_job_completed.__parent_job, + script_lang = v2_job_completed.__language, + script_entrypoint_override = NULLIF(v2_job_completed.__args->>'_ENTRYPOINT_OVERRIDE', '__WM_PREPROCESSOR'), + trigger = v2_job_completed.__schedule_path, + trigger_kind = CASE WHEN v2_job_completed.__schedule_path IS NOT NULL THEN 'schedule'::job_trigger_kind END, + tag = v2_job_completed.__tag, + visible_to_owner = v2_job_completed.__visible_to_owner, + priority = v2_job_completed.__priority, + preprocessed = CASE WHEN v2_job_completed.flow_status->'_metadata'->'preprocessed_args' = 'true'::JSONB THEN TRUE END, + args = v2_job_completed.__args, + raw_code = COALESCE(v2_job.raw_code, v2_job_completed.__raw_code), + raw_lock = COALESCE(v2_job.raw_lock, v2_job_completed.__raw_lock), + raw_flow = COALESCE(v2_job.raw_flow, v2_job_completed.__raw_flow), + labels = CASE + WHEN jsonb_typeof(v2_job_completed.result->'wm_labels') = 'array' AND ( + SELECT bool_and(jsonb_typeof(elem) = 'string') + FROM jsonb_array_elements(v2_job_completed.result->'wm_labels') AS elem + ) THEN ( + SELECT array_agg(DISTINCT all_labels) + FROM unnest( + coalesce(v2_job.labels, ARRAY[]::TEXT[]) + || translate(v2_job_completed.result->>'wm_labels', '[]', '{}')::TEXT[] + ) all_labels + ) + END +FROM v2_job_completed +WHERE v2_job.id = v2_job_completed.id AND v2_job.created_by = 'missing'; + +-- Migrate `v2_job_queue` moved columns to `v2_job_runtime`: +INSERT INTO v2_job_runtime (id, ping, memory_peak) +SELECT id, __last_ping, __mem_peak +FROM v2_job_queue +WHERE __last_ping IS NOT NULL OR __mem_peak IS NOT NULL + -- Locked ones will sync within triggers + FOR UPDATE SKIP LOCKED +ON CONFLICT (id) DO NOTHING; + +-- Migrate `v2_job_queue` moved columns to `v2_job_status` for `workflow_as_code`: +INSERT INTO v2_job_status (id, workflow_as_code_status) +SELECT id, __flow_status +FROM v2_job_queue +WHERE __flow_status IS NOT NULL AND __is_flow_step = false AND __parent_job IS NOT NULL + -- Locked ones will sync within triggers + FOR UPDATE SKIP LOCKED +ON CONFLICT (id) DO NOTHING; + +-- Migrate `v2_job_queue` moved columns to `v2_job_status`: +INSERT INTO v2_job_status (id, flow_status, flow_leaf_jobs) +SELECT id, __flow_status, __leaf_jobs +FROM v2_job_queue +WHERE __flow_status IS NOT NULL OR __leaf_jobs IS NOT NULL + -- Locked ones will sync within triggers + FOR UPDATE SKIP LOCKED +ON CONFLICT (id) DO NOTHING; + +-- Migrate old `v2_job_queue.__logs` to `job_logs` +INSERT INTO job_logs (job_id, workspace_id, logs) +SELECT id, workspace_id, __logs +FROM v2_job_queue +WHERE __logs IS NOT NULL +ON CONFLICT (job_id) DO UPDATE SET + logs = CONCAT(job_logs.logs, EXCLUDED.logs) +; + +-- Migrate old `v2_job_completed.__logs` to `job_logs` +INSERT INTO job_logs (job_id, workspace_id, logs) +SELECT id, workspace_id, __logs +FROM v2_job_completed +WHERE __logs IS NOT NULL AND __logs IS DISTINCT FROM '##DELETED##' +ON CONFLICT (job_id) DO UPDATE SET + logs = CONCAT(job_logs.logs, EXCLUDED.logs) +; diff --git a/backend/migrations/20250201124804_v2_job_runtime_sync.down.sql b/backend/migrations/20250201124804_v2_job_runtime_sync.down.sql new file mode 100644 index 0000000000..9c9fe30241 --- /dev/null +++ b/backend/migrations/20250201124804_v2_job_runtime_sync.down.sql @@ -0,0 +1,3 @@ +-- Add down migration script here +DROP FUNCTION v2_job_runtime_before_insert() CASCADE; +DROP FUNCTION v2_job_runtime_before_update() CASCADE; diff --git a/backend/migrations/20250201124804_v2_job_runtime_sync.up.sql b/backend/migrations/20250201124804_v2_job_runtime_sync.up.sql new file mode 100644 index 0000000000..26947c3615 --- /dev/null +++ b/backend/migrations/20250201124804_v2_job_runtime_sync.up.sql @@ -0,0 +1,31 @@ +-- Add up migration script here + +-- On every insert/update to `v2_job_runtime`, reflect to `v2_job_queue` as well +-- This triggers will be removed once all server(s)/worker(s) are updated to use `v2_*` tables +CREATE OR REPLACE FUNCTION v2_job_runtime_before_insert() RETURNS TRIGGER AS $$ BEGIN + UPDATE v2_job_queue + SET __last_ping = NEW.ping, __mem_peak = NEW.memory_peak + WHERE id = NEW.id; + RETURN NEW; +END $$ LANGUAGE plpgsql; + +CREATE OR REPLACE TRIGGER v2_job_runtime_before_insert_trigger + BEFORE INSERT ON v2_job_runtime + FOR EACH ROW + WHEN (pg_trigger_depth() < 1) -- Prevent infinite loop v2 <-> v1 +EXECUTE FUNCTION v2_job_runtime_before_insert(); + +CREATE OR REPLACE FUNCTION v2_job_runtime_before_update() RETURNS TRIGGER AS $$ BEGIN + IF NEW.ping IS DISTINCT FROM OLD.ping OR NEW.memory_peak IS DISTINCT FROM OLD.memory_peak THEN + UPDATE v2_job_queue + SET __last_ping = NEW.ping, __mem_peak = NEW.memory_peak + WHERE id = NEW.id; + END IF; + RETURN NEW; +END $$ LANGUAGE plpgsql; + +CREATE OR REPLACE TRIGGER v2_job_runtime_before_update_trigger + BEFORE UPDATE ON v2_job_runtime + FOR EACH ROW + WHEN (pg_trigger_depth() < 1) -- Prevent infinite loop v2 <-> v1 +EXECUTE FUNCTION v2_job_runtime_before_update(); diff --git a/backend/migrations/20250201124805_v2_job_status_sync.down.sql b/backend/migrations/20250201124805_v2_job_status_sync.down.sql new file mode 100644 index 0000000000..95ce39e16b --- /dev/null +++ b/backend/migrations/20250201124805_v2_job_status_sync.down.sql @@ -0,0 +1,3 @@ +-- Add down migration script here +DROP FUNCTION v2_job_status_before_insert() CASCADE; +DROP FUNCTION v2_job_status_before_update() CASCADE; diff --git a/backend/migrations/20250201124805_v2_job_status_sync.up.sql b/backend/migrations/20250201124805_v2_job_status_sync.up.sql new file mode 100644 index 0000000000..08e7e73dd2 --- /dev/null +++ b/backend/migrations/20250201124805_v2_job_status_sync.up.sql @@ -0,0 +1,37 @@ +-- Add up migration script here + +-- On every insert/update to `v2_job_status`, reflect to `v2_job_queue` as well +-- This triggers will be removed once all server(s)/worker(s) are updated to use `v2_*` tables +CREATE OR REPLACE FUNCTION v2_job_status_before_insert() RETURNS TRIGGER AS $$ BEGIN + UPDATE v2_job_queue + SET __flow_status = NEW.flow_status, __leaf_jobs = NEW.flow_leaf_jobs + WHERE id = NEW.id; + RETURN NEW; +END $$ LANGUAGE plpgsql; + +CREATE OR REPLACE TRIGGER v2_job_status_before_insert_trigger + BEFORE INSERT ON v2_job_status + FOR EACH ROW + WHEN (pg_trigger_depth() < 1) -- Prevent infinite loop v2 <-> v1 +EXECUTE FUNCTION v2_job_status_before_insert(); + +CREATE OR REPLACE FUNCTION v2_job_status_before_update() RETURNS TRIGGER AS $$ BEGIN + IF NEW.flow_status::TEXT IS DISTINCT FROM OLD.flow_status::TEXT OR + NEW.flow_leaf_jobs::TEXT IS DISTINCT FROM OLD.flow_leaf_jobs::TEXT THEN + UPDATE v2_job_queue + SET __flow_status = NEW.flow_status, __leaf_jobs = NEW.flow_leaf_jobs + WHERE id = NEW.id; + END IF; + IF NEW.workflow_as_code_status::TEXT IS DISTINCT FROM OLD.workflow_as_code_status::TEXT THEN + UPDATE v2_job_queue + SET __flow_status = NEW.workflow_as_code_status + WHERE id = NEW.id; + END IF; + RETURN NEW; +END $$ LANGUAGE plpgsql; + +CREATE OR REPLACE TRIGGER v2_job_status_before_update_trigger + BEFORE UPDATE ON v2_job_status + FOR EACH ROW + WHEN (pg_trigger_depth() < 1) -- Prevent infinite loop v2 <-> v1 +EXECUTE FUNCTION v2_job_status_before_update(); diff --git a/backend/migrations/20250201124919_v2_job_constraints.down.sql b/backend/migrations/20250201124919_v2_job_constraints.down.sql new file mode 100644 index 0000000000..72796ff008 --- /dev/null +++ b/backend/migrations/20250201124919_v2_job_constraints.down.sql @@ -0,0 +1,4 @@ +-- Add down migration script here +ALTER TABLE v2_job ALTER COLUMN workspace_id DROP NOT NULL; +ALTER TABLE v2_job ALTER COLUMN tag DROP DEFAULT; +ALTER TABLE v2_job ALTER COLUMN tag DROP NOT NULL; diff --git a/backend/migrations/20250201124919_v2_job_constraints.up.sql b/backend/migrations/20250201124919_v2_job_constraints.up.sql new file mode 100644 index 0000000000..b0ba090fa9 --- /dev/null +++ b/backend/migrations/20250201124919_v2_job_constraints.up.sql @@ -0,0 +1,4 @@ +-- Add up migration script here +ALTER TABLE v2_job ALTER COLUMN workspace_id SET NOT NULL; +ALTER TABLE v2_job ALTER COLUMN tag SET DEFAULT 'other'; +ALTER TABLE v2_job ALTER COLUMN tag SET NOT NULL; diff --git a/backend/migrations/20250201145628_v2_job_queue_constraints.down.sql b/backend/migrations/20250201145628_v2_job_queue_constraints.down.sql new file mode 100644 index 0000000000..9f13ce1b41 --- /dev/null +++ b/backend/migrations/20250201145628_v2_job_queue_constraints.down.sql @@ -0,0 +1,2 @@ +-- Add up migration script here +ALTER TABLE v2_job_queue ALTER COLUMN __created_by SET NOT NULL; diff --git a/backend/migrations/20250201145628_v2_job_queue_constraints.up.sql b/backend/migrations/20250201145628_v2_job_queue_constraints.up.sql new file mode 100644 index 0000000000..5eac732ab0 --- /dev/null +++ b/backend/migrations/20250201145628_v2_job_queue_constraints.up.sql @@ -0,0 +1,2 @@ +-- Add up migration script here +ALTER TABLE v2_job_queue ALTER COLUMN __created_by DROP NOT NULL; diff --git a/backend/migrations/20250201145629_v2_job_completed_constraints.down.sql b/backend/migrations/20250201145629_v2_job_completed_constraints.down.sql new file mode 100644 index 0000000000..bf4a399470 --- /dev/null +++ b/backend/migrations/20250201145629_v2_job_completed_constraints.down.sql @@ -0,0 +1,10 @@ +-- Add down migration script here +LOCK TABLE v2_job_completed IN ACCESS EXCLUSIVE MODE; +UPDATE v2_job_completed SET started_at = completed_at WHERE started_at IS NULL; +ALTER TABLE v2_job_completed ALTER COLUMN status DROP NOT NULL; +ALTER TABLE v2_job_completed ALTER COLUMN completed_at DROP DEFAULT; +ALTER TABLE v2_job_completed ALTER COLUMN completed_at DROP NOT NULL; +ALTER TABLE v2_job_completed ALTER COLUMN started_at SET NOT NULL; +ALTER TABLE v2_job_completed ALTER COLUMN __created_at SET NOT NULL; +ALTER TABLE v2_job_completed ALTER COLUMN __created_by SET NOT NULL; +ALTER TABLE v2_job_completed ALTER COLUMN __success SET NOT NULL; diff --git a/backend/migrations/20250201145629_v2_job_completed_constraints.up.sql b/backend/migrations/20250201145629_v2_job_completed_constraints.up.sql new file mode 100644 index 0000000000..9c4772ec72 --- /dev/null +++ b/backend/migrations/20250201145629_v2_job_completed_constraints.up.sql @@ -0,0 +1,8 @@ +-- Add up migration script here +ALTER TABLE v2_job_completed ALTER COLUMN status SET NOT NULL; +ALTER TABLE v2_job_completed ALTER COLUMN completed_at SET DEFAULT now(); +ALTER TABLE v2_job_completed ALTER COLUMN completed_at SET NOT NULL; +ALTER TABLE v2_job_completed ALTER COLUMN started_at DROP NOT NULL; +ALTER TABLE v2_job_completed ALTER COLUMN __created_at DROP NOT NULL; +ALTER TABLE v2_job_completed ALTER COLUMN __created_by DROP NOT NULL; +ALTER TABLE v2_job_completed ALTER COLUMN __success DROP NOT NULL; diff --git a/backend/migrations/20250201145633_v2_fix_no_runtime_2.down.sql b/backend/migrations/20250201145633_v2_fix_no_runtime_2.down.sql new file mode 100644 index 0000000000..d2f607c5b8 --- /dev/null +++ b/backend/migrations/20250201145633_v2_fix_no_runtime_2.down.sql @@ -0,0 +1 @@ +-- Add down migration script here diff --git a/backend/migrations/20250201145633_v2_fix_no_runtime_2.up.sql b/backend/migrations/20250201145633_v2_fix_no_runtime_2.up.sql new file mode 100644 index 0000000000..f18ad51d1c --- /dev/null +++ b/backend/migrations/20250201145633_v2_fix_no_runtime_2.up.sql @@ -0,0 +1,12 @@ +-- Add up migration script here +UPDATE v2_job_queue SET + running = false, + started_at = NULL +WHERE running = true AND NOT EXISTS (SELECT 1 FROM v2_job_runtime WHERE id = v2_job_queue.id); + +INSERT INTO v2_job_runtime (id, ping, memory_peak) +SELECT id, __last_ping, __mem_peak +FROM v2_job_queue + -- Locked ones will sync within triggers + FOR UPDATE SKIP LOCKED +ON CONFLICT (id) DO NOTHING; diff --git a/backend/migrations/20250205131513_http_static_websites.down.sql b/backend/migrations/20250205131513_http_static_websites.down.sql new file mode 100644 index 0000000000..72a7ae8a15 --- /dev/null +++ b/backend/migrations/20250205131513_http_static_websites.down.sql @@ -0,0 +1,11 @@ +ALTER TABLE http_trigger DROP COLUMN is_static_website; + +CREATE OR REPLACE FUNCTION prevent_route_path_change() +RETURNS TRIGGER AS $$ +BEGIN + IF CURRENT_USER <> 'windmill_admin' AND NEW.route_path <> OLD.route_path THEN + RAISE EXCEPTION 'Modification of route_path is only allowed by admins'; + END IF; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; \ No newline at end of file diff --git a/backend/migrations/20250205131513_http_static_websites.up.sql b/backend/migrations/20250205131513_http_static_websites.up.sql new file mode 100644 index 0000000000..e4edc0255d --- /dev/null +++ b/backend/migrations/20250205131513_http_static_websites.up.sql @@ -0,0 +1,12 @@ +ALTER TABLE http_trigger ADD COLUMN is_static_website BOOLEAN NOT NULL DEFAULT FALSE; + + +CREATE OR REPLACE FUNCTION prevent_route_path_change() +RETURNS TRIGGER AS $$ +BEGIN + IF CURRENT_USER = 'windmill_user' AND NEW.route_path <> OLD.route_path THEN + RAISE EXCEPTION 'Modification of route_path is only allowed by admins'; + END IF; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; \ No newline at end of file diff --git a/backend/migrations/20250205131514_v2_queue_compatibility_view.down.sql b/backend/migrations/20250205131514_v2_queue_compatibility_view.down.sql new file mode 100644 index 0000000000..23bb72031f --- /dev/null +++ b/backend/migrations/20250205131514_v2_queue_compatibility_view.down.sql @@ -0,0 +1,2 @@ +-- Add down migration script here +DROP VIEW v2_as_queue; diff --git a/backend/migrations/20250205131514_v2_queue_compatibility_view.up.sql b/backend/migrations/20250205131514_v2_queue_compatibility_view.up.sql new file mode 100644 index 0000000000..c0b79bf9b6 --- /dev/null +++ b/backend/migrations/20250205131514_v2_queue_compatibility_view.up.sql @@ -0,0 +1,52 @@ +-- Add up migration script here +CREATE OR REPLACE VIEW v2_as_queue AS +SELECT + j.id, + j.workspace_id, + j.parent_job, + j.created_by, + j.created_at, + q.started_at, + q.scheduled_for, + q.running, + j.runnable_id AS script_hash, + j.runnable_path AS script_path, + j.args, + j.raw_code, + q.canceled_by IS NOT NULL AS canceled, + q.canceled_by, + q.canceled_reason, + r.ping AS last_ping, + j.kind AS job_kind, + CASE WHEN j.trigger_kind = 'schedule'::job_trigger_kind THEN j.trigger END + AS schedule_path, + j.permissioned_as, + COALESCE(s.flow_status, s.workflow_as_code_status) AS flow_status, + j.raw_flow, + j.flow_step_id IS NOT NULL AS is_flow_step, + j.script_lang AS language, + q.suspend, + q.suspend_until, + j.same_worker, + j.raw_lock, + j.pre_run_error, + j.permissioned_as_email AS email, + j.visible_to_owner, + r.memory_peak AS mem_peak, + j.flow_innermost_root_job AS root_job, + s.flow_leaf_jobs AS leaf_jobs, + j.tag, + j.concurrent_limit, + j.concurrency_time_window_s, + j.timeout, + j.flow_step_id, + j.cache_ttl, + j.priority, + NULL::TEXT AS logs, + j.script_entrypoint_override, + j.preprocessed +FROM v2_job_queue q + JOIN v2_job j USING (id) + LEFT JOIN v2_job_runtime r USING (id) + LEFT JOIN v2_job_status s USING (id) +; diff --git a/backend/migrations/20250205131515_v2_completed_job_compatibility_view.down.sql b/backend/migrations/20250205131515_v2_completed_job_compatibility_view.down.sql new file mode 100644 index 0000000000..90ce271f4a --- /dev/null +++ b/backend/migrations/20250205131515_v2_completed_job_compatibility_view.down.sql @@ -0,0 +1,2 @@ +-- Add up migration script here +DROP VIEW v2_as_completed_job; diff --git a/backend/migrations/20250205131515_v2_completed_job_compatibility_view.up.sql b/backend/migrations/20250205131515_v2_completed_job_compatibility_view.up.sql new file mode 100644 index 0000000000..51709c42c7 --- /dev/null +++ b/backend/migrations/20250205131515_v2_completed_job_compatibility_view.up.sql @@ -0,0 +1,42 @@ +-- Add up migration script here +CREATE OR REPLACE VIEW v2_as_completed_job AS +SELECT + j.id, + j.workspace_id, + j.parent_job, + j.created_by, + j.created_at, + c.duration_ms, + c.status = 'success' AS success, + j.runnable_id AS script_hash, + j.runnable_path AS script_path, + j.args, + c.result, + FALSE AS deleted, + j.raw_code, + c.status = 'canceled' AS canceled, + c.canceled_by, + c.canceled_reason, + j.kind AS job_kind, + CASE WHEN j.trigger_kind = 'schedule'::job_trigger_kind THEN j.trigger END + AS schedule_path, + j.permissioned_as, + COALESCE(c.flow_status, c.workflow_as_code_status) AS flow_status, + j.raw_flow, + j.flow_step_id IS NOT NULL AS is_flow_step, + j.script_lang AS language, + c.started_at, + c.status = 'skipped' AS is_skipped, + j.raw_lock, + j.permissioned_as_email AS email, + j.visible_to_owner, + c.memory_peak AS mem_peak, + j.tag, + j.priority, + NULL::TEXT AS logs, + c.result_columns, + j.script_entrypoint_override, + j.preprocessed +FROM v2_job_completed c + JOIN v2_job j USING (id) +; diff --git a/backend/parsers/windmill-parser-py-imports/src/lib.rs b/backend/parsers/windmill-parser-py-imports/src/lib.rs index 58f857ef08..99b3b37cf5 100644 --- a/backend/parsers/windmill-parser-py-imports/src/lib.rs +++ b/backend/parsers/windmill-parser-py-imports/src/lib.rs @@ -65,6 +65,7 @@ static PYTHON_IMPORTS_REPLACEMENT: phf::Map<&'static str, &'static str> = phf_ma "pythonjsonlogger" => "python-json-logger", "socks" => "PySocks", "taiga" => "python-taiga", + "docx" => "python-docx", }; fn replace_import(x: String) -> String { diff --git a/backend/src/main.rs b/backend/src/main.rs index d4459a3371..d9ae5c6e58 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -320,7 +320,7 @@ async fn windmill_main() -> anyhow::Result<()> { .unwrap_or(DEFAULT_NUM_WORKERS as i32) }; - if num_workers > 1 { + if num_workers > 1 && !std::env::var("WORKER_GROUP").is_ok_and(|x| x == "native") { println!( "We STRONGLY recommend using at most 1 worker per container, use at your own risks" ); @@ -344,8 +344,17 @@ async fn windmill_main() -> anyhow::Result<()> { }; println!("Connecting to database..."); - let db = windmill_common::connect_db(server_mode, indexer_mode).await?; + let db = windmill_common::initial_connection().await?; + let num_version = sqlx::query_scalar!("SELECT version()").fetch_one(&db).await; + + tracing::info!( + "PostgreSQL version: {} (windmill require PG >= 14)", + num_version + .ok() + .flatten() + .unwrap_or_else(|| "UNKNOWN".to_string()) + ); load_otel(&db).await; tracing::info!("Database connected"); @@ -362,19 +371,14 @@ async fn windmill_main() -> anyhow::Result<()> { let _guard = windmill_common::tracing_init::initialize_tracing(&hostname, &mode, &environment); - let num_version = sqlx::query_scalar!("SELECT version()").fetch_one(&db).await; - - tracing::info!( - "PostgreSQL version: {} (windmill require PG >= 14)", - num_version - .ok() - .flatten() - .unwrap_or_else(|| "UNKNOWN".to_string()) - ); - let is_agent = mode == Mode::Agent; - if !is_agent { + #[cfg(feature = "parquet")] + let disable_s3_store = std::env::var("DISABLE_S3_STORE") + .ok() + .is_some_and(|x| x == "1" || x == "true"); + + if !is_agent && !indexer_mode { let skip_migration = std::env::var("SKIP_MIGRATION") .map(|val| val == "true") .unwrap_or(false); @@ -387,10 +391,15 @@ async fn windmill_main() -> anyhow::Result<()> { } } + drop(db); + let worker_mode = num_workers > 0; + + let db = windmill_common::connect_db(server_mode, indexer_mode, worker_mode).await?; + let (killpill_tx, mut killpill_rx) = tokio::sync::broadcast::channel::<()>(2); let mut monitor_killpill_rx = killpill_tx.subscribe(); - let server_killpill_rx = killpill_tx.subscribe(); let (killpill_phase2_tx, _killpill_phase2_rx) = tokio::sync::broadcast::channel::<()>(2); + let server_killpill_rx = killpill_phase2_tx.subscribe(); let shutdown_signal = windmill_common::shutdown_signal(killpill_tx.clone(), killpill_tx.subscribe()); @@ -450,8 +459,6 @@ Windmill Community Edition {GIT_VERSION} } } - let worker_mode = num_workers > 0; - if server_mode || worker_mode || indexer_mode { let port_var = std::env::var("PORT").ok().and_then(|x| x.parse().ok()); @@ -474,7 +481,15 @@ Windmill Community Edition {GIT_VERSION} default_base_internal_url.clone() }; - initial_load(&db, killpill_tx.clone(), worker_mode, server_mode, is_agent).await; + initial_load( + &db, + killpill_tx.clone(), + worker_mode, + server_mode, + #[cfg(feature = "parquet")] + disable_s3_store, + ) + .await; monitor_db( &db, @@ -635,7 +650,7 @@ Windmill Community Edition {GIT_VERSION} killpill_tx.clone(), num_workers, base_internal_url.clone(), - mode.clone() == Mode::Agent, + is_agent, hostname.clone(), ) .await?; @@ -646,9 +661,13 @@ Windmill Community Edition {GIT_VERSION} } } if killpill_phase2_tx.receiver_count() > 0 { - tracing::info!("Starting phase 2 of shutdown"); + if worker_mode { + tracing::info!("Starting phase 2 of shutdown"); + } killpill_phase2_tx.send(())?; - tracing::info!("Phase 2 of shutdown completed"); + if worker_mode { + tracing::info!("Phase 2 of shutdown completed"); + } } Ok(()) }; @@ -753,8 +772,10 @@ Windmill Community Edition {GIT_VERSION} reload_job_default_timeout_setting(&db).await }, #[cfg(feature = "parquet")] - OBJECT_STORE_CACHE_CONFIG_SETTING if !is_agent => { - reload_s3_cache_setting(&db).await + OBJECT_STORE_CACHE_CONFIG_SETTING => { + if !disable_s3_store { + reload_s3_cache_setting(&db).await + } }, SCIM_TOKEN_SETTING => { reload_scim_token_setting(&db).await diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index b77d097ab1..a3f117721f 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -19,6 +19,7 @@ use tokio::{ join, sync::{mpsc, RwLock}, }; +use uuid::Uuid; #[cfg(feature = "embedding")] use windmill_api::embeddings::update_embeddings_db; @@ -36,7 +37,7 @@ use windmill_common::{ auth::JWT_SECRET, ee::CriticalErrorChannel, error, - flow_status::FlowStatusModule, + flow_status::{FlowStatus, FlowStatusModule}, global_settings::{ BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING, CRITICAL_ERROR_CHANNELS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, @@ -135,7 +136,7 @@ pub async fn initial_load( tx: tokio::sync::broadcast::Sender<()>, worker_mode: bool, server_mode: bool, - _is_agent: bool, + #[cfg(feature = "parquet")] disable_s3_store: bool, ) { if let Err(e) = load_metrics_enabled(db).await { tracing::error!("Error loading expose metrics: {e:#}"); @@ -179,7 +180,7 @@ pub async fn initial_load( } #[cfg(feature = "parquet")] - if !_is_agent { + if !disable_s3_store { reload_s3_cache_setting(&db).await; } @@ -735,11 +736,17 @@ pub async fn delete_expired_items(db: &DB) -> () { match db.begin().await { Ok(mut tx) => { let deleted_jobs = sqlx::query_scalar!( - "DELETE FROM completed_job WHERE created_at <= now() - ($1::bigint::text || ' s')::interval AND started_at + ((duration_ms/1000 + $1::bigint) || ' s')::interval <= now() RETURNING id", - job_retention_secs - ) - .fetch_all(&mut *tx) - .await; + "DELETE FROM v2_job_completed c + USING v2_job j + WHERE + created_at <= now() - ($1::bigint::text || ' s')::interval + AND completed_at + ($1::bigint::text || ' s')::interval <= now() + AND c.id = j.id + RETURNING c.id", + job_retention_secs + ) + .fetch_all(&mut *tx) + .await; match deleted_jobs { Ok(deleted_jobs) => { @@ -787,7 +794,7 @@ pub async fn delete_expired_items(db: &DB) -> () { } if let Err(e) = - sqlx::query!("DELETE FROM job WHERE id = ANY($1)", &deleted_jobs) + sqlx::query!("DELETE FROM v2_job WHERE id = ANY($1)", &deleted_jobs) .execute(&mut *tx) .await { @@ -1342,13 +1349,19 @@ pub async fn expose_queue_metrics(db: &Pool) { .ok(); if count > 0 { sqlx::query!( - "INSERT INTO metrics (id, value) - VALUES ($1, to_jsonb((SELECT EXTRACT(EPOCH FROM now() - scheduled_for) - FROM queue WHERE tag = $2 AND running = false AND scheduled_for <= now() - ('3 seconds')::interval - ORDER BY priority DESC NULLS LAST, scheduled_for LIMIT 1)))", - format!("queue_delay_{}", tag), - tag - ).execute(db).await.ok(); + "INSERT INTO metrics (id, value) + VALUES ($1, to_jsonb(( + SELECT EXTRACT(EPOCH FROM now() - scheduled_for) + FROM v2_job_queue + WHERE tag = $2 AND running = false AND scheduled_for <= now() - ('3 seconds')::interval + ORDER BY priority DESC NULLS LAST, scheduled_for LIMIT 1 + )))", + format!("queue_delay_{}", tag), + tag + ) + .execute(db) + .await + .ok(); } } } @@ -1507,26 +1520,30 @@ pub async fn reload_base_url_setting(db: &DB) -> error::Result<()> { async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, worker_name: &str) { if *RESTART_ZOMBIE_JOBS { let restarted = sqlx::query!( - "WITH zombie_jobs AS ( - UPDATE queue SET running = false, started_at = null - WHERE last_ping < now() - ($1 || ' seconds')::interval - AND running = true AND job_kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlescriptflow') AND same_worker = false - RETURNING id, workspace_id, last_ping - ), - update_concurrency AS ( - UPDATE concurrency_counter cc - SET job_uuids = job_uuids - zj.id::text - FROM zombie_jobs zj - INNER JOIN concurrency_key ck ON ck.job_id = zj.id - WHERE cc.concurrency_id = ck.key - ) - SELECT id, workspace_id, last_ping FROM zombie_jobs", - *ZOMBIE_JOB_TIMEOUT, + "WITH zombie_jobs AS ( + UPDATE v2_job_queue q SET running = false, started_at = null + FROM v2_job j, v2_job_runtime r + WHERE j.id = q.id AND j.id = r.id + AND ping < now() - ($1 || ' seconds')::interval + AND running = true + AND kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlescriptflow') + AND same_worker = false + RETURNING q.id, q.workspace_id, ping + ), + update_concurrency AS ( + UPDATE concurrency_counter cc + SET job_uuids = job_uuids - zj.id::text + FROM zombie_jobs zj + INNER JOIN concurrency_key ck ON ck.job_id = zj.id + WHERE cc.concurrency_id = ck.key ) - .fetch_all(db) - .await - .ok() - .unwrap_or_else(|| vec![]); + SELECT id, workspace_id, ping FROM zombie_jobs", + *ZOMBIE_JOB_TIMEOUT, + ) + .fetch_all(db) + .await + .ok() + .unwrap_or_else(|| vec![]); #[cfg(feature = "prometheus")] if METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed) { @@ -1535,7 +1552,7 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, worker let base_url = BASE_URL.read().await.clone(); for r in restarted { - let last_ping = if let Some(x) = r.last_ping { + let last_ping = if let Some(x) = r.ping { format!("last ping at {x}") } else { "no last ping".to_string() @@ -1547,36 +1564,101 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, worker ); let _ = sqlx::query!(" - INSERT INTO job_logs (job_id, logs) VALUES ($1,'Restarted job after not receiving job''s ping for too long the ' || now() || '\n\n') - ON CONFLICT (job_id) DO UPDATE SET logs = job_logs.logs || '\nRestarted job after not receiving job''s ping for too long the ' || now() || '\n\n' WHERE job_logs.job_id = $1", r.id) - .execute(db).await; + INSERT INTO job_logs (job_id, logs) + VALUES ($1, 'Restarted job after not receiving job''s ping for too long the ' || now() || '\n\n') + ON CONFLICT (job_id) DO UPDATE SET logs = job_logs.logs || '\n' || EXCLUDED.logs + WHERE job_logs.job_id = $1", + r.id + ) + .execute(db) + .await; tracing::error!(error_message); report_critical_error(error_message, db.clone(), Some(&r.workspace_id), None).await; } } - let mut timeout_query = - "SELECT * FROM queue WHERE last_ping < now() - ($1 || ' seconds')::interval - AND running = true AND job_kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlescriptflow')" - .to_string(); - if *RESTART_ZOMBIE_JOBS { - timeout_query.push_str(" AND same_worker = true"); - }; - let timeouts = sqlx::query_as::<_, QueuedJob>(&timeout_query) - .bind(ZOMBIE_JOB_TIMEOUT.as_str()) + let same_worker_timeout_jobs = { + let long_same_worker_jobs = sqlx::query!( + "SELECT worker, array_agg(v2_job_queue.id) as ids FROM v2_job_queue LEFT JOIN v2_job ON v2_job_queue.id = v2_job.id LEFT JOIN v2_job_runtime ON v2_job_queue.id = v2_job_runtime.id WHERE v2_job_queue.created_at < now() - ('60 seconds')::interval + AND running = true AND ping IS NULL AND same_worker = true AND worker IS NOT NULL GROUP BY worker", + ) .fetch_all(db) .await .ok() .unwrap_or_else(|| vec![]); + let worker_ids = long_same_worker_jobs + .iter() + .map(|x| x.worker.clone().unwrap_or_default()) + .collect::>(); + + let long_dead_workers: std::collections::HashSet = sqlx::query_scalar!( + "WITH worker_ids AS (SELECT unnest($1::text[]) as worker) + SELECT worker_ids.worker FROM worker_ids + LEFT JOIN worker_ping ON worker_ids.worker = worker_ping.worker + WHERE worker_ping.worker IS NULL OR worker_ping.ping_at < now() - ('60 seconds')::interval", + &worker_ids[..] + ) + .fetch_all(db) + .await + .ok() + .unwrap_or_else(|| vec![]) + .into_iter() + .filter_map(|x| x) + .collect(); + + let mut timeouts: Vec = vec![]; + for worker in long_same_worker_jobs { + if worker.worker.is_some() && long_dead_workers.contains(&worker.worker.unwrap()) { + if let Some(ids) = worker.ids { + timeouts.extend(ids); + } + } + } + if !timeouts.is_empty() { + tracing::error!( + "Failing same worker zombie jobs: {:?}", + timeouts + .iter() + .map(|x| x.hyphenated().to_string()) + .collect::>() + .join(",") + ); + } + + let jobs = sqlx::query_as::<_, QueuedJob>("SELECT * FROM v2_as_queue WHERE id = ANY($1)") + .bind(&timeouts[..]) + .fetch_all(db) + .await + .map_err(|e| tracing::error!("Error fetching same worker jobs: {:?}", e)) + .unwrap_or_default(); + + jobs + }; + + let non_restartable_jobs = if *RESTART_ZOMBIE_JOBS { + vec![] + } else { + sqlx::query_as::<_, QueuedJob>("SELECT * FROM v2_as_queue WHERE last_ping < now() - ($1 || ' seconds')::interval + AND running = true AND job_kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlescriptflow') AND same_worker = false") + .bind(ZOMBIE_JOB_TIMEOUT.as_str()) + .fetch_all(db) + .await + .ok() + .unwrap_or_else(|| vec![]) + }; + + let timeouts = non_restartable_jobs + .into_iter() + .chain(same_worker_timeout_jobs) + .collect::>(); + #[cfg(feature = "prometheus")] if METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed) { QUEUE_ZOMBIE_DELETE_COUNT.inc_by(timeouts.len() as _); } for job in timeouts { - tracing::info!("timedout zombie job {} {}", job.id, job.workspace_id,); - // since the job is unrecoverable, the same worker queue should never be sent anything let (same_worker_tx_never_used, _same_worker_rx_never_used) = mpsc::channel::(1); @@ -1618,11 +1700,12 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, worker 0, None, error::Error::ExecutionErr(format!( - "Job timed out after no ping from job since {} (ZOMBIE_JOB_TIMEOUT: {})", + "Job timed out after no ping from job since {} (ZOMBIE_JOB_TIMEOUT: {}, same_worker: {})", last_ping .map(|x| x.to_string()) .unwrap_or_else(|| "no ping".to_string()), - *ZOMBIE_JOB_TIMEOUT + *ZOMBIE_JOB_TIMEOUT, + job.same_worker )), true, same_worker_tx_never_used, @@ -1630,27 +1713,35 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, worker worker_name, send_result_never_used, #[cfg(feature = "benchmark")] - &mut windmill_common::bench::BenchmarkIter::new(), + &mut windmill_worker::bench::BenchmarkIter::new(), ) .await; } } async fn handle_zombie_flows(db: &DB) -> error::Result<()> { - let flows = sqlx::query_as::<_, QueuedJob>( + let flows = sqlx::query!( r#" - SELECT * - FROM queue - WHERE running = true AND suspend = 0 AND suspend_until IS null AND scheduled_for <= now() AND (job_kind = 'flow' OR job_kind = 'flowpreview' OR job_kind = 'flownode') - AND last_ping IS NOT NULL AND last_ping < NOW() - ($1 || ' seconds')::interval AND canceled = false + SELECT + id AS "id!", workspace_id AS "workspace_id!", parent_job, is_flow_step, + flow_status AS "flow_status: Box", last_ping, same_worker + FROM v2_as_queue + WHERE running = true AND suspend = 0 AND suspend_until IS null AND scheduled_for <= now() + AND (job_kind = 'flow' OR job_kind = 'flowpreview' OR job_kind = 'flownode') + AND last_ping IS NOT NULL AND last_ping < NOW() - ($1 || ' seconds')::interval + AND canceled = false "#, - ).bind(FLOW_ZOMBIE_TRANSITION_TIMEOUT.as_str()) + FLOW_ZOMBIE_TRANSITION_TIMEOUT.as_str() + ) .fetch_all(db) .await?; for flow in flows { - let status = flow.parse_flow_status(); - if !flow.same_worker + let status = flow + .flow_status + .as_deref() + .and_then(|x| serde_json::from_str::(x).ok()); + if !flow.same_worker.unwrap_or(false) && status.is_some_and(|s| { s.modules .get(0) @@ -1682,7 +1773,8 @@ async fn handle_zombie_flows(db: &DB) -> error::Result<()> { } sqlx::query!( - "UPDATE queue SET running = false, started_at = null WHERE id = $1 AND canceled = false", + "UPDATE v2_job_queue SET running = false, started_at = null + WHERE id = $1 AND canceled_by IS NULL", flow.id ) .execute(&mut *tx) @@ -1695,44 +1787,40 @@ async fn handle_zombie_flows(db: &DB) -> error::Result<()> { let now = now_from_db(db).await?; let reason = format!( "{} was hanging in between 2 steps. Last ping: {last_ping:?} (now: {now})", - if flow.is_flow_step && flow.parent_job.is_some() { + if flow.is_flow_step.unwrap_or(false) && flow.parent_job.is_some() { format!("Flow was cancelled because subflow {id}") } else { format!("Flow {id} was cancelled because it") } ); report_critical_error(reason.clone(), db.clone(), Some(&flow.workspace_id), None).await; - cancel_zombie_flow_job(db, flow, reason).await?; + cancel_zombie_flow_job(db, flow.id, &flow.workspace_id, reason).await?; } } let flows2 = sqlx::query!( - " - DELETE - FROM parallel_monitor_lock - WHERE last_ping IS NOT NULL AND last_ping < NOW() - ($1 || ' seconds')::interval - RETURNING parent_flow_id, job_id, last_ping - ", + r#" + DELETE + FROM parallel_monitor_lock + WHERE last_ping IS NOT NULL AND last_ping < NOW() - ($1 || ' seconds')::interval + RETURNING parent_flow_id, job_id, last_ping, (SELECT workspace_id FROM v2_job_queue q + WHERE q.id = parent_flow_id AND q.running = true AND q.canceled_by IS NULL + ) AS workspace_id + "#, FLOW_ZOMBIE_TRANSITION_TIMEOUT.as_str() ) .fetch_all(db) .await?; for flow in flows2 { - let in_queue = sqlx::query_as::<_, QueuedJob>( - "SELECT * FROM queue WHERE id = $1 AND running = true AND canceled = false", - ) - .bind(flow.parent_flow_id) - .fetch_optional(db) - .await?; - if let Some(job) = in_queue { + if let Some(parent_flow_workspace_id) = flow.workspace_id { tracing::error!( "parallel Zombie flow detected: {} in workspace {}. Last ping was: {:?}.", - job.id, - job.workspace_id, + flow.parent_flow_id, + parent_flow_workspace_id, flow.last_ping ); - cancel_zombie_flow_job(db, job, + cancel_zombie_flow_job(db, flow.parent_flow_id, &parent_flow_workspace_id, format!("Flow {} cancelled as one of the parallel branch {} was unable to make the last transition ", flow.parent_flow_id, flow.job_id)) .await?; } else { @@ -1744,27 +1832,28 @@ async fn handle_zombie_flows(db: &DB) -> error::Result<()> { async fn cancel_zombie_flow_job( db: &Pool, - flow: QueuedJob, + id: Uuid, + workspace_id: &str, message: String, ) -> Result<(), error::Error> { - let tx = db.begin().await.unwrap(); + let mut tx = db.begin().await?; tracing::error!( "zombie flow detected: {} in workspace {}. Cancelling it.", - flow.id, - flow.workspace_id + id, + workspace_id ); - let (ntx, _) = cancel_job( + (tx, _) = cancel_job( "monitor", Some(message), - flow.id, - flow.workspace_id.as_str(), + id, + workspace_id, tx, db, true, false, ) .await?; - ntx.commit().await?; + tx.commit().await?; Ok(()) } diff --git a/backend/tests/fixtures/base.sql b/backend/tests/fixtures/base.sql index f406e603d4..0e38b5c7bd 100644 --- a/backend/tests/fixtures/base.sql +++ b/backend/tests/fixtures/base.sql @@ -23,13 +23,13 @@ GRANT ALL PRIVILEGES ON TABLE workspace_key TO windmill_user; CREATE FUNCTION "notify_insert_on_completed_job" () RETURNS TRIGGER AS $$ BEGIN - PERFORM pg_notify('insert on completed_job', NEW.id::text); + PERFORM pg_notify('completed', NEW.id::text); RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER "notify_insert_on_completed_job" - AFTER INSERT ON "completed_job" + AFTER INSERT ON "v2_job_completed" FOR EACH ROW EXECUTE FUNCTION "notify_insert_on_completed_job" (); @@ -37,18 +37,199 @@ EXECUTE FUNCTION "notify_insert_on_completed_job" (); CREATE FUNCTION "notify_queue" () RETURNS TRIGGER AS $$ BEGIN - PERFORM pg_notify('queue', NEW.id::text); + PERFORM pg_notify('queued', NEW.id::text); RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER "notify_queue_after_insert" - AFTER INSERT ON "queue" + AFTER INSERT ON "v2_job_queue" FOR EACH ROW EXECUTE FUNCTION "notify_queue" (); CREATE TRIGGER "notify_queue_after_flow_status_update" - AFTER UPDATE ON "queue" + AFTER UPDATE ON "v2_job_status" FOR EACH ROW WHEN (NEW.flow_status IS DISTINCT FROM OLD.flow_status) -EXECUTE FUNCTION "notify_queue" (); \ No newline at end of file +EXECUTE FUNCTION "notify_queue" (); + +-- TODO(uael): remove before phase 4 +CREATE OR REPLACE FUNCTION zzz_v2_job_queue_integrity_check() RETURNS TRIGGER AS $$ +DECLARE job v2_job; +DECLARE job_runtime v2_job_runtime; +DECLARE job_status v2_job_status; +BEGIN + IF (OLD.canceled_by IS NOT NULL) IS DISTINCT FROM OLD.__canceled THEN + RAISE EXCEPTION 'canceled mismatch'; + END IF; + -- v2_job: + SELECT * INTO job FROM v2_job WHERE id = OLD.id; + IF job.tag IS DISTINCT FROM OLD.tag THEN + RAISE EXCEPTION 'tag mismatch'; + END IF; + IF job.workspace_id IS DISTINCT FROM OLD.workspace_id THEN + RAISE EXCEPTION 'workspace_id mismatch'; + END IF; + IF job.created_at IS DISTINCT FROM OLD.created_at THEN + RAISE EXCEPTION 'created_at mismatch'; + END IF; + IF job.created_by IS DISTINCT FROM OLD.__created_by THEN + RAISE EXCEPTION 'created_by mismatch'; + END IF; + IF job.permissioned_as IS DISTINCT FROM OLD.__permissioned_as THEN + RAISE EXCEPTION 'permissioned_as mismatch'; + END IF; + IF job.permissioned_as_email IS DISTINCT FROM OLD.__email THEN + RAISE EXCEPTION 'permissioned_as_email mismatch'; + END IF; + IF job.kind IS DISTINCT FROM OLD.__job_kind THEN + RAISE EXCEPTION 'kind mismatch'; + END IF; + IF job.runnable_id IS DISTINCT FROM OLD.__script_hash THEN + RAISE EXCEPTION 'runnable_id mismatch'; + END IF; + IF job.runnable_path IS DISTINCT FROM OLD.__script_path THEN + RAISE EXCEPTION 'runnable_path mismatch'; + END IF; + IF job.parent_job IS DISTINCT FROM OLD.__parent_job THEN + RAISE EXCEPTION 'parent_job mismatch'; + END IF; + IF job.script_lang IS DISTINCT FROM OLD.__language THEN + RAISE EXCEPTION 'script_lang mismatch'; + END IF; + IF job.script_entrypoint_override IS DISTINCT FROM NULLIF(OLD.__args->>'_ENTRYPOINT_OVERRIDE', '__WM_PREPROCESSOR') + AND OLD.__args->>'reason' IS DISTINCT FROM 'PREPROCESSOR_ARGS_ARE_DISCARDED' + THEN + RAISE EXCEPTION 'script_entrypoint_override mismatch'; + END IF; + IF job.flow_step_id IS DISTINCT FROM OLD.__flow_step_id THEN + RAISE EXCEPTION 'flow_step_id mismatch'; + END IF; + IF (job.flow_step_id IS NOT NULL) IS DISTINCT FROM OLD.__is_flow_step THEN + RAISE EXCEPTION 'is_flow_step mismatch'; + END IF; + IF job.flow_innermost_root_job IS DISTINCT FROM OLD.__root_job THEN + RAISE EXCEPTION 'flow_innermost_root_job mismatch'; + END IF; + IF job.trigger IS DISTINCT FROM OLD.__schedule_path THEN + RAISE EXCEPTION 'trigger mismatch'; + END IF; + IF job.same_worker IS DISTINCT FROM OLD.__same_worker THEN + RAISE EXCEPTION 'same_worker mismatch'; + END IF; + IF job.visible_to_owner IS DISTINCT FROM OLD.__visible_to_owner THEN + RAISE EXCEPTION 'visible_to_owner mismatch'; + END IF; + IF job.concurrent_limit IS DISTINCT FROM OLD.__concurrent_limit THEN + RAISE EXCEPTION 'concurrent_limit mismatch'; + END IF; + IF job.concurrency_time_window_s IS DISTINCT FROM OLD.__concurrency_time_window_s THEN + RAISE EXCEPTION 'concurrency_time_window_s mismatch'; + END IF; + IF job.cache_ttl IS DISTINCT FROM OLD.__cache_ttl THEN + RAISE EXCEPTION 'cache_ttl mismatch'; + END IF; + IF job.timeout IS DISTINCT FROM OLD.__timeout THEN + RAISE EXCEPTION 'timeout mismatch'; + END IF; + IF job.priority IS DISTINCT FROM OLD.priority THEN + RAISE EXCEPTION 'priority mismatch'; + END IF; + IF job.args::TEXT IS DISTINCT FROM OLD.__args::TEXT AND OLD.__args->>'_ENTRYPOINT_OVERRIDE' IS DISTINCT FROM '__WM_PREPROCESSOR' THEN + RAISE EXCEPTION 'args mismatch'; + END IF; + IF job.pre_run_error IS DISTINCT FROM OLD.__pre_run_error THEN + RAISE EXCEPTION 'pre_run_error mismatch'; + END IF; + -- v2_job_runtime: + SELECT * INTO job_runtime FROM v2_job_runtime WHERE id = OLD.id; + IF job_runtime.ping IS DISTINCT FROM OLD.__last_ping THEN + RAISE EXCEPTION 'ping mismatch'; + END IF; + IF job_runtime.memory_peak IS DISTINCT FROM OLD.__mem_peak THEN + RAISE EXCEPTION 'memory_peak mismatch'; + END IF; + -- v2_job_status: + IF EXISTS(SELECT 1 FROM v2_job_status WHERE id = OLD.id) THEN + SELECT * INTO job_status FROM v2_job_status WHERE id = OLD.id; + IF COALESCE(job_status.flow_status, job_status.workflow_as_code_status)::TEXT IS DISTINCT FROM OLD.__flow_status::TEXT + THEN + RAISE EXCEPTION 'flow_status mismatch'; + END IF; + IF job_status.flow_leaf_jobs::TEXT IS DISTINCT FROM OLD.__leaf_jobs::TEXT THEN + RAISE EXCEPTION 'leaf_jobs mismatch'; + END IF; + END IF; + RETURN OLD; +END $$ LANGUAGE PLPGSQL; + +CREATE OR REPLACE TRIGGER zzz_v2_job_queue_integrity_check_before_delete + BEFORE DELETE ON v2_job_queue + FOR EACH ROW +EXECUTE FUNCTION zzz_v2_job_queue_integrity_check(); + +-- TODO(uael): remove before phase 4 +CREATE OR REPLACE FUNCTION zzz_v2_job_completed_integrity_check() RETURNS TRIGGER AS $$ +DECLARE job v2_job; +BEGIN + IF (NEW.canceled_by IS NOT NULL) IS DISTINCT FROM NEW.__canceled THEN + RAISE EXCEPTION 'canceled mismatch'; + END IF; + SELECT * INTO job FROM v2_job WHERE id = NEW.id; + IF job.tag IS DISTINCT FROM NEW.__tag THEN + RAISE EXCEPTION 'tag mismatch % %', job.tag, NEW.__tag; + END IF; + IF job.workspace_id IS DISTINCT FROM NEW.workspace_id THEN + RAISE EXCEPTION 'workspace_id mismatch'; + END IF; + IF job.created_at IS DISTINCT FROM NEW.__created_at THEN + RAISE EXCEPTION 'created_at mismatch'; + END IF; + IF job.created_by IS DISTINCT FROM NEW.__created_by THEN + RAISE EXCEPTION 'created_by mismatch'; + END IF; + IF job.permissioned_as IS DISTINCT FROM NEW.__permissioned_as THEN + RAISE EXCEPTION 'permissioned_as mismatch'; + END IF; + IF job.permissioned_as_email IS DISTINCT FROM NEW.__email THEN + RAISE EXCEPTION 'permissioned_as_email mismatch'; + END IF; + IF job.kind IS DISTINCT FROM NEW.__job_kind THEN + RAISE EXCEPTION 'kind mismatch'; + END IF; + IF job.runnable_id IS DISTINCT FROM NEW.__script_hash THEN + RAISE EXCEPTION 'runnable_id mismatch'; + END IF; + IF job.runnable_path IS DISTINCT FROM NEW.__script_path THEN + RAISE EXCEPTION 'runnable_path mismatch'; + END IF; + IF job.parent_job IS DISTINCT FROM NEW.__parent_job THEN + RAISE EXCEPTION 'parent_job mismatch'; + END IF; + IF job.script_lang IS DISTINCT FROM NEW.__language THEN + RAISE EXCEPTION 'script_lang mismatch'; + END IF; + IF job.script_entrypoint_override IS DISTINCT FROM NULLIF(NEW.__args->>'_ENTRYPOINT_OVERRIDE', '__WM_PREPROCESSOR') + AND NEW.__args->>'reason' IS DISTINCT FROM 'PREPROCESSOR_ARGS_ARE_DISCARDED' + THEN + RAISE EXCEPTION 'script_entrypoint_override mismatch'; + END IF; + IF (job.flow_step_id IS NOT NULL) IS DISTINCT FROM NEW.__is_flow_step THEN + RAISE EXCEPTION 'is_flow_step mismatch'; + END IF; + IF job.trigger IS DISTINCT FROM NEW.__schedule_path THEN + RAISE EXCEPTION 'trigger mismatch'; + END IF; + IF job.visible_to_owner IS DISTINCT FROM NEW.__visible_to_owner THEN + RAISE EXCEPTION 'visible_to_owner mismatch'; + END IF; + IF job.args::TEXT IS DISTINCT FROM NEW.__args::TEXT AND NEW.__args->>'_ENTRYPOINT_OVERRIDE' IS DISTINCT FROM '__WM_PREPROCESSOR' THEN + RAISE EXCEPTION 'args mismatch'; + END IF; + RETURN NEW; +END $$ LANGUAGE PLPGSQL; + +CREATE OR REPLACE TRIGGER zzz_v2_job_completed_integrity_check_after_insert + AFTER INSERT ON v2_job_completed + FOR EACH ROW +EXECUTE FUNCTION zzz_v2_job_completed_integrity_check(); diff --git a/backend/tests/fixtures/hello.sql b/backend/tests/fixtures/hello.sql index 31490ce087..f273cdfeaa 100644 --- a/backend/tests/fixtures/hello.sql +++ b/backend/tests/fixtures/hello.sql @@ -13,6 +13,23 @@ export function main(world: string) { '', 'f/system/hello', 123412, 'deno', ''); +INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( +'test-workspace', +'system', +' +export async function preprocessor() { + return { foo: ''bar'', bar: ''baz'' }; +} + +export async function main(foo: string, bar: string) { + return ''Hello '' + foo + '' '' + bar; +} +', +'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"foo":{"default":null,"description":"","originalType":"string","type":"string"},"bar":{"default":null,"description":"","originalType":"string","type":"string"}},"required":["foo","bar"],"type":"object"}', +'', +'', +'f/system/hello_with_preprocessor', 123413, 'deno', ''); + INSERT INTO public.flow(workspace_id, summary, description, path, versions, schema, value, edited_by) VALUES ( 'test-workspace', '', @@ -40,7 +57,7 @@ INSERT INTO public.flow(workspace_id, summary, description, path, versions, sche 'f/system/hello_with_nodes_flow', '{1443253234253454}', '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"world":{"default":"world","description":"","type":"string"}},"required":[],"type":"object"}', -E'{"modules":[{"id":"a","value":{"type":"forloopflow","modules":[{"id":"b","value":{"type":"rawscript","content":"export function main(world: string) {\\n const greet = `Hello ${world}!`;\\n console.log(greet)\\n return greet\\n}\\n","language":"deno","input_transforms":{"world":{"type":"javascript","expr":"flow_input.iter.value"}},"is_trigger":false}},{"id":"c","value":{"type":"rawscript","content":"export function main(hello: string) {\\n const dareyou = `Did you just say \\"${hello}\\"??!`;\\n console.log(dareyou)\\n return dareyou\\n}","language":"deno","input_transforms":{"hello":{"type":"javascript","value":"${results.b}","expr":"`${results.b}`"}},"is_trigger":false}}],"iterator":{"type":"javascript","expr":"[\'foo\', \'bar\', \'baz\']"},"skip_failures":true,"parallel":false}}],"same_worker":false}', +E'{"modules":[{"id":"a","value":{"type":"forloopflow","modules":[{"id":"b","value":{"type":"rawscript","content":"export function main(world: string) {\\n const greet = `Hello ${world}!`;\\n console.log(greet)\\n return greet\\n}\\n","language":"deno","input_transforms":{"world":{"type":"javascript","expr":"flow_input.iter.value"}},"is_trigger":false}},{"id":"c","value":{"type":"rawscript","content":"export function main(hello: string) {\\n const dareyou = `Did you just say \\"${hello}\\"??!`;\\n console.log(dareyou)\\n return dareyou\\n}","language":"deno","input_transforms":{"hello":{"type":"javascript","value":"${results.b}","expr":"`${results.b}`"}},"is_trigger":false}}],"iterator":{"type":"javascript","expr":"[''foo'', ''bar'', ''baz'']"},"skip_failures":true,"parallel":false}}],"same_worker":false}', 'system' ); @@ -49,6 +66,26 @@ INSERT INTO public.flow_version(id, workspace_id, path, schema, value, created_b 'test-workspace', 'f/system/hello_with_nodes_flow', '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"world":{"default":"world","description":"","type":"string"}},"required":[],"type":"object"}', -E'{"modules":[{"id":"a","value":{"type":"forloopflow","modules":[{"id":"b","value":{"type":"rawscript","content":"export function main(world: string) {\\n const greet = `Hello ${world}!`;\\n console.log(greet)\\n return greet\\n}\\n","language":"deno","input_transforms":{"world":{"type":"javascript","expr":"flow_input.iter.value"}},"is_trigger":false}},{"id":"c","value":{"type":"rawscript","content":"export function main(hello: string) {\\n const dareyou = `Did you just say \\"${hello}\\"??!`;\\n console.log(dareyou)\\n return dareyou\\n}","language":"deno","input_transforms":{"hello":{"type":"javascript","value":"${results.b}","expr":"`${results.b}`"}},"is_trigger":false}}],"iterator":{"type":"javascript","expr":"[\'foo\', \'bar\', \'baz\']"},"skip_failures":true,"parallel":false}}],"same_worker":false}', +E'{"modules":[{"id":"a","value":{"type":"forloopflow","modules":[{"id":"b","value":{"type":"rawscript","content":"export function main(world: string) {\\n const greet = `Hello ${world}!`;\\n console.log(greet)\\n return greet\\n}\\n","language":"deno","input_transforms":{"world":{"type":"javascript","expr":"flow_input.iter.value"}},"is_trigger":false}},{"id":"c","value":{"type":"rawscript","content":"export function main(hello: string) {\\n const dareyou = `Did you just say \\"${hello}\\"??!`;\\n console.log(dareyou)\\n return dareyou\\n}","language":"deno","input_transforms":{"hello":{"type":"javascript","value":"${results.b}","expr":"`${results.b}`"}},"is_trigger":false}}],"iterator":{"type":"javascript","expr":"[''foo'', ''bar'', ''baz'']"},"skip_failures":true,"parallel":false}}],"same_worker":false}', +'system' +); + +INSERT INTO public.flow(workspace_id, summary, description, path, versions, schema, value, edited_by) VALUES ( +'test-workspace', +'', +'', +'f/system/hello_with_preprocessor', +'{1443253234253456}', +'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"foo":{"type":"string","description":"","default":""},"bar":{"type":"string","description":"","default":""}},"required":[],"type":"object","order":["foo","bar"]}', +E'{"modules":[{"id":"a","value":{"type":"rawscript","content":"export async function main(foo: string, bar: string) {\\n return ''Hello '' + foo + ''-'' + bar\\n}\\n","language":"deno","is_trigger":false,"input_transforms":{"bar":{"expr":"flow_input.bar","type":"javascript"},"foo":{"expr":"flow_input.foo","type":"javascript"}}}}],"preprocessor_module":{"id":"preprocessor","value":{"type":"rawscript","content":"export async function preprocessor(\\n\\twm_trigger: {\\n\\t\\tkind: ''http'' | ''email'' | ''webhook'' | ''websocket'' | ''kafka'' | ''nats'',\\n\\t\\thttp?: {\\n\\t\\t\\troute: string\\n\\t\\t\\tpath: string\\n\\t\\t\\tmethod: string\\n\\t\\t\\tparams: Record\\n\\t\\t\\tquery: Record\\n\\t\\t\\theaders: Record\\n\\t\\t},\\n\\t\\twebsocket?: {\\n\\t\\t\\turl: string\\n\\t\\t},\\n\\t\\tkafka?: {\\n\\t\\t\\tbrokers: string[]\\n\\t\\t\\ttopic: string\\n\\t\\t\\tgroup_id: string\\n\\t\\t},\\n\\t\\tnats?: {\\n\\t\\t\\tservers: string[]\\n\\t\\t\\tsubject: string\\n\\t\\t\\theaders?: Record\\n\\t\\t\\tstatus?: number\\n\\t\\t\\tdescription?: string\\n\\t\\t\\tlength: number\\n\\t\\t}\\n\\t},\\n) {\\n\\treturn {\\n\\t\\tfoo: ''bar'',\\n\\t\\tbar: ''baz''\\n\\t}\\n}","language":"deno","is_trigger":false,"input_transforms":{"wm_trigger":{"type":"static"}}}}}', +'system' +); + +INSERT INTO public.flow_version(id, workspace_id, path, schema, value, created_by) VALUES ( +1443253234253456, +'test-workspace', +'f/system/hello_with_preprocessor', +'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"foo":{"type":"string","description":"","default":""},"bar":{"type":"string","description":"","default":""}},"required":[],"type":"object","order":["foo","bar"]}', +E'{"modules":[{"id":"a","value":{"type":"rawscript","content":"export async function main(foo: string, bar: string) {\\n return ''Hello '' + foo + ''-'' + bar\\n}\\n","language":"deno","is_trigger":false,"input_transforms":{"bar":{"expr":"flow_input.bar","type":"javascript"},"foo":{"expr":"flow_input.foo","type":"javascript"}}}}],"preprocessor_module":{"id":"preprocessor","value":{"type":"rawscript","content":"export async function preprocessor(\\n\\twm_trigger: {\\n\\t\\tkind: ''http'' | ''email'' | ''webhook'' | ''websocket'' | ''kafka'' | ''nats'',\\n\\t\\thttp?: {\\n\\t\\t\\troute: string\\n\\t\\t\\tpath: string\\n\\t\\t\\tmethod: string\\n\\t\\t\\tparams: Record\\n\\t\\t\\tquery: Record\\n\\t\\t\\theaders: Record\\n\\t\\t},\\n\\t\\twebsocket?: {\\n\\t\\t\\turl: string\\n\\t\\t},\\n\\t\\tkafka?: {\\n\\t\\t\\tbrokers: string[]\\n\\t\\t\\ttopic: string\\n\\t\\t\\tgroup_id: string\\n\\t\\t},\\n\\t\\tnats?: {\\n\\t\\t\\tservers: string[]\\n\\t\\t\\tsubject: string\\n\\t\\t\\theaders?: Record\\n\\t\\t\\tstatus?: number\\n\\t\\t\\tdescription?: string\\n\\t\\t\\tlength: number\\n\\t\\t}\\n\\t},\\n) {\\n\\treturn {\\n\\t\\tfoo: ''bar'',\\n\\t\\tbar: ''baz''\\n\\t}\\n}","language":"deno","is_trigger":false,"input_transforms":{"wm_trigger":{"type":"static"}}}}}', 'system' ); diff --git a/backend/tests/fixtures/result_format.sql b/backend/tests/fixtures/result_format.sql index 1f990e2d86..2cf8fa7299 100644 --- a/backend/tests/fixtures/result_format.sql +++ b/backend/tests/fixtures/result_format.sql @@ -1,3 +1,9 @@ +INSERT INTO public.v2_job ( + id, workspace_id, created_by, created_at, kind, script_lang +) VALUES ( + '1eecb96a-c8b0-4a3d-b1b6-087878c55e41', 'test-workspace', 'test-user', '2023-01-01 00:00:00', 'script', 'postgresql' +); + INSERT INTO public.completed_job ( id, workspace_id, created_by, created_at, duration_ms, success, flow_status, result, job_kind, language ) VALUES ( diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index 001b15ac45..7573e139ee 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -18,8 +18,6 @@ use tokio::time::{timeout, Duration}; use windmill_api_client::types::{CreateFlowBody, RawScript}; -use sqlx::query; - #[cfg(feature = "enterprise")] use windmill_api_client::types::{EditSchedule, NewSchedule, ScriptArgs}; @@ -191,7 +189,6 @@ async fn set_jwt_secret() -> () { mod suspend_resume { use serde_json::json; - use sqlx::query_scalar; use super::*; @@ -202,11 +199,13 @@ mod suspend_resume { ) { loop { queue.by_ref().find(&flow).await.unwrap(); - if query_scalar("SELECT suspend > 0 FROM queue WHERE id = $1") - .bind(flow) - .fetch_one(db) - .await - .unwrap() + if sqlx::query_scalar!( + "SELECT suspend > 0 AS \"r!\" FROM v2_job_queue WHERE id = $1", + flow + ) + .fetch_one(db) + .await + .unwrap() { break; } @@ -360,7 +359,7 @@ mod suspend_resume { // ensure resumes are cleaned up through CASCADE when the flow is finished assert_eq!( 0, - query_scalar::<_, i64>("SELECT count(*) FROM resume_job") + sqlx::query_scalar!("SELECT count(*) AS \"count!\" FROM resume_job") .fetch_one(&db) .await .unwrap() @@ -927,7 +926,7 @@ impl RunJob { /* root job */ None, /* job_id */ None, /* is_flow_step */ false, - /* running */ false, + /* same_worker */ false, None, true, None, @@ -951,6 +950,21 @@ impl RunJob { let r = completed_job(uuid, db).await; r } + + /// push the job, spawn a worker, wait until the job is in completed_job + async fn run_until_complete_with>( + self, + db: &Pool, + port: u16, + test: impl Fn(Uuid) -> F, + ) -> CompletedJob { + let uuid = self.push(db).await; + let listener = listen_for_completed_jobs(db).await; + test(uuid).await; + in_test_worker(db, listener.find(&uuid), port).await; + let r = completed_job(uuid, db).await; + r + } } async fn run_job_in_new_worker_until_complete( @@ -1049,11 +1063,11 @@ fn spawn_test_worker( } async fn listen_for_completed_jobs(db: &Pool) -> impl Stream + Unpin { - listen_for_uuid_on(db, "insert on completed_job").await + listen_for_uuid_on(db, "completed").await } async fn listen_for_queue(db: &Pool) -> impl Stream + Unpin { - listen_for_uuid_on(db, "queue").await + listen_for_uuid_on(db, "queued").await } async fn listen_for_uuid_on( @@ -1078,7 +1092,7 @@ async fn listen_for_uuid_on( async fn completed_job(uuid: Uuid, db: &Pool) -> CompletedJob { sqlx::query_as::<_, CompletedJob>( - "SELECT *, result->'wm_labels' as labels FROM completed_job WHERE id = $1", + "SELECT *, result->'wm_labels' as labels FROM v2_as_completed_job WHERE id = $1", ) .bind(uuid) .fetch_one(db) @@ -3190,11 +3204,13 @@ async fn test_script_schedule_handlers(db: Pool) { let uuid = uuid.unwrap().unwrap(); - let completed_job = - query!("SELECT script_path FROM completed_job WHERE id = $1", uuid) - .fetch_one(&db2) - .await - .unwrap(); + let completed_job = sqlx::query!( + "SELECT script_path FROM v2_as_completed_job WHERE id = $1", + uuid + ) + .fetch_one(&db2) + .await + .unwrap(); if completed_job.script_path.is_none() || completed_job.script_path != Some("f/system/schedule_error_handler".to_string()) @@ -3259,7 +3275,7 @@ async fn test_script_schedule_handlers(db: Pool) { let uuid = uuid.unwrap().unwrap(); let completed_job = - query!("SELECT script_path FROM completed_job WHERE id = $1", uuid) + sqlx::query!("SELECT script_path FROM v2_as_completed_job WHERE id = $1", uuid) .fetch_one(&db2) .await .unwrap(); @@ -3342,11 +3358,13 @@ async fn test_flow_schedule_handlers(db: Pool) { let uuid = uuid.unwrap().unwrap(); - let completed_job = - query!("SELECT script_path FROM completed_job WHERE id = $1", uuid) - .fetch_one(&db2) - .await - .unwrap(); + let completed_job = sqlx::query!( + "SELECT script_path FROM v2_as_completed_job WHERE id = $1", + uuid + ) + .fetch_one(&db2) + .await + .unwrap(); if completed_job.script_path.is_none() || completed_job.script_path != Some("f/system/schedule_error_handler".to_string()) @@ -3412,7 +3430,7 @@ async fn test_flow_schedule_handlers(db: Pool) { let uuid = uuid.unwrap().unwrap(); let completed_job = - query!("SELECT script_path FROM completed_job WHERE id = $1", uuid) + sqlx::query!("SELECT script_path FROM v2_as_completed_job WHERE id = $1", uuid) .fetch_one(&db2) .await .unwrap(); @@ -3487,7 +3505,7 @@ async fn run_deployed_relative_imports( async move { completed.next().await; // deployed script - let script = query!( + let script = sqlx::query!( "SELECT hash FROM script WHERE path = $1", "f/system/test_import".to_string() ) @@ -3743,6 +3761,162 @@ async fn test_result_format(db: Pool) { assert_eq!(result.get(), correct_result); } +#[sqlx::test(fixtures("base"))] +async fn test_job_labels(db: Pool) { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await; + let port = server.addr.port(); + + let db = &db; + let test = |original_labels: &'static [&'static str]| async move { + let job = RunJob::from(JobPayload::RawFlow { + value: serde_json::from_value(json!({ + "modules": [{ + "id": "a", + "value": { + "type": "rawscript", + "content": r#"export function main(world: string) { + const greet = `Hello ${world}!`; + console.log(greet) + return { greet, wm_labels: ["yolo", "greet", "greet", world] }; + }"#, + "language": "deno", + "input_transforms": { + "world": { "type": "javascript", "expr": "flow_input.world" } + } + } + }], + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { "world": { "type": "string" } }, + "type": "object", + "order": [ "world" ] + } + })) + .unwrap(), + path: None, + restarted_from: None, + }) + .arg("world", json!("you")) + .run_until_complete_with(&db, port, |id| async move { + sqlx::query!( + "UPDATE v2_job SET labels = $2 WHERE id = $1 AND $2::TEXT[] IS NOT NULL", + id, + original_labels as &[&str], + ) + .execute(db) + .await + .unwrap(); + }) + .await; + + let result = job.json_result().unwrap(); + assert_eq!(result.get("greet"), Some(&json!("Hello you!"))); + let labels = sqlx::query_scalar!("SELECT labels FROM v2_job WHERE id = $1", job.id) + .fetch_one(db) + .await + .unwrap(); + let mut expected_labels = original_labels + .iter() + .chain(&["yolo", "greet", "you"]) + .map(ToString::to_string) + .collect::>(); + expected_labels.sort(); + assert_eq!(labels, Some(expected_labels)); + }; + test(&[]).await; + test(&["z", "a", "x"]).await; +} + +#[cfg(feature = "python")] +const WORKFLOW_AS_CODE: &str = r#" +from wmill import task + +import pandas as pd +import numpy as np + +@task() +def heavy_compute(n: int): + df = pd.DataFrame(np.random.randn(100, 4), columns=list('ABCD')) + return df.sum().sum() + +@task +def send_result(res: int, email: str): + print(f"Sending result {res} to {email}") + return "OK" + +def main(n: int): + l = [] + for i in range(n): + l.append(heavy_compute(i)) + print(l) + return [send_result(sum(l), "example@example.com"), n] +"#; + +#[cfg(feature = "python")] +#[sqlx::test(fixtures("base", "hello"))] +async fn test_workflow_as_code(db: Pool) { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await; + let port = server.addr.port(); + + // workflow as code require at least 2 workers: + let db = &db; + in_test_worker( + &db, + async move { + let job = RunJob::from(JobPayload::Code(RawCode { + language: ScriptLang::Python3, + content: WORKFLOW_AS_CODE.into(), + ..RawCode::default() + })) + .arg("n", json!(3)) + .run_until_complete(&db, port) + .await; + + assert_eq!(job.json_result().unwrap(), json!(["OK", 3])); + + let workflow_as_code_status = sqlx::query_scalar!( + "SELECT workflow_as_code_status FROM v2_job_completed WHERE id = $1", + job.id + ) + .fetch_one(db) + .await + .unwrap() + .unwrap(); + + #[derive(Deserialize)] + #[allow(dead_code)] + struct WorkflowJobStatus { + name: String, + started_at: String, + scheduled_for: String, + duration_ms: i64, + } + + let workflow_as_code_status: std::collections::HashMap = + serde_json::from_value(workflow_as_code_status).unwrap(); + + let uuids = sqlx::query_scalar!("SELECT id FROM v2_job WHERE parent_job = $1", job.id) + .fetch_all(db) + .await + .unwrap(); + + assert_eq!(uuids.len(), 4); + for uuid in uuids { + let status = workflow_as_code_status.get(&uuid.to_string()); + assert!(status.is_some()); + assert!( + status.unwrap().name == "send_result" + || status.unwrap().name == "heavy_compute" + ); + } + }, + port, + ) + .await; +} + async fn test_for_versions>( version_flags: impl Iterator>>, test: impl Fn() -> F, @@ -3799,6 +3973,53 @@ mod job_payload { test_for_versions(VERSION_FLAGS.iter().cloned(), test).await; } + #[sqlx::test(fixtures("base", "hello"))] + async fn test_script_hash_payload_with_preprocessor(db: Pool) { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await; + let port = server.addr.port(); + + let test = || async { + let db = &db; + let job = RunJob::from(JobPayload::ScriptHash { + hash: ScriptHash(123413), + path: "f/system/hello_with_preprocessor".to_string(), + custom_concurrency_key: None, + concurrent_limit: None, + concurrency_time_window_s: None, + cache_ttl: None, + dedicated_worker: None, + language: ScriptLang::Deno, + priority: None, + apply_preprocessor: true, + }) + .run_until_complete_with(db, port, |id| async move { + let job = sqlx::query!("SELECT preprocessed FROM v2_job WHERE id = $1", id) + .fetch_one(db) + .await + .unwrap(); + assert_eq!(job.preprocessed, Some(false)); + }) + .await; + + let args = job.args.as_ref().unwrap(); + assert_eq!(args.get("foo"), Some(&json!("bar"))); + assert_eq!(args.get("bar"), Some(&json!("baz"))); + // TODO: remove this check on v2 phase 4 + assert_eq!( + job.flow_status.as_ref().unwrap().get("_metadata"), + Some(&json!({"preprocessed_args": true})) + ); + assert_eq!(job.json_result().unwrap(), json!("Hello bar baz")); + let job = sqlx::query!("SELECT preprocessed FROM v2_job WHERE id = $1", job.id) + .fetch_one(db) + .await + .unwrap(); + assert_eq!(job.preprocessed, Some(true)); + }; + test_for_versions(VERSION_FLAGS.iter().cloned(), test).await; + } + #[sqlx::test(fixtures("base", "hello"))] async fn test_flow_script_payload(db: Pool) { initialize_tracing().await; @@ -4081,7 +4302,7 @@ mod job_payload { let result = RunJob::from(JobPayload::Flow { path: "f/system/hello_with_nodes_flow".to_string(), dedicated_worker: None, - apply_preprocessor: true, + apply_preprocessor: false, }) .run_until_complete(&db, port) .await @@ -4113,6 +4334,73 @@ mod job_payload { test_for_versions(VERSION_FLAGS.iter().cloned(), test).await; } + #[sqlx::test(fixtures("base", "hello"))] + async fn test_flow_payload_with_preprocessor(db: Pool) { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await; + let port = server.addr.port(); + + let db = &db; + let test = || async { + let job = RunJob::from(JobPayload::Flow { + path: "f/system/hello_with_preprocessor".to_string(), + dedicated_worker: None, + apply_preprocessor: true, + }) + .run_until_complete_with(db, port, |id| async move { + let job = sqlx::query!("SELECT preprocessed FROM v2_job WHERE id = $1", id) + .fetch_one(db) + .await + .unwrap(); + assert_eq!(job.preprocessed, Some(false)); + }) + .await; + + let args = job.args.as_ref().unwrap(); + let flow_status = job.flow_status.as_ref().unwrap(); + assert_eq!(args.get("foo"), Some(&json!("bar"))); + assert_eq!(args.get("bar"), Some(&json!("baz"))); + assert_eq!(job.json_result().unwrap(), json!("Hello bar-baz")); + let job = sqlx::query!("SELECT preprocessed FROM v2_job WHERE id = $1", job.id) + .fetch_one(db) + .await + .unwrap(); + assert_eq!(job.preprocessed, Some(true)); + let flow_status = serde_json::from_value::(flow_status.clone()).unwrap(); + let FlowStatusModule::Success { job, .. } = flow_status.preprocessor_module.unwrap() + else { + panic!("Expected a success preprocessor module"); + }; + let pp_id = job; + let job = sqlx::query!( + "SELECT preprocessed, script_entrypoint_override FROM v2_job WHERE id = $1", + pp_id + ) + .fetch_one(db) + .await + .unwrap(); + assert_eq!(job.preprocessed, Some(true)); + assert_eq!( + job.script_entrypoint_override.as_deref(), + Some("preprocessor") + ); + }; + // Test the not "lite" flow. + test_for_versions(VERSION_FLAGS.iter().cloned(), test).await; + // Deploy the flow to produce the "lite" version. + let _ = RunJob::from(JobPayload::FlowDependencies { + path: "f/system/hello_with_preprocessor".to_string(), + dedicated_worker: None, + version: 1443253234253456, + }) + .run_until_complete(db, port) + .await + .json_result() + .unwrap(); + // Test the "lite" flow. + test_for_versions(VERSION_FLAGS.iter().cloned(), test).await; + } + #[sqlx::test(fixtures("base", "hello"))] async fn test_restarted_flow_payload(db: Pool) { initialize_tracing().await; @@ -4210,4 +4498,124 @@ mod job_payload { }; test_for_versions(VERSION_FLAGS.iter().cloned(), test).await; } + + #[sqlx::test(fixtures("base", "hello"))] + async fn test_raw_flow_payload_with_restarted_from(db: Pool) { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await; + let port = server.addr.port(); + + let db = &db; + let test = |restarted_from, arg, result| async move { + let job = RunJob::from(JobPayload::RawFlow { + value: serde_json::from_value(json!({ + "modules": [{ + "id": "a", + "value": { + "type": "rawscript", + "content": r#"export function main(world: string) { + return `Hello ${world}!`; + }"#, + "language": "deno", + "input_transforms": { + "world": { "type": "javascript", "expr": "flow_input.world" } + } + } + }, { + "id": "b", + "value": { + "type": "rawscript", + "content": r#"export function main(world: string, a: string) { + return `${a} ${world}!`; + }"#, + "language": "deno", + "input_transforms": { + "world": { "type": "javascript", "expr": "flow_input.world" }, + "a": { "type": "javascript", "expr": "results.a" } + } + } + }, { + "id": "c", + "value": { + "type": "forloopflow", + "iterator": { "type": "javascript", "expr": "['a', 'b', 'c']" }, + "modules": [{ + "value": { + "input_transforms": { + "world": { "type": "javascript", "expr": "flow_input.world" }, + "b": { "type": "javascript", "expr": "results.b" }, + "x": { "type": "javascript", "expr": "flow_input.iter.value" } + }, + "type": "rawscript", + "language": "deno", + "content": r#"export function main(world: string, b: string, x: string) { + return `${x}: ${b} ${world}!`; + }"#, + }, + }], + } + }], + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { "world": { "type": "string" } }, + "type": "object", + "order": [ "world" ] + } + })) + .unwrap(), + path: None, + restarted_from, + }) + .arg("world", arg) + .run_until_complete(db, port) + .await; + + assert_eq!(job.json_result().unwrap(), result); + job.id + }; + let flow_job_id = test( + None, + json!("foo"), + json!([ + "a: Hello foo! foo! foo!", + "b: Hello foo! foo! foo!", + "c: Hello foo! foo! foo!" + ]), + ) + .await; + let flow_job_id = test( + Some(RestartedFrom { flow_job_id, step_id: "a".into(), branch_or_iteration_n: None }), + json!("foo"), + json!([ + "a: Hello foo! foo! foo!", + "b: Hello foo! foo! foo!", + "c: Hello foo! foo! foo!" + ]), + ) + .await; + let flow_job_id = test( + Some(RestartedFrom { flow_job_id, step_id: "b".into(), branch_or_iteration_n: None }), + json!("bar"), + json!([ + "a: Hello foo! bar! bar!", + "b: Hello foo! bar! bar!", + "c: Hello foo! bar! bar!" + ]), + ) + .await; + let _ = test( + Some(RestartedFrom { + flow_job_id, + step_id: "c".into(), + branch_or_iteration_n: Some(1), + }), + json!("yolo"), + json!([ + "a: Hello foo! bar! bar!", + "b: Hello foo! bar! yolo!", + "c: Hello foo! bar! yolo!" + ]), + ) + .await; + } } diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 1b6184e662..9f2ece7d4b 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.456.0 + version: 1.458.4 title: Windmill API contact: @@ -1745,9 +1745,13 @@ paths: deploy_to: type: string ai_resource: - $ref: "#/components/schemas/AiResource" - code_completion_enabled: - type: boolean + $ref: "#/components/schemas/AIResource" + code_completion_model: + type: string + ai_models: + type: array + items: + type: string error_handler: type: string error_handler_extra_args: @@ -1771,7 +1775,7 @@ paths: operator_settings: $ref: "#/components/schemas/OperatorSettings" required: - - code_completion_enabled + - ai_models - automatic_billing - error_handler_muted_on_cancel @@ -2084,12 +2088,16 @@ paths: schema: type: object required: - - code_completion_enabled + - ai_models properties: ai_resource: - $ref: "#/components/schemas/AiResource" - code_completion_enabled: - type: boolean + $ref: "#/components/schemas/AIResource" + code_completion_model: + type: string + ai_models: + type: array + items: + type: string responses: "200": description: status @@ -2116,15 +2124,18 @@ paths: type: object properties: ai_provider: - type: string + $ref: "#/components/schemas/AIProvider" exists_ai_resource: type: boolean - code_completion_enabled: - type: boolean + code_completion_model: + type: string + ai_models: + type: array + items: + type: string required: - - ai_provider - exists_ai_resource - - code_completion_enabled + - ai_models /w/{workspace}/workspaces/edit_error_handler: post: @@ -7770,8 +7781,9 @@ paths: http_method: type: string enum: ["get", "post", "put", "delete", "patch"] + trigger_path: + type: string required: - - kind - route_path - http_method responses: @@ -11546,16 +11558,21 @@ components: schemas: $ref: "../../openflow.openapi.yaml#/components/schemas" - AiResource: + AIProvider: + type: string + enum: [openai, anthropic, mistral, deepseek, groq, openrouter, customai] + + AIResource: type: object properties: path: type: string provider: - type: string + $ref: "#/components/schemas/AIProvider" required: - path - provider + Script: type: object properties: @@ -11908,6 +11925,8 @@ components: type: number suspend: type: number + preprocessed: + type: boolean required: - id - running @@ -12015,6 +12034,8 @@ components: type: number aggregate_wait_time_ms: type: number + preprocessed: + type: boolean required: - id - created_by @@ -12978,6 +12999,8 @@ components: type: boolean requires_auth: type: boolean + is_static_website: + type: boolean required: - path @@ -12992,6 +13015,7 @@ components: - is_async - requires_auth - http_method + - is_static_website NewHttpTrigger: type: object @@ -13027,6 +13051,8 @@ components: type: boolean requires_auth: type: boolean + is_static_website: + type: boolean required: - path @@ -13036,6 +13062,7 @@ components: - is_async - requires_auth - http_method + - is_static_website EditHttpTrigger: type: object @@ -13071,6 +13098,8 @@ components: type: boolean requires_auth: type: boolean + is_static_website: + type: boolean required: - path - script_path @@ -13079,6 +13108,7 @@ components: - is_async - requires_auth - http_method + - is_static_website TriggersCount: type: object diff --git a/backend/windmill-api/src/ai.rs b/backend/windmill-api/src/ai.rs index 3b97ba18ce..d77943df9d 100644 --- a/backend/windmill-api/src/ai.rs +++ b/backend/windmill-api/src/ai.rs @@ -4,27 +4,21 @@ use crate::{ }; use anthropic::AnthropicCache; -use axum::{ - body::Bytes, - extract::{Path, Query}, - response::IntoResponse, - routing::post, - Extension, Router, -}; +use anyhow::Context; +use axum::{body::Bytes, extract::Path, response::IntoResponse, routing::post, Extension, Router}; +use http::HeaderMap; use lazy_static::lazy_static; -use mistral::MistralCache; -use openai::OpenaiCache; use quick_cache::sync::Cache; use reqwest::{Client, RequestBuilder}; -use serde::{Deserialize, Deserializer}; -use windmill_audit::audit_ee::audit_log; -use windmill_audit::ActionKind; -use windmill_common::error::{to_anyhow, Result}; - -use windmill_common::error::Error; - +use serde::{Deserialize, Serialize}; use serde_json::value::{RawValue, Value}; use std::collections::HashMap; +use windmill_audit::{audit_ee::audit_log, ActionKind}; +use windmill_common::error::{to_anyhow, Error, Result}; + +use mistral::MistralCache; +use openai::OpenaiCache; +use openai_api_compatible::OpenaiApiCompatibleCache; lazy_static::lazy_static! { static ref HTTP_CLIENT: Client = reqwest::ClientBuilder::new() @@ -33,16 +27,63 @@ lazy_static::lazy_static! { .build().unwrap(); } -trait AiRequest { - fn prepare_request(self, path: &str, body: Bytes) -> Result; +mod openai_api_compatible { + use super::*; + + #[derive(Deserialize, Clone, Debug)] + pub struct OpenaiApiCompatibleCache { + pub base_url: String, + pub api_key: Option, + } + + impl OpenaiApiCompatibleCache { + pub fn prepare_request(self, path: &str, body: Bytes) -> Result { + let url = format!("{}/{}", self.base_url, path); + + let mut request = HTTP_CLIENT + .post(url) + .header("content-type", "application/json") + .body(body); + + if let Some(api_key) = self.api_key { + request = request.header("Authorization", format!("Bearer {}", api_key)); + } + + Ok(request) + } + } + + pub async fn get_cached_value( + db: &DB, + w_id: &str, + resource: Value, + base_url: Option, + ) -> Result { + let mut resource: OpenaiApiCompatibleCache = if let Some(base_url) = base_url { + let api_key = match resource { + Value::Object(mut obj) => obj + .remove("api_key") + .map(|v| serde_json::from_value::(v.clone()).ok()) + .flatten(), + _ => None, + }; + OpenaiApiCompatibleCache { base_url, api_key } + } else { + serde_json::from_value(resource).with_context(|| "validating custom AI resource")? + }; + + if let Some(api_key) = resource.api_key { + resource.api_key = Some(get_variable_or_self(api_key, db, w_id).await?); + } + + Ok(KeyCache::OpenaiApiCompatible(resource)) + } } mod openai { use super::*; - use super::{get_variable_or_self, KeyCache}; - - const API_VERSION: &str = "2023-05-15"; + const API_VERSION: &str = "2024-10-21"; #[derive(Deserialize, Debug)] struct OpenaiResource { @@ -94,14 +135,14 @@ mod openai { } const BASE_URL: &str = "https://api.openai.com/v1"; - impl AiRequest for OpenaiCache { - fn prepare_request(self, openai_path: &str, mut body: Bytes) -> Result { + impl OpenaiCache { + pub fn prepare_request(self, openai_path: &str, mut body: Bytes) -> Result { let OpenaiCache { api_key, azure_base_path, organization_id, user } = self; if user.is_some() { tracing::debug!("Adding user to request body"); let mut json_body: HashMap> = serde_json::from_slice(&body) .map_err(|e| { - Error::InternalErr(format!("Failed to parse request body: {}", e)) + Error::internal_err(format!("Failed to parse request body: {}", e)) })?; let user_json_string = serde_json::Value::String(user.unwrap()).to_string(); // makes sure to escape characters @@ -109,12 +150,12 @@ mod openai { json_body.insert( "user".to_string(), RawValue::from_string(user_json_string) - .map_err(|e| Error::InternalErr(format!("Failed to parse user: {}", e)))?, + .map_err(|e| Error::internal_err(format!("Failed to parse user: {}", e)))?, ); body = serde_json::to_vec(&json_body) .map_err(|e| { - Error::InternalErr(format!("Failed to reserialize request body: {}", e)) + Error::internal_err(format!("Failed to reserialize request body: {}", e)) })? .into(); } @@ -163,13 +204,13 @@ mod openai { .send() .await .map_err(|err| { - Error::InternalErr(format!( + Error::internal_err(format!( "Failed to get OpenAI credentials using credentials flow: {}", err )) })?; let response = response.json::().await.map_err(|err| { - Error::InternalErr(format!( + Error::internal_err(format!( "Failed to parse OpenAI credentials from credentials flow: {}", err )) @@ -179,7 +220,7 @@ mod openai { pub async fn get_cached_value(db: &DB, w_id: &str, resource: Value) -> Result { let config = serde_json::from_value(resource) - .map_err(|e| Error::InternalErr(format!("validating openai resource {e:#}")))?; + .map_err(|e| Error::internal_err(format!("validating openai resource {e:#}")))?; let mut user = None::; let mut resource = match config { @@ -216,7 +257,7 @@ mod openai { let azure_base_path = if let Some(azure_base_path) = azure_base_path { Some( serde_json::from_value::(azure_base_path).map_err(|e| { - Error::InternalErr(format!("validating openai azure base path {e:#}")) + Error::internal_err(format!("validating openai azure base path {e:#}")) })?, ) } else { @@ -245,15 +286,9 @@ mod anthropic { const API_VERSION: &str = "2023-06-01"; - impl AnthropicCache { - pub fn new(api_key: String) -> Self { - Self { api_key } - } - } - const BASE_URL: &str = "https://api.anthropic.com"; - impl AiRequest for AnthropicCache { - fn prepare_request(self, anthropic_path: &str, body: Bytes) -> Result { + impl AnthropicCache { + pub fn prepare_request(self, anthropic_path: &str, body: Bytes) -> Result { let AnthropicCache { api_key } = self; let url = format!("{}/{}", BASE_URL, anthropic_path); let request = HTTP_CLIENT @@ -268,10 +303,9 @@ mod anthropic { pub async fn get_cached_value(db: &DB, w_id: &str, resource: Value) -> Result { let mut resource: AnthropicCache = serde_json::from_value(resource) - .map_err(|e| Error::InternalErr(format!("validating anthropic resource {e:#}")))?; + .map_err(|e| Error::internal_err(format!("validating anthropic resource {e:#}")))?; resource.api_key = get_variable_or_self(resource.api_key, db, w_id).await?; - let workspace_cache = AnthropicCache::new(resource.api_key); - Ok(KeyCache::Anthropic(workspace_cache)) + Ok(KeyCache::Anthropic(resource)) } } @@ -283,15 +317,9 @@ mod mistral { pub api_key: String, } - impl MistralCache { - pub fn new(api_key: String) -> Self { - Self { api_key } - } - } - const BASE_URL: &str = "https://api.mistral.ai"; - impl AiRequest for MistralCache { - fn prepare_request(self, mistral_path: &str, body: Bytes) -> Result { + impl MistralCache { + pub fn prepare_request(self, mistral_path: &str, body: Bytes) -> Result { let MistralCache { api_key } = self; let url = format!("{}/{}", BASE_URL, mistral_path); @@ -307,11 +335,9 @@ mod mistral { pub async fn get_cached_value(db: &DB, w_id: &str, resource: Value) -> Result { let mut resource: MistralCache = serde_json::from_value(resource) - .map_err(|e| Error::InternalErr(format!("validating mistral resource {e:#}")))?; + .map_err(|e| Error::internal_err(format!("validating mistral resource {e:#}")))?; resource.api_key = get_variable_or_self(resource.api_key, db, w_id).await?; - - let workspace_cache = MistralCache::new(resource.api_key); - Ok(KeyCache::Mistral(workspace_cache)) + Ok(KeyCache::Mistral(resource)) } } @@ -320,16 +346,17 @@ pub enum KeyCache { Openai(OpenaiCache), Anthropic(AnthropicCache), Mistral(MistralCache), + OpenaiApiCompatible(OpenaiApiCompatibleCache), } #[derive(Clone, Debug)] -pub struct AiCache { +pub struct AICache { pub path: String, pub cached_key: KeyCache, pub expires_at: std::time::Instant, } -impl AiCache { +impl AICache { pub fn new(path: String, cached_key: KeyCache) -> Self { Self { path, @@ -343,35 +370,57 @@ impl AiCache { } lazy_static! { - pub static ref AI_KEY_CACHE: Cache = Cache::new(500); + pub static ref AI_KEY_CACHE: Cache = Cache::new(500); } -#[derive(Deserialize, Debug)] -struct ProxyQueryParams { - no_cache: Option, +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "lowercase")] +pub enum AIProvider { + OpenAI, + Anthropic, + Mistral, + DeepSeek, + Groq, + OpenRouter, + CustomAI, } -#[derive(Deserialize, Debug)] -pub struct AiResource { - pub path: String, - #[serde(deserialize_with = "check_if_valid_ai_provider")] - pub provider: String, -} - -fn check_if_valid_ai_provider<'de, D>(provider: D) -> std::result::Result -where - D: Deserializer<'de>, -{ - let provider = String::deserialize(provider)?; - match provider.as_str() { - "anthropic" | "openai" | "mistral" => Ok(provider), - _ => Err(serde::de::Error::custom( - "Only the following Ai providers are supported: openai, anthropic and mistral" - .to_string(), - )), +impl AIProvider { + pub fn get_openai_compatible_base_url(&self) -> Result> { + match self { + AIProvider::DeepSeek => Ok(Some("https://api.deepseek.com/v1".to_string())), + AIProvider::Groq => Ok(Some("https://api.groq.com/openai/v1".to_string())), + AIProvider::OpenRouter => Ok(Some("https://openrouter.ai/api/v1".to_string())), + AIProvider::CustomAI => Ok(None), + _ => Err(Error::BadRequest( + "Please use the specific provider instead of the OpenAI compatible one".to_string(), + )), + } } } +impl TryFrom<&str> for AIProvider { + type Error = Error; + fn try_from(s: &str) -> Result { + match s { + "openai" => Ok(AIProvider::OpenAI), + "anthropic" => Ok(AIProvider::Anthropic), + "mistral" => Ok(AIProvider::Mistral), + "groq" => Ok(AIProvider::Groq), + "openrouter" => Ok(AIProvider::OpenRouter), + "deepseek" => Ok(AIProvider::DeepSeek), + "customai" => Ok(AIProvider::CustomAI), + _ => Err(Error::BadRequest(format!("Invalid AI provider: {}", s))), + } + } +} + +#[derive(Deserialize, Debug)] +pub struct AIResource { + pub path: String, + pub provider: AIProvider, +} + pub fn workspaced_service() -> Router { let router = Router::new().route("/proxy/*ai", post(proxy)); @@ -382,74 +431,105 @@ async fn proxy( authed: ApiAuthed, Extension(db): Extension, Path((w_id, ai_path)): Path<(String, String)>, - Query(query_params): Query, + headers: HeaderMap, body: Bytes, ) -> impl IntoResponse { let workspace_cache = AI_KEY_CACHE.get(&w_id); + let forced_resource_path = headers + .get("X-Resource-Path") + .map(|v| v.to_str().unwrap_or("").to_string()); let ai_cache = match workspace_cache { - Some(cache) if !cache.is_expired() && !query_params.no_cache.unwrap_or(false) => { - cache.cached_key - } + Some(cache) if !cache.is_expired() && forced_resource_path.is_none() => cache.cached_key, _ => { - let ai_resource = sqlx::query_scalar!( - "SELECT ai_resource FROM workspace_settings WHERE workspace_id = $1", - &w_id - ) - .fetch_one(&db) - .await?; + let (resource, resource_path, ai_provider) = if let Some(resource_path) = + forced_resource_path + { + // guess the provider from the resource type + let record = sqlx::query!( + "SELECT value, resource_type FROM resource WHERE path = $1 AND workspace_id = $2", + &resource_path, + &w_id + ) + .fetch_optional(&db) + .await? + .ok_or_else(|| { + Error::NotFound(format!( + "Could not find the resource {}, update the resource path in the workspace settings", resource_path + )) + })?; - if ai_resource.is_none() { - return Err(Error::InternalErr("AI resource not configured".to_string())); - } + ( + record.value, + resource_path, + AIProvider::try_from(record.resource_type.as_str())?, + ) + } else { + let ai_resource = sqlx::query_scalar!( + "SELECT ai_resource FROM workspace_settings WHERE workspace_id = $1", + &w_id + ) + .fetch_one(&db) + .await?; - let ai_resource = serde_json::from_value::(ai_resource.unwrap()) - .map_err(|e| Error::BadRequest(e.to_string()))?; - let ai_resource_path = ai_resource.path; + if ai_resource.is_none() { + return Err(Error::internal_err("AI resource not configured".to_string())); + } - let resource = sqlx::query_scalar!( - "SELECT value - FROM resource - WHERE path = $1 AND workspace_id = $2", - &ai_resource_path, - &w_id - ) - .fetch_optional(&db) - .await? - .ok_or_else(|| { - Error::InternalErr(format!( - "Could not find the {} resource at path {ai_resource_path}, update the resource path in the workspace settings", ai_resource.provider - )) - })?; + let ai_resource = serde_json::from_value::(ai_resource.unwrap()) + .map_err(|e| Error::BadRequest(e.to_string()))?; + + let resource = sqlx::query_scalar!( + "SELECT value + FROM resource + WHERE path = $1 AND workspace_id = $2", + &ai_resource.path, + &w_id + ) + .fetch_optional(&db) + .await? + .ok_or_else(|| { + Error::NotFound(format!( + "Could not find the {:?} resource at path {}, update the resource path in the workspace settings", ai_resource.provider, ai_resource.path + )) + })?; + + (resource, ai_resource.path, ai_resource.provider) + }; if resource.is_none() { - return Err(Error::InternalErr(format!( - "{} resource missing value", - ai_resource.provider + return Err(Error::internal_err(format!( + "{:?} resource missing value", + ai_provider ))); } let resource = resource.unwrap(); - let ai_cache = match ai_resource.provider.as_str() { - "openai" => openai::get_cached_value(&db, &w_id, resource).await, - "anthropic" => anthropic::get_cached_value(&db, &w_id, resource).await, - "mistral" => mistral::get_cached_value(&db, &w_id, resource).await, - provider => { - return Err(Error::BadRequest(format!("{} is not supported", provider))) + let ai_cache = match ai_provider { + AIProvider::OpenAI => openai::get_cached_value(&db, &w_id, resource).await, + AIProvider::Anthropic => anthropic::get_cached_value(&db, &w_id, resource).await, + AIProvider::Mistral => mistral::get_cached_value(&db, &w_id, resource).await, + _ => { + openai_api_compatible::get_cached_value( + &db, + &w_id, + resource, + ai_provider.get_openai_compatible_base_url()?, + ) + .await } }; let ai_cache = ai_cache?; - AI_KEY_CACHE.insert( - w_id.clone(), - AiCache::new(ai_resource_path, ai_cache.clone()), - ); + AI_KEY_CACHE.insert(w_id.clone(), AICache::new(resource_path, ai_cache.clone())); ai_cache } }; - let (path, request) = match ai_cache { - KeyCache::Openai(cached) => ("openai_path", cached.prepare_request(&ai_path, body)), - KeyCache::Anthropic(cached) => ("anthropic_path", cached.prepare_request(&ai_path, body)), - KeyCache::Mistral(cached) => ("mistral_path", cached.prepare_request(&ai_path, body)), + + let request = match ai_cache { + KeyCache::Openai(cached) => cached.prepare_request(&ai_path, body), + KeyCache::Anthropic(cached) => cached.prepare_request(&ai_path, body), + KeyCache::Mistral(cached) => cached.prepare_request(&ai_path, body), + KeyCache::OpenaiApiCompatible(cached) => cached.prepare_request(&ai_path, body), }; let response = request?.send().await.map_err(to_anyhow)?; @@ -463,7 +543,7 @@ async fn proxy( ActionKind::Execute, &w_id, Some(&authed.email), - Some([(path, &format!("{:?}", ai_path)[..])].into()), + Some([("ai_resource_path", &format!("{:?}", ai_path)[..])].into()), ) .await?; tx.commit().await?; diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index bf134adc7f..952ff6d77e 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -360,7 +360,7 @@ async fn list_apps( .fields(&["dm.deployment_msg"]); } - let sql = sqlb.sql().map_err(|e| Error::InternalErr(e.to_string()))?; + let sql = sqlb.sql().map_err(|e| Error::internal_err(e.to_string()))?; let mut tx = user_db.begin(&authed).await?; let rows = sqlx::query_as::<_, ListableApp>(&sql) .fetch_all(&mut *tx) @@ -611,7 +611,7 @@ async fn get_public_app_by_secret( let decrypted = mc .decrypt_bytes_to_bytes(&(hex::decode(secret)?)) - .map_err(|e| Error::InternalErr(e.to_string()))?; + .map_err(|e| Error::internal_err(e.to_string()))?; let bytes = str::from_utf8(&decrypted).map_err(to_anyhow)?; let id: i64 = bytes.parse().map_err(to_anyhow)?; @@ -958,7 +958,7 @@ async fn delete_app( .execute(&db) .await .map_err(|e| { - Error::InternalErr(format!( + Error::internal_err(format!( "error deleting deployment metadata for script with path {path} in workspace {w_id}: {e:#}" )) })?; @@ -1061,7 +1061,7 @@ async fn update_app( sqlb.returning("path"); - let sql = sqlb.sql().map_err(|e| Error::InternalErr(e.to_string()))?; + let sql = sqlb.sql().map_err(|e| Error::internal_err(e.to_string()))?; let npath_o: Option = sqlx::query_scalar(&sql).fetch_optional(&mut *tx).await?; not_found_if_none(npath_o, "App", path)? } else { @@ -1710,7 +1710,7 @@ async fn upload_s3_file_from_app( } }; - let s3_resource = s3_resource_opt.ok_or(Error::InternalErr( + let s3_resource = s3_resource_opt.ok_or(Error::internal_err( "No files storage resource defined at the workspace level".to_string(), ))?; let s3_client = build_object_store_client(&s3_resource).await?; @@ -1794,7 +1794,7 @@ async fn check_if_allowed_to_access_s3_file_from_app( let allowed = opt_authed.is_some() || sqlx::query_scalar!( r#"SELECT EXISTS ( - SELECT 1 FROM completed_job + SELECT 1 FROM v2_as_completed_job WHERE workspace_id = $2 AND (job_kind = 'appscript' OR job_kind = 'preview') AND created_by = 'anonymous' @@ -2034,7 +2034,10 @@ async fn build_args( safe_args.insert( k.to_string(), to_raw_value(&value.unwrap_or(Ok(serde_json::Value::Null)).map_err(|e| { - Error::InternalErr(format!("failed to serialize ctx variable for {}: {}", k, e)) + Error::internal_err(format!( + "failed to serialize ctx variable for {}: {}", + k, e + )) })?), ); } else if !arg_str.contains("\"$var:") && !arg_str.contains("\"$res:") { @@ -2054,7 +2057,7 @@ async fn build_args( ), ) .map_err(|e| { - Error::InternalErr(format!( + Error::internal_err(format!( "failed to remove sensitive variable(s)/resource(s) with error: {}", e )) diff --git a/backend/windmill-api/src/auth.rs b/backend/windmill-api/src/auth.rs index 64024f1ab1..71c6270477 100644 --- a/backend/windmill-api/src/auth.rs +++ b/backend/windmill-api/src/auth.rs @@ -151,12 +151,16 @@ impl AuthCache { } } _ => { - let user_o = sqlx::query_as::<_, (Option, Option, bool, Option>, Option)>( - "UPDATE token SET last_used_at = now() WHERE token = $1 AND (expiration > NOW() \ - OR expiration IS NULL) AND (workspace_id IS NULL OR workspace_id = $2) RETURNING owner, email, super_admin, scopes, label", + let user_o = sqlx::query!( + "UPDATE token SET last_used_at = now() WHERE + token = $1 + AND (expiration > NOW() OR expiration IS NULL) + AND (workspace_id IS NULL OR workspace_id = $2) + RETURNING owner, email, super_admin, scopes, label", + token, + w_id.as_ref(), ) - .bind(token) - .bind(w_id.as_ref()) + .map(|x| (x.owner, x.email, x.super_admin, x.scopes, x.label)) .fetch_optional(&self.db) .await .ok() @@ -251,12 +255,13 @@ impl AuthCache { (_, Some(email), super_admin, scopes, label) => { let username_override = username_override_from_label(label); if w_id.is_some() { - let row_o = sqlx::query_as::<_, (String, bool, bool)>( - "SELECT username, is_admin, operator FROM usr where email = $1 AND \ - workspace_id = $2 AND disabled = false", + let row_o = sqlx::query!( + "SELECT username, is_admin, operator FROM usr WHERE + email = $1 AND workspace_id = $2 AND disabled = false", + &email, + w_id.as_ref().unwrap() ) - .bind(&email) - .bind(&w_id.as_ref().unwrap()) + .map(|x| (x.username, x.is_admin, x.operator)) .fetch_optional(&self.db) .await .unwrap_or(Some(("error".to_string(), false, false))); diff --git a/backend/windmill-api/src/capture.rs b/backend/windmill-api/src/capture.rs index fe752336f3..4b933b01b4 100644 --- a/backend/windmill-api/src/capture.rs +++ b/backend/windmill-api/src/capture.rs @@ -412,7 +412,7 @@ async fn get_capture_trigger_config_and_owner( Ok(( serde_json::from_str(trigger_config.get()).map_err(|e| { - Error::InternalErr(format!( + Error::internal_err(format!( "error parsing capture config for {} trigger: {}", kind, e )) diff --git a/backend/windmill-api/src/concurrency_groups.rs b/backend/windmill-api/src/concurrency_groups.rs index 945df75573..3f7e41617f 100644 --- a/backend/windmill-api/src/concurrency_groups.rs +++ b/backend/windmill-api/src/concurrency_groups.rs @@ -16,7 +16,7 @@ use sql_builder::bind::Bind; use sql_builder::SqlBuilder; use uuid::Uuid; use windmill_common::db::UserDB; -use windmill_common::error::Error::{InternalErr, PermissionDenied}; +use windmill_common::error::Error::PermissionDenied; use windmill_common::error::{self, JsonResult}; use windmill_common::utils::require_admin; @@ -82,7 +82,7 @@ async fn prune_concurrency_group( if n_job_uuids > 0 { tx.commit().await?; - return Err(InternalErr( + return Err(error::Error::internal_err( "Concurrency group is currently in use, unable to remove it. Retry later.".to_string(), )); } @@ -157,22 +157,22 @@ async fn get_concurrent_intervals( let lq = ListCompletedQuery { order_desc: Some(true), ..lq }; let lqc = lq.clone(); let lqq: ListQueueQuery = lqc.into(); - let mut sqlb_q = SqlBuilder::select_from("queue") + let mut sqlb_q = SqlBuilder::select_from("v2_as_queue") .fields(UnifiedJob::queued_job_fields()) .order_by("created_at", lq.order_desc.unwrap_or(true)) .limit(row_limit) .clone(); - let mut sqlb_c = SqlBuilder::select_from("completed_job") + let mut sqlb_c = SqlBuilder::select_from("v2_as_completed_job") .fields(UnifiedJob::completed_job_fields()) .order_by("started_at", lq.order_desc.unwrap_or(true)) .limit(row_limit) .clone(); - let mut sqlb_q_user = SqlBuilder::select_from("queue") + let mut sqlb_q_user = SqlBuilder::select_from("v2_as_queue") .fields(&["id"]) .order_by("created_at", lq.order_desc.unwrap_or(true)) .limit(row_limit) .clone(); - let mut sqlb_c_user = SqlBuilder::select_from("completed_job") + let mut sqlb_c_user = SqlBuilder::select_from("v2_as_completed_job") .fields(&["id"]) .order_by("started_at", lq.order_desc.unwrap_or(true)) .limit(row_limit) diff --git a/backend/windmill-api/src/db.rs b/backend/windmill-api/src/db.rs index 6cff3872e7..a2a641b802 100644 --- a/backend/windmill-api/src/db.rs +++ b/backend/windmill-api/src/db.rs @@ -84,7 +84,7 @@ impl Migrate for CustomMigrator { })? .unwrap_or(false); if !r { - tracing::info!("PG lock already acquired by another server or worker, retrying in 5s. (look for the advisory lock in pg_lock with granted = true)"); + tracing::info!("PG migration lock already acquired by another server or worker, a migration is in progress, this may take a long time if you have many jobs and be normal, rechecking in 5s."); tokio::time::sleep(std::time::Duration::from_secs(5)).await; } } @@ -180,6 +180,24 @@ pub async fn migrate(db: &DB) -> Result<(), Error> { tracing::info!("Could not remove sqlx migration with version=20250131115248: {err:#}"); } + // Remove the migration `v2_fix_no_runtime` in favor of `v2_fix_no_runtime_2`. + if let Err(err) = sqlx::query!("DELETE FROM _sqlx_migrations WHERE version=20250201145632") + .execute(db) + .await + { + tracing::info!("Could not remove sqlx migration with version=20250201145632: {err:#}"); + } + + // New version of `v2_as_queue` and `v2_as_completed_job` VIEWs. + if let Err(err) = sqlx::query!( + "DELETE FROM _sqlx_migrations WHERE version=20250201145630 OR version=20250201145631" + ) + .execute(db) + .await + { + tracing::info!("Could not remove sqlx migration with version=[20250201145630, 20250201145631] : {err:#}"); + } + match sqlx::migrate!("../migrations") .run_direct(&mut custom_migrator) .await @@ -405,60 +423,60 @@ async fn fix_job_completed_index(db: &DB) -> Result<(), Error> { .await?; }); - run_windmill_migration!("fix_job_completed_index_4", &db, { + run_windmill_migration!("fix_job_index_1", &db, { let migration_job_name = "fix_job_completed_index_4"; let mut i = 1; tracing::info!("step {i} of {migration_job_name} migration"); - sqlx::query("create index concurrently if not exists ix_completed_job_workspace_id_created_at_new_3 ON completed_job (workspace_id, created_at DESC)") + sqlx::query!("create index concurrently if not exists ix_job_workspace_id_created_at_new_3 ON v2_job (workspace_id, created_at DESC)") .execute(db) .await?; i += 1; tracing::info!("step {i} of {migration_job_name} migration"); - sqlx::query("create index concurrently if not exists ix_completed_job_workspace_id_created_at_new_8 ON completed_job (workspace_id, created_at DESC) where job_kind in ('deploymentcallback') AND parent_job IS NULL") + sqlx::query!("create index concurrently if not exists ix_job_workspace_id_created_at_new_8 ON v2_job (workspace_id, created_at DESC) where kind in ('deploymentcallback') AND parent_job IS NULL") .execute(db) .await?; i += 1; tracing::info!("step {i} of {migration_job_name} migration"); - sqlx::query("create index concurrently if not exists ix_completed_job_workspace_id_created_at_new_9 ON completed_job (workspace_id, created_at DESC) where job_kind in ('dependencies', 'flowdependencies', 'appdependencies') AND parent_job IS NULL") + sqlx::query!("create index concurrently if not exists ix_job_workspace_id_created_at_new_9 ON v2_job (workspace_id, created_at DESC) where kind in ('dependencies', 'flowdependencies', 'appdependencies') AND parent_job IS NULL") .execute(db) .await?; i += 1; tracing::info!("step {i} of {migration_job_name} migration"); - sqlx::query("create index concurrently if not exists ix_completed_job_workspace_id_created_at_new_5 ON completed_job (workspace_id, created_at DESC) where job_kind in ('preview', 'flowpreview') AND parent_job IS NULL") + sqlx::query!("create index concurrently if not exists ix_job_workspace_id_created_at_new_5 ON v2_job (workspace_id, created_at DESC) where kind in ('preview', 'flowpreview') AND parent_job IS NULL") .execute(db) .await?; i += 1; tracing::info!("step {i} of {migration_job_name} migration"); - sqlx::query("create index concurrently if not exists ix_completed_job_workspace_id_created_at_new_6 ON completed_job (workspace_id, created_at DESC) where job_kind in ('script', 'flow') AND parent_job IS NULL") + sqlx::query!("create index concurrently if not exists ix_job_workspace_id_created_at_new_6 ON v2_job (workspace_id, created_at DESC) where kind in ('script', 'flow') AND parent_job IS NULL") .execute(db) .await?; i += 1; tracing::info!("step {i} of {migration_job_name} migration"); - sqlx::query("create index concurrently if not exists ix_completed_job_workspace_id_created_at_new_7 ON completed_job (workspace_id, success, created_at DESC) where job_kind in ('script', 'flow') AND parent_job IS NULL") + sqlx::query!("create index concurrently if not exists ix_job_workspace_id_created_at_new_7 ON v2_job (workspace_id, created_at DESC) where kind in ('script', 'flow') AND parent_job IS NULL") .execute(db) .await?; i += 1; tracing::info!("step {i} of {migration_job_name} migration"); - sqlx::query("create index concurrently if not exists ix_completed_job_workspace_id_started_at_new_2 ON completed_job (workspace_id, started_at DESC)") + sqlx::query!("create index concurrently if not exists ix_completed_job_workspace_id_started_at_new_2 ON v2_job_completed (workspace_id, started_at DESC)") .execute(db) .await?; i += 1; tracing::info!("step {i} of {migration_job_name} migration"); - sqlx::query("create index concurrently if not exists root_job_index_by_path_2 ON completed_job (workspace_id, script_path, created_at desc) WHERE parent_job IS NULL") + sqlx::query!("create index concurrently if not exists root_job_index_by_path_2 ON v2_job (workspace_id, runnable_path, created_at desc) WHERE parent_job IS NULL") .execute(db) .await?; i += 1; tracing::info!("step {i} of {migration_job_name} migration"); - sqlx::query("create index concurrently if not exists ix_completed_job_created_at ON completed_job (created_at DESC)") + sqlx::query!("create index concurrently if not exists ix_job_created_at ON v2_job (created_at DESC)") .execute(db) .await?; @@ -492,10 +510,21 @@ async fn fix_job_completed_index(db: &DB) -> Result<(), Error> { .execute(db) .await?; sqlx::query!( - "CREATE INDEX CONCURRENTLY labeled_jobs_on_jobs ON completed_job USING GIN ((result -> 'wm_labels')) WHERE result ? 'wm_labels'" + "CREATE INDEX CONCURRENTLY labeled_jobs_on_jobs ON v2_job_completed USING GIN ((result -> 'wm_labels')) WHERE result ? 'wm_labels'" ).execute(db).await?; }); + run_windmill_migration!("v2_labeled_jobs_index", &db, { + tracing::info!("Special migration to add index concurrently on job labels"); + sqlx::query!( + "CREATE INDEX CONCURRENTLY ix_v2_job_labels ON v2_job + USING GIN (labels) + WHERE labels IS NOT NULL" + ) + .execute(db) + .await?; + }); + Ok(()) } diff --git a/backend/windmill-api/src/embeddings.rs b/backend/windmill-api/src/embeddings.rs index fd222bdbfa..30415156ce 100644 --- a/backend/windmill-api/src/embeddings.rs +++ b/backend/windmill-api/src/embeddings.rs @@ -90,7 +90,7 @@ async fn query_hub_scripts( Ok(Json(results)) } else { - Err(windmill_common::error::Error::InternalErr( + Err(windmill_common::error::Error::internal_err( "Embeddings db not initialized".to_string(), )) } @@ -124,7 +124,7 @@ async fn query_resource_types( Ok(Json(results)) } else { - Err(windmill_common::error::Error::InternalErr( + Err(windmill_common::error::Error::internal_err( "Embeddings db not initialized".to_string(), )) } diff --git a/backend/windmill-api/src/flows.rs b/backend/windmill-api/src/flows.rs index 1b283d43f3..5e779119a8 100644 --- a/backend/windmill-api/src/flows.rs +++ b/backend/windmill-api/src/flows.rs @@ -187,7 +187,7 @@ async fn list_flows( .fields(&["dm.deployment_msg"]); } - let sql = sqlb.sql().map_err(|e| Error::InternalErr(e.to_string()))?; + let sql = sqlb.sql().map_err(|e| Error::internal_err(e.to_string()))?; let mut tx = user_db.begin(&authed).await?; let rows = sqlx::query_as::<_, ListableFlow>(&sql) .fetch_all(&mut *tx) @@ -704,7 +704,7 @@ async fn update_flow( w_id, ) .execute(&mut *tx) - .await.map_err(|e| error::Error::InternalErr(format!("Error updating flow due to flow update: {e:#}")))?; + .await.map_err(|e| error::Error::internal_err(format!("Error updating flow due to flow update: {e:#}")))?; if is_new_path { // if new path, must clone flow to new path and delete old flow for flow_version foreign key constraint @@ -721,7 +721,7 @@ async fn update_flow( .execute(&mut *tx) .await .map_err(|e| { - error::Error::InternalErr(format!("Error updating flow due to create new flow: {e:#}")) + error::Error::internal_err(format!("Error updating flow due to create new flow: {e:#}")) })?; sqlx::query!( @@ -733,7 +733,7 @@ async fn update_flow( .execute(&mut *tx) .await .map_err(|e| { - error::Error::InternalErr(format!( + error::Error::internal_err(format!( "Error updating flow due to updating flow history path: {e:#}" )) })?; @@ -746,7 +746,7 @@ async fn update_flow( .execute(&mut *tx) .await .map_err(|e| { - error::Error::InternalErr(format!( + error::Error::internal_err(format!( "Error updating flow due to deleting old flow: {e:#}" )) })?; @@ -781,7 +781,7 @@ async fn update_flow( .fetch_one(&mut *tx) .await .map_err(|e| { - error::Error::InternalErr(format!( + error::Error::internal_err(format!( "Error updating flow due to flow history insert: {e:#}" )) })?; @@ -805,7 +805,7 @@ async fn update_flow( .bind(&flow_path) .bind(&w_id) .fetch_all(&mut *tx) - .await.map_err(|e| error::Error::InternalErr(format!("Error updating flow due to related schedules update: {e:#}")))?; + .await.map_err(|e| error::Error::internal_err(format!("Error updating flow due to related schedules update: {e:#}")))?; let schedule = sqlx::query_as::<_, Schedule>( "UPDATE schedule SET path = $1, script_path = $1 WHERE path = $2 AND workspace_id = $3 AND is_flow IS true RETURNING *") @@ -813,7 +813,7 @@ async fn update_flow( .bind(&flow_path) .bind(&w_id) .fetch_optional(&mut *tx) - .await.map_err(|e| error::Error::InternalErr(format!("Error updating flow due to related schedule update: {e:#}")))?; + .await.map_err(|e| error::Error::internal_err(format!("Error updating flow due to related schedule update: {e:#}")))?; if let Some(schedule) = schedule { clear_schedule(&mut tx, &flow_path, &w_id).await?; @@ -907,19 +907,23 @@ async fn update_flow( .execute(&mut *new_tx) .await .map_err(|e| { - error::Error::InternalErr(format!( + error::Error::internal_err(format!( "Error updating flow due to updating dependency job field: {e:#}" )) })?; if let Some(old_dep_job) = old_dep_job { sqlx::query!( - "UPDATE queue SET canceled = true WHERE id = $1", - old_dep_job + "UPDATE v2_job_queue SET + canceled_by = $2, + canceled_reason = 're-deployment' + WHERE id = $1", + old_dep_job, + &authed.username ) .execute(&mut *new_tx) .await .map_err(|e| { - error::Error::InternalErr(format!( + error::Error::internal_err(format!( "Error updating flow due to cancelling dependency job: {e:#}" )) })?; @@ -1204,7 +1208,7 @@ async fn delete_flow_by_path( .execute(&db) .await .map_err(|e| { - Error::InternalErr(format!( + Error::internal_err(format!( "error deleting deployment metadata for script with path {path} in workspace {w_id}: {e:#}" )) })?; diff --git a/backend/windmill-api/src/folders.rs b/backend/windmill-api/src/folders.rs index 208ec63985..cab039c64f 100644 --- a/backend/windmill-api/src/folders.rs +++ b/backend/windmill-api/src/folders.rs @@ -340,7 +340,7 @@ async fn update_folder( let sql = sqlb .sql() - .map_err(|e| error::Error::InternalErr(e.to_string()))?; + .map_err(|e| error::Error::internal_err(e.to_string()))?; let nfolder = sqlx::query_as::<_, Folder>(&sql) .fetch_optional(&mut *tx) .await?; diff --git a/backend/windmill-api/src/groups.rs b/backend/windmill-api/src/groups.rs index b63d0f9de8..e25d3ee7da 100644 --- a/backend/windmill-api/src/groups.rs +++ b/backend/windmill-api/src/groups.rs @@ -72,6 +72,7 @@ pub struct NewGroup { pub struct GroupInfo { pub workspace_id: String, pub name: String, + #[serde(skip_serializing_if = "Option::is_none")] pub summary: Option, pub members: Vec, pub extra_perms: serde_json::Value, diff --git a/backend/windmill-api/src/http_triggers.rs b/backend/windmill-api/src/http_triggers.rs index ef62928f2c..f09451c3af 100644 --- a/backend/windmill-api/src/http_triggers.rs +++ b/backend/windmill-api/src/http_triggers.rs @@ -36,7 +36,8 @@ use windmill_common::{ }; lazy_static::lazy_static! { - static ref ROUTE_PATH_KEY_RE: regex::Regex = regex::Regex::new(r"/:\w+").unwrap(); + static ref ROUTE_PATH_KEY_RE: regex::Regex = regex::Regex::new(r"/?:[-\w]+").unwrap(); + static ref VALID_ROUTE_PATH_RE: regex::Regex = regex::Regex::new(r"^:?[-\w]+(/:?[-\w]+)*$").unwrap(); } pub fn routes_global_service() -> Router { @@ -109,24 +110,27 @@ struct NewTrigger { requires_auth: bool, http_method: HttpMethod, static_asset_config: Option>, + is_static_website: bool, } #[derive(FromRow, Serialize)] -struct Trigger { - workspace_id: String, - path: String, - route_path: String, - route_path_key: String, - script_path: String, - is_flow: bool, - edited_by: String, - email: String, - edited_at: chrono::DateTime, - extra_perms: serde_json::Value, - is_async: bool, - requires_auth: bool, - http_method: HttpMethod, - static_asset_config: Option>, +pub struct HttpTrigger { + pub workspace_id: String, + pub path: String, + pub route_path: String, + pub route_path_key: String, + pub script_path: String, + pub is_flow: bool, + pub edited_by: String, + pub email: String, + pub edited_at: chrono::DateTime, + pub extra_perms: serde_json::Value, + pub is_async: bool, + pub requires_auth: bool, + pub http_method: HttpMethod, + #[serde(skip_serializing_if = "Option::is_none")] + pub static_asset_config: Option>, + pub is_static_website: bool, } #[derive(Deserialize)] @@ -139,6 +143,7 @@ struct EditTrigger { requires_auth: bool, http_method: HttpMethod, static_asset_config: Option>, + is_static_website: bool, } #[derive(Deserialize)] @@ -155,7 +160,7 @@ async fn list_triggers( Extension(user_db): Extension, Path(w_id): Path, Query(lst): Query, -) -> error::JsonResult> { +) -> error::JsonResult> { let mut tx = user_db.begin(&authed).await?; let (per_page, offset) = paginate(Pagination { per_page: lst.per_page, page: lst.page }); let mut sqlb = SqlBuilder::select_from("http_trigger") @@ -176,8 +181,8 @@ async fn list_triggers( } let sql = sqlb .sql() - .map_err(|e| error::Error::InternalErr(e.to_string()))?; - let rows = sqlx::query_as::<_, Trigger>(&sql) + .map_err(|e| error::Error::internal_err(e.to_string()))?; + let rows = sqlx::query_as::<_, HttpTrigger>(&sql) .fetch_all(&mut *tx) .await?; tx.commit().await?; @@ -189,12 +194,12 @@ async fn get_trigger( authed: ApiAuthed, Extension(user_db): Extension, Path((w_id, path)): Path<(String, StripPath)>, -) -> error::JsonResult { +) -> error::JsonResult { let mut tx = user_db.begin(&authed).await?; let path = path.to_path(); let trigger = sqlx::query_as!( - Trigger, - r#"SELECT workspace_id, path, route_path, route_path_key, script_path, is_flow, http_method as "http_method: _", edited_by, email, edited_at, extra_perms, is_async, requires_auth, static_asset_config as "static_asset_config: _" + HttpTrigger, + r#"SELECT workspace_id, path, route_path, route_path_key, script_path, is_flow, http_method as "http_method: _", edited_by, email, edited_at, extra_perms, is_async, requires_auth, static_asset_config as "static_asset_config: _", is_static_website FROM http_trigger WHERE workspace_id = $1 AND path = $2"#, w_id, @@ -211,17 +216,38 @@ async fn get_trigger( async fn create_trigger( authed: ApiAuthed, + Extension(db): Extension, Extension(user_db): Extension, Path(w_id): Path, Json(ct): Json, ) -> error::Result<(StatusCode, String)> { require_admin(authed.is_admin, &authed.username)?; + if !VALID_ROUTE_PATH_RE.is_match(&ct.route_path) { + return Err(error::Error::BadRequest("Invalid route path".to_string())); + } + + // route path key is extracted from the route path to check for uniqueness + // it replaces /?:{key} with :key + // it will also remove the leading / if present, not an issue as we only allow : after slashes let route_path_key = ROUTE_PATH_KEY_RE.replace_all(ct.route_path.as_str(), ":key"); + let exists = route_path_key_exists(&route_path_key, &ct.http_method, &w_id, None, &db).await?; + if exists { + return Err(error::Error::BadRequest( + "A route already exists with this path".to_string(), + )); + } + + if *CLOUD_HOSTED && (ct.is_static_website || ct.static_asset_config.is_some()) { + return Err(error::Error::BadRequest( + "Static website and static asset are not supported on cloud".to_string(), + )); + } + let mut tx = user_db.begin(&authed).await?; sqlx::query!( - "INSERT INTO http_trigger (workspace_id, path, route_path, route_path_key, script_path, is_flow, is_async, requires_auth, http_method, static_asset_config, edited_by, email, edited_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, now())", + "INSERT INTO http_trigger (workspace_id, path, route_path, route_path_key, script_path, is_flow, is_async, requires_auth, http_method, static_asset_config, edited_by, email, edited_at, is_static_website) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, now(), $13)", w_id, ct.path, ct.route_path, @@ -233,7 +259,8 @@ async fn create_trigger( ct.http_method as _, ct.static_asset_config as _, &authed.username, - &authed.email + &authed.email, + ct.is_static_website, ) .execute(&mut *tx).await?; @@ -256,27 +283,48 @@ async fn create_trigger( async fn update_trigger( authed: ApiAuthed, Extension(user_db): Extension, + Extension(db): Extension, Path((w_id, path)): Path<(String, StripPath)>, Json(ct): Json, ) -> error::Result { let path = path.to_path(); - let mut tx = user_db.begin(&authed).await?; + if *CLOUD_HOSTED && (ct.is_static_website || ct.static_asset_config.is_some()) { + return Err(error::Error::BadRequest( + "Static website and static asset are not supported on cloud".to_string(), + )); + } + + let mut tx; if authed.is_admin { - if ct.route_path.is_none() { + let Some(route_path) = ct.route_path else { return Err(error::Error::BadRequest( "route_path is required".to_string(), )); + }; + + if !VALID_ROUTE_PATH_RE.is_match(&route_path) { + return Err(error::Error::BadRequest("Invalid route path".to_string())); } - let route_path_key = - ROUTE_PATH_KEY_RE.replace_all(ct.route_path.as_ref().unwrap().as_str(), ":key"); + let route_path_key = ROUTE_PATH_KEY_RE.replace_all(&route_path, ":key"); + + let exists = + route_path_key_exists(&route_path_key, &ct.http_method, &w_id, Some(&path), &db) + .await?; + if exists { + return Err(error::Error::BadRequest( + "A route already exists with this path".to_string(), + )); + } + + tx = user_db.begin(&authed).await?; sqlx::query!( "UPDATE http_trigger - SET route_path = $1, route_path_key = $2, script_path = $3, path = $4, is_flow = $5, http_method = $6, static_asset_config = $7, edited_by = $8, email = $9, is_async = $10, requires_auth = $11, edited_at = now() - WHERE workspace_id = $12 AND path = $13", - ct.route_path, + SET route_path = $1, route_path_key = $2, script_path = $3, path = $4, is_flow = $5, http_method = $6, static_asset_config = $7, edited_by = $8, email = $9, is_async = $10, requires_auth = $11, edited_at = now(), is_static_website = $12 + WHERE workspace_id = $13 AND path = $14", + route_path, &route_path_key, ct.script_path, ct.path, @@ -287,14 +335,16 @@ async fn update_trigger( &authed.email, ct.is_async, ct.requires_auth, + ct.is_static_website, w_id, path, ) .execute(&mut *tx).await?; } else { + tx = user_db.begin(&authed).await?; sqlx::query!( - "UPDATE http_trigger SET script_path = $1, path = $2, is_flow = $3, http_method = $4, static_asset_config = $5, edited_by = $6, email = $7, is_async = $8, requires_auth = $9, edited_at = now() - WHERE workspace_id = $10 AND path = $11", + "UPDATE http_trigger SET script_path = $1, path = $2, is_flow = $3, http_method = $4, static_asset_config = $5, edited_by = $6, email = $7, is_async = $8, requires_auth = $9, edited_at = now(), is_static_website = $10 + WHERE workspace_id = $11 AND path = $12", ct.script_path, ct.path, ct.is_flow, @@ -304,6 +354,7 @@ async fn update_trigger( &authed.email, ct.is_async, ct.requires_auth, + ct.is_static_website, w_id, path, ) @@ -378,34 +429,57 @@ async fn exists_trigger( struct RouteExists { route_path: String, http_method: HttpMethod, + trigger_path: Option, +} + +async fn route_path_key_exists( + route_path_key: &str, + http_method: &HttpMethod, + w_id: &str, + trigger_path: Option<&str>, + db: &DB, +) -> error::Result { + let exists = if *CLOUD_HOSTED { + sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM http_trigger WHERE route_path_key = $1 AND workspace_id = $2 AND http_method = $3 AND ($4::TEXT IS NULL OR path != $4))", + &route_path_key, + w_id, + http_method as &HttpMethod, + trigger_path + ) + .fetch_one(db) + .await? + .unwrap_or(false) + } else { + sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM http_trigger WHERE route_path_key = $1 AND http_method = $2 AND ($3::TEXT IS NULL OR path != $3))", + &route_path_key, + http_method as &HttpMethod, + trigger_path + ) + .fetch_one(db) + .await? + .unwrap_or(false) + }; + Ok(exists) } async fn exists_route( Extension(db): Extension, Path(w_id): Path, - Json(RouteExists { route_path, http_method }): Json, + Json(RouteExists { route_path, http_method, trigger_path }): Json, ) -> JsonResult { let route_path_key = ROUTE_PATH_KEY_RE.replace_all(route_path.as_str(), ":key"); - let exists = if *CLOUD_HOSTED { - sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM http_trigger WHERE route_path_key = $1 AND workspace_id = $2 AND http_method = $3)", - &route_path_key, - w_id, - http_method as HttpMethod - ) - .fetch_one(&db) - .await? - .unwrap_or(false) - } else { - sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM http_trigger WHERE route_path_key = $1 AND http_method = $2)", - &route_path_key, - http_method as HttpMethod - ) - .fetch_one(&db) - .await? - .unwrap_or(false) - }; + + let exists = route_path_key_exists( + &route_path_key, + &http_method, + &w_id, + trigger_path.as_deref(), + &db, + ) + .await?; + Ok(Json(exists)) } @@ -420,6 +494,7 @@ struct TriggerRoute { edited_by: String, email: String, static_asset_config: Option>, + is_static_website: bool, } async fn get_http_route_trigger( @@ -439,7 +514,7 @@ async fn get_http_route_trigger( let route_path = StripPath(splitted.collect::>().join("/")); let triggers = sqlx::query_as!( TriggerRoute, - r#"SELECT path, script_path, is_flow, route_path, workspace_id, is_async, requires_auth, edited_by, email, static_asset_config as "static_asset_config: _" FROM http_trigger WHERE workspace_id = $1 AND http_method = $2"#, + r#"SELECT path, script_path, is_flow, route_path, workspace_id, is_async, requires_auth, edited_by, email, static_asset_config as "static_asset_config: _", is_static_website FROM http_trigger WHERE workspace_id = $1 AND http_method = $2"#, w_id, http_method as HttpMethod ) @@ -449,7 +524,7 @@ async fn get_http_route_trigger( } else { let triggers = sqlx::query_as!( TriggerRoute, - r#"SELECT path, script_path, is_flow, route_path, workspace_id, is_async, requires_auth, edited_by, email, static_asset_config as "static_asset_config: _" FROM http_trigger WHERE http_method = $1"#, + r#"SELECT path, script_path, is_flow, route_path, workspace_id, is_async, requires_auth, edited_by, email, static_asset_config as "static_asset_config: _", is_static_website FROM http_trigger WHERE http_method = $1"#, http_method as HttpMethod ) .fetch_all(db) @@ -461,6 +536,17 @@ async fn get_http_route_trigger( for (idx, trigger) in triggers.iter().enumerate() { let route_path = trigger.route_path.clone(); + if trigger.is_static_website { + router + .insert(format!("{}/*wm_subpath", route_path), idx) + .unwrap_or_else(|e| { + tracing::warn!( + "Failed to consider http trigger route {}: {:?}", + route_path, + e, + ); + }); + } router.insert(route_path.as_str(), idx).unwrap_or_else(|e| { tracing::warn!( "Failed to consider http trigger route {}: {:?}", @@ -565,7 +651,7 @@ async fn route_job( headers: HeaderMap, args: WebhookArgs, ) -> impl IntoResponse { - let route_path = route_path.to_path(); + let route_path = route_path.to_path().trim_end_matches("/"); let (trigger, called_path, params, authed) = match get_http_route_trigger( route_path, &auth_cache, @@ -590,7 +676,7 @@ async fn route_job( #[cfg(not(feature = "parquet"))] if trigger.static_asset_config.is_some() { - return error::Error::InternalErr( + return error::Error::internal_err( "Static asset configuration is not supported in this build".to_string(), ) .into_response(); @@ -608,15 +694,41 @@ async fn route_job( config.storage, ) .await?; - let s3_resource = s3_resource_opt.ok_or(error::Error::InternalErr( + let s3_resource = s3_resource_opt.ok_or(error::Error::internal_err( "No files storage resource defined at the workspace level".to_string(), ))?; let s3_client = build_object_store_client(&s3_resource).await?; - let path = object_store::path::Path::from(config.s3); - let s3_object = s3_client.get(&path).await.map_err(|err| { + + let path = if trigger.is_static_website { + let subpath = params + .get("wm_subpath") + .cloned() + .unwrap_or("index.html".to_string()); + tracing::info!("subpath: {}", subpath); + format!("{}/{}", config.s3.trim_end_matches('/'), subpath) + } else { + config.s3.clone() + }; + let path = object_store::path::Path::from(path); + let s3_object = s3_client.get(&path).await; + + let s3_object = match s3_object { + Err(object_store::Error::NotFound { .. }) if trigger.is_static_website => { + // fallback to index.html if the file is not found + let path = object_store::path::Path::from(format!( + "{}/index.html", + config.s3.trim_end_matches('/') + )); + s3_client.get(&path).await + } + r => r, + }; + + let s3_object = s3_object.map_err(|err| { tracing::warn!("Error retrieving file from S3: {:?}", err); - error::Error::InternalErr(format!("Error retrieving file: {}", err.to_string())) + error::Error::internal_err(format!("Error retrieving file: {}", err.to_string())) })?; + let mut response_headers = http::HeaderMap::new(); if let Some(ref e_tag) = s3_object.meta.e_tag { if let Some(if_none_match) = headers.get(IF_NONE_MATCH) { @@ -641,24 +753,26 @@ async fn route_job( .flatten() .unwrap_or("application/octet-stream".parse().unwrap()), ); - response_headers.insert( - "content-disposition", - config.filename.as_ref().map_or_else( - || { - s3_object - .attributes - .get(&object_store::Attribute::ContentDisposition) - .map(|s| s.parse().ok()) - .flatten() - .unwrap_or("inline".parse().unwrap()) - }, - |filename| { - format!("inline; filename=\"{}\"", filename) - .parse() - .unwrap_or("inline".parse().unwrap()) - }, - ), - ); + if !trigger.is_static_website { + response_headers.insert( + "content-disposition", + config.filename.as_ref().map_or_else( + || { + s3_object + .attributes + .get(&object_store::Attribute::ContentDisposition) + .map(|s| s.parse().ok()) + .flatten() + .unwrap_or("inline".parse().unwrap()) + }, + |filename| { + format!("inline; filename=\"{}\"", filename) + .parse() + .unwrap_or("inline".parse().unwrap()) + }, + ), + ); + } let body_stream = axum::body::Body::from_stream(s3_object.into_stream()); Ok::<_, error::Error>((StatusCode::OK, response_headers, body_stream)) diff --git a/backend/windmill-api/src/inputs.rs b/backend/windmill-api/src/inputs.rs index 1271be1620..1ce3fa9226 100644 --- a/backend/windmill-api/src/inputs.rs +++ b/backend/windmill-api/src/inputs.rs @@ -133,7 +133,7 @@ async fn get_input_history( let mut tx = user_db.begin(&authed).await?; let sql = &format!( - "select id, created_at, created_by, 'null'::jsonb as args, success from completed_job \ + "select id, created_at, created_by, 'null'::jsonb as args, success from v2_as_completed_job \ where {} = $1 and job_kind = any($2) and workspace_id = $3 \ order by created_at desc limit $4 offset $5", r.runnable_type.column_name() @@ -204,16 +204,16 @@ async fn get_args_from_history_or_saved_input( let result_o = if let Some(input) = g.input { if input { sqlx::query_scalar!( - "SELECT CASE WHEN pg_column_size(args) < 40000 OR $3 THEN args ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as args FROM input WHERE id = $1 AND workspace_id = $2", - job_or_input_id, - w_id, - g.allow_large.unwrap_or(true) - ) - .fetch_optional(&mut *tx) - .await? + "SELECT CASE WHEN pg_column_size(args) < 40000 OR $3 THEN args ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as args FROM input WHERE id = $1 AND workspace_id = $2", + job_or_input_id, + w_id, + g.allow_large.unwrap_or(true) + ) + .fetch_optional(&mut *tx) + .await? } else { sqlx::query_scalar!( - "SELECT CASE WHEN pg_column_size(args) < 40000 OR $3 THEN args ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as args FROM completed_job WHERE id = $1 AND workspace_id = $2", + "SELECT CASE WHEN pg_column_size(args) < 40000 OR $3 THEN args ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as args FROM v2_job WHERE id = $1 AND workspace_id = $2", job_or_input_id, w_id, g.allow_large.unwrap_or(true) @@ -223,13 +223,15 @@ async fn get_args_from_history_or_saved_input( } } else { sqlx::query_scalar!( - "SELECT CASE WHEN pg_column_size(args) < 40000 OR $3 THEN args ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as args FROM completed_job WHERE id = $1 AND workspace_id = $2 UNION ALL SELECT CASE WHEN pg_column_size(args) < 40000 OR $3 THEN args ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as args FROM input WHERE id = $1 AND workspace_id = $2", - job_or_input_id, - w_id, - g.allow_large.unwrap_or(true) - ) - .fetch_optional(&mut *tx) - .await? + "SELECT CASE WHEN pg_column_size(args) < 40000 OR $3 THEN args ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as args FROM v2_job WHERE id = $1 AND workspace_id = $2 + UNION ALL + SELECT CASE WHEN pg_column_size(args) < 40000 OR $3 THEN args ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as args FROM input WHERE id = $1 AND workspace_id = $2", + job_or_input_id, + w_id, + g.allow_large.unwrap_or(true) + ) + .fetch_optional(&mut *tx) + .await? }; tx.commit().await?; diff --git a/backend/windmill-api/src/job_helpers_ee.rs b/backend/windmill-api/src/job_helpers_ee.rs index 7b074a300a..b35f40d661 100644 --- a/backend/windmill-api/src/job_helpers_ee.rs +++ b/backend/windmill-api/src/job_helpers_ee.rs @@ -69,7 +69,7 @@ pub async fn get_s3_resource<'c>( _resource_type: Option, _job_id: Option, ) -> error::Result { - Err(error::Error::InternalErr( + Err(error::Error::internal_err( "Not implemented in Windmill's Open Source repository".to_string(), )) } @@ -81,7 +81,7 @@ pub async fn upload_file_from_req( _req: axum::extract::Request, _options: PutMultipartOpts, ) -> error::Result<()> { - Err(error::Error::InternalErr( + Err(error::Error::internal_err( "Not implemented in Windmill's Open Source repository".to_string(), )) } @@ -93,7 +93,7 @@ pub async fn upload_file_internal( _stream: impl Stream> + Unpin, _options: PutMultipartOpts, ) -> error::Result<()> { - Err(error::Error::InternalErr( + Err(error::Error::internal_err( "Not implemented in Windmill's Open Source repository".to_string(), )) } @@ -107,7 +107,7 @@ pub async fn download_s3_file_internal( _w_id: &str, _query: DownloadFileQuery, ) -> error::Result { - Err(error::Error::InternalErr( + Err(error::Error::internal_err( "Not implemented in Windmill's Open Source repository".to_string(), )) } @@ -120,7 +120,7 @@ pub async fn load_image_preview_internal( _w_id: &str, _query: LoadImagePreviewQuery, ) -> error::Result { - Err(error::Error::InternalErr( + Err(error::Error::internal_err( "Not implemented in Windmill's Open Source repository".to_string(), )) } diff --git a/backend/windmill-api/src/job_metrics.rs b/backend/windmill-api/src/job_metrics.rs index 2af59383a8..70e68ba3ed 100644 --- a/backend/windmill-api/src/job_metrics.rs +++ b/backend/windmill-api/src/job_metrics.rs @@ -171,7 +171,7 @@ async fn set_job_progress( if let Some(flow_job_id) = flow_job_id { // TODO: Return error if trying to set completed job? sqlx::query!( - "UPDATE queue + "UPDATE v2_job_status SET flow_status = JSONB_SET(flow_status, ARRAY['modules', flow_status->>'step', 'progress'], $1) WHERE id = $2", serde_json::json!(percent.clamp(0, 99)), diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index f039c220fb..90caf92518 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -87,8 +87,8 @@ use windmill_common::{METRICS_DEBUG_ENABLED, METRICS_ENABLED}; use windmill_common::{get_latest_deployed_hash_for_path, BASE_URL}; use windmill_queue::{ - cancel_job, get_queued_job, get_result_and_success_by_id_from_flow, job_is_complete, push, - PushArgs, PushArgsOwned, PushIsolationLevel, + cancel_job, get_result_and_success_by_id_from_flow, job_is_complete, push, PushArgs, + PushArgsOwned, PushIsolationLevel, }; #[cfg(feature = "prometheus")] @@ -336,16 +336,21 @@ async fn get_root_job( Ok(Json(res)) } -async fn compute_root_job_for_flow(db: &DB, w_id: &str, job_id: Uuid) -> error::Result { - let mut job = get_queued_job(&job_id, w_id, db).await?; - while let Some(j) = job { - if let Some(uuid) = j.parent_job { - job = get_queued_job(&uuid, w_id, db).await?; - } else { - return Ok(j.id.to_string()); +async fn compute_root_job_for_flow(db: &DB, w_id: &str, mut job_id: Uuid) -> error::Result { + // TODO: use `root_job` ? + loop { + job_id = match sqlx::query_scalar!( + "SELECT parent_job FROM v2_job WHERE id = $1 AND workspace_id = $2", + job_id, + w_id + ) + .fetch_one(db) + .await + { + Ok(Some(job_id)) => job_id, + _ => return Ok(job_id.to_string()), } } - Ok(job_id.to_string()) } async fn get_db_clock(Extension(db): Extension) -> windmill_common::error::JsonResult { @@ -384,7 +389,7 @@ async fn cancel_job_api( ) .await .map_err(|e| { - Error::InternalErr(format!( + Error::internal_err(format!( "timeout after 120s while cancelling job {id} in {w_id}: {e:#}" )) })??; @@ -495,7 +500,7 @@ async fn force_cancel( ) .await .map_err(|e| { - Error::InternalErr(format!( + Error::internal_err(format!( "timeout after 120s while cancelling job {id} in {w_id}: {e:#}" )) })??; @@ -554,7 +559,7 @@ pub async fn get_path_tag_limits_cache_for_hash( .fetch_optional(&mut *tx) .await .map_err(|e| { - Error::InternalErr(format!( + Error::internal_err(format!( "querying getting path for hash {hash} in {w_id}: {e:#}" )) })?.ok_or_else(|| Error::NotFound(format!( @@ -602,7 +607,7 @@ async fn get_flow_job_debug_info( let mut job_ids = vec![]; let jobs_with_root = sqlx::query_scalar!( - "SELECT id FROM queue WHERE workspace_id = $1 and root_job = $2", + "SELECT id FROM v2_job WHERE workspace_id = $1 and flow_innermost_root_job = $2", &w_id, &id, ) @@ -672,23 +677,24 @@ async fn get_job( } macro_rules! get_job_query { - ("completed_job_view", $($opts:tt)*) => { + ("v2_as_completed_job", $($opts:tt)*) => { get_job_query!( - @impl "completed_job_view", ($($opts)*), - "duration_ms, success, result, deleted, is_skipped, result->'wm_labels' as labels, \ + @impl "v2_as_completed_job", ($($opts)*), + "duration_ms, success, result, result_columns, deleted, is_skipped, result->'wm_labels' as labels, \ CASE WHEN result is null or pg_column_size(result) < 90000 THEN result ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as result", ) }; - ("queue_view", $($opts:tt)*) => { + ("v2_as_queue", $($opts:tt)*) => { get_job_query!( - @impl "queue_view", ($($opts)*), + @impl "v2_as_queue", ($($opts)*), "scheduled_for, running, last_ping, suspend, suspend_until, same_worker, pre_run_error, visible_to_owner, \ - root_job, leaf_jobs, concurrent_limit, concurrency_time_window_s, timeout, flow_step_id, cache_ttl", + root_job, leaf_jobs, concurrent_limit, concurrency_time_window_s, timeout, flow_step_id, cache_ttl,\ + script_entrypoint_override", ) }; (@impl $table:literal, (with_logs: $with_logs:expr, $($rest:tt)*), $additional_fields:literal, $($args:tt)*) => { if $with_logs { - get_job_query!(@impl $table, ($($rest)*), $additional_fields, logs = const_format::formatcp!("right({}.logs, 20000)", $table), $($args)*) + get_job_query!(@impl $table, ($($rest)*), $additional_fields, logs = "right(job_logs.logs, 20000)", $($args)*) } else { get_job_query!(@impl $table, ($($rest)*), $additional_fields, logs = "null", $($args)*) } @@ -714,8 +720,8 @@ macro_rules! get_job_query { CASE WHEN args is null or pg_column_size(args) < 90000 THEN args ELSE '{{\"reason\": \"WINDMILL_TOO_BIG\"}}'::jsonb END as args, \ {logs} as logs, {code} as raw_code, canceled, canceled_by, canceled_reason, job_kind, \ schedule_path, permissioned_as, flow_status, {flow} as raw_flow, is_flow_step, language, \ - {lock} as raw_lock, email, visible_to_owner, mem_peak, tag, priority, {additional_fields} \ - FROM {table} \ + {lock} as raw_lock, email, visible_to_owner, mem_peak, tag, priority, preprocessed, {additional_fields} \ + FROM {table} LEFT JOIN job_logs ON id = job_id \ WHERE id = $1 AND {table}.workspace_id = $2 AND ($3::text[] IS NULL OR tag = ANY($3)) LIMIT 1", table = $table, additional_fields = $additional_fields, @@ -829,7 +835,7 @@ impl<'a> GetQuery<'a> { job_id: Uuid, workspace_id: &str, ) -> error::Result>> { - let query = get_job_query!("queue_view", + let query = get_job_query!("v2_as_queue", with_logs: self.with_logs, with_code: self.with_code, with_flow: self.with_flow, @@ -861,7 +867,7 @@ impl<'a> GetQuery<'a> { job_id: Uuid, workspace_id: &str, ) -> error::Result>> { - let query = get_job_query!("completed_job_view", + let query = get_job_query!("v2_as_completed_job", with_logs: self.with_logs, with_code: self.with_code, with_flow: self.with_flow, @@ -1009,10 +1015,10 @@ async fn get_job_logs( .flatten(); let record = sqlx::query!( - "SELECT created_by, CONCAT(coalesce(completed_job.logs, ''), coalesce(job_logs.logs, '')) as logs, job_logs.log_offset, job_logs.log_file_index - FROM completed_job - LEFT JOIN job_logs ON job_logs.job_id = completed_job.id - WHERE completed_job.id = $1 AND completed_job.workspace_id = $2 AND ($3::text[] IS NULL OR completed_job.tag = ANY($3))", + "SELECT created_by AS \"created_by!\", CONCAT(coalesce(v2_as_completed_job.logs, ''), coalesce(job_logs.logs, '')) as logs, job_logs.log_offset, job_logs.log_file_index + FROM v2_as_completed_job + LEFT JOIN job_logs ON job_logs.job_id = v2_as_completed_job.id + WHERE v2_as_completed_job.id = $1 AND v2_as_completed_job.workspace_id = $2 AND ($3::text[] IS NULL OR v2_as_completed_job.tag = ANY($3))", id, w_id, tags.as_ref().map(|v| v.as_slice()) @@ -1046,10 +1052,10 @@ async fn get_job_logs( Ok(content_plain(Body::from(logs))) } else { let text = sqlx::query!( - "SELECT created_by, CONCAT(coalesce(queue.logs, ''), coalesce(job_logs.logs, '')) as logs, coalesce(job_logs.log_offset, 0) as log_offset, job_logs.log_file_index - FROM queue - LEFT JOIN job_logs ON job_logs.job_id = queue.id - WHERE queue.id = $1 AND queue.workspace_id = $2 AND ($3::text[] IS NULL OR queue.tag = ANY($3))", + "SELECT created_by AS \"created_by!\", CONCAT(coalesce(v2_as_queue.logs, ''), coalesce(job_logs.logs, '')) as logs, coalesce(job_logs.log_offset, 0) as log_offset, job_logs.log_file_index + FROM v2_as_queue + LEFT JOIN job_logs ON job_logs.job_id = v2_as_queue.id + WHERE v2_as_queue.id = $1 AND v2_as_queue.workspace_id = $2 AND ($3::text[] IS NULL OR v2_as_queue.tag = ANY($3))", id, w_id, tags.as_ref().map(|v| v.as_slice()) @@ -1087,29 +1093,23 @@ async fn get_job_logs( } } -#[derive(FromRow)] -pub struct RawArgs { - pub args: Option>>, - pub created_by: String, -} - async fn get_args( OptAuthed(opt_authed): OptAuthed, Extension(db): Extension, Path((w_id, id)): Path<(String, Uuid)>, -) -> error::JsonResult> { +) -> JsonResult> { let tags = opt_authed .as_ref() .map(|authed| get_scope_tags(authed)) .flatten(); - let record = sqlx::query_as::<_, RawArgs>( - "SELECT created_by, args - FROM completed_job - WHERE completed_job.id = $1 AND completed_job.workspace_id = $2 AND ($3::text[] IS NULL OR completed_job.tag = ANY($3))", + let record = sqlx::query!( + "SELECT created_by AS \"created_by!\", args as \"args: sqlx::types::Json>\" + FROM v2_as_completed_job + WHERE id = $1 AND workspace_id = $2 AND ($3::text[] IS NULL OR tag = ANY($3))", + id, + &w_id, + tags.as_ref().map(|v| v.as_slice()) as Option<&[&str]>, ) - .bind(&id) - .bind(&w_id) - .bind(tags.as_ref().map(|v| v.as_slice())) .fetch_optional(&db) .await?; @@ -1124,14 +1124,14 @@ async fn get_args( Ok(Json(record.args.map(|x| x.0).unwrap_or_default())) } else { - let record = sqlx::query_as::<_, RawArgs>( - "SELECT created_by, args - FROM queue - WHERE queue.id = $1 AND queue.workspace_id = $2 AND ($3::text[] IS NULL OR queue.tag = ANY($3))", + let record = sqlx::query!( + "SELECT created_by AS \"created_by!\", args as \"args: sqlx::types::Json>\" + FROM v2_job + WHERE id = $1 AND workspace_id = $2 AND ($3::text[] IS NULL OR tag = ANY($3))", + id, + &w_id, + tags.as_ref().map(|v| v.as_slice()) as Option<&[&str]>, ) - .bind(&id) - .bind(&w_id) - .bind(tags.as_ref().map(|v| v.as_slice())) .fetch_optional(&db) .await?; let record = not_found_if_none(record, "Job Args", id.to_string())?; @@ -1399,7 +1399,7 @@ pub fn list_queue_jobs_query( tags: Option>, ) -> SqlBuilder { let (limit, offset) = paginate_without_limits(pagination); - let mut sqlb = SqlBuilder::select_from("queue") + let mut sqlb = SqlBuilder::select_from("v2_as_queue") .fields(fields) .order_by("created_at", lq.order_desc.unwrap_or(true)) .limit(limit) @@ -1488,75 +1488,36 @@ async fn cancel_jobs( ) -> error::JsonResult> { let mut uuids = vec![]; let mut tx = db.begin().await?; - let trivial_jobs = sqlx::query!("INSERT INTO completed_job AS cj + let trivial_jobs = sqlx::query!("INSERT INTO v2_job_completed AS cj ( workspace_id , id - , parent_job - , created_by - , created_at - , started_at , duration_ms - , success - , script_hash - , script_path - , args , result - , raw_code - , raw_lock - , canceled , canceled_by , canceled_reason - , job_kind - , schedule_path - , permissioned_as , flow_status - , raw_flow - , is_flow_step - , is_skipped - , language - , email - , visible_to_owner - , mem_peak - , tag - , priority + , status + , worker ) - SELECT workspace_id - , id - , parent_job - , created_by - , created_at - , now() + SELECT q.workspace_id + , q.id , 0 - , false - , script_hash - , script_path - , args , $4 - , raw_code - , raw_lock - , true , $1 - , canceled_reason - , job_kind - , schedule_path - , permissioned_as - , flow_status - , raw_flow - , is_flow_step - , false - , language - , email - , visible_to_owner - , mem_peak - , tag - , priority FROM queue - WHERE id = any($2) AND running = false AND parent_job IS NULL AND workspace_id = $3 AND schedule_path IS NULL FOR UPDATE SKIP LOCKED - ON CONFLICT (id) DO NOTHING RETURNING id", username, &jobs, w_id, serde_json::json!({"error": { "message": format!("Job canceled: cancel all by {username}"), "name": "Canceled", "reason": "cancel all", "canceler": username}})) + , 'cancel all' + , (SELECT flow_status FROM v2_job_status WHERE id = q.id) + , 'canceled'::job_status + , worker + FROM v2_job_queue q + JOIN v2_job USING (id) + WHERE q.id = any($2) AND running = false AND parent_job IS NULL AND q.workspace_id = $3 AND trigger IS NULL + FOR UPDATE SKIP LOCKED + ON CONFLICT (id) DO NOTHING RETURNING id AS \"id!\"", username, &jobs, w_id, serde_json::json!({"error": { "message": format!("Job canceled: cancel all by {username}"), "name": "Canceled", "reason": "cancel all", "canceler": username}})) .fetch_all(&mut *tx) .await?.into_iter().map(|x| x.id).collect::>(); sqlx::query!( - "DELETE FROM queue WHERE id = any($1) AND workspace_id = $2", + "DELETE FROM v2_job_queue WHERE id = any($1) AND workspace_id = $2", &trivial_jobs, w_id ) @@ -1624,7 +1585,7 @@ async fn cancel_selection( let mut tx = user_db.begin(&authed).await?; let tags = get_scope_tags(&authed).map(|v| v.iter().map(|s| s.to_string()).collect_vec()); let jobs_to_cancel = sqlx::query_scalar!( - "SELECT id FROM queue WHERE id = ANY($1) AND schedule_path IS NULL AND ($2::text[] IS NULL OR tag = ANY($2))", + "SELECT id AS \"id!\" FROM v2_as_queue WHERE id = ANY($1) AND schedule_path IS NULL AND ($2::text[] IS NULL OR tag = ANY($2))", &jobs, tags.as_ref().map(|v| v.as_slice()) ) @@ -1644,7 +1605,9 @@ async fn list_filtered_uuids( ) -> error::JsonResult> { require_admin(authed.is_admin, &authed.username)?; - let mut sqlb = SqlBuilder::select_from("queue").fields(&["id"]).clone(); + let mut sqlb = SqlBuilder::select_from("v2_as_queue") + .fields(&["id"]) + .clone(); sqlb = join_concurrency_key(lq.concurrency_key.as_ref(), sqlb); @@ -1681,7 +1644,7 @@ async fn count_queue_jobs( Ok(Json( sqlx::query_as!( QueueStats, - "SELECT coalesce(COUNT(*) FILTER(WHERE suspend = 0 AND running = false), 0) as \"database_length!\", coalesce(COUNT(*) FILTER(WHERE suspend > 0), 0) as \"suspended!\" FROM queue WHERE (workspace_id = $1 OR $2) AND scheduled_for <= now()", + "SELECT coalesce(COUNT(*) FILTER(WHERE suspend = 0 AND running = false), 0) as \"database_length!\", coalesce(COUNT(*) FILTER(WHERE suspend > 0), 0) as \"suspended!\" FROM v2_as_queue WHERE (workspace_id = $1 OR $2) AND scheduled_for <= now()", w_id, w_id == "admins" && cq.all_workspaces.unwrap_or(false), ) @@ -1703,7 +1666,7 @@ async fn count_completed_jobs_detail( Path(w_id): Path, Query(query): Query, ) -> error::JsonResult { - let mut sqlb = SqlBuilder::select_from("completed_job"); + let mut sqlb = SqlBuilder::select_from("v2_as_completed_job"); sqlb.field("COUNT(*) as count"); if !query.all_workspaces.unwrap_or(false) { @@ -1745,7 +1708,7 @@ async fn count_completed_jobs( Ok(Json( sqlx::query_as!( QueueStats, - "SELECT coalesce(COUNT(*), 0) as \"database_length!\", null::bigint as suspended FROM completed_job WHERE workspace_id = $1", + "SELECT coalesce(COUNT(*), 0) as \"database_length!\", null::bigint as suspended FROM v2_job_completed WHERE workspace_id = $1", w_id ) .fetch_one(&db) @@ -1959,7 +1922,7 @@ async fn resume_suspended_job_internal( if !approved { sqlx::query!( - "UPDATE queue SET suspend = 0 WHERE id = $1", + "UPDATE v2_job_queue SET suspend = 0 WHERE id = $1", parent_flow_info.id ) .execute(&mut *tx) @@ -2045,7 +2008,7 @@ async fn resume_immediately_if_relevant<'c>( if matches!(status.current_step(), Some(FlowStatusModule::WaitingForEvents { job, .. }) if job == &job_id) { sqlx::query!( - "UPDATE queue SET suspend = $1 WHERE id = $2", + "UPDATE v2_job_queue SET suspend = $1 WHERE id = $2", suspend, flow.id, ) @@ -2096,9 +2059,11 @@ async fn get_suspended_parent_flow_info(job_id: Uuid, db: &DB) -> error::Result< let flow = sqlx::query_as!( FlowInfo, r#" - SELECT id, flow_status, suspend, script_path - FROM queue - WHERE id = ( SELECT parent_job FROM queue WHERE id = $1 UNION ALL SELECT parent_job FROM completed_job WHERE id = $1) + SELECT q.id, f.flow_status, q.suspend, j.runnable_path AS script_path + FROM v2_job_queue q + JOIN v2_job j USING (id) + JOIN v2_job_status f USING (id) + WHERE id = ( SELECT parent_job FROM v2_job WHERE id = $1 ) FOR UPDATE "#, job_id, @@ -2116,8 +2081,8 @@ async fn get_suspended_flow_info<'c>( let flow = sqlx::query_as!( FlowInfo, r#" - SELECT id, flow_status, suspend, script_path - FROM queue + SELECT id AS "id!", flow_status, suspend AS "suspend!", script_path + FROM v2_as_queue WHERE id = $1 "#, job_id, @@ -2176,11 +2141,7 @@ pub async fn get_suspended_job_flow( let flow_id = sqlx::query_scalar!( r#" SELECT parent_job - FROM queue - WHERE id = $1 AND workspace_id = $2 - UNION ALL - SELECT parent_job - FROM completed_job + FROM v2_job WHERE id = $1 AND workspace_id = $2 "#, job, @@ -2286,8 +2247,10 @@ fn conditionally_require_authed_user( return Ok(()); } } - let error_msg = format!("Only users from one of the following groups are allowed to approve this workflow: {}", - approval_conditions.user_groups_required.join(", ")); + let error_msg = format!( + "Only users from one of the following groups are allowed to approve this workflow: {}", + approval_conditions.user_groups_required.join(", ") + ); return Err(Error::PermissionDenied(error_msg)); } } @@ -2317,7 +2280,7 @@ pub async fn get_flow_user_state( let r = sqlx::query_scalar!( r#" SELECT flow_status->'user_states'->$1 - FROM queue + FROM v2_as_queue WHERE id = $2 AND workspace_id = $3 "#, key, @@ -2339,8 +2302,9 @@ pub async fn set_flow_user_state( let mut tx = user_db.begin(&authed).await?; let r = sqlx::query_scalar!( r#" - UPDATE queue SET flow_status = JSONB_SET(flow_status, ARRAY['user_states'], JSONB_SET(COALESCE(flow_status->'user_states', '{}'::jsonb), ARRAY[$1], $2)) - WHERE id = $3 AND workspace_id = $4 AND job_kind IN ('flow', 'flowpreview', 'flownode') RETURNING 1 + UPDATE v2_job_status f SET flow_status = JSONB_SET(flow_status, ARRAY['user_states'], JSONB_SET(COALESCE(flow_status->'user_states', '{}'::jsonb), ARRAY[$1], $2)) + FROM v2_job j + WHERE f.id = $3 AND f.id = j.id AND j.workspace_id = $4 AND kind IN ('flow', 'flowpreview', 'flownode') RETURNING 1 "#, key, value, @@ -2682,6 +2646,7 @@ pub struct UnifiedJob { pub labels: Option, pub self_wait_time_ms: Option, pub aggregate_wait_time_ms: Option, + pub preprocessed: Option, } const CJ_FIELDS: &[&str] = &[ @@ -2719,6 +2684,7 @@ const CJ_FIELDS: &[&str] = &[ "result->'wm_labels' as labels", "self_wait_time_ms", "aggregate_wait_time_ms", + "preprocessed", ]; const QJ_FIELDS: &[&str] = &[ "'QueuedJob' as typ", @@ -2755,6 +2721,7 @@ const QJ_FIELDS: &[&str] = &[ "null as labels", "self_wait_time_ms", "aggregate_wait_time_ms", + "preprocessed", ]; impl UnifiedJob { @@ -2778,13 +2745,14 @@ impl<'a> From for Job { parent_job: uj.parent_job, created_by: uj.created_by, created_at: uj.created_at, - started_at: uj.started_at.unwrap_or(uj.created_at), + started_at: uj.started_at, duration_ms: uj.duration_ms.unwrap(), success: uj.success.unwrap(), script_hash: uj.script_hash, script_path: uj.script_path, args: None, result: None, + result_columns: None, logs: None, flow_status: None, deleted: uj.deleted, @@ -2803,6 +2771,7 @@ impl<'a> From for Job { tag: uj.tag, priority: uj.priority, labels: uj.labels, + preprocessed: uj.preprocessed, }, )), "QueuedJob" => Job::QueuedJob(JobExtended::new( @@ -2831,6 +2800,7 @@ impl<'a> From for Job { permissioned_as: uj.permissioned_as, is_flow_step: uj.is_flow_step, language: uj.language, + script_entrypoint_override: None, same_worker: false, pre_run_error: None, email: uj.email, @@ -2846,6 +2816,7 @@ impl<'a> From for Job { flow_step_id: None, cache_ttl: None, priority: uj.priority, + preprocessed: uj.preprocessed, }, )), t => panic!("job type {} not valid", t), @@ -3150,11 +3121,15 @@ pub async fn restart_flow( check_license_key_valid().await?; let mut tx = user_db.clone().begin(&authed).await?; - let completed_job = sqlx::query_as::<_, CompletedJob>( - "SELECT *, result->'wm_labels' as labels from completed_job WHERE id = $1 and workspace_id = $2", + let completed_job = sqlx::query!( + "SELECT + script_path, args AS \"args: sqlx::types::Json>>\", + tag AS \"tag!\", priority + FROM v2_as_completed_job + WHERE id = $1 and workspace_id = $2", + job_id, + &w_id, ) - .bind(job_id) - .bind(&w_id) .fetch_optional(&mut *tx) .await? .with_context(|| "Unable to find completed job with the given job UUID")?; @@ -3180,11 +3155,7 @@ pub async fn restart_flow( &db, tx, &w_id, - JobPayload::RestartedFlow { - completed_job_id: job_id, - step_id: step_id, - branch_or_iteration_n: branch_or_iteration_n, - }, + JobPayload::RestartedFlow { completed_job_id: job_id, step_id, branch_or_iteration_n }, push_args, &authed.username, &authed.email, @@ -3454,14 +3425,29 @@ pub async fn run_workflow_as_code( if !wkflow_query.skip_update.unwrap_or(false) { sqlx::query!( - "UPDATE queue SET flow_status = jsonb_set(COALESCE(flow_status, '{}'::jsonb), array[$1], jsonb_set(jsonb_set('{}'::jsonb, '{scheduled_for}', to_jsonb(now()::text)), '{name}', to_jsonb($4::text))) WHERE id = $2 AND workspace_id = $3", - uuid.to_string(), + "INSERT INTO v2_job_status (id, workflow_as_code_status) + VALUES ($1, JSONB_SET('{}'::JSONB, array[$2], $3)) + ON CONFLICT (id) DO UPDATE SET + workflow_as_code_status = JSONB_SET( + COALESCE(v2_job_status.workflow_as_code_status, '{}'::JSONB), + array[$2], + $3 + )", job_id, - w_id, - entrypoint - ).execute(&mut *tx).await?; + uuid.to_string(), + serde_json::json!({ "scheduled_for": Utc::now(), "name": entrypoint }), + ) + .execute(&mut *tx) + .await?; } else { tracing::info!("Skipping update of flow status for job {job_id} in workspace {w_id}"); + sqlx::query!( + "INSERT INTO v2_job_status (id, workflow_as_code_status) VALUES ($1, '{}'::JSONB) + ON CONFLICT (id) DO NOTHING", + job_id, + ) + .execute(&mut *tx) + .await?; } if *CLOUD_HOSTED { @@ -3587,17 +3573,21 @@ pub async fn run_wait_result_internal( } if result.is_none() { - let row = sqlx::query_as::<_, RawResultWithSuccess>( - "SELECT '' as created_by, result, language, flow_status, success FROM completed_job WHERE id = $1 AND workspace_id = $2", + let row = sqlx::query!( + "SELECT + result AS \"result: sqlx::types::Json>\", + result_columns, + status = 'success' AS \"success!\" + FROM v2_job_completed + WHERE id = $1 AND workspace_id = $2", + uuid, + &w_id ) - .bind(uuid) - .bind(&w_id) .fetch_optional(db) .await?; if let Some(mut raw_result) = row { format_result( - raw_result.language.as_ref(), - raw_result.flow_status.as_ref(), + raw_result.result_columns.as_ref(), raw_result.result.as_mut(), ); result = raw_result.result.map(|x| x.0); @@ -3682,10 +3672,10 @@ pub async fn run_wait_result( if let Some(windmill_headers) = windmill_headers { for (k, v) in windmill_headers { let k = HeaderName::from_str(k.as_str()).map_err(|err| { - Error::InternalErr(format!("Invalid header name {k}: {err}")) + Error::internal_err(format!("Invalid header name {k}: {err}")) })?; let v = HeaderValue::from_str(v.as_str()).map_err(|err| { - Error::InternalErr(format!("Invalid header value {v}: {err}")) + Error::internal_err(format!("Invalid header value {v}: {err}")) })?; headers.insert(k, v); } @@ -3704,7 +3694,7 @@ pub async fn run_wait_result( headers.insert( http::header::CONTENT_TYPE, HeaderValue::from_str(content_type.as_str()).map_err(|err| { - Error::InternalErr(format!("Invalid content type {content_type}: {err}")) + Error::internal_err(format!("Invalid content type {content_type}: {err}")) })?, ); return Ok((status_code_or_default, headers, serialized_result).into_response()); @@ -3729,17 +3719,19 @@ pub async fn run_wait_result( async fn delete_job_metadata_after_use(db: &DB, job_uuid: Uuid) -> Result<(), Error> { sqlx::query!( - "UPDATE completed_job - SET logs = '##DELETED##', args = '{}'::jsonb, result = '{}'::jsonb - WHERE id = $1", + "UPDATE v2_job SET args = '{}'::jsonb WHERE id = $1", job_uuid, ) .execute(db) .await?; sqlx::query!( - "UPDATE job_logs - SET logs = '##DELETED##' - WHERE job_id = $1", + "UPDATE v2_job_completed SET result = '{}'::jsonb WHERE id = $1", + job_uuid, + ) + .execute(db) + .await?; + sqlx::query!( + "UPDATE job_logs SET logs = '##DELETED##' WHERE job_id = $1", job_uuid, ) .execute(db) @@ -3750,7 +3742,7 @@ async fn delete_job_metadata_after_use(db: &DB, job_uuid: Uuid) -> Result<(), Er pub async fn check_queue_too_long(db: &DB, queue_limit: Option) -> error::Result<()> { if let Some(limit) = queue_limit { let count = sqlx::query_scalar!( - "SELECT COUNT(*) FROM queue WHERE canceled = false AND (scheduled_for <= now() + "SELECT COUNT(*) FROM v2_as_queue WHERE canceled = false AND (scheduled_for <= now() OR (suspend_until IS NOT NULL AND ( suspend <= 0 OR suspend_until <= now())))", @@ -3760,7 +3752,7 @@ pub async fn check_queue_too_long(db: &DB, queue_limit: Option) -> error::R .unwrap_or(0); if count > queue_limit.unwrap() { - return Err(Error::InternalErr(format!( + return Err(Error::internal_err(format!( "Number of queued job is too high: {count} > {limit}" ))); } @@ -3872,7 +3864,7 @@ pub async fn run_wait_result_job_by_path_get( return Ok(Json(serde_json::json!("")).into_response()); } let payload_r = run_query.payload.map(decode_payload).map(|x| { - x.map_err(|e| Error::InternalErr(format!("Impossible to decode query payload: {e:#?}"))) + x.map_err(|e| Error::internal_err(format!("Impossible to decode query payload: {e:#?}"))) }); let mut payload_args = if let Some(payload) = payload_r { @@ -3967,7 +3959,7 @@ pub async fn run_wait_result_flow_by_path_get( } let payload_r = run_query.payload.clone().map(decode_payload).map(|x| { x.map_err(|e| { - error::Error::InternalErr(format!("Impossible to decode query payload: {e:#?}")) + error::Error::internal_err(format!("Impossible to decode query payload: {e:#?}")) }) }); @@ -4551,7 +4543,7 @@ async fn run_dependencies_job( } if req.raw_scripts.len() != 1 || req.raw_scripts[0].script_path != req.entrypoint { - return Err(error::Error::InternalErr( + return Err(error::Error::internal_err( "For now only a single raw script can be passed to this endpoint, and the entrypoint should be set to the script path".to_string(), )); } @@ -4790,10 +4782,10 @@ async fn add_batch_jobs( ) .fetch_optional(&mut *tx) .await? - .ok_or_else(|| Error::InternalErr(format!("not found flow at path {:?}", path)))?; + .ok_or_else(|| Error::internal_err(format!("not found flow at path {:?}", path)))?; let value = serde_json::from_str::(value_json.value.get()).map_err(|err| { - Error::InternalErr(format!( + Error::internal_err(format!( "could not convert json to flow for {path}: {err:?}" )) })?; @@ -4860,47 +4852,68 @@ async fn add_batch_jobs( let uuids = sqlx::query_scalar!( r#"WITH uuid_table as ( - select gen_random_uuid() as uuid from generate_series(1, $5) + select gen_random_uuid() as uuid from generate_series(1, $16) ) - INSERT INTO job - (id, workspace_id, raw_code, raw_lock, raw_flow) - (SELECT uuid, $1, $2, $3, $4 FROM uuid_table) - RETURNING id"#, + INSERT INTO v2_job + (id, workspace_id, raw_code, raw_lock, raw_flow, tag, runnable_id, runnable_path, kind, + script_lang, created_by, permissioned_as, permissioned_as_email, concurrent_limit, + concurrency_time_window_s, timeout, args) + (SELECT uuid, $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, + ('{ "uuid": "' || uuid || '" }')::jsonb FROM uuid_table) + RETURNING id AS "id!""#, w_id, raw_code, raw_lock, raw_flow.map(sqlx::types::Json) as Option>, - n + tag, + hash.map(|h| h.0), + path, + job_kind.clone() as JobKind, + language as ScriptLang, + authed.username, + username_to_permissioned_as(&authed.username), + authed.email, + concurrent_limit, + concurrent_time_window_s, + timeout, + n, ) .fetch_all(&mut *tx) .await?; let uuids = sqlx::query_scalar!( r#"WITH uuid_table as ( - select unnest($11::uuid[]) as uuid + select unnest($4::uuid[]) as uuid ) - INSERT INTO queue - (id, script_hash, script_path, job_kind, language, args, tag, created_by, permissioned_as, email, scheduled_for, workspace_id, concurrent_limit, concurrency_time_window_s, timeout, flow_status) - (SELECT uuid, $1, $2, $3, $4, ('{ "uuid": "' || uuid || '" }')::jsonb, $5, $6, $7, $8, $9, $10, $12, $13, $14, $15 FROM uuid_table) + INSERT INTO v2_job_queue + (id, workspace_id, scheduled_for, tag) + (SELECT uuid, $1, $2, $3 FROM uuid_table) RETURNING id"#, - hash.map(|h| h.0), - path, - job_kind.clone() as JobKind, - language as ScriptLang, - tag, - authed.username, - username_to_permissioned_as(&authed.username), - authed.email, - Utc::now(), - w_id, + w_id, + Utc::now(), + tag, + &uuids + ) + .fetch_all(&mut *tx) + .await?; + + sqlx::query!( + "INSERT INTO v2_job_runtime (id, ping) SELECT unnest($1::uuid[]), null", + &uuids, + ) + .execute(&mut *tx) + .await?; + + if let Some(flow_status) = flow_status { + sqlx::query!( + "INSERT INTO v2_job_status (id, flow_status) + SELECT unnest($1::uuid[]), $2", &uuids, - concurrent_limit, - concurrent_time_window_s, - timeout, - flow_status.map(sqlx::types::Json) as Option> + sqlx::types::Json(flow_status) as sqlx::types::Json ) - .fetch_all(&mut *tx) + .execute(&mut *tx) .await?; + } if let Some(custom_concurrency_key) = custom_concurrency_key { sqlx::query!( @@ -5138,7 +5151,7 @@ async fn get_log_file(Path((_w_id, file_p)): Path<(String, String)>) -> error::R .unwrap(); return Ok(res); } else { - return Err(error::Error::InternalErr(format!( + return Err(error::Error::internal_err(format!( "Error getting bytes from file: {}", file_p ))); @@ -5150,7 +5163,7 @@ async fn get_log_file(Path((_w_id, file_p)): Path<(String, String)>) -> error::R ))); } } else { - return Err(error::Error::InternalErr(format!( + return Err(error::Error::internal_err(format!( "Object store client not present and file not found on server logs volume at {local_file}" ))); } @@ -5162,107 +5175,76 @@ async fn get_log_file(Path((_w_id, file_p)): Path<(String, String)>) -> error::R ))); } -#[derive(Deserialize, sqlx::FromRow)] -pub struct JobUpdateRow { - pub running: bool, - pub logs: Option, - pub mem_peak: Option, - pub flow_status: Option>>, - pub log_offset: Option, - pub created_by: String, -} async fn get_job_update( OptAuthed(opt_authed): OptAuthed, Extension(db): Extension, Path((w_id, job_id)): Path<(String, Uuid)>, - Query(JobUpdateQuery { running, log_offset, get_progress }): Query, -) -> error::JsonResult { - let record = sqlx::query_as::<_, JobUpdateRow>( - "SELECT running, substr(concat(coalesce(queue.logs, ''), job_logs.logs), greatest($1 - job_logs.log_offset, 0)) as logs, mem_peak, - CASE WHEN is_flow_step is true then NULL else flow_status END as flow_status, - job_logs.log_offset + char_length(job_logs.logs) + 1 as log_offset, created_by - FROM queue - LEFT JOIN job_logs ON job_logs.job_id = queue.id - WHERE queue.workspace_id = $2 AND queue.id = $3", + Query(JobUpdateQuery { log_offset, get_progress, running }): Query, +) -> JsonResult { + let record = sqlx::query!( + "SELECT + c.id IS NOT NULL AS completed, + CASE + WHEN q.id IS NOT NULL THEN (CASE WHEN NOT $5 AND q.running THEN true ELSE null END) + ELSE false + END AS running, + SUBSTR(logs, GREATEST($1 - log_offset, 0)) AS logs, + COALESCE(r.memory_peak, c.memory_peak) AS mem_peak, + CASE + -- flow step: + WHEN flow_step_id IS NOT NULL THEN NULL + -- completed: + WHEN c.id IS NOT NULL THEN COALESCE( + c.workflow_as_code_status || c.flow_status, + c.workflow_as_code_status, + c.flow_status + ) + -- not completed: + ELSE COALESCE( + f.workflow_as_code_status || f.flow_status, + f.workflow_as_code_status, + f.flow_status + ) + END AS \"flow_status: sqlx::types::Json>\", + job_logs.log_offset + CHAR_LENGTH(job_logs.logs) + 1 AS log_offset, + created_by AS \"created_by!\", + CASE WHEN $4::BOOLEAN THEN ( + SELECT scalar_int FROM job_stats WHERE job_id = $3 AND metric_id = 'progress_perc' + ) END AS progress + FROM v2_job j + LEFT JOIN v2_job_queue q USING (id) + LEFT JOIN v2_job_runtime r USING (id) + LEFT JOIN v2_job_status f USING (id) + LEFT JOIN v2_job_completed c USING (id) + LEFT JOIN job_logs ON job_logs.job_id = $3 + WHERE j.workspace_id = $2 AND j.id = $3", + log_offset, + &w_id, + job_id, + get_progress.unwrap_or(false), + running, ) - .bind(log_offset) - .bind(&w_id) - .bind(&job_id) .fetch_optional(&db) - .await?; + .await? + .ok_or_else(|| Error::NotFound(format!("Job not found: {}", job_id)))?; - let progress: Option = if get_progress == Some(true) { - sqlx::query_scalar!( - "SELECT scalar_int FROM job_stats WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3", - &w_id, - job_id, - "progress_perc" - ) - .fetch_optional(&db) - .await?.and_then(|inner| inner) - } else { - None - }; - - if let Some(record) = record { - if opt_authed.is_none() && record.created_by != "anonymous" { - return Err(Error::BadRequest( - "As a non logged in user, you can only see jobs ran by anonymous users".to_string(), - )); - } - log_job_view(&db, opt_authed.as_ref(), &w_id, &job_id).await?; - Ok(Json(JobUpdate { - running: if !running && record.running { - Some(true) - } else { - None - }, - log_offset: record.log_offset, - completed: None, - new_logs: record.logs, - mem_peak: record.mem_peak, - progress, - flow_status: record - .flow_status - .map(|x: sqlx::types::Json>| x.0), - })) - } else { - let record = sqlx::query_as::<_, JobUpdateRow>( - "SELECT false as running, substr(concat(coalesce(completed_job.logs, ''), job_logs.logs), greatest($1 - job_logs.log_offset, 0)) as logs, mem_peak, - CASE WHEN is_flow_step is true then NULL else flow_status END as flow_status, - job_logs.log_offset + char_length(job_logs.logs) + 1 as log_offset, created_by - FROM completed_job - LEFT JOIN job_logs ON job_logs.job_id = completed_job.id - WHERE completed_job.workspace_id = $2 AND id = $3", - ) - .bind(log_offset) - .bind(&w_id) - .bind(&job_id) - .fetch_optional(&db) - .await?; - if let Some(record) = record { - if opt_authed.is_none() && record.created_by != "anonymous" { - return Err(Error::BadRequest( - "As a non logged in user, you can only see jobs ran by anonymous users" - .to_string(), - )); - } - log_job_view(&db, opt_authed.as_ref(), &w_id, &job_id).await?; - Ok(Json(JobUpdate { - running: Some(false), - completed: Some(true), - log_offset: record.log_offset, - new_logs: record.logs, - mem_peak: record.mem_peak, - progress, - flow_status: record - .flow_status - .map(|x: sqlx::types::Json>| x.0), - })) - } else { - Err(error::Error::NotFound(format!("Job not found: {}", job_id))) - } + if opt_authed.is_none() && record.created_by != "anonymous" { + return Err(Error::BadRequest( + "As a non logged in user, you can only see jobs ran by anonymous users".to_string(), + )); } + log_job_view(&db, opt_authed.as_ref(), &w_id, &job_id).await?; + Ok(Json(JobUpdate { + running: record.running, + completed: record.completed, + log_offset: record.log_offset, + new_logs: record.logs, + mem_peak: record.mem_peak, + progress: record.progress, + flow_status: record + .flow_status + .map(|x: sqlx::types::Json>| x.0), + })) } pub fn filter_list_completed_query( @@ -5380,7 +5362,7 @@ pub fn list_completed_jobs_query( join_outstanding_wait_times: bool, tags: Option>, ) -> SqlBuilder { - let mut sqlb = SqlBuilder::select_from("completed_job") + let mut sqlb = SqlBuilder::select_from("v2_as_completed_job") .fields(fields) .order_by("created_at", lq.order_desc.unwrap_or(true)) .offset(offset) @@ -5523,20 +5505,10 @@ async fn get_completed_job<'a>( #[derive(FromRow)] pub struct RawResult { pub result: Option>>, - pub flow_status: Option>>, - pub language: Option, + pub result_columns: Option>, pub created_by: Option, } -#[derive(FromRow)] -pub struct RawResultWithSuccess { - pub result: Option>>, - pub flow_status: Option>>, - pub language: Option, - pub success: bool, - pub created_by: String, -} - async fn get_completed_job_result( OptAuthed(opt_authed): OptAuthed, Extension(db): Extension, @@ -5548,27 +5520,38 @@ async fn get_completed_job_result( .map(|authed| get_scope_tags(authed)) .flatten(); let result_o = if let Some(json_path) = json_path { - sqlx::query_as::<_, RawResult>( - "SELECT result #> $3 as result, flow_status, language, created_by FROM completed_job WHERE id = $1 AND workspace_id = $2 AND ($4::text[] IS NULL OR tag = ANY($4))", + sqlx::query_as!( + RawResult, + "SELECT + result #> $3 AS \"result: sqlx::types::Json>\", + result_columns, + created_by AS \"created_by!\" + FROM v2_job_completed c + JOIN v2_job USING (id) + WHERE c.id = $1 AND c.workspace_id = $2 AND ($4::text[] IS NULL OR tag = ANY($4))", + id, + &w_id, + json_path.split(".").collect::>() as Vec<&str>, + tags.as_ref().map(|v| v.as_slice()) as Option<&[&str]>, ) - .bind(id) - .bind(&w_id) - .bind( - json_path - .split(".") - .map(|x| x.to_string()) - .collect::>(), - ) - .bind(tags.as_ref().map(|v| v.as_slice())) .fetch_optional(&db) .await? } else { - sqlx::query_as::<_, RawResult>("SELECT result, flow_status, language, created_by FROM completed_job WHERE id = $1 AND workspace_id = $2 AND ($3::text[] IS NULL OR tag = ANY($3))") - .bind(id) - .bind(&w_id) - .bind(tags.as_ref().map(|v| v.as_slice())) - .fetch_optional(&db) - .await? + sqlx::query_as!( + RawResult, + "SELECT + result AS \"result: sqlx::types::Json>\", + result_columns, + created_by AS \"created_by!\" + FROM v2_job_completed c + JOIN v2_job USING (id) + WHERE c.id = $1 AND c.workspace_id = $2 AND ($3::text[] IS NULL OR tag = ANY($3))", + id, + &w_id, + tags.as_ref().map(|v| v.as_slice()) as Option<&[&str]>, + ) + .fetch_optional(&db) + .await? }; let mut raw_result = not_found_if_none(result_o, "Completed Job", id.to_string())?; @@ -5579,7 +5562,7 @@ async fn get_completed_job_result( let mut parent_job = id; while parent_job != suspended_job { let p_job = sqlx::query_scalar!( - "SELECT parent_job FROM queue WHERE id = $1 AND workspace_id = $2", + "SELECT parent_job FROM v2_job WHERE id = $1 AND workspace_id = $2", parent_job, &w_id ) @@ -5612,8 +5595,7 @@ async fn get_completed_job_result( } format_result( - raw_result.language.as_ref(), - raw_result.flow_status.as_ref(), + raw_result.result_columns.as_ref(), raw_result.result.as_mut(), ); @@ -5646,7 +5628,7 @@ async fn count_by_tag( TagCount, r#" SELECT tag as "tag!", COUNT(*) as "count!" - FROM completed_job + FROM v2_as_completed_job WHERE started_at > NOW() - make_interval(secs => $1) AND ($2::text IS NULL OR workspace_id = $2) GROUP BY tag ORDER BY "count!" DESC @@ -5683,21 +5665,24 @@ async fn get_completed_job_result_maybe( .as_ref() .map(|authed| get_scope_tags(authed)) .flatten(); - let result_o = sqlx::query_as::<_, RawResultWithSuccess>( - "SELECT result, success, language, flow_status, created_by FROM completed_job WHERE id = $1 AND workspace_id = $2 AND ($3::text[] IS NULL OR tag = ANY($3))", + let result_o = sqlx::query!( + "SELECT + result AS \"result: sqlx::types::Json>\", + result_columns, + status = 'success' AS \"success!\", + created_by AS \"created_by!\" + FROM v2_job_completed c + JOIN v2_job j USING (id) + WHERE c.id = $1 AND c.workspace_id = $2 AND ($3::text[] IS NULL OR tag = ANY($3))", + id, + &w_id, + tags.as_ref().map(|v| v.as_slice()) as Option<&[&str]>, ) - .bind(id) - .bind(&w_id) - .bind(tags.as_ref().map(|v| v.as_slice())) .fetch_optional(&db) .await?; if let Some(mut res) = result_o { - format_result( - res.language.as_ref(), - res.flow_status.as_ref(), - res.result.as_mut(), - ); + format_result(res.result_columns.as_ref(), res.result.as_mut()); if opt_authed.is_none() && res.created_by != "anonymous" { return Err(Error::BadRequest( "As a non logged in user, you can only see jobs ran by anonymous users".to_string(), @@ -5715,7 +5700,7 @@ async fn get_completed_job_result_maybe( .into_response()) } else if get_started.is_some_and(|x| x) { let started = sqlx::query_scalar!( - "SELECT running FROM queue WHERE id = $1 AND workspace_id = $2", + "SELECT running AS \"running!\" FROM v2_job_queue WHERE id = $1 AND workspace_id = $2", id, w_id ) @@ -5752,8 +5737,17 @@ async fn delete_completed_job<'a>( require_admin(authed.is_admin, &authed.username)?; let tags = get_scope_tags(&authed); let job_o = sqlx::query_as::<_, CompletedJob>( - "UPDATE completed_job SET args = null, logs = '', result = null, deleted = true WHERE id = $1 AND workspace_id = $2 AND ($3::text[] IS NULL OR tag = ANY($3)) \ - RETURNING *, null as labels", + "WITH mark_as_deleted AS ( + UPDATE v2_job_completed c SET + result = NULL, + deleted = TRUE + FROM v2_job j + WHERE c.id = $1 + AND j.id = c.id + AND c.workspace_id = $2 + AND ($3::TEXT[] IS NULL OR tag = ANY($3)) + RETURNING c.id + ) SELECT * FROM v2_as_completed_job WHERE id = (SELECT id FROM mark_as_deleted)", ) .bind(id) .bind(&w_id) @@ -5763,6 +5757,9 @@ async fn delete_completed_job<'a>( let cj = not_found_if_none(job_o, "Completed Job", id.to_string())?; + sqlx::query!("UPDATE v2_job SET args = NULL WHERE id = $1", id) + .execute(&mut *tx) + .await?; sqlx::query!("DELETE FROM job_logs WHERE job_id = $1", id) .execute(&mut *tx) .await?; diff --git a/backend/windmill-api/src/kafka_triggers_ee.rs b/backend/windmill-api/src/kafka_triggers_ee.rs index d02ea970e3..de50eea53b 100644 --- a/backend/windmill-api/src/kafka_triggers_ee.rs +++ b/backend/windmill-api/src/kafka_triggers_ee.rs @@ -18,3 +18,25 @@ pub fn start_kafka_consumers( #[derive(Serialize, Deserialize)] pub enum KafkaTriggerConfigConnection {} + +#[derive(Serialize, Clone)] +pub struct KafkaTrigger { + pub workspace_id: String, + pub path: String, + pub kafka_resource_path: String, + pub group_id: String, + pub topics: Vec, + pub script_path: String, + pub is_flow: bool, + pub edited_by: String, + pub email: String, + pub edited_at: chrono::DateTime, + #[serde(skip_serializing_if = "Option::is_none")] + pub server_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub last_server_ping: Option>, + pub extra_perms: serde_json::Value, + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, + pub enabled: bool, +} diff --git a/backend/windmill-api/src/nats_triggers_ee.rs b/backend/windmill-api/src/nats_triggers_ee.rs index bfa5fbdd73..92894a2a34 100644 --- a/backend/windmill-api/src/nats_triggers_ee.rs +++ b/backend/windmill-api/src/nats_triggers_ee.rs @@ -15,3 +15,29 @@ pub fn start_nats_consumers(_db: DB, mut _killpill_rx: tokio::sync::broadcast::R #[derive(Serialize, Deserialize)] pub enum NatsTriggerConfigConnection {} + +#[derive(Serialize, Clone)] +pub struct NatsTrigger { + pub workspace_id: String, + pub path: String, + pub nats_resource_path: String, + pub subjects: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub stream_name: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub consumer_name: Option, + pub use_jetstream: bool, + pub script_path: String, + pub is_flow: bool, + pub edited_by: String, + pub email: String, + pub edited_at: chrono::DateTime, + #[serde(skip_serializing_if = "Option::is_none")] + pub server_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub last_server_ping: Option>, + pub extra_perms: serde_json::Value, + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, + pub enabled: bool, +} diff --git a/backend/windmill-api/src/postgres_triggers/handler.rs b/backend/windmill-api/src/postgres_triggers/handler.rs index 5bcfb69acc..dcf350bd91 100644 --- a/backend/windmill-api/src/postgres_triggers/handler.rs +++ b/backend/windmill-api/src/postgres_triggers/handler.rs @@ -153,7 +153,7 @@ pub async fn get_raw_postgres_connection(db: &Database) -> Result, + #[serde(skip_serializing_if = "Option::is_none")] pub extra_perms: Option, pub postgres_resource_path: String, + #[serde(skip_serializing_if = "Option::is_none")] pub error: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub server_id: Option, pub replication_slot_name: String, pub publication_name: String, + #[serde(skip_serializing_if = "Option::is_none")] pub last_server_ping: Option>, pub enabled: bool, } @@ -651,7 +655,7 @@ pub async fn get_publication_info( let (all_table, transaction_to_track) = match publication_data { Ok(pub_data) => pub_data, - Err(Error::SqlErr(sqlx::Error::RowNotFound)) => { + Err(Error::SqlErr { error: sqlx::Error::RowNotFound, .. }) => { return Err(Error::NotFound( "Publication was not found, please create a new publication".to_string(), )) @@ -1378,7 +1382,7 @@ pub async fn create_template_script( let rows: Vec = sqlx::query_as(&query) .fetch_all(&mut connection) .await - .map_err(error::Error::SqlErr)?; + .map_err(|e| error::Error::SqlErr { error: e, location: "pg_trigger".to_string() })?; let mut mapper: HashMap>> = HashMap::new(); diff --git a/backend/windmill-api/src/postgres_triggers/mod.rs b/backend/windmill-api/src/postgres_triggers/mod.rs index 0bd87ed262..12d7b558e0 100644 --- a/backend/windmill-api/src/postgres_triggers/mod.rs +++ b/backend/windmill-api/src/postgres_triggers/mod.rs @@ -16,7 +16,7 @@ use handler::{ create_template_script, delete_postgres_trigger, delete_publication, drop_slot_name, exists_postgres_trigger, get_postgres_trigger, get_publication_info, get_template_script, is_database_in_logical_level, list_database_publication, list_postgres_triggers, - list_slot_name, set_enabled, update_postgres_trigger, Database, PostgresTrigger, + list_slot_name, set_enabled, update_postgres_trigger, Database, }; use windmill_common::{db::UserDB, error::Error, utils::StripPath}; use windmill_queue::PushArgsOwned; @@ -30,6 +30,7 @@ mod relation; mod replication_message; mod trigger; +pub use handler::PostgresTrigger; pub use trigger::start_database; pub async fn get_database_resource( @@ -52,7 +53,7 @@ pub async fn get_database_resource( .map_err(|_| Error::NotFound("Database resource do not exist".to_string()))?; let resource = match resource { - Some(resource) => serde_json::from_value::(resource).map_err(Error::SerdeJson)?, + Some(resource) => serde_json::from_value::(resource)?, None => { return { Err(Error::NotFound( diff --git a/backend/windmill-api/src/raw_apps.rs b/backend/windmill-api/src/raw_apps.rs index 59f2471c4d..bb2763b73d 100644 --- a/backend/windmill-api/src/raw_apps.rs +++ b/backend/windmill-api/src/raw_apps.rs @@ -110,7 +110,7 @@ async fn list_apps( sqlb.and_where_eq("app.path", "?".bind(path_exact)); } - let sql = sqlb.sql().map_err(|e| Error::InternalErr(e.to_string()))?; + let sql = sqlb.sql().map_err(|e| Error::internal_err(e.to_string()))?; let mut tx = user_db.begin(&authed).await?; let rows = sqlx::query_as::<_, ListableApp>(&sql) .fetch_all(&mut *tx) @@ -293,7 +293,7 @@ async fn update_app( sqlb.returning("path"); - let sql = sqlb.sql().map_err(|e| Error::InternalErr(e.to_string()))?; + let sql = sqlb.sql().map_err(|e| Error::internal_err(e.to_string()))?; let npath_o: Option = sqlx::query_scalar(&sql).fetch_optional(&mut *tx).await?; not_found_if_none(npath_o, "Raw App", path)?; diff --git a/backend/windmill-api/src/resources.rs b/backend/windmill-api/src/resources.rs index 88f8d79c29..f3f2285cd0 100644 --- a/backend/windmill-api/src/resources.rs +++ b/backend/windmill-api/src/resources.rs @@ -31,7 +31,6 @@ use windmill_audit::ActionKind; use windmill_common::{ db::UserDB, error::{Error, JsonResult, Result}, - jobs::QueuedJob, utils::{not_found_if_none, paginate, require_admin, Pagination, StripPath}, variables, }; @@ -262,7 +261,7 @@ async fn list_resources( sqlb.and_where_like_left("resource.path", path_start); } - let sql = sqlb.sql().map_err(|e| Error::InternalErr(e.to_string()))?; + let sql = sqlb.sql().map_err(|e| Error::internal_err(e.to_string()))?; let mut tx = user_db.begin(&authed).await?; let rows = sqlx::query_as::<_, ListableResource>(&sql) .fetch_all(&mut *tx) @@ -516,7 +515,7 @@ pub async fn transform_json_value<'c>( Value::String(y) if y.starts_with("$res:") => { let path = y.strip_prefix("$res:").unwrap(); if path.split("/").count() < 2 { - return Err(Error::InternalErr(format!("Invalid resource path: {path}"))); + return Err(Error::internal_err(format!("Invalid resource path: {path}"))); } let mut tx: Transaction<'_, Postgres> = authed_transaction_or_default(authed, user_db.clone(), db).await?; @@ -537,21 +536,28 @@ pub async fn transform_json_value<'c>( } Value::String(y) if y.starts_with("$") && job_id.is_some() => { let mut tx = authed_transaction_or_default(authed, user_db.clone(), db).await?; - let job = sqlx::query_as::<_, QueuedJob>( - "SELECT * FROM queue WHERE id = $1 AND workspace_id = $2", + let job_id = job_id.unwrap(); + let job = sqlx::query!( + "SELECT + email AS \"email!\", + created_by AS \"created_by!\", + parent_job, permissioned_as AS \"permissioned_as!\", + script_path, schedule_path, flow_step_id, root_job, + scheduled_for AS \"scheduled_for!: chrono::DateTime\" + FROM v2_as_queue WHERE id = $1 AND workspace_id = $2", + job_id, + workspace ) - .bind(job_id.unwrap()) - .bind(workspace) .fetch_optional(&mut *tx) .await?; tx.commit().await?; - let job = not_found_if_none(job, "Job", job_id.unwrap().to_string())?; + let job = not_found_if_none(job, "Job", job_id.to_string())?; let flow_path = if let Some(uuid) = job.parent_job { let mut tx: Transaction<'_, Postgres> = authed_transaction_or_default(authed, user_db.clone(), db).await?; - let p = sqlx::query_scalar!("SELECT script_path FROM queue WHERE id = $1", uuid) + let p = sqlx::query_scalar!("SELECT runnable_path FROM v2_job WHERE id = $1", uuid) .fetch_optional(&mut *tx) .await? .flatten(); @@ -563,11 +569,11 @@ pub async fn transform_json_value<'c>( let variables = variables::get_reserved_variables( db, - &job.workspace_id, + workspace, token, &job.email, &job.created_by, - &job.id.to_string(), + &job_id.to_string(), &job.permissioned_as, job.script_path.clone(), job.parent_job.map(|x| x.to_string()), @@ -815,7 +821,7 @@ async fn update_resource( } } - let sql = sqlb.sql().map_err(|e| Error::InternalErr(e.to_string()))?; + let sql = sqlb.sql().map_err(|e| Error::internal_err(e.to_string()))?; let npath_o: Option = sqlx::query_scalar(&sql).fetch_optional(&mut *tx).await?; let npath = not_found_if_none(npath_o, "Resource", path)?; @@ -1165,7 +1171,7 @@ async fn update_resource_type( sqlb.set_str("description", ndesc); } sqlb.set_str("edited_at", "now()"); - let sql = sqlb.sql().map_err(|e| Error::InternalErr(e.to_string()))?; + let sql = sqlb.sql().map_err(|e| Error::internal_err(e.to_string()))?; let mut tx = user_db.begin(&authed).await?; sqlx::query(&sql).execute(&mut *tx).await?; diff --git a/backend/windmill-api/src/schedule.rs b/backend/windmill-api/src/schedule.rs index 79e8c4dcb4..109957da04 100644 --- a/backend/windmill-api/src/schedule.rs +++ b/backend/windmill-api/src/schedule.rs @@ -196,7 +196,7 @@ async fn create_schedule( .bind(&ns.cron_version.unwrap_or("v2".to_string())) .fetch_one(&mut *tx) .await - .map_err(|e| Error::InternalErr(format!("inserting schedule in {w_id}: {e:#}")))?; + .map_err(|e| Error::internal_err(format!("inserting schedule in {w_id}: {e:#}")))?; handle_deployment_metadata( &authed.email, @@ -282,7 +282,7 @@ async fn edit_schedule( .bind(&es.cron_version) .fetch_one(&mut *tx) .await - .map_err(|e| Error::InternalErr(format!("updating schedule in {w_id}: {e:#}")))?; + .map_err(|e| Error::internal_err(format!("updating schedule in {w_id}: {e:#}")))?; handle_deployment_metadata( &authed.email, @@ -356,7 +356,7 @@ async fn list_schedule( if let Some(path_start) = &lsq.path_start { sqlb.and_where_like_left("path", path_start); } - let sql = sqlb.sql().map_err(|e| Error::InternalErr(e.to_string()))?; + let sql = sqlb.sql().map_err(|e| Error::internal_err(e.to_string()))?; let rows = sqlx::query_as::<_, Schedule>(&sql) .fetch_all(&mut *tx) .await?; @@ -407,8 +407,8 @@ async fn list_schedule_with_jobs( let mut tx = user_db.begin(&authed).await?; let (per_page, offset) = paginate(pagination); let rows = sqlx::query_as!(ScheduleWJobs, - "SELECT schedule.*, t.jobs FROM schedule, LATERAL ( SELECT ARRAY (SELECT json_build_object('id', id, 'success', success, 'duration_ms', duration_ms) FROM completed_job WHERE - completed_job.schedule_path = schedule.path AND completed_job.workspace_id = $1 AND parent_job IS NULL AND is_skipped = False ORDER BY started_at DESC LIMIT 20) AS jobs ) t + "SELECT schedule.*, t.jobs FROM schedule, LATERAL ( SELECT ARRAY (SELECT json_build_object('id', id, 'success', success, 'duration_ms', duration_ms) FROM v2_as_completed_job WHERE + v2_as_completed_job.schedule_path = schedule.path AND v2_as_completed_job.workspace_id = $1 AND parent_job IS NULL AND is_skipped = False ORDER BY started_at DESC LIMIT 20) AS jobs ) t WHERE schedule.workspace_id = $1 ORDER BY schedule.edited_at desc LIMIT $2 OFFSET $3", w_id, per_page as i64, @@ -832,17 +832,30 @@ pub struct EditSchedule { } pub async fn clear_schedule<'c>( - db: &mut Transaction<'c, Postgres>, + tx: &mut Transaction<'c, Postgres>, path: &str, w_id: &str, ) -> Result<()> { tracing::info!("Clearing schedule {}", path); sqlx::query!( - "DELETE FROM queue WHERE schedule_path = $1 AND running = false AND workspace_id = $2 AND is_flow_step = false", + "WITH to_delete AS ( + SELECT id FROM v2_job_queue + JOIN v2_job j USING (id) + WHERE trigger_kind = 'schedule' + AND trigger = $1 + AND j.workspace_id = $2 + AND flow_step_id IS NULL + AND running = false + FOR UPDATE + ), deleted AS ( + DELETE FROM v2_job_queue + WHERE id IN (SELECT id FROM to_delete) + RETURNING id + ) DELETE FROM v2_job WHERE id IN (SELECT id FROM deleted)", path, w_id ) - .execute(&mut **db) + .execute(&mut **tx) .await?; Ok(()) } diff --git a/backend/windmill-api/src/scripts.rs b/backend/windmill-api/src/scripts.rs index 10ac5aa387..cbeb13e106 100644 --- a/backend/windmill-api/src/scripts.rs +++ b/backend/windmill-api/src/scripts.rs @@ -311,7 +311,7 @@ async fn list_scripts( .fields(&["dm.deployment_msg"]); } - let sql = sqlb.sql().map_err(|e| Error::InternalErr(e.to_string()))?; + let sql = sqlb.sql().map_err(|e| Error::internal_err(e.to_string()))?; let mut tx = user_db.begin(&authed).await?; let rows = sqlx::query_as::<_, ListableScript>(&sql) .fetch_all(&mut *tx) @@ -1337,7 +1337,7 @@ async fn archive_script_by_path( ) .fetch_one(&db) .await - .map_err(|e| Error::InternalErr(format!("archiving script in {w_id}: {e:#}")))?; + .map_err(|e| Error::internal_err(format!("archiving script in {w_id}: {e:#}")))?; audit_log( &mut *tx, &authed, @@ -1387,7 +1387,7 @@ async fn archive_script_by_hash( .bind(&hash.0) .fetch_one(&mut *tx) .await - .map_err(|e| Error::InternalErr(format!("archiving script in {w_id}: {e:#}")))?; + .map_err(|e| Error::internal_err(format!("archiving script in {w_id}: {e:#}")))?; audit_log( &mut *tx, @@ -1427,7 +1427,7 @@ async fn delete_script_by_hash( .bind(&w_id) .fetch_one(&db) .await - .map_err(|e| Error::InternalErr(format!("deleting script by hash {w_id}: {e:#}")))?; + .map_err(|e| Error::internal_err(format!("deleting script by hash {w_id}: {e:#}")))?; audit_log( &mut *tx, @@ -1487,7 +1487,7 @@ async fn delete_script_by_path( ) .fetch_one(&db) .await - .map_err(|e| Error::InternalErr(format!("deleting script by path {w_id}: {e:#}")))? + .map_err(|e| Error::internal_err(format!("deleting script by path {w_id}: {e:#}")))? } else { // If the script is draft only, we can delete it without admin permissions but we still need write permissions sqlx::query_scalar!( @@ -1497,7 +1497,7 @@ async fn delete_script_by_path( ) .fetch_one(&mut *tx) .await - .map_err(|e| Error::InternalErr(format!("deleting script by path {w_id}: {e:#}")))? + .map_err(|e| Error::internal_err(format!("deleting script by path {w_id}: {e:#}")))? }; sqlx::query!( @@ -1559,7 +1559,7 @@ async fn delete_script_by_path( .execute(&db) .await .map_err(|e| { - Error::InternalErr(format!( + Error::internal_err(format!( "error deleting deployment metadata for script with path {path} in workspace {w_id}: {e:#}" )) })?; diff --git a/backend/windmill-api/src/service_logs.rs b/backend/windmill-api/src/service_logs.rs index 277a08bca9..b11646fbe0 100644 --- a/backend/windmill-api/src/service_logs.rs +++ b/backend/windmill-api/src/service_logs.rs @@ -83,7 +83,7 @@ async fn list_files( if let Some(true) = lq.with_error { sqlb.and_where("err_lines > 0"); } - let sql = sqlb.sql().map_err(|e| Error::InternalErr(e.to_string()))?; + let sql = sqlb.sql().map_err(|e| Error::internal_err(e.to_string()))?; let rows = sqlx::query_as::<_, LogFile>(&sql).fetch_all(&db).await?; Ok(Json(rows)) } @@ -114,7 +114,7 @@ async fn get_log_file( return Ok(content_plain(Body::from(bytes::Bytes::from(bytes)))); } Err(e) => { - return Err(Error::InternalErr(format!( + return Err(Error::internal_err(format!( "Error pulling the bytes: {}", e ))); @@ -122,7 +122,7 @@ async fn get_log_file( } } Err(e) => { - return Err(Error::InternalErr(format!( + return Err(Error::internal_err(format!( "Error fetching the file: {}", e ))); diff --git a/backend/windmill-api/src/settings.rs b/backend/windmill-api/src/settings.rs index c527eb481a..37a33ee81a 100644 --- a/backend/windmill-api/src/settings.rs +++ b/backend/windmill-api/src/settings.rs @@ -132,7 +132,7 @@ pub async fn test_s3_bucket( if first_file.is_some() { if let Err(e) = first_file.as_ref().unwrap() { tracing::error!("error listing bucket: {e:#}"); - error::Error::InternalErr(format!("Failed to list files in blob storage: {e:#}")); + error::Error::internal_err(format!("Failed to list files in blob storage: {e:#}")); } tracing::info!("Listed files: {:?}", first_file.unwrap()); } else { @@ -156,7 +156,7 @@ pub async fn test_s3_bucket( .await .map_err(to_anyhow)?; if content != Bytes::from_static(b"hello") { - return Err(error::Error::InternalErr( + return Err(error::Error::internal_err( "Failed to read back from blob storage".to_string(), )); } @@ -232,7 +232,7 @@ pub async fn set_global_setting_internal( generate_instance_username_for_all_users(db) .await .map_err(|err| { - error::Error::InternalErr(format!( + error::Error::internal_err(format!( "Failed to generate instance wide usernames: {}", err )) @@ -349,7 +349,7 @@ pub async fn get_latest_key_renewal_attempt( Some(last_attempt) => { let last_attempt_result = serde_json::from_value::(last_attempt.value) .map_err(|e| { - error::Error::InternalErr(format!("Failed to parse last attempt: {}", e)) + error::Error::internal_err(format!("Failed to parse last attempt: {}", e)) })?; Ok(Json(Some(KeyRenewalAttempt { result: last_attempt_result, diff --git a/backend/windmill-api/src/slack_approvals.rs b/backend/windmill-api/src/slack_approvals.rs index 8d39131e7d..652c7b1721 100644 --- a/backend/windmill-api/src/slack_approvals.rs +++ b/backend/windmill-api/src/slack_approvals.rs @@ -977,17 +977,17 @@ async fn get_modal_blocks( let (job_kind, script_hash, raw_flow, parent_job_id, created_at, created_by, script_path, args) = sqlx::query!( "SELECT - queue.job_kind AS \"job_kind: JobKind\", - queue.script_hash AS \"script_hash: ScriptHash\", - queue.raw_flow AS \"raw_flow: sqlx::types::Json>\", - completed_job.parent_job AS \"parent_job: Uuid\", - completed_job.created_at AS \"created_at: chrono::NaiveDateTime\", - completed_job.created_by AS \"created_by!\", - queue.script_path, - queue.args AS \"args: sqlx::types::Json>\" - FROM queue - JOIN completed_job ON completed_job.parent_job = queue.id - WHERE completed_job.id = $1 AND completed_job.workspace_id = $2 + v2_as_queue.job_kind AS \"job_kind!: JobKind\", + v2_as_queue.script_hash AS \"script_hash: ScriptHash\", + v2_as_queue.raw_flow AS \"raw_flow: sqlx::types::Json>\", + v2_as_completed_job.parent_job AS \"parent_job: Uuid\", + v2_as_completed_job.created_at AS \"created_at!: chrono::NaiveDateTime\", + v2_as_completed_job.created_by AS \"created_by!\", + v2_as_queue.script_path, + v2_as_queue.args AS \"args: sqlx::types::Json>\" + FROM v2_as_queue + JOIN v2_as_completed_job ON v2_as_completed_job.parent_job = v2_as_queue.id + WHERE v2_as_completed_job.id = $1 AND v2_as_completed_job.workspace_id = $2 LIMIT 1", job_id, &w_id diff --git a/backend/windmill-api/src/tracing_init.rs b/backend/windmill-api/src/tracing_init.rs index d099eb1598..9f34e93592 100644 --- a/backend/windmill-api/src/tracing_init.rs +++ b/backend/windmill-api/src/tracing_init.rs @@ -31,7 +31,7 @@ impl OnResponse for MyOnResponse { if *LOG_REQUESTS { let latency = latency.as_millis(); let status = response.status().as_u16(); - if response.status().is_success() { + if response.status().is_success() || response.status().is_redirection() { tracing::info!(latency = latency, status = status, "response") } else { tracing::error!(latency = latency, status = status, "response") diff --git a/backend/windmill-api/src/users.rs b/backend/windmill-api/src/users.rs index b603259664..71bfeb2c2a 100644 --- a/backend/windmill-api/src/users.rs +++ b/backend/windmill-api/src/users.rs @@ -10,6 +10,7 @@ use std::sync::atomic::AtomicBool; use std::sync::Arc; +use std::time::Duration; use crate::db::ApiAuthed; @@ -464,14 +465,16 @@ async fn list_user_usage( require_admin(authed.is_admin, &authed.username)?; } let mut tx = user_db.begin(&authed).await?; - let rows = sqlx::query_as!( - UserWithUsage, - " + let rows = tokio::time::timeout( + Duration::from_secs(300), + sqlx::query_as!( + UserWithUsage, + " SELECT usr.email, usage.executions FROM usr , LATERAL ( SELECT COALESCE(SUM(duration_ms + 1000)/1000 , 0)::BIGINT executions - FROM completed_job + FROM v2_as_completed_job WHERE workspace_id = $1 AND job_kind NOT IN ('flow', 'flowpreview', 'flownode') AND email = usr.email @@ -479,10 +482,12 @@ async fn list_user_usage( ) usage WHERE workspace_id = $1 ", - w_id + w_id + ) + .fetch_all(&mut *tx), ) - .fetch_all(&mut *tx) - .await?; + .await + .map_err(|e| Error::internal_err(format!("Timed out while fetching user usage: {e:#}")))??; tx.commit().await?; Ok(Json(rows)) } @@ -690,7 +695,7 @@ async fn global_whoami( ) .fetch_one(&db) .await - .map_err(|e| Error::InternalErr(format!("fetching global identity: {e:#}"))); + .map_err(|e| Error::internal_err(format!("fetching global identity: {e:#}"))); if let Ok(user) = user { Ok(Json(user)) @@ -1586,7 +1591,7 @@ async fn login( if let Some((email, hash, super_admin, first_time_user)) = email_w_h { let parsed_hash = - PasswordHash::new(&hash).map_err(|e| Error::InternalErr(e.to_string()))?; + PasswordHash::new(&hash).map_err(|e| Error::internal_err(e.to_string()))?; if argon2 .verify_password(password.as_bytes(), &parsed_hash) .is_err() @@ -2310,25 +2315,9 @@ async fn update_username_in_workpsace<'c>( .execute(&mut **tx) .await?; - // ---- queue ---- + // ---- v2_job ---- sqlx::query!( - r#"UPDATE queue SET script_path = REGEXP_REPLACE(script_path,'u/' || $2 || '/(.*)','u/' || $1 || '/\1') WHERE script_path LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#, - new_username, - old_username, - w_id - ).execute(&mut **tx) - .await?; - - sqlx::query!( - r#"UPDATE queue SET schedule_path = REGEXP_REPLACE(schedule_path,'u/' || $2 || '/(.*)','u/' || $1 || '/\1') WHERE schedule_path LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#, - new_username, - old_username, - w_id - ).execute(&mut **tx) - .await?; - - sqlx::query!( - "UPDATE queue SET permissioned_as = ('u/' || $1) WHERE permissioned_as = ('u/' || $2) AND workspace_id = $3", + r#"UPDATE v2_job SET runnable_path = REGEXP_REPLACE(runnable_path,'u/' || $2 || '/(.*)','u/' || $1 || '/\1') WHERE runnable_path LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#, new_username, old_username, w_id @@ -2337,44 +2326,7 @@ async fn update_username_in_workpsace<'c>( .await?; sqlx::query!( - "UPDATE queue SET canceled_by = $1 WHERE canceled_by = $2 AND workspace_id = $3", - new_username, - old_username, - w_id - ) - .execute(&mut **tx) - .await - .unwrap(); - - sqlx::query!( - "UPDATE queue SET created_by = $1 WHERE created_by = $2 AND workspace_id = $3", - new_username, - old_username, - w_id - ) - .execute(&mut **tx) - .await - .unwrap(); - - // ---- completed_job ---- - sqlx::query!( - r#"UPDATE completed_job SET script_path = REGEXP_REPLACE(script_path,'u/' || $2 || '/(.*)','u/' || $1 || '/\1') WHERE script_path LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#, - new_username, - old_username, - w_id - ).execute(&mut **tx) - .await?; - - sqlx::query!( - r#"UPDATE completed_job SET schedule_path = REGEXP_REPLACE(schedule_path,'u/' || $2 || '/(.*)','u/' || $1 || '/\1') WHERE schedule_path LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#, - new_username, - old_username, - w_id - ).execute(&mut **tx) - .await?; - - sqlx::query!( - "UPDATE completed_job SET permissioned_as = ('u/' || $1) WHERE permissioned_as = ('u/' || $2) AND workspace_id = $3", + r#"UPDATE v2_job SET trigger = REGEXP_REPLACE(trigger,'u/' || $2 || '/(.*)','u/' || $1 || '/\1') WHERE trigger LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#, new_username, old_username, w_id @@ -2383,24 +2335,42 @@ async fn update_username_in_workpsace<'c>( .await?; sqlx::query!( - "UPDATE completed_job SET created_by = $1 WHERE created_by = $2 AND workspace_id = $3", + "UPDATE v2_job SET permissioned_as = ('u/' || $1) WHERE permissioned_as = ('u/' || $2) AND workspace_id = $3", new_username, old_username, w_id ) .execute(&mut **tx) - .await - .unwrap(); + .await?; sqlx::query!( - "UPDATE completed_job SET canceled_by = $1 WHERE canceled_by = $2 AND workspace_id = $3", + "UPDATE v2_job SET created_by = $1 WHERE created_by = $2 AND workspace_id = $3", new_username, old_username, w_id ) .execute(&mut **tx) - .await - .unwrap(); + .await?; + + // ---- v2_job_queue ---- + sqlx::query!( + "UPDATE v2_job_queue SET canceled_by = $1 WHERE canceled_by = $2 AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + + // ---- v2_job_completed ---- + sqlx::query!( + "UPDATE v2_job_completed SET canceled_by = $1 WHERE canceled_by = $2 AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; // ---- resources---- sqlx::query!( diff --git a/backend/windmill-api/src/users_ee.rs b/backend/windmill-api/src/users_ee.rs index 0e20d5bf84..7a11239a2f 100644 --- a/backend/windmill-api/src/users_ee.rs +++ b/backend/windmill-api/src/users_ee.rs @@ -17,7 +17,7 @@ pub async fn create_user( _argon2: Arc>, mut _nu: NewUser, ) -> Result<(StatusCode, String)> { - Err(Error::InternalErr( + Err(Error::internal_err( "Not implemented in Windmill's Open Source repository".to_string(), )) } @@ -29,7 +29,7 @@ pub async fn set_password( _user_email: &str, _ep: EditPassword, ) -> Result { - Err(Error::InternalErr( + Err(Error::internal_err( "Not implemented in Windmill's Open Source repository".to_string(), )) } diff --git a/backend/windmill-api/src/utils.rs b/backend/windmill-api/src/utils.rs index 5e7adde1e6..439944e2b2 100644 --- a/backend/windmill-api/src/utils.rs +++ b/backend/windmill-api/src/utils.rs @@ -76,7 +76,7 @@ pub async fn generate_instance_wide_unique_username<'c>( let mut i = 1; while username_conflict { if i > 1000 { - return Err(Error::InternalErr(format!( + return Err(Error::internal_err(format!( "too many username conflicts for {}", email ))); @@ -168,7 +168,7 @@ pub async fn get_instance_username_or_create_pending<'c>( ) .execute(&mut **tx) .await - .map_err(|e| Error::InternalErr(format!("creating pending user: {e:#}")))?; + .map_err(|e| Error::internal_err(format!("creating pending user: {e:#}")))?; Ok(username) } diff --git a/backend/windmill-api/src/variables.rs b/backend/windmill-api/src/variables.rs index 48800621b6..37960e8583 100644 --- a/backend/windmill-api/src/variables.rs +++ b/backend/windmill-api/src/variables.rs @@ -196,7 +196,7 @@ async fn get_variable( ) } #[cfg(not(feature = "oauth2"))] - return Err(Error::InternalErr("Require oauth2 feature".to_string())); + return Err(Error::internal_err("Require oauth2 feature".to_string())); } else if !value.is_empty() && decrypt_secret { let _ = tx.commit().await; let mc = build_crypt(&db, &w_id).await?; @@ -558,7 +558,7 @@ async fn update_variable( } } - let sql = sqlb.sql().map_err(|e| Error::InternalErr(e.to_string()))?; + let sql = sqlb.sql().map_err(|e| Error::internal_err(e.to_string()))?; let npath_o: Option = sqlx::query_scalar(&sql).fetch_optional(&mut *tx).await?; @@ -662,7 +662,7 @@ pub async fn get_value_internal<'c>( .await? } #[cfg(not(feature = "oauth2"))] - return Err(Error::InternalErr("Require oauth2 feature".to_string())); + return Err(Error::internal_err("Require oauth2 feature".to_string())); } else if !value.is_empty() { tx.commit().await?; let mc = build_crypt(&db, &w_id).await?; diff --git a/backend/windmill-api/src/websocket_triggers.rs b/backend/windmill-api/src/websocket_triggers.rs index f90bb8def0..4de01f14f3 100644 --- a/backend/windmill-api/src/websocket_triggers.rs +++ b/backend/windmill-api/src/websocket_triggers.rs @@ -88,23 +88,27 @@ enum InitialMessage { #[derive(FromRow, Serialize, Clone)] pub struct WebsocketTrigger { - workspace_id: String, - path: String, - url: String, - script_path: String, - is_flow: bool, - edited_by: String, - email: String, - edited_at: chrono::DateTime, - server_id: Option, - last_server_ping: Option>, - extra_perms: serde_json::Value, - error: Option, - enabled: bool, - filters: Vec>>, - initial_messages: Option>>>, - url_runnable_args: Option>>, - can_return_message: bool, + pub workspace_id: String, + pub path: String, + pub url: String, + pub script_path: String, + pub is_flow: bool, + pub edited_by: String, + pub email: String, + pub edited_at: chrono::DateTime, + #[serde(skip_serializing_if = "Option::is_none")] + pub server_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub last_server_ping: Option>, + pub extra_perms: serde_json::Value, + pub error: Option, + pub enabled: bool, + pub filters: Vec>>, + #[serde(skip_serializing_if = "Option::is_none")] + pub initial_messages: Option>>>, + #[serde(skip_serializing_if = "Option::is_none")] + pub url_runnable_args: Option>>, + pub can_return_message: bool, } #[derive(Deserialize)] @@ -154,7 +158,7 @@ async fn list_websocket_triggers( } let sql = sqlb .sql() - .map_err(|e| error::Error::InternalErr(e.to_string()))?; + .map_err(|e| error::Error::internal_err(e.to_string()))?; let rows = sqlx::query_as::<_, WebsocketTrigger>(&sql) .fetch_all(&mut *tx) .await?; diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index 3758f5def4..f832b85f14 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -8,7 +8,7 @@ use std::collections::HashMap; -use crate::ai::{AiResource, AI_KEY_CACHE}; +use crate::ai::{AIProvider, AIResource, AI_KEY_CACHE}; use crate::db::ApiAuthed; use crate::users_ee::send_email_if_possible; use crate::utils::get_instance_username_or_create_pending; @@ -165,30 +165,54 @@ struct Workspace { #[derive(FromRow, Serialize, Debug)] pub struct WorkspaceSettings { pub workspace_id: String, + #[serde(skip_serializing_if = "Option::is_none")] pub slack_team_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub slack_name: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub slack_command_script: Option, pub slack_email: String, + #[serde(skip_serializing_if = "Option::is_none")] pub auto_invite_domain: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub auto_invite_operator: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub auto_add: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub customer_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub plan: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub webhook: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub deploy_to: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub ai_resource: Option, - pub code_completion_enabled: bool, + pub ai_models: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub code_completion_model: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub error_handler: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub error_handler_extra_args: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub error_handler_muted_on_cancel: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub large_file_storage: Option, // effectively: DatasetsStorage - pub git_sync: Option, // effectively: WorkspaceGitSyncSettings - pub deploy_ui: Option, // effectively: WorkspaceDeploymentUISettings + #[serde(skip_serializing_if = "Option::is_none")] + pub git_sync: Option, // effectively: WorkspaceGitSyncSettings + #[serde(skip_serializing_if = "Option::is_none")] + pub deploy_ui: Option, // effectively: WorkspaceDeploymentUISettings + #[serde(skip_serializing_if = "Option::is_none")] pub default_app: Option, pub automatic_billing: bool, + #[serde(skip_serializing_if = "Option::is_none")] pub default_scripts: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub mute_critical_alerts: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub color: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub operator_settings: Option, } @@ -246,7 +270,8 @@ struct EditWebhook { #[derive(Deserialize)] struct EditCopilotConfig { ai_resource: Option, - code_completion_enabled: bool, + code_completion_model: Option, + ai_models: Vec, } #[derive(Deserialize, Serialize, Debug)] @@ -409,7 +434,7 @@ async fn get_settings( ) .fetch_one(&mut *tx) .await - .map_err(|e| Error::InternalErr(format!("getting settings: {e:#}")))?; + .map_err(|e| Error::internal_err(format!("getting settings: {e:#}")))?; tx.commit().await?; Ok(Json(settings)) @@ -432,7 +457,7 @@ async fn get_deploy_to( ) .fetch_one(&mut *tx) .await - .map_err(|e| Error::InternalErr(format!("getting deploy_to: {e:#}")))?; + .map_err(|e| Error::internal_err(format!("getting deploy_to: {e:#}")))?; tx.commit().await?; Ok(Json(settings)) @@ -660,27 +685,37 @@ async fn edit_copilot_config( let mut tx = db.begin().await?; if let Some(ai_resource) = &eo.ai_resource { - let path = serde_json::from_value::(ai_resource.clone()) - .map_err(|e| Error::BadRequest(e.to_string()))? - .path; + let parsed_ai_resource = serde_json::from_value::(ai_resource.clone()) + .map_err(|e| Error::BadRequest(e.to_string()))?; + + #[cfg(not(feature = "enterprise"))] + { + if matches!(parsed_ai_resource.provider, AIProvider::CustomAI) { + return Err(Error::BadRequest( + "Custom AI is only available on EE".to_string(), + )); + } + } + sqlx::query!( - "UPDATE workspace_settings SET ai_resource = $1, code_completion_enabled = $2 WHERE workspace_id = $3", + "UPDATE workspace_settings SET ai_resource = $1, code_completion_model = $2, ai_models = $3 WHERE workspace_id = $4", ai_resource, - eo.code_completion_enabled, + eo.code_completion_model, + eo.ai_models.as_slice(), &w_id ) .execute(&mut *tx) .await?; if let Some(cached) = AI_KEY_CACHE.get(&w_id) { - if cached.path != path { + if cached.path != parsed_ai_resource.path { AI_KEY_CACHE.remove(&w_id); } } } else { sqlx::query!( - "UPDATE workspace_settings SET ai_resource = NULL, code_completion_enabled = $1 WHERE workspace_id = $2", - eo.code_completion_enabled, + "UPDATE workspace_settings SET ai_resource = NULL, code_completion_model = $1, ai_models = '{}' WHERE workspace_id = $2", + eo.code_completion_model, &w_id, ) .execute(&mut *tx) @@ -698,8 +733,8 @@ async fn edit_copilot_config( [ ("ai_resource", &format!("{:?}", eo.ai_resource)[..]), ( - "code_completion_enabled", - &format!("{:?}", eo.code_completion_enabled)[..], + "code_completion_model", + &format!("{:?}", eo.code_completion_model)[..], ), ] .into(), @@ -713,9 +748,12 @@ async fn edit_copilot_config( #[derive(Serialize)] struct CopilotInfo { - pub ai_provider: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub ai_provider: Option, pub exists_ai_resource: bool, - pub code_completion_enabled: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub code_completion_model: Option, + pub ai_models: Vec, } async fn get_copilot_info( Extension(db): Extension, @@ -723,33 +761,26 @@ async fn get_copilot_info( ) -> JsonResult { let mut tx = db.begin().await?; let record = sqlx::query!( - "SELECT ai_resource, code_completion_enabled FROM workspace_settings WHERE workspace_id = $1", + "SELECT ai_resource, code_completion_model, ai_models FROM workspace_settings WHERE workspace_id = $1", &w_id ) .fetch_one(&mut *tx) .await - .map_err(|e| Error::InternalErr(format!("getting ai_resource and code_completion_enabled: {e:#}")))?; + .map_err(|e| Error::internal_err(format!("getting ai_resource and code_completion_model: {e:#}")))?; tx.commit().await?; let (ai_provider, exists_ai_resource) = if let Some(ai_resource) = record.ai_resource { - let ai_resource = serde_json::from_value::(ai_resource); - let exist = ai_resource.is_ok(); - ( - if exist { - ai_resource.unwrap().provider - } else { - "".to_string() - }, - exist, - ) + let ai_resource = serde_json::from_value::(ai_resource)?; + (Some(ai_resource.provider), true) } else { - ("".to_string(), false) + (None, false) }; Ok(Json(CopilotInfo { ai_provider, exists_ai_resource, - code_completion_enabled: record.code_completion_enabled, + code_completion_model: record.code_completion_model, + ai_models: record.ai_models, })) } @@ -779,7 +810,7 @@ async fn edit_large_file_storage_config( if let Some(lfs_config) = new_config.large_file_storage { let serialized_lfs_config = serde_json::to_value::(lfs_config) - .map_err(|err| Error::InternalErr(err.to_string()))?; + .map_err(|err| Error::internal_err(err.to_string()))?; sqlx::query!( "UPDATE workspace_settings SET large_file_storage = $1 WHERE workspace_id = $2", @@ -848,7 +879,7 @@ async fn edit_git_sync_config( if let Some(git_sync_settings) = new_config.git_sync_settings { let serialized_config = serde_json::to_value::(git_sync_settings) - .map_err(|err| Error::InternalErr(err.to_string()))?; + .map_err(|err| Error::internal_err(err.to_string()))?; sqlx::query!( "UPDATE workspace_settings SET git_sync = $1 WHERE workspace_id = $2", @@ -914,7 +945,7 @@ async fn edit_deploy_ui_config( if let Some(deploy_ui_settings) = new_config.deploy_ui_settings { let serialized_config = serde_json::to_value::(deploy_ui_settings) - .map_err(|err| Error::InternalErr(err.to_string()))?; + .map_err(|err| Error::internal_err(err.to_string()))?; sqlx::query!( "UPDATE workspace_settings SET deploy_ui = $1 WHERE workspace_id = $2", @@ -1009,7 +1040,7 @@ async fn get_default_scripts( ) .fetch_optional(&mut *tx) .await - .map_err(|err| Error::InternalErr(format!("getting default_app: {err}")))?; + .map_err(|err| Error::internal_err(format!("getting default_app: {err}")))?; tx.commit().await?; Ok(Json(default_scripts.flatten())) @@ -1083,7 +1114,7 @@ async fn get_default_app( ) .fetch_one(&mut *tx) .await - .map_err(|err| Error::InternalErr(format!("getting default_app: {err}")))?; + .map_err(|err| Error::internal_err(format!("getting default_app: {err}")))?; tx.commit().await?; Ok(Json(WorkspaceDefaultApp { default_app_path })) @@ -1219,7 +1250,7 @@ async fn get_encryption_key( Extension(db): Extension, Path(w_id): Path, ) -> JsonResult { - require_super_admin(&db, &authed.email).await?; + require_admin(authed.is_admin, &authed.username)?; let encryption_key_opt = sqlx::query_scalar!( "SELECT key FROM workspace_key WHERE workspace_id = $1", diff --git a/backend/windmill-api/src/workspaces_ee.rs b/backend/windmill-api/src/workspaces_ee.rs index d5fd2cdda1..aa8799e233 100644 --- a/backend/windmill-api/src/workspaces_ee.rs +++ b/backend/windmill-api/src/workspaces_ee.rs @@ -9,7 +9,7 @@ pub async fn edit_auto_invite( _w_id: String, _ea: EditAutoInvite, ) -> windmill_common::error::Result { - Err(windmill_common::error::Error::InternalErr( + Err(windmill_common::error::Error::internal_err( "Not implemented on OSS".to_string(), )) } diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index 0a417ff916..40a80025ea 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -148,6 +148,7 @@ pub(crate) struct ArchiveQueryParams { skip_variables: Option, skip_resources: Option, include_schedules: Option, + include_triggers: Option, include_users: Option, include_groups: Option, include_settings: Option, @@ -185,6 +186,8 @@ where "has_draft", "draft_only", "error", + "last_server_ping", + "server_id", ], ignore_keys.unwrap_or(vec![]), ] @@ -221,6 +224,7 @@ struct SimplifiedUser { #[derive(Serialize)] struct SimplifiedGroup { name: String, + #[serde(skip_serializing_if = "Option::is_none")] summary: Option, members: Vec, admins: Vec, @@ -235,18 +239,35 @@ struct SimplifiedSettings { auto_invite_enabled: bool, auto_invite_as: String, auto_invite_mode: String, + #[serde(skip_serializing_if = "Option::is_none")] webhook: Option, + #[serde(skip_serializing_if = "Option::is_none")] deploy_to: Option, + #[serde(skip_serializing_if = "Option::is_none")] error_handler: Option, + #[serde(skip_serializing_if = "Option::is_none")] error_handler_extra_args: Option, error_handler_muted_on_cancel: bool, + #[serde(skip_serializing_if = "Option::is_none")] ai_resource: Option, - code_completion_enabled: bool, + ai_models: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + code_completion_model: Option, + #[serde(skip_serializing_if = "Option::is_none")] large_file_storage: Option, + #[serde(skip_serializing_if = "Option::is_none")] git_sync: Option, + #[serde(skip_serializing_if = "Option::is_none")] default_app: Option, + #[serde(skip_serializing_if = "Option::is_none")] default_scripts: Option, name: String, + #[serde(skip_serializing_if = "Option::is_none")] + mute_critical_alerts: Option, + #[serde(skip_serializing_if = "Option::is_none")] + color: Option, + #[serde(skip_serializing_if = "Option::is_none")] + operator_settings: Option, } pub(crate) async fn tarball_workspace( @@ -262,6 +283,7 @@ pub(crate) async fn tarball_workspace( skip_secrets, skip_variables, include_schedules, + include_triggers, include_users, include_groups, include_settings, @@ -508,6 +530,113 @@ pub(crate) async fn tarball_workspace( } } + if include_triggers.unwrap_or(false) { + #[cfg(feature = "http_trigger")] + { + let http_triggers = sqlx::query_as!( + crate::http_triggers::HttpTrigger, + "SELECT workspace_id, path, route_path, route_path_key, script_path, is_flow, edited_by, edited_at, email, extra_perms, is_async, requires_auth, http_method as \"http_method: _\", static_asset_config as \"static_asset_config: _\", is_static_website FROM http_trigger + WHERE workspace_id = $1", + &w_id + ) + .fetch_all(&mut *tx) + .await?; + + for trigger in http_triggers { + let trigger_str = &to_string_without_metadata(&trigger, false, None).unwrap(); + archive + .write_to_archive(&trigger_str, &format!("{}.http_trigger.json", trigger.path)) + .await?; + } + } + + #[cfg(feature = "websocket")] + { + let websocket_triggers = sqlx::query_as!( + crate::websocket_triggers::WebsocketTrigger, + "SELECT workspace_id, path, url, script_path, is_flow, edited_by, email, edited_at, server_id, last_server_ping, extra_perms, error, enabled, filters as \"filters: _\", initial_messages as \"initial_messages: _\", url_runnable_args as \"url_runnable_args: _\", can_return_message FROM websocket_trigger + WHERE workspace_id = $1", + &w_id + ) + .fetch_all(&mut *tx) + .await?; + + for trigger in websocket_triggers { + let trigger_str = &to_string_without_metadata(&trigger, false, None).unwrap(); + archive + .write_to_archive( + &trigger_str, + &format!("{}.websocket_trigger.json", trigger.path), + ) + .await?; + } + } + + #[cfg(all(feature = "enterprise", feature = "kafka"))] + { + let kafka_triggers = sqlx::query_as!( + crate::kafka_triggers_ee::KafkaTrigger, + "SELECT * FROM kafka_trigger + WHERE workspace_id = $1", + &w_id + ) + .fetch_all(&mut *tx) + .await?; + + for trigger in kafka_triggers { + let trigger_str = &to_string_without_metadata(&trigger, false, None).unwrap(); + archive + .write_to_archive( + &trigger_str, + &format!("{}.kafka_trigger.json", trigger.path), + ) + .await?; + } + } + + #[cfg(all(feature = "enterprise", feature = "nats"))] + { + let nats_triggers = sqlx::query_as!( + crate::nats_triggers_ee::NatsTrigger, + "SELECT * FROM nats_trigger + WHERE workspace_id = $1", + &w_id + ) + .fetch_all(&mut *tx) + .await?; + + for trigger in nats_triggers { + let trigger_str: &String = + &to_string_without_metadata(&trigger, false, None).unwrap(); + archive + .write_to_archive(&trigger_str, &format!("{}.nats_trigger.json", trigger.path)) + .await?; + } + } + + #[cfg(feature = "postgres_trigger")] + { + let postgres_triggers = sqlx::query_as!( + crate::postgres_triggers::PostgresTrigger, + "SELECT * FROM postgres_trigger + WHERE workspace_id = $1", + &w_id + ) + .fetch_all(&mut *tx) + .await?; + + for trigger in postgres_triggers { + let trigger_str = &to_string_without_metadata(&trigger, false, None).unwrap(); + archive + .write_to_archive( + &trigger_str, + &format!("{}.postgres_trigger.json", trigger.path), + ) + .await?; + } + } + } + if include_users.unwrap_or(false) { let users = sqlx::query!( "SELECT * FROM usr @@ -530,12 +659,7 @@ pub(crate) async fn tarball_workspace( disabled: user.disabled, email: user.email, }; - let user_str = &to_string_without_metadata( - &user, - false, - Some(vec!["is_admin", "operator", "email"]), - ) - .unwrap(); + let user_str = &to_string_without_metadata(&user, false, Some(vec!["email"])).unwrap(); archive .write_to_archive(&user_str, &format!("users/{}.user.json", user.email)) .await?; @@ -558,7 +682,7 @@ pub(crate) async fn tarball_workspace( for group in groups { let extra_perms: HashMap = serde_json::from_value(group.extra_perms) .map_err(|e| { - Error::InternalErr(format!( + Error::internal_err(format!( "Error parsing extra_perms for group {}: {}", group.name, e )) @@ -617,14 +741,18 @@ pub(crate) async fn tarball_workspace( deploy_to, error_handler, ai_resource, - code_completion_enabled, + ai_models, + code_completion_model, error_handler_extra_args, error_handler_muted_on_cancel, large_file_storage, git_sync, default_app, default_scripts, - workspace.name + workspace.name, + mute_critical_alerts, + color, + operator_settings FROM workspace_settings LEFT JOIN workspace ON workspace.id = workspace_settings.workspace_id WHERE workspace_id = $1"#, @@ -635,7 +763,7 @@ pub(crate) async fn tarball_workspace( .map(|v| serde_json::to_string_pretty(&v).ok()) .ok() .flatten() - .ok_or_else(|| Error::InternalErr("Error serializing settings".to_string()))?; + .ok_or_else(|| Error::internal_err("Error serializing settings".to_string()))?; archive .write_to_archive(&settings_str, "settings.json") @@ -654,7 +782,7 @@ pub(crate) async fn tarball_workspace( .map(|v| serde_json::to_string_pretty(&v).ok()) .ok() .flatten() - .ok_or_else(|| Error::InternalErr("Error serializing enryption key".to_string()))?; + .ok_or_else(|| Error::internal_err("Error serializing enryption key".to_string()))?; archive .write_to_archive(&key_json, "encryption_key.json") .await?; diff --git a/backend/windmill-api/src/workspaces_extra.rs b/backend/windmill-api/src/workspaces_extra.rs index a8f0c8721d..2378e85c02 100644 --- a/backend/windmill-api/src/workspaces_extra.rs +++ b/backend/windmill-api/src/workspaces_extra.rs @@ -144,7 +144,7 @@ pub(crate) async fn change_workspace_id( .await?; sqlx::query!( - "UPDATE completed_job SET workspace_id = $1 WHERE workspace_id = $2", + "UPDATE v2_job_completed SET workspace_id = $1 WHERE workspace_id = $2", &rw.new_id, &old_id ) @@ -269,7 +269,7 @@ pub(crate) async fn change_workspace_id( .await?; sqlx::query!( - "UPDATE queue SET workspace_id = $1 WHERE workspace_id = $2", + "UPDATE v2_job_queue SET workspace_id = $1 WHERE workspace_id = $2", &rw.new_id, &old_id ) @@ -277,7 +277,7 @@ pub(crate) async fn change_workspace_id( .await?; sqlx::query!( - "UPDATE job SET workspace_id = $1 WHERE workspace_id = $2", + "UPDATE v2_job SET workspace_id = $1 WHERE workspace_id = $2", &rw.new_id, &old_id ) @@ -430,7 +430,10 @@ pub(crate) async fn delete_workspace( sqlx::query!("DELETE FROM dependency_map WHERE workspace_id = $1", &w_id) .execute(&mut *tx) .await?; - sqlx::query!("DELETE FROM queue WHERE workspace_id = $1", &w_id) + sqlx::query!("DELETE FROM v2_job_queue WHERE workspace_id = $1", &w_id) + .execute(&mut *tx) + .await?; + sqlx::query!("DELETE FROM v2_job WHERE workspace_id = $1", &w_id) .execute(&mut *tx) .await?; sqlx::query!("DELETE FROM capture WHERE workspace_id = $1", &w_id) @@ -468,9 +471,12 @@ pub(crate) async fn delete_workspace( .execute(&mut *tx) .await?; - sqlx::query!("DELETE FROM completed_job WHERE workspace_id = $1", &w_id) - .execute(&mut *tx) - .await?; + sqlx::query!( + "DELETE FROM v2_job_completed WHERE workspace_id = $1", + &w_id + ) + .execute(&mut *tx) + .await?; sqlx::query!("DELETE FROM job_stats WHERE workspace_id = $1", &w_id) .execute(&mut *tx) diff --git a/backend/windmill-common/src/auth.rs b/backend/windmill-common/src/auth.rs index 017131388d..52a966b105 100644 --- a/backend/windmill-common/src/auth.rs +++ b/backend/windmill-common/src/auth.rs @@ -67,7 +67,7 @@ pub async fn is_super_admin_email(db: &DB, email: &str) -> Result { let is_admin = sqlx::query_scalar!("SELECT super_admin FROM password WHERE email = $1", email) .fetch_optional(db) .await - .map_err(|e| Error::InternalErr(format!("fetching super admin: {e:#}")))? + .map_err(|e| Error::internal_err(format!("fetching super admin: {e:#}")))? .unwrap_or(false); Ok(is_admin) @@ -81,7 +81,7 @@ pub async fn is_devops_email(db: &DB, email: &str) -> Result { let is_devops = sqlx::query_scalar!("SELECT devops FROM password WHERE email = $1", email) .fetch_optional(db) .await - .map_err(|e| Error::InternalErr(format!("fetching super admin: {e:#}")))? + .map_err(|e| Error::internal_err(format!("fetching super admin: {e:#}")))? .unwrap_or(false); Ok(is_devops) @@ -123,7 +123,7 @@ pub async fn fetch_authed_from_permissioned_as( if let Some(r) = r { (r.is_admin, r.operator) } else { - return Err(Error::InternalErr(format!( + return Err(Error::internal_err(format!( "user {name} not found in workspace {w_id}" ))); } diff --git a/backend/windmill-common/src/cache.rs b/backend/windmill-common/src/cache.rs index 83cec0b417..7447213605 100644 --- a/backend/windmill-common/src/cache.rs +++ b/backend/windmill-common/src/cache.rs @@ -389,7 +389,7 @@ pub mod flow { async move { fetch_node.await.and_then(|data| match data { RawData::Script(data) => Ok(data), - RawData::Flow(_) => Err(error::Error::InternalErr(format!( + RawData::Flow(_) => Err(error::Error::internal_err(format!( "Flow node ({:x}) isn't a script node.", node.0 ))), @@ -410,7 +410,7 @@ pub mod flow { async move { fetch_node.await.and_then(|data| match data { RawData::Flow(data) => Ok(data), - RawData::Script(_) => Err(error::Error::InternalErr(format!( + RawData::Script(_) => Err(error::Error::internal_err(format!( "Flow node ({:x}) isn't a flow node.", node.0 ))), @@ -639,7 +639,7 @@ pub mod job { async move { fetch_preview.await.and_then(|data| match data { RawData::Flow(data) => Ok(data), - RawData::Script(_) => Err(error::Error::InternalErr(format!( + RawData::Script(_) => Err(error::Error::internal_err(format!( "Job ({job}) isn't a flow job." ))), }) @@ -659,7 +659,7 @@ pub mod job { async move { fetch_preview.await.and_then(|data| match data { RawData::Script(data) => Ok(data), - RawData::Flow(_) => Err(error::Error::InternalErr(format!( + RawData::Flow(_) => Err(error::Error::internal_err(format!( "Job ({job}) isn't a script job." ))), }) @@ -681,7 +681,7 @@ pub mod job { match (raw_lock, raw_code, raw_flow) { (None, None, None) => sqlx::query!( "SELECT raw_code, raw_lock, raw_flow AS \"raw_flow: Json>\" \ - FROM job WHERE id = $1 LIMIT 1", + FROM v2_job WHERE id = $1 LIMIT 1", job ) .fetch_optional(e) @@ -723,7 +723,7 @@ pub mod job { .await .map(|(data, _meta)| data), (AppScript, Some(id)) => app::fetch_script(e, AppScriptId(id)).await, - _ => Err(error::Error::InternalErr(format!( + _ => Err(error::Error::internal_err(format!( "Isn't a script job: {:?}", kind ))), @@ -748,7 +748,7 @@ pub mod job { Ok(raw_flow) => Ok(raw_flow), Err(_) => flow::fetch_version(e, id).await, }, - _ => Err(error::Error::InternalErr(format!( + _ => Err(error::Error::internal_err(format!( "Isn't a flow job {:?}", kind ))), @@ -807,7 +807,7 @@ const _: () = { fn resolve(mut src: Self::Untrusted) -> error::Result { let Some(meta) = src.meta.take() else { - return Err(error::Error::InternalErr("Invalid script src".to_string())); + return Err(error::Error::internal_err("Invalid script src".to_string())); }; Ok(ScriptFull { data: Arc::new(ScriptData { code: src.content, lock: src.lock }), @@ -842,7 +842,7 @@ const _: () = { RawNode { raw_code: Some(code), raw_lock: lock, .. } => { Ok(Self::Script(Arc::new(ScriptData { code, lock }))) } - _ => Err(error::Error::InternalErr( + _ => Err(error::Error::internal_err( "Invalid raw data src".to_string(), )), } diff --git a/backend/windmill-common/src/error.rs b/backend/windmill-common/src/error.rs index 426354a13d..faf1a72d1b 100644 --- a/backend/windmill-common/src/error.rs +++ b/backend/windmill-common/src/error.rs @@ -22,8 +22,6 @@ pub type JsonResult = std::result::Result, Error>; #[derive(Debug, Error)] pub enum Error { - #[error("Uuid Error {0}")] - UuidErr(#[from] uuid::Error), #[error("Bad config: {0}")] BadConfig(String), #[error("Connecting to database: {0}")] @@ -40,26 +38,34 @@ pub enum Error { RequireAdmin(String), #[error("{0}")] ExecutionErr(String), - #[error("IO error: {0}")] - IoErr(#[from] io::Error), - #[error("Sql error: {0}")] - SqlErr(#[from] sqlx::Error), + #[error("IoErr: {error:#} @{location:#}")] + IoErr { error: io::Error, location: String }, + #[error("Utf8Err: {error:#} @{location:#}")] + Utf8Err { error: std::string::FromUtf8Error, location: String }, + #[error("UuidErr: {error:#} @{location:#}")] + UuidErr { error: uuid::Error, location: String }, + #[error("SqlErr: {error:#} @{location:#}")] + SqlErr { error: sqlx::Error, location: String }, + #[error("SerdeJson: {error:#} @{location:#}")] + SerdeJson { error: serde_json::Error, location: String }, #[error("Bad request: {0}")] BadRequest(String), #[error("Quota exceeded: {0}")] QuotaExceeded(String), #[error("Internal: {0}")] InternalErr(String), + #[error("Internal: {message} @{location}")] + InternalErrLoc { message: String, location: String }, #[error("Internal: {0}: {1}")] InternalErrAt(&'static Location<'static>, String), - #[error("Hexadecimal decoding error: {0}")] - HexErr(#[from] hex::FromHexError), + #[error("HexErr: {error:#} @{location:#}")] + HexErr { error: hex::FromHexError, location: String }, #[error("Migrating database: {0}")] DatabaseMigration(#[from] MigrateError), - #[error("Non-zero exit status: {0}")] - ExitStatus(i32), - #[error("Err: {0:#}")] - Anyhow(#[from] anyhow::Error), + #[error("Non-zero exit status for {0}: {1}")] + ExitStatus(String, i32), + #[error("Error: {error:#} @{location:#}")] + Anyhow { error: anyhow::Error, location: String }, #[error("Error: {0:#?}")] JsonErr(serde_json::Value), #[error("{0}")] @@ -68,10 +74,63 @@ pub enum Error { AlreadyCompleted(String), #[error("Find python error: {0}")] FindPythonError(String), - #[error("{0}")] - Utf8(#[from] std::string::FromUtf8Error), - #[error("Encoding/decoding error: {0}")] - SerdeJson(#[from] serde_json::Error), +} + +fn prettify_location(location: &'static Location<'static>) -> String { + location + .to_string() + .split("/") + .last() + .unwrap_or("unknown") + .to_string() +} +impl From for Error { + #[track_caller] + fn from(e: anyhow::Error) -> Self { + Self::Anyhow { error: e, location: prettify_location(std::panic::Location::caller()) } + } +} + +impl From for Error { + #[track_caller] + fn from(e: sqlx::Error) -> Self { + Self::SqlErr { error: e, location: prettify_location(std::panic::Location::caller()) } + } +} + +impl From for Error { + #[track_caller] + fn from(e: uuid::Error) -> Self { + Self::UuidErr { error: e, location: prettify_location(std::panic::Location::caller()) } + } +} + +impl From for Error { + #[track_caller] + fn from(e: std::string::FromUtf8Error) -> Self { + Self::Utf8Err { error: e, location: prettify_location(std::panic::Location::caller()) } + } +} + +impl From for Error { + #[track_caller] + fn from(e: io::Error) -> Self { + Self::IoErr { error: e, location: prettify_location(std::panic::Location::caller()) } + } +} + +impl From for Error { + #[track_caller] + fn from(e: hex::FromHexError) -> Self { + Self::HexErr { error: e, location: prettify_location(std::panic::Location::caller()) } + } +} + +impl From for Error { + #[track_caller] + fn from(e: serde_json::Error) -> Self { + Self::SerdeJson { error: e, location: prettify_location(std::panic::Location::caller()) } + } } impl Error { @@ -86,10 +145,20 @@ impl Error { pub fn relocate_internal(self, loc: &'static Location<'static>) -> Self { match self { - Self::InternalErr(s) | Self::InternalErrAt(_, s) => Self::InternalErrAt(loc, s), + Self::InternalErrLoc { message, .. } + | Self::InternalErrAt(_, message) + | Self::InternalErr(message) => Self::InternalErrAt(loc, message), _ => self, } } + + #[track_caller] + pub fn internal_err>(msg: T) -> Self { + Self::InternalErrLoc { + message: msg.as_ref().to_string(), + location: prettify_location(std::panic::Location::caller()), + } + } } pub fn relocate_internal(loc: &'static Location<'static>) -> impl FnOnce(Error) -> Error { @@ -109,9 +178,10 @@ impl IntoResponse for Error { Self::NotFound(_) => axum::http::StatusCode::NOT_FOUND, Self::NotAuthorized(_) => axum::http::StatusCode::UNAUTHORIZED, Self::RequireAdmin(_) => axum::http::StatusCode::FORBIDDEN, - Self::SqlErr(_) | Self::BadRequest(_) | Self::AiError(_) | Self::QuotaExceeded(_) => { - axum::http::StatusCode::BAD_REQUEST - } + Self::SqlErr { .. } + | Self::BadRequest(_) + | Self::AiError(_) + | Self::QuotaExceeded(_) => axum::http::StatusCode::BAD_REQUEST, _ => axum::http::StatusCode::INTERNAL_SERVER_ERROR, }; diff --git a/backend/windmill-common/src/flows.rs b/backend/windmill-common/src/flows.rs index b4e23bb342..31152aa938 100644 --- a/backend/windmill-common/src/flows.rs +++ b/backend/windmill-common/src/flows.rs @@ -736,7 +736,7 @@ pub async fn resolve_value( with_code: bool, ) -> Result<(), Error> { let mut val = serde_json::from_str::(value.get()).map_err(|err| { - Error::InternalErr(format!("resolve: Failed to parse flow value: {}", err)) + Error::internal_err(format!("resolve: Failed to parse flow value: {}", err)) })?; for module in &mut val.modules { resolve_module(e, workspace_id, &mut module.value, with_code).await?; @@ -755,7 +755,7 @@ pub async fn resolve_module( use FlowModuleValue::*; let mut val = serde_json::from_str::(value.get()).map_err(|err| { - Error::InternalErr(format!( + Error::internal_err(format!( "resolve: Failed to parse flow module value: {}", err )) diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index 3829826da6..31d8214e0c 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -40,7 +40,7 @@ pub const JWT_SECRET_SETTING: &str = "jwt_secret"; pub const EMAIL_DOMAIN_SETTING: &str = "email_domain"; pub const OTEL_SETTING: &str = "otel"; -pub const ENV_SETTINGS: [&str; 55] = [ +pub const ENV_SETTINGS: [&str; 56] = [ "DISABLE_NSJAIL", "MODE", "NUM_WORKERS", @@ -96,11 +96,12 @@ pub const ENV_SETTINGS: [&str; 55] = [ "OTEL_METRICS", "OTEL_TRACING", "OTEL_LOGS", + "DISABLE_S3_STORE", ]; use crate::error; -use sqlx::Pool; use sqlx::postgres::Postgres; +use sqlx::Pool; pub async fn load_value_from_global_settings( db: &Pool, @@ -114,4 +115,4 @@ pub async fn load_value_from_global_settings( .await? .map(|x| x.value); Ok(r) -} \ No newline at end of file +} diff --git a/backend/windmill-common/src/jobs.rs b/backend/windmill-common/src/jobs.rs index 13a8dd7e8a..49de1c2415 100644 --- a/backend/windmill-common/src/jobs.rs +++ b/backend/windmill-common/src/jobs.rs @@ -11,8 +11,6 @@ use uuid::Uuid; pub const ENTRYPOINT_OVERRIDE: &str = "_ENTRYPOINT_OVERRIDE"; -pub const PREPROCESSOR_FAKE_ENTRYPOINT: &str = "__WM_PREPROCESSOR"; - use crate::{ apps::AppScriptId, error::{self, to_anyhow, Error}, @@ -71,6 +69,7 @@ pub struct QueuedJob { pub script_hash: Option, #[serde(skip_serializing_if = "Option::is_none")] pub script_path: Option, + pub script_entrypoint_override: Option, pub args: Option>>>, #[serde(skip_serializing_if = "Option::is_none")] pub logs: Option, @@ -116,6 +115,8 @@ pub struct QueuedJob { pub cache_ttl: Option, #[serde(skip_serializing_if = "Option::is_none")] pub priority: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub preprocessed: Option, } impl QueuedJob { @@ -170,6 +171,7 @@ impl Default for QueuedJob { flow_status: None, is_flow_step: false, language: None, + script_entrypoint_override: None, same_worker: false, pre_run_error: None, email: "".to_string(), @@ -185,6 +187,7 @@ impl Default for QueuedJob { flow_step_id: None, cache_ttl: None, priority: None, + preprocessed: None, } } } @@ -197,7 +200,7 @@ pub struct CompletedJob { pub parent_job: Option, pub created_by: String, pub created_at: chrono::DateTime, - pub started_at: chrono::DateTime, + pub started_at: Option>, pub duration_ms: i64, pub success: bool, #[serde(skip_serializing_if = "Option::is_none")] @@ -207,6 +210,7 @@ pub struct CompletedJob { pub args: Option>>>, #[serde(skip_serializing_if = "Option::is_none")] pub result: Option>>, + pub result_columns: Option>, #[serde(skip_serializing_if = "Option::is_none")] pub logs: Option, pub deleted: bool, @@ -234,6 +238,8 @@ pub struct CompletedJob { pub priority: Option, #[serde(skip_serializing_if = "Option::is_none")] pub labels: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub preprocessed: Option, } impl CompletedJob { @@ -251,12 +257,6 @@ impl CompletedJob { } } -#[derive(sqlx::FromRow)] -pub struct BranchResults { - pub result: sqlx::types::Json>, - pub id: Uuid, -} - #[derive(Debug, Clone)] pub enum JobPayload { ScriptHub { @@ -353,7 +353,7 @@ pub enum JobPayload { Noop, } -#[derive(Clone, Serialize, Deserialize, Debug)] +#[derive(Clone, Serialize, Deserialize, Debug, Default)] pub struct RawCode { pub content: String, pub path: Option, @@ -480,7 +480,7 @@ pub async fn script_hash_to_tag_and_limits<'c>( .fetch_one(&mut **db) .await .map_err(|e| { - Error::InternalErr(format!( + Error::internal_err(format!( "querying getting tag for hash {script_hash}: {e:#}" )) })?; @@ -535,19 +535,9 @@ pub async fn get_payload_tag_from_prefixed_path( Ok((payload, tag, on_behalf_of)) } -#[derive(Deserialize)] -struct FlowStatusMetadata { - column_order: Vec, -} - -#[derive(Deserialize)] -struct FlowStatusWithMetadataOnly { - _metadata: FlowStatusMetadata, -} - pub fn order_columns( rows: Option>>, - column_order: Vec, + column_order: &Vec, ) -> Option> { if let Some(mut rows) = rows { if let Some(first_row) = rows.get(0) { @@ -555,7 +545,7 @@ pub fn order_columns( if let Ok(first_row) = first_row { let mut new_first_row = IndexMap::new(); for col in column_order { - if let Some(val) = first_row.get(&col) { + if let Some(val) = first_row.get(col) { new_first_row.insert(col.clone(), val.clone()); } } @@ -572,39 +562,21 @@ pub fn order_columns( } pub fn format_result( - language: Option<&ScriptLang>, - flow_status: Option<&sqlx::types::Json>>, + result_columns: Option<&Vec>, result: Option<&mut sqlx::types::Json>>, ) -> () { - match language { - Some(&ScriptLang::Postgresql) - | Some(&ScriptLang::Mysql) - | Some(&ScriptLang::Snowflake) - | Some(&ScriptLang::Bigquery) => { - if let Some(Ok(flow_status)) = - flow_status.map(|x| serde_json::from_str::(x.get())) - { - if let Some(result) = result { - let rows = serde_json::from_str::>>(result.get()).ok(); - if let Some(ordered_result) = - order_columns(rows, flow_status._metadata.column_order) - { - *result = sqlx::types::Json(ordered_result); - } - } + if let Some(result_columns) = result_columns { + if let Some(result) = result { + let rows = serde_json::from_str::>>(result.get()).ok(); + if let Some(ordered_result) = order_columns(rows, result_columns) { + *result = sqlx::types::Json(ordered_result); } } - _ => {} } } pub fn format_completed_job_result(mut cj: CompletedJob) -> CompletedJob { - format_result( - cj.language.as_ref(), - cj.flow_status.as_ref(), - cj.result.as_mut(), - ); - + format_result(cj.result_columns.as_ref(), cj.result.as_mut()); cj } diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index e47c4fc87f..802af6196b 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -8,6 +8,7 @@ use std::{ net::SocketAddr, + str::FromStr, sync::{atomic::AtomicBool, Arc}, }; @@ -18,8 +19,6 @@ use sqlx::{Pool, Postgres}; pub mod apps; pub mod auth; -#[cfg(feature = "benchmark")] -pub mod bench; pub mod cache; pub mod db; pub mod ee; @@ -36,7 +35,6 @@ pub mod job_s3_helpers_ee; pub mod jobs; pub mod more_serde; pub mod oauth2; -pub mod teams_ee; pub mod otel_ee; pub mod queue; pub mod s3_helpers; @@ -44,6 +42,7 @@ pub mod schedule; pub mod scripts; pub mod server; pub mod stats_ee; +pub mod teams_ee; pub mod tracing_init; pub mod users; pub mod utils; @@ -220,28 +219,42 @@ async fn reset() -> () { todo!() } -pub async fn connect_db( - server_mode: bool, - indexer_mode: bool, -) -> anyhow::Result> { - use anyhow::Context; +pub async fn get_database_url() -> Result { use std::env::var; use tokio::fs::File; use tokio::io::AsyncReadExt; - - let database_url = match var("DATABASE_URL_FILE") { + match var("DATABASE_URL_FILE") { Ok(file_path) => { let mut file = File::open(file_path).await?; let mut contents = String::new(); file.read_to_string(&mut contents).await?; - contents.trim().to_string() + Ok(contents.trim().to_string()) } Err(_) => var("DATABASE_URL").map_err(|_| { Error::BadConfig( "Either DATABASE_URL_FILE or DATABASE_URL env var is missing".to_string(), ) - })?, - }; + }), + } +} + +pub async fn initial_connection() -> Result, error::Error> { + let database_url = get_database_url().await?; + sqlx::postgres::PgPoolOptions::new() + .max_connections(2) + .connect_with(sqlx::postgres::PgConnectOptions::from_str(&database_url)?) + .await + .map_err(|err| Error::ConnectingToDatabase(err.to_string())) +} + +pub async fn connect_db( + server_mode: bool, + indexer_mode: bool, + worker_mode: bool, +) -> anyhow::Result> { + use anyhow::Context; + + let database_url = get_database_url().await?; let max_connections = match std::env::var("DATABASE_CONNECTIONS") { Ok(n) => n.parse::().context("invalid DATABASE_CONNECTIONS")?, @@ -262,20 +275,35 @@ pub async fn connect_db( } }; - Ok(connect(&database_url, max_connections).await?) + Ok(connect(&database_url, max_connections, worker_mode).await?) } pub async fn connect( database_url: &str, max_connections: u32, + worker_mode: bool, ) -> Result, error::Error> { use std::time::Duration; sqlx::postgres::PgPoolOptions::new() - .min_connections(3) + .min_connections((max_connections / 5).clamp(3, max_connections)) .max_connections(max_connections) .max_lifetime(Duration::from_secs(30 * 60)) // 30 mins - .connect(database_url) + .after_connect(move |conn, _| { + if worker_mode { + Box::pin(async move { + sqlx::query("SET enable_seqscan = OFF;") + .execute(conn) + .await?; + Ok(()) + }) + } else { + Box::pin(async move { Ok(()) }) + } + }) + .connect_with( + sqlx::postgres::PgConnectOptions::from_str(database_url)?.statement_cache_capacity(400), + ) .await .map_err(|err| Error::ConnectingToDatabase(err.to_string())) } diff --git a/backend/windmill-common/src/queue.rs b/backend/windmill-common/src/queue.rs index 6cbb5611c5..39ec349c91 100644 --- a/backend/windmill-common/src/queue.rs +++ b/backend/windmill-common/src/queue.rs @@ -3,14 +3,14 @@ use std::collections::HashMap; use sqlx::{Pool, Postgres}; pub async fn get_queue_counts(db: &Pool) -> HashMap { - sqlx::query_as::<_, (String, i64)>( - "SELECT tag, count(*) as count FROM queue WHERE + sqlx::query!( + "SELECT tag AS \"tag!\", count(*) AS \"count!\" FROM v2_job_queue WHERE scheduled_for <= now() - ('3 seconds')::interval AND running = false GROUP BY tag", ) .fetch_all(db) .await .ok() - .map(|v| v.into_iter().map(|(k, v)| (k, v as u32)).collect()) + .map(|v| v.into_iter().map(|x| (x.tag, x.count as u32)).collect()) .unwrap_or_else(|| HashMap::new()) } diff --git a/backend/windmill-common/src/s3_helpers.rs b/backend/windmill-common/src/s3_helpers.rs index 0274d0f38d..01507f5b6a 100644 --- a/backend/windmill-common/src/s3_helpers.rs +++ b/backend/windmill-common/src/s3_helpers.rs @@ -298,7 +298,7 @@ pub async fn build_s3_client(s3_resource_ref: &S3Resource) -> error::Result Result workspace_id, err ); - Err(Error::SqlErr(err)) + return Err(err.into()); } } } diff --git a/backend/windmill-common/src/variables.rs b/backend/windmill-common/src/variables.rs index 353b26ecb5..48746152ef 100644 --- a/backend/windmill-common/src/variables.rs +++ b/backend/windmill-common/src/variables.rs @@ -105,7 +105,7 @@ pub async fn get_workspace_key(w_id: &str, db: &DB) -> crate::error::Result String { pub fn decrypt(mc: &MagicCrypt256, value: String) -> error::Result { mc.decrypt_base64_to_string(value).map_err(|e| match e { - MagicCryptError::DecryptError(_) => error::Error::InternalErr( + MagicCryptError::DecryptError(_) => error::Error::internal_err( "Could not decrypt value. The value may have been encrypted with a different key." .to_string(), ), - _ => error::Error::InternalErr(e.to_string()), + _ => error::Error::internal_err(e.to_string()), }) } diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 241679301f..23b5ed8f3e 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -103,33 +103,67 @@ lazy_static::lazy_static! { pub static ref DISABLE_FLOW_SCRIPT: bool = std::env::var("DISABLE_FLOW_SCRIPT").ok().is_some_and(|x| x == "1" || x == "true"); } +fn format_pull_query(peek: String) -> String { + let r = format!( + "WITH peek AS ( + {} + ), q AS NOT MATERIALIZED ( + UPDATE v2_job_queue SET + running = true, + started_at = coalesce(started_at, now()), + suspend_until = null, + worker = $1 + WHERE id = (SELECT id FROM peek) + RETURNING + started_at, scheduled_for, running, + canceled_by, canceled_reason, canceled_by IS NOT NULL AS canceled, + suspend, suspend_until + ), r AS NOT MATERIALIZED ( + UPDATE v2_job_runtime SET + ping = now() + WHERE id = (SELECT id FROM peek) + ), j AS NOT MATERIALIZED ( + SELECT + id, workspace_id, parent_job, created_by, created_at, runnable_id AS script_hash, + runnable_path AS script_path, args, kind AS job_kind, + CASE WHEN trigger_kind = 'schedule' THEN trigger END AS schedule_path, + permissioned_as, permissioned_as_email AS email, script_lang AS language, + flow_innermost_root_job AS root_job, flow_step_id, flow_step_id IS NOT NULL AS is_flow_step, + same_worker, pre_run_error, visible_to_owner, tag, concurrent_limit, + concurrency_time_window_s, timeout, cache_ttl, priority, raw_code, raw_lock, + raw_flow, script_entrypoint_override, preprocessed + FROM v2_job + WHERE id = (SELECT id FROM peek) + ) SELECT id, workspace_id, parent_job, created_by, created_at, started_at, scheduled_for, + running, script_hash, script_path, args, null as logs, canceled, canceled_by, + canceled_reason, null as last_ping, job_kind, schedule_path, permissioned_as, + flow_status, is_flow_step, language, suspend, suspend_until, + same_worker, pre_run_error, email, visible_to_owner, null as mem_peak, + root_job, flow_leaf_jobs as leaf_jobs, tag, concurrent_limit, concurrency_time_window_s, + timeout, flow_step_id, cache_ttl, priority, raw_code, raw_lock, raw_flow, + script_entrypoint_override, preprocessed + FROM q, j + LEFT JOIN v2_job_status f USING (id)", + peek + ); + tracing::debug!("pull query: {}", r); + r +} + pub async fn make_suspended_pull_query(wc: &WorkerConfig) { if wc.worker_tags.len() == 0 { tracing::error!("Empty tags in worker tags, skipping"); return; } - let query = format!( - "UPDATE queue - SET running = true - , started_at = coalesce(started_at, now()) - , last_ping = now() - , suspend_until = null - WHERE id = ( - SELECT id - FROM queue - WHERE suspend_until IS NOT NULL AND (suspend <= 0 OR suspend_until <= now()) AND tag IN ({}) - ORDER BY priority DESC NULLS LAST, created_at - FOR UPDATE SKIP LOCKED - LIMIT 1 - ) - RETURNING id, workspace_id, parent_job, created_by, created_at, started_at, scheduled_for, - running, script_hash, script_path, args, null as logs, canceled, canceled_by, - canceled_reason, last_ping, job_kind, schedule_path, permissioned_as, - flow_status, is_flow_step, language, suspend, suspend_until, - same_worker, pre_run_error, email, visible_to_owner, mem_peak, - root_job, leaf_jobs, tag, concurrent_limit, concurrency_time_window_s, - timeout, flow_step_id, cache_ttl, priority, - raw_code, raw_lock, raw_flow", wc.worker_tags.iter().map(|x| format!("'{x}'")).join(", ")); + let query = format_pull_query(format!( + "SELECT id + FROM v2_job_queue + WHERE suspend_until IS NOT NULL AND (suspend <= 0 OR suspend_until <= now()) AND tag IN ({}) + ORDER BY priority DESC NULLS LAST, created_at + FOR UPDATE SKIP LOCKED + LIMIT 1", + wc.worker_tags.iter().map(|x| format!("'{x}'")).join(", ") + )); let mut l = WORKER_SUSPENDED_PULL_QUERY.write().await; *l = query; } @@ -141,31 +175,17 @@ pub async fn make_pull_query(wc: &WorkerConfig) { tracing::error!("Empty tags in priority tags, skipping"); continue; } - let query = format!("UPDATE queue - SET running = true - , started_at = coalesce(started_at, now()) - , last_ping = now() - , suspend_until = null - WHERE id = ( - SELECT id - FROM queue + let query = format_pull_query(format!( + "SELECT id + FROM v2_job_queue WHERE running = false AND tag IN ({}) AND scheduled_for <= now() ORDER BY priority DESC NULLS LAST, scheduled_for FOR UPDATE SKIP LOCKED - LIMIT 1 - ) - RETURNING id, workspace_id, parent_job, created_by, created_at, started_at, scheduled_for, - running, script_hash, script_path, args, null as logs, canceled, canceled_by, - canceled_reason, last_ping, job_kind, schedule_path, permissioned_as, - flow_status, is_flow_step, language, suspend, suspend_until, - same_worker, pre_run_error, email, visible_to_owner, mem_peak, - root_job, leaf_jobs, tag, concurrent_limit, concurrency_time_window_s, - timeout, flow_step_id, cache_ttl, priority, - raw_code, raw_lock, raw_flow", tags.tags.iter().map(|x| format!("'{x}'")).join(", ")); - + LIMIT 1", + tags.tags.iter().map(|x| format!("'{x}'")).join(", ") + )); queries.push(query); } - let mut l = WORKER_PULL_QUERIES.write().await; *l = queries; } @@ -907,7 +927,7 @@ impl Default for WorkerConfigOpt { } } -#[derive(PartialEq, Debug, Clone)] +#[derive(PartialEq, Clone)] pub struct WorkerConfig { pub worker_tags: Vec, pub priority_tags_sorted: Vec, @@ -919,6 +939,13 @@ pub struct WorkerConfig { pub env_vars: HashMap, } +impl std::fmt::Debug for WorkerConfig { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "WorkerConfig {{ worker_tags: {:?}, priority_tags_sorted: {:?}, dedicated_worker: {:?}, init_bash: {:?}, cache_clear: {:?}, additional_python_paths: {:?}, pip_local_dependencies: {:?}, env_vars: {:?} }}", + self.worker_tags, self.priority_tags_sorted, self.dedicated_worker, self.init_bash, self.cache_clear, self.additional_python_paths, self.pip_local_dependencies, self.env_vars.iter().map(|(k, v)| format!("{}: {}{} ({} chars)", k, &v[..3.min(v.len())], "***", v.len())).collect::>().join(", ")) + } +} + #[derive(PartialEq, Debug, Clone)] pub struct PriorityTags { pub priority: u8, diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 2405be5b61..f561549af8 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -34,23 +34,21 @@ use windmill_common::{ error::{self, to_anyhow, Error}, flow_status::{ BranchAllStatus, FlowCleanupModule, FlowStatus, FlowStatusModule, FlowStatusModuleWParent, - Iterator, JobResult, RestartedFrom, RetryStatus, MAX_RETRY_ATTEMPTS, MAX_RETRY_INTERVAL, + Iterator as FlowIterator, JobResult, RestartedFrom, RetryStatus, MAX_RETRY_ATTEMPTS, + MAX_RETRY_INTERVAL, }, flows::{ add_virtual_items_if_necessary, FlowModule, FlowModuleValue, FlowValue, InputTransform, }, - jobs::{ - get_payload_tag_from_prefixed_path, CompletedJob, JobKind, JobPayload, QueuedJob, RawCode, - ENTRYPOINT_OVERRIDE, PREPROCESSOR_FAKE_ENTRYPOINT, - }, + jobs::{get_payload_tag_from_prefixed_path, JobKind, JobPayload, QueuedJob, RawCode}, schedule::Schedule, scripts::{get_full_hub_script_by_path, ScriptHash, ScriptLang}, users::{SUPERADMIN_NOTIFICATION_EMAIL, SUPERADMIN_SECRET_EMAIL}, utils::{not_found_if_none, report_critical_error, StripPath, WarnAfterExt}, worker::{ to_raw_value, CLOUD_HOSTED, DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES, - DISABLE_FLOW_SCRIPT, MIN_VERSION_IS_AT_LEAST_1_427, MIN_VERSION_IS_AT_LEAST_1_432, - MIN_VERSION_IS_AT_LEAST_1_440, NO_LOGS, WORKER_PULL_QUERIES, WORKER_SUSPENDED_PULL_QUERY, + DISABLE_FLOW_SCRIPT, MIN_VERSION_IS_AT_LEAST_1_432, MIN_VERSION_IS_AT_LEAST_1_440, NO_LOGS, + WORKER_PULL_QUERIES, WORKER_SUSPENDED_PULL_QUERY, }, DB, METRICS_ENABLED, }; @@ -173,7 +171,7 @@ pub async fn cancel_single_job<'c>( }); } else { let id: Option = sqlx::query_scalar!( - "UPDATE queue SET canceled = true, canceled_by = $1, canceled_reason = $2, scheduled_for = now(), suspend = 0 WHERE id = $3 AND workspace_id = $4 AND (canceled = false OR canceled_reason != $2) RETURNING id", + "UPDATE v2_job_queue SET canceled_by = $1, canceled_reason = $2, scheduled_for = now(), suspend = 0 WHERE id = $3 AND workspace_id = $4 AND (canceled_by IS NULL OR canceled_reason != $2) RETURNING id", username, reason, job_running.id, @@ -249,7 +247,7 @@ pub async fn cancel_job<'c>( while !jobs.is_empty() { let p_job = jobs.pop(); let new_jobs = sqlx::query_scalar!( - "SELECT id FROM queue WHERE parent_job = $1 AND workspace_id = $2", + "SELECT id AS \"id!\" FROM v2_job WHERE parent_job = $1 AND workspace_id = $2", p_job, w_id ) @@ -361,7 +359,7 @@ async fn cancel_persistent_script_jobs_internal<'c>( // we could have retrieved the job IDs in the first query where we retrieve the hashes, but just in case a job was inserted in the queue right in-between the two above query, we re-do the fetch here let jobs_to_cancel = sqlx::query_scalar::<_, Uuid>( - "SELECT id FROM queue WHERE workspace_id = $1 AND script_path = $2 AND canceled = false", + "SELECT id FROM v2_as_queue WHERE workspace_id = $1 AND script_path = $2 AND canceled = false", ) .bind(w_id) .bind(script_path) @@ -395,23 +393,43 @@ pub struct WrappedError { pub trait ValidableJson { fn is_valid_json(&self) -> bool; + fn wm_labels(&self) -> Option>; +} + +#[derive(serde::Deserialize)] +struct ResultLabels { + wm_labels: Vec, } impl ValidableJson for WrappedError { fn is_valid_json(&self) -> bool { true } + + fn wm_labels(&self) -> Option> { + None + } } impl ValidableJson for Box { fn is_valid_json(&self) -> bool { !self.get().is_empty() } + + fn wm_labels(&self) -> Option> { + serde_json::from_str::(self.get()) + .ok() + .map(|r| r.wm_labels) + } } -impl ValidableJson for Arc> { +impl ValidableJson for Arc { fn is_valid_json(&self) -> bool { - !self.get().is_empty() + T::is_valid_json(&self) + } + + fn wm_labels(&self) -> Option> { + T::wm_labels(&self) } } @@ -419,12 +437,22 @@ impl ValidableJson for serde_json::Value { fn is_valid_json(&self) -> bool { true } + + fn wm_labels(&self) -> Option> { + serde_json::from_value::(self.clone()) + .ok() + .map(|r| r.wm_labels) + } } impl ValidableJson for Json { fn is_valid_json(&self) -> bool { self.0.is_valid_json() } + + fn wm_labels(&self) -> Option> { + self.0.wm_labels() + } } pub async fn register_metric( @@ -499,6 +527,7 @@ pub async fn add_completed_job_error( false, false, Json(&result), + None, mem_peak, canceled_by, flow_is_done, @@ -518,6 +547,7 @@ pub async fn add_completed_job( success: bool, skipped: bool, result: Json<&T>, + result_columns: Option>, mem_peak: i32, canceled_by: Option, flow_is_done: bool, @@ -528,11 +558,12 @@ pub async fn add_completed_job( // add_time!(bench, "add_completed_job start"); if !result.is_valid_json() { - return Err(Error::InternalErr( + return Err(Error::internal_err( "Result of job is invalid json (empty)".to_string(), )); } + let result_columns = result_columns.as_ref(); let _job_id = queued_job.id; let (opt_uuid, _duration, _skip_downstream_error_handlers) = (|| async { let mut tx = db.begin().await?; @@ -546,133 +577,87 @@ pub async fn add_completed_job( serde_json::to_string(&result).unwrap_or_else(|_| "".to_string()) ); - let (raw_code, raw_lock, raw_flow) = if !*MIN_VERSION_IS_AT_LEAST_1_427.read().await { - sqlx::query!( - "SELECT raw_code, raw_lock, raw_flow AS \"raw_flow: Json>\" - FROM job WHERE id = $1 AND workspace_id = $2 LIMIT 1", - &job_id, - &queued_job.workspace_id - ) - .fetch_one(db) - .map_ok(|record| (record.raw_code, record.raw_lock, record.raw_flow)) - .or_else(|_| { - sqlx::query!( - "SELECT raw_code, raw_lock, raw_flow AS \"raw_flow: Json>\" - FROM queue WHERE id = $1 AND workspace_id = $2 LIMIT 1", - &job_id, - &queued_job.workspace_id - ) - .fetch_one(db) - .map_ok(|record| (record.raw_code, record.raw_lock, record.raw_flow)) - }) - .await - .unwrap_or_default() - } else { - (None, None, None) - }; - let mem_peak = mem_peak.max(queued_job.mem_peak.unwrap_or(0)); // add_time!(bench, "add_completed_job query START"); let _duration = sqlx::query_scalar!( - "INSERT INTO completed_job AS cj + "INSERT INTO v2_job_completed AS cj ( workspace_id , id - , parent_job - , created_by - , created_at , started_at , duration_ms - , success - , script_hash - , script_path - , args , result - , raw_code - , raw_lock - , canceled + , result_columns , canceled_by , canceled_reason - , job_kind - , schedule_path - , permissioned_as , flow_status - , raw_flow - , is_flow_step - , is_skipped - , language - , email - , visible_to_owner - , mem_peak - , tag - , priority + , workflow_as_code_status + , memory_peak + , status + , worker ) - VALUES ($1, $2, $3, $4, $5, COALESCE($6, now()), COALESCE($30::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE($6, now()))))*1000), $7, $8, $9,\ - $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29) - ON CONFLICT (id) DO UPDATE SET success = $7, result = $11 RETURNING duration_ms", - queued_job.workspace_id, - queued_job.id, - queued_job.parent_job, - queued_job.created_by, - queued_job.created_at, - queued_job.started_at, - success, - queued_job.script_hash.map(|x| x.0), - queued_job.script_path, - &queued_job.args as &Option>>>, - result as Json<&T>, - raw_code, - raw_lock, - canceled_by.is_some(), - canceled_by.clone().map(|cb| cb.username).flatten(), - canceled_by.clone().map(|cb| cb.reason).flatten(), - queued_job.job_kind.clone() as JobKind, - queued_job.schedule_path, - queued_job.permissioned_as, - &queued_job.flow_status as &Option>>, - &raw_flow as &Option>>, - queued_job.is_flow_step, - skipped, - queued_job.language.clone() as Option, - queued_job.email, - queued_job.visible_to_owner, - if mem_peak > 0 { Some(mem_peak) } else { None }, - queued_job.tag, - queued_job.priority, - duration, + SELECT q.workspace_id, q.id, started_at, COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(started_at, now()))))*1000), $3, $10, $5, $6, + flow_status, workflow_as_code_status, + $8, CASE WHEN $4::BOOL THEN 'canceled'::job_status + WHEN $7::BOOL THEN 'skipped'::job_status + WHEN $2::BOOL THEN 'success'::job_status + ELSE 'failure'::job_status END AS status, + q.worker + FROM v2_job_queue q LEFT JOIN v2_job_status USING (id) WHERE q.id = $1 + ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = $3 RETURNING duration_ms AS \"duration_ms!\"", + /* $1 */ queued_job.id, + /* $2 */ success, + /* $3 */ result as Json<&T>, + /* $4 */ canceled_by.is_some(), + /* $5 */ canceled_by.clone().map(|cb| cb.username).flatten(), + /* $6 */ canceled_by.clone().map(|cb| cb.reason).flatten(), + /* $7 */ skipped, + /* $8 */ if mem_peak > 0 { Some(mem_peak) } else { None }, + /* $9 */ duration, + /* $10 */ result_columns as Option<&Vec>, ) .fetch_one(&mut *tx) .await - .map_err(|e| Error::InternalErr(format!("Could not add completed job {job_id}: {e:#}")))?; + .map_err(|e| Error::internal_err(format!("Could not add completed job {job_id}: {e:#}")))?; + if let Some(labels) = result.wm_labels() { + sqlx::query!( + "UPDATE v2_job SET labels = ( + SELECT array_agg(DISTINCT all_labels) + FROM unnest(coalesce(labels, ARRAY[]::TEXT[]) || $2) all_labels + ) WHERE id = $1", + job_id, + labels as Vec + ) + .execute(&mut *tx) + .await + .map_err(|e| Error::InternalErr(format!("Could not update job labels: {e:#}")))?; + } if !queued_job.is_flow_step { - if _duration > 500 - && (queued_job.job_kind == JobKind::Script - || queued_job.job_kind == JobKind::Preview) - { - if let Err(e) = sqlx::query!( - "UPDATE completed_job SET flow_status = q.flow_status FROM queue q WHERE completed_job.id = $1 AND q.id = $1 AND q.workspace_id = $2 AND completed_job.workspace_id = $2 AND q.flow_status IS NOT NULL", - &queued_job.id, - &queued_job.workspace_id - ) - .execute(&mut *tx) - .await { - tracing::error!("Could not update job duration: {}", e); - } - } if let Some(parent_job) = queued_job.parent_job { - if let Err(e) = sqlx::query_scalar!( - "UPDATE queue SET flow_status = jsonb_set(jsonb_set(COALESCE(flow_status, '{}'::jsonb), array[$1], COALESCE(flow_status->$1, '{}'::jsonb)), array[$1, 'duration_ms'], to_jsonb($2::bigint)) WHERE id = $3 AND workspace_id = $4", - &queued_job.id.to_string(), - _duration, - parent_job, - &queued_job.workspace_id - ) - .execute(&mut *tx) - .await { - tracing::error!("Could not update parent job flow_status: {}", e); - } + let _ = sqlx::query_scalar!( + "UPDATE v2_job_status SET + workflow_as_code_status = jsonb_set( + jsonb_set( + COALESCE(workflow_as_code_status, '{}'::jsonb), + array[$1], + COALESCE(workflow_as_code_status->$1, '{}'::jsonb) + ), + array[$1, 'duration_ms'], + to_jsonb($2::bigint) + ) + WHERE id = $3", + &queued_job.id.to_string(), + _duration, + parent_job + ) + .execute(&mut *tx) + .await + .inspect_err(|e| tracing::error!( + "Could not update parent job `duration_ms` in workflow as code status: {}", + e, + )); } } // tracing::error!("Added completed job {:#?}", queued_job); @@ -689,12 +674,17 @@ pub async fn add_completed_job( parent_job ); sqlx::query!( - "UPDATE queue SET last_ping = now() WHERE id = $1 AND workspace_id = $2 AND canceled = false", - parent_job, - &queued_job.workspace_id - ) - .execute(&mut *tx) - .await?; + "UPDATE v2_job_runtime r SET + ping = now() + FROM v2_job_queue q + WHERE r.id = $1 AND q.id = r.id + AND q.workspace_id = $2 + AND canceled_by IS NULL", + parent_job, + &queued_job.workspace_id + ) + .execute(&mut *tx) + .await?; if flow_is_done { let r = sqlx::query_scalar!( "UPDATE parallel_monitor_lock SET last_ping = now() WHERE parent_flow_id = $1 and job_id = $2 RETURNING 1", @@ -703,9 +693,9 @@ pub async fn add_completed_job( ).fetch_optional(&mut *tx).await?; if r.is_some() { tracing::info!( - "parallel flow iteration is done, setting parallel monitor last ping lock for job {}", - &queued_job.id - ); + "parallel flow iteration is done, setting parallel monitor last ping lock for job {}", + &queued_job.id + ); } } } @@ -728,21 +718,21 @@ pub async fn add_completed_job( // otherwise flow rescheduling is done inside handle_flow let schedule_next_tick = !queued_job.is_flow() || !success && sqlx::query_scalar!( - "SELECT - flow_status->>'step' = '0' - AND ( - jsonb_array_length(flow_status->'modules') = 0 - OR flow_status->'modules'->0->>'type' = 'WaitingForPriorSteps' - OR ( - flow_status->'modules'->0->>'type' = 'Failure' - AND flow_status->'modules'->0->>'job' = $1 - ) - ) - FROM completed_job WHERE id = $2 AND workspace_id = $3", - Uuid::nil().to_string(), - &queued_job.id, - &queued_job.workspace_id - ).fetch_optional(&mut *tx).await?.flatten().unwrap_or(false); + "SELECT + flow_status->>'step' = '0' + AND ( + jsonb_array_length(flow_status->'modules') = 0 + OR flow_status->'modules'->0->>'type' = 'WaitingForPriorSteps' + OR ( + flow_status->'modules'->0->>'type' = 'Failure' + AND flow_status->'modules'->0->>'job' = $1 + ) + ) + FROM v2_job_completed WHERE id = $2 AND workspace_id = $3", + Uuid::nil().to_string(), + &queued_job.id, + &queued_job.workspace_id + ).fetch_optional(&mut *tx).await?.flatten().unwrap_or(false); if schedule_next_tick { if let Err(err) = handle_maybe_scheduled_job( @@ -782,7 +772,7 @@ pub async fn add_completed_job( let w_id: &String = &queued_job.workspace_id; if !matches!(err, Error::QuotaExceeded(_)) { report_error_to_workspace_handler_or_critical_side_channel( - &queued_job, + &queued_job, db, format!( "Failed to push schedule error handler job to handle failed job ({base_url}/run/{}?workspace={w_id}): {}", @@ -798,9 +788,9 @@ pub async fn add_completed_job( }; } else { tracing::error!( - "Schedule {schedule_path} in {} not found. Impossible to schedule again and apply schedule handlers", - &queued_job.workspace_id - ); + "Schedule {schedule_path} in {} not found. Impossible to schedule again and apply schedule handlers", + &queued_job.workspace_id + ); } } } @@ -818,10 +808,10 @@ pub async fn add_completed_job( } }; if let Err(e) = sqlx::query_scalar!( - "UPDATE concurrency_counter SET job_uuids = job_uuids - $2 WHERE concurrency_id = $1", - concurrency_key, - queued_job.id.hyphenated().to_string(), - ) + "UPDATE concurrency_counter SET job_uuids = job_uuids - $2 WHERE concurrency_id = $1", + concurrency_key, + queued_job.id.hyphenated().to_string(), + ) .execute(&mut *tx) .await { @@ -835,9 +825,9 @@ pub async fn add_completed_job( .execute(&mut *tx) .await .map_err(|e| { - Error::InternalErr(format!( - "Error updating to add ended_at timestamp concurrency_key={concurrency_key}: {e:#}" - )) + Error::internal_err(format!( + "Error updating to add ended_at timestamp concurrency_key={concurrency_key}: {e:#}" + )) }) { tracing::error!("Could not update concurrency_key: {}", e); } @@ -898,16 +888,19 @@ pub async fn add_completed_job( sqlx::query_scalar!("SELECT premium FROM workspace WHERE id = $1", w_id) .fetch_one(db) .await - .map_err(|e| Error::InternalErr(format!("fetching if {w_id} is premium: {e:#}")))?; + .map_err(|e| { + Error::internal_err(format!("fetching if {w_id} is premium: {e:#}")) + })?; let _ = sqlx::query!( - "INSERT INTO usage (id, is_workspace, month_, usage) - VALUES ($1, TRUE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), $2) - ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + $2", - w_id, - additional_usage as i32) - .execute(db) - .await - .map_err(|e| Error::InternalErr(format!("updating usage: {e:#}"))); + "INSERT INTO usage (id, is_workspace, month_, usage) + VALUES ($1, TRUE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), $2) + ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + $2", + w_id, + additional_usage as i32 + ) + .execute(db) + .await + .map_err(|e| Error::internal_err(format!("updating usage: {e:#}"))); if !premium_workspace { let _ = sqlx::query!( @@ -915,10 +908,11 @@ pub async fn add_completed_job( VALUES ($1, FALSE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), $2) ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + $2", queued_job.email, - additional_usage as i32) - .execute(db) - .await - .map_err(|e| Error::InternalErr(format!("updating usage: {e:#}"))); + additional_usage as i32 + ) + .execute(db) + .await + .map_err(|e| Error::internal_err(format!("updating usage: {e:#}"))); } } @@ -928,15 +922,14 @@ pub async fn add_completed_job( if let Ok(flow) = cache::job::fetch_flow(db, job.job_kind, job.script_hash).await { return flow.value().failure_module.is_some(); } - sqlx::query_scalar!("SELECT raw_flow->'failure_module' != 'null'::jsonb FROM job WHERE id = $1", job.id) - .fetch_one(db) - .or_else(|_| - sqlx::query_scalar!("SELECT raw_flow->'failure_module' != 'null'::jsonb FROM completed_job WHERE id = $1", job.id) - .fetch_one(db) - ) - .await - .unwrap_or(Some(false)) - .unwrap_or(false) + sqlx::query_scalar!( + "SELECT raw_flow->'failure_module' != 'null'::jsonb FROM v2_job WHERE id = $1", + job.id + ) + .fetch_one(db) + .await + .unwrap_or(Some(false)) + .unwrap_or(false) } if queued_job.email == ERROR_HANDLER_USER_EMAIL { @@ -1157,7 +1150,8 @@ pub async fn report_error_to_workspace_handler_or_critical_side_channel( let w_id = &queued_job.workspace_id; let (error_handler, error_handler_extra_args) = sqlx::query_as::<_, (Option, Option>>)>( "SELECT error_handler, error_handler_extra_args FROM workspace_settings WHERE workspace_id = $1", - ).bind(&w_id) + ) + .bind(&w_id) .fetch_optional(db) .await .ok() @@ -1209,11 +1203,12 @@ pub async fn send_error_to_workspace_handler<'a, 'c, T: Serialize + Send + Sync> let w_id = &queued_job.workspace_id; let (error_handler, error_handler_extra_args, error_handler_muted_on_cancel) = sqlx::query_as::<_, (Option, Option>>, bool)>( "SELECT error_handler, error_handler_extra_args, error_handler_muted_on_cancel FROM workspace_settings WHERE workspace_id = $1", - ).bind(&w_id) + ) + .bind(&w_id) .fetch_optional(db) .await .context("fetching error handler info from workspace_settings")? - .ok_or_else(|| Error::InternalErr(format!("no workspace settings for id {w_id}")))?; + .ok_or_else(|| Error::internal_err(format!("no workspace settings for id {w_id}")))?; if is_canceled && error_handler_muted_on_cancel { return Ok(()); @@ -1291,7 +1286,7 @@ pub async fn handle_maybe_scheduled_job<'c>( tx.commit().await?; Ok::<(), Error>(()) }) - .map_err(|e| Error::InternalErr(format!("Pushing next scheduled job timedout: {e:#}"))) + .map_err(|e| Error::internal_err(format!("Pushing next scheduled job timedout: {e:#}"))) .unwrap_or_else(|e| Err(e)) }) .retry( @@ -1324,7 +1319,7 @@ pub async fn handle_maybe_scheduled_job<'c>( Error::QuotaExceeded(_) => {} _ => { report_error_to_workspace_handler_or_critical_side_channel(job, db, - format!("Could not schedule next job for {} with err {}. Schedule disabled", schedule.path, err.to_string()) + format!("Could not schedule next job for {} with err {}. Schedule disabled", schedule.path, err.to_string()), ).await; } } @@ -1334,8 +1329,8 @@ pub async fn handle_maybe_scheduled_job<'c>( Error::QuotaExceeded(_) => Err(err), _ => { report_error_to_workspace_handler_or_critical_side_channel(job, db, - format!("Could not schedule next job for {} and could not disable schedule with err {}.", schedule.path, disable_err) - ).await; + format!("Could not schedule next job for {} and could not disable schedule with err {}.", schedule.path, disable_err), + ).await; Err(to_anyhow(disable_err).into()) } }, @@ -1381,15 +1376,23 @@ async fn apply_schedule_handlers<'a, 'c, T: Serialize + Send + Sync>( let times = schedule.on_failure_times.unwrap_or(1).max(1); let exact = schedule.on_failure_exact.unwrap_or(false); if times > 1 || exact { - let past_jobs = sqlx::query_as::<_, CompletedJobSubset>( - "SELECT success, result, started_at FROM completed_job WHERE workspace_id = $1 AND schedule_path = $2 AND script_path = $3 AND id != $4 ORDER BY created_at DESC LIMIT $5", + let past_jobs = sqlx::query!( + "SELECT + success AS \"success!\", + result AS \"result: Json>\", + started_at AS \"started_at!\" + FROM v2_as_completed_job + WHERE workspace_id = $1 AND schedule_path = $2 AND script_path = $3 AND id != $4 + ORDER BY created_at DESC + LIMIT $5", + &schedule.workspace_id, + &schedule.path, + script_path, + job_id, + if exact { times } else { times - 1 } as i64 ) - .bind(&schedule.workspace_id) - .bind(&schedule.path) - .bind(script_path) - .bind(job_id) - .bind(if exact { times } else { times - 1 } as i64,) - .fetch_all(db).await?; + .fetch_all(db) + .await?; let match_times = if exact { past_jobs.len() == times as usize @@ -1444,15 +1447,22 @@ async fn apply_schedule_handlers<'a, 'c, T: Serialize + Send + Sync>( if let Some(ref on_recovery_path) = schedule.on_recovery.clone() { let tx = db.begin().await?; let times = schedule.on_recovery_times.unwrap_or(1).max(1); - let past_jobs = sqlx::query_as::<_, CompletedJobSubset>( - "SELECT success, result, started_at FROM completed_job WHERE workspace_id = $1 AND schedule_path = $2 AND script_path = $3 AND id != $4 ORDER BY created_at DESC LIMIT $5", + let past_jobs = sqlx::query!( + "SELECT + success AS \"success!\", + result AS \"result: Json>\", + started_at AS \"started_at!\"\ + FROM v2_as_completed_job WHERE workspace_id = $1 AND schedule_path = $2 AND script_path = $3 AND id != $4 + ORDER BY created_at DESC + LIMIT $5", + &schedule.workspace_id, + &schedule.path, + script_path, + job_id, + times as i64 ) - .bind(&schedule.workspace_id) - .bind(&schedule.path) - .bind(script_path) - .bind(job_id) - .bind(times as i64) - .fetch_all(db).await?; + .fetch_all(db) + .await?; if past_jobs.len() < times as usize { return Ok(()); @@ -1464,7 +1474,7 @@ async fn apply_schedule_handlers<'a, 'c, T: Serialize + Send + Sync>( return Ok(()); } - let failed_job = past_jobs[past_jobs.len() - 1].clone(); + let failed_job = &past_jobs[past_jobs.len() - 1]; if !failed_job.success { handle_recovered_schedule( @@ -1476,7 +1486,8 @@ async fn apply_schedule_handlers<'a, 'c, T: Serialize + Send + Sync>( schedule.is_flow, w_id, &on_recovery_path, - failed_job, + failed_job.result.as_ref().map(AsRef::as_ref), + failed_job.started_at, result, times, started_at, @@ -1622,7 +1633,8 @@ async fn handle_recovered_schedule<'a, 'c, T: Serialize + Send + Sync>( is_flow: bool, w_id: &str, on_recovery_path: &str, - error_job: CompletedJobSubset, + result: Option<&Box>, + started_at: DateTime, successful_job_result: Json<&'a T>, successful_times: i32, successful_job_started_at: DateTime, @@ -1632,10 +1644,7 @@ async fn handle_recovered_schedule<'a, 'c, T: Serialize + Send + Sync>( get_payload_tag_from_prefixed_path(on_recovery_path, db, w_id).await?; let mut extra = HashMap::new(); - extra.insert( - "error_started_at".to_string(), - to_raw_value(&error_job.started_at), - ); + extra.insert("error_started_at".to_string(), to_raw_value(&started_at)); extra.insert("schedule_path".to_string(), to_raw_value(&schedule_path)); extra.insert("path".to_string(), to_raw_value(&script_path)); extra.insert("is_flow".to_string(), to_raw_value(&is_flow)); @@ -1661,9 +1670,8 @@ async fn handle_recovered_schedule<'a, 'c, T: Serialize + Send + Sync>( } } - let args = error_job - .result - .and_then(|x| serde_json::from_str::>>(x.0.get()).ok()) + let args = result + .and_then(|x| serde_json::from_str::>>(x.get()).ok()) .unwrap_or_else(HashMap::new); let (email, permissioned_as) = if let Some(on_behalf_of) = on_behalf_of.as_ref() { @@ -1820,10 +1828,15 @@ impl std::ops::Deref for PulledJob { pub async fn pull( db: &Pool, suspend_first: bool, + worker_name: &str, ) -> windmill_common::error::Result<(Option, bool)> { loop { - let (job, suspended) = - pull_single_job_and_mark_as_running_no_concurrency_limit(db, suspend_first).await?; + let (job, suspended) = pull_single_job_and_mark_as_running_no_concurrency_limit( + db, + suspend_first, + worker_name, + ) + .await?; let Some(job) = job else { return Ok((None, suspended)); @@ -1878,7 +1891,7 @@ pub async fn pull( ); sqlx::query_scalar!( - "SELECT null FROM queue WHERE id = $1 FOR UPDATE", + "SELECT null FROM v2_job_queue WHERE id = $1 FOR UPDATE", pulled_job.id ) .fetch_one(&mut *tx) @@ -1901,7 +1914,7 @@ pub async fn pull( .fetch_one(&mut *tx) .await .map_err(|e| { - Error::InternalErr(format!( + Error::internal_err(format!( "Error getting concurrency count for script path {job_script_path}: {e:#}" )) })?; @@ -1912,14 +1925,14 @@ pub async fn pull( job_concurrency_key, f64::from(job_custom_concurrency_time_window_s), ).fetch_one(&mut *tx).await.map_err(|e| { - Error::InternalErr(format!( + Error::internal_err(format!( "Error getting completed count for key {job_concurrency_key}: {e:#}" )) })?; let min_started_at = sqlx::query!( "SELECT COALESCE((SELECT MIN(started_at) as min_started_at - FROM queue + FROM v2_as_queue WHERE script_path = $1 AND job_kind != 'dependencies' AND running = true AND workspace_id = $2 AND canceled = false AND concurrent_limit > 0), $3) as min_started_at, now() AS now", job_script_path, &pulled_job.workspace_id, @@ -1928,7 +1941,7 @@ pub async fn pull( .fetch_one(&mut *tx) .await .map_err(|e| { - Error::InternalErr(format!( + Error::internal_err(format!( "Error getting concurrency count for script path {job_script_path}: {e:#}" )) })?; @@ -1956,7 +1969,7 @@ pub async fn pull( .fetch_one(&mut *tx) .await .map_err(|e| { - Error::InternalErr(format!( + Error::internal_err(format!( "Error decreasing concurrency count for script path {job_script_path}: {e:#}" )) })?; @@ -1966,7 +1979,7 @@ pub async fn pull( let job_uuid: Uuid = pulled_job.id; let avg_script_duration: Option = sqlx::query_scalar!( "SELECT CAST(ROUND(AVG(duration_ms), 0) AS BIGINT) AS avg_duration_s FROM - (SELECT duration_ms FROM concurrency_key LEFT JOIN completed_job ON completed_job.id = concurrency_key.job_id WHERE key = $1 AND ended_at IS NOT NULL + (SELECT duration_ms FROM concurrency_key LEFT JOIN v2_as_completed_job ON v2_as_completed_job.id = concurrency_key.job_id WHERE key = $1 AND ended_at IS NOT NULL ORDER BY ended_at DESC LIMIT 10) AS t", job_concurrency_key @@ -1998,7 +2011,7 @@ pub async fn pull( loop { let nestimated = estimated_next_schedule_timestamp + inc; let jobs_in_window = sqlx::query_scalar!( - "SELECT COUNT(*) FROM queue LEFT JOIN concurrency_key ON concurrency_key.job_id = queue.id + "SELECT COUNT(*) FROM v2_as_queue LEFT JOIN concurrency_key ON concurrency_key.job_id = v2_as_queue.id WHERE key = $1 AND running = false AND canceled = false AND scheduled_for >= $2 AND scheduled_for < $3", job_concurrency_key, estimated_next_schedule_timestamp, @@ -2022,18 +2035,19 @@ pub async fn pull( // if using posgtres, then we're able to re-queue the entire batch of scheduled job for this script_path, so we do it sqlx::query!( - "UPDATE queue - SET running = false - , started_at = null - , scheduled_for = $1 - , last_ping = null - WHERE id = $2", - estimated_next_schedule_timestamp, - job_uuid, - ) - .fetch_all(&mut *tx) - .await - .map_err(|e| Error::InternalErr(format!("Could not update and re-queue job {job_uuid}. The job will be marked as running but it is not running: {e:#}")))?; + "WITH ping AS (UPDATE v2_job_runtime SET ping = NULL WHERE id = $2 RETURNING id) + UPDATE v2_job_queue SET + running = false, + started_at = null, + scheduled_for = $1 + WHERE id = $2", + estimated_next_schedule_timestamp, + job_uuid, + ) + .execute(&mut *tx) + .await + .map_err(|e| Error::internal_err(format!("Could not update and re-queue job {job_uuid}. The job will be marked as running but it is not running: {e:#}")))?; + tx.commit().await? } } @@ -2041,6 +2055,7 @@ pub async fn pull( async fn pull_single_job_and_mark_as_running_no_concurrency_limit<'c>( db: &Pool, suspend_first: bool, + worker_name: &str, ) -> windmill_common::error::Result<(Option, bool)> { let job_and_suspended: (Option, bool) = { /* Jobs can be started if they: @@ -2060,6 +2075,7 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<'c>( let r = if suspend_first { // tracing::info!("Pulling job with query: {}", query); sqlx::query_as::<_, PulledJob>(&query) + .bind(worker_name) .fetch_optional(db) .await? } else { @@ -2080,6 +2096,7 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<'c>( for query in queries.iter() { // tracing::info!("Pulling job with query: {}", query); let r = sqlx::query_as::<_, PulledJob>(query) + .bind(worker_name) .fetch_optional(db) .await?; @@ -2193,17 +2210,6 @@ fn fullpath_with_workspace( ) } -#[derive(FromRow)] -pub struct ResultR { - result: Option>>, -} - -#[derive(FromRow)] -pub struct ResultWithId { - result: Option>>, - id: Uuid, -} - pub async fn get_result_by_id( db: Pool, w_id: String, @@ -2222,25 +2228,29 @@ pub async fn get_result_by_id( { Ok(res) => Ok(res), Err(_) => { - let running_flow_job =sqlx::query_as::<_, QueuedJob>( - "SELECT * FROM queue WHERE COALESCE((SELECT root_job FROM queue WHERE id = $1), $1) = id AND workspace_id = $2" - ).bind(flow_id) - .bind(&w_id) - .fetch_optional(&db).await?; - match running_flow_job { - Some(job) => { - let restarted_from = windmill_common::utils::not_found_if_none( - job.parse_flow_status() - .map(|status| status.restarted_from) - .flatten(), + let root = sqlx::query!( + "SELECT + id As \"id!\", + flow_status->'restarted_from'->'flow_job_id' AS \"restarted_from: Json\" + FROM v2_as_queue + WHERE COALESCE((SELECT flow_innermost_root_job FROM v2_job WHERE id = $1), $1) = id AND workspace_id = $2", + flow_id, + &w_id + ) + .fetch_optional(&db) + .await?; + match root { + Some(root) => { + let restarted_from_id = not_found_if_none( + root.restarted_from, "Id not found in the result's mapping of the root job and root job had no restarted from information", - format!("parent: {}, root: {}, id: {}", flow_id, job.id, node_id), + format!("parent: {}, root: {}, id: {}", flow_id, root.id, node_id), )?; get_result_by_id_from_original_flow( &db, w_id.as_str(), - &restarted_from.flow_job_id, + &restarted_from_id, node_id.as_str(), json_path.clone(), ) @@ -2261,12 +2271,6 @@ pub async fn get_result_by_id( } } -#[derive(FromRow)] -struct FlowJobResult { - leaf_jobs: Option>>, - parent_job: Option, -} - pub async fn get_result_and_success_by_id_from_flow( db: &Pool, w_id: &str, @@ -2307,7 +2311,8 @@ pub async fn get_result_and_success_by_id_from_flow( let success = match &job_result { JobResult::SingleJob(job_id) => { sqlx::query_scalar!( - "SELECT success FROM completed_job WHERE id = $1 AND workspace_id = $2", + "SELECT success AS \"success!\" + FROM v2_as_completed_job WHERE id = $1 AND workspace_id = $2", job_id, w_id ) @@ -2324,7 +2329,11 @@ pub async fn get_result_and_success_by_id_from_flow( SELECT module->>'type' = 'Success' FROM modules WHERE module->>'id' = $3"#, - if completed { "completed_job" } else { "queue" } + if completed { + "v2_as_completed_job" + } else { + "v2_as_queue" + } ); sqlx::query_scalar(&query) .bind(flow_id) @@ -2333,7 +2342,9 @@ pub async fn get_result_and_success_by_id_from_flow( .fetch_optional(db) .await? .ok_or_else(|| { - error::Error::InternalErr(format!("Could not get success from flow job status")) + error::Error::internal_err(format!( + "Could not get success from flow job status" + )) })? } }; @@ -2363,11 +2374,14 @@ pub async fn get_result_by_id_from_running_flow_inner( flow_id: &Uuid, node_id: &str, ) -> error::Result { - let flow_job_result = sqlx::query_as::<_, FlowJobResult>( - "SELECT leaf_jobs->$1::text as leaf_jobs, parent_job FROM queue WHERE COALESCE((SELECT root_job FROM queue WHERE id = $2), $2) = id AND workspace_id = $3") - .bind(node_id) - .bind(flow_id) - .bind(w_id) + let flow_job_result = sqlx::query!( + "SELECT leaf_jobs->$1::text AS \"leaf_jobs: Json>\", parent_job + FROM v2_as_queue + WHERE COALESCE((SELECT flow_innermost_root_job FROM v2_job WHERE id = $2), $2) = id AND workspace_id = $3", + node_id, + flow_id, + w_id, + ) .fetch_optional(db) .await?; @@ -2384,11 +2398,14 @@ pub async fn get_result_by_id_from_running_flow_inner( if job_result.is_none() && flow_job_result.parent_job.is_some() { let parent_job = flow_job_result.parent_job.unwrap(); - let root_job = sqlx::query_scalar!("SELECT root_job FROM queue WHERE id = $1", parent_job) - .fetch_optional(db) - .await? - .flatten() - .unwrap_or(parent_job); + let root_job = sqlx::query_scalar!( + "SELECT flow_innermost_root_job FROM v2_job WHERE id = $1", + parent_job + ) + .fetch_optional(db) + .await? + .flatten() + .unwrap_or(parent_job); return get_result_by_id_from_running_flow_inner(db, w_id, &root_job, node_id).await; } @@ -2401,18 +2418,13 @@ pub async fn get_result_by_id_from_running_flow_inner( Ok(result_id) } -#[async_recursion] async fn get_completed_flow_node_result_rec( db: &Pool, w_id: &str, - subflows: Vec, + subflows: impl std::iter::Iterator, node_id: &str, ) -> error::Result> { - for subflow in subflows { - let flow_status = subflow.parse_flow_status().ok_or_else(|| { - error::Error::InternalErr(format!("Could not parse flow status of {}", subflow.id)) - })?; - + for (id, flow_status) in subflows { if let Some(node_status) = flow_status .modules .iter() @@ -2423,15 +2435,27 @@ async fn get_completed_flow_node_result_rec( (Some(_), Some(jobs)) => Ok(Some(JobResult::ListJob(jobs))), _ => Err(error::Error::NotFound(format!( "Flow result by id not found going top-down in subflows (currently: {}), (id: {})", - subflow.id, + id, node_id, ))), }; } else { - let subflows = sqlx::query_as::<_, CompletedJob>( - "SELECT *, null as labels FROM completed_job WHERE parent_job = $1 AND workspace_id = $2 AND flow_status IS NOT NULL", - ).bind(subflow.id).bind(w_id).fetch_all(db).await?; - match get_completed_flow_node_result_rec(db, w_id, subflows, node_id).await? { + let subflows = sqlx::query!( + "SELECT id AS \"id!\", flow_status AS \"flow_status!: Json\" + FROM v2_as_completed_job + WHERE parent_job = $1 AND workspace_id = $2 AND flow_status IS NOT NULL", + id, + w_id + ) + .map(|record| (record.id, record.flow_status.0)) + .fetch_all(db) + .await? + .into_iter(); + match Box::pin(get_completed_flow_node_result_rec( + db, w_id, subflows, node_id, + )) + .await? + { Some(res) => return Ok(Some(res)), None => continue, }; @@ -2447,23 +2471,25 @@ async fn get_result_by_id_from_original_flow_inner( completed_flow_id: &Uuid, node_id: &str, ) -> error::Result { - let flow_job = sqlx::query_as::<_, CompletedJob>( - "SELECT *, null as labels FROM completed_job WHERE id = $1 AND workspace_id = $2", + let flow_job = sqlx::query!( + "SELECT id, flow_status AS \"flow_status!: Json\" + FROM v2_job_completed WHERE id = $1 AND workspace_id = $2", + completed_flow_id, + w_id ) - .bind(completed_flow_id) - .bind(w_id) + .map(|record| (record.id, record.flow_status.0)) .fetch_optional(db) .await?; - let flow_job = windmill_common::utils::not_found_if_none( + let flow_job = not_found_if_none( flow_job, "Root completed job", format!("root: {}, id: {}", completed_flow_id, node_id), )?; - match get_completed_flow_node_result_rec(db, w_id, vec![flow_job], node_id).await? { + match get_completed_flow_node_result_rec(db, w_id, [flow_job].into_iter(), node_id).await? { Some(res) => Ok(res), - None => Err(error::Error::NotFound(format!( + None => Err(Error::NotFound(format!( "Flow result by id not found going top-down from {}, (id: {})", completed_flow_id, node_id ))), @@ -2499,26 +2525,26 @@ async fn extract_result_from_job_result( let Some(job_id) = job_ids.get(idx).cloned() else { return Ok(to_raw_value(&serde_json::Value::Null)); }; - Ok(sqlx::query_as::<_, ResultR>( - "SELECT result #> $3 as result FROM completed_job WHERE id = $1 AND workspace_id = $2", - ) - .bind(job_id) - .bind(w_id) - .bind( - parts.map(|x| x.to_string()).collect::>() + Ok(sqlx::query_scalar!( + "SELECT result #> $3 AS \"result: Json>\" + FROM v2_job_completed WHERE id = $1 AND workspace_id = $2", + job_id, + w_id, + parts.collect::>() as Vec<&str> ) .fetch_optional(db) .await? - .map(|r| r.result.map(|x| x.0)) .flatten() + .map(|x| x.0) .unwrap_or_else(|| to_raw_value(&serde_json::Value::Null))) } None => { - let rows = sqlx::query_as::<_, ResultWithId>( - "SELECT id, result FROM completed_job WHERE id = ANY($1) AND workspace_id = $2", + let rows = sqlx::query!( + "SELECT id, result AS \"result: Json>\" + FROM v2_job_completed WHERE id = ANY($1) AND workspace_id = $2", + job_ids.as_slice(), + w_id ) - .bind(job_ids.as_slice()) - .bind(w_id) .fetch_all(db) .await? .into_iter() @@ -2535,15 +2561,15 @@ async fn extract_result_from_job_result( Ok(to_raw_value(&result)) } }, - JobResult::SingleJob(x) => Ok(sqlx::query_as::<_, ResultR>( - "SELECT result #> $3 as result FROM completed_job WHERE id = $1 AND workspace_id = $2", - ) - .bind(x) - .bind(w_id) - .bind( + JobResult::SingleJob(x) => Ok(sqlx::query!( + "SELECT result #> $3 AS \"result: Json>\" + FROM v2_job_completed WHERE id = $1 AND workspace_id = $2", + x, + w_id, json_path - .map(|x| x.split(".").map(|x| x.to_string()).collect::>()) - .unwrap_or_default(), + .as_ref() + .map(|x| x.split(".").collect::>()) + .unwrap_or_default() as Vec<&str>, ) .fetch_optional(db) .await? @@ -2564,7 +2590,7 @@ pub async fn delete_job<'c>( } let job_removed = sqlx::query_scalar!( - "DELETE FROM queue WHERE workspace_id = $1 AND id = $2 RETURNING 1", + "DELETE FROM v2_job_queue WHERE workspace_id = $1 AND id = $2 RETURNING 1", w_id, job_id ) @@ -2588,7 +2614,7 @@ pub async fn delete_job<'c>( pub async fn job_is_complete(db: &DB, id: Uuid, w_id: &str) -> error::Result { Ok(sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM completed_job WHERE id = $1 AND workspace_id = $2)", + "SELECT EXISTS(SELECT 1 FROM v2_job_completed WHERE id = $1 AND workspace_id = $2)", id, w_id ) @@ -2604,7 +2630,7 @@ async fn get_queued_job_tx<'c>( ) -> error::Result> { sqlx::query_as::<_, QueuedJob>( "SELECT * - FROM queue WHERE id = $1 AND workspace_id = $2", + FROM v2_as_queue WHERE id = $1 AND workspace_id = $2", ) .bind(id) .bind(w_id) @@ -2616,7 +2642,7 @@ async fn get_queued_job_tx<'c>( pub async fn get_queued_job(id: &Uuid, w_id: &str, db: &DB) -> error::Result> { sqlx::query_as::<_, QueuedJob>( "SELECT * - FROM queue WHERE id = $1 AND workspace_id = $2", + FROM v2_as_queue WHERE id = $1 AND workspace_id = $2", ) .bind(id) .bind(w_id) @@ -2744,7 +2770,7 @@ pub async fn push<'c, 'd>( parent_job: Option, root_job: Option, job_id: Option, - is_flow_step: bool, + _is_flow_step: bool, mut same_worker: bool, pre_run_error: Option<&windmill_common::error::Error>, visible_to_owner: bool, @@ -2761,7 +2787,7 @@ pub async fn push<'c, 'd>( .fetch_one(_db) .await .map_err(|e| { - Error::InternalErr(format!( + Error::internal_err(format!( "fetching if {workspace_id} is premium and overquota: {e:#}" )) })?; @@ -2780,7 +2806,7 @@ pub async fn push<'c, 'd>( ) .fetch_one(_db) .await - .map_err(|e| Error::InternalErr(format!("updating usage: {e:#}")))?; + .map_err(|e| Error::internal_err(format!("updating usage: {e:#}")))?; let user_usage = if !premium_workspace { Some(sqlx::query_scalar!( @@ -2792,7 +2818,7 @@ pub async fn push<'c, 'd>( ) .fetch_one(_db) .await - .map_err(|e| Error::InternalErr(format!("updating usage: {e:#}")))?) + .map_err(|e| Error::internal_err(format!("updating usage: {e:#}")))?) } else { None }; @@ -2843,11 +2869,13 @@ pub async fn push<'c, 'd>( ))); } - let in_queue = - sqlx::query_scalar!("SELECT COUNT(id) FROM queue WHERE email = $1", email) - .fetch_one(_db) - .await? - .unwrap_or(0); + let in_queue = sqlx::query_scalar!( + "SELECT COUNT(id) FROM v2_as_queue WHERE email = $1", + email + ) + .fetch_one(_db) + .await? + .unwrap_or(0); if in_queue > MAX_FREE_EXECS.into() { return Err(error::Error::QuotaExceeded(format!( @@ -2856,7 +2884,7 @@ pub async fn push<'c, 'd>( } let concurrent_runs = sqlx::query_scalar!( - "SELECT COUNT(id) FROM queue WHERE running = true AND email = $1", + "SELECT COUNT(id) FROM v2_as_queue WHERE running = true AND email = $1", email ) .fetch_one(_db) @@ -2875,16 +2903,16 @@ pub async fn push<'c, 'd>( workspace_usage } else { sqlx::query_scalar!( - "SELECT usage.usage + 1 FROM usage - WHERE is_workspace IS TRUE AND - month_ = EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date) - AND id = $1", - workspace_id - ) - .fetch_optional(_db) - .await? - .flatten() - .unwrap_or(1) + "SELECT usage.usage + 1 FROM usage + WHERE is_workspace IS TRUE AND + month_ = EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date) + AND id = $1", + workspace_id + ) + .fetch_optional(_db) + .await? + .flatten() + .unwrap_or(1) }; if workspace_usage > MAX_FREE_EXECS @@ -2898,7 +2926,7 @@ pub async fn push<'c, 'd>( } let in_queue_workspace = sqlx::query_scalar!( - "SELECT COUNT(id) FROM queue WHERE workspace_id = $1", + "SELECT COUNT(id) FROM v2_job_queue WHERE workspace_id = $1", workspace_id ) .fetch_one(_db) @@ -2912,7 +2940,7 @@ pub async fn push<'c, 'd>( } let concurrent_runs_workspace = sqlx::query_scalar!( - "SELECT COUNT(id) FROM queue WHERE running = true AND workspace_id = $1", + "SELECT COUNT(id) FROM v2_job_queue WHERE running = true AND workspace_id = $1", workspace_id ) .fetch_one(_db) @@ -2929,6 +2957,7 @@ pub async fn push<'c, 'd>( } } + let mut preprocessed = None; let ( script_hash, script_path, @@ -2958,10 +2987,7 @@ pub async fn push<'c, 'd>( } => { let extra = args.extra.get_or_insert_with(HashMap::new); if apply_preprocessor { - extra.insert( - ENTRYPOINT_OVERRIDE.to_string(), - to_raw_value(&PREPROCESSOR_FAKE_ENTRYPOINT), - ); + preprocessed = Some(false); extra.entry("wm_trigger".to_string()).or_insert_with(|| { to_raw_value(&serde_json::json!({ "kind": "webhook", @@ -3344,7 +3370,7 @@ pub async fn push<'c, 'd>( ) .fetch_optional(&mut *ntx) .await? - .ok_or_else(|| Error::InternalErr(format!("not found flow at path {:?}", path)))?; + .ok_or_else(|| Error::internal_err(format!("not found flow at path {:?}", path)))?; // Do not use the lite version unless all workers are updated. let data = if *DISABLE_FLOW_SCRIPT @@ -3376,6 +3402,7 @@ pub async fn push<'c, 'd>( status.preprocessor_module = None; extra.remove("wm_trigger"); } else { + preprocessed = Some(false); extra.entry("wm_trigger".to_string()).or_insert_with(|| { to_raw_value(&serde_json::json!({ "kind": "webhook", @@ -3650,12 +3677,9 @@ pub async fn push<'c, 'd>( let mut tx = tx.into_tx().await?; let job_id: Uuid = if let Some(job_id) = job_id { - let conflicting_id = sqlx::query_scalar!( - "SELECT 1 FROM queue WHERE id = $1 UNION ALL select 1 FROM completed_job WHERE id = $1", - job_id - ) - .fetch_optional(&mut *tx) - .await?; + let conflicting_id = sqlx::query_scalar!("SELECT 1 FROM v2_job WHERE id = $1", job_id) + .fetch_optional(&mut *tx) + .await?; if conflicting_id.is_some() { return Err(Error::BadRequest(format!( @@ -3683,12 +3707,12 @@ pub async fn push<'c, 'd>( ) .execute(&mut *tx) .await - .map_err(|e| Error::InternalErr(format!("Could not insert concurrency_key={concurrency_key} for job_id={job_id} script_path={script_path:?} workspace_id={workspace_id}: {e:#}")))?; + .map_err(|e| Error::internal_err(format!("Could not insert concurrency_key={concurrency_key} for job_id={job_id} script_path={script_path:?} workspace_id={workspace_id}: {e:#}")))?; } let stringified_args = if *JOB_ARGS_AUDIT_LOGS { Some(serde_json::to_string(&args).map_err(|e| { - Error::InternalErr(format!( + Error::internal_err(format!( "Could not serialize args for audit log of job {job_id}: {e:#}" )) })?) @@ -3697,72 +3721,90 @@ pub async fn push<'c, 'd>( }; let raw_flow = raw_flow.map(Json); + let preprocessed = preprocessed.or_else(|| match flow_step_id.as_deref() { + Some("preprocessor") => Some(false), + _ => None, + }); sqlx::query!( - "INSERT INTO job (id, workspace_id, raw_code, raw_lock, raw_flow, tag) - VALUES ($1, $2, $3, $4, $5, $6)", + "INSERT INTO v2_job (id, workspace_id, raw_code, raw_lock, raw_flow, tag, parent_job, + created_by, permissioned_as, runnable_id, runnable_path, args, kind, trigger, + script_lang, same_worker, pre_run_error, permissioned_as_email, visible_to_owner, + flow_innermost_root_job, concurrent_limit, concurrency_time_window_s, timeout, flow_step_id, + cache_ttl, priority, trigger_kind, script_entrypoint_override, preprocessed) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, + $19, $20, $21, $22, $23, $24, $25, $26, + CASE WHEN $14::VARCHAR IS NOT NULL THEN 'schedule'::job_trigger_kind END, + ($12::JSONB)->>'_ENTRYPOINT_OVERRIDE', $27)", job_id, workspace_id, raw_code, raw_lock, - raw_flow.as_ref() as Option<&Json>, + raw_flow as Option>, tag, - ) - .execute(&mut *tx) - .warn_after_seconds(1) - .await?; - - let (raw_code, raw_lock, raw_flow) = if !*MIN_VERSION_IS_AT_LEAST_1_427.read().await { - (raw_code, raw_lock, raw_flow) - } else { - (None, None, None) - }; - - tracing::debug!("Pushing job {job_id} with tag {tag}, schedule_path {schedule_path:?}, script_path: {script_path:?}, email {email}, workspace_id {workspace_id}"); - let uuid = sqlx::query_scalar!( - "INSERT INTO queue - (workspace_id, id, running, parent_job, created_by, permissioned_as, scheduled_for, - script_hash, script_path, raw_code, raw_lock, args, job_kind, schedule_path, raw_flow, \ - flow_status, is_flow_step, language, started_at, same_worker, pre_run_error, email, \ - visible_to_owner, root_job, tag, concurrent_limit, concurrency_time_window_s, timeout, \ - flow_step_id, cache_ttl, priority, last_ping) - VALUES ($1, $2, $3, $4, $5, $6, COALESCE($7, now()), $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, CASE WHEN $3 THEN now() END, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, NULL) \ - RETURNING id", - workspace_id, - job_id, - is_running, parent_job, user, permissioned_as, - scheduled_for_o, script_hash, script_path.clone(), - raw_code, - raw_lock, Json(args) as Json, job_kind.clone() as JobKind, schedule_path, - raw_flow.as_ref() as Option<&Json>, - flow_status.map(Json) as Option>, - is_flow_step, language as Option, same_worker, pre_run_error.map(|e| e.to_string()), email, visible_to_owner, root_job, - tag, concurrent_limit, - if concurrent_limit.is_some() { concurrency_time_window_s } else { None }, + if concurrent_limit.is_some() { + concurrency_time_window_s + } else { + None + }, custom_timeout, flow_step_id, cache_ttl, final_priority, + preprocessed, + ) + .execute(&mut *tx) + .warn_after_seconds(1) + .await?; + + tracing::debug!("Pushing job {job_id} with tag {tag}, schedule_path {schedule_path:?}, script_path: {script_path:?}, email {email}, workspace_id {workspace_id}"); + let uuid = sqlx::query_scalar!( + "INSERT INTO v2_job_queue + (workspace_id, id, running, scheduled_for, started_at, tag, priority) + VALUES ($1, $2, $3, COALESCE($4, now()), CASE WHEN $3 THEN now() END, $5, $6) \ + RETURNING id AS \"id!\"", + workspace_id, + job_id, + is_running, + scheduled_for_o, + tag, + final_priority, ) .fetch_one(&mut *tx) .warn_after_seconds(1) .await - .map_err(|e| Error::InternalErr(format!("Could not insert into queue {job_id} with tag {tag}, schedule_path {schedule_path:?}, script_path: {script_path:?}, email {email}, workspace_id {workspace_id}: {e:#}")))?; + .map_err(|e| Error::internal_err(format!("Could not insert into queue {job_id} with tag {tag}, schedule_path {schedule_path:?}, script_path: {script_path:?}, email {email}, workspace_id {workspace_id}: {e:#}")))?; + + sqlx::query!( + "INSERT INTO v2_job_runtime (id, ping) VALUES ($1, null)", + job_id + ) + .execute(&mut *tx) + .await?; + if let Some(flow_status) = flow_status { + sqlx::query!( + "INSERT INTO v2_job_status (id, flow_status) VALUES ($1, $2)", + job_id, + Json(flow_status) as Json, + ) + .execute(&mut *tx) + .await?; + } tracing::debug!("Pushed {job_id}"); // TODO: technically the job isn't queued yet, as the transaction can be rolled back. Should be solved when moving these metrics to the queue abstraction. @@ -3791,7 +3833,7 @@ pub async fn push<'c, 'd>( ) .await .map_err(|e| { - Error::InternalErr(format!( + Error::internal_err(format!( "Could not get permissions directly for job {job_id}: {e:#}" )) })? @@ -3912,17 +3954,17 @@ async fn restarted_flows_resolution( let row = sqlx::query!( "SELECT script_path, script_hash AS \"script_hash: ScriptHash\", - job_kind AS \"job_kind: JobKind\", + job_kind AS \"job_kind!: JobKind\", flow_status AS \"flow_status: Json>\", raw_flow AS \"raw_flow: Json>\" - FROM completed_job WHERE id = $1 and workspace_id = $2", + FROM v2_as_completed_job WHERE id = $1 and workspace_id = $2", completed_flow_id, workspace_id, ) .fetch_one(db) // TODO: should we try to use the passed-in `tx` here? .await .map_err(|err| { - Error::InternalErr(format!( + Error::internal_err(format!( "completed job not found for UUID {} in workspace {}: {}", completed_flow_id, workspace_id, err )) @@ -3936,7 +3978,7 @@ async fn restarted_flows_resolution( .flow_status .as_ref() .and_then(|v| serde_json::from_str::(v.get()).ok()) - .ok_or(Error::InternalErr(format!( + .ok_or(Error::internal_err(format!( "Unable to parse flow status for job {} in workspace {}", completed_flow_id, workspace_id, )))?; @@ -3966,14 +4008,14 @@ async fn restarted_flows_resolution( match module_definition.get_value() { Ok(FlowModuleValue::BranchAll { branches, parallel, .. }) => { if parallel { - return Err(Error::InternalErr(format!( + return Err(Error::internal_err(format!( "Module {} is a parallel branchall. It can only be restarted at a given branch if it's sequential", restart_step_id, ))); } let total_branch_number = module.flow_jobs().map(|v| v.len()).unwrap_or(0); if total_branch_number <= branch_or_iteration_n { - return Err(Error::InternalErr(format!( + return Err(Error::internal_err(format!( "Branch-all module {} has only {} branches. It can't be restarted on branch {}", restart_step_id, total_branch_number, @@ -4004,14 +4046,14 @@ async fn restarted_flows_resolution( } Ok(FlowModuleValue::ForloopFlow { parallel, .. }) => { if parallel { - return Err(Error::InternalErr(format!( + return Err(Error::internal_err(format!( "Module {} is not parallel loop. It can only be restarted at a given iteration if it's sequential", restart_step_id, ))); } let total_iterations = module.flow_jobs().map(|v| v.len()).unwrap_or(0); if total_iterations <= branch_or_iteration_n { - return Err(Error::InternalErr(format!( + return Err(Error::internal_err(format!( "For-loop module {} doesn't cannot be restarted on iteration number {} as it has only {} iterations", restart_step_id, branch_or_iteration_n, @@ -4027,7 +4069,7 @@ async fn restarted_flows_resolution( truncated_modules.push(FlowStatusModule::InProgress { id: module.id(), job: new_flow_jobs[new_flow_jobs.len() - 1], // set to last finished job from completed flow - iterator: Some(Iterator { + iterator: Some(FlowIterator { index: branch_or_iteration_n - 1, // same deal as above, this refers to the last finished job itered: vec![], // Setting itered to empty array here, such that input transforms will be re-computed by worker_flows }), @@ -4041,7 +4083,7 @@ async fn restarted_flows_resolution( }); } _ => { - return Err(Error::InternalErr(format!( + return Err(Error::internal_err(format!( "Module {} is not a branchall or forloop, unable to restart it at step {:?}", restart_step_id, branch_or_iteration_n @@ -4057,7 +4099,7 @@ async fn restarted_flows_resolution( step_n = step_n + 1; match module.clone() { FlowStatusModule::Success { .. } => Ok(truncated_modules.push(module)), - _ => Err(Error::InternalErr(format!( + _ => Err(Error::internal_err(format!( "Flow cannot be restarted from a non successful module", ))), }?; @@ -4066,7 +4108,7 @@ async fn restarted_flows_resolution( if !dependent_module { // step not found in flow. - return Err(Error::InternalErr(format!( + return Err(Error::internal_err(format!( "Flow cannot be restarted from step {} as it could not be found.", restart_step_id ))); diff --git a/backend/windmill-queue/src/schedule.rs b/backend/windmill-queue/src/schedule.rs index ea9f476ab6..fd4bcab4b4 100644 --- a/backend/windmill-queue/src/schedule.rs +++ b/backend/windmill-queue/src/schedule.rs @@ -71,7 +71,7 @@ pub async fn push_scheduled_job<'c>( let next = next.with_timezone(&chrono::Utc); let already_exists: bool = query_scalar!( - "SELECT EXISTS (SELECT 1 FROM queue WHERE workspace_id = $1 AND schedule_path = $2 AND scheduled_for = $3)", + "SELECT EXISTS (SELECT 1 FROM v2_as_queue WHERE workspace_id = $1 AND schedule_path = $2 AND scheduled_for = $3)", &schedule.workspace_id, &schedule.path, next @@ -149,7 +149,7 @@ pub async fn push_scheduled_job<'c>( if schedule.retry.is_some() { let parsed_retry = serde_json::from_value::(schedule.retry.clone().unwrap()) .map_err(|err| { - error::Error::InternalErr(format!( + error::Error::internal_err(format!( "Unable to parse retry information from schedule: {}", err.to_string(), )) diff --git a/backend/windmill-worker/nsjail/download.py.config.proto b/backend/windmill-worker/nsjail/download.py.config.proto index b910e1c690..5c2989417f 100644 --- a/backend/windmill-worker/nsjail/download.py.config.proto +++ b/backend/windmill-worker/nsjail/download.py.config.proto @@ -22,6 +22,7 @@ keep_caps: true keep_env: true mount_proc: true + mount { src: "/bin" dst: "/bin" @@ -75,11 +76,12 @@ mount { } mount { - src: "{CACHE_DIR}" - dst: "{CACHE_DIR}" + src: "{TARGET_DIR}" + dst: "{TARGET_DIR}" is_bind: true rw: true } + mount { src: "{PY_INSTALL_DIR}" dst: "{PY_INSTALL_DIR}" diff --git a/backend/windmill-worker/nsjail/download.py.pip.config.proto b/backend/windmill-worker/nsjail/download.py.pip.config.proto index 15e5c6c7f6..6e2a8a1974 100644 --- a/backend/windmill-worker/nsjail/download.py.pip.config.proto +++ b/backend/windmill-worker/nsjail/download.py.pip.config.proto @@ -73,13 +73,6 @@ mount { is_bind: true } -mount { - src: "{CACHE_DIR}" - dst: "{CACHE_DIR}" - is_bind: true - rw: true -} - mount { src: "/dev/urandom" dst: "/dev/urandom" diff --git a/backend/windmill-worker/nsjail/run.bun.config.proto b/backend/windmill-worker/nsjail/run.bun.config.proto index 751452251d..527d49cf7a 100644 --- a/backend/windmill-worker/nsjail/run.bun.config.proto +++ b/backend/windmill-worker/nsjail/run.bun.config.proto @@ -156,14 +156,6 @@ mount { iface_no_lo: true -mount { - src: "{CACHE_DIR}" - dst: "/tmp/windmill/cache_nomount/bun" - is_bind: true - mandatory: false - rw: true -} - {SHARED_MOUNT} envar: "HOME=/tmp/{LANG}" diff --git a/backend/windmill-worker/nsjail/run.go.config.proto b/backend/windmill-worker/nsjail/run.go.config.proto index db77c0e982..1ba6f52d5e 100644 --- a/backend/windmill-worker/nsjail/run.go.config.proto +++ b/backend/windmill-worker/nsjail/run.go.config.proto @@ -96,14 +96,6 @@ mount { iface_no_lo: true -mount { - src: "{CACHE_DIR}" - dst: "/tmp/.cache/go" - is_bind: true - rw: true - mandatory: false -} - {SHARED_MOUNT} envar: "GOPATH=/tmp/.cache/go" diff --git a/backend/windmill-worker/nsjail/run.php.config.proto b/backend/windmill-worker/nsjail/run.php.config.proto index 2a6722cfaf..d3752d33b7 100644 --- a/backend/windmill-worker/nsjail/run.php.config.proto +++ b/backend/windmill-worker/nsjail/run.php.config.proto @@ -124,14 +124,6 @@ mount { iface_no_lo: true -mount { - src: "{CACHE_DIR}" - dst: "/tmp/windmill/cache/php" - is_bind: true - rw: true - mandatory: false -} - {SHARED_MOUNT} envar: "HOME=/tmp" diff --git a/backend/windmill-worker/nsjail/run.powershell.config.proto b/backend/windmill-worker/nsjail/run.powershell.config.proto index a0d77ca75e..27a36548b4 100644 --- a/backend/windmill-worker/nsjail/run.powershell.config.proto +++ b/backend/windmill-worker/nsjail/run.powershell.config.proto @@ -121,7 +121,7 @@ mount { src: "{CACHE_DIR}" dst: "/tmp/windmill/cache/powershell" is_bind: true - rw: true + rw: false mandatory: false } diff --git a/backend/windmill-common/src/bench.rs b/backend/windmill-worker/src/bench.rs similarity index 62% rename from backend/windmill-common/src/bench.rs rename to backend/windmill-worker/src/bench.rs index 0e89af571c..5006d5a34c 100644 --- a/backend/windmill-common/src/bench.rs +++ b/backend/windmill-worker/src/bench.rs @@ -1,9 +1,9 @@ -use crate::{ +use serde::Serialize; +use tokio::time::Instant; +use windmill_common::{ worker::{write_file, TMP_DIR}, DB, }; -use serde::Serialize; -use tokio::time::Instant; #[derive(Serialize)] pub struct BenchmarkInfo { @@ -79,11 +79,12 @@ impl BenchmarkIter { } pub async fn benchmark_init(benchmark_jobs: i32, db: &DB) { - use crate::{jobs::JobKind, scripts::ScriptLang}; + use windmill_common::{jobs::JobKind, scripts::ScriptLang}; let benchmark_kind = std::env::var("BENCHMARK_KIND").unwrap_or("noop".to_string()); if benchmark_jobs > 0 { + let mut tx = db.begin().await.unwrap(); match benchmark_kind.as_str() { "dedicated" => { // you need to create the script first, check https://github.com/windmill-labs/windmill/blob/b76a92cfe454c686f005c65f534e29e039f3c706/benchmarks/lib.ts#L47 @@ -92,10 +93,10 @@ pub async fn benchmark_init(benchmark_jobs: i32, db: &DB) { "f/benchmarks/dedicated", "admins" ) - .fetch_one(db) + .fetch_one(&mut *tx) .await .unwrap_or_else(|_e| panic!("failed to insert dedicated jobs")); - sqlx::query!("INSERT INTO queue (id, script_hash, script_path, job_kind, language, tag, created_by, permissioned_as, email, scheduled_for, workspace_id) (SELECT gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8, $9, $10 FROM generate_series(1, $11))", + let uuids = sqlx::query_scalar!("INSERT INTO v2_job (id, runnable_id, runnable_path, kind, script_lang, tag, created_by, permissioned_as, permissioned_as_email, workspace_id) (SELECT gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8, $9 FROM generate_series(1, $10)) RETURNING id", hash, "f/benchmarks/dedicated", JobKind::Script as JobKind, @@ -104,12 +105,21 @@ pub async fn benchmark_init(benchmark_jobs: i32, db: &DB) { "admin", "u/admin", "admin@windmill.dev", - chrono::Utc::now(), "admins", benchmark_jobs ) - .execute(db) - .await.unwrap_or_else(|_e| panic!("failed to insert dedicated jobs")); + .fetch_all(&mut *tx) + .await.unwrap_or_else(|_e| panic!("failed to insert dedicated jobs (1)")); + sqlx::query!("INSERT INTO v2_job_queue (id, workspace_id, scheduled_for, tag) SELECT unnest($1::uuid[]), $2, now(), $3", &uuids, "admins", "admins:f/benchmarks/dedicated") + .execute(&mut *tx) + .await.unwrap_or_else(|_e| panic!("failed to insert dedicated jobs (2)")); + sqlx::query!( + "INSERT INTO v2_job_runtime (id) SELECT unnest($1::uuid[])", + &uuids + ) + .execute(&mut *tx) + .await + .unwrap_or_else(|_e| panic!("failed to insert dedicated jobs (3)")); } "parallelflow" => { //create dedicated script @@ -124,9 +134,9 @@ pub async fn benchmark_init(benchmark_jobs: i32, db: &DB) { "flow", "admin", ) - .execute(db) + .execute(&mut *tx) .await.unwrap_or_else(|_e| panic!("failed to insert parallelflow jobs {_e:#}")); - sqlx::query!("INSERT INTO queue (id, script_hash, script_path, job_kind, language, tag, created_by, permissioned_as, email, scheduled_for, workspace_id, raw_flow, flow_status) (SELECT gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12 FROM generate_series(1, 1))", + let uuids = sqlx::query_scalar!("INSERT INTO v2_job (id, runnable_id, runnable_path, kind, script_lang, tag, created_by, permissioned_as, permissioned_as_email, workspace_id, raw_flow) (SELECT gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8, $9, $10 FROM generate_series(1, 1)) RETURNING id", None::, None::, JobKind::FlowPreview as JobKind, @@ -135,7 +145,6 @@ pub async fn benchmark_init(benchmark_jobs: i32, db: &DB) { "admin", "u/admin", "admin@windmill.dev", - chrono::Utc::now(), "admins", serde_json::from_str::(r#" { @@ -169,7 +178,24 @@ pub async fn benchmark_init(benchmark_jobs: i32, db: &DB) { "preprocessor_module": null } "#).unwrap(), - serde_json::from_str::(r#" + ) + .fetch_all(&mut *tx) + .await.unwrap_or_else(|_e| panic!("failed to insert parallelflow jobs (1)")); + sqlx::query!("INSERT INTO v2_job_queue (id, workspace_id, scheduled_for, tag) SELECT unnest($1::uuid[]), $2, now(), $3", &uuids, "admins", "flow") + .execute(&mut *tx) + .await.unwrap_or_else(|_e| panic!("failed to insert parallelflow jobs (2)")); + sqlx::query!( + "INSERT INTO v2_job_runtime (id) SELECT unnest($1::uuid[])", + &uuids + ) + .execute(&mut *tx) + .await + .unwrap_or_else(|_e| panic!("failed to insert parallelflow jobs (3)")); + sqlx::query!( + "INSERT INTO v2_job_status (id, flow_status) SELECT unnest($1::uuid[]), $2", + &uuids, + serde_json::from_str::( + r#" { "step": 0, "modules": [ @@ -186,13 +212,16 @@ pub async fn benchmark_init(benchmark_jobs: i32, db: &DB) { "preprocessor_module": null } - "#).unwrap() + "# + ) + .unwrap() ) - .execute(db) - .await.unwrap_or_else(|_e| panic!("failed to insert parallelflow jobs")); + .execute(&mut *tx) + .await + .unwrap_or_else(|_e| panic!("failed to insert parallelflow jobs (4)")); } _ => { - sqlx::query!("INSERT INTO queue (id, script_hash, script_path, job_kind, language, tag, created_by, permissioned_as, email, scheduled_for, workspace_id) (SELECT gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8, $9, $10 FROM generate_series(1, $11))", + let uuids = sqlx::query_scalar!("INSERT INTO v2_job (id, runnable_id, runnable_path, kind, script_lang, tag, created_by, permissioned_as, permissioned_as_email, workspace_id) (SELECT gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8, $9 FROM generate_series(1, $10)) RETURNING id", None::, None::, JobKind::Noop as JobKind, @@ -201,13 +230,23 @@ pub async fn benchmark_init(benchmark_jobs: i32, db: &DB) { "admin", "u/admin", "admin@windmill.dev", - chrono::Utc::now(), "admins", benchmark_jobs ) - .execute(db) - .await.unwrap_or_else(|_e| panic!("failed to insert noop jobs")); + .fetch_all(&mut *tx) + .await.unwrap_or_else(|_e| panic!("failed to insert noop jobs (1)")); + sqlx::query!("INSERT INTO v2_job_queue (id, workspace_id, scheduled_for, tag) SELECT unnest($1::uuid[]), $2, now(), $3", &uuids, "admins", "deno") + .execute(&mut *tx) + .await.unwrap_or_else(|_e| panic!("failed to insert noop jobs (2)")); + sqlx::query!( + "INSERT INTO v2_job_runtime (id) SELECT unnest($1::uuid[])", + &uuids + ) + .execute(&mut *tx) + .await + .unwrap_or_else(|_e| panic!("failed to insert noop jobs (3)")); } } + tx.commit().await.unwrap(); } } diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 77bad32a5d..4fc471df91 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -1,12 +1,11 @@ #[cfg(feature = "deno_core")] use std::time::Instant; -use std::{collections::HashMap, fs, io, path::Path, process::Stdio}; +use std::{collections::HashMap, fs, path::Path, process::Stdio}; +use anyhow::Context; use base64::Engine; use itertools::Itertools; -#[cfg(not(feature = "deno_core"))] -use serde_json::value::to_raw_value; use serde_json::value::RawValue; use sha2::Digest; @@ -19,9 +18,8 @@ use crate::common::build_envs_map; use crate::{ common::{ - create_args_and_out_file, get_main_override, get_reserved_variables, parse_npm_config, - read_file, read_file_content, read_result, start_child_process, write_file_binary, - OccupancyMetrics, + create_args_and_out_file, get_reserved_variables, parse_npm_config, read_file, + read_file_content, read_result, start_child_process, write_file_binary, OccupancyMetrics, }, handle_child::handle_child, AuthedClientBackgroundTask, BUNFIG_INSTALL_SCOPES, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, @@ -45,7 +43,7 @@ use windmill_common::variables; use windmill_common::{ error::{self, Result}, get_latest_hash_for_path, - jobs::{QueuedJob, PREPROCESSOR_FAKE_ENTRYPOINT}, + jobs::QueuedJob, scripts::ScriptLang, worker::{exists_in_cache, save_cache, write_file}, DB, @@ -531,7 +529,7 @@ pub async fn generate_wrapper_mjs( format!("{job_dir}/wrapper.js"), format!("{job_dir}/wrapper.mjs"), ) - .map_err(|e| error::Error::InternalErr(format!("Could not move wrapper to mjs: {e:#}")))?; + .map_err(|e| error::Error::internal_err(format!("Could not move wrapper to mjs: {e:#}")))?; Ok(()) } @@ -644,7 +642,7 @@ pub fn copy_recursively( source: impl AsRef, destination: impl AsRef, skip: Option<&Vec>, -) -> io::Result<()> { +) -> Result<()> { let mut stack = Vec::new(); stack.push(( source.as_ref().to_path_buf(), @@ -652,7 +650,9 @@ pub fn copy_recursively( 0, )); while let Some((current_source, current_destination, level)) = stack.pop() { - for entry in fs::read_dir(¤t_source)? { + for entry in fs::read_dir(¤t_source) + .context(format!("reading directory {current_source:?}"))? + { let entry = entry?; let filetype = entry.file_type()?; let destination = current_destination.join(entry.file_name()); @@ -670,7 +670,11 @@ pub fn copy_recursively( fs::create_dir_all(&destination)?; stack.push((entry.path(), destination, level + 1)); } else { - fs::hard_link(&original, &destination)? + fs::hard_link(&original, &destination).map_err(|e| { + error::Error::internal_err(format!( + "hard linking from {original:?} to {destination:?}: {e:#}" + )) + })?; } } } @@ -816,7 +820,7 @@ async fn write_lock(splitted_lockb_2: &str, job_dir: &str, is_binary: bool) -> R "bun.lockb", &base64::engine::general_purpose::STANDARD .decode(splitted_lockb_2) - .map_err(|_| error::Error::InternalErr(format!("Could not decode bun.lockb")))?, + .map_err(|_| error::Error::internal_err(format!("Could not decode bun.lockb")))?, ) .await?; } else { @@ -875,16 +879,8 @@ pub async fn handle_bun_job( if codebase.is_some() { annotation.nodejs = true } - let (main_override, apply_preprocessor) = match get_main_override(job.args.as_ref()) { - Some(main_override) => { - if main_override == PREPROCESSOR_FAKE_ENTRYPOINT { - (None, true) - } else { - (Some(main_override), false) - } - } - None => (None, false), - }; + let main_override = job.script_entrypoint_override.as_deref(); + let apply_preprocessor = !job.is_flow_step && job.preprocessed == Some(false); #[cfg(not(feature = "enterprise"))] if annotation.nodejs || annotation.npm { @@ -984,7 +980,7 @@ pub async fn handle_bun_job( "bunfig.toml".to_string(), ]), ) { - fs::remove_dir_all(&buntar_path)?; + fs::remove_dir_all(&buntar_path).context("deleting buntar directory")?; tracing::error!("Could not create buntar: {e}"); } } @@ -1051,9 +1047,12 @@ pub async fn handle_bun_job( return Ok(()) as error::Result<()>; } // let mut start = Instant::now(); - let args = - windmill_parser_ts::parse_deno_signature(inner_content, true, main_override.clone())? - .args; + let args = windmill_parser_ts::parse_deno_signature( + inner_content, + true, + main_override.map(ToString::to_string), + )? + .args; let pre_args = if apply_preprocessor { Some( @@ -1087,7 +1086,7 @@ pub async fn handle_bun_job( let spread = args.into_iter().map(|x| x.name).join(","); // logs.push_str(format!("infer args: {:?}\n", start.elapsed().as_micros()).as_str()); // we cannot use Bun.read and Bun.write because it results in an EBADF error on cloud - let main_name = main_override.unwrap_or("main".to_string()); + let main_name = main_override.unwrap_or("main"); let main_import = if codebase.is_some() || has_bundle_cache { "./main.js" @@ -1297,7 +1296,7 @@ try {{ tracing::error!( r#""deno_core" feature is not activated, but "//native" annotation used. Returning empty value..."# ); - return Ok(to_raw_value("").unwrap()); + return Err(error::Error::internal_err("deno_core feature is not activated, but //native annotation used. Returning empty value...".to_string())); } #[cfg(feature = "deno_core")] @@ -1355,7 +1354,6 @@ try {{ &NSJAIL_CONFIG_RUN_BUN_CONTENT .replace("{LANG}", if annotation.nodejs { "nodejs" } else { "bun" }) .replace("{JOB_DIR}", job_dir) - .replace("{CACHE_DIR}", BUN_CACHE_DIR) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) .replace( "{SHARED_MOUNT}", @@ -1496,13 +1494,13 @@ try {{ let args = read_file(&format!("{job_dir}/args.json")) .await .map_err(|e| { - error::Error::InternalErr(format!( + error::Error::internal_err(format!( "error while reading args from preprocessing: {e:#}" )) })?; let args: HashMap> = serde_json::from_str(args.get()).map_err(|e| { - error::Error::InternalErr(format!( + error::Error::internal_err(format!( "error while deserializing args from preprocessing: {e:#}" )) })?; @@ -1625,7 +1623,7 @@ pub async fn start_worker( &base64::engine::general_purpose::STANDARD .decode(lock) .map_err(|_| { - error::Error::InternalErr("Could not decode bun.lockb".to_string()) + error::Error::internal_err("Could not decode bun.lockb".to_string()) })?, ) .await?; diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index 099300f621..a64a32c869 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -12,7 +12,6 @@ use sqlx::types::Json; use sqlx::{Pool, Postgres}; use tokio::process::Command; use tokio::{fs::File, io::AsyncReadExt}; -use windmill_common::jobs::ENTRYPOINT_OVERRIDE; #[cfg(feature = "parquet")] use windmill_common::s3_helpers::{ @@ -149,13 +148,13 @@ pub async fn transform_json<'a>( let inner_vs = v.get(); if (*RE_RES_VAR).is_match(inner_vs) { let value = serde_json::from_str(inner_vs).map_err(|e| { - error::Error::InternalErr(format!("Error while parsing inner arg: {e:#}")) + error::Error::internal_err(format!("Error while parsing inner arg: {e:#}")) })?; let transformed = transform_json_value(&k, &client.get_authed().await, workspace, value, job, db) .await?; let as_raw = serde_json::from_value(transformed).map_err(|e| { - error::Error::InternalErr(format!("Error while parsing inner arg: {e:#}")) + error::Error::internal_err(format!("Error while parsing inner arg: {e:#}")) })?; r.insert(k.to_string(), as_raw); } else { @@ -177,13 +176,13 @@ pub async fn transform_json_as_values<'a>( let inner_vs = v.get(); if (*RE_RES_VAR).is_match(inner_vs) { let value = serde_json::from_str(inner_vs).map_err(|e| { - error::Error::InternalErr(format!("Error while parsing inner arg: {e:#}")) + error::Error::internal_err(format!("Error while parsing inner arg: {e:#}")) })?; let transformed = transform_json_value(&k, &client.get_authed().await, workspace, value, job, db) .await?; let as_raw = serde_json::from_value(transformed).map_err(|e| { - error::Error::InternalErr(format!("Error while parsing inner arg: {e:#}")) + error::Error::internal_err(format!("Error while parsing inner arg: {e:#}")) })?; r.insert(k.to_string(), as_raw); } else { @@ -237,7 +236,7 @@ pub async fn transform_json_value( Value::String(y) if y.starts_with("$res:") => { let path = y.strip_prefix("$res:").unwrap(); if path.split("/").count() < 2 { - return Err(Error::InternalErr(format!( + return Err(Error::internal_err(format!( "Argument `{name}` is an invalid resource path: {path}", ))); } @@ -256,14 +255,14 @@ pub async fn transform_json_value( let mc = build_crypt_with_key_suffix(&db, &job.workspace_id, &job.id.to_string()).await?; decrypt(&mc, encrypted.to_string()).and_then(|x| { - serde_json::from_str(&x).map_err(|e| Error::InternalErr(e.to_string())) + serde_json::from_str(&x).map_err(|e| Error::internal_err(e.to_string())) }) // let path = y.strip_prefix("$res:").unwrap(); } Value::String(y) if y.starts_with("$") => { let flow_path = if let Some(uuid) = job.parent_job { - sqlx::query_scalar!("SELECT script_path FROM queue WHERE id = $1", uuid) + sqlx::query_scalar!("SELECT runnable_path FROM v2_job WHERE id = $1", uuid) .fetch_optional(db) .await? .flatten() @@ -397,7 +396,7 @@ pub async fn get_reserved_variables( db: &sqlx::Pool, ) -> Result, Error> { let flow_path = if let Some(uuid) = job.parent_job { - sqlx::query_scalar!("SELECT script_path FROM queue WHERE id = $1", uuid) + sqlx::query_scalar!("SELECT runnable_path FROM v2_job WHERE id = $1", uuid) .fetch_optional(db) .await? .flatten() @@ -440,15 +439,6 @@ pub async fn build_envs_map(context: Vec) -> HashMap>>>) -> Option { - return args - .map(|x| { - x.0.get(ENTRYPOINT_OVERRIDE) - .map(|x| x.get().to_string().replace("\"", "")) - }) - .flatten(); -} - pub fn sizeof_val(v: &serde_json::Value) -> usize { std::mem::size_of::() + match v { @@ -570,7 +560,7 @@ impl OccupancyMetrics { pub async fn start_child_process(mut cmd: Command, executable: &str) -> Result { return cmd .spawn() - .map_err(|err| tentatively_improve_error(Error::IoErr(err), executable)); + .map_err(|err| tentatively_improve_error(err.into(), executable)); } pub async fn resolve_job_timeout( @@ -930,7 +920,7 @@ fn tentatively_improve_error(err: Error, executable: &str) -> Error { let err_msg = "program not found"; if err.to_string().contains(&err_msg) { - return Error::InternalErr(format!( + return Error::internal_err(format!( "Executable {executable} not found on worker. PATH: {}", *PATH_ENV )); @@ -967,5 +957,5 @@ pub fn build_http_client(timeout_duration: std::time::Duration) -> error::Result .timeout(timeout_duration) .connect_timeout(std::time::Duration::from_secs(10)) .build() - .map_err(|e| Error::InternalErr(format!("Error building http client: {e:#}"))) + .map_err(|e| Error::internal_err(format!("Error building http client: {e:#}"))) } diff --git a/backend/windmill-worker/src/dedicated_worker.rs b/backend/windmill-worker/src/dedicated_worker.rs index 56ea01c038..fb3a30176c 100644 --- a/backend/windmill-worker/src/dedicated_worker.rs +++ b/backend/windmill-worker/src/dedicated_worker.rs @@ -78,7 +78,7 @@ pub async fn handle_dedicated_process( //do not cache local dependencies use crate::{handle_child::process_status, PROXY_ENVS}; - + let cmd_name = format!("dedicated {command_path}"); let mut child = { let mut cmd = Command::new(command_path); cmd.current_dir(job_dir) @@ -126,7 +126,7 @@ pub async fn handle_dedicated_process( .wait() .await .expect("child process encountered an error"); - if let Err(e) = process_status(status) { + if let Err(e) = process_status(&cmd_name, status) { tracing::error!("child exit status was not success: {e:#}"); } else { tracing::info!("child exit status was success"); @@ -185,14 +185,14 @@ pub async fn handle_dedicated_process( let result = Arc::new(result); append_logs(&job.id, &job.workspace_id, logs.clone(), db).await; if line.starts_with("wm_res[success]:") { - job_completed_tx.send(JobCompleted { job , result, mem_peak: 0, canceled_by: None, success: true, cached_res_path: None, token: token.to_string(), duration: None }).await.unwrap() + job_completed_tx.send(JobCompleted { job , result, result_columns: None, mem_peak: 0, canceled_by: None, success: true, cached_res_path: None, token: token.to_string(), duration: None }).await.unwrap() } else { - job_completed_tx.send(JobCompleted { job , result, mem_peak: 0, canceled_by: None, success: false, cached_res_path: None, token: token.to_string(), duration: None }).await.unwrap() + job_completed_tx.send(JobCompleted { job , result, result_columns: None, mem_peak: 0, canceled_by: None, success: false, cached_res_path: None, token: token.to_string(), duration: None }).await.unwrap() } }, Err(e) => { tracing::error!("Could not deserialize job result `{line}`: {e:?}"); - job_completed_tx.send(JobCompleted { job , result: Arc::new(to_raw_value(&serde_json::json!({"error": format!("Could not deserialize job result `{line}`: {e:?}")}))), mem_peak: 0, canceled_by: None, success: false, cached_res_path: None, token: token.to_string(), duration: None }).await.unwrap(); + job_completed_tx.send(JobCompleted { job , result: Arc::new(to_raw_value(&serde_json::json!({"error": format!("Could not deserialize job result `{line}`: {e:?}")}))), result_columns: None, mem_peak: 0, canceled_by: None, success: false, cached_res_path: None, token: token.to_string(), duration: None }).await.unwrap(); }, }; logs = init_log.clone(); @@ -470,7 +470,7 @@ pub async fn create_dedicated_worker_map( if let Ok(v) = value { if let Some(v) = v { let value = serde_json::from_str::(v.get()).map_err(|err| { - Error::InternalErr(format!( + Error::internal_err(format!( "could not convert json to flow for {flow_path}: {err:?}" )) }); @@ -622,7 +622,7 @@ async fn spawn_dedicated_worker( .bind(&w_id) .fetch_optional(&db) .await - .map_err(|e| Error::InternalErr(format!("expected content and lock: {e:#}"))) + .map_err(|e| Error::internal_err(format!("expected content and lock: {e:#}"))) .map(|x| x.map(|y| (y.0, y.1, y.2, y.3, if y.4 { y.5.map(|z| z.to_string()) } else { None }))) }; if let Ok(q) = q { diff --git a/backend/windmill-worker/src/deno_executor.rs b/backend/windmill-worker/src/deno_executor.rs index 2385771a15..6a7bed5c38 100644 --- a/backend/windmill-worker/src/deno_executor.rs +++ b/backend/windmill-worker/src/deno_executor.rs @@ -7,17 +7,15 @@ use windmill_queue::{append_logs, CanceledBy}; use crate::{ common::{ - create_args_and_out_file, get_main_override, get_reserved_variables, parse_npm_config, - read_file, read_result, start_child_process, OccupancyMetrics, + create_args_and_out_file, get_reserved_variables, parse_npm_config, read_file, read_result, + start_child_process, OccupancyMetrics, }, handle_child::handle_child, AuthedClientBackgroundTask, DENO_CACHE_DIR, DENO_PATH, DISABLE_NSJAIL, HOME_ENV, NPM_CONFIG_REGISTRY, PATH_ENV, TZ_ENV, }; use tokio::{fs::File, io::AsyncReadExt, process::Command}; -use windmill_common::{ - error::Result, jobs::PREPROCESSOR_FAKE_ENTRYPOINT, worker::write_file, BASE_URL, -}; +use windmill_common::{error::Result, worker::write_file, BASE_URL}; use windmill_common::{ error::{self}, jobs::QueuedJob, @@ -197,24 +195,19 @@ pub async fn handle_deno_job( let logs1 = "\n\n--- DENO CODE EXECUTION ---\n".to_string(); append_logs(&job.id, &job.workspace_id, logs1, db).await; - let (main_override, apply_preprocessor) = match get_main_override(job.args.as_ref()) { - Some(main_override) => { - if main_override == PREPROCESSOR_FAKE_ENTRYPOINT { - (None, true) - } else { - (Some(main_override), false) - } - } - None => (None, false), - }; + let main_override = job.script_entrypoint_override.as_deref(); + let apply_preprocessor = !job.is_flow_step && job.preprocessed == Some(false); write_file(job_dir, "main.ts", inner_content)?; let write_wrapper_f = async { // let mut start = Instant::now(); - let args = - windmill_parser_ts::parse_deno_signature(inner_content, true, main_override.clone())? - .args; + let args = windmill_parser_ts::parse_deno_signature( + inner_content, + true, + main_override.map(ToString::to_string), + )? + .args; let pre_args = if apply_preprocessor { Some( @@ -246,7 +239,7 @@ pub async fn handle_deno_job( .join("\n "); let spread = args.into_iter().map(|x| x.name).join(","); - let main_name = main_override.unwrap_or("main".to_string()); + let main_name = main_override.unwrap_or("main"); // logs.push_str(format!("infer args: {:?}\n", start.elapsed().as_micros()).as_str()); let (preprocessor_import, preprocessor) = if let Some(pre_args) = pre_args { let pre_spread = pre_args.into_iter().map(|x| x.name).join(","); @@ -432,13 +425,13 @@ try {{ let args = read_file(&format!("{job_dir}/args.json")) .await .map_err(|e| { - error::Error::InternalErr(format!( + error::Error::internal_err(format!( "error while reading args from preprocessing: {e:#}" )) })?; let args: HashMap> = serde_json::from_str(args.get()).map_err(|e| { - error::Error::InternalErr(format!( + error::Error::internal_err(format!( "error while deserializing args from preprocessing: {e:#}" )) })?; diff --git a/backend/windmill-worker/src/global_cache.rs b/backend/windmill-worker/src/global_cache.rs index 57ded13ee7..8239ab77d4 100644 --- a/backend/windmill-worker/src/global_cache.rs +++ b/backend/windmill-worker/src/global_cache.rs @@ -13,6 +13,9 @@ use windmill_common::error; #[cfg(all(feature = "enterprise", feature = "parquet", unix))] use std::sync::Arc; +#[cfg(all(feature = "enterprise", feature = "parquet"))] +pub const TARGET: &str = const_format::concatcp!(std::env::consts::OS, "_", std::env::consts::ARCH); + #[cfg(all(feature = "enterprise", feature = "parquet", unix))] pub async fn build_tar_and_push( s3_client: Arc, @@ -57,7 +60,7 @@ pub async fn build_tar_and_push( if let Err(e) = s3_client .put( &Path::from(format!( - "/tar/{}/{folder_name}.tar", + "/tar/{TARGET}/{}/{folder_name}.tar", if no_uv { "pip" } else { &python_xyz } )), std::fs::read(&tar_path)?.into(), @@ -100,7 +103,7 @@ pub async fn pull_from_tar( let start = Instant::now(); let tar_path = format!( - "tar/{}/{folder_name}.tar", + "tar/{TARGET}/{}/{folder_name}.tar", if no_uv { "pip".to_owned() } else { python_xyz } ); let bytes = attempt_fetch_bytes(client, &tar_path).await?; diff --git a/backend/windmill-worker/src/go_executor.rs b/backend/windmill-worker/src/go_executor.rs index 9a52da78cd..1a8156a1bb 100644 --- a/backend/windmill-worker/src/go_executor.rs +++ b/backend/windmill-worker/src/go_executor.rs @@ -259,7 +259,6 @@ func Run(req Req) (interface{{}}, error){{ "run.config.proto", &NSJAIL_CONFIG_RUN_GO_CONTENT .replace("{JOB_DIR}", job_dir) - .replace("{CACHE_DIR}", GO_CACHE_DIR) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) .replace("{SHARED_MOUNT}", shared_mount), )?; diff --git a/backend/windmill-worker/src/handle_child.rs b/backend/windmill-worker/src/handle_child.rs index 5dcdbfa287..7294950363 100644 --- a/backend/windmill-worker/src/handle_child.rs +++ b/backend/windmill-worker/src/handle_child.rs @@ -206,17 +206,14 @@ pub async fn handle_child( let set_reason = async { if matches!(kill_reason, KillReason::Timeout { .. }) { - if let Err(err) = sqlx::query( - r#" - UPDATE queue - SET canceled = true - , canceled_by = 'timeout' - , canceled_reason = $1 - WHERE id = $2 - "#, + if let Err(err) = sqlx::query!( + "UPDATE v2_job_queue + SET canceled_by = 'timeout' + , canceled_reason = $1 + WHERE id = $2", + format!("duration > {}", timeout_duration.as_secs()), + job_id ) - .bind(format!("duration > {}", timeout_duration.as_secs())) - .bind(job_id) .execute(&db) .await { @@ -426,7 +423,7 @@ pub async fn handle_child( _ if *too_many_logs.borrow() => Err(Error::ExecutionErr(format!( "logs or result reached limit. (current max size: {MAX_RESULT_SIZE} characters)" ))), - Ok(Ok(status)) => process_status(status), + Ok(Ok(status)) => process_status(&child_name, status), Ok(Err(kill_reason)) => match kill_reason { KillReason::AlreadyCompleted => { Err(Error::AlreadyCompleted("Job already completed".to_string())) @@ -644,23 +641,31 @@ where } } if job_id != Uuid::nil() { - let (canceled, canceled_by, canceled_reason, already_completed) = sqlx::query_as::<_, (bool, Option, Option, bool)>("UPDATE queue SET mem_peak = $1, last_ping = now() WHERE id = $2 RETURNING canceled, canceled_by, canceled_reason, false") - .bind(*mem_peak) - .bind(job_id) + let (canceled_by, canceled_reason, already_completed) = sqlx::query!( + "UPDATE v2_job_runtime r SET + memory_peak = $1, + ping = now() + FROM v2_job_queue q + WHERE r.id = $2 AND q.id = r.id + RETURNING canceled_by, canceled_reason", + *mem_peak, + job_id + ) + .map(|x| (x.canceled_by, x.canceled_reason, false)) .fetch_optional(&db) .await .unwrap_or_else(|e| { tracing::error!(%e, "error updating job {job_id}: {e:#}"); - Some((false, None, None, false)) + Some((None, None, false)) }) .unwrap_or_else(|| { // if the job is not in queue, it can only be in the completed_job so it is already complete - (false, None, None, true) + (None, None, true) }); if already_completed { return UpdateJobPollingExit::AlreadyCompleted } - if canceled { + if canceled_by.is_some() { canceled_by_ref.replace(CanceledBy { username: canceled_by.clone(), reason: canceled_reason.clone(), @@ -714,11 +719,11 @@ pub fn lines_to_stream( }) } -pub fn process_status(status: ExitStatus) -> error::Result<()> { +pub fn process_status(program: &str, status: ExitStatus) -> error::Result<()> { if status.success() { Ok(()) } else if let Some(code) = status.code() { - Err(error::Error::ExitStatus(code)) + Err(error::Error::ExitStatus(program.to_string(), code)) } else { #[cfg(any(target_os = "linux", target_os = "macos"))] return Err(error::Error::ExecutionErr(format!( diff --git a/backend/windmill-worker/src/js_eval.rs b/backend/windmill-worker/src/js_eval.rs index 482bfea3ed..9aab144296 100644 --- a/backend/windmill-worker/src/js_eval.rs +++ b/backend/windmill-worker/src/js_eval.rs @@ -42,6 +42,8 @@ use uuid::Uuid; #[cfg(feature = "deno_core")] use windmill_common::error::Error; +#[cfg(feature = "deno_core")] +use windmill_common::worker::{write_file, TMP_DIR}; use windmill_common::{flow_status::JobResult, DB}; use windmill_queue::CanceledBy; @@ -885,7 +887,7 @@ pub async fn eval_fetch_timeout( let future = async { let r = tokio::select! { - r = eval_fetch(&mut js_runtime, &js_expr, Some(env_code), load_client) => Ok(r), + r = eval_fetch(&mut js_runtime, &js_expr, Some(env_code), load_client, &job_id) => Ok(r), _ = memory_limit_rx.recv() => Err(Error::ExecutionErr("Memory limit reached, killing isolate".to_string())) }; @@ -934,12 +936,44 @@ pub async fn eval_fetch_timeout( #[cfg(feature = "deno_core")] const WINDMILL_CLIENT: &str = include_str!("./windmill-client.js"); +#[cfg(feature = "deno_core")] +const ERROR_DIR: &str = const_format::concatcp!(TMP_DIR, "/native_errors"); + +#[cfg(feature = "deno_core")] +fn write_error_expr(expr: &str, uuid: &Uuid) { + if let Err(e) = std::fs::create_dir_all(ERROR_DIR) { + tracing::error!("failed to create error dir {ERROR_DIR}: {e}"); + return; + } + let dir_entries = match std::fs::read_dir(ERROR_DIR) { + Ok(entries) => entries.count(), + Err(_) => { + tracing::error!("failed to read error dir {ERROR_DIR}"); + return; + } + }; + + if dir_entries >= 100 { + tracing::info!("Too many error files in {ERROR_DIR}, skipping write"); + return; + } + + let path = format!("/{uuid}.js"); + tracing::info!( + "nativets job {uuid} failed, writing error expr to {ERROR_DIR}/{path} for debugging: {path}" + ); + if let Err(e) = write_file(ERROR_DIR, &path, expr) { + tracing::error!("failed to write error expr to file {path}: {e}"); + } +} + #[cfg(feature = "deno_core")] async fn eval_fetch( js_runtime: &mut JsRuntime, expr: &str, env_code: Option, load_client: bool, + job_id: &Uuid, ) -> anyhow::Result> { if load_client { if let Some(env_code) = env_code.as_ref() { @@ -951,26 +985,42 @@ async fn eval_fetch( .await?; } } - + use anyhow::Context; let _ = js_runtime .load_side_es_module_from_code( &deno_core::resolve_url("file:///eval.ts")?, format!("{}\n{expr}", env_code.unwrap_or_default()), ) - .await?; + .await + .map_err(|e| { + write_error_expr(expr, &job_id); + e + }) + .context("failed to load module")?; - let script = js_runtime.execute_script( - "", - r#" + let script = js_runtime + .execute_script( + "", + r#" let args = Deno.core.ops.op_get_static_args().map(JSON.parse) import("file:///eval.ts").then((module) => module.main(...args)).then(JSON.stringify) "#, - )?; + ) + .map_err(|e| { + write_error_expr(expr, &job_id); + e + }) + .context("native script initialization")?; let fut = js_runtime.resolve(script); let global = js_runtime .with_event_loop_promise(fut, PollEventLoopOptions::default()) - .await?; + .await + .map_err(|e| { + write_error_expr(expr, &job_id); + e + }) + .context("native script event loop")?; let scope = &mut js_runtime.handle_scope(); let local = v8::Local::new(scope, global); diff --git a/backend/windmill-worker/src/lib.rs b/backend/windmill-worker/src/lib.rs index cc34dd72ab..9b77ca840f 100644 --- a/backend/windmill-worker/src/lib.rs +++ b/backend/windmill-worker/src/lib.rs @@ -9,6 +9,8 @@ mod snowflake_executor; mod ansible_executor; mod bash_executor; +#[cfg(feature = "benchmark")] +pub mod bench; mod bun_executor; pub mod common; mod config; @@ -38,6 +40,7 @@ mod rust_executor; mod worker; mod worker_flow; mod worker_lockfiles; + pub use worker::*; pub use result_processor::handle_job_error; diff --git a/backend/windmill-worker/src/mssql_executor.rs b/backend/windmill-worker/src/mssql_executor.rs index be5ea1e905..42d9c3aa8c 100644 --- a/backend/windmill-worker/src/mssql_executor.rs +++ b/backend/windmill-worker/src/mssql_executor.rs @@ -120,9 +120,11 @@ pub async fn do_mssql( let tcp = TcpStream::connect(config.get_addr()).await?; tcp.set_nodelay(true)?; - Client::connect(config, tcp.compat_write()).await.map_err(to_anyhow)? + Client::connect(config, tcp.compat_write()) + .await + .map_err(to_anyhow)? } - Err(e) => return Err(Error::Anyhow(to_anyhow(e))), + Err(e) => return Err(to_anyhow(e).into()), }; let sig = parse_mssql_sig(&query) diff --git a/backend/windmill-worker/src/mysql_executor.rs b/backend/windmill-worker/src/mysql_executor.rs index b98ec365ee..d733901f93 100644 --- a/backend/windmill-worker/src/mysql_executor.rs +++ b/backend/windmill-worker/src/mysql_executor.rs @@ -133,7 +133,7 @@ pub async fn do_mysql( .await?; let as_raw = serde_json::from_value(val).map_err(|e| { - Error::InternalErr(format!("Error while parsing inline resource: {e:#}")) + Error::internal_err(format!("Error while parsing inline resource: {e:#}")) })?; Some(as_raw) diff --git a/backend/windmill-worker/src/oracledb_executor.rs b/backend/windmill-worker/src/oracledb_executor.rs index f6e64a60a1..e6292181b2 100644 --- a/backend/windmill-worker/src/oracledb_executor.rs +++ b/backend/windmill-worker/src/oracledb_executor.rs @@ -328,7 +328,7 @@ pub async fn do_oracledb( .await?; let as_raw = serde_json::from_value(val).map_err(|e| { - Error::InternalErr(format!("Error while parsing inline resource: {e:#}")) + Error::internal_err(format!("Error while parsing inline resource: {e:#}")) })?; Some(as_raw) diff --git a/backend/windmill-worker/src/php_executor.rs b/backend/windmill-worker/src/php_executor.rs index 7eb0d95429..4f579a54a8 100644 --- a/backend/windmill-worker/src/php_executor.rs +++ b/backend/windmill-worker/src/php_executor.rs @@ -15,8 +15,8 @@ use windmill_queue::{append_logs, CanceledBy}; use crate::{ common::{ - check_executor_binary_exists, create_args_and_out_file, get_main_override, - get_reserved_variables, read_result, start_child_process, OccupancyMetrics, + check_executor_binary_exists, create_args_and_out_file, get_reserved_variables, + read_result, start_child_process, OccupancyMetrics, }, handle_child::handle_child, AuthedClientBackgroundTask, COMPOSER_CACHE_DIR, COMPOSER_PATH, DISABLE_NSJAIL, DISABLE_NUSER, @@ -190,11 +190,14 @@ pub async fn handle_php_job( let _ = write_file(job_dir, "main.php", inner_content)?; - let main_override = get_main_override(job.args.as_ref()); + let main_override = job.script_entrypoint_override.as_deref(); let write_wrapper_f = async { - let args = - windmill_parser_php::parse_php_signature(inner_content, main_override.clone())?.args; + let args = windmill_parser_php::parse_php_signature( + inner_content, + main_override.map(ToString::to_string), + )? + .args; let args_to_include = args .iter() @@ -220,7 +223,7 @@ pub async fn handle_php_job( .collect::>() .join("\n\n"); - let main_name = main_override.unwrap_or("main".to_string()); + let main_name = main_override.unwrap_or("main"); let wrapper_content: String = format!( r#" diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 8c134166a0..d86c84dd0d 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -10,7 +10,7 @@ use anyhow::anyhow; use itertools::Itertools; use regex::Regex; use serde_json::value::RawValue; -use sqlx::{types::Json, Pool, Postgres}; +use sqlx::{Pool, Postgres}; use tokio::{ fs::{metadata, DirBuilder, File}, io::AsyncReadExt, @@ -26,7 +26,7 @@ use windmill_common::{ self, Error::{self}, }, - jobs::{QueuedJob, PREPROCESSOR_FAKE_ENTRYPOINT}, + jobs::QueuedJob, utils::calculate_hash, worker::{write_file, PythonAnnotations, WORKER_CONFIG}, DB, @@ -35,8 +35,8 @@ use windmill_common::{ #[cfg(feature = "enterprise")] use windmill_common::variables::get_secret_value_as_admin; -use windmill_queue::{append_logs, CanceledBy}; use std::env::var; +use windmill_queue::{append_logs, CanceledBy}; lazy_static::lazy_static! { static ref PYTHON_PATH: String = @@ -84,8 +84,8 @@ use windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS; use crate::{ common::{ - create_args_and_out_file, get_main_override, get_reserved_variables, read_file, - read_result, start_child_process, OccupancyMetrics, + create_args_and_out_file, get_reserved_variables, read_file, read_result, + start_child_process, OccupancyMetrics, }, handle_child::handle_child, AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, INSTANCE_PYTHON_VERSION, @@ -308,12 +308,26 @@ impl PyVersion { let mut child_cmd = Command::new(uv_cmd); child_cmd + .env_clear() + .env("HOME", HOME_ENV.to_string()) + .env("PATH", PATH_ENV.to_string()) .args(["python", "install", v, "--python-preference=only-managed"]) // TODO: Do we need these? .envs([("UV_PYTHON_INSTALL_DIR", PY_INSTALL_DIR)]) .stdout(Stdio::piped()) .stderr(Stdio::piped()); + #[cfg(windows)] + { + child_cmd + .env("SystemRoot", SYSTEM_ROOT.as_str()) + .env("USERPROFILE", crate::USERPROFILE_ENV.as_str()) + .env( + "TMP", + std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")), + ); + } + let child_process = start_child_process(child_cmd, "uv").await?; append_logs(&job_id, &w_id, logs, db).await; @@ -341,8 +355,23 @@ impl PyVersion { let uv_cmd = UV_PATH.as_str(); let mut child_cmd = Command::new(uv_cmd); + + #[cfg(windows)] + { + child_cmd + .env("SystemRoot", SYSTEM_ROOT.as_str()) + .env("USERPROFILE", crate::USERPROFILE_ENV.as_str()) + .env( + "TMP", + std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")), + ); + } + let output = child_cmd // .current_dir(job_dir) + .env_clear() + .env("HOME", HOME_ENV.to_string()) + .env("PATH", PATH_ENV.to_string()) .args([ "python", "find", @@ -632,7 +661,7 @@ pub async fn uv_pip_compile( if *NATIVE_CERT { args.extend(["--native-tls"]); } - tracing::error!("uv args: {:?}", args); + tracing::debug!("uv args: {:?}", args); #[cfg(windows)] let uv_cmd = "uv"; @@ -902,8 +931,6 @@ pub async fn handle_python_job( tracing::debug!("Finished deps postinstall stage"); } - - if no_uv { append_logs( &job.id, @@ -937,10 +964,11 @@ pub async fn handle_python_job( pre_spread, ) = prepare_wrapper( job_dir, + job.is_flow_step, + job.preprocessed, + job.script_entrypoint_override.as_deref(), inner_content, &script_path, - job.args.as_ref(), - false, ) .await?; @@ -1057,7 +1085,7 @@ except BaseException as e: // Usefull if certain wheels needs to be preinstalled before execution. let global_site_packages_path = py_version.to_cache_dir() + "/global-site-packages"; let additional_python_paths_folders = { - let mut paths= additional_python_paths.clone(); + let mut paths = additional_python_paths.clone(); if std::fs::metadata(&global_site_packages_path).is_ok() { // We want global_site_packages_path to be included in additonal_python_paths_folders, but // we don't want it to be included in global_site_packages_path. @@ -1189,13 +1217,13 @@ mount {{ let args = read_file(&format!("{job_dir}/args.json")) .await .map_err(|e| { - error::Error::InternalErr(format!( + error::Error::internal_err(format!( "error while reading args from preprocessing: {e:#}" )) })?; let args: HashMap> = serde_json::from_str(args.get()).map_err(|e| { - error::Error::InternalErr(format!( + error::Error::internal_err(format!( "error while deserializing args from preprocessing: {e:#}" )) })?; @@ -1207,10 +1235,11 @@ mount {{ async fn prepare_wrapper( job_dir: &str, + job_is_flow_step: bool, + job_preprocessed: Option, + job_script_entrypoint_override: Option<&str>, inner_content: &str, script_path: &str, - args: Option<&Json>>>, - skip_preprocessor: bool, ) -> error::Result<( &'static str, &'static str, @@ -1223,16 +1252,8 @@ async fn prepare_wrapper( Option, Option, )> { - let (main_override, apply_preprocessor) = match get_main_override(args) { - Some(main_override) => { - if !skip_preprocessor && main_override == PREPROCESSOR_FAKE_ENTRYPOINT { - (None, true) - } else { - (Some(main_override), false) - } - } - None => (None, false), - }; + let main_override = job_script_entrypoint_override.as_deref(); + let apply_preprocessor = !job_is_flow_step && job_preprocessed == Some(false); let relative_imports = RELATIVE_IMPORT_REGEX.is_match(&inner_content); @@ -1272,7 +1293,10 @@ async fn prepare_wrapper( let _ = write_file(job_dir, "loader.py", RELATIVE_PYTHON_LOADER)?; } - let sig = windmill_parser_py::parse_python_signature(inner_content, main_override.clone())?; + let sig = windmill_parser_py::parse_python_signature( + inner_content, + main_override.map(ToString::to_string), + )?; let pre_sig = if apply_preprocessor { Some(windmill_parser_py::parse_python_signature( @@ -1366,12 +1390,12 @@ async fn prepare_wrapper( } else { format!( r#"pre_args["{name}"] = kwargs.get("{name}") - if pre_args["{name}"] is None: - del pre_args["{name}"]"# + if pre_args["{name}"] is None: + del pre_args["{name}"]"# ) } }) - .join("\n ") + .join("\n ") }; Some(spread) } else { @@ -1389,7 +1413,7 @@ async fn prepare_wrapper( last, transforms, spread, - main_override, + main_override.map(ToString::to_string), pre_spread, )) } @@ -1409,7 +1433,7 @@ async fn replace_pip_secret( let capture = PIP_SECRET_VARIABLE.captures(req); let variable = capture.unwrap().get(1).unwrap().as_str(); if !variable.contains("/PIP_SECRET_") { - return Err(error::Error::InternalErr(format!( + return Err(error::Error::internal_err(format!( "invalid secret variable in pip requirements, (last part of path ma): {}", req ))); @@ -1586,6 +1610,7 @@ async fn spawn_uv_install( // If none, it is system python py_path: Option, no_uv_install: bool, + worker_dir: &str, ) -> Result { if !*DISABLE_NSJAIL { tracing::info!( @@ -1620,13 +1645,30 @@ async fn spawn_uv_install( vars.push(("REQ", &req)); vars.push(("TARGET", venv_p)); + std::fs::create_dir_all(venv_p)?; + let nsjail_proto = format!("{req}.config.proto"); + // Prepare NSJAIL + let _ = write_file( + job_dir, + &nsjail_proto, + &(if no_uv_install { + NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT_FALLBACK + } else { + NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT + }) + .replace("{WORKER_DIR}", worker_dir) + .replace("{PY_INSTALL_DIR}", &PY_INSTALL_DIR) + .replace("{TARGET_DIR}", &venv_p) + .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()), + )?; + let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); nsjail_cmd .current_dir(job_dir) .env_clear() .envs(vars) .envs(PROXY_ENVS.clone()) - .args(vec!["--config", "download.config.proto"]) + .args(vec!["--config", &nsjail_proto]) .stdout(Stdio::piped()) .stderr(Stdio::piped()); start_child_process(nsjail_cmd, NSJAIL_PATH.as_str()).await @@ -1721,7 +1763,7 @@ async fn spawn_uv_install( // Track https://github.com/astral-sh/uv/issues/6715 if let Some(cert_path) = INDEX_CERT.as_ref() { // Once merged --cert can be used instead - // + // // command_args.extend(["--cert", cert_path]); envs.push(("SSL_CERT_FILE", cert_path)); } @@ -1812,6 +1854,8 @@ pub async fn handle_python_reqs( // TODO: Remove (Deprecated) mut no_uv_install: bool, ) -> error::Result> { + let worker_dir = worker_dir.to_string(); + let counter_arc = Arc::new(tokio::sync::Mutex::new(0)); // Append logs with line like this: // [9/21] + requests==2.32.3 << (S3) | in 57ms @@ -1895,30 +1939,6 @@ pub async fn handle_python_reqs( .map(handle_ephemeral_token), ); - // Prepare NSJAIL - if !*DISABLE_NSJAIL { - let _ = write_file( - job_dir, - "download.config.proto", - &(if no_uv_install { - NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT_FALLBACK - } else { - NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT - }) - .replace("{WORKER_DIR}", &worker_dir) - .replace("{PY_INSTALL_DIR}", &PY_INSTALL_DIR) - .replace( - "{CACHE_DIR}", - &(if no_uv_install { - PIP_CACHE_DIR.to_owned() - } else { - py_version.to_cache_dir() - }), - ) - .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()), - )?; - }; - // Cached paths let mut req_with_penv: Vec<(String, String)> = vec![]; // Requirements to pull (not cached) @@ -2020,29 +2040,26 @@ pub async fn handle_python_reqs( // Notify server that we are still alive // Detect if job has been canceled - let canceled = - sqlx::query_scalar::<_, bool> - (r#" - - UPDATE queue - SET last_ping = now() - , mem_peak = $1 - WHERE id = $2 - RETURNING canceled - - "#) - .bind(mem_peak_actual) - .bind(job_id_2) - .fetch_optional(&db_2) - .await - .unwrap_or_else(|e| { - tracing::error!(%e, "error updating job {job_id_2}: {e:#}"); - Some(false) - }) - .unwrap_or_else(|| { - // if the job is not in queue, it can only be in the completed_job so it is already complete - false - }); + let canceled = sqlx::query_scalar!( + "UPDATE v2_job_runtime r SET + memory_peak = $1, + ping = now() + FROM v2_job_queue q + WHERE r.id = $2 AND q.id = r.id + RETURNING canceled_by IS NOT NULL AS \"canceled!\"", + mem_peak_actual, + job_id_2 + ) + .fetch_optional(&db_2) + .await + .unwrap_or_else(|e| { + tracing::error!(%e, "error updating job {job_id_2}: {e:#}"); + Some(false) + }) + .unwrap_or_else(|| { + // if the job is not in queue, it can only be in the completed_job so it is already complete + false + }); if canceled { @@ -2155,7 +2172,7 @@ pub async fn handle_python_reqs( let pip_indexes = pip_indexes.clone(); let py_path = py_path.clone(); let pids = pids.clone(); - + let worker_dir = worker_dir.clone(); handles.push(task::spawn(async move { // permit will be dropped anyway if this thread exits at any point // so we dont have to drop it manually @@ -2223,7 +2240,8 @@ pub async fn handle_python_reqs( &job_dir, pip_indexes, py_path, - no_uv_install + no_uv_install, + &worker_dir ).await { Ok(r) => r, Err(e) => { @@ -2263,12 +2281,12 @@ pub async fn handle_python_reqs( uv_install_proccess.kill().await?; pids.lock().await.get_mut(i).and_then(|e| e.take()); return Err(anyhow::anyhow!("uv pip install was canceled")); - }, + }, (_, exitstatus) = async { // See tokio::process::Child::wait_with_output() for more context // Sometimes uv_install_proccess.wait() is not exiting if stderr is not awaited before it :/ (stderr_future.await, uv_install_proccess.wait().await) - } => match exitstatus { + } => match exitstatus { Ok(status) => if !status.success() { tracing::warn!( workspace_id = %w_id, @@ -2307,6 +2325,10 @@ pub async fn handle_python_reqs( #[cfg(not(all(feature = "enterprise", feature = "parquet", unix)))] let s3_push = false; + if !*DISABLE_NSJAIL { + let _ = std::fs::remove_file(format!("{job_dir}/{req}.config.proto")); + } + print_success( false, s3_push, @@ -2449,7 +2471,6 @@ pub async fn start_worker( ) .await?; - let _args = None; let ( import_loader, import_base64, @@ -2461,7 +2482,7 @@ pub async fn start_worker( spread, _, _, - ) = prepare_wrapper(job_dir, inner_content, script_path, _args.as_ref(), true).await?; + ) = prepare_wrapper(job_dir, false, None, None, inner_content, script_path).await?; { let indented_transforms = transforms diff --git a/backend/windmill-worker/src/result_processor.rs b/backend/windmill-worker/src/result_processor.rs index 5812b12175..1b7ec63a1c 100644 --- a/backend/windmill-worker/src/result_processor.rs +++ b/backend/windmill-worker/src/result_processor.rs @@ -26,7 +26,7 @@ use windmill_common::{ }; #[cfg(feature = "benchmark")] -use windmill_common::bench::{BenchmarkInfo, BenchmarkIter}; +use crate::bench::{BenchmarkInfo, BenchmarkIter}; use windmill_queue::{append_logs, get_queued_job, CanceledBy, WrappedError}; @@ -232,6 +232,7 @@ async fn send_job_completed( job_completed_tx: JobCompletedSender, job: Arc, result: Arc>, + result_columns: Option>, mem_peak: i32, canceled_by: Option, success: bool, @@ -242,6 +243,7 @@ async fn send_job_completed( let jc = JobCompleted { job, result, + result_columns, mem_peak, canceled_by, success, @@ -272,52 +274,22 @@ pub async fn process_result( ) -> error::Result { match result { Ok(r) => { - let job = if column_order.is_some() || new_args.is_some() { - let mut updated_job = (*job).clone(); - if let Some(column_order) = column_order { - match updated_job.flow_status { - Some(_) => { - tracing::warn!("flow_status was expected to be none"); - } - None => { - updated_job.flow_status = - Some(sqlx::types::Json(to_raw_value(&serde_json::json!({ - "_metadata": { - "column_order": column_order - } - })))); - } - } - } - if let Some(new_args) = new_args { - match updated_job.flow_status { - Some(_) => { - tracing::warn!("flow_status was expected to be none"); - } - None => { - // TODO save original args somewhere - // if let Some(args) = updated_job.args.as_mut() { - // args.0.remove(ENTRYPOINT_OVERRIDE); - // } - updated_job.flow_status = - Some(sqlx::types::Json(to_raw_value(&serde_json::json!({ - "_metadata": { - "preprocessed_args": true - } - })))); - } - } - updated_job.args = Some(Json(new_args)); - } - Arc::new(updated_job) - } else { - job - }; + // Update script args to preprocessed args + if let Some(preprocessed_args) = new_args { + sqlx::query!( + "UPDATE v2_job SET args = $1, preprocessed = TRUE WHERE id = $2", + Json(preprocessed_args) as Json>>, + job.id + ) + .execute(db) + .await?; + } send_job_completed( job_completed_tx, job, r, + column_order, mem_peak, canceled_by, true, @@ -331,7 +303,7 @@ pub async fn process_result( } Err(e) => { let error_value = match e { - Error::ExitStatus(i) => { + Error::ExitStatus(program, i) => { let res = read_result(job_dir).await.ok(); if res.as_ref().is_some_and(|x| !x.get().is_empty()) { @@ -348,11 +320,11 @@ pub async fn process_result( .last() .unwrap_or(&last_10_log_lines); - extract_error_value(log_lines, i, job.flow_step_id.clone()) + extract_error_value(&program, log_lines, i, job.flow_step_id.clone()) } } err @ _ => to_raw_value(&SerializedError { - message: format!("error during execution of the script:\n{}", err), + message: format!("error during execution of the script:\n{err:#}",), name: "ExecutionErr".to_string(), step_id: job.flow_step_id.clone(), exit_code: None, @@ -363,6 +335,7 @@ pub async fn process_result( job_completed_tx, job, Arc::new(to_raw_value(&error_value)), + None, mem_peak, canceled_by, false, @@ -435,7 +408,17 @@ pub async fn handle_receive_completed_job( } pub async fn process_completed_job( - JobCompleted { job, result, mem_peak, success, cached_res_path, canceled_by, duration, .. }: JobCompleted, + JobCompleted { + job, + result, + mem_peak, + success, + cached_res_path, + canceled_by, + duration, + result_columns, + .. + }: JobCompleted, client: &AuthedClient, db: &DB, worker_dir: &str, @@ -455,12 +438,32 @@ pub async fn process_completed_job( let job_id = job.id.clone(); let workspace_id = job.workspace_id.clone(); + if job.flow_step_id.as_deref() == Some("preprocessor") { + // Do this before inserting to `v2_job_completed` for backwards compatibility + // when we set `flow_status->_metadata->preprocessed_args` to true. + sqlx::query!( + r#"UPDATE v2_job SET + args = '{"reason":"PREPROCESSOR_ARGS_ARE_DISCARDED"}'::jsonb, + preprocessed = TRUE + WHERE id = $1 AND preprocessed = FALSE"#, + job.id + ) + .execute(db) + .await + .map_err(|e| { + Error::InternalErr(format!( + "error while deleting args of preprocessing step: {e:#}" + )) + })?; + } + add_completed_job( db, &job, true, false, Json(&result), + result_columns, mem_peak.to_owned(), canceled_by, false, @@ -662,10 +665,15 @@ pub struct SerializedError { #[serde(skip_serializing_if = "Option::is_none")] pub exit_code: Option, } -pub fn extract_error_value(log_lines: &str, i: i32, step_id: Option) -> Box { +pub fn extract_error_value( + program: &str, + log_lines: &str, + i: i32, + step_id: Option, +) -> Box { return to_raw_value(&SerializedError { message: format!( - "ExitCode: {i}, last log lines:\n{}", + "exit code for \"{program}\": {i}, last log lines:\n{}", ANSI_ESCAPE_RE.replace_all(log_lines.trim(), "").to_string() ), name: "ExecutionErr".to_string(), diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 2c2aefe6b3..90b0444901 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -146,7 +146,7 @@ use crate::mssql_executor::do_mssql; use crate::bigquery_executor::do_bigquery; #[cfg(feature = "benchmark")] -use windmill_common::bench::{benchmark_init, BenchmarkInfo, BenchmarkIter}; +use crate::bench::{benchmark_init, BenchmarkInfo, BenchmarkIter}; use windmill_common::add_time; @@ -208,7 +208,7 @@ pub async fn create_token_for_owner( let jwt_secret = JWT_SECRET.read().await; if jwt_secret.is_empty() { - return Err(Error::InternalErr("No JWT secret found".to_string())); + return Err(Error::internal_err("No JWT secret found".to_string())); } let job_authed = match sqlx::query_as!( @@ -226,7 +226,7 @@ pub async fn create_token_for_owner( fetch_authed_from_permissioned_as(owner.to_string(), email.to_string(), w_id, db) .await .map_err(|e| { - Error::InternalErr(format!( + Error::internal_err(format!( "Could not get permissions directly for job {job_id}: {e:#}" )) })? @@ -254,7 +254,7 @@ pub async fn create_token_for_owner( &jsonwebtoken::EncodingKey::from_secret(jwt_secret.as_bytes()), ) .map_err(|err| { - Error::InternalErr(format!( + Error::internal_err(format!( "Could not encode JWT token for job {job_id}: {:?}", err )) @@ -1278,7 +1278,7 @@ pub async fn run_worker( tokio::task::spawn( (async move { tracing::info!(worker = %worker_name, hostname = %hostname, "vacuuming queue"); - if let Err(e) = sqlx::query!("VACUUM (skip_locked) queue") + if let Err(e) = sqlx::query!("VACUUM (skip_locked) v2_job_queue, v2_job_runtime, v2_job_status") .execute(&db2) .await { @@ -1325,12 +1325,28 @@ pub async fn run_worker( same_worker_job.job_id ); let r = sqlx::query_as::<_, PulledJob>( - "UPDATE queue SET last_ping = now() WHERE id = $1 RETURNING *", + " + WITH ping AS ( + UPDATE v2_job_runtime SET ping = NOW() WHERE id = $1 RETURNING id + ) + SELECT * FROM v2_as_queue WHERE id = (SELECT id FROM ping) + ", ) .bind(same_worker_job.job_id) .fetch_optional(db) .await - .map_err(|_| Error::InternalErr("Impossible to fetch same_worker job".to_string())); + .map_err(|e| { + Error::internal_err(format!( + "Impossible to fetch same_worker job {}: {}", + same_worker_job.job_id, e + )) + }); + let _ = sqlx::query!( + "UPDATE v2_job_queue SET started_at = NOW() WHERE id = $1", + same_worker_job.job_id + ) + .execute(db) + .await; if r.is_err() && !same_worker_job.recoverable { tracing::error!( worker = %worker_name, hostname = %hostname, @@ -1377,7 +1393,7 @@ pub async fn run_worker( last_suspend_first = Instant::now(); } - let job = pull(&db, suspend_first).await; + let job = pull(&db, suspend_first, &worker_name).await; add_time!(bench, "job pulled from DB"); let duration_pull_s = pull_time.elapsed().as_secs_f64(); @@ -1484,6 +1500,7 @@ pub async fn run_worker( job: Arc::new(job.job), success: true, result: Arc::new(empty_result()), + result_columns: None, mem_peak: 0, cached_res_path: None, token: "".to_string(), @@ -1880,6 +1897,7 @@ pub enum SendResult { pub struct JobCompleted { pub job: Arc, pub result: Arc>, + pub result_columns: Option>, pub mem_peak: i32, pub success: bool, pub cached_res_path: Option, @@ -1994,23 +2012,36 @@ async fn handle_queued_job( db, &job.workspace_id, job.parent_job - .ok_or_else(|| Error::InternalErr(format!("expected parent job")))?, + .ok_or_else(|| Error::internal_err(format!("expected parent job")))?, job.id, ) .warn_after_seconds(5) .await?; } else if let Some(parent_job) = job.parent_job { - if let Err(e) = sqlx::query_scalar!( - "UPDATE queue SET flow_status = jsonb_set(jsonb_set(COALESCE(flow_status, '{}'::jsonb), array[$1], COALESCE(flow_status->$1, '{}'::jsonb)), array[$1, 'started_at'], to_jsonb(now()::text)) WHERE id = $2 AND workspace_id = $3", + let _ = sqlx::query_scalar!( + "UPDATE v2_job_status SET + workflow_as_code_status = jsonb_set( + jsonb_set( + COALESCE(workflow_as_code_status, '{}'::jsonb), + array[$1], + COALESCE(workflow_as_code_status->$1, '{}'::jsonb) + ), + array[$1, 'started_at'], + to_jsonb(now()::text) + ) + WHERE id = $2", &job.id.to_string(), - parent_job, - &job.workspace_id + parent_job ) .execute(db) .warn_after_seconds(5) - .await { - tracing::error!("Could not update parent job started_at flow_status: {}", e); - } + .await + .inspect_err(|e| { + tracing::error!( + "Could not update parent job `started_at` in workflow as code status: {}", + e + ) + }); } let started = Instant::now(); @@ -2056,6 +2087,7 @@ async fn handle_queued_job( .send(JobCompleted { job, result, + result_columns: None, mem_peak: 0, canceled_by: None, success: true, @@ -2084,6 +2116,7 @@ async fn handle_queued_job( same_worker_tx, worker_dir, job_completed_tx.0.clone(), + worker_name, ) .warn_after_seconds(10) .await?; @@ -2280,7 +2313,7 @@ pub async fn get_hub_script_content_and_requirements( ) -> error::Result { let script_path = script_path .clone() - .ok_or_else(|| Error::InternalErr(format!("expected script path for hub script")))?; + .ok_or_else(|| Error::internal_err(format!("expected script path for hub script")))?; let script = get_full_hub_script_by_path(StripPath(script_path.to_string()), &HTTP_CLIENT, db).await?; @@ -2331,7 +2364,7 @@ async fn handle_code_execution_job( ) -> error::Result> { let script_hash = || { job.script_hash - .ok_or_else(|| Error::InternalErr("expected script hash".into())) + .ok_or_else(|| Error::internal_err("expected script hash")) }; let (arc_data, arc_metadata, data, metadata): ( Arc, @@ -2349,7 +2382,8 @@ async fn handle_code_execution_job( _ => None, }; - arc_data = preview.ok_or_else(|| Error::InternalErr("expected preview".to_string()))?; + arc_data = + preview.ok_or_else(|| Error::internal_err("expected preview".to_string()))?; metadata = ScriptMetadata { language: job.language, codebase, envs: None }; (arc_data.as_ref(), &metadata) } @@ -2378,7 +2412,7 @@ async fn handle_code_execution_job( let script_path = job .script_path .as_ref() - .ok_or_else(|| Error::InternalErr("expected script path".to_string()))?; + .ok_or_else(|| Error::internal_err("expected script path".to_string()))?; if script_path.starts_with("hub/") { let ContentReqLangEnvs { content, lockfile, language, envs, codebase } = get_hub_script_content_and_requirements(Some(script_path), Some(db)).await?; @@ -2394,7 +2428,7 @@ async fn handle_code_execution_job( ) .fetch_optional(db) .await? - .ok_or_else(|| Error::InternalErr("expected script hash".to_string()))?; + .ok_or_else(|| Error::internal_err("expected script hash".to_string()))?; (arc_data, arc_metadata) = cache::script::fetch(db, ScriptHash(hash)).await?; (arc_data.as_ref(), arc_metadata.as_ref()) @@ -2421,7 +2455,7 @@ async fn handle_code_execution_job( .await; } else if language == Some(ScriptLang::Mysql) { #[cfg(not(feature = "mysql"))] - return Err(Error::InternalErr( + return Err(Error::internal_err( "MySQL requires the mysql feature to be enabled".to_string(), )); @@ -2449,7 +2483,7 @@ async fn handle_code_execution_job( #[allow(unreachable_code)] #[cfg(not(feature = "bigquery"))] { - return Err(Error::InternalErr( + return Err(Error::internal_err( "Bigquery requires the bigquery feature to be enabled".to_string(), )); } @@ -2503,7 +2537,7 @@ async fn handle_code_execution_job( #[allow(unreachable_code)] #[cfg(not(feature = "mssql"))] { - return Err(Error::InternalErr( + return Err(Error::internal_err( "Microsoft SQL server requires the mssql feature to be enabled".to_string(), )); } @@ -2533,7 +2567,7 @@ async fn handle_code_execution_job( #[allow(unreachable_code)] #[cfg(not(feature = "oracledb"))] { - return Err(Error::InternalErr( + return Err(Error::internal_err( "Oracle DB requires the oracledb feature to be enabled".to_string(), )); } @@ -2644,7 +2678,7 @@ mount {{ } Some(ScriptLang::Python3) => { #[cfg(not(feature = "python"))] - return Err(Error::InternalErr( + return Err(Error::internal_err( "Python requires the python feature to be enabled".to_string(), )); @@ -2761,7 +2795,7 @@ mount {{ } Some(ScriptLang::Php) => { #[cfg(not(feature = "php"))] - return Err(Error::InternalErr( + return Err(Error::internal_err( "PHP requires the php feature to be enabled".to_string(), )); @@ -2785,7 +2819,7 @@ mount {{ } Some(ScriptLang::Rust) => { #[cfg(not(feature = "rust"))] - return Err(Error::InternalErr( + return Err(Error::internal_err( "Rust requires the rust feature to be enabled".to_string(), )); @@ -2809,7 +2843,7 @@ mount {{ } Some(ScriptLang::Ansible) => { #[cfg(not(feature = "python"))] - return Err(Error::InternalErr( + return Err(Error::internal_err( "Ansible requires the python feature to be enabled".to_string(), )); diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index cf24258b9e..42dd66c217 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -11,6 +11,8 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; use std::time::Duration; +#[cfg(feature = "benchmark")] +use crate::bench::BenchmarkIter; use crate::common::{cached_result_path, save_in_cache}; use crate::js_eval::{eval_timeout, IdContext}; use crate::{ @@ -30,17 +32,15 @@ use tracing::instrument; use uuid::Uuid; use windmill_common::add_time; use windmill_common::auth::JobPerms; -#[cfg(feature = "benchmark")] -use windmill_common::bench::BenchmarkIter; use windmill_common::cache::{self, RawData}; use windmill_common::db::Authed; use windmill_common::flow_status::{ - ApprovalConditions, FlowStatusModuleWParent, Iterator, JobResult, + ApprovalConditions, FlowStatusModuleWParent, Iterator as FlowIterator, JobResult, }; use windmill_common::flows::{add_virtual_items_if_necessary, Branch, FlowNodeId}; use windmill_common::jobs::{ - script_hash_to_tag_and_limits, script_path_to_payload, BranchResults, JobKind, JobPayload, - OnBehalfOf, QueuedJob, RawCode, ENTRYPOINT_OVERRIDE, + script_hash_to_tag_and_limits, script_path_to_payload, JobKind, JobPayload, OnBehalfOf, + QueuedJob, RawCode, ENTRYPOINT_OVERRIDE, }; use windmill_common::scripts::ScriptHash; use windmill_common::users::username_to_permissioned_as; @@ -179,11 +179,6 @@ pub struct RecUpdateFlowStatusAfterJobCompletion { skip_error_handler: bool, } -#[derive(FromRow)] -pub struct RowArgs { - pub args: Option>>>, -} - #[derive(Deserialize)] struct RecoveryObject { recover: Option, @@ -222,18 +217,18 @@ pub async fn update_flow_status_after_job_completion_internal( let (job_kind, script_hash, old_status, raw_flow) = sqlx::query!( "SELECT - job_kind AS \"job_kind: JobKind\", + job_kind AS \"job_kind!: JobKind\", script_hash AS \"script_hash: ScriptHash\", flow_status AS \"flow_status!: Json>\", raw_flow AS \"raw_flow: Json>\" - FROM queue WHERE id = $1 AND workspace_id = $2 LIMIT 1", + FROM v2_as_queue WHERE id = $1 AND workspace_id = $2 LIMIT 1", flow, w_id ) .fetch_one(db) .await .map_err(|e| { - Error::InternalErr(format!( + Error::internal_err(format!( "fetching flow status {flow} while reporting {success} {result:?}: {e:#}" )) }) @@ -242,7 +237,7 @@ pub async fn update_flow_status_after_job_completion_internal( record.job_kind, record.script_hash, serde_json::from_str::(record.flow_status.0.get()).map_err(|e| { - Error::InternalErr(format!( + Error::internal_err(format!( "requiring current module to be parsable as FlowStatus: {e:?}" )) })?, @@ -264,12 +259,12 @@ pub async fn update_flow_status_after_job_completion_internal( Step::PreprocessorStep => old_status .preprocessor_module .as_ref() - .ok_or_else(|| Error::InternalErr(format!("preprocessor module not found")))?, + .ok_or_else(|| Error::internal_err(format!("preprocessor module not found")))?, Step::FailureStep => &old_status.failure_module.module_status, Step::Step(i) => old_status .modules .get(i as usize) - .ok_or_else(|| Error::InternalErr(format!("module {i} not found")))?, + .ok_or_else(|| Error::internal_err(format!("module {i} not found")))?, }; // tracing::debug!( @@ -355,22 +350,21 @@ pub async fn update_flow_status_after_job_completion_internal( )), _ => None, }; - let args = sqlx::query_as::<_, RowArgs>( + let args = sqlx::query_scalar!( "SELECT - args - FROM queue - WHERE id = $2", + args AS \"args: Json>>\" + FROM v2_job + WHERE id = $1", + flow ) - .bind(old_status.step) - .bind(flow) .fetch_one(db) .await .map_err(|e| { - Error::InternalErr(format!("retrieval of args from state: {e:#}")) + Error::internal_err(format!("retrieval of args from state: {e:#}")) })?; compute_bool_from_expr( &expr, - Marc::new(args.args.unwrap_or_default().0), + Marc::new(args.unwrap_or_default().0), result.clone(), all_iters, None, @@ -414,22 +408,18 @@ pub async fn update_flow_status_after_job_completion_internal( if matches!(module_step, Step::PreprocessorStep) { sqlx::query!( - "UPDATE queue SET args = (select result FROM completed_job WHERE id = $1) WHERE id = $2", + "UPDATE v2_job SET + args = (SELECT result FROM v2_job_completed WHERE id = $1), + preprocessed = TRUE + WHERE id = $2", job_id_for_status, flow - ).execute(db).await.map_err(|e| { - Error::InternalErr(format!("error while updating args in preprocessing step: {e:#}")) - })?; - - sqlx::query!( - r#"UPDATE completed_job SET args = '{"reason":"PREPROCESSOR_ARGS_ARE_DISCARDED"}'::jsonb WHERE id = $1"#, - job_id_for_status ) .execute(db) .await .map_err(|e| { - Error::InternalErr(format!( - "error while deleting args of preprocessing step: {e:#}" + Error::internal_err(format!( + "error while updating args in preprocessing step: {e:#}" )) })?; } @@ -448,7 +438,7 @@ pub async fn update_flow_status_after_job_completion_internal( .. } if *parallel => { let (nindex, len) = match (iterator, branchall) { - (Some(Iterator { itered, .. }), _) => { + (Some(FlowIterator { itered, .. }), _) => { let position = if flow_jobs_success.is_some() { find_flow_job_index(jobs, job_id_for_status) } else { @@ -457,24 +447,27 @@ pub async fn update_flow_status_after_job_completion_internal( let nindex = if let Some(position) = position { sqlx::query_scalar!( - "UPDATE queue - SET flow_status = JSONB_SET( - JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4), - ARRAY['modules', $1::TEXT, 'iterator', 'index'], - ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb - ), - last_ping = NULL - WHERE id = $2 - RETURNING (flow_status->'modules'->$1::int->'iterator'->>'index')::int", - old_status.step, - flow, - position as i32, - json!(success) - )} else { + "UPDATE v2_job_status SET + flow_status = JSONB_SET( + JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4), + ARRAY['modules', $1::TEXT, 'iterator', 'index'], + ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb + ) + WHERE id = $2 + RETURNING (flow_status->'modules'->$1::int->'iterator'->>'index')::int", + old_status.step, + flow, + position as i32, + json!(success) + ) + } else { sqlx::query_scalar!( - "UPDATE queue - SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'iterator', 'index'], ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb), - last_ping = NULL + "UPDATE v2_job_status SET + flow_status = JSONB_SET( + flow_status, + ARRAY['modules', $1::TEXT, 'iterator', 'index'], + ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb + ) WHERE id = $2 RETURNING (flow_status->'modules'->$1::int->'iterator'->>'index')::int", old_status.step, @@ -483,11 +476,11 @@ pub async fn update_flow_status_after_job_completion_internal( } .fetch_one(&mut *tx) .await.map_err(|e| { - Error::InternalErr(format!( + Error::internal_err(format!( "error while fetching iterator index: {e:#}" )) })? - .ok_or_else(|| Error::InternalErr(format!("requiring an index in InProgress")))?; + .ok_or_else(|| Error::internal_err(format!("requiring an index in InProgress")))?; tracing::info!( "parallel iteration {job_id_for_status} of flow {flow} update nindex: {nindex} len: {len}", nindex = nindex, @@ -504,11 +497,12 @@ pub async fn update_flow_status_after_job_completion_internal( let nindex = if let Some(position) = position { sqlx::query_scalar!( - "UPDATE queue - SET flow_status = JSONB_SET( - JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4), - ARRAY['modules', $1::TEXT, 'branchall', 'branch'], ((flow_status->'modules'->$1::int->'branchall'->>'branch')::int + 1)::text::jsonb), - last_ping = NULL + "UPDATE v2_job_status SET + flow_status = JSONB_SET( + JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4), + ARRAY['modules', $1::TEXT, 'branchall', 'branch'], + ((flow_status->'modules'->$1::int->'branchall'->>'branch')::int + 1)::text::jsonb + ) WHERE id = $2 RETURNING (flow_status->'modules'->$1::int->'branchall'->>'branch')::int", old_status.step, @@ -516,29 +510,38 @@ pub async fn update_flow_status_after_job_completion_internal( position as i32, json!(success) ) - } else { sqlx::query_scalar!( - "UPDATE queue - SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'branchall', 'branch'], ((flow_status->'modules'->$1::int->'branchall'->>'branch')::int + 1)::text::jsonb), - last_ping = NULL - WHERE id = $2 - RETURNING (flow_status->'modules'->$1::int->'branchall'->>'branch')::int", - old_status.step, - flow - )} + } else { + sqlx::query_scalar!( + "UPDATE v2_job_status SET + flow_status = JSONB_SET( + flow_status, + ARRAY['modules', $1::TEXT, 'branchall', 'branch'], + ((flow_status->'modules'->$1::int->'branchall'->>'branch')::int + 1)::text::jsonb + ) + WHERE id = $2 + RETURNING (flow_status->'modules'->$1::int->'branchall'->>'branch')::int", + old_status.step, + flow + ) + } .fetch_one(&mut *tx) .await .map_err(|e| { - Error::InternalErr(format!( + Error::internal_err(format!( "error while fetching branchall index: {e:#}" )) })? - .ok_or_else(|| Error::InternalErr(format!("requiring an index in InProgress")))?; + .ok_or_else(|| Error::internal_err(format!("requiring an index in InProgress")))?; (nindex, *len as i32) } - _ => Err(Error::InternalErr(format!( + _ => Err(Error::internal_err(format!( "unexpected status for parallel module" )))?, }; + // reset ping after updating flow status: + let _ = sqlx::query!("UPDATE v2_job_runtime SET ping = NULL WHERE id = $1", flow) + .execute(&mut *tx) + .await?; if nindex == len { let mut flow_jobs_success = flow_jobs_success.clone(); if let Some(flow_job_success) = flow_jobs_success.as_mut() { @@ -552,13 +555,13 @@ pub async fn update_flow_status_after_job_completion_internal( let new_status = if skip_loop_failures || sqlx::query_scalar!( - "SELECT success FROM completed_job WHERE id = ANY($1)", + "SELECT success AS \"success!\" FROM v2_as_completed_job WHERE id = ANY($1)", jobs.as_slice() ) .fetch_all(&mut *tx) .await .map_err(|e| { - Error::InternalErr(format!( + Error::internal_err(format!( "error while fetching sucess from completed_jobs: {e:#}" )) })? @@ -591,7 +594,7 @@ pub async fn update_flow_status_after_job_completion_internal( "DELETE FROM parallel_monitor_lock WHERE parent_flow_id = $1 RETURNING last_ping", flow, ).fetch_optional(db).await.map_err(|e| { - Error::InternalErr(format!( + Error::internal_err(format!( "error while deleting parallel_monitor_lock: {e:#}" )) })?; @@ -611,14 +614,18 @@ pub async fn update_flow_status_after_job_completion_internal( if parallelism.is_some() { sqlx::query!( - "UPDATE queue SET suspend = 0 WHERE parent_job = $1 AND suspend = $2 AND (flow_status->'step')::int = 0", + "UPDATE v2_job_queue q SET suspend = 0 + FROM v2_job j, v2_job_status f + WHERE parent_job = $1 + AND f.id = j.id AND q.id = j.id + AND suspend = $2 AND (f.flow_status->'step')::int = 0", flow, nindex ) .execute(db) .await .map_err(|e| { - Error::InternalErr(format!( + Error::internal_err(format!( "error resuming job at suspend {nindex} and parent {flow}: {e:#}" )) })?; @@ -629,7 +636,7 @@ pub async fn update_flow_status_after_job_completion_internal( flow, job_id_for_status ).fetch_optional(db).await.map_err(|e| { - Error::InternalErr(format!("error while removing parallel_monitor_lock: {e:#}")) + Error::internal_err(format!("error while removing parallel_monitor_lock: {e:#}")) })?; if r.is_some() { tracing::info!( @@ -642,7 +649,7 @@ pub async fn update_flow_status_after_job_completion_internal( } } FlowStatusModule::InProgress { - iterator: Some(windmill_common::flow_status::Iterator { index, itered, .. }), + iterator: Some(FlowIterator { index, itered, .. }), flow_jobs_success, flow_jobs, while_loop, @@ -715,12 +722,14 @@ pub async fn update_flow_status_after_job_completion_internal( let is_skipped = if current_module.as_ref().is_some_and(|m| m.skip_if.is_some()) { sqlx::query_scalar!( - "SELECT job_kind = 'identity' FROM completed_job WHERE id = $1", + "SELECT kind = 'identity' FROM v2_job WHERE id = $1", job_id_for_status ) .fetch_one(db) .await - .map_err(|e| Error::InternalErr(format!("error during skip check: {e:#}")))? + .map_err(|e| { + Error::internal_err(format!("error during skip check: {e:#}")) + })? .unwrap_or(false) } else { false @@ -768,7 +777,7 @@ pub async fn update_flow_status_after_job_completion_internal( let step_counter = if inc_step_counter { sqlx::query!( - "UPDATE queue + "UPDATE v2_job_status SET flow_status = JSONB_SET(flow_status, ARRAY['step'], $1) WHERE id = $2", json!(old_status.step + 1), @@ -777,7 +786,7 @@ pub async fn update_flow_status_after_job_completion_internal( .execute(&mut *tx) .await .map_err(|e| { - Error::InternalErr(format!("error while setting flow index for {flow}: {e:#}")) + Error::internal_err(format!("error while setting flow index for {flow}: {e:#}")) })?; old_status.step + 1 } else { @@ -792,18 +801,18 @@ pub async fn update_flow_status_after_job_completion_internal( if let Some(new_status) = new_status.as_ref() { if is_failure_step { let parent_module = sqlx::query_scalar!( - "SELECT flow_status->'failure_module'->>'parent_module' FROM queue WHERE id = $1", + "SELECT flow_status->'failure_module'->>'parent_module' FROM v2_job_status WHERE id = $1", flow ) .fetch_one(&mut *tx) .await.map_err(|e| { - Error::InternalErr(format!( + Error::internal_err(format!( "error while fetching failure module: {e:#}" )) })?; sqlx::query!( - "UPDATE queue + "UPDATE v2_job_status SET flow_status = JSONB_SET(flow_status, ARRAY['failure_module'], $1) WHERE id = $2", json!(FlowStatusModuleWParent { @@ -815,13 +824,13 @@ pub async fn update_flow_status_after_job_completion_internal( .execute(&mut *tx) .await .map_err(|e| { - Error::InternalErr(format!( + Error::internal_err(format!( "error while setting flow status in failure step: {e:#}" )) })?; } else if matches!(module_step, Step::PreprocessorStep) { sqlx::query!( - "UPDATE queue + "UPDATE v2_job_status SET flow_status = JSONB_SET(flow_status, ARRAY['preprocessor_module'], $1) WHERE id = $2", json!(new_status), @@ -830,13 +839,13 @@ pub async fn update_flow_status_after_job_completion_internal( .execute(&mut *tx) .await .map_err(|e| { - Error::InternalErr(format!( + Error::internal_err(format!( "error while setting flow status in preprocessing step: {e:#}" )) })?; } else { sqlx::query!( - "UPDATE queue + "UPDATE v2_job_status SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT], $2) WHERE id = $3", old_status.step.to_string(), @@ -846,21 +855,21 @@ pub async fn update_flow_status_after_job_completion_internal( .execute(&mut *tx) .await .map_err(|e| { - Error::InternalErr(format!("error while setting new flow status: {e:#}")) + Error::internal_err(format!("error while setting new flow status: {e:#}")) })?; if let Some(job_result) = new_status.job_result() { sqlx::query!( - "UPDATE queue - SET leaf_jobs = JSONB_SET(coalesce(leaf_jobs, '{}'::jsonb), ARRAY[$1::TEXT], $2) - WHERE COALESCE((SELECT root_job FROM queue WHERE id = $3), $3) = id", + "UPDATE v2_job_status + SET flow_leaf_jobs = JSONB_SET(coalesce(flow_leaf_jobs, '{}'::jsonb), ARRAY[$1::TEXT], $2) + WHERE COALESCE((SELECT flow_innermost_root_job FROM v2_job WHERE id = $3), $3) = id", new_status.id(), json!(job_result), flow ) .execute(&mut *tx) .await.map_err(|e| { - Error::InternalErr(format!( + Error::internal_err(format!( "error while setting leaf jobs: {e:#}" )) })?; @@ -882,23 +891,20 @@ pub async fn update_flow_status_after_job_completion_internal( .as_ref() .and_then(|m| m.stop_after_all_iters_if.as_ref().map(|x| x.expr.clone())) { - let args = sqlx::query_as::<_, RowArgs>( - "SELECT - args - FROM queue - WHERE id = $2", + let args = sqlx::query_scalar!( + "SELECT args AS \"args: Json>>\" + FROM v2_job WHERE id = $1", + flow ) - .bind(old_status.step) - .bind(flow) .fetch_one(db) .await .map_err(|e| { - Error::InternalErr(format!("retrieval of args from state: {e:#}")) + Error::internal_err(format!("retrieval of args from state: {e:#}")) })?; let should_stop = compute_bool_from_expr( &expr, - Marc::new(args.args.unwrap_or_default().0), + Marc::new(args.unwrap_or_default().0), nresult.clone(), None, None, @@ -927,27 +933,26 @@ pub async fn update_flow_status_after_job_completion_internal( if old_status.retry.fail_count > 0 && matches!(&new_status, Some(FlowStatusModule::Success { .. })) { - sqlx::query( - "UPDATE queue + sqlx::query!( + "UPDATE v2_job_status SET flow_status = flow_status - 'retry' - WHERE id = $1 - RETURNING flow_status", + WHERE id = $1", + flow ) - .bind(flow) .execute(&mut *tx) .await .context("remove flow status retry")?; } let flow_job = sqlx::query_as::<_, QueuedJob>( - "SELECT * FROM queue WHERE id = $1 AND workspace_id = $2", + "SELECT * FROM v2_as_queue WHERE id = $1 AND workspace_id = $2", ) .bind(flow) .bind(w_id) .fetch_optional(&mut *tx) .await .map_err(Into::::into)? - .ok_or_else(|| Error::InternalErr(format!("requiring flow to be in the queue")))?; + .ok_or_else(|| Error::internal_err(format!("requiring flow to be in the queue")))?; tx.commit().await?; let job_root = flow_job @@ -1036,15 +1041,22 @@ pub async fn update_flow_status_after_job_completion_internal( _cleanup_module.flow_jobs_to_clean ); sqlx::query!( - "UPDATE completed_job - SET logs = '##DELETED##', args = '{}'::jsonb, result = '{}'::jsonb - WHERE id = ANY($1)", + "UPDATE v2_job SET args = '{}'::jsonb WHERE id = ANY($1)", &_cleanup_module.flow_jobs_to_clean, ) .execute(db) .await .map_err(|e| { - Error::InternalErr(format!("error while cleaning up completed_job: {e:#}")) + Error::InternalErr(format!("error while cleaning up completed job: {e:#}")) + })?; + sqlx::query!( + "UPDATE v2_job_completed SET result = '{}'::jsonb WHERE id = ANY($1)", + &_cleanup_module.flow_jobs_to_clean, + ) + .execute(db) + .await + .map_err(|e| { + Error::internal_err(format!("error while cleaning up completed job: {e:#}")) })?; } } @@ -1086,6 +1098,7 @@ pub async fn update_flow_status_after_job_completion_internal( success, stop_early && skip_if_stop_early, Json(&nresult), + None, 0, None, true, @@ -1103,6 +1116,7 @@ pub async fn update_flow_status_after_job_completion_internal( |e| json!({"error": format!("Impossible to serialize error: {e:#}")}), ), ), + None, 0, None, true, @@ -1123,6 +1137,7 @@ pub async fn update_flow_status_after_job_completion_internal( same_worker_tx.clone(), worker_dir, job_completed_tx, + worker_name, ) .warn_after_seconds(10) .await @@ -1192,18 +1207,23 @@ async fn set_success_in_flow_job_success<'c>( let position = find_flow_job_index(flow_jobs, job_id_for_status); if let Some(position) = position { sqlx::query!( - "UPDATE queue SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4) WHERE id = $2", - old_status.step as i32, - flow, - position as i32, - json!(success) - ) - .execute(&mut **tx) - .await.map_err(|e| { - Error::InternalErr(format!( - "error while setting flow_jobs_success: {e:#}" - )) - })?; + "UPDATE v2_job_status SET + flow_status = JSONB_SET( + flow_status, + ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], + $4 + ) + WHERE id = $2", + old_status.step as i32, + flow, + position as i32, + json!(success) + ) + .execute(&mut **tx) + .await + .map_err(|e| { + Error::internal_err(format!("error while setting flow_jobs_success: {e:#}")) + })?; } } Ok(()) @@ -1214,13 +1234,13 @@ async fn retrieve_flow_jobs_results( w_id: &str, job_uuids: &Vec, ) -> error::Result> { - let results = sqlx::query_as::<_, BranchResults>( + let results = sqlx::query!( "SELECT result, id - FROM completed_job + FROM v2_job_completed WHERE id = ANY($1) AND workspace_id = $2", + job_uuids.as_slice(), + w_id ) - .bind(job_uuids.as_slice()) - .bind(w_id) .fetch_all(db) .await? .into_iter() @@ -1232,7 +1252,7 @@ async fn retrieve_flow_jobs_results( .map(|j| { results .get(j) - .ok_or_else(|| Error::InternalErr(format!("missing job result for {}", j))) + .ok_or_else(|| Error::internal_err(format!("missing job result for {}", j))) }) .collect::, _>>()?; @@ -1248,23 +1268,23 @@ async fn compute_skip_branchall_failure<'c>( flow_module: Option<&FlowModule>, ) -> Result, Error> { let branch = if parallel { - sqlx::query_scalar!("SELECT script_path FROM completed_job WHERE id = $1", job) + sqlx::query_scalar!("SELECT runnable_path FROM v2_job WHERE id = $1", job) .fetch_one(db) .await .map_err(|e| { - Error::InternalErr(format!("error during retrieval of branchall index: {e:#}")) + Error::internal_err(format!("error during retrieval of branchall index: {e:#}")) })? .map(|p| { BRANCHALL_INDEX_RE .captures(&p) .map(|x| x.get(1).unwrap().as_str().parse::().ok()) .flatten() - .ok_or(Error::InternalErr(format!( + .ok_or(Error::internal_err(format!( "could not parse branchall index from path: {p}" ))) }) .ok_or_else(|| { - Error::InternalErr(format!("no branchall script path found for job {job}")) + Error::internal_err(format!("no branchall script path found for job {job}")) })?? } else { branch as i32 @@ -1288,12 +1308,12 @@ async fn compute_skip_branchall_failure<'c>( // ) // .fetch_one(db) // .await -// .map_err(|e| Error::InternalErr(format!("error during retrieval of cleanup module: {e:#}")))?; +// .map_err(|e| Error::internal_err(format!("error during retrieval of cleanup module: {e:#}")))?; // raw_value // .clone() // .and_then(|rv| serde_json::from_value::(rv).ok()) -// .ok_or(Error::InternalErr(format!( +// .ok_or(Error::internal_err(format!( // "Unable to parse flow cleanup module {:?}", // raw_value // ))) @@ -1350,48 +1370,55 @@ async fn compute_bool_from_expr( pub async fn update_flow_status_in_progress( db: &DB, - w_id: &str, + _w_id: &str, flow: Uuid, job_in_progress: Uuid, ) -> error::Result { let step = get_step_of_flow_status(db, flow).await?; match step { Step::Step(step) => { - sqlx::query(&format!( - "UPDATE queue - SET flow_status = jsonb_set(jsonb_set(flow_status, '{{modules, {step}, job}}', $1), '{{modules, {step}, type}}', $2) - WHERE id = $3 AND workspace_id = $4", - )) - .bind(json!(job_in_progress.to_string())) - .bind(json!("InProgress")) - .bind(flow) - .bind(w_id) + sqlx::query!( + "UPDATE v2_job_status SET + flow_status = jsonb_set( + jsonb_set(flow_status, ARRAY['modules', $3::INTEGER::TEXT, 'job'], to_jsonb($1::UUID::TEXT)), + ARRAY['modules', $3::INTEGER::TEXT, 'type'], + to_jsonb('InProgress'::text) + ) + WHERE id = $2", + job_in_progress, + flow, + step as i32 + ) .execute(db) .await?; } Step::PreprocessorStep => { - sqlx::query(&format!( - "UPDATE queue - SET flow_status = jsonb_set(jsonb_set(flow_status, '{{preprocessor_module, job}}', $1), '{{preprocessor_module, type}}', $2) - WHERE id = $3 AND workspace_id = $4", - )) - .bind(json!(job_in_progress.to_string())) - .bind(json!("InProgress")) - .bind(flow) - .bind(w_id) + sqlx::query!( + "UPDATE v2_job_status SET + flow_status = jsonb_set( + jsonb_set(flow_status, ARRAY['preprocessor_module', 'job'], to_jsonb($1::UUID::TEXT)), + ARRAY['preprocessor_module', 'type'], + to_jsonb('InProgress'::text) + ) + WHERE id = $2", + job_in_progress, + flow + ) .execute(db) .await?; } Step::FailureStep => { - sqlx::query(&format!( - "UPDATE queue - SET flow_status = jsonb_set(jsonb_set(flow_status, '{{failure_module, job}}', $1), '{{failure_module, type}}', $2) - WHERE id = $3 AND workspace_id = $4", - )) - .bind(json!(job_in_progress.to_string())) - .bind(json!("InProgress")) - .bind(flow) - .bind(w_id) + sqlx::query!( + "UPDATE v2_job_status SET + flow_status = jsonb_set( + jsonb_set(flow_status, ARRAY['failure_module', 'job'], to_jsonb($1::UUID::TEXT)), + ARRAY['failure_module', 'type'], + to_jsonb('InProgress'::text) + ) + WHERE id = $2", + job_in_progress, + flow + ) .execute(db) .await?; } @@ -1400,7 +1427,7 @@ pub async fn update_flow_status_in_progress( Ok(step) } -#[derive(Debug)] +#[derive(Debug, Copy, Clone)] pub enum Step { Step(usize), PreprocessorStep, @@ -1422,17 +1449,18 @@ impl Step { #[instrument(level = "trace", skip_all)] pub async fn get_step_of_flow_status(db: &DB, id: Uuid) -> error::Result { let r = sqlx::query!( - "SELECT (flow_status->'step')::integer as step, jsonb_array_length(flow_status->'modules') as len FROM queue WHERE id = $1", + "SELECT (flow_status->'step')::integer as step, jsonb_array_length(flow_status->'modules') as len + FROM v2_job_status WHERE id = $1", id ) .fetch_one(db) .await - .map_err(|e| Error::InternalErr(format!("fetching step flow status: {e:#}")))?; + .map_err(|e| Error::internal_err(format!("fetching step flow status: {e:#}")))?; if let Some(step) = r.step { Ok(Step::from_i32_and_len(step, r.len.unwrap_or(0) as usize)) } else { - Err(Error::InternalErr("step is null".to_string())) + Err(Error::internal_err("step is null".to_string())) } } @@ -1501,6 +1529,7 @@ pub async fn handle_flow( same_worker_tx: SameWorkerSender, worker_dir: &str, job_completed_tx: Sender, + worker_name: &str, ) -> anyhow::Result<()> { let flow = flow_data.value(); let status = flow_job @@ -1554,6 +1583,7 @@ pub async fn handle_flow( same_worker_tx, worker_dir, job_completed_tx, + worker_name, ) .warn_after_seconds(10) .await?; @@ -1574,11 +1604,6 @@ pub struct ResumeRow { pub resume_id: i32, } -#[derive(FromRow)] -pub struct RawArgs { - pub args: Option>>>, -} - lazy_static::lazy_static! { static ref CRASH_FORCEFULLY_AT_STEP: Option = std::env::var("CRASH_FORCEFULLY_AT_STEP") .ok() @@ -1616,6 +1641,7 @@ async fn push_next_flow_job( same_worker_tx: SameWorkerSender, worker_dir: &str, job_completed_tx: Sender, + worker_name: &str, ) -> error::Result<()> { let job_root = flow_job .root_job @@ -1666,7 +1692,7 @@ async fn push_next_flow_job( }) .await .map_err(|e| { - Error::InternalErr(format!( + Error::internal_err(format!( "error sending update flow message to job completed channel: {e:#}" )) })?; @@ -1685,7 +1711,7 @@ async fn push_next_flow_job( .await?; if no_flow_overlap { let overlapping = sqlx::query_scalar!( - "SELECT id FROM queue WHERE schedule_path = $1 AND workspace_id = $2 AND id != $3 AND running = true", + "SELECT id AS \"id!\" FROM v2_as_queue WHERE schedule_path = $1 AND workspace_id = $2 AND id != $3 AND running = true", flow_job.schedule_path.as_ref().unwrap(), flow_job.workspace_id.as_str(), flow_job.id @@ -1711,7 +1737,7 @@ async fn push_next_flow_job( }) .await .map_err(|e| { - Error::InternalErr(format!( + Error::internal_err(format!( "error sending update flow message to job completed channel: {e:#}" )) })?; @@ -1748,7 +1774,7 @@ async fn push_next_flow_job( }) .await .map_err(|e| { - Error::InternalErr(format!( + Error::internal_err(format!( "error sending update flow message to job completed channel: {e:#}" )) })?; @@ -1798,7 +1824,7 @@ async fn push_next_flow_job( * * This only works because jobs::resume_job does the same thing. */ sqlx::query_scalar!( - "SELECT null FROM queue WHERE id = $1 FOR UPDATE", + "SELECT null FROM v2_job_queue WHERE id = $1 FOR UPDATE", flow_job.id ) .fetch_one(&mut *tx) @@ -1870,17 +1896,17 @@ async fn push_next_flow_job( } } let approval_conditions = ApprovalConditions { - user_auth_required: user_auth_required, - user_groups_required: user_groups_required, - self_approval_disabled: self_approval_disabled, + user_auth_required, + user_groups_required, + self_approval_disabled, }; - sqlx::query( - "UPDATE queue + sqlx::query!( + "UPDATE v2_job_status SET flow_status = JSONB_SET(flow_status, ARRAY['approval_conditions'], $1) WHERE id = $2", + json!(approval_conditions), + flow_job.id ) - .bind(json!(approval_conditions)) - .bind(flow_job.id) .execute(&mut *tx) .await?; } @@ -1897,7 +1923,10 @@ async fn push_next_flow_job( && suspend.continue_on_disapprove_timeout.unwrap_or(false); let audit_author = AuditAuthor { - username: flow_job.permissioned_as.trim_start_matches("u/").to_string(), + username: flow_job + .permissioned_as + .trim_start_matches("u/") + .to_string(), email: flow_job.email.clone(), username_override: None, }; @@ -1923,32 +1952,33 @@ async fn push_next_flow_job( .await?; } - sqlx::query( - "UPDATE queue + sqlx::query!( + "UPDATE v2_job_status SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'approvers'], $2) WHERE id = $3", + (status.step - 1).to_string(), + json!(resumes + .into_iter() + .map(|r| Approval { + resume_id: r.resume_id as u16, + approver: r + .approver.clone() + .unwrap_or_else(|| "unknown".to_string()) + }) + .collect::>() + ), + flow_job.id ) - .bind(status.step - 1) - .bind(json!(resumes - .into_iter() - .map(|r| Approval { - resume_id: r.resume_id as u16, - approver: r - .approver.clone() - .unwrap_or_else(|| "unknown".to_string()) - }) - .collect::>())) - .bind(flow_job.id) .execute(&mut *tx) .await?; // Remove the approval conditions from the flow status - sqlx::query( - "UPDATE queue + sqlx::query!( + "UPDATE v2_job_status SET flow_status = flow_status - 'approval_conditions' WHERE id = $1", + flow_job.id ) - .bind(flow_job.id) .execute(&mut *tx) .await?; @@ -1961,24 +1991,33 @@ async fn push_next_flow_job( FlowStatusModule::WaitingForPriorSteps { .. } ) && is_disapproved.is_none() { - sqlx::query( - "UPDATE queue SET - flow_status = JSONB_SET(flow_status, ARRAY['modules', flow_status->>'step'::text], $1), - suspend = $2, - suspend_until = now() + $3 - WHERE id = $4", + sqlx::query!( + "WITH suspend AS ( + UPDATE v2_job_queue SET suspend = $2, suspend_until = now() + $3 + WHERE id = $4 + RETURNING id + ) UPDATE v2_job_status SET flow_status = JSONB_SET( + flow_status, + ARRAY['modules', flow_status->>'step'::TEXT], + $1 + ) WHERE id = (SELECT id FROM suspend)", + json!(FlowStatusModule::WaitingForEvents { + id: status_module.id(), + count: required_events, + job: last + }), + (required_events - resume_messages.len() as u16) as i32, + Duration::from_secs( + suspend.timeout.map(|t| t.into()).unwrap_or_else(|| 30 * 60) + ) as Duration, + flow_job.id, ) - .bind(json!(FlowStatusModule::WaitingForEvents { id: status_module.id(), count: required_events, job: last })) - .bind((required_events - resume_messages.len() as u16) as i32) - .bind(Duration::from_secs(suspend.timeout.map(|t| t.into()).unwrap_or_else(|| 30 * 60))) - .bind(flow_job.id) .execute(&mut *tx) .await?; sqlx::query!( - "UPDATE queue - SET last_ping = null - WHERE id = $1 AND last_ping = $2", + "UPDATE v2_job_runtime SET ping = NULL + WHERE id = $1 AND ping = $2", flow_job.id, flow_job.last_ping ) @@ -2038,7 +2077,7 @@ async fn push_next_flow_job( }) .await .map_err(|e| { - Error::InternalErr(format!( + Error::internal_err(format!( "error sending update flow message to job completed channel: {e:#}" )) })?; @@ -2159,15 +2198,15 @@ async fn push_next_flow_job( scheduled_for_o = Some(from_now(retry_in)); status.retry.failed_jobs.push(job.clone()); - sqlx::query( - "UPDATE queue + sqlx::query!( + "UPDATE v2_job_status SET flow_status = JSONB_SET(JSONB_SET(flow_status, ARRAY['retry'], $1), ARRAY['modules', $3::TEXT, 'failed_retries'], $4) WHERE id = $2", + json!(RetryStatus { fail_count, ..status.retry.clone() }), + flow_job.id, + status.step.to_string(), + json!(status.retry.failed_jobs) ) - .bind(json!(RetryStatus { fail_count, ..status.retry.clone() })) - .bind(flow_job.id) - .bind(status.step) - .bind(json!(status.retry.failed_jobs)) .execute(db) .warn_after_seconds(2) .await @@ -2196,13 +2235,13 @@ async fn push_next_flow_job( status_module = status.failure_module.module_status.clone(); if module.retry.as_ref().is_some_and(|x| x.has_attempts()) { - sqlx::query( - "UPDATE queue + sqlx::query!( + "UPDATE v2_job_status SET flow_status = JSONB_SET(flow_status, ARRAY['retry'], $1) WHERE id = $2", + json!(RetryStatus { fail_count: 0, failed_jobs: vec![] }), + flow_job.id ) - .bind(json!(RetryStatus { fail_count: 0, failed_jobs: vec![] })) - .bind(flow_job.id) .execute(db) .await .context("update flow retry")?; @@ -2255,17 +2294,16 @@ async fn push_next_flow_job( ); Ok(Marc::new(hm)) } else if let Some(id) = get_args_from_id { - let row = sqlx::query_as::<_, RawArgs>( - "SELECT args FROM completed_job WHERE id = $1 AND workspace_id = $2", + let args = sqlx::query_scalar!( + "SELECT args AS \"args: Json>>\" + FROM v2_job WHERE id = $1 AND workspace_id = $2", + id, + &flow_job.workspace_id ) - .bind(id) - .bind(&flow_job.workspace_id) .fetch_optional(db) .await?; - if let Some(raw_args) = row { - Ok(Marc::new( - raw_args.args.map(|x| x.0).unwrap_or_else(HashMap::new), - )) + if let Some(args) = args { + Ok(Marc::new(args.map(|x| x.0).unwrap_or_else(HashMap::new))) } else { Ok(Marc::new(HashMap::new())) } @@ -2287,7 +2325,7 @@ async fn push_next_flow_job( .unwrap(), ) .map(Marc::new) - .map_err(|e| error::Error::InternalErr(format!("identity: {e:#}"))) + .map_err(|e| error::Error::internal_err(format!("identity: {e:#}"))) } Ok( FlowModuleValue::Script { input_transforms, .. } @@ -2313,7 +2351,7 @@ async fn push_next_flow_job( } Ok(_) => Ok(arc_flow_job_args.clone()), Err(e) => { - return Err(error::Error::InternalErr(format!( + return Err(error::Error::internal_err(format!( "module was not convertible to acceptable value {e:?}" ))) } @@ -2344,23 +2382,23 @@ async fn push_next_flow_job( let (job_payloads, next_status) = match next_flow_transform { NextFlowTransform::Continue(job_payload, next_state) => (job_payload, next_state), NextFlowTransform::EmptyInnerFlows => { - sqlx::query( - "UPDATE queue + sqlx::query!( + "UPDATE v2_job_status SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT], $2) WHERE id = $3", + status.step.to_string(), + json!(FlowStatusModule::Success { + id: status_module.id(), + job: Uuid::nil(), + flow_jobs: Some(vec![]), + flow_jobs_success: Some(vec![]), + branch_chosen: None, + approvers: vec![], + failed_retries: vec![], + skipped: false, + }), + flow_job.id ) - .bind(status.step) - .bind(json!(FlowStatusModule::Success { - id: status_module.id(), - job: Uuid::nil(), - flow_jobs: Some(vec![]), - flow_jobs_success: Some(vec![]), - branch_chosen: None, - approvers: vec![], - failed_retries: vec![], - skipped: false, - })) - .bind(flow_job.id) .execute(db) .await?; // flow is reprocessed by the worker in a state where the module has completed successfully. @@ -2396,9 +2434,7 @@ async fn push_next_flow_job( if i % 100 == 0 && i != 0 { tracing::info!(id = %flow_job.id, root_id = %job_root, "pushed (non-commited yet) first {i} subflows of {len}"); sqlx::query!( - "UPDATE queue - SET last_ping = now() - WHERE id = $1 AND last_ping < now()", + "UPDATE v2_job_runtime SET ping = now() WHERE id = $1 AND ping < now()", flow_job.id, ) .execute(db) @@ -2472,7 +2508,7 @@ async fn push_next_flow_job( } NextStatus::AllFlowJobs { branchall: None, - iterator: Some(Iterator { itered, .. }), + iterator: Some(FlowIterator { itered, .. }), simple_input_transforms, } => { if let Ok(args) = args.as_ref() { @@ -2614,6 +2650,16 @@ async fn push_next_flow_job( .warn_after_seconds(2) .await?; + if continue_on_same_worker { + let _ = sqlx::query!( + "UPDATE v2_job_queue SET worker = $2 WHERE id = $1", + uuid, + worker_name + ) + .execute(&mut *inner_tx) + .await; + } + tracing::debug!(id = %flow_job.id, root_id = %job_root, "pushed next flow job: {uuid}"); if value_with_parallel.type_ == "forloopflow" { @@ -2622,9 +2668,11 @@ async fn push_next_flow_job( if i as u16 >= p { sqlx::query!( - "UPDATE queue - SET suspend = $1, suspend_until = now() + interval '14 day', running = true - WHERE id = $2", + "UPDATE v2_job_queue SET + suspend = $1, + suspend_until = now() + interval '14 day', + running = true + WHERE id = $2", (i as u16 - p + 1) as i32, uuid, ) @@ -2637,16 +2685,16 @@ async fn push_next_flow_job( if payload_tag.delete_after_use { let uuid_singleton_json = serde_json::to_value(&[uuid]).map_err(|e| { - error::Error::InternalErr(format!("Unable to serialize uuid: {e:#}")) + error::Error::internal_err(format!("Unable to serialize uuid: {e:#}")) })?; - sqlx::query( - "UPDATE queue + sqlx::query!( + "UPDATE v2_job_status SET flow_status = JSONB_SET(flow_status, ARRAY['cleanup_module', 'flow_jobs_to_clean'], COALESCE(flow_status->'cleanup_module'->'flow_jobs_to_clean', '[]'::jsonb) || $1) WHERE id = $2", + uuid_singleton_json, + root_job.unwrap_or(flow_job.id) ) - .bind(uuid_singleton_json) - .bind(root_job.unwrap_or(flow_job.id)) .execute(&mut *inner_tx) .await?; } @@ -2700,7 +2748,7 @@ async fn push_next_flow_job( } FlowStatusModule::InProgress { job: uuid, - iterator: Some(windmill_common::flow_status::Iterator { index, itered }), + iterator: Some(FlowIterator { index, itered }), flow_jobs: Some(flow_jobs), flow_jobs_success, branch_chosen: None, @@ -2770,9 +2818,12 @@ async fn push_next_flow_job( match step { Step::FailureStep => { sqlx::query!( - "UPDATE queue - SET flow_status = JSONB_SET( - JSONB_SET(flow_status, ARRAY['failure_module'], $1), ARRAY['step'], $2) + "UPDATE v2_job_status SET + flow_status = JSONB_SET( + JSONB_SET(flow_status, ARRAY['failure_module'], $1), + ARRAY['step'], + $2 + ) WHERE id = $3", json!(FlowStatusModuleWParent { parent_module: Some(current_id.clone()), @@ -2786,9 +2837,12 @@ async fn push_next_flow_job( } Step::PreprocessorStep => { sqlx::query!( - "UPDATE queue - SET flow_status = JSONB_SET( - JSONB_SET(flow_status, ARRAY['preprocessor_module'], $1), ARRAY['step'], $2) + "UPDATE v2_job_status SET + flow_status = JSONB_SET( + JSONB_SET(flow_status, ARRAY['preprocessor_module'], $1), + ARRAY['step'], + $2 + ) WHERE id = $3", json!(new_status), json!(-1), @@ -2799,9 +2853,12 @@ async fn push_next_flow_job( } Step::Step(i) => { sqlx::query!( - "UPDATE queue - SET flow_status = JSONB_SET( - JSONB_SET(flow_status, ARRAY['modules', $1::TEXT], $2), ARRAY['step'], $3) + "UPDATE v2_job_status SET + flow_status = JSONB_SET( + JSONB_SET(flow_status, ARRAY['modules', $1::TEXT], $2), + ARRAY['step'], + $3 + ) WHERE id = $4", i as i32, json!(new_status), @@ -2816,9 +2873,7 @@ async fn push_next_flow_job( potentially_crash_for_testing(); sqlx::query!( - "UPDATE queue - SET last_ping = null - WHERE id = $1", + "UPDATE v2_job_runtime SET ping = null WHERE id = $1", flow_job.id ) .execute(&mut *tx) @@ -2941,7 +2996,7 @@ enum NextStatus { }, AllFlowJobs { branchall: Option, - iterator: Option, + iterator: Option, simple_input_transforms: Option>, }, } @@ -3022,7 +3077,7 @@ fn get_path(flow_job: &QueuedJob, status: &FlowStatus, module: &FlowModule) -> S { format!("{}/preprocessor", flow_job.script_path()) } else { - format!("{}/step-{}", flow_job.script_path(), status.step) + format!("{}/{}", flow_job.script_path(), module.id) } } @@ -3284,10 +3339,7 @@ async fn compute_next_flow_transform( ContinuePayload::ParallelJobs(payloads), NextStatus::AllFlowJobs { branchall: None, - iterator: Some(windmill_common::flow_status::Iterator { - index: 0, - itered, - }), + iterator: Some(FlowIterator { index: 0, itered }), // we removed the is_simple_case for simple_input_transforms // if is_simple { // match value { @@ -3634,7 +3686,7 @@ async fn next_forloop_status( } FlowStatusModule::InProgress { - iterator: Some(windmill_common::flow_status::Iterator { itered, index }), + iterator: Some(FlowIterator { itered, index }), flow_jobs: Some(flow_jobs), flow_jobs_success, .. @@ -3978,11 +4030,12 @@ async fn get_previous_job_result( Ok(Some(retrieve_flow_jobs_results(db, w_id, flow_jobs).await?)) } Some(FlowStatusModule::Success { job, .. }) => Ok(Some( - sqlx::query_scalar::<_, Json>>( - "SELECT result FROM completed_job WHERE id = $1 AND workspace_id = $2", + sqlx::query_scalar!( + "SELECT result AS \"result!: Json>\" + FROM v2_job_completed WHERE id = $1 AND workspace_id = $2", + job, + w_id ) - .bind(job) - .bind(w_id) .fetch_one(db) .await? .0, diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index d4df729910..03ba19d2ad 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -265,13 +265,13 @@ pub async fn handle_dependency_job( Some(hash) => &cache::script::fetch(db, hash).await?.0, _ => match preview_data { Some(RawData::Script(data)) => data, - _ => return Err(Error::InternalErr("expected script hash".into())), + _ => return Err(Error::internal_err("expected script hash")), }, }; let content = capture_dependency_job( &job.id, job.language.as_ref().map(|v| Ok(v)).unwrap_or_else(|| { - Err(Error::InternalErr( + Err(Error::internal_err( "Job Language required for dependency jobs".to_owned(), )) })?, @@ -551,7 +551,7 @@ pub async fn handle_flow_dependency_job( occupancy_metrics: &mut OccupancyMetrics, ) -> error::Result> { let job_path = job.script_path.clone().ok_or_else(|| { - error::Error::InternalErr( + error::Error::internal_err( "Cannot resolve flow dependencies for flow without path".to_string(), ) })?; @@ -574,7 +574,7 @@ pub async fn handle_flow_dependency_job( job.script_hash .clone() .ok_or_else(|| { - Error::InternalErr( + Error::internal_err( "Flow Dependency requires script hash (flow version)".to_owned(), ) })? @@ -602,7 +602,7 @@ pub async fn handle_flow_dependency_job( Some(ScriptHash(id)) => cache::flow::fetch_version(db, id).await?, _ => match preview_data { Some(RawData::Flow(data)) => data.clone(), - _ => return Err(Error::InternalErr("expected script hash".into())), + _ => return Err(Error::internal_err("expected script hash")), }, } .value() @@ -633,15 +633,17 @@ pub async fn handle_flow_dependency_job( let new_flow_value = Json(serde_json::value::to_raw_value(&flow).map_err(to_anyhow)?); // Re-check cancellation to ensure we don't accidentally override a flow. - if sqlx::query_scalar!("SELECT canceled FROM queue WHERE id = $1", job.id) - .fetch_optional(db) - .await - .map(|v| Some(true) == v) - .unwrap_or_else(|err| { - tracing::error!(%job.id, %err, "error checking cancellation for job {0}: {err}", job.id); - false - }) - { + if sqlx::query_scalar!( + "SELECT canceled_by IS NOT NULL AS \"canceled!\" FROM v2_job_queue WHERE id = $1", + job.id + ) + .fetch_optional(db) + .await + .map(|v| Some(true) == v) + .unwrap_or_else(|err| { + tracing::error!(%job.id, %err, "error checking cancellation for job {0}: {err}", job.id); + false + }) { return Ok(to_raw_value_owned(json!({ "status": "Flow lock generation was canceled", }))); @@ -649,7 +651,7 @@ pub async fn handle_flow_dependency_job( if !skip_flow_update { let version = version.ok_or_else(|| { - Error::InternalErr("Flow Dependency requires script hash (flow version)".to_owned()) + Error::internal_err("Flow Dependency requires script hash (flow version)".to_owned()) })?; sqlx::query!( @@ -1128,7 +1130,6 @@ async fn insert_flow_modules<'c>( same_worker, )) .await?; - add_virtual_items_if_necessary(modules); if modules.is_empty() || crate::worker_flow::is_simple_modules(modules, failure_module) { return Ok(tx); } @@ -1163,7 +1164,7 @@ async fn reduce_flow<'c>( for module in &mut *modules { let mut val = serde_json::from_str::(module.value.get()).map_err(|err| { - Error::InternalErr(format!( + Error::internal_err(format!( "reduce_flow: Failed to parse flow module value: {}", err )) @@ -1257,6 +1258,7 @@ async fn reduce_flow<'c>( } module.value = to_raw_value(&val); } + add_virtual_items_if_necessary(&mut *modules); Ok(tx) } @@ -1274,7 +1276,7 @@ async fn reduce_app(db: &sqlx::Pool, value: &mut Value, app: i64 // replace `content` with an empty string: let Some(Value::String(code)) = script.get_mut("content").map(std::mem::take) else { - return Err(error::Error::InternalErr( + return Err(error::Error::internal_err( "Missing `content` in inlineScript".to_string(), )); }; @@ -1477,7 +1479,7 @@ pub async fn handle_app_dependency_job( occupancy_metrics: &mut OccupancyMetrics, ) -> error::Result<()> { let job_path = job.script_path.clone().ok_or_else(|| { - error::Error::InternalErr( + error::Error::internal_err( "Cannot resolve app dependencies for app without path".to_string(), ) })?; @@ -1485,7 +1487,7 @@ pub async fn handle_app_dependency_job( let id = job .script_hash .clone() - .ok_or_else(|| Error::InternalErr("App Dependency requires script hash".to_owned()))? + .ok_or_else(|| Error::internal_err("App Dependency requires script hash".to_owned()))? .0; let record = sqlx::query!("SELECT app_id, value FROM app_version WHERE id = $1", id) .fetch_optional(db) @@ -1525,15 +1527,17 @@ pub async fn handle_app_dependency_job( .await?; // Re-check cancelation to ensure we don't accidentially override an app. - if sqlx::query_scalar!("SELECT canceled FROM queue WHERE id = $1", job.id) - .fetch_optional(db) - .await - .map(|v| Some(true) == v) - .unwrap_or_else(|err| { - tracing::error!(%job.id, %err, "error checking cancelation for job {0}: {err}", job.id); - false - }) - { + if sqlx::query_scalar!( + "SELECT canceled_by IS NOT NULL AS \"canceled!\" FROM v2_job_queue WHERE id = $1", + job.id + ) + .fetch_optional(db) + .await + .map(|v| Some(true) == v) + .unwrap_or_else(|err| { + tracing::error!(%job.id, %err, "error checking cancelation for job {0}: {err}", job.id); + false + }) { return Ok(()); } @@ -1572,7 +1576,7 @@ pub async fn handle_app_dependency_job( // match tx { // PushIsolationLevel::Transaction(tx) => tx.commit().await?, // _ => { - // return Err(Error::InternalErr( + // return Err(Error::internal_err( // "Expected a transaction here".to_string(), // )); // } @@ -1682,7 +1686,7 @@ async fn capture_dependency_job( match job_language { ScriptLang::Python3 => { #[cfg(not(feature = "python"))] - return Err(Error::InternalErr( + return Err(Error::internal_err( "Python requires the python feature to be enabled".to_string(), )); #[cfg(feature = "python")] @@ -1749,7 +1753,7 @@ async fn capture_dependency_job( } ScriptLang::Ansible => { #[cfg(not(feature = "python"))] - return Err(Error::InternalErr( + return Err(Error::internal_err( "Ansible requires the python feature to be enabled".to_string(), )); @@ -1887,7 +1891,7 @@ async fn capture_dependency_job( } ScriptLang::Php => { #[cfg(not(feature = "php"))] - return Err(Error::InternalErr( + return Err(Error::internal_err( "PHP requires the php feature to be enabled".to_string(), )); @@ -1929,7 +1933,7 @@ async fn capture_dependency_job( } #[cfg(not(feature = "rust"))] - return Err(Error::InternalErr( + return Err(Error::internal_err( "Rust requires the rust feature to be enabled".to_string(), )); diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 47f2684855..7221d958bb 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.456.0"; +export const VERSION = "v1.458.4"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/conf.ts b/cli/conf.ts index 2bb3a69b9b..a8598d5d0b 100644 --- a/cli/conf.ts +++ b/cli/conf.ts @@ -12,6 +12,7 @@ export interface SyncOptions { skipResources?: boolean; skipSecrets?: boolean; includeSchedules?: boolean; + includeTriggers?: boolean; includeUsers?: boolean; includeGroups?: boolean; includeSettings?: boolean; diff --git a/cli/gen/core/OpenAPI.ts b/cli/gen/core/OpenAPI.ts index f9debf88b1..4488a855bf 100644 --- a/cli/gen/core/OpenAPI.ts +++ b/cli/gen/core/OpenAPI.ts @@ -54,7 +54,7 @@ export const OpenAPI: OpenAPIConfig = { PASSWORD: undefined, TOKEN: getEnv("WM_TOKEN"), USERNAME: undefined, - VERSION: '1.447.5', + VERSION: '1.457.1', WITH_CREDENTIALS: true, interceptors: { request: new Interceptors(), diff --git a/cli/gen/services.gen.ts b/cli/gen/services.gen.ts index 49b5e1482c..4992329350 100644 --- a/cli/gen/services.gen.ts +++ b/cli/gen/services.gen.ts @@ -3,7 +3,7 @@ import type { CancelablePromise } from './core/CancelablePromise.ts'; import { OpenAPI } from './core/OpenAPI.ts'; import { request as __request } from './core/request.ts'; -import type { BackendVersionResponse, BackendUptodateResponse, GetLicenseIdResponse, GetOpenApiYamlResponse, GetAuditLogData, GetAuditLogResponse, ListAuditLogsData, ListAuditLogsResponse, LoginData, LoginResponse, LogoutResponse, GetUserData, GetUserResponse, UpdateUserData, UpdateUserResponse, IsOwnerOfPathData, IsOwnerOfPathResponse, SetPasswordData, SetPasswordResponse, SetPasswordForUserData, SetPasswordForUserResponse, SetLoginTypeForUserData, SetLoginTypeForUserResponse, CreateUserGloballyData, CreateUserGloballyResponse, GlobalUserUpdateData, GlobalUserUpdateResponse, GlobalUsernameInfoData, GlobalUsernameInfoResponse, GlobalUserRenameData, GlobalUserRenameResponse, GlobalUserDeleteData, GlobalUserDeleteResponse, GlobalUsersOverwriteData, GlobalUsersOverwriteResponse, GlobalUsersExportResponse, DeleteUserData, DeleteUserResponse, ListWorkspacesResponse, IsDomainAllowedResponse, ListUserWorkspacesResponse, ListWorkspacesAsSuperAdminData, ListWorkspacesAsSuperAdminResponse, CreateWorkspaceData, CreateWorkspaceResponse, ExistsWorkspaceData, ExistsWorkspaceResponse, ExistsUsernameData, ExistsUsernameResponse, GetGlobalData, GetGlobalResponse, SetGlobalData, SetGlobalResponse, GetLocalResponse, TestSmtpData, TestSmtpResponse, TestCriticalChannelsData, TestCriticalChannelsResponse, GetCriticalAlertsData, GetCriticalAlertsResponse, AcknowledgeCriticalAlertData, AcknowledgeCriticalAlertResponse, AcknowledgeAllCriticalAlertsResponse, TestLicenseKeyData, TestLicenseKeyResponse, TestObjectStorageConfigData, TestObjectStorageConfigResponse, SendStatsResponse, GetLatestKeyRenewalAttemptResponse, RenewLicenseKeyData, RenewLicenseKeyResponse, CreateCustomerPortalSessionData, CreateCustomerPortalSessionResponse, TestMetadataData, TestMetadataResponse, ListGlobalSettingsResponse, GetCurrentEmailResponse, RefreshUserTokenData, RefreshUserTokenResponse, GetTutorialProgressResponse, UpdateTutorialProgressData, UpdateTutorialProgressResponse, LeaveInstanceResponse, GetUsageResponse, GetRunnableResponse, GlobalWhoamiResponse, ListWorkspaceInvitesResponse, WhoamiData, WhoamiResponse, AcceptInviteData, AcceptInviteResponse, DeclineInviteData, DeclineInviteResponse, InviteUserData, InviteUserResponse, AddUserData, AddUserResponse, DeleteInviteData, DeleteInviteResponse, ArchiveWorkspaceData, ArchiveWorkspaceResponse, UnarchiveWorkspaceData, UnarchiveWorkspaceResponse, DeleteWorkspaceData, DeleteWorkspaceResponse, LeaveWorkspaceData, LeaveWorkspaceResponse, GetWorkspaceNameData, GetWorkspaceNameResponse, ChangeWorkspaceNameData, ChangeWorkspaceNameResponse, ChangeWorkspaceIdData, ChangeWorkspaceIdResponse, ChangeWorkspaceColorData, ChangeWorkspaceColorResponse, WhoisData, WhoisResponse, ExistsEmailData, ExistsEmailResponse, ListUsersAsSuperAdminData, ListUsersAsSuperAdminResponse, ListPendingInvitesData, ListPendingInvitesResponse, GetSettingsData, GetSettingsResponse, GetDeployToData, GetDeployToResponse, GetIsPremiumData, GetIsPremiumResponse, GetPremiumInfoData, GetPremiumInfoResponse, SetAutomaticBillingData, SetAutomaticBillingResponse, GetThresholdAlertData, GetThresholdAlertResponse, SetThresholdAlertData, SetThresholdAlertResponse, EditSlackCommandData, EditSlackCommandResponse, RunSlackMessageTestJobData, RunSlackMessageTestJobResponse, EditDeployToData, EditDeployToResponse, EditAutoInviteData, EditAutoInviteResponse, EditWebhookData, EditWebhookResponse, EditCopilotConfigData, EditCopilotConfigResponse, GetCopilotInfoData, GetCopilotInfoResponse, EditErrorHandlerData, EditErrorHandlerResponse, EditLargeFileStorageConfigData, EditLargeFileStorageConfigResponse, EditWorkspaceGitSyncConfigData, EditWorkspaceGitSyncConfigResponse, EditWorkspaceDeployUiSettingsData, EditWorkspaceDeployUiSettingsResponse, EditWorkspaceDefaultAppData, EditWorkspaceDefaultAppResponse, EditDefaultScriptsData, EditDefaultScriptsResponse, GetDefaultScriptsData, GetDefaultScriptsResponse, SetEnvironmentVariableData, SetEnvironmentVariableResponse, GetWorkspaceEncryptionKeyData, GetWorkspaceEncryptionKeyResponse, SetWorkspaceEncryptionKeyData, SetWorkspaceEncryptionKeyResponse, GetWorkspaceDefaultAppData, GetWorkspaceDefaultAppResponse, GetLargeFileStorageConfigData, GetLargeFileStorageConfigResponse, GetWorkspaceUsageData, GetWorkspaceUsageResponse, GetUsedTriggersData, GetUsedTriggersResponse, ListUsersData, ListUsersResponse, ListUsersUsageData, ListUsersUsageResponse, ListUsernamesData, ListUsernamesResponse, UsernameToEmailData, UsernameToEmailResponse, CreateTokenData, CreateTokenResponse, CreateTokenImpersonateData, CreateTokenImpersonateResponse, DeleteTokenData, DeleteTokenResponse, ListTokensData, ListTokensResponse, GetOidcTokenData, GetOidcTokenResponse, CreateVariableData, CreateVariableResponse, EncryptValueData, EncryptValueResponse, DeleteVariableData, DeleteVariableResponse, UpdateVariableData, UpdateVariableResponse, GetVariableData, GetVariableResponse, GetVariableValueData, GetVariableValueResponse, ExistsVariableData, ExistsVariableResponse, ListVariableData, ListVariableResponse, ListContextualVariablesData, ListContextualVariablesResponse, WorkspaceGetCriticalAlertsData, WorkspaceGetCriticalAlertsResponse, WorkspaceAcknowledgeCriticalAlertData, WorkspaceAcknowledgeCriticalAlertResponse, WorkspaceAcknowledgeAllCriticalAlertsData, WorkspaceAcknowledgeAllCriticalAlertsResponse, WorkspaceMuteCriticalAlertsUiData, WorkspaceMuteCriticalAlertsUiResponse, LoginWithOauthData, LoginWithOauthResponse, ConnectSlackCallbackData, ConnectSlackCallbackResponse, ConnectSlackCallbackInstanceData, ConnectSlackCallbackInstanceResponse, ConnectCallbackData, ConnectCallbackResponse, CreateAccountData, CreateAccountResponse, RefreshTokenData, RefreshTokenResponse, DisconnectAccountData, DisconnectAccountResponse, DisconnectSlackData, DisconnectSlackResponse, ListOauthLoginsResponse, ListOauthConnectsResponse, GetOauthConnectData, GetOauthConnectResponse, CreateResourceData, CreateResourceResponse, DeleteResourceData, DeleteResourceResponse, UpdateResourceData, UpdateResourceResponse, UpdateResourceValueData, UpdateResourceValueResponse, GetResourceData, GetResourceResponse, GetResourceValueInterpolatedData, GetResourceValueInterpolatedResponse, GetResourceValueData, GetResourceValueResponse, ExistsResourceData, ExistsResourceResponse, ListResourceData, ListResourceResponse, ListSearchResourceData, ListSearchResourceResponse, ListResourceNamesData, ListResourceNamesResponse, CreateResourceTypeData, CreateResourceTypeResponse, FileResourceTypeToFileExtMapData, FileResourceTypeToFileExtMapResponse, DeleteResourceTypeData, DeleteResourceTypeResponse, UpdateResourceTypeData, UpdateResourceTypeResponse, GetResourceTypeData, GetResourceTypeResponse, ExistsResourceTypeData, ExistsResourceTypeResponse, ListResourceTypeData, ListResourceTypeResponse, ListResourceTypeNamesData, ListResourceTypeNamesResponse, QueryResourceTypesData, QueryResourceTypesResponse, ListHubIntegrationsData, ListHubIntegrationsResponse, ListHubFlowsResponse, GetHubFlowByIdData, GetHubFlowByIdResponse, ListHubAppsResponse, GetHubAppByIdData, GetHubAppByIdResponse, GetPublicAppByCustomPathData, GetPublicAppByCustomPathResponse, GetHubScriptContentByPathData, GetHubScriptContentByPathResponse, GetHubScriptByPathData, GetHubScriptByPathResponse, GetTopHubScriptsData, GetTopHubScriptsResponse, QueryHubScriptsData, QueryHubScriptsResponse, ListSearchScriptData, ListSearchScriptResponse, ListScriptsData, ListScriptsResponse, ListScriptPathsData, ListScriptPathsResponse, CreateDraftData, CreateDraftResponse, DeleteDraftData, DeleteDraftResponse, CreateScriptData, CreateScriptResponse, ToggleWorkspaceErrorHandlerForScriptData, ToggleWorkspaceErrorHandlerForScriptResponse, GetCustomTagsData, GetCustomTagsResponse, GeDefaultTagsResponse, IsDefaultTagsPerWorkspaceResponse, ArchiveScriptByPathData, ArchiveScriptByPathResponse, ArchiveScriptByHashData, ArchiveScriptByHashResponse, DeleteScriptByHashData, DeleteScriptByHashResponse, DeleteScriptByPathData, DeleteScriptByPathResponse, GetScriptByPathData, GetScriptByPathResponse, GetTriggersCountOfScriptData, GetTriggersCountOfScriptResponse, ListTokensOfScriptData, ListTokensOfScriptResponse, GetScriptByPathWithDraftData, GetScriptByPathWithDraftResponse, GetScriptHistoryByPathData, GetScriptHistoryByPathResponse, GetScriptLatestVersionData, GetScriptLatestVersionResponse, UpdateScriptHistoryData, UpdateScriptHistoryResponse, RawScriptByPathData, RawScriptByPathResponse, RawScriptByPathTokenedData, RawScriptByPathTokenedResponse, ExistsScriptByPathData, ExistsScriptByPathResponse, GetScriptByHashData, GetScriptByHashResponse, RawScriptByHashData, RawScriptByHashResponse, GetScriptDeploymentStatusData, GetScriptDeploymentStatusResponse, RunScriptByPathData, RunScriptByPathResponse, OpenaiSyncScriptByPathData, OpenaiSyncScriptByPathResponse, RunWaitResultScriptByPathData, RunWaitResultScriptByPathResponse, RunWaitResultScriptByPathGetData, RunWaitResultScriptByPathGetResponse, OpenaiSyncFlowByPathData, OpenaiSyncFlowByPathResponse, RunWaitResultFlowByPathData, RunWaitResultFlowByPathResponse, ResultByIdData, ResultByIdResponse, ListFlowPathsData, ListFlowPathsResponse, ListSearchFlowData, ListSearchFlowResponse, ListFlowsData, ListFlowsResponse, GetFlowHistoryData, GetFlowHistoryResponse, GetFlowLatestVersionData, GetFlowLatestVersionResponse, GetFlowVersionData, GetFlowVersionResponse, UpdateFlowHistoryData, UpdateFlowHistoryResponse, GetFlowByPathData, GetFlowByPathResponse, GetTriggersCountOfFlowData, GetTriggersCountOfFlowResponse, ListTokensOfFlowData, ListTokensOfFlowResponse, ToggleWorkspaceErrorHandlerForFlowData, ToggleWorkspaceErrorHandlerForFlowResponse, GetFlowByPathWithDraftData, GetFlowByPathWithDraftResponse, ExistsFlowByPathData, ExistsFlowByPathResponse, CreateFlowData, CreateFlowResponse, UpdateFlowData, UpdateFlowResponse, ArchiveFlowByPathData, ArchiveFlowByPathResponse, DeleteFlowByPathData, DeleteFlowByPathResponse, ListRawAppsData, ListRawAppsResponse, ExistsRawAppData, ExistsRawAppResponse, GetRawAppDataData, GetRawAppDataResponse, ListSearchAppData, ListSearchAppResponse, ListAppsData, ListAppsResponse, CreateAppData, CreateAppResponse, ExistsAppData, ExistsAppResponse, GetAppByPathData, GetAppByPathResponse, GetAppLiteByPathData, GetAppLiteByPathResponse, GetAppByPathWithDraftData, GetAppByPathWithDraftResponse, GetAppHistoryByPathData, GetAppHistoryByPathResponse, GetAppLatestVersionData, GetAppLatestVersionResponse, UpdateAppHistoryData, UpdateAppHistoryResponse, GetPublicAppBySecretData, GetPublicAppBySecretResponse, GetPublicResourceData, GetPublicResourceResponse, GetPublicSecretOfAppData, GetPublicSecretOfAppResponse, GetAppByVersionData, GetAppByVersionResponse, CreateRawAppData, CreateRawAppResponse, UpdateRawAppData, UpdateRawAppResponse, DeleteRawAppData, DeleteRawAppResponse, DeleteAppData, DeleteAppResponse, UpdateAppData, UpdateAppResponse, CustomPathExistsData, CustomPathExistsResponse, ExecuteComponentData, ExecuteComponentResponse, RunFlowByPathData, RunFlowByPathResponse, RestartFlowAtStepData, RestartFlowAtStepResponse, RunScriptByHashData, RunScriptByHashResponse, RunScriptPreviewData, RunScriptPreviewResponse, RunCodeWorkflowTaskData, RunCodeWorkflowTaskResponse, RunRawScriptDependenciesData, RunRawScriptDependenciesResponse, RunFlowPreviewData, RunFlowPreviewResponse, ListQueueData, ListQueueResponse, GetQueueCountData, GetQueueCountResponse, GetCompletedCountData, GetCompletedCountResponse, CountCompletedJobsData, CountCompletedJobsResponse, ListFilteredUuidsData, ListFilteredUuidsResponse, CancelSelectionData, CancelSelectionResponse, ListCompletedJobsData, ListCompletedJobsResponse, ListJobsData, ListJobsResponse, GetDbClockResponse, CountJobsByTagData, CountJobsByTagResponse, GetJobData, GetJobResponse, GetRootJobIdData, GetRootJobIdResponse, GetJobLogsData, GetJobLogsResponse, GetJobArgsData, GetJobArgsResponse, GetJobUpdatesData, GetJobUpdatesResponse, GetLogFileFromStoreData, GetLogFileFromStoreResponse, GetFlowDebugInfoData, GetFlowDebugInfoResponse, GetCompletedJobData, GetCompletedJobResponse, GetCompletedJobResultData, GetCompletedJobResultResponse, GetCompletedJobResultMaybeData, GetCompletedJobResultMaybeResponse, DeleteCompletedJobData, DeleteCompletedJobResponse, CancelQueuedJobData, CancelQueuedJobResponse, CancelPersistentQueuedJobsData, CancelPersistentQueuedJobsResponse, ForceCancelQueuedJobData, ForceCancelQueuedJobResponse, CreateJobSignatureData, CreateJobSignatureResponse, GetResumeUrlsData, GetResumeUrlsResponse, GetSlackApprovalPayloadData, GetSlackApprovalPayloadResponse, ResumeSuspendedJobGetData, ResumeSuspendedJobGetResponse, ResumeSuspendedJobPostData, ResumeSuspendedJobPostResponse, SetFlowUserStateData, SetFlowUserStateResponse, GetFlowUserStateData, GetFlowUserStateResponse, ResumeSuspendedFlowAsOwnerData, ResumeSuspendedFlowAsOwnerResponse, CancelSuspendedJobGetData, CancelSuspendedJobGetResponse, CancelSuspendedJobPostData, CancelSuspendedJobPostResponse, GetSuspendedJobFlowData, GetSuspendedJobFlowResponse, PreviewScheduleData, PreviewScheduleResponse, CreateScheduleData, CreateScheduleResponse, UpdateScheduleData, UpdateScheduleResponse, SetScheduleEnabledData, SetScheduleEnabledResponse, DeleteScheduleData, DeleteScheduleResponse, GetScheduleData, GetScheduleResponse, ExistsScheduleData, ExistsScheduleResponse, ListSchedulesData, ListSchedulesResponse, ListSchedulesWithJobsData, ListSchedulesWithJobsResponse, SetDefaultErrorOrRecoveryHandlerData, SetDefaultErrorOrRecoveryHandlerResponse, CreateHttpTriggerData, CreateHttpTriggerResponse, UpdateHttpTriggerData, UpdateHttpTriggerResponse, DeleteHttpTriggerData, DeleteHttpTriggerResponse, GetHttpTriggerData, GetHttpTriggerResponse, ListHttpTriggersData, ListHttpTriggersResponse, ExistsHttpTriggerData, ExistsHttpTriggerResponse, ExistsRouteData, ExistsRouteResponse, CreateWebsocketTriggerData, CreateWebsocketTriggerResponse, UpdateWebsocketTriggerData, UpdateWebsocketTriggerResponse, DeleteWebsocketTriggerData, DeleteWebsocketTriggerResponse, GetWebsocketTriggerData, GetWebsocketTriggerResponse, ListWebsocketTriggersData, ListWebsocketTriggersResponse, ExistsWebsocketTriggerData, ExistsWebsocketTriggerResponse, SetWebsocketTriggerEnabledData, SetWebsocketTriggerEnabledResponse, CreateKafkaTriggerData, CreateKafkaTriggerResponse, UpdateKafkaTriggerData, UpdateKafkaTriggerResponse, DeleteKafkaTriggerData, DeleteKafkaTriggerResponse, GetKafkaTriggerData, GetKafkaTriggerResponse, ListKafkaTriggersData, ListKafkaTriggersResponse, ExistsKafkaTriggerData, ExistsKafkaTriggerResponse, SetKafkaTriggerEnabledData, SetKafkaTriggerEnabledResponse, CreateNatsTriggerData, CreateNatsTriggerResponse, UpdateNatsTriggerData, UpdateNatsTriggerResponse, DeleteNatsTriggerData, DeleteNatsTriggerResponse, GetNatsTriggerData, GetNatsTriggerResponse, ListNatsTriggersData, ListNatsTriggersResponse, ExistsNatsTriggerData, ExistsNatsTriggerResponse, SetNatsTriggerEnabledData, SetNatsTriggerEnabledResponse, ListInstanceGroupsResponse, GetInstanceGroupData, GetInstanceGroupResponse, CreateInstanceGroupData, CreateInstanceGroupResponse, UpdateInstanceGroupData, UpdateInstanceGroupResponse, DeleteInstanceGroupData, DeleteInstanceGroupResponse, AddUserToInstanceGroupData, AddUserToInstanceGroupResponse, RemoveUserFromInstanceGroupData, RemoveUserFromInstanceGroupResponse, ExportInstanceGroupsResponse, OverwriteInstanceGroupsData, OverwriteInstanceGroupsResponse, ListGroupsData, ListGroupsResponse, ListGroupNamesData, ListGroupNamesResponse, CreateGroupData, CreateGroupResponse, UpdateGroupData, UpdateGroupResponse, DeleteGroupData, DeleteGroupResponse, GetGroupData, GetGroupResponse, AddUserToGroupData, AddUserToGroupResponse, RemoveUserToGroupData, RemoveUserToGroupResponse, ListFoldersData, ListFoldersResponse, ListFolderNamesData, ListFolderNamesResponse, CreateFolderData, CreateFolderResponse, UpdateFolderData, UpdateFolderResponse, DeleteFolderData, DeleteFolderResponse, GetFolderData, GetFolderResponse, GetFolderUsageData, GetFolderUsageResponse, AddOwnerToFolderData, AddOwnerToFolderResponse, RemoveOwnerToFolderData, RemoveOwnerToFolderResponse, ListWorkersData, ListWorkersResponse, ExistsWorkerWithTagData, ExistsWorkerWithTagResponse, GetQueueMetricsResponse, GetCountsOfJobsWaitingPerTagResponse, ListWorkerGroupsResponse, GetConfigData, GetConfigResponse, UpdateConfigData, UpdateConfigResponse, DeleteConfigData, DeleteConfigResponse, ListConfigsResponse, ListAutoscalingEventsData, ListAutoscalingEventsResponse, GetGranularAclsData, GetGranularAclsResponse, AddGranularAclsData, AddGranularAclsResponse, RemoveGranularAclsData, RemoveGranularAclsResponse, SetCaptureConfigData, SetCaptureConfigResponse, PingCaptureConfigData, PingCaptureConfigResponse, GetCaptureConfigsData, GetCaptureConfigsResponse, ListCapturesData, ListCapturesResponse, DeleteCaptureData, DeleteCaptureResponse, StarData, StarResponse, UnstarData, UnstarResponse, GetInputHistoryData, GetInputHistoryResponse, GetArgsFromHistoryOrSavedInputData, GetArgsFromHistoryOrSavedInputResponse, ListInputsData, ListInputsResponse, CreateInputData, CreateInputResponse, UpdateInputData, UpdateInputResponse, DeleteInputData, DeleteInputResponse, DuckdbConnectionSettingsData, DuckdbConnectionSettingsResponse, DuckdbConnectionSettingsV2Data, DuckdbConnectionSettingsV2Response, PolarsConnectionSettingsData, PolarsConnectionSettingsResponse, PolarsConnectionSettingsV2Data, PolarsConnectionSettingsV2Response, S3ResourceInfoData, S3ResourceInfoResponse, DatasetStorageTestConnectionData, DatasetStorageTestConnectionResponse, ListStoredFilesData, ListStoredFilesResponse, LoadFileMetadataData, LoadFileMetadataResponse, LoadFilePreviewData, LoadFilePreviewResponse, LoadParquetPreviewData, LoadParquetPreviewResponse, LoadTableRowCountData, LoadTableRowCountResponse, LoadCsvPreviewData, LoadCsvPreviewResponse, DeleteS3FileData, DeleteS3FileResponse, MoveS3FileData, MoveS3FileResponse, FileUploadData, FileUploadResponse, FileDownloadData, FileDownloadResponse, FileDownloadParquetAsCsvData, FileDownloadParquetAsCsvResponse, GetJobMetricsData, GetJobMetricsResponse, SetJobProgressData, SetJobProgressResponse, GetJobProgressData, GetJobProgressResponse, ListLogFilesData, ListLogFilesResponse, GetLogFileData, GetLogFileResponse, ListConcurrencyGroupsResponse, DeleteConcurrencyGroupData, DeleteConcurrencyGroupResponse, GetConcurrencyKeyData, GetConcurrencyKeyResponse, ListExtendedJobsData, ListExtendedJobsResponse, SearchJobsIndexData, SearchJobsIndexResponse, SearchLogsIndexData, SearchLogsIndexResponse, CountSearchLogsIndexData, CountSearchLogsIndexResponse, ClearIndexData, ClearIndexResponse } from './types.gen.ts'; +import type { BackendVersionResponse, BackendUptodateResponse, GetLicenseIdResponse, GetOpenApiYamlResponse, GetAuditLogData, GetAuditLogResponse, ListAuditLogsData, ListAuditLogsResponse, LoginData, LoginResponse, LogoutResponse, GetUserData, GetUserResponse, UpdateUserData, UpdateUserResponse, IsOwnerOfPathData, IsOwnerOfPathResponse, SetPasswordData, SetPasswordResponse, SetPasswordForUserData, SetPasswordForUserResponse, SetLoginTypeForUserData, SetLoginTypeForUserResponse, CreateUserGloballyData, CreateUserGloballyResponse, GlobalUserUpdateData, GlobalUserUpdateResponse, GlobalUsernameInfoData, GlobalUsernameInfoResponse, GlobalUserRenameData, GlobalUserRenameResponse, GlobalUserDeleteData, GlobalUserDeleteResponse, GlobalUsersOverwriteData, GlobalUsersOverwriteResponse, GlobalUsersExportResponse, DeleteUserData, DeleteUserResponse, ListWorkspacesResponse, IsDomainAllowedResponse, ListUserWorkspacesResponse, ListWorkspacesAsSuperAdminData, ListWorkspacesAsSuperAdminResponse, CreateWorkspaceData, CreateWorkspaceResponse, ExistsWorkspaceData, ExistsWorkspaceResponse, ExistsUsernameData, ExistsUsernameResponse, GetGlobalData, GetGlobalResponse, SetGlobalData, SetGlobalResponse, GetLocalResponse, TestSmtpData, TestSmtpResponse, TestCriticalChannelsData, TestCriticalChannelsResponse, GetCriticalAlertsData, GetCriticalAlertsResponse, AcknowledgeCriticalAlertData, AcknowledgeCriticalAlertResponse, AcknowledgeAllCriticalAlertsResponse, TestLicenseKeyData, TestLicenseKeyResponse, TestObjectStorageConfigData, TestObjectStorageConfigResponse, SendStatsResponse, GetLatestKeyRenewalAttemptResponse, RenewLicenseKeyData, RenewLicenseKeyResponse, CreateCustomerPortalSessionData, CreateCustomerPortalSessionResponse, TestMetadataData, TestMetadataResponse, ListGlobalSettingsResponse, GetCurrentEmailResponse, RefreshUserTokenData, RefreshUserTokenResponse, GetTutorialProgressResponse, UpdateTutorialProgressData, UpdateTutorialProgressResponse, LeaveInstanceResponse, GetUsageResponse, GetRunnableResponse, GlobalWhoamiResponse, ListWorkspaceInvitesResponse, WhoamiData, WhoamiResponse, AcceptInviteData, AcceptInviteResponse, DeclineInviteData, DeclineInviteResponse, InviteUserData, InviteUserResponse, AddUserData, AddUserResponse, DeleteInviteData, DeleteInviteResponse, ArchiveWorkspaceData, ArchiveWorkspaceResponse, UnarchiveWorkspaceData, UnarchiveWorkspaceResponse, DeleteWorkspaceData, DeleteWorkspaceResponse, LeaveWorkspaceData, LeaveWorkspaceResponse, GetWorkspaceNameData, GetWorkspaceNameResponse, ChangeWorkspaceNameData, ChangeWorkspaceNameResponse, ChangeWorkspaceIdData, ChangeWorkspaceIdResponse, ChangeWorkspaceColorData, ChangeWorkspaceColorResponse, WhoisData, WhoisResponse, UpdateOperatorSettingsData, UpdateOperatorSettingsResponse, ExistsEmailData, ExistsEmailResponse, ListUsersAsSuperAdminData, ListUsersAsSuperAdminResponse, ListPendingInvitesData, ListPendingInvitesResponse, GetSettingsData, GetSettingsResponse, GetDeployToData, GetDeployToResponse, GetIsPremiumData, GetIsPremiumResponse, GetPremiumInfoData, GetPremiumInfoResponse, SetAutomaticBillingData, SetAutomaticBillingResponse, GetThresholdAlertData, GetThresholdAlertResponse, SetThresholdAlertData, SetThresholdAlertResponse, EditSlackCommandData, EditSlackCommandResponse, RunSlackMessageTestJobData, RunSlackMessageTestJobResponse, EditDeployToData, EditDeployToResponse, EditAutoInviteData, EditAutoInviteResponse, EditWebhookData, EditWebhookResponse, EditCopilotConfigData, EditCopilotConfigResponse, GetCopilotInfoData, GetCopilotInfoResponse, EditErrorHandlerData, EditErrorHandlerResponse, EditLargeFileStorageConfigData, EditLargeFileStorageConfigResponse, EditWorkspaceGitSyncConfigData, EditWorkspaceGitSyncConfigResponse, EditWorkspaceDeployUiSettingsData, EditWorkspaceDeployUiSettingsResponse, EditWorkspaceDefaultAppData, EditWorkspaceDefaultAppResponse, EditDefaultScriptsData, EditDefaultScriptsResponse, GetDefaultScriptsData, GetDefaultScriptsResponse, SetEnvironmentVariableData, SetEnvironmentVariableResponse, GetWorkspaceEncryptionKeyData, GetWorkspaceEncryptionKeyResponse, SetWorkspaceEncryptionKeyData, SetWorkspaceEncryptionKeyResponse, GetWorkspaceDefaultAppData, GetWorkspaceDefaultAppResponse, GetLargeFileStorageConfigData, GetLargeFileStorageConfigResponse, GetWorkspaceUsageData, GetWorkspaceUsageResponse, GetUsedTriggersData, GetUsedTriggersResponse, ListUsersData, ListUsersResponse, ListUsersUsageData, ListUsersUsageResponse, ListUsernamesData, ListUsernamesResponse, UsernameToEmailData, UsernameToEmailResponse, CreateTokenData, CreateTokenResponse, CreateTokenImpersonateData, CreateTokenImpersonateResponse, DeleteTokenData, DeleteTokenResponse, ListTokensData, ListTokensResponse, GetOidcTokenData, GetOidcTokenResponse, CreateVariableData, CreateVariableResponse, EncryptValueData, EncryptValueResponse, DeleteVariableData, DeleteVariableResponse, UpdateVariableData, UpdateVariableResponse, GetVariableData, GetVariableResponse, GetVariableValueData, GetVariableValueResponse, ExistsVariableData, ExistsVariableResponse, ListVariableData, ListVariableResponse, ListContextualVariablesData, ListContextualVariablesResponse, WorkspaceGetCriticalAlertsData, WorkspaceGetCriticalAlertsResponse, WorkspaceAcknowledgeCriticalAlertData, WorkspaceAcknowledgeCriticalAlertResponse, WorkspaceAcknowledgeAllCriticalAlertsData, WorkspaceAcknowledgeAllCriticalAlertsResponse, WorkspaceMuteCriticalAlertsUiData, WorkspaceMuteCriticalAlertsUiResponse, LoginWithOauthData, LoginWithOauthResponse, ConnectSlackCallbackData, ConnectSlackCallbackResponse, ConnectSlackCallbackInstanceData, ConnectSlackCallbackInstanceResponse, ConnectCallbackData, ConnectCallbackResponse, CreateAccountData, CreateAccountResponse, RefreshTokenData, RefreshTokenResponse, DisconnectAccountData, DisconnectAccountResponse, DisconnectSlackData, DisconnectSlackResponse, ListOauthLoginsResponse, ListOauthConnectsResponse, GetOauthConnectData, GetOauthConnectResponse, SyncTeamsResponse, CreateResourceData, CreateResourceResponse, DeleteResourceData, DeleteResourceResponse, UpdateResourceData, UpdateResourceResponse, UpdateResourceValueData, UpdateResourceValueResponse, GetResourceData, GetResourceResponse, GetResourceValueInterpolatedData, GetResourceValueInterpolatedResponse, GetResourceValueData, GetResourceValueResponse, ExistsResourceData, ExistsResourceResponse, ListResourceData, ListResourceResponse, ListSearchResourceData, ListSearchResourceResponse, ListResourceNamesData, ListResourceNamesResponse, CreateResourceTypeData, CreateResourceTypeResponse, FileResourceTypeToFileExtMapData, FileResourceTypeToFileExtMapResponse, DeleteResourceTypeData, DeleteResourceTypeResponse, UpdateResourceTypeData, UpdateResourceTypeResponse, GetResourceTypeData, GetResourceTypeResponse, ExistsResourceTypeData, ExistsResourceTypeResponse, ListResourceTypeData, ListResourceTypeResponse, ListResourceTypeNamesData, ListResourceTypeNamesResponse, QueryResourceTypesData, QueryResourceTypesResponse, ListHubIntegrationsData, ListHubIntegrationsResponse, ListHubFlowsResponse, GetHubFlowByIdData, GetHubFlowByIdResponse, ListHubAppsResponse, GetHubAppByIdData, GetHubAppByIdResponse, GetPublicAppByCustomPathData, GetPublicAppByCustomPathResponse, GetHubScriptContentByPathData, GetHubScriptContentByPathResponse, GetHubScriptByPathData, GetHubScriptByPathResponse, GetTopHubScriptsData, GetTopHubScriptsResponse, QueryHubScriptsData, QueryHubScriptsResponse, ListSearchScriptData, ListSearchScriptResponse, ListScriptsData, ListScriptsResponse, ListScriptPathsData, ListScriptPathsResponse, CreateDraftData, CreateDraftResponse, DeleteDraftData, DeleteDraftResponse, CreateScriptData, CreateScriptResponse, ToggleWorkspaceErrorHandlerForScriptData, ToggleWorkspaceErrorHandlerForScriptResponse, GetCustomTagsData, GetCustomTagsResponse, GeDefaultTagsResponse, IsDefaultTagsPerWorkspaceResponse, ArchiveScriptByPathData, ArchiveScriptByPathResponse, ArchiveScriptByHashData, ArchiveScriptByHashResponse, DeleteScriptByHashData, DeleteScriptByHashResponse, DeleteScriptByPathData, DeleteScriptByPathResponse, GetScriptByPathData, GetScriptByPathResponse, GetTriggersCountOfScriptData, GetTriggersCountOfScriptResponse, ListTokensOfScriptData, ListTokensOfScriptResponse, GetScriptByPathWithDraftData, GetScriptByPathWithDraftResponse, GetScriptHistoryByPathData, GetScriptHistoryByPathResponse, GetScriptLatestVersionData, GetScriptLatestVersionResponse, UpdateScriptHistoryData, UpdateScriptHistoryResponse, RawScriptByPathData, RawScriptByPathResponse, RawScriptByPathTokenedData, RawScriptByPathTokenedResponse, ExistsScriptByPathData, ExistsScriptByPathResponse, GetScriptByHashData, GetScriptByHashResponse, RawScriptByHashData, RawScriptByHashResponse, GetScriptDeploymentStatusData, GetScriptDeploymentStatusResponse, RunScriptByPathData, RunScriptByPathResponse, OpenaiSyncScriptByPathData, OpenaiSyncScriptByPathResponse, RunWaitResultScriptByPathData, RunWaitResultScriptByPathResponse, RunWaitResultScriptByPathGetData, RunWaitResultScriptByPathGetResponse, OpenaiSyncFlowByPathData, OpenaiSyncFlowByPathResponse, RunWaitResultFlowByPathData, RunWaitResultFlowByPathResponse, ResultByIdData, ResultByIdResponse, ListFlowPathsData, ListFlowPathsResponse, ListSearchFlowData, ListSearchFlowResponse, ListFlowsData, ListFlowsResponse, GetFlowHistoryData, GetFlowHistoryResponse, GetFlowLatestVersionData, GetFlowLatestVersionResponse, GetFlowVersionData, GetFlowVersionResponse, UpdateFlowHistoryData, UpdateFlowHistoryResponse, GetFlowByPathData, GetFlowByPathResponse, GetTriggersCountOfFlowData, GetTriggersCountOfFlowResponse, ListTokensOfFlowData, ListTokensOfFlowResponse, ToggleWorkspaceErrorHandlerForFlowData, ToggleWorkspaceErrorHandlerForFlowResponse, GetFlowByPathWithDraftData, GetFlowByPathWithDraftResponse, ExistsFlowByPathData, ExistsFlowByPathResponse, CreateFlowData, CreateFlowResponse, UpdateFlowData, UpdateFlowResponse, ArchiveFlowByPathData, ArchiveFlowByPathResponse, DeleteFlowByPathData, DeleteFlowByPathResponse, ListRawAppsData, ListRawAppsResponse, ExistsRawAppData, ExistsRawAppResponse, GetRawAppDataData, GetRawAppDataResponse, ListSearchAppData, ListSearchAppResponse, ListAppsData, ListAppsResponse, CreateAppData, CreateAppResponse, ExistsAppData, ExistsAppResponse, GetAppByPathData, GetAppByPathResponse, GetAppLiteByPathData, GetAppLiteByPathResponse, GetAppByPathWithDraftData, GetAppByPathWithDraftResponse, GetAppHistoryByPathData, GetAppHistoryByPathResponse, GetAppLatestVersionData, GetAppLatestVersionResponse, UpdateAppHistoryData, UpdateAppHistoryResponse, GetPublicAppBySecretData, GetPublicAppBySecretResponse, GetPublicResourceData, GetPublicResourceResponse, GetPublicSecretOfAppData, GetPublicSecretOfAppResponse, GetAppByVersionData, GetAppByVersionResponse, CreateRawAppData, CreateRawAppResponse, UpdateRawAppData, UpdateRawAppResponse, DeleteRawAppData, DeleteRawAppResponse, DeleteAppData, DeleteAppResponse, UpdateAppData, UpdateAppResponse, CustomPathExistsData, CustomPathExistsResponse, ExecuteComponentData, ExecuteComponentResponse, RunFlowByPathData, RunFlowByPathResponse, RestartFlowAtStepData, RestartFlowAtStepResponse, RunScriptByHashData, RunScriptByHashResponse, RunScriptPreviewData, RunScriptPreviewResponse, RunCodeWorkflowTaskData, RunCodeWorkflowTaskResponse, RunRawScriptDependenciesData, RunRawScriptDependenciesResponse, RunFlowPreviewData, RunFlowPreviewResponse, ListQueueData, ListQueueResponse, GetQueueCountData, GetQueueCountResponse, GetCompletedCountData, GetCompletedCountResponse, CountCompletedJobsData, CountCompletedJobsResponse, ListFilteredUuidsData, ListFilteredUuidsResponse, CancelSelectionData, CancelSelectionResponse, ListCompletedJobsData, ListCompletedJobsResponse, ListJobsData, ListJobsResponse, GetDbClockResponse, CountJobsByTagData, CountJobsByTagResponse, GetJobData, GetJobResponse, GetRootJobIdData, GetRootJobIdResponse, GetJobLogsData, GetJobLogsResponse, GetJobArgsData, GetJobArgsResponse, GetJobUpdatesData, GetJobUpdatesResponse, GetLogFileFromStoreData, GetLogFileFromStoreResponse, GetFlowDebugInfoData, GetFlowDebugInfoResponse, GetCompletedJobData, GetCompletedJobResponse, GetCompletedJobResultData, GetCompletedJobResultResponse, GetCompletedJobResultMaybeData, GetCompletedJobResultMaybeResponse, DeleteCompletedJobData, DeleteCompletedJobResponse, CancelQueuedJobData, CancelQueuedJobResponse, CancelPersistentQueuedJobsData, CancelPersistentQueuedJobsResponse, ForceCancelQueuedJobData, ForceCancelQueuedJobResponse, CreateJobSignatureData, CreateJobSignatureResponse, GetResumeUrlsData, GetResumeUrlsResponse, GetSlackApprovalPayloadData, GetSlackApprovalPayloadResponse, ResumeSuspendedJobGetData, ResumeSuspendedJobGetResponse, ResumeSuspendedJobPostData, ResumeSuspendedJobPostResponse, SetFlowUserStateData, SetFlowUserStateResponse, GetFlowUserStateData, GetFlowUserStateResponse, ResumeSuspendedFlowAsOwnerData, ResumeSuspendedFlowAsOwnerResponse, CancelSuspendedJobGetData, CancelSuspendedJobGetResponse, CancelSuspendedJobPostData, CancelSuspendedJobPostResponse, GetSuspendedJobFlowData, GetSuspendedJobFlowResponse, PreviewScheduleData, PreviewScheduleResponse, CreateScheduleData, CreateScheduleResponse, UpdateScheduleData, UpdateScheduleResponse, SetScheduleEnabledData, SetScheduleEnabledResponse, DeleteScheduleData, DeleteScheduleResponse, GetScheduleData, GetScheduleResponse, ExistsScheduleData, ExistsScheduleResponse, ListSchedulesData, ListSchedulesResponse, ListSchedulesWithJobsData, ListSchedulesWithJobsResponse, SetDefaultErrorOrRecoveryHandlerData, SetDefaultErrorOrRecoveryHandlerResponse, CreateHttpTriggerData, CreateHttpTriggerResponse, UpdateHttpTriggerData, UpdateHttpTriggerResponse, DeleteHttpTriggerData, DeleteHttpTriggerResponse, GetHttpTriggerData, GetHttpTriggerResponse, ListHttpTriggersData, ListHttpTriggersResponse, ExistsHttpTriggerData, ExistsHttpTriggerResponse, ExistsRouteData, ExistsRouteResponse, CreateWebsocketTriggerData, CreateWebsocketTriggerResponse, UpdateWebsocketTriggerData, UpdateWebsocketTriggerResponse, DeleteWebsocketTriggerData, DeleteWebsocketTriggerResponse, GetWebsocketTriggerData, GetWebsocketTriggerResponse, ListWebsocketTriggersData, ListWebsocketTriggersResponse, ExistsWebsocketTriggerData, ExistsWebsocketTriggerResponse, SetWebsocketTriggerEnabledData, SetWebsocketTriggerEnabledResponse, TestWebsocketConnectionData, TestWebsocketConnectionResponse, CreateKafkaTriggerData, CreateKafkaTriggerResponse, UpdateKafkaTriggerData, UpdateKafkaTriggerResponse, DeleteKafkaTriggerData, DeleteKafkaTriggerResponse, GetKafkaTriggerData, GetKafkaTriggerResponse, ListKafkaTriggersData, ListKafkaTriggersResponse, ExistsKafkaTriggerData, ExistsKafkaTriggerResponse, SetKafkaTriggerEnabledData, SetKafkaTriggerEnabledResponse, TestKafkaConnectionData, TestKafkaConnectionResponse, CreateNatsTriggerData, CreateNatsTriggerResponse, UpdateNatsTriggerData, UpdateNatsTriggerResponse, DeleteNatsTriggerData, DeleteNatsTriggerResponse, GetNatsTriggerData, GetNatsTriggerResponse, ListNatsTriggersData, ListNatsTriggersResponse, ExistsNatsTriggerData, ExistsNatsTriggerResponse, SetNatsTriggerEnabledData, SetNatsTriggerEnabledResponse, TestNatsConnectionData, TestNatsConnectionResponse, IsValidPostgresConfigurationData, IsValidPostgresConfigurationResponse, CreateTemplateScriptData, CreateTemplateScriptResponse, GetTemplateScriptData, GetTemplateScriptResponse, ListPostgresReplicationSlotData, ListPostgresReplicationSlotResponse, CreatePostgresReplicationSlotData, CreatePostgresReplicationSlotResponse, DeletePostgresReplicationSlotData, DeletePostgresReplicationSlotResponse, ListPostgresPublicationData, ListPostgresPublicationResponse, GetPostgresPublicationData, GetPostgresPublicationResponse, CreatePostgresPublicationData, CreatePostgresPublicationResponse, UpdatePostgresPublicationData, UpdatePostgresPublicationResponse, DeletePostgresPublicationData, DeletePostgresPublicationResponse, CreatePostgresTriggerData, CreatePostgresTriggerResponse, UpdatePostgresTriggerData, UpdatePostgresTriggerResponse, DeletePostgresTriggerData, DeletePostgresTriggerResponse, GetPostgresTriggerData, GetPostgresTriggerResponse, ListPostgresTriggersData, ListPostgresTriggersResponse, ExistsPostgresTriggerData, ExistsPostgresTriggerResponse, SetPostgresTriggerEnabledData, SetPostgresTriggerEnabledResponse, ListInstanceGroupsResponse, GetInstanceGroupData, GetInstanceGroupResponse, CreateInstanceGroupData, CreateInstanceGroupResponse, UpdateInstanceGroupData, UpdateInstanceGroupResponse, DeleteInstanceGroupData, DeleteInstanceGroupResponse, AddUserToInstanceGroupData, AddUserToInstanceGroupResponse, RemoveUserFromInstanceGroupData, RemoveUserFromInstanceGroupResponse, ExportInstanceGroupsResponse, OverwriteInstanceGroupsData, OverwriteInstanceGroupsResponse, ListGroupsData, ListGroupsResponse, ListGroupNamesData, ListGroupNamesResponse, CreateGroupData, CreateGroupResponse, UpdateGroupData, UpdateGroupResponse, DeleteGroupData, DeleteGroupResponse, GetGroupData, GetGroupResponse, AddUserToGroupData, AddUserToGroupResponse, RemoveUserToGroupData, RemoveUserToGroupResponse, ListFoldersData, ListFoldersResponse, ListFolderNamesData, ListFolderNamesResponse, CreateFolderData, CreateFolderResponse, UpdateFolderData, UpdateFolderResponse, DeleteFolderData, DeleteFolderResponse, GetFolderData, GetFolderResponse, GetFolderUsageData, GetFolderUsageResponse, AddOwnerToFolderData, AddOwnerToFolderResponse, RemoveOwnerToFolderData, RemoveOwnerToFolderResponse, ListWorkersData, ListWorkersResponse, ExistsWorkerWithTagData, ExistsWorkerWithTagResponse, GetQueueMetricsResponse, GetCountsOfJobsWaitingPerTagResponse, ListWorkerGroupsResponse, GetConfigData, GetConfigResponse, UpdateConfigData, UpdateConfigResponse, DeleteConfigData, DeleteConfigResponse, ListConfigsResponse, ListAutoscalingEventsData, ListAutoscalingEventsResponse, GetGranularAclsData, GetGranularAclsResponse, AddGranularAclsData, AddGranularAclsResponse, RemoveGranularAclsData, RemoveGranularAclsResponse, SetCaptureConfigData, SetCaptureConfigResponse, PingCaptureConfigData, PingCaptureConfigResponse, GetCaptureConfigsData, GetCaptureConfigsResponse, ListCapturesData, ListCapturesResponse, GetCaptureData, GetCaptureResponse, DeleteCaptureData, DeleteCaptureResponse, StarData, StarResponse, UnstarData, UnstarResponse, GetInputHistoryData, GetInputHistoryResponse, GetArgsFromHistoryOrSavedInputData, GetArgsFromHistoryOrSavedInputResponse, ListInputsData, ListInputsResponse, CreateInputData, CreateInputResponse, UpdateInputData, UpdateInputResponse, DeleteInputData, DeleteInputResponse, DuckdbConnectionSettingsData, DuckdbConnectionSettingsResponse, DuckdbConnectionSettingsV2Data, DuckdbConnectionSettingsV2Response, PolarsConnectionSettingsData, PolarsConnectionSettingsResponse, PolarsConnectionSettingsV2Data, PolarsConnectionSettingsV2Response, S3ResourceInfoData, S3ResourceInfoResponse, DatasetStorageTestConnectionData, DatasetStorageTestConnectionResponse, ListStoredFilesData, ListStoredFilesResponse, LoadFileMetadataData, LoadFileMetadataResponse, LoadFilePreviewData, LoadFilePreviewResponse, LoadParquetPreviewData, LoadParquetPreviewResponse, LoadTableRowCountData, LoadTableRowCountResponse, LoadCsvPreviewData, LoadCsvPreviewResponse, DeleteS3FileData, DeleteS3FileResponse, MoveS3FileData, MoveS3FileResponse, FileUploadData, FileUploadResponse, FileDownloadData, FileDownloadResponse, FileDownloadParquetAsCsvData, FileDownloadParquetAsCsvResponse, GetJobMetricsData, GetJobMetricsResponse, SetJobProgressData, SetJobProgressResponse, GetJobProgressData, GetJobProgressResponse, ListLogFilesData, ListLogFilesResponse, GetLogFileData, GetLogFileResponse, ListConcurrencyGroupsResponse, DeleteConcurrencyGroupData, DeleteConcurrencyGroupResponse, GetConcurrencyKeyData, GetConcurrencyKeyResponse, ListExtendedJobsData, ListExtendedJobsResponse, SearchJobsIndexData, SearchJobsIndexResponse, SearchLogsIndexData, SearchLogsIndexResponse, CountSearchLogsIndexData, CountSearchLogsIndexResponse, ClearIndexData, ClearIndexResponse } from './types.gen.ts'; /** * get backend version @@ -997,6 +997,25 @@ export const whois = (data: WhoisData): CancelablePromise => { re } }); }; +/** + * Update operator settings for a workspace + * Updates the operator settings for a specific workspace. Requires workspace admin privileges. + * @param data The data for the request. + * @param data.workspace + * @param data.requestBody + * @returns string Operator settings updated successfully + * @throws ApiError + */ +export const updateOperatorSettings = (data: UpdateOperatorSettingsData): CancelablePromise => { return __request(OpenAPI, { + method: 'POST', + url: '/w/{workspace}/workspaces/operator_settings', + path: { + workspace: data.workspace + }, + body: data.requestBody, + mediaType: 'application/json' +}); }; + /** * exists email * @param data The data for the request. @@ -2081,6 +2100,16 @@ export const getOauthConnect = (data: GetOauthConnectData): CancelablePromise => { return __request(OpenAPI, { + method: 'POST', + url: '/teams/sync' +}); }; + /** * create resource * @param data The data for the request. @@ -2866,7 +2895,7 @@ export const deleteScriptByHash = (data: DeleteScriptByHashData): CancelableProm }); }; /** - * delete all scripts at a given path (require admin) + * delete script at a given path (require admin) * @param data The data for the request. * @param data.workspace * @param data.path @@ -5736,6 +5765,24 @@ export const setWebsocketTriggerEnabled = (data: SetWebsocketTriggerEnabledData) mediaType: 'application/json' }); }; +/** + * test websocket connection + * @param data The data for the request. + * @param data.workspace + * @param data.requestBody test websocket connection + * @returns string successfuly connected to websocket + * @throws ApiError + */ +export const testWebsocketConnection = (data: TestWebsocketConnectionData): CancelablePromise => { return __request(OpenAPI, { + method: 'POST', + url: '/w/{workspace}/websocket_triggers/test', + path: { + workspace: data.workspace + }, + body: data.requestBody, + mediaType: 'application/json' +}); }; + /** * create kafka trigger * @param data The data for the request. @@ -5872,6 +5919,24 @@ export const setKafkaTriggerEnabled = (data: SetKafkaTriggerEnabledData): Cancel mediaType: 'application/json' }); }; +/** + * test kafka connection + * @param data The data for the request. + * @param data.workspace + * @param data.requestBody test kafka connection + * @returns string successfuly connected to kafka brokers + * @throws ApiError + */ +export const testKafkaConnection = (data: TestKafkaConnectionData): CancelablePromise => { return __request(OpenAPI, { + method: 'POST', + url: '/w/{workspace}/kafka_triggers/test', + path: { + workspace: data.workspace + }, + body: data.requestBody, + mediaType: 'application/json' +}); }; + /** * create nats trigger * @param data The data for the request. @@ -6008,6 +6073,368 @@ export const setNatsTriggerEnabled = (data: SetNatsTriggerEnabledData): Cancelab mediaType: 'application/json' }); }; +/** + * test NATS connection + * @param data The data for the request. + * @param data.workspace + * @param data.requestBody test nats connection + * @returns string successfuly connected to NATS servers + * @throws ApiError + */ +export const testNatsConnection = (data: TestNatsConnectionData): CancelablePromise => { return __request(OpenAPI, { + method: 'POST', + url: '/w/{workspace}/nats_triggers/test', + path: { + workspace: data.workspace + }, + body: data.requestBody, + mediaType: 'application/json' +}); }; + +/** + * check if postgres configuration is set to logical + * @param data The data for the request. + * @param data.workspace + * @param data.path + * @returns boolean boolean that indicates if postgres is set to logical level or not + * @throws ApiError + */ +export const isValidPostgresConfiguration = (data: IsValidPostgresConfigurationData): CancelablePromise => { return __request(OpenAPI, { + method: 'GET', + url: '/w/{workspace}/postgres_triggers/is_valid_postgres_configuration/{path}', + path: { + workspace: data.workspace, + path: data.path + } +}); }; + +/** + * create template script + * @param data The data for the request. + * @param data.workspace + * @param data.requestBody template script + * @returns string custom id to retrieve template script + * @throws ApiError + */ +export const createTemplateScript = (data: CreateTemplateScriptData): CancelablePromise => { return __request(OpenAPI, { + method: 'POST', + url: '/w/{workspace}/postgres_triggers/create_template_script', + path: { + workspace: data.workspace + }, + body: data.requestBody, + mediaType: 'application/json' +}); }; + +/** + * get template script + * @param data The data for the request. + * @param data.workspace + * @param data.id + * @returns string template script + * @throws ApiError + */ +export const getTemplateScript = (data: GetTemplateScriptData): CancelablePromise => { return __request(OpenAPI, { + method: 'GET', + url: '/w/{workspace}/postgres_triggers/get_template_script/{id}', + path: { + workspace: data.workspace, + id: data.id + } +}); }; + +/** + * list postgres replication slot + * @param data The data for the request. + * @param data.workspace + * @param data.path + * @returns SlotList list postgres slot + * @throws ApiError + */ +export const listPostgresReplicationSlot = (data: ListPostgresReplicationSlotData): CancelablePromise => { return __request(OpenAPI, { + method: 'GET', + url: '/w/{workspace}/postgres_triggers/slot/list/{path}', + path: { + workspace: data.workspace, + path: data.path + } +}); }; + +/** + * create replication slot for postgres + * @param data The data for the request. + * @param data.workspace + * @param data.path + * @param data.requestBody new slot for postgres + * @returns string slot created + * @throws ApiError + */ +export const createPostgresReplicationSlot = (data: CreatePostgresReplicationSlotData): CancelablePromise => { return __request(OpenAPI, { + method: 'POST', + url: '/w/{workspace}/postgres_triggers/slot/create/{path}', + path: { + workspace: data.workspace, + path: data.path + }, + body: data.requestBody, + mediaType: 'application/json' +}); }; + +/** + * delete postgres replication slot + * @param data The data for the request. + * @param data.workspace + * @param data.path + * @param data.requestBody replication slot of postgres + * @returns string postgres replication slot deleted + * @throws ApiError + */ +export const deletePostgresReplicationSlot = (data: DeletePostgresReplicationSlotData): CancelablePromise => { return __request(OpenAPI, { + method: 'DELETE', + url: '/w/{workspace}/postgres_triggers/slot/delete/{path}', + path: { + workspace: data.workspace, + path: data.path + }, + body: data.requestBody, + mediaType: 'application/json' +}); }; + +/** + * list postgres publication + * @param data The data for the request. + * @param data.workspace + * @param data.path + * @returns string database publication list + * @throws ApiError + */ +export const listPostgresPublication = (data: ListPostgresPublicationData): CancelablePromise => { return __request(OpenAPI, { + method: 'GET', + url: '/w/{workspace}/postgres_triggers/publication/list/{path}', + path: { + workspace: data.workspace, + path: data.path + } +}); }; + +/** + * get postgres publication + * @param data The data for the request. + * @param data.workspace + * @param data.path + * @param data.publication + * @returns PublicationData postgres publication get + * @throws ApiError + */ +export const getPostgresPublication = (data: GetPostgresPublicationData): CancelablePromise => { return __request(OpenAPI, { + method: 'GET', + url: '/w/{workspace}/postgres_triggers/publication/get/{publication}/{path}', + path: { + workspace: data.workspace, + path: data.path, + publication: data.publication + } +}); }; + +/** + * create publication for postgres + * @param data The data for the request. + * @param data.workspace + * @param data.path + * @param data.publication + * @param data.requestBody new publication for postgres + * @returns string publication created + * @throws ApiError + */ +export const createPostgresPublication = (data: CreatePostgresPublicationData): CancelablePromise => { return __request(OpenAPI, { + method: 'POST', + url: '/w/{workspace}/postgres_triggers/publication/create/{publication}/{path}', + path: { + workspace: data.workspace, + path: data.path, + publication: data.publication + }, + body: data.requestBody, + mediaType: 'application/json' +}); }; + +/** + * update publication for postgres + * @param data The data for the request. + * @param data.workspace + * @param data.path + * @param data.publication + * @param data.requestBody update publication for postgres + * @returns string publication updated + * @throws ApiError + */ +export const updatePostgresPublication = (data: UpdatePostgresPublicationData): CancelablePromise => { return __request(OpenAPI, { + method: 'POST', + url: '/w/{workspace}/postgres_triggers/publication/update/{publication}/{path}', + path: { + workspace: data.workspace, + path: data.path, + publication: data.publication + }, + body: data.requestBody, + mediaType: 'application/json' +}); }; + +/** + * delete postgres publication + * @param data The data for the request. + * @param data.workspace + * @param data.path + * @param data.publication + * @returns string postgres publication deleted + * @throws ApiError + */ +export const deletePostgresPublication = (data: DeletePostgresPublicationData): CancelablePromise => { return __request(OpenAPI, { + method: 'DELETE', + url: '/w/{workspace}/postgres_triggers/publication/delete/{publication}/{path}', + path: { + workspace: data.workspace, + path: data.path, + publication: data.publication + } +}); }; + +/** + * create postgres trigger + * @param data The data for the request. + * @param data.workspace + * @param data.requestBody new postgres trigger + * @returns string postgres trigger created + * @throws ApiError + */ +export const createPostgresTrigger = (data: CreatePostgresTriggerData): CancelablePromise => { return __request(OpenAPI, { + method: 'POST', + url: '/w/{workspace}/postgres_triggers/create', + path: { + workspace: data.workspace + }, + body: data.requestBody, + mediaType: 'application/json' +}); }; + +/** + * update postgres trigger + * @param data The data for the request. + * @param data.workspace + * @param data.path + * @param data.requestBody updated trigger + * @returns string postgres trigger updated + * @throws ApiError + */ +export const updatePostgresTrigger = (data: UpdatePostgresTriggerData): CancelablePromise => { return __request(OpenAPI, { + method: 'POST', + url: '/w/{workspace}/postgres_triggers/update/{path}', + path: { + workspace: data.workspace, + path: data.path + }, + body: data.requestBody, + mediaType: 'application/json' +}); }; + +/** + * delete postgres trigger + * @param data The data for the request. + * @param data.workspace + * @param data.path + * @returns string postgres trigger deleted + * @throws ApiError + */ +export const deletePostgresTrigger = (data: DeletePostgresTriggerData): CancelablePromise => { return __request(OpenAPI, { + method: 'DELETE', + url: '/w/{workspace}/postgres_triggers/delete/{path}', + path: { + workspace: data.workspace, + path: data.path + } +}); }; + +/** + * get postgres trigger + * @param data The data for the request. + * @param data.workspace + * @param data.path + * @returns PostgresTrigger get postgres trigger + * @throws ApiError + */ +export const getPostgresTrigger = (data: GetPostgresTriggerData): CancelablePromise => { return __request(OpenAPI, { + method: 'GET', + url: '/w/{workspace}/postgres_triggers/get/{path}', + path: { + workspace: data.workspace, + path: data.path + } +}); }; + +/** + * list postgres triggers + * @param data The data for the request. + * @param data.workspace + * @param data.page which page to return (start at 1, default 1) + * @param data.perPage number of items to return for a given page (default 30, max 100) + * @param data.path filter by path + * @param data.isFlow + * @param data.pathStart + * @returns PostgresTrigger postgres trigger list + * @throws ApiError + */ +export const listPostgresTriggers = (data: ListPostgresTriggersData): CancelablePromise => { return __request(OpenAPI, { + method: 'GET', + url: '/w/{workspace}/postgres_triggers/list', + path: { + workspace: data.workspace + }, + query: { + page: data.page, + per_page: data.perPage, + path: data.path, + is_flow: data.isFlow, + path_start: data.pathStart + } +}); }; + +/** + * does postgres trigger exists + * @param data The data for the request. + * @param data.workspace + * @param data.path + * @returns boolean postgres trigger exists + * @throws ApiError + */ +export const existsPostgresTrigger = (data: ExistsPostgresTriggerData): CancelablePromise => { return __request(OpenAPI, { + method: 'GET', + url: '/w/{workspace}/postgres_triggers/exists/{path}', + path: { + workspace: data.workspace, + path: data.path + } +}); }; + +/** + * set enabled postgres trigger + * @param data The data for the request. + * @param data.workspace + * @param data.path + * @param data.requestBody updated postgres trigger enable + * @returns string postgres trigger enabled set + * @throws ApiError + */ +export const setPostgresTriggerEnabled = (data: SetPostgresTriggerEnabledData): CancelablePromise => { return __request(OpenAPI, { + method: 'POST', + url: '/w/{workspace}/postgres_triggers/setenabled/{path}', + path: { + workspace: data.workspace, + path: data.path + }, + body: data.requestBody, + mediaType: 'application/json' +}); }; + /** * list instance groups * @returns InstanceGroup instance group list @@ -6746,6 +7173,23 @@ export const listCaptures = (data: ListCapturesData): CancelablePromise => { return __request(OpenAPI, { + method: 'GET', + url: '/w/{workspace}/capture/{id}', + path: { + workspace: data.workspace, + id: data.id + } +}); }; + /** * delete a capture * @param data The data for the request. diff --git a/cli/gen/types.gen.ts b/cli/gen/types.gen.ts index ee075449a9..075aa848ee 100644 --- a/cli/gen/types.gen.ts +++ b/cli/gen/types.gen.ts @@ -1,8 +1,10 @@ // This file is auto-generated by @hey-api/openapi-ts -export type AiResource = { +export type AIProvider = 'openai' | 'anthropic' | 'mistral' | 'deepseek' | 'groq' | 'openrouter' | 'customai'; + +export type AIResource = { path: string; - provider: string; + provider: AIProvider; }; export type Script = { @@ -30,7 +32,7 @@ export type Script = { }; lock?: string; lock_error_logs?: string; - language: 'python3' | 'deno' | 'go' | 'bash' | 'powershell' | 'postgresql' | 'mysql' | 'bigquery' | 'snowflake' | 'mssql' | 'oracledb' | 'graphql' | 'nativets' | 'bun' | 'php' | 'rust' | 'ansible' | 'csharp'; + language: ScriptLang; kind: 'script' | 'failure' | 'trigger' | 'command' | 'approval' | 'preprocessor'; starred: boolean; tag?: string; @@ -54,8 +56,6 @@ export type Script = { on_behalf_of_email?: string; }; -export type language = 'python3' | 'deno' | 'go' | 'bash' | 'powershell' | 'postgresql' | 'mysql' | 'bigquery' | 'snowflake' | 'mssql' | 'oracledb' | 'graphql' | 'nativets' | 'bun' | 'php' | 'rust' | 'ansible' | 'csharp'; - export type kind = 'script' | 'failure' | 'trigger' | 'command' | 'approval' | 'preprocessor'; export type NewScript = { @@ -69,7 +69,7 @@ export type NewScript = { }; is_template?: boolean; lock?: string; - language: 'python3' | 'deno' | 'go' | 'bash' | 'powershell' | 'postgresql' | 'mysql' | 'bigquery' | 'snowflake' | 'mssql' | 'oracledb' | 'graphql' | 'nativets' | 'bun' | 'php' | 'rust' | 'ansible' | 'csharp'; + language: ScriptLang; kind?: 'script' | 'failure' | 'trigger' | 'command' | 'approval' | 'preprocessor'; tag?: string; draft_only?: boolean; @@ -159,7 +159,7 @@ export type QueuedJob = { flow_status?: FlowStatus; raw_flow?: FlowValue; is_flow_step: boolean; - language?: 'python3' | 'deno' | 'go' | 'bash' | 'powershell' | 'postgresql' | 'mysql' | 'bigquery' | 'snowflake' | 'mssql' | 'oracledb' | 'graphql' | 'nativets' | 'bun' | 'php' | 'rust' | 'ansible' | 'csharp'; + language?: ScriptLang; email: string; visible_to_owner: boolean; mem_peak?: number; @@ -168,6 +168,7 @@ export type QueuedJob = { self_wait_time_ms?: number; aggregate_wait_time_ms?: number; suspend?: number; + preprocessed?: boolean; }; export type job_kind = 'script' | 'preview' | 'dependencies' | 'flowdependencies' | 'appdependencies' | 'flow' | 'flowpreview' | 'script_hub' | 'identity' | 'deploymentcallback' | 'singlescriptflow' | 'flowscript' | 'flownode' | 'appscript'; @@ -202,7 +203,7 @@ export type CompletedJob = { flow_status?: FlowStatus; raw_flow?: FlowValue; is_flow_step: boolean; - language?: 'python3' | 'deno' | 'go' | 'bash' | 'powershell' | 'postgresql' | 'mysql' | 'bigquery' | 'snowflake' | 'mssql' | 'oracledb' | 'graphql' | 'nativets' | 'bun' | 'php' | 'rust' | 'ansible' | 'csharp'; + language?: ScriptLang; is_skipped: boolean; email: string; visible_to_owner: boolean; @@ -212,6 +213,7 @@ export type CompletedJob = { labels?: Array<(string)>; self_wait_time_ms?: number; aggregate_wait_time_ms?: number; + preprocessed?: boolean; }; export type ObscuredJob = { @@ -372,11 +374,13 @@ export type MainArgSignature = { export type type2 = 'Valid' | 'Invalid'; +export type ScriptLang = 'python3' | 'deno' | 'go' | 'bash' | 'powershell' | 'postgresql' | 'mysql' | 'bigquery' | 'snowflake' | 'mssql' | 'oracledb' | 'graphql' | 'nativets' | 'bun' | 'php' | 'rust' | 'ansible' | 'csharp'; + export type Preview = { content?: string; path?: string; args: ScriptArgs; - language?: 'python3' | 'deno' | 'go' | 'bash' | 'powershell' | 'postgresql' | 'mysql' | 'bigquery' | 'snowflake' | 'mssql' | 'oracledb' | 'graphql' | 'nativets' | 'bun' | 'php' | 'rust' | 'ansible' | 'csharp'; + language?: ScriptLang; tag?: string; kind?: 'code' | 'identity' | 'http'; dedicated_worker?: boolean; @@ -550,10 +554,18 @@ export type EditSchedule = { cron_version?: string; }; -export type HttpTrigger = { - path: string; +export type TriggerExtraProperty = { + email: string; + extra_perms: { + [key: string]: (boolean); + }; + workspace_id: string; edited_by: string; edited_at: string; +}; + +export type HttpTrigger = TriggerExtraProperty & { + path: string; script_path: string; route_path: string; static_asset_config?: { @@ -562,14 +574,10 @@ export type HttpTrigger = { filename?: string; }; is_flow: boolean; - extra_perms: { - [key: string]: (boolean); - }; - email: string; - workspace_id: string; http_method: 'get' | 'post' | 'put' | 'delete' | 'patch'; is_async: boolean; requires_auth: boolean; + is_static_website: boolean; }; export type http_method = 'get' | 'post' | 'put' | 'delete' | 'patch'; @@ -587,6 +595,7 @@ export type NewHttpTrigger = { http_method: 'get' | 'post' | 'put' | 'delete' | 'patch'; is_async: boolean; requires_auth: boolean; + is_static_website: boolean; }; export type EditHttpTrigger = { @@ -602,6 +611,7 @@ export type EditHttpTrigger = { http_method: 'get' | 'post' | 'put' | 'delete' | 'patch'; is_async: boolean; requires_auth: boolean; + is_static_website: boolean; }; export type TriggersCount = { @@ -613,22 +623,16 @@ export type TriggersCount = { webhook_count?: number; email_count?: number; websocket_count?: number; + postgres_count?: number; kafka_count?: number; nats_count?: number; }; -export type WebsocketTrigger = { +export type WebsocketTrigger = TriggerExtraProperty & { path: string; - edited_by: string; - edited_at: string; script_path: string; url: string; is_flow: boolean; - extra_perms: { - [key: string]: (boolean); - }; - email: string; - workspace_id: string; server_id?: string; last_server_ping?: string; error?: string; @@ -639,6 +643,7 @@ export type WebsocketTrigger = { }>; initial_messages?: Array; url_runnable_args?: ScriptArgs; + can_return_message: boolean; }; export type NewWebsocketTrigger = { @@ -653,6 +658,7 @@ export type NewWebsocketTrigger = { }>; initial_messages?: Array; url_runnable_args?: ScriptArgs; + can_return_message: boolean; }; export type EditWebsocketTrigger = { @@ -666,6 +672,7 @@ export type EditWebsocketTrigger = { }>; initial_messages?: Array; url_runnable_args?: ScriptArgs; + can_return_message: boolean; }; export type WebsocketTriggerInitialMessage = { @@ -678,6 +685,74 @@ export type WebsocketTriggerInitialMessage = { }; }; +export type Slot = { + name?: string; +}; + +export type SlotList = { + slot_name?: string; + active?: boolean; +}; + +export type PublicationData = { + table_to_track?: Array; + transaction_to_track: Array<(string)>; +}; + +export type TableToTrack = Array<{ + table_name: string; + columns_name?: Array<(string)>; + where_clause?: string; +}>; + +export type Relations = { + schema_name: string; + table_to_track: TableToTrack; +}; + +export type Language = 'Typescript'; + +export type TemplateScript = { + postgres_resource_path: string; + relations: Array; + language: Language; +}; + +export type PostgresTrigger = TriggerExtraProperty & { + path: string; + script_path: string; + is_flow: boolean; + enabled: boolean; + postgres_resource_path: string; + publication_name: string; + server_id?: string; + replication_slot_name: string; + error?: string; + last_server_ping?: string; +}; + +export type NewPostgresTrigger = { + replication_slot_name?: string; + publication_name?: string; + path: string; + script_path: string; + is_flow: boolean; + enabled: boolean; + postgres_resource_path: string; + publication?: PublicationData; +}; + +export type EditPostgresTrigger = { + replication_slot_name: string; + publication_name: string; + path: string; + script_path: string; + is_flow: boolean; + enabled: boolean; + postgres_resource_path: string; + publication?: PublicationData; +}; + export type KafkaTrigger = { path: string; edited_by: string; @@ -817,6 +892,7 @@ export type UserWorkspaceList = { name: string; username: string; color: string; + operator_settings?: OperatorSettings; }>; }; @@ -1111,7 +1187,7 @@ export type MetricDataPoint = { export type RawScriptForDependencies = { raw_code: string; path: string; - language: 'python3' | 'deno' | 'go' | 'bash' | 'powershell' | 'postgresql' | 'mysql' | 'bigquery' | 'snowflake' | 'mssql' | 'oracledb' | 'graphql' | 'nativets' | 'bun' | 'php' | 'rust' | 'ansible' | 'csharp'; + language: ScriptLang; }; export type ConcurrencyGroup = { @@ -1223,6 +1299,79 @@ export type CaptureConfig = { last_server_ping?: string; }; +export type OperatorSettings = { + /** + * Whether operators can view runs + */ + runs: boolean; + /** + * Whether operators can view schedules + */ + schedules: boolean; + /** + * Whether operators can view resources + */ + resources: boolean; + /** + * Whether operators can view variables + */ + variables: boolean; + /** + * Whether operators can view audit logs + */ + audit_logs: boolean; + /** + * Whether operators can view triggers + */ + triggers: boolean; + /** + * Whether operators can view groups page + */ + groups: boolean; + /** + * Whether operators can view folders page + */ + folders: boolean; + /** + * Whether operators can view workers page + */ + workers: boolean; +} | null; + +export type TeamInfo = { + /** + * The unique identifier of the Microsoft Teams team + */ + team_id: string; + /** + * The display name of the Microsoft Teams team + */ + team_name: string; + /** + * List of channels within the team + */ + channels: Array; +}; + +export type ChannelInfo = { + /** + * The unique identifier of the channel + */ + channel_id: string; + /** + * The display name of the channel + */ + channel_name: string; + /** + * The Microsoft Teams tenant identifier + */ + tenant_id: string; + /** + * The service URL for the channel + */ + service_url: string; +}; + export type OpenFlow = { summary: string; description?: string; @@ -1331,7 +1480,7 @@ export type RawScript = { is_trigger?: boolean; }; -export type language2 = 'deno' | 'bun' | 'python3' | 'go' | 'bash' | 'powershell' | 'postgresql' | 'mysql' | 'bigquery' | 'snowflake' | 'mssql' | 'oracledb' | 'graphql' | 'nativets' | 'php'; +export type language = 'deno' | 'bun' | 'python3' | 'go' | 'bash' | 'powershell' | 'postgresql' | 'mysql' | 'bigquery' | 'snowflake' | 'mssql' | 'oracledb' | 'graphql' | 'nativets' | 'php'; export type PathScript = { input_transforms: { @@ -1441,10 +1590,14 @@ export type FlowStatusModule = { export type type4 = 'WaitingForPriorSteps' | 'WaitingForEvents' | 'WaitingForExecutor' | 'InProgress' | 'Success' | 'Failure'; +export type ParameterId = string; + export type ParameterKey = string; export type ParameterWorkspaceId = string; +export type ParameterPublicationName = string; + export type ParameterVersionId = number; export type ParameterToken = string; @@ -2239,6 +2392,13 @@ export type WhoisData = { export type WhoisResponse = (User); +export type UpdateOperatorSettingsData = { + requestBody: OperatorSettings; + workspace: string; +}; + +export type UpdateOperatorSettingsResponse = (string); + export type ExistsEmailData = { email: string; }; @@ -2285,8 +2445,9 @@ export type GetSettingsResponse = ({ customer_id?: string; webhook?: string; deploy_to?: string; - ai_resource?: AiResource; - code_completion_enabled: boolean; + ai_resource?: AIResource; + code_completion_model?: string; + ai_models: Array<(string)>; error_handler?: string; error_handler_extra_args?: ScriptArgs; error_handler_muted_on_cancel: boolean; @@ -2297,6 +2458,7 @@ export type GetSettingsResponse = ({ default_scripts?: WorkspaceDefaultScripts; mute_critical_alerts?: boolean; color?: string; + operator_settings?: OperatorSettings; }); export type GetDeployToData = { @@ -2427,8 +2589,9 @@ export type EditCopilotConfigData = { * WorkspaceCopilotConfig */ requestBody: { - ai_resource?: AiResource; - code_completion_enabled: boolean; + ai_resource?: AIResource; + code_completion_model?: string; + ai_models: Array<(string)>; }; workspace: string; }; @@ -2440,9 +2603,10 @@ export type GetCopilotInfoData = { }; export type GetCopilotInfoResponse = ({ - ai_provider: string; + ai_provider?: AIProvider; exists_ai_resource: boolean; - code_completion_enabled: boolean; + code_completion_model?: string; + ai_models: Array<(string)>; }); export type EditErrorHandlerData = { @@ -2586,6 +2750,7 @@ export type GetUsedTriggersResponse = ({ websocket_used: boolean; kafka_used: boolean; nats_used: boolean; + postgres_used: boolean; }); export type ListUsersData = { @@ -2916,6 +3081,8 @@ export type GetOauthConnectResponse = ({ scopes?: Array<(string)>; }); +export type SyncTeamsResponse = (Array); + export type CreateResourceData = { /** * new resource @@ -5409,6 +5576,7 @@ export type ExistsRouteData = { requestBody: { route_path: string; http_method: 'get' | 'post' | 'put' | 'delete' | 'patch'; + trigger_path?: string; }; workspace: string; }; @@ -5490,6 +5658,20 @@ export type SetWebsocketTriggerEnabledData = { export type SetWebsocketTriggerEnabledResponse = (string); +export type TestWebsocketConnectionData = { + /** + * test websocket connection + */ + requestBody: { + url: string; + url_runnable_args?: ScriptArgs; + can_return_message: boolean; + }; + workspace: string; +}; + +export type TestWebsocketConnectionResponse = (string); + export type CreateKafkaTriggerData = { /** * new kafka trigger @@ -5565,6 +5747,20 @@ export type SetKafkaTriggerEnabledData = { export type SetKafkaTriggerEnabledResponse = (string); +export type TestKafkaConnectionData = { + /** + * test kafka connection + */ + requestBody: { + connection: { + [key: string]: unknown; + }; + }; + workspace: string; +}; + +export type TestKafkaConnectionResponse = (string); + export type CreateNatsTriggerData = { /** * new nats trigger @@ -5640,6 +5836,195 @@ export type SetNatsTriggerEnabledData = { export type SetNatsTriggerEnabledResponse = (string); +export type TestNatsConnectionData = { + /** + * test nats connection + */ + requestBody: { + connection: { + [key: string]: unknown; + }; + }; + workspace: string; +}; + +export type TestNatsConnectionResponse = (string); + +export type IsValidPostgresConfigurationData = { + path: string; + workspace: string; +}; + +export type IsValidPostgresConfigurationResponse = (boolean); + +export type CreateTemplateScriptData = { + /** + * template script + */ + requestBody: TemplateScript; + workspace: string; +}; + +export type CreateTemplateScriptResponse = (string); + +export type GetTemplateScriptData = { + id: string; + workspace: string; +}; + +export type GetTemplateScriptResponse = (string); + +export type ListPostgresReplicationSlotData = { + path: string; + workspace: string; +}; + +export type ListPostgresReplicationSlotResponse = (Array); + +export type CreatePostgresReplicationSlotData = { + path: string; + /** + * new slot for postgres + */ + requestBody: Slot; + workspace: string; +}; + +export type CreatePostgresReplicationSlotResponse = (string); + +export type DeletePostgresReplicationSlotData = { + path: string; + /** + * replication slot of postgres + */ + requestBody: Slot; + workspace: string; +}; + +export type DeletePostgresReplicationSlotResponse = (string); + +export type ListPostgresPublicationData = { + path: string; + workspace: string; +}; + +export type ListPostgresPublicationResponse = (Array<(string)>); + +export type GetPostgresPublicationData = { + path: string; + publication: string; + workspace: string; +}; + +export type GetPostgresPublicationResponse = (PublicationData); + +export type CreatePostgresPublicationData = { + path: string; + publication: string; + /** + * new publication for postgres + */ + requestBody: PublicationData; + workspace: string; +}; + +export type CreatePostgresPublicationResponse = (string); + +export type UpdatePostgresPublicationData = { + path: string; + publication: string; + /** + * update publication for postgres + */ + requestBody: PublicationData; + workspace: string; +}; + +export type UpdatePostgresPublicationResponse = (string); + +export type DeletePostgresPublicationData = { + path: string; + publication: string; + workspace: string; +}; + +export type DeletePostgresPublicationResponse = (string); + +export type CreatePostgresTriggerData = { + /** + * new postgres trigger + */ + requestBody: NewPostgresTrigger; + workspace: string; +}; + +export type CreatePostgresTriggerResponse = (string); + +export type UpdatePostgresTriggerData = { + path: string; + /** + * updated trigger + */ + requestBody: EditPostgresTrigger; + workspace: string; +}; + +export type UpdatePostgresTriggerResponse = (string); + +export type DeletePostgresTriggerData = { + path: string; + workspace: string; +}; + +export type DeletePostgresTriggerResponse = (string); + +export type GetPostgresTriggerData = { + path: string; + workspace: string; +}; + +export type GetPostgresTriggerResponse = (PostgresTrigger); + +export type ListPostgresTriggersData = { + isFlow?: boolean; + /** + * which page to return (start at 1, default 1) + */ + page?: number; + /** + * filter by path + */ + path?: string; + pathStart?: string; + /** + * number of items to return for a given page (default 30, max 100) + */ + perPage?: number; + workspace: string; +}; + +export type ListPostgresTriggersResponse = (Array); + +export type ExistsPostgresTriggerData = { + path: string; + workspace: string; +}; + +export type ExistsPostgresTriggerResponse = (boolean); + +export type SetPostgresTriggerEnabledData = { + path: string; + /** + * updated postgres trigger enable + */ + requestBody: { + enabled: boolean; + }; + workspace: string; +}; + +export type SetPostgresTriggerEnabledResponse = (string); + export type ListInstanceGroupsResponse = (Array); export type GetInstanceGroupData = { @@ -5987,7 +6372,7 @@ export type ListAutoscalingEventsData = { export type ListAutoscalingEventsResponse = (Array); export type GetGranularAclsData = { - kind: 'script' | 'group_' | 'resource' | 'schedule' | 'variable' | 'flow' | 'folder' | 'app' | 'raw_app' | 'http_trigger' | 'websocket_trigger' | 'kafka_trigger' | 'nats_trigger'; + kind: 'script' | 'group_' | 'resource' | 'schedule' | 'variable' | 'flow' | 'folder' | 'app' | 'raw_app' | 'http_trigger' | 'websocket_trigger' | 'kafka_trigger' | 'nats_trigger' | 'postgres_trigger'; path: string; workspace: string; }; @@ -5997,7 +6382,7 @@ export type GetGranularAclsResponse = ({ }); export type AddGranularAclsData = { - kind: 'script' | 'group_' | 'resource' | 'schedule' | 'variable' | 'flow' | 'folder' | 'app' | 'raw_app' | 'http_trigger' | 'websocket_trigger' | 'kafka_trigger' | 'nats_trigger'; + kind: 'script' | 'group_' | 'resource' | 'schedule' | 'variable' | 'flow' | 'folder' | 'app' | 'raw_app' | 'http_trigger' | 'websocket_trigger' | 'kafka_trigger' | 'nats_trigger' | 'postgres_trigger'; path: string; /** * acl to add @@ -6012,7 +6397,7 @@ export type AddGranularAclsData = { export type AddGranularAclsResponse = (string); export type RemoveGranularAclsData = { - kind: 'script' | 'group_' | 'resource' | 'schedule' | 'variable' | 'flow' | 'folder' | 'app' | 'raw_app' | 'http_trigger' | 'websocket_trigger' | 'kafka_trigger' | 'nats_trigger'; + kind: 'script' | 'group_' | 'resource' | 'schedule' | 'variable' | 'flow' | 'folder' | 'app' | 'raw_app' | 'http_trigger' | 'websocket_trigger' | 'kafka_trigger' | 'nats_trigger' | 'postgres_trigger'; path: string; /** * acl to add @@ -6076,6 +6461,13 @@ export type ListCapturesData = { export type ListCapturesResponse = (Array); +export type GetCaptureData = { + id: number; + workspace: string; +}; + +export type GetCaptureResponse = (Capture); + export type DeleteCaptureData = { id: number; workspace: string; @@ -6200,6 +6592,7 @@ export type DuckdbConnectionSettingsV2Data = { export type DuckdbConnectionSettingsV2Response = ({ connection_settings_str: string; + azure_container_path?: string; }); export type PolarsConnectionSettingsData = { diff --git a/cli/instance.ts b/cli/instance.ts index e80db19700..5a68f29c30 100644 --- a/cli/instance.ts +++ b/cli/instance.ts @@ -363,6 +363,7 @@ async function instancePull(opts: InstanceSyncOptions) { baseUrl: undefined, includeGroups: true, includeSchedules: true, + includeTriggers: true, includeSettings: true, includeUsers: true, includeKey: true, @@ -529,6 +530,7 @@ async function instancePush(opts: InstanceSyncOptions) { baseUrl: undefined, includeGroups: true, includeSchedules: true, + includeTriggers: true, includeSettings: true, includeUsers: true, includeKey: true, diff --git a/cli/main.ts b/cli/main.ts index f02dbbe841..aad1a2ec83 100644 --- a/cli/main.ts +++ b/cli/main.ts @@ -17,6 +17,7 @@ import variable from "./variable.ts"; import hub from "./hub.ts"; import folder from "./folder.ts"; import schedule from "./schedule.ts"; +import trigger from "./trigger.ts"; import sync from "./sync.ts"; import instance from "./instance.ts"; import workerGroups from "./worker_groups.ts"; @@ -44,6 +45,7 @@ export { hub, folder, schedule, + trigger, sync, instance, dev, @@ -60,7 +62,7 @@ export { // } // }); -export const VERSION = "1.456.0"; +export const VERSION = "1.458.4"; const command = new Command() .name("wmill") @@ -108,6 +110,7 @@ const command = new Command() skipResources: true, skipSecrets: true, includeSchedules: false, + includeTriggers: false, }) ); log.info(colors.green("wmill.yaml created")); @@ -122,6 +125,7 @@ const command = new Command() .command("hub", hub) .command("folder", folder) .command("schedule", schedule) + .command("trigger", trigger) .command("dev", dev) .command("sync", sync) .command("instance", instance) diff --git a/cli/pull.ts b/cli/pull.ts index 769f46bf0b..51474f9eb8 100644 --- a/cli/pull.ts +++ b/cli/pull.ts @@ -11,6 +11,7 @@ export async function downloadZip( skipResources?: boolean, skipSecrets?: boolean, includeSchedules?: boolean, + includeTriggers?: boolean, includeUsers?: boolean, includeGroups?: boolean, includeSettings?: boolean, @@ -38,11 +39,11 @@ export async function downloadZip( skipResources ?? false }&skip_secrets=${skipSecrets ?? false}&include_schedules=${ includeSchedules ?? false - }&include_users=${includeUsers ?? false}&include_groups=${ - includeGroups ?? false - }&include_settings=${includeSettings ?? false}&include_key=${ - includeKey ?? false - }&default_ts=${defaultTs ?? "bun"}`, + }&include_triggers=${includeTriggers ?? false}&include_users=${ + includeUsers ?? false + }&include_groups=${includeGroups ?? false}&include_settings=${ + includeSettings ?? false + }&include_key=${includeKey ?? false}&default_ts=${defaultTs ?? "bun"}`, { headers: requestHeaders, method: "GET", diff --git a/cli/settings.ts b/cli/settings.ts index 4952b7ac7a..054763a16b 100644 --- a/cli/settings.ts +++ b/cli/settings.ts @@ -1,15 +1,11 @@ -import { yamlStringify } from "./deps.ts"; -import { Confirm } from "./deps.ts"; -import { colors } from "./deps.ts"; -import { yamlParseFile } from "./deps.ts"; -import { log } from "./deps.ts"; +import process from "node:process"; +import { colors, Confirm, log, yamlParseFile, yamlStringify } from "./deps.ts"; +import * as wmill from "./gen/services.gen.ts"; +import { AIResource, Config, GlobalSetting } from "./gen/types.gen.ts"; import { compareInstanceObjects, InstanceSyncOptions } from "./instance.ts"; import { isSuperset } from "./types.ts"; import { deepEqual } from "./utils.ts"; -import * as wmill from "./gen/services.gen.ts"; -import { AiResource, Config, GlobalSetting } from "./gen/types.gen.ts"; import { removeWorkerPrefix } from "./worker_groups.ts"; -import process from "node:process"; export interface SimplifiedSettings { // slack_team_id?: string; @@ -24,13 +20,17 @@ export interface SimplifiedSettings { error_handler?: string; error_handler_extra_args?: any; error_handler_muted_on_cancel?: boolean; - ai_resource?: AiResource; - code_completion_enabled: boolean; + ai_resource?: AIResource; + code_completion_model?: string; + ai_models: string[]; large_file_storage?: any; git_sync?: any; default_app?: string; default_scripts?: any; name: string; + mute_critical_alerts?: boolean; + color?: string; + operator_settings?: any; } const INSTANCE_SETTINGS_PATH = "instance_settings.yaml"; @@ -81,12 +81,16 @@ export async function pushWorkspaceSettings( error_handler_muted_on_cancel: remoteSettings.error_handler_muted_on_cancel, ai_resource: remoteSettings.ai_resource, - code_completion_enabled: remoteSettings.code_completion_enabled, + code_completion_model: remoteSettings.code_completion_model, + ai_models: remoteSettings.ai_models, large_file_storage: remoteSettings.large_file_storage, git_sync: remoteSettings.git_sync, default_app: remoteSettings.default_app, default_scripts: remoteSettings.default_scripts, name: workspaceName, + mute_critical_alerts: remoteSettings.mute_critical_alerts, + color: remoteSettings.color, + operator_settings: remoteSettings.operator_settings, }; } catch (err) { throw new Error(`Failed to get workspace settings: ${err}`); @@ -153,25 +157,27 @@ export async function pushWorkspaceSettings( } } if ( - localSettings.ai_resource !== settings.ai_resource || - localSettings.code_completion_enabled !== settings.code_completion_enabled + localSettings.ai_resource != settings.ai_resource || + localSettings.code_completion_model != settings.code_completion_model || + !deepEqual(localSettings.ai_models, settings.ai_models) ) { - log.debug(`Updating openai settings...`); + log.debug(`Updating copilot settings...`); await wmill.editCopilotConfig({ workspace, requestBody: { ai_resource: localSettings.ai_resource, - code_completion_enabled: localSettings.code_completion_enabled, + code_completion_model: localSettings.code_completion_model, + ai_models: localSettings.ai_models, }, }); } if ( - localSettings.error_handler !== settings.error_handler || + localSettings.error_handler != settings.error_handler || !deepEqual( localSettings.error_handler_extra_args, settings.error_handler_extra_args ) || - localSettings.error_handler_muted_on_cancel !== + localSettings.error_handler_muted_on_cancel != settings.error_handler_muted_on_cancel ) { log.debug(`Updating error handler...`); @@ -185,7 +191,7 @@ export async function pushWorkspaceSettings( }, }); } - if (localSettings.deploy_to !== settings.deploy_to) { + if (localSettings.deploy_to != settings.deploy_to) { log.debug(`Updating deploy to...`); await wmill.editDeployTo({ workspace, @@ -221,7 +227,7 @@ export async function pushWorkspaceSettings( requestBody: localSettings.default_scripts, }); } - if (localSettings.default_app !== settings.default_app) { + if (localSettings.default_app != settings.default_app) { log.debug(`Updating default app...`); await wmill.editWorkspaceDefaultApp({ workspace, @@ -231,7 +237,7 @@ export async function pushWorkspaceSettings( }); } - if (localSettings.name !== settings.name) { + if (localSettings.name != settings.name) { log.debug(`Updating workspace name...`); await wmill.changeWorkspaceName({ workspace, @@ -240,6 +246,34 @@ export async function pushWorkspaceSettings( }, }); } + + if (localSettings.mute_critical_alerts != settings.mute_critical_alerts) { + log.debug(`Updating mute critical alerts...`); + await wmill.workspaceMuteCriticalAlertsUi({ + workspace, + requestBody: { + mute_critical_alerts: localSettings.mute_critical_alerts, + }, + }); + } + + if (localSettings.color != settings.color) { + log.debug(`Updating workspace color...`); + await wmill.changeWorkspaceColor({ + workspace, + requestBody: { + color: localSettings.color, + }, + }); + } + + if (localSettings.operator_settings != settings.operator_settings) { + log.debug(`Updating operator settings...`); + await wmill.updateOperatorSettings({ + workspace, + requestBody: localSettings.operator_settings, + }); + } } export async function pushWorkspaceKey( @@ -282,7 +316,9 @@ export async function readInstanceSettings(opts: InstanceSyncOptions) { await checkInstanceSettingsPath(opts); try { - localSettings = (await yamlParseFile(instanceSettingsPath)) as GlobalSetting[]; + localSettings = (await yamlParseFile( + instanceSettingsPath + )) as GlobalSetting[]; } catch { log.warn(`No ${instanceSettingsPath} found`); } @@ -445,9 +481,7 @@ export async function pushInstanceSettings( } } -export async function readLocalConfigs( - opts: InstanceSyncOptions -) { +export async function readLocalConfigs(opts: InstanceSyncOptions) { let localConfigs: Config[] = []; await checkInstanceConfigPath(opts); diff --git a/cli/sync.ts b/cli/sync.ts index b691a0d42c..e350b94fef 100644 --- a/cli/sync.ts +++ b/cli/sync.ts @@ -116,7 +116,7 @@ async function addCodebaseDigestIfRelevant( const parsed: any = yamlParseContent(path, content); if (parsed && typeof parsed == "object") { parsed["codebase"] = c.digest; - parsed["lock"] = ''; + parsed["lock"] = ""; return yamlStringify(parsed, yamlOptions); } else { throw Error( @@ -159,8 +159,12 @@ export async function FSFSElement( // }, async getContentText(): Promise { const content = await Deno.readTextFile(localP); - const itemPath = localP.substring(p.length + 1) - const r = await addCodebaseDigestIfRelevant(itemPath, content, codebases); + const itemPath = localP.substring(p.length + 1); + const r = await addCodebaseDigestIfRelevant( + itemPath, + content, + codebases + ); return r; }, }; @@ -352,12 +356,12 @@ function ZipFSElement( ) ? "flow" : p.endsWith("app.json") - ? "app" - : p.endsWith("script.json") - ? "script" - : p.endsWith("resource.json") - ? "resource" - : "other"; + ? "app" + : p.endsWith("script.json") + ? "script" + : p.endsWith("resource.json") + ? "resource" + : "other"; const isJson = p.endsWith(".json"); @@ -387,7 +391,7 @@ function ZipFSElement( yield { isDirectory: false, path: path.join(finalPath, s.path), - async *getChildren() { }, + async *getChildren() {}, // deno-lint-ignore require-await async getContentText() { return s.content; @@ -398,7 +402,7 @@ function ZipFSElement( yield { isDirectory: false, path: path.join(finalPath, "flow.yaml"), - async *getChildren() { }, + async *getChildren() {}, // deno-lint-ignore require-await async getContentText() { return yamlStringify(flow, yamlOptions); @@ -414,7 +418,7 @@ function ZipFSElement( yield { isDirectory: false, path: path.join(finalPath, s.path), - async *getChildren() { }, + async *getChildren() {}, // deno-lint-ignore require-await async getContentText() { return s.content; @@ -425,7 +429,7 @@ function ZipFSElement( yield { isDirectory: false, path: path.join(finalPath, "app.yaml"), - async *getChildren() { }, + async *getChildren() {}, // deno-lint-ignore require-await async getContentText() { return yamlStringify(app, yamlOptions); @@ -490,7 +494,7 @@ function ZipFSElement( r.push({ isDirectory: false, path: removeSuffix(finalPath, ".json") + ".lock", - async *getChildren() { }, + async *getChildren() {}, // deno-lint-ignore require-await async getContentText() { return lock; @@ -513,7 +517,7 @@ function ZipFSElement( removeSuffix(finalPath, ".resource.json") + ".resource.file." + formatExtension, - async *getChildren() { }, + async *getChildren() {}, // deno-lint-ignore require-await async getContentText() { return fileContent; @@ -574,19 +578,19 @@ export async function* readDirRecursiveWithIgnore( // getContentBytes(): Promise; getContentText(): Promise; }[] = [ - { - path: root.path, - ignored: ignore(root.path, root.isDirectory), - isDirectory: root.isDirectory, - c: root.getChildren, - // getContentBytes(): Promise { - // throw undefined; - // }, - getContentText(): Promise { - throw undefined; - }, + { + path: root.path, + ignored: ignore(root.path, root.isDirectory), + isDirectory: root.isDirectory, + c: root.getChildren, + // getContentBytes(): Promise { + // throw undefined; + // }, + getContentText(): Promise { + throw undefined; }, - ]; + }, + ]; while (stack.length > 0) { const e = stack.pop()!; @@ -606,7 +610,13 @@ export async function* readDirRecursiveWithIgnore( type Added = { name: "added"; path: string; content: string }; type Deleted = { name: "deleted"; path: string }; -type Edit = { name: "edited"; path: string; before: string; after: string; codebase?: string }; +type Edit = { + name: "edited"; + path: string; + before: string; + after: string; + codebase?: string; +}; type Change = Added | Deleted | Edit; @@ -624,6 +634,15 @@ export async function elementsToMap( if (!json && path.endsWith(".json") && !isFileResource(path)) continue; const ext = json ? ".json" : ".yaml"; if (!skips.includeSchedules && path.endsWith(".schedule" + ext)) continue; + if ( + !skips.includeTriggers && + (path.endsWith(".http_trigger" + ext) || + path.endsWith(".websocket_trigger" + ext) || + path.endsWith(".kafka_trigger" + ext) || + path.endsWith(".nats_trigger" + ext) || + path.endsWith(".postgres_trigger" + ext)) + ) + continue; if (!skips.includeUsers && path.endsWith(".user" + ext)) continue; if (!skips.includeGroups && path.endsWith(".group" + ext)) continue; if (!skips.includeSettings && path === "settings" + ext) continue; @@ -683,6 +702,7 @@ export interface Skips { skipSecrets?: boolean | undefined; skipScriptsMetadata?: boolean | undefined; includeSchedules?: boolean | undefined; + includeTriggers?: boolean | undefined; includeUsers?: boolean | undefined; includeGroups?: boolean | undefined; includeSettings?: boolean | undefined; @@ -700,9 +720,9 @@ async function compareDynFSElement( ): Promise { const [m1, m2] = els2 ? await Promise.all([ - elementsToMap(els1, ignore, json, skips), - elementsToMap(els2, ignore, json, skips), - ]) + elementsToMap(els1, ignore, json, skips), + elementsToMap(els2, ignore, json, skips), + ]) : [await elementsToMap(els1, ignore, json, skips), {}]; const changes: Change[] = []; @@ -738,7 +758,8 @@ async function compareDynFSElement( } const codebaseChanges: Record = {}; for (let [k, v] of Object.entries(m1)) { - const isScriptMetadata = k.endsWith(".script.yaml") || k.endsWith(".script.json"); + const isScriptMetadata = + k.endsWith(".script.yaml") || k.endsWith(".script.json"); const skipMetadata = skips.skipScriptsMetadata && isScriptMetadata; if (m2[k] === undefined) { @@ -749,8 +770,7 @@ async function compareDynFSElement( } else { if (m2[k] == v) { continue; - } - else if (k.endsWith(".json")) { + } else if (k.endsWith(".json")) { if (deepEqual(JSON.parse(v), JSON.parse(m2[k]))) { continue; } @@ -775,14 +795,23 @@ async function compareDynFSElement( continue; } } - changes.push({ name: "edited", path: k, after: v, before: m2[k], codebase: codebaseChanges[k] }); + changes.push({ + name: "edited", + path: k, + after: v, + before: m2[k], + codebase: codebaseChanges[k], + }); } } const remoteCodebase: Record = {}; for (const [k] of Object.entries(m2)) { if (m1[k] === undefined) { - if (!ignoreMetadataDeletion || (!k?.endsWith(".script.yaml") && !k?.endsWith(".script.json"))) { + if ( + !ignoreMetadataDeletion || + (!k?.endsWith(".script.yaml") && !k?.endsWith(".script.json")) + ) { changes.push({ name: "deleted", path: k }); } else if (k?.endsWith(".script.yaml")) { let o = parseYaml(k, m2[k]); @@ -795,12 +824,22 @@ async function compareDynFSElement( for (const [k, v] of Object.entries(remoteCodebase)) { const tsFile = k.replace(".script.yaml", ".ts"); - if (changes.find(c => c.path == tsFile && (c.name == "edited" || c.name == "deleted"))) { + if ( + changes.find( + (c) => c.path == tsFile && (c.name == "edited" || c.name == "deleted") + ) + ) { continue; } let c = findCodebase(tsFile, codebases); if (c?.digest != v) { - changes.push({ name: "edited", path: tsFile, codebase: v, before: m1[tsFile], after: m2[tsFile] }); + changes.push({ + name: "edited", + path: tsFile, + codebase: v, + before: m1[tsFile], + after: m2[tsFile], + }); } } @@ -841,16 +880,24 @@ function getOrderFromPath(p: string) { return 6; } else if (typ == "schedule") { return 7; - } else if (typ == "variable") { + } else if ( + typ == "http_trigger" || + typ == "websocket_trigger" || + typ == "kafka_trigger" || + typ == "nats_trigger" || + typ == "postgres_trigger" + ) { return 8; - } else if (typ == "user") { + } else if (typ == "variable") { return 9; - } else if (typ == "group") { + } else if (typ == "user") { return 10; - } else if (typ == "encryption_key") { + } else if (typ == "group") { return 11; - } else { + } else if (typ == "encryption_key") { return 12; + } else { + return 13; } } @@ -956,8 +1003,7 @@ async function addToChangedIfNotExists(p: string, tracker: ChangeTracker) { const isScript = exts.some((e) => p.endsWith(e)); if (isScript) { if (p.includes(".flow" + SEP)) { - const folder = - p.substring(0, p.indexOf(".flow" + SEP)) + ".flow" + SEP; + const folder = p.substring(0, p.indexOf(".flow" + SEP)) + ".flow" + SEP; if (!tracker.flows.includes(folder)) { tracker.flows.push(folder); } @@ -1031,6 +1077,7 @@ export async function pull(opts: GlobalOptions & SyncOptions) { opts.skipResources, opts.skipSecrets, opts.includeSchedules, + opts.includeTriggers, opts.includeUsers, opts.includeGroups, opts.includeSettings, @@ -1071,8 +1118,6 @@ export async function pull(opts: GlobalOptions & SyncOptions) { const conflicts = []; - - log.info(colors.gray(`Applying changes to files ...`)); for await (const change of changes) { const target = path.join(Deno.cwd(), change.path); @@ -1224,7 +1269,11 @@ function prettyChanges(changes: Change[]) { ); } else if (change.name === "edited") { log.info( - colors.yellow(`~ ${getTypeStrFromPath(change.path)} ` + change.path + (change.codebase ? ` (codebase changed)` : "")) + colors.yellow( + `~ ${getTypeStrFromPath(change.path)} ` + + change.path + + (change.codebase ? ` (codebase changed)` : "") + ) ); if (change.before != change.after) { showDiff(change.before, change.after); @@ -1300,6 +1349,7 @@ export async function push(opts: GlobalOptions & SyncOptions) { opts.skipResources, opts.skipSecrets, opts.includeSchedules, + opts.includeTriggers, opts.includeUsers, opts.includeGroups, opts.includeSettings, @@ -1322,7 +1372,6 @@ export async function push(opts: GlobalOptions & SyncOptions) { codebases ); - const globalDeps = await findGlobalDeps(); const tracker: ChangeTracker = await buildTracker(changes); @@ -1346,28 +1395,38 @@ export async function push(opts: GlobalOptions & SyncOptions) { } if (staleScripts.length > 0) { - log.info("") - log.warn("Stale scripts metadata found, you may want to update them using 'wmill script generate-metadata' before pushing:"); + log.info(""); + log.warn( + "Stale scripts metadata found, you may want to update them using 'wmill script generate-metadata' before pushing:" + ); for (const stale of staleScripts) { log.warn(stale); } - log.info("") + log.info(""); } for (const change of tracker.flows) { - const stale = await generateFlowLockInternal(change, true, workspace, false, true); + const stale = await generateFlowLockInternal( + change, + true, + workspace, + false, + true + ); if (stale) { staleFlows.push(stale); } } if (staleFlows.length > 0) { - log.warn("Stale flows locks found, you may want to update them using 'wmill flow generate-locks' before pushing:"); + log.warn( + "Stale flows locks found, you may want to update them using 'wmill flow generate-locks' before pushing:" + ); for (const stale of staleFlows) { log.warn(stale); } - log.info("") + log.info(""); } const version = await fetchVersion(workspace.remote); @@ -1393,7 +1452,6 @@ export async function push(opts: GlobalOptions & SyncOptions) { const start = performance.now(); log.info(colors.gray(`Applying changes to files ...`)); - let stateful = opts.stateful; if (stateful) { try { @@ -1406,21 +1464,26 @@ export async function push(opts: GlobalOptions & SyncOptions) { // Group changes by base path (before first dot) const groupedChanges = new Map(); for (const change of changes) { - const basePath = change.path.split('.')[0]; + const basePath = change.path.split(".")[0]; if (!groupedChanges.has(basePath)) { groupedChanges.set(basePath, []); } groupedChanges.get(basePath)!.push(change); } - - let parallelizationFactor = opts.parallel ?? 1; if (parallelizationFactor <= 0) { parallelizationFactor = 1; } const groupedChangesArray = Array.from(groupedChanges.entries()); - log.info(`found changes for ${groupedChangesArray.length} items with a total of ${groupedChangesArray.reduce((acc, [_, changes]) => acc + changes.length, 0)} files to process`); + log.info( + `found changes for ${ + groupedChangesArray.length + } items with a total of ${groupedChangesArray.reduce( + (acc, [_, changes]) => acc + changes.length, + 0 + )} files to process` + ); if (parallelizationFactor > 1) { log.info(`Parallelizing ${parallelizationFactor} changes at a time`); } @@ -1481,7 +1544,9 @@ export async function push(opts: GlobalOptions & SyncOptions) { } if (stateTarget) { await ensureDir(path.dirname(stateTarget)); - log.info(`Editing ${getTypeStrFromPath(change.path)} ${change.path}`); + log.info( + `Editing ${getTypeStrFromPath(change.path)} ${change.path}` + ); } if (isFileResource(change.path)) { @@ -1545,7 +1610,9 @@ export async function push(opts: GlobalOptions & SyncOptions) { } if (stateTarget) { await ensureDir(path.dirname(stateTarget)); - log.info(`Adding ${getTypeStrFromPath(change.path)} ${change.path}`); + log.info( + `Adding ${getTypeStrFromPath(change.path)} ${change.path}` + ); } const obj = parseFromPath(change.path, change.content); await pushObj( @@ -1622,6 +1689,36 @@ export async function push(opts: GlobalOptions & SyncOptions) { path: removeSuffix(target, ".schedule.json"), }); break; + case "http_trigger": + await wmill.deleteHttpTrigger({ + workspace: workspaceId, + path: removeSuffix(target, ".http_trigger.json"), + }); + break; + case "websocket_trigger": + await wmill.deleteWebsocketTrigger({ + workspace: workspaceId, + path: removeSuffix(target, ".websocket_trigger.json"), + }); + break; + case "kafka_trigger": + await wmill.deleteKafkaTrigger({ + workspace: workspaceId, + path: removeSuffix(target, ".kafka_trigger.json"), + }); + break; + case "nats_trigger": + await wmill.deleteNatsTrigger({ + workspace: workspaceId, + path: removeSuffix(target, ".nats_trigger.json"), + }); + break; + case "postgres_trigger": + await wmill.deletePostgresTrigger({ + workspace: workspaceId, + path: removeSuffix(target, ".postgres_trigger.json"), + }); + break; case "variable": await wmill.deleteVariable({ workspace: workspaceId, @@ -1682,7 +1779,9 @@ export async function push(opts: GlobalOptions & SyncOptions) { } log.info( colors.bold.green.underline( - `\nDone! All ${changes.length} changes pushed to the remote workspace ${workspace.workspaceId} named ${workspace.name} (${(performance.now() - start).toFixed(0)}ms)` + `\nDone! All ${changes.length} changes pushed to the remote workspace ${ + workspace.workspaceId + } named ${workspace.name} (${(performance.now() - start).toFixed(0)}ms)` ) ); } @@ -1706,6 +1805,7 @@ const command = new Command() .option("--skip-resources", "Skip syncing resources") // .option("--skip-scripts-metadata", "Skip syncing scripts metadata, focus solely on logic") .option("--include-schedules", "Include syncing schedules") + .option("--include-triggers", "Include syncing triggers") .option("--include-users", "Include syncing users") .option("--include-groups", "Include syncing groups") .option("--include-settings", "Include syncing workspace settings") @@ -1734,6 +1834,7 @@ const command = new Command() .option("--skip-resources", "Skip syncing resources") // .option("--skip-scripts-metadata", "Skip syncing scripts metadata, focus solely on logic") .option("--include-schedules", "Include syncing schedules") + .option("--include-triggers", "Include syncing triggers") .option("--include-users", "Include syncing users") .option("--include-groups", "Include syncing groups") .option("--include-settings", "Include syncing workspace settings") @@ -1754,10 +1855,7 @@ const command = new Command() "--message ", "Include a message that will be added to all scripts/flows/apps updated during this push" ) - .option( - "--parallel ", - "Number of changes to process in parallel" - ) + .option("--parallel ", "Number of changes to process in parallel") // deno-lint-ignore no-explicit-any .action(push as any); diff --git a/cli/trigger.ts b/cli/trigger.ts new file mode 100644 index 0000000000..9f0b9d887b --- /dev/null +++ b/cli/trigger.ts @@ -0,0 +1,245 @@ +import * as wmill from "./gen/services.gen.ts"; +import { + HttpTrigger, + KafkaTrigger, + NatsTrigger, + PostgresTrigger, + WebsocketTrigger, +} from "./gen/types.gen.ts"; +import { colors, Command, log, SEP, Table } from "./deps.ts"; +import { + GlobalOptions, + isSuperset, + parseFromFile, + removeType, +} from "./types.ts"; +import { requireLogin } from "./context.ts"; +import { validatePath } from "./context.ts"; +import { resolveWorkspace } from "./context.ts"; + +type Trigger = { + http: HttpTrigger; + websocket: WebsocketTrigger; + kafka: KafkaTrigger; + nats: NatsTrigger; + postgres: PostgresTrigger; +}; + +type TriggerFile = Omit< + Trigger[K], + | "path" + | "workspace" + | "edited_by" + | "edited_at" + | "error" + | "last_server_ping" + | "server_id" +>; + +type TriggerType = keyof Trigger; + +async function getTrigger( + triggerType: K, + workspace: string, + path: string +): Promise { + const triggerFunctions: { + [K in TriggerType]: (args: { + workspace: string; + path: string; + }) => Promise; + } = { + http: wmill.getHttpTrigger, + websocket: wmill.getWebsocketTrigger, + kafka: wmill.getKafkaTrigger, + nats: wmill.getNatsTrigger, + postgres: wmill.getPostgresTrigger, + }; + const triggerFunction = triggerFunctions[triggerType]; + + const trigger = await triggerFunction({ workspace, path }); + return trigger; +} + +async function updateTrigger( + triggerType: K, + workspace: string, + path: string, + trigger: Trigger[K] +): Promise { + const triggerFunctions: { + [K in TriggerType]: (args: { + workspace: string; + path: string; + requestBody: Trigger[K]; + }) => Promise; + } = { + http: wmill.updateHttpTrigger, + websocket: wmill.updateWebsocketTrigger, + kafka: wmill.updateKafkaTrigger, + nats: wmill.updateNatsTrigger, + postgres: wmill.updatePostgresTrigger, + }; + const triggerFunction = triggerFunctions[triggerType]; + await triggerFunction({ workspace, path, requestBody: trigger }); +} + +async function createTrigger( + triggerType: K, + workspace: string, + path: string, + trigger: Trigger[K] +): Promise { + const triggerFunctions: { + [K in TriggerType]: (args: { + workspace: string; + path: string; + requestBody: Trigger[K]; + }) => Promise; + } = { + http: wmill.createHttpTrigger, + websocket: wmill.createWebsocketTrigger, + kafka: wmill.createKafkaTrigger, + nats: wmill.createNatsTrigger, + postgres: wmill.createPostgresTrigger, + }; + const triggerFunction = triggerFunctions[triggerType]; + await triggerFunction({ workspace, path, requestBody: trigger }); +} + +export async function pushTrigger( + triggerType: K, + workspace: string, + path: string, + trigger: TriggerFile | Trigger[K] | undefined, + localTrigger: TriggerFile +): Promise { + path = removeType(path, triggerType + "_trigger").replaceAll(SEP, "/"); + log.debug(`Processing local ${triggerType} trigger ${path}`); + + try { + trigger = await getTrigger(triggerType, workspace, path); + log.debug(`${triggerType} trigger ${path} exists on remote`); + } catch { + log.debug(`${triggerType} trigger ${path} does not exist on remote`); + //ignore + } + + if (trigger) { + if (isSuperset(localTrigger, trigger)) { + log.debug(`${triggerType} trigger ${path} is up to date`); + return; + } + log.debug(`${triggerType} trigger ${path} is not up-to-date, updating...`); + try { + await updateTrigger(triggerType, workspace, path, { + ...localTrigger, + path, + } as Trigger[K]); + } catch (e) { + console.error(e.body); + throw e; + } + } else { + console.log( + colors.bold.yellow(`Creating new ${triggerType} trigger: ${path}`) + ); + try { + await createTrigger(triggerType, workspace, path, { + ...localTrigger, + path, + } as Trigger[K]); + } catch (e) { + console.error(e.body); + throw e; + } + } +} + +async function list(opts: GlobalOptions) { + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + + const httpTriggers = await wmill.listHttpTriggers({ + workspace: workspace.workspaceId, + }); + const websocketTriggers = await wmill.listWebsocketTriggers({ + workspace: workspace.workspaceId, + }); + const kafkaTriggers = await wmill.listKafkaTriggers({ + workspace: workspace.workspaceId, + }); + const natsTriggers = await wmill.listNatsTriggers({ + workspace: workspace.workspaceId, + }); + const postgresTriggers = await wmill.listPostgresTriggers({ + workspace: workspace.workspaceId, + }); + + const triggers = [ + ...httpTriggers.map((x) => ({ path: x.path, kind: "http" })), + ...websocketTriggers.map((x) => ({ path: x.path, kind: "websocket" })), + ...kafkaTriggers.map((x) => ({ path: x.path, kind: "kafka" })), + ...natsTriggers.map((x) => ({ path: x.path, kind: "nats" })), + ...postgresTriggers.map((x) => ({ path: x.path, kind: "postgres" })), + ]; + + new Table() + .header(["Path", "Kind"]) + .padding(2) + .border(true) + .body(triggers.map((x) => [x.path, x.kind])) + .render(); +} + +function checkIfValidTrigger(kind: string | undefined): kind is TriggerType { + if ( + kind && + ["http", "websocket", "kafka", "nats", "postgres"].includes(kind) + ) { + return true; + } else { + return false; + } +} + +async function push(opts: GlobalOptions, filePath: string, remotePath: string) { + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + + if (!validatePath(remotePath)) { + return; + } + + const fstat = await Deno.stat(filePath); + if (!fstat.isFile) { + throw new Error("file path must refer to a file."); + } + + console.log(colors.bold.yellow("Pushing trigger...")); + + const triggerKind = filePath.split(".")[1].split("_")[0]; + if (!checkIfValidTrigger(triggerKind)) { + throw new Error("Invalid trigger kind: " + triggerKind); + } + await pushTrigger( + triggerKind, + workspace.workspaceId, + remotePath, + undefined, + parseFromFile(filePath) + ); + console.log(colors.bold.underline.green("Trigger pushed")); +} + +const command = new Command() + .description("trigger related commands") + .action(list as any) + .command( + "push", + "push a local trigger spec. This overrides any remote versions." + ) + .arguments(" ") + .action(push as any); + +export default command; diff --git a/cli/types.ts b/cli/types.ts index 0ba5a1c476..c075ca51d3 100644 --- a/cli/types.ts +++ b/cli/types.ts @@ -22,6 +22,7 @@ import { pushSchedule } from "./schedule.ts"; import { pushWorkspaceUser } from "./user.ts"; import { pushGroup } from "./user.ts"; import { pushWorkspaceSettings, pushWorkspaceKey } from "./settings.ts"; +import { pushTrigger } from "./trigger.ts"; export interface DifferenceCreate { type: "CREATE"; @@ -80,7 +81,7 @@ export function showDiff(local: string, remote: string) { return; } - for (const part of Diff.diffLines(local ?? '', remote ?? '')) { + for (const part of Diff.diffLines(local ?? "", remote ?? "")) { if (part.removed) { // print red if removed without newline finalString += `\x1b[31m${part.value}\x1b[0m`; @@ -139,6 +140,16 @@ export async function pushObj( await pushResourceType(workspace, p, befObj, newObj); } else if (typeEnding === "schedule") { await pushSchedule(workspace, p, befObj, newObj); + } else if (typeEnding === "http_trigger") { + await pushTrigger("http", workspace, p, befObj, newObj); + } else if (typeEnding === "websocket_trigger") { + await pushTrigger("websocket", workspace, p, befObj, newObj); + } else if (typeEnding === "kafka_trigger") { + await pushTrigger("kafka", workspace, p, befObj, newObj); + } else if (typeEnding === "nats_trigger") { + await pushTrigger("nats", workspace, p, befObj, newObj); + } else if (typeEnding === "postgres_trigger") { + await pushTrigger("postgres", workspace, p, befObj, newObj); } else if (typeEnding === "user") { await pushWorkspaceUser(workspace, p, befObj, newObj); } else if (typeEnding === "group") { @@ -158,8 +169,8 @@ export function parseFromPath(p: string, content: string): any { return p.endsWith(".yaml") ? yamlParseContent(p, content) : p.endsWith(".json") - ? JSON.parse(content) - : content; + ? JSON.parse(content) + : content; } export function parseFromFile(p: string): any { if (p.endsWith(".json")) { @@ -181,6 +192,11 @@ export function getTypeStrFromPath( | "folder" | "app" | "schedule" + | "http_trigger" + | "websocket_trigger" + | "kafka_trigger" + | "nats_trigger" + | "postgres_trigger" | "user" | "group" | "settings" @@ -221,6 +237,11 @@ export function getTypeStrFromPath( typeEnding === "resource-type" || typeEnding === "app" || typeEnding === "schedule" || + typeEnding === "http_trigger" || + typeEnding === "websocket_trigger" || + typeEnding === "kafka_trigger" || + typeEnding === "nats_trigger" || + typeEnding === "postgres_trigger" || typeEnding === "user" || typeEnding === "group" || typeEnding === "settings" || diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 95f0c35a97..1d4f18629b 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.456.0", + "version": "1.458.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.456.0", + "version": "1.458.4", "license": "AGPL-3.0", "dependencies": { "@anthropic-ai/sdk": "^0.32.1", diff --git a/frontend/package.json b/frontend/package.json index 3ea5fd90b8..5f29a86923 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.456.0", + "version": "1.458.4", "scripts": { "dev": "vite dev", "build": "vite build", diff --git a/frontend/src/lib/components/AppConnectInner.svelte b/frontend/src/lib/components/AppConnectInner.svelte index fc644a3cd2..02136a91dc 100644 --- a/frontend/src/lib/components/AppConnectInner.svelte +++ b/frontend/src/lib/components/AppConnectInner.svelte @@ -25,6 +25,7 @@ import { Pen } from 'lucide-svelte' import GfmMarkdown from './GfmMarkdown.svelte' import { apiTokenApps, forceSecretValue, linkedSecretValue } from './app_connect' + import type { SchemaProperty } from '$lib/common' export let step = 1 export let resourceType = '' @@ -55,18 +56,20 @@ let args: any = {} let renderDescription = true - function computeCandidates(resourceType: string, argsKeys: string[]) { + function computeCandidates(resourceType: string, argsKeys: string[], passwords: string[]) { return apiTokenApps[resourceType]?.linkedSecret ? ([apiTokenApps[resourceType]?.linkedSecret] as string[]) - : argsKeys.filter((x) => - ['token', 'secret', 'key', 'pass', 'private'].some((y) => x.toLowerCase().includes(y)) + : argsKeys.filter( + (x) => + passwords.includes(x) || + ['token', 'secret', 'key', 'pass', 'private'].some((y) => x.toLowerCase().includes(y)) ) } let linkedSecret: string | undefined = undefined let linkedSecretCandidates: string[] | undefined = undefined - function computeLinkedSecret(resourceType: string, argsKeys: string[]) { - linkedSecretCandidates = computeCandidates(resourceType, argsKeys) + function computeLinkedSecret(resourceType: string, argsKeys: string[], passwords: string[]) { + linkedSecretCandidates = computeCandidates(resourceType, argsKeys, passwords) return ( forceSecretValue(resourceType) ?? linkedSecretCandidates?.sort((ua, ub) => linkedSecretValue(ub) - linkedSecretValue(ua))?.[0] @@ -240,9 +243,13 @@ workspace: $workspaceStore!, path: resourceType }) - const newArgsKeys = Object.keys(resourceTypeInfo?.schema?.['properties'] ?? {}) ?? [] + const props: Record = resourceTypeInfo?.schema?.['properties'] ?? {} + const newArgsKeys = Object.keys(props) ?? [] + const passwords = newArgsKeys.filter((x) => { + return props?.[x]?.password + }) if (!linkedSecret) { - linkedSecret = computeLinkedSecret(resourceType, newArgsKeys) + linkedSecret = computeLinkedSecret(resourceType, newArgsKeys, passwords) } } export async function next() { diff --git a/frontend/src/lib/components/ArgInput.svelte b/frontend/src/lib/components/ArgInput.svelte index 5ebb4b6cc2..f2d652286e 100644 --- a/frontend/src/lib/components/ArgInput.svelte +++ b/frontend/src/lib/components/ArgInput.svelte @@ -230,7 +230,11 @@ let oldDefaultValue = structuredClone(defaultValue) function handleDefaultValueChange() { - if (deepEqual(value, oldDefaultValue)) { + if ( + deepEqual(value, oldDefaultValue) && + !deepEqual(value, defaultValue) && + !deepEqual(defaultValue, oldDefaultValue) + ) { value = defaultValue } oldDefaultValue = structuredClone(defaultValue) @@ -489,6 +493,7 @@
-{#if canEditResourceType} +{#if canEditResourceType || originalType == 'string[]' || originalType == 'object[]'} {:else if itemsType?.resourceType} @@ -121,31 +124,33 @@
{/each} -
- - -
+ {#if canEditResourceType || originalType == 'string[]' || originalType == 'object[]'} +
+ + +
+ {/if} {/if} diff --git a/frontend/src/lib/components/Auth0Setting.svelte b/frontend/src/lib/components/Auth0Setting.svelte new file mode 100644 index 0000000000..5ff0781c72 --- /dev/null +++ b/frontend/src/lib/components/Auth0Setting.svelte @@ -0,0 +1,102 @@ + + +
+ + + {#if enabled} +
+ + + + + +
+ From your Admin page, setup a Windmill application
+ Create a new application
+ For "application type" select "Regular Web Application"
+ Copy down the "Client ID" and "Client Secret" and paste them into the fields above
+ Under "Application URIs", set the following:
+ a. Application Login URI: `BASE_URL/user/login`
+ b. Allowed Callback URLs: `BASE_URL/user/login_callback/auth0`
+ c. Allowed Logout URLs: `BASE_URL/auth/logout`
+ d. Allowed Web Origins: `BASE_URL`
+ e. Allowed Origins (CORS): `BASE_URL`
+
+ +
+ {/if} +
diff --git a/frontend/src/lib/components/AuthSettings.svelte b/frontend/src/lib/components/AuthSettings.svelte index 195e760114..f6db65526a 100644 --- a/frontend/src/lib/components/AuthSettings.svelte +++ b/frontend/src/lib/components/AuthSettings.svelte @@ -4,6 +4,7 @@ import OAuthSetting from '$lib/components/OAuthSetting.svelte' import OktaSetting from './OktaSetting.svelte' + import Auth0Setting from './Auth0Setting.svelte' import CloseButton from './common/CloseButton.svelte' import KeycloakSetting from './KeycloakSetting.svelte' import CustomSso from './CustomSso.svelte' @@ -81,6 +82,7 @@ + @@ -90,7 +92,7 @@ {#each Object.keys(oauths) as k} - {#if !['authelia', 'authentik', 'google', 'microsoft', 'github', 'gitlab', 'jumpcloud', 'okta', 'keycloak', 'slack', 'kanidm', 'zitadel'].includes(k) && 'login_config' in oauths[k]} + {#if !['authelia', 'authentik', 'google', 'microsoft', 'github', 'gitlab', 'jumpcloud', 'okta', 'auth0', 'keycloak', 'slack', 'kanidm', 'zitadel'].includes(k) && 'login_config' in oauths[k]} {#if oauths[k]}
diff --git a/frontend/src/lib/components/Dev.svelte b/frontend/src/lib/components/Dev.svelte index ccdda7524e..e53d49eb72 100644 --- a/frontend/src/lib/components/Dev.svelte +++ b/frontend/src/lib/components/Dev.svelte @@ -48,7 +48,7 @@ } from '$lib/relative_imports' import Tooltip from './Tooltip.svelte' import type { ScheduleTrigger, TriggerContext } from './triggers' - import { initAllAiWorkspace } from './copilot/lib' + import { workspaceAIClients } from './copilot/lib' import type { FlowPropPickerConfig, PropPickerContext } from './prop_picker' import type { PickableProperties } from './flows/previousResults' $: token = $page.url.searchParams.get('wm_token') ?? undefined @@ -110,14 +110,19 @@ async function setCopilotInfo() { if (workspace) { - initAllAiWorkspace(workspace) + workspaceAIClients.init(workspace) try { - copilotInfo.set(await WorkspaceService.getCopilotInfo({ workspace })) + const info = await WorkspaceService.getCopilotInfo({ workspace }) + copilotInfo.set({ + ...info, + ai_provider: info.ai_provider ?? 'openai' + }) } catch (err) { copilotInfo.set({ - ai_provider: '', + ai_provider: 'openai', exists_ai_resource: false, - code_completion_enabled: false + code_completion_model: undefined, + ai_models: [] }) console.error('Could not get copilot info') diff --git a/frontend/src/lib/components/DisplayResult.svelte b/frontend/src/lib/components/DisplayResult.svelte index b5088c0a5f..80f7ce999f 100644 --- a/frontend/src/lib/components/DisplayResult.svelte +++ b/frontend/src/lib/components/DisplayResult.svelte @@ -677,16 +677,22 @@ preview rendered
{:else if result?.s3?.endsWith('.pdf')}
{/if} diff --git a/frontend/src/lib/components/EditableSchemaForm.svelte b/frontend/src/lib/components/EditableSchemaForm.svelte index 2c8a2af555..a5e991393a 100644 --- a/frontend/src/lib/components/EditableSchemaForm.svelte +++ b/frontend/src/lib/components/EditableSchemaForm.svelte @@ -36,6 +36,7 @@ export let jsonEnabled: boolean = true export let isAppInput: boolean = false export let displayWebhookWarning: boolean = false + export let onlyMaskPassword: boolean = false export let dndType: string | undefined = undefined export let editTab: | 'inputEditor' @@ -282,6 +283,7 @@ schema={previewSchema ? previewSchema : schema} {dndType} {disableDnd} + {onlyMaskPassword} bind:args on:click={(e) => { opened = e.detail diff --git a/frontend/src/lib/components/Editor.svelte b/frontend/src/lib/components/Editor.svelte index 350d26b643..b5f6565362 100644 --- a/frontend/src/lib/components/Editor.svelte +++ b/frontend/src/lib/components/Editor.svelte @@ -172,7 +172,6 @@ import { initVim } from './monaco_keybindings' import { buildWorkerDefinition } from '$lib/monaco_workers/build_workers' import { parseTypescriptDeps } from '$lib/relative_imports' - import type { AiProviderTypes } from './copilot/lib' // import EditorTheme from './EditorTheme.svelte' @@ -640,7 +639,7 @@ textAfterPosition, lang, abortController, - aiProvider as AiProviderTypes + aiProvider ) if (insertText) { items = [ @@ -668,7 +667,7 @@ } $: $copilotInfo.exists_ai_resource && - $copilotInfo.code_completion_enabled && + $copilotInfo.code_completion_model && $codeCompletionSessionEnabled && initialized && addCopilotSuggestions() diff --git a/frontend/src/lib/components/FlowBuilder.svelte b/frontend/src/lib/components/FlowBuilder.svelte index 3db9804bb8..a713835829 100644 --- a/frontend/src/lib/components/FlowBuilder.svelte +++ b/frontend/src/lib/components/FlowBuilder.svelte @@ -86,7 +86,6 @@ import type { FlowBuilderWhitelabelCustomUi } from './custom_ui' import FlowYamlEditor from './flows/header/FlowYamlEditor.svelte' import { type TriggerContext, type ScheduleTrigger } from './triggers' - import type { AiProviderTypes } from './copilot/lib' export let initialPath: string = '' export let pathStoreInit: string | undefined = undefined @@ -821,7 +820,7 @@ try { push(history, $flowStore) let module = stepOnly ? $copilotModulesStore[0] : $copilotModulesStore[idx] - const aiProvider = $copilotInfo.ai_provider as AiProviderTypes + const aiProvider = $copilotInfo.ai_provider copilotLoading = true copilotStatus = "Generating code for step '" + module.id + "'..." diff --git a/frontend/src/lib/components/FlowHistoryJobPicker.svelte b/frontend/src/lib/components/FlowHistoryJobPicker.svelte index a631a2a595..5a9a661a74 100644 --- a/frontend/src/lib/components/FlowHistoryJobPicker.svelte +++ b/frontend/src/lib/components/FlowHistoryJobPicker.svelte @@ -20,6 +20,8 @@ }) if (jobs.length > 0) { dispatch('select', { jobId: jobs[0].id, initial: true }) + } else { + dispatch('nohistory') } } diff --git a/frontend/src/lib/components/FlowPreviewContent.svelte b/frontend/src/lib/components/FlowPreviewContent.svelte index 2863a246ed..93dbb0f6a6 100644 --- a/frontend/src/lib/components/FlowPreviewContent.svelte +++ b/frontend/src/lib/components/FlowPreviewContent.svelte @@ -169,6 +169,34 @@ } $: selectedJobStep !== undefined && onSelectedJobStepChange() + + async function loadIndividualStepsStates() { + dfs($flowStore.value.modules, async (module) => { + if ($flowStateStore[module.id]?.previewResult) { + return + } + const previousJobId = await JobService.listJobs({ + workspace: $workspaceStore!, + scriptPathExact: (initialPath == '' ? $pathStore : initialPath) + '/' + module.id, + jobKinds: ['preview', 'script', 'flowpreview', 'flow'].join(','), + page: 1, + perPage: 1 + }) + + if (previousJobId.length > 0) { + const getJobResult = await JobService.getCompletedJobResultMaybe({ + workspace: $workspaceStore!, + id: previousJobId[0].id + }) + if (getJobResult.result) { + $flowStateStore[module.id] = { + ...($flowStateStore[module.id] ?? {}), + previewResult: getJobResult.result + } + } + } + }) + } @@ -401,6 +429,9 @@ class="absolute top-[22px] right-2 border p-1.5 hover:bg-surface-hover rounded-md center-center" > { + loadIndividualStepsStates() + }} on:select={(e) => { if (!currentJobId) { currentJobId = jobId @@ -439,11 +470,14 @@ {flowStateStore} {jobId} on:done={() => { - console.log('done') $executionCount = $executionCount + 1 }} on:jobsLoaded={({ detail }) => { job = detail + if (initial) { + console.log('loading initial steps after initial job loaded') + loadIndividualStepsStates() + } }} bind:selectedJobStep /> diff --git a/frontend/src/lib/components/HistoricInputs.svelte b/frontend/src/lib/components/HistoricInputs.svelte index d03ea8a8fe..e2d633ba7c 100644 --- a/frontend/src/lib/components/HistoricInputs.svelte +++ b/frontend/src/lib/components/HistoricInputs.svelte @@ -121,9 +121,9 @@ /> {/each} {#if jobs?.length == 5} -
... there may be more runs not displayed here as the limit is 5
+ + limited to 5 runs + {/if} {:else} diff --git a/frontend/src/lib/components/Login.svelte b/frontend/src/lib/components/Login.svelte index 8b073e4d8a..3046afd53c 100644 --- a/frontend/src/lib/components/Login.svelte +++ b/frontend/src/lib/components/Login.svelte @@ -5,6 +5,7 @@ import Google from '$lib/components/icons/brands/Google.svelte' import Microsoft from '$lib/components/icons/brands/Microsoft.svelte' import Okta from '$lib/components/icons/brands/Okta.svelte' + import Auth0 from '$lib/components/icons/brands/Auth0.svelte' import { OauthService, UserService, WorkspaceService } from '$lib/gen' import { usersWorkspaceStore, workspaceStore, userStore } from '$lib/stores' @@ -49,6 +50,11 @@ type: 'okta', name: 'Okta', icon: Okta + }, + { + type: 'auth0', + name: 'Auth0', + icon: Auth0 } ] as const diff --git a/frontend/src/lib/components/ResourceEditor.svelte b/frontend/src/lib/components/ResourceEditor.svelte index 8f882baa59..8a3a952186 100644 --- a/frontend/src/lib/components/ResourceEditor.svelte +++ b/frontend/src/lib/components/ResourceEditor.svelte @@ -204,7 +204,6 @@
{/if} - {#if can_write && editDescription}
GH Markdown
diff --git a/frontend/src/lib/components/S3FilePicker.svelte b/frontend/src/lib/components/S3FilePicker.svelte index 3ee14c1986..8bfaa82763 100644 --- a/frontend/src/lib/components/S3FilePicker.svelte +++ b/frontend/src/lib/components/S3FilePicker.svelte @@ -43,6 +43,7 @@ export let initialFileKey: { s3: string } | undefined = undefined let initialFileKeyInternalCopy: { s3: string } export let selectedFileKey: { s3: string } | undefined = undefined + export let folderOnly = false let csvSeparatorChar: string = ',' let csvHasHeader: boolean = true @@ -369,6 +370,11 @@ let item_key = displayedFileKeys[index] let item = allFilesByKey[item_key] if (item.type === 'folder') { + if (folderOnly) { + selectedFileKey = { + s3: item_key + } + } if (toggleCollapsed) { item.collapsed = !item.collapsed } @@ -709,7 +715,9 @@ > {#if !fromWorkspaceSettings} {/if} diff --git a/frontend/src/lib/components/SchemaForm.svelte b/frontend/src/lib/components/SchemaForm.svelte index 3720bc585b..3991f1c3e5 100644 --- a/frontend/src/lib/components/SchemaForm.svelte +++ b/frontend/src/lib/components/SchemaForm.svelte @@ -265,7 +265,10 @@ dispatch('click', argName) }} > - {#if typeof args == 'object' && schema?.properties[argName]} + {#if args && typeof args == 'object' && schema?.properties[argName]} + {#if !hidden[argName]} { diff --git a/frontend/src/lib/components/SchemaFormWithArgPicker.svelte b/frontend/src/lib/components/SchemaFormWithArgPicker.svelte index 29ec803557..040af5cd96 100644 --- a/frontend/src/lib/components/SchemaFormWithArgPicker.svelte +++ b/frontend/src/lib/components/SchemaFormWithArgPicker.svelte @@ -119,7 +119,7 @@ {runnableId} {runnableType} on:select={(e) => { - dispatch('select', { payload: e.detail.args, type: 'history' }) + dispatch('select', { payload: e.detail?.args, type: 'history' }) }} /> diff --git a/frontend/src/lib/components/Section.svelte b/frontend/src/lib/components/Section.svelte index f102c18fe6..d4babb09c1 100644 --- a/frontend/src/lib/components/Section.svelte +++ b/frontend/src/lib/components/Section.svelte @@ -1,8 +1,9 @@
@@ -27,11 +29,14 @@ > {#if collapsable} {:else} @@ -58,7 +63,10 @@
{/if} {#if !collapsable || !collapsed} -
+
{/if} diff --git a/frontend/src/lib/components/StringTypeNarrowing.svelte b/frontend/src/lib/components/StringTypeNarrowing.svelte index b960653b64..ae9b4e9a9d 100644 --- a/frontend/src/lib/components/StringTypeNarrowing.svelte +++ b/frontend/src/lib/components/StringTypeNarrowing.svelte @@ -120,6 +120,8 @@ } if (e.detail == 'date-time') { format = 'date-time' + } else if (format) { + format = undefined } if (e.detail == 'none') { pattern = undefined diff --git a/frontend/src/lib/components/apps/components/buttons/AppButton.svelte b/frontend/src/lib/components/apps/components/buttons/AppButton.svelte index ce9cd67c36..065a236a02 100644 --- a/frontend/src/lib/components/apps/components/buttons/AppButton.svelte +++ b/frontend/src/lib/components/apps/components/buttons/AppButton.svelte @@ -76,12 +76,29 @@ $: resolvedConfig.beforeIcon && beforeIconComponent && handleBeforeIcon() $: resolvedConfig.afterIcon && afterIconComponent && handleAfterIcon() + function getIconSize() { + switch (resolvedConfig.size as 'xs' | 'sm' | 'md' | 'lg' | 'xl') { + case 'xs': + return 14 + case 'sm': + return 16 + case 'md': + return 20 + case 'lg': + return 24 + case 'xl': + return 26 + default: + return 24 + } + } + async function handleBeforeIcon() { if (resolvedConfig.beforeIcon) { beforeIconComponent = await loadIcon( resolvedConfig.beforeIcon, beforeIconComponent, - 14, + getIconSize(), undefined, undefined ) @@ -93,7 +110,7 @@ afterIconComponent = await loadIcon( resolvedConfig.afterIcon, afterIconComponent, - 14, + getIconSize(), undefined, undefined ) diff --git a/frontend/src/lib/components/apps/components/display/AppAccordionList.svelte b/frontend/src/lib/components/apps/components/display/AppAccordionList.svelte index 9697a1f651..2966cf418d 100644 --- a/frontend/src/lib/components/apps/components/display/AppAccordionList.svelte +++ b/frontend/src/lib/components/apps/components/display/AppAccordionList.svelte @@ -102,7 +102,7 @@ )} > {activeIndex === index ? '-' : '+'} - {accordionInput?.value[index]?.header || `Header ${index}`} + {result[index]?.header || `Header ${index}`} {#if activeIndex === index}
diff --git a/frontend/src/lib/components/apps/components/display/AppDisplayComponent.svelte b/frontend/src/lib/components/apps/components/display/AppDisplayComponent.svelte index 4b09156da2..c15855f873 100644 --- a/frontend/src/lib/components/apps/components/display/AppDisplayComponent.svelte +++ b/frontend/src/lib/components/apps/components/display/AppDisplayComponent.svelte @@ -15,6 +15,7 @@ import ResolveStyle from '../helpers/ResolveStyle.svelte' import { components } from '../../editor/component' import ResolveConfig from '../helpers/ResolveConfig.svelte' + import { userStore } from '$lib/stores' export let id: string export let componentInput: AppInput | undefined @@ -96,7 +97,7 @@ {result} {requireHtmlApproval} disableExpand={resolvedConfig?.hideDetails} - appPath={$appPath} + appPath={$userStore ? undefined : $appPath} />
diff --git a/frontend/src/lib/components/apps/components/display/AppDisplayComponentByJobId.svelte b/frontend/src/lib/components/apps/components/display/AppDisplayComponentByJobId.svelte index bb5aa888d6..f61db83517 100644 --- a/frontend/src/lib/components/apps/components/display/AppDisplayComponentByJobId.svelte +++ b/frontend/src/lib/components/apps/components/display/AppDisplayComponentByJobId.svelte @@ -16,6 +16,7 @@ import ResolveStyle from '../helpers/ResolveStyle.svelte' import InitializeComponent from '../helpers/InitializeComponent.svelte' import DisplayResult from '$lib/components/DisplayResult.svelte' + import { userStore } from '$lib/stores' export let id: string export let initializing: boolean | undefined = false @@ -23,7 +24,7 @@ export let configuration: RichConfigurations export let render: boolean - const { app, worldStore, workspace } = getContext('AppViewerContext') + const { app, worldStore, workspace, appPath } = getContext('AppViewerContext') const requireHtmlApproval = getContext(IS_APP_PUBLIC_CONTEXT_KEY) let resolvedConfig = initConfig( @@ -115,6 +116,7 @@ {result} {requireHtmlApproval} disableExpand={resolvedConfig?.hideDetails} + appPath={$userStore ? undefined : $appPath} />
diff --git a/frontend/src/lib/components/apps/components/display/AppMenu.svelte b/frontend/src/lib/components/apps/components/display/AppMenu.svelte index de56cd22cd..ac0fdebb11 100644 --- a/frontend/src/lib/components/apps/components/display/AppMenu.svelte +++ b/frontend/src/lib/components/apps/components/display/AppMenu.svelte @@ -105,6 +105,7 @@ on:close on:open class={resolvedConfig.fillContainer ? 'w-full h-full' : ''} + usePointerDownOutside={true} >
@@ -145,7 +146,7 @@
-
+
{#if menuItems.length > 0} {#each menuItems as actionButton, actionIndex (actionButton?.id)} {#if actionButton.type == 'buttoncomponent'} diff --git a/frontend/src/lib/components/apps/utils.ts b/frontend/src/lib/components/apps/utils.ts index cb30aac94f..5e41d4ccfa 100644 --- a/frontend/src/lib/components/apps/utils.ts +++ b/frontend/src/lib/components/apps/utils.ts @@ -248,7 +248,7 @@ declare function setValue(id: string, value: any): void; */ declare function setSelectedIndex(id: string, index: number): void; -/** close a drawer or modal +/** open a drawer or modal * @param id component's id */ declare function open(id: string): void; diff --git a/frontend/src/lib/components/common/fileInput/FileInput.svelte b/frontend/src/lib/components/common/fileInput/FileInput.svelte index b123fbb3a4..0212f5f534 100644 --- a/frontend/src/lib/components/common/fileInput/FileInput.svelte +++ b/frontend/src/lib/components/common/fileInput/FileInput.svelte @@ -20,30 +20,24 @@ export let submittedText: string | undefined = undefined export let defaultFile: string | undefined = undefined export let disabled: boolean | undefined = undefined + export let folderOnly = false const dispatch = createEventDispatcher() let input: HTMLInputElement - export let files: File[] | undefined = undefined + type FileWithPath = File & { path?: string } + export let files: FileWithPath[] | undefined = undefined - async function onChange(fileList: FileList | null) { + async function onChange(fileList: FileWithPath[] | null) { if (!fileList || !fileList.length) { files = undefined dispatch('change', files) return } - if (!multiple || !files) { - files = [] - } - for (let i = 0; i < fileList.length; i++) { - const file = fileList.item(i) - if (file) { - files.push(file) - } - } - - if (!files.length) { - files = undefined + if (multiple && files) { + files = [...files, ...fileList] + } else { + files = fileList } // Needs to be reset so the same file can be selected @@ -86,15 +80,70 @@ } } - function handleDrop(event: DragEvent) { + async function handleFile(fileEntry: FileSystemFileEntry): Promise { + return new Promise((resolve, reject) => { + fileEntry.file(resolve, reject) + }) + } + + async function handleDirectory( + dirEntry: FileSystemDirectoryEntry, + path: string + ): Promise { + const files: FileWithPath[] = [] + const dirReader = dirEntry.createReader() + + async function readEntries() { + return new Promise((resolve) => { + dirReader.readEntries(async (entries) => { + if (entries.length === 0) { + resolve(files) + return + } + + const filePromises = entries.map(async (entry) => { + return traverseFileTree(entry, path + dirEntry.name + '/') + }) + const nestedFiles = await Promise.all(filePromises) + files.push(...nestedFiles.flat()) + // readEntries only return up to 100 files + // continue reading if more files exist + resolve(await readEntries()) + }) + }) + } + + return readEntries() + } + + async function traverseFileTree(entry: FileSystemEntry, path = ''): Promise { + if (entry.isFile) { + const file = await handleFile(entry as FileSystemFileEntry) + file.path = path + file.name + return [file] + } else if (entry.isDirectory) { + return handleDirectory(entry as FileSystemDirectoryEntry, path) + } + return [] + } + + async function handleDrop(event: DragEvent) { event.preventDefault() if (event.dataTransfer) { - if (event.dataTransfer.files && event.dataTransfer.files.length) { - if (!multiple && event.dataTransfer.files.length > 1) { - sendUserToast('Only one file can be uploaded at a time') - return - } else { - onChange(event.dataTransfer.files) + if (folderOnly) { + const item = event.dataTransfer.items[0]?.webkitGetAsEntry() + if (item) { + const files = await traverseFileTree(item, '') + onChange(files) + } + } else { + if (event.dataTransfer.files && event.dataTransfer.files.length) { + if (!multiple && event.dataTransfer.files.length > 1) { + sendUserToast('Only one file can be uploaded at a time') + return + } else { + onChange(Array.from(event.dataTransfer.files)) + } } } } @@ -174,22 +223,22 @@
{:else} - Drag and drop {multiple ? 'files' : 'a file'} + Drag and drop {folderOnly ? 'a folder' : multiple ? 'files' : 'a file'} {/if} 1 ? 's' : ''} chosen` : 'No file chosen'} bind:this={input} on:change={({ currentTarget }) => { - onChange(currentTarget.files) + onChange(currentTarget.files ? Array.from(currentTarget.files) : null) }} {accept} {multiple} {...$$restProps} /> - {#if defaultFile}
Default file: {defaultFile} diff --git a/frontend/src/lib/components/common/fileUpload/FileUpload.svelte b/frontend/src/lib/components/common/fileUpload/FileUpload.svelte index dccdbed4a4..09d6e639ac 100644 --- a/frontend/src/lib/components/common/fileUpload/FileUpload.svelte +++ b/frontend/src/lib/components/common/fileUpload/FileUpload.svelte @@ -14,7 +14,10 @@ export let acceptedFileTypes: string[] | undefined = ['*'] export let allowMultiple: boolean = true - export let containerText: string = allowMultiple + export let folderOnly = false + export let containerText: string = folderOnly + ? 'Drag and drop a folder here or click to browse' + : allowMultiple ? 'Drag and drop files here or click to browse' : 'Drag and drop a file here or click to browse' export let customResourcePath: string | undefined = undefined @@ -53,20 +56,61 @@ } async function handleChange(files: File[] | undefined) { - for (const file of files ?? []) { - uploadFileToS3(file, file.name) + if (folderOnly) { + uniqueFolderPrefix = getRandomFolderPrefix() + uploadedFolderRoot = undefined + await Promise.all( + files?.map(async (file) => { + await uploadFileToS3(file, file.name) + }) ?? [] + ) + if (uploadedFolderRoot) { + dispatch('addition', { + path: uniqueFolderPrefix + uploadedFolderRoot, + filename: undefined + }) + sendUserToast('Folder uploaded!') + } + } else { + for (const file of files ?? []) { + uploadFileToS3(file, file.name) + } } } let activeUploads: { xhr: XMLHttpRequest; fileName: string }[] = [] - async function uploadFileToS3(fileToUpload: File, fileToUploadKey: string) { + function getRandomFolderPrefix(): string { + const now = Date.now() + const randomNum = Math.floor(Math.random() * 65536) + return `windmill_uploads/upload_${now}_${randomNum}/` + } + + let uniqueFolderPrefix = getRandomFolderPrefix() + let uploadedFolderRoot: string | undefined = undefined + + async function uploadFileToS3(fileToUpload: File & { path?: string }, fileToUploadKey: string) { if (fileToUpload === undefined || fileToUploadKey === undefined) { return } let path: string | undefined = undefined let fileExtension: string | undefined = undefined - if (randomFileKey) { + if (folderOnly) { + const relativePath = fileToUpload.webkitRelativePath || fileToUpload.path + if (!relativePath) { + throw new Error('Missing file relative path') + } + const rootFolder = relativePath.split('/')[0] + if (!rootFolder) { + throw new Error('Missing root folder') + } + if (uploadedFolderRoot && rootFolder !== uploadedFolderRoot) { + throw new Error('Uploading a file in a different root folder than the previous files') + } else { + uploadedFolderRoot = rootFolder + } + path = uniqueFolderPrefix + relativePath + } else if (randomFileKey) { fileExtension = fileToUpload.name.split('.').pop() if (emptyString(fileExtension)) { fileExtension = undefined @@ -219,8 +263,10 @@ }) return } - dispatch('addition', { path: uploadData.path, filename: fileToUpload.name }) - sendUserToast('File upload finished!') + if (!folderOnly) { + dispatch('addition', { path: uploadData.path, filename: fileToUpload.name }) + sendUserToast('File uploaded!') + } uploadData.progress = 100 $fileUploads = $fileUploads.map((fileUpload) => { @@ -442,6 +488,7 @@
{:else} -{#if $copilotInfo.exists_ai_resource && $copilotInfo.code_completion_enabled} +{#if $copilotInfo.exists_ai_resource && $copilotInfo.code_completion_model} import { ExternalLink, Wand2 } from 'lucide-svelte' import Button from '../common/button/Button.svelte' - import { getNonStreamingCompletion, type AiProviderTypes } from './lib' + import { getNonStreamingCompletion } from './lib' import Popup from '../common/popup/Popup.svelte' import { sendUserToast } from '$lib/toast' import { copilotInfo } from '$lib/stores' import { base } from '$lib/base' import type { ChatCompletionMessageParam } from 'openai/resources/index.mjs' + import type { AIProvider } from '$lib/gen' export let schedule: string export let cronVersion: string @@ -56,7 +57,7 @@ const response = await getNonStreamingCompletion( messages, abortController, - aiProvider as AiProviderTypes + aiProvider as AIProvider ) if (response.startsWith('ERROR:')) { diff --git a/frontend/src/lib/components/copilot/IteratorGen.svelte b/frontend/src/lib/components/copilot/IteratorGen.svelte index 361a6a5c1a..a53423670b 100644 --- a/frontend/src/lib/components/copilot/IteratorGen.svelte +++ b/frontend/src/lib/components/copilot/IteratorGen.svelte @@ -1,7 +1,7 @@ @@ -21,8 +24,10 @@ setTimeout(() => { abortController.abort() }, 10000) + await testKey({ apiKey, + resourcePath, messages: [ { role: 'user', @@ -30,7 +35,8 @@ } ], abortController, - aiProvider + aiProvider, + model }) sendUserToast('Valid key') } catch (err) { @@ -42,5 +48,11 @@ } finally { loading = false } - }}>Test key + {#if apiKey} + Test key + {:else} + Test + {/if} + diff --git a/frontend/src/lib/components/copilot/completion.ts b/frontend/src/lib/components/copilot/completion.ts index b30a3fe7e0..01c90035c6 100644 --- a/frontend/src/lib/components/copilot/completion.ts +++ b/frontend/src/lib/components/copilot/completion.ts @@ -1,6 +1,7 @@ -import type { ChatCompletionMessageParam } from 'openai/resources/chat/index.mjs' -import { getNonStreamingCompletion, type AiProviderTypes } from './lib' +import type { AIProvider } from '$lib/gen' import { codeCompletionLoading } from '$lib/stores' +import type { ChatCompletionMessageParam } from 'openai/resources/chat/index.mjs' +import { getNonStreamingCompletion } from './lib' const systemPrompt = `You are a code completion assistant, return the code that should go instead of the . @@ -74,7 +75,7 @@ export async function editorCodeCompletion( after: string, lang: string, abortController: AbortController, - aiProvider: AiProviderTypes + aiProvider: AIProvider ) { codeCompletionLoading.set(true) const messages: ChatCompletionMessageParam[] = [ diff --git a/frontend/src/lib/components/copilot/flow.ts b/frontend/src/lib/components/copilot/flow.ts index 7867a2f2d5..1c5f8685bd 100644 --- a/frontend/src/lib/components/copilot/flow.ts +++ b/frontend/src/lib/components/copilot/flow.ts @@ -1,21 +1,17 @@ import { - type FlowModule, ScriptService, - type RawScript, - type PathScript, + type AIProvider, + type FlowModule, type InputTransform, + type PathScript, + type RawScript, type Script } from '$lib/gen' -import { - addResourceTypes, - deltaCodeCompletion, - getNonStreamingCompletion, - type AiProviderTypes -} from './lib' +import { scriptLangToEditorLang } from '$lib/scripts' import type { Writable } from 'svelte/store' import type Editor from '../Editor.svelte' import type { Drawer } from '../common' -import { scriptLangToEditorLang } from '$lib/scripts' +import { addResourceTypes, deltaCodeCompletion, getNonStreamingCompletion } from './lib' export type FlowCopilotModule = { id: string @@ -259,7 +255,7 @@ export async function stepCopilot( | undefined, isFirstInLoop: boolean, abortController: AbortController, - aiProvider: AiProviderTypes + aiProvider: AIProvider ) { if (module.source !== 'custom') { throw new Error('Not a custom module') @@ -328,7 +324,7 @@ export async function glueCopilot( }, isFirstInLoop: boolean, abortController: AbortController, - aiProvider: AiProviderTypes + aiProvider: AIProvider ) { const { prevCode, prevLang } = await getPreviousStepContent(pastModule, workspace) diff --git a/frontend/src/lib/components/copilot/lib.ts b/frontend/src/lib/components/copilot/lib.ts index fe7028dc1e..69d5ac9b7d 100644 --- a/frontend/src/lib/components/copilot/lib.ts +++ b/frontend/src/lib/components/copilot/lib.ts @@ -1,56 +1,130 @@ -import { OpenAI } from 'openai' -import { OpenAPI, ResourceService, type Script } from '../../gen' -import type { Writable } from 'svelte/store' +import type { AIProvider } from '$lib/gen' +import { + copilotInfo, + copilotSessionModel, + type DBSchema, + type GraphqlSchema, + type SQLSchema +} from '$lib/stores' import { Anthropic } from '@anthropic-ai/sdk' -import type { DBSchema, GraphqlSchema, SQLSchema } from '$lib/stores' -import { formatResourceTypes } from './utils' -import { EDIT_CONFIG, FIX_CONFIG, GEN_CONFIG } from './prompts' import { Mistral } from '@mistralai/mistralai' import { buildClientSchema, printSchema } from 'graphql' +import { OpenAI } from 'openai' import type { ChatCompletionCreateParamsStreaming, ChatCompletionMessageParam } from 'openai/resources/index.mjs' +import { get, type Writable } from 'svelte/store' +import { OpenAPI, ResourceService, type Script } from '../../gen' +import { EDIT_CONFIG, FIX_CONFIG, GEN_CONFIG } from './prompts' +import { formatResourceTypes } from './utils' import type { MessageCreateParams, MessageParam } from '@anthropic-ai/sdk/resources/messages.mjs' -import type { ChatCompletionRequest } from '@mistralai/mistralai/models/components/chatcompletionrequest' import type { - SystemMessage, - UserMessage, AssistantMessage, - ToolMessage, CompletionEvent, - ContentChunk + ContentChunk, + SystemMessage, + ToolMessage, + UserMessage } from '@mistralai/mistralai/models/components' +import type { ChatCompletionRequest } from '@mistralai/mistralai/models/components/chatcompletionrequest' export const SUPPORTED_LANGUAGES = new Set(Object.keys(GEN_CONFIG.prompts)) -export type AiProviderTypes = 'openai' | 'anthropic' | 'mistral' - -interface AiProvider { - init: (workspace: string, updateClient: boolean, token?: string) => void +export const AI_DEFAULT_MODELS: Record = { + openai: ['gpt-4o', 'gpt-4o-mini'], + anthropic: ['claude-3-5-sonnet-latest', 'claude-3-5-haiku-latest'], + mistral: ['codestral-latest'], + deepseek: ['deepseek-chat', 'deepseek-reasoner'], + groq: ['llama-3.3-70b-versatile', 'llama-3.1-8b-instant'], + openrouter: ['meta-llama/llama-3.2-3b-instruct:free'], + customai: [] } -class WorkspacedMistral implements AiProvider { - private client: Mistral | undefined +export const OPENAI_COMPATIBLE_BASE_URLS = { + groq: 'https://api.groq.com/openai/v1', + openrouter: 'https://openrouter.ai/api/v1', + deepseek: 'https://api.deepseek.com/v1' +} as const - init(workspace: string, updateClient: boolean, token?: string) { - if (!this.client || updateClient) { - this.client = initWorkspaceAiProvider(workspace, 'mistral', token) as unknown as Mistral - } +class WorkspacedAIClients { + private openaiClient: OpenAI | undefined + private anthropicClient: Anthropic | undefined + private mistralClient: Mistral | undefined + + init(workspace: string) { + this.initOpenai(workspace) + this.initAnthropic(workspace) + this.initMistral(workspace) } - getClient() { - if (!this.client) { - throw new Error('AnthropicAi not initialized') + private getBaseURL(workspace: string) { + return `${location.origin}${OpenAPI.BASE}/w/${workspace}/ai/proxy` + } + + private initOpenai(workspace: string) { + const baseURL = this.getBaseURL(workspace) + this.openaiClient = new OpenAI({ + baseURL, + apiKey: 'fake-key', + defaultHeaders: { + Authorization: '' // a non empty string will be unable to access Windmill backend proxy + }, + dangerouslyAllowBrowser: true + }) + } + + private initAnthropic(workspace: string) { + const baseURL = this.getBaseURL(workspace) + this.anthropicClient = new Anthropic({ + baseURL, + apiKey: 'fake-key', + dangerouslyAllowBrowser: true + }) + } + + private initMistral(workspace: string) { + const baseURL = this.getBaseURL(workspace) + this.mistralClient = new Mistral({ + serverURL: baseURL + }) + } + + getOpenaiClient() { + if (!this.openaiClient) { + throw new Error('OpenAI not initialized') } - return this.client + return this.openaiClient + } + + getAnthropicClient() { + if (!this.anthropicClient) { + throw new Error('Anthropic not initialized') + } + return this.anthropicClient + } + + getMistralClient() { + if (!this.mistralClient) { + throw new Error('Mistral not initialized') + } + return this.mistralClient } } -export namespace MistralAi { - export let workspace = new WorkspacedMistral() +export const workspaceAIClients = new WorkspacedAIClients() +const DEFAULT_COMPLETION_CONFIG: ChatCompletionCreateParamsStreaming = { + model: '', + max_tokens: 8000, //TODO: make this dynamic + temperature: 0, + seed: 42, + stream: true, + messages: [] +} + +namespace MistralAI { export const mistralConfig: ChatCompletionRequest = { temperature: 0, model: null, @@ -78,30 +152,11 @@ export namespace MistralAi { } } -class WorkspacedAnthropic implements AiProvider { - private client: Anthropic | undefined - - init(workspace: string, updateClient: boolean, token: string | undefined = undefined) { - if (!this.client || updateClient) { - this.client = initWorkspaceAiProvider(workspace, 'anthropic', token) as unknown as Anthropic - } - } - - getClient() { - if (!this.client) { - throw new Error('AnthropicAi not initialized') - } - return this.client - } -} - -export namespace AnthropicAi { - export let workspace = new WorkspacedAnthropic() - +export namespace AnthropicAI { export const config: MessageCreateParams = { temperature: 0, max_tokens: 8192, - model: 'claude-3-5-sonnet-20241022', + model: '', messages: [] } @@ -135,30 +190,11 @@ export namespace AnthropicAi { } } -class WorkspacedOpenai implements AiProvider { - private client: OpenAI | undefined - - init(workspace: string, updateClient: boolean, token: string | undefined = undefined) { - if (!this.client || updateClient) { - this.client = initWorkspaceAiProvider(workspace, 'openai', token) as unknown as OpenAI - } - } - - getClient() { - if (!this.client) { - throw new Error('OpenAI not initialized') - } - return this.client - } -} - -export namespace OpenAi { - export let workspace = new WorkspacedOpenai() - +namespace OpenAi { export const openaiConfig: ChatCompletionCreateParamsStreaming = { temperature: 0, max_tokens: 16384, - model: 'gpt-4o-2024-08-06', + model: '', seed: 42, stream: true, messages: [] @@ -169,125 +205,38 @@ export namespace OpenAi { } } -export function initAllAiWorkspace(workspace: string, updateClient: boolean = false) { - OpenAi.workspace.init(workspace, updateClient) - AnthropicAi.workspace.init(workspace, updateClient) - MistralAi.workspace.init(workspace, updateClient) -} - -function initWorkspaceAiProvider( - workspace: string, - aiProvider: AiProviderTypes, - token: string | undefined = undefined -): Anthropic | OpenAI | Mistral { - const baseURL = `${location.origin}${OpenAPI.BASE}/w/${workspace}/ai/proxy` - let client - switch (aiProvider) { - case 'openai': { - client = new OpenAI({ - baseURL, - apiKey: 'fake-key', - defaultHeaders: { - Authorization: token ? `Bearer ${token}` : '' - }, - dangerouslyAllowBrowser: true - }) - break - } - case 'anthropic': { - client = new Anthropic({ - baseURL, - apiKey: 'fake-key', - defaultHeaders: { - Authorization: token ? `Bearer ${token}` : '' - }, - dangerouslyAllowBrowser: true - }) - break - } - case 'mistral': { - client = new Mistral({ - serverURL: baseURL - }) - } - } - return client -} - export async function testKey({ apiKey, + resourcePath, + model, abortController, messages, aiProvider }: { apiKey?: string + resourcePath?: string + model: string | undefined messages: ChatCompletionMessageParam[] abortController: AbortController - aiProvider: AiProviderTypes + aiProvider: AIProvider }) { - if (apiKey) { - switch (aiProvider) { - case 'openai': { - const openai = new OpenAI({ - apiKey, - dangerouslyAllowBrowser: true - }) - await openai.chat.completions.create( - { - ...OpenAi.openaiConfig, - messages, - stream: false - }, - { - signal: abortController.signal - } - ) - break - } - case 'anthropic': { - const anthropic = new Anthropic({ - apiKey, - dangerouslyAllowBrowser: true - }) - const [, anthropicMessages] = AnthropicAi.getSystemPromptAndArrayMessages(messages) - await anthropic.messages.create( - { - ...AnthropicAi.config, - messages: anthropicMessages, - stream: false - }, - { - signal: abortController.signal - } - ) - break - } - case 'mistral': { - const mistral = new Mistral({ - apiKey - }) - await mistral.chat.complete( - { - ...MistralAi.mistralConfig, - model: 'codestral-latest', - stream: false, - messages: messages as MistralAi.MistralParamsMessage[] - }, - { - fetchOptions: { - signal: abortController.signal, - headers: { - 'content-type': 'application/json' - } - } - } - ) - break - } - } - } else { - await getNonStreamingCompletion(messages, abortController, aiProvider, undefined, true) + if (!apiKey && !resourcePath) { + throw new Error('API key or resource path is required') } + const modelToTest = model ?? AI_DEFAULT_MODELS[aiProvider][0] + + if (!modelToTest) { + throw new Error('Missing a model to test') + } + + await getNonStreamingCompletion( + messages, + abortController, + aiProvider, + apiKey, + resourcePath, + modelToTest + ) } interface BaseOptions { @@ -410,7 +359,9 @@ function addDBSChema(scriptOptions: CopilotOptions, prompt: string) { const { dbSchema, language } = scriptOptions if ( dbSchema && - ['postgresql', 'mysql', 'snowflake', 'bigquery', 'mssql', 'graphql', 'oracledb'].includes(language) && // make sure we are using a SQL/query language + ['postgresql', 'mysql', 'snowflake', 'bigquery', 'mssql', 'graphql', 'oracledb'].includes( + language + ) && // make sure we are using a SQL/query language language === dbSchema.lang // make sure we are using the same language as the schema ) { let { stringified } = dbSchema @@ -467,66 +418,109 @@ const PROMPTS_CONFIGS = { export async function getNonStreamingCompletion( messages: ChatCompletionMessageParam[], abortController: AbortController, - aiProvider: AiProviderTypes, - model = OpenAi.openaiConfig.model, - noCache?: boolean + aiProvider: AIProvider, + apiKey?: string, // testing API KEY directly from the frontend + resourcePath?: string, // testing resource path passed as a header to the backend proxy + forceModel?: string ) { let response: string | undefined = '' - const queryOptions = { - query: { - no_cache: noCache - }, + let model = forceModel + + if (!model) { + model = get(copilotSessionModel) + let info = get(copilotInfo) + const { ai_models: aiModels } = info + + if (!model || !aiModels.includes(model)) { + console.warn('Invalid model, using default model:', aiModels[0]) + model = aiModels[0] + } + } + + if (!model) { + throw new Error('No model found') + } + + const fetchOptions: { + signal: AbortSignal + headers?: Record + } = { signal: abortController.signal } - switch (aiProvider) { - case 'openai': { - const openaiClient = OpenAi.workspace.getClient() - const completion = await openaiClient.chat.completions.create( - { - ...OpenAi.openaiConfig, - messages, - stream: false, - model - }, - queryOptions - ) - response = completion.choices[0]?.message.content || '' - break + if (resourcePath) { + fetchOptions.headers = { + 'X-Resource-Path': resourcePath } + } + switch (aiProvider) { case 'anthropic': { - const anthropicClient = AnthropicAi.workspace.getClient() - const [system, anthropicMessages] = AnthropicAi.getSystemPromptAndArrayMessages(messages) + const anthropicClient = apiKey + ? new Anthropic({ + apiKey, + dangerouslyAllowBrowser: true + }) + : workspaceAIClients.getAnthropicClient() + const [system, anthropicMessages] = AnthropicAI.getSystemPromptAndArrayMessages(messages) const message = await anthropicClient.messages.create( { - ...AnthropicAi.config, + ...AnthropicAI.config, system, + model, messages: anthropicMessages, stream: false }, - queryOptions + fetchOptions ) response = message.content[0].type === 'text' ? message.content[0].text : '' break } case 'mistral': { - const mistralClient = MistralAi.workspace.getClient() + const mistralClient = apiKey + ? new Mistral({ + apiKey + }) + : workspaceAIClients.getMistralClient() const message = await mistralClient.chat.complete( { - ...MistralAi.mistralConfig, - model: 'codestral-latest', + ...MistralAI.mistralConfig, + model, stream: false, - messages: messages as MistralAi.MistralParamsMessage[] + messages: messages as MistralAI.MistralParamsMessage[] }, { - fetchOptions: { - signal: abortController.signal, - cache: 'no-store' - } + fetchOptions: fetchOptions } ) - response = MistralAi.retrieveTextValue(message.choices && message.choices[0].message.content) + response = MistralAI.retrieveTextValue(message.choices && message.choices[0].message.content) break } + default: { + if (aiProvider === 'customai' && apiKey) { + throw new Error('Cannot test API key for Custom AI, only resource path is supported') + } + const baseURL = OPENAI_COMPATIBLE_BASE_URLS[aiProvider] + + if (apiKey && aiProvider !== 'openai' && !baseURL) { + throw new Error('No base URL for this provider: ' + aiProvider) + } + const openaiClient = apiKey + ? new OpenAI({ + apiKey, + baseURL, + dangerouslyAllowBrowser: true + }) + : workspaceAIClients.getOpenaiClient() + const completion = await openaiClient.chat.completions.create( + { + ...(aiProvider === 'openai' ? OpenAi.openaiConfig : DEFAULT_COMPLETION_CONFIG), + messages, + model, + stream: false + }, + fetchOptions + ) + response = completion.choices[0]?.message.content || '' + } } return response } @@ -534,17 +528,30 @@ export async function getNonStreamingCompletion( export async function getCompletion( messages: ChatCompletionMessageParam[], abortController: AbortController, - aiProvider: AiProviderTypes, - model = OpenAi.openaiConfig.model + aiProvider: AIProvider ) { + let model = get(copilotSessionModel) + let info = get(copilotInfo) + const { ai_models: aiModels } = info + + if (!model || !aiModels.includes(model)) { + console.warn('Invalid model, using default model:', aiModels[0]) + model = aiModels[0] + } + + if (!model) { + throw new Error('No model found') + } + switch (aiProvider) { case 'anthropic': { - const anthropicClient = AnthropicAi.workspace.getClient() - const [system, anthropicMessages] = AnthropicAi.getSystemPromptAndArrayMessages(messages) + const anthropicClient = workspaceAIClients.getAnthropicClient() + const [system, anthropicMessages] = AnthropicAI.getSystemPromptAndArrayMessages(messages) const completion = await anthropicClient.messages.create( { - ...AnthropicAi.config, + ...AnthropicAI.config, + model, system, messages: anthropicMessages, stream: true @@ -553,13 +560,29 @@ export async function getCompletion( ) return completion } - case 'openai': { - const openaiClient = OpenAi.workspace.getClient() + case 'mistral': { + const mistralClient = workspaceAIClients.getMistralClient() + const message = await mistralClient.chat.stream( + { + ...MistralAI.mistralConfig, + model, + messages: messages as MistralAI.MistralParamsMessage[] + }, + { + fetchOptions: { + signal: abortController.signal + } + } + ) + return message + } + default: { + const openaiClient = workspaceAIClients.getOpenaiClient() const completion = await openaiClient.chat.completions.create( { - ...OpenAi.openaiConfig, - messages, - model + ...(aiProvider === 'openai' ? OpenAi.openaiConfig : DEFAULT_COMPLETION_CONFIG), + model, + messages }, { signal: abortController.signal @@ -567,23 +590,6 @@ export async function getCompletion( ) return completion } - case 'mistral': { - const mistralClient = MistralAi.workspace.getClient() - const message = await mistralClient.chat.stream( - { - ...MistralAi.mistralConfig, - model: 'codestral-latest', - messages: messages as MistralAi.MistralParamsMessage[] - }, - { - fetchOptions: { - signal: abortController.signal, - cache: 'no-store' - } - } - ) - return message - } } } @@ -592,21 +598,20 @@ export function getResponseFromEvent( | Anthropic.Messages.RawMessageStreamEvent | OpenAI.Chat.Completions.ChatCompletionChunk | CompletionEvent, - aiProvider: AiProviderTypes + aiProvider: AIProvider ): string { switch (aiProvider) { - case 'openai': { - const messages = part as OpenAI.Chat.Completions.ChatCompletionChunk - return OpenAi.retrieveTextValue(messages) - } case 'anthropic': { const messages = part as Anthropic.Messages.RawMessageStreamEvent - return AnthropicAi.retrieveTextValue(messages) + return AnthropicAI.retrieveTextValue(messages) } case 'mistral': { const messages = part as CompletionEvent - return MistralAi.retrieveTextValue(messages.data.choices[0].delta.content) + return MistralAI.retrieveTextValue(messages.data.choices[0].delta.content) } + default: + const messages = part as OpenAI.Chat.Completions.ChatCompletionChunk + return OpenAi.retrieveTextValue(messages) } } @@ -614,7 +619,7 @@ export async function copilot( scriptOptions: CopilotOptions, generatedCode: Writable, abortController: AbortController, - aiProvider: AiProviderTypes, + aiProvider: AIProvider, generatedExplanation?: Writable ) { const { prompt, systemPrompt } = await getPrompts(scriptOptions) @@ -704,7 +709,7 @@ export async function deltaCodeCompletion( messages: ChatCompletionMessageParam[], generatedCodeDelta: Writable, abortController: AbortController, - aiProvider: AiProviderTypes + aiProvider: AIProvider ) { const completion = await getCompletion(messages, abortController, aiProvider) diff --git a/frontend/src/lib/components/details/EmailTriggerPanel.svelte b/frontend/src/lib/components/details/EmailTriggerPanel.svelte index 9b31103702..3687c1e710 100644 --- a/frontend/src/lib/components/details/EmailTriggerPanel.svelte +++ b/frontend/src/lib/components/details/EmailTriggerPanel.svelte @@ -74,6 +74,7 @@ on:updateSchema on:testWithArgs {newItem} + alwaysOpened={true} /> {:else}
diff --git a/frontend/src/lib/components/flows/content/FlowInput.svelte b/frontend/src/lib/components/flows/content/FlowInput.svelte index 00f4083f12..5ae31191b1 100644 --- a/frontend/src/lib/components/flows/content/FlowInput.svelte +++ b/frontend/src/lib/components/flows/content/FlowInput.svelte @@ -304,7 +304,7 @@ function resetArgs() { if (!previewSchema) { - previewArguments = undefined + // previewArguments = undefined savedPreviewArgs = undefined } } diff --git a/frontend/src/lib/components/flows/content/FlowModuleSuspend.svelte b/frontend/src/lib/components/flows/content/FlowModuleSuspend.svelte index 1c2ad440c6..13b0752380 100644 --- a/frontend/src/lib/components/flows/content/FlowModuleSuspend.svelte +++ b/frontend/src/lib/components/flows/content/FlowModuleSuspend.svelte @@ -122,8 +122,13 @@ + {#if Boolean(flowModule.suspend?.continue_on_disapprove_timeout)} + + We recommend using the expr resume?.error to handle null payload values. +
+ To filter timeout, use resume?.error?.name === "SuspendedTimedOut".
+ To filter disapproval, use resume?.error?.name === "SuspendedDisapproved" + + {/if}
{:else if suspendTabSelected === 'permissions'}
diff --git a/frontend/src/lib/components/flows/map/FlowJobsMenu.svelte b/frontend/src/lib/components/flows/map/FlowJobsMenu.svelte index b709d9e47b..97156f4149 100644 --- a/frontend/src/lib/components/flows/map/FlowJobsMenu.svelte +++ b/frontend/src/lib/components/flows/map/FlowJobsMenu.svelte @@ -62,7 +62,7 @@ {/if} - + -
+
import { melt } from '@melt-ui/svelte' - import { createEventDispatcher } from 'svelte' export let href: string | undefined = undefined export let disabled: boolean = false - + export let target: string | undefined = undefined export let item: any - - const dispatch = createEventDispatcher() {#if href} - + {:else} - {/if} diff --git a/frontend/src/lib/components/meltComponents/index.ts b/frontend/src/lib/components/meltComponents/index.ts index 3af39b5433..f38c7aeb3a 100644 --- a/frontend/src/lib/components/meltComponents/index.ts +++ b/frontend/src/lib/components/meltComponents/index.ts @@ -1,3 +1,4 @@ export { default as Menu } from './Menu.svelte' export { default as Menubar } from './Menubar.svelte' +export { default as MenuSingleItem } from './MenuSingleItem.svelte' export { default as MenuItem } from './MenuItem.svelte' diff --git a/frontend/src/lib/components/propertyPicker/ObjectViewer.svelte b/frontend/src/lib/components/propertyPicker/ObjectViewer.svelte index 87ab307034..31cb5d554a 100644 --- a/frontend/src/lib/components/propertyPicker/ObjectViewer.svelte +++ b/frontend/src/lib/components/propertyPicker/ObjectViewer.svelte @@ -93,7 +93,7 @@ size="xs2" variant="border" on:click={collapse} - wrapperClasses="inline-flex w-fit" + wrapperClasses="!inline-flex w-fit" btnClasses="font-mono h-4 text-2xs px-1 font-thin text-primary rounded-[0.275rem]" >- @@ -218,7 +218,7 @@ size="xs2" variant="border" on:click={collapse} - wrapperClasses="inline-flex w-fit" + wrapperClasses="!inline-flex w-fit" btnClasses="h-4 text-[9px] px-1 text-primary rounded-[0.275rem]" > {openBracket}{collapsedSymbol}{closeBracket} diff --git a/frontend/src/lib/components/runs/PreprocessedArgsDisplay.svelte b/frontend/src/lib/components/runs/PreprocessedArgsDisplay.svelte index 27ab382b51..03ce501164 100644 --- a/frontend/src/lib/components/runs/PreprocessedArgsDisplay.svelte +++ b/frontend/src/lib/components/runs/PreprocessedArgsDisplay.svelte @@ -9,15 +9,14 @@ // import { copyToClipboard } from '$lib/utils' // import { deepEqual } from 'fast-equals' - export let flowStatus: any + export let preprocessed: boolean | undefined // $: args = // '_metadata' in flowStatus && 'original_args' in flowStatus['_metadata'] // ? flowStatus['_metadata']['original_args'] // : undefined - $: hasPreprocessedArgs = - '_metadata' in flowStatus && !!flowStatus['_metadata']['preprocessed_args'] + $: hasPreprocessedArgs = preprocessed === true // $: argsStr = args !== undefined ? JSON.stringify(args, null, 4) : undefined diff --git a/frontend/src/lib/components/schema/EditableSchemaWrapper.svelte b/frontend/src/lib/components/schema/EditableSchemaWrapper.svelte index 3df59deeb9..b029f9bd83 100644 --- a/frontend/src/lib/components/schema/EditableSchemaWrapper.svelte +++ b/frontend/src/lib/components/schema/EditableSchemaWrapper.svelte @@ -104,6 +104,7 @@ {/if} dispatch('change', schema)} diff --git a/frontend/src/lib/components/schema/FlowPropertyEditor.svelte b/frontend/src/lib/components/schema/FlowPropertyEditor.svelte index e8bc5d6fb5..921f912366 100644 --- a/frontend/src/lib/components/schema/FlowPropertyEditor.svelte +++ b/frontend/src/lib/components/schema/FlowPropertyEditor.svelte @@ -304,7 +304,7 @@ {/if} {:else if type === 'object' && format !== 'resource-s3_object'} { if (e.detail === 'custom-object') { format = '' @@ -327,7 +327,7 @@ {/if} - {#if !(type === 'object' && oneOf && oneOf.length >= 2)} + {#if !(type === 'object' && oneOf && oneOf.length >= 2) && !(type == 'object' && initialObjectSelected == 'custom-object')}
diff --git a/frontend/src/lib/components/sidebar/SidebarContent.svelte b/frontend/src/lib/components/sidebar/SidebarContent.svelte index d17a992af0..7c2197cf4d 100644 --- a/frontend/src/lib/components/sidebar/SidebarContent.svelte +++ b/frontend/src/lib/components/sidebar/SidebarContent.svelte @@ -50,12 +50,9 @@ import SideBarNotification from './SideBarNotification.svelte' import KafkaIcon from '../icons/KafkaIcon.svelte' import NatsIcon from '../icons/NatsIcon.svelte' - import Menubar from '../meltComponents/Menubar.svelte' - import Menu from '../meltComponents/Menu.svelte' + import { Menubar, Menu, MenuSingleItem, MenuItem } from '$lib/components/meltComponents' import { melt } from '@melt-ui/svelte' - import MenuButtonMelt from './MenuButtonMelt.svelte' - import MenuSingleItem from '../meltComponents/MenuSingleItem.svelte' - import MenuLinkMelt from './MenuLinkMelt.svelte' + import MenuButton from './MenuButton.svelte' export let numUnacknowledgedCriticalAlerts = 0 $: mainMenuLinks = [ @@ -88,7 +85,7 @@ goto('/user/workspaces') } - const defaultExtraTriggerLinks = [ + $: defaultExtraTriggerLinks = [ { label: 'HTTP', href: '/routes', @@ -292,7 +289,6 @@ ] export let isCollapsed: boolean = false - export let noGap: boolean = false let leaveWorkspaceModal = false @@ -303,15 +299,18 @@ } return count } + + const itemClass = twMerge( + 'text-secondary font-normal w-full block px-4 py-2 text-2xs data-[highlighted]:bg-surface-hover data-[highlighted]:text-primary' + ) + {/if} + {/each} +
diff --git a/frontend/src/lib/components/sidebar/UserMenu.svelte b/frontend/src/lib/components/sidebar/UserMenu.svelte index f160696fc0..ccf3eaed4a 100644 --- a/frontend/src/lib/components/sidebar/UserMenu.svelte +++ b/frontend/src/lib/components/sidebar/UserMenu.svelte @@ -13,19 +13,25 @@ import { twMerge } from 'tailwind-merge' import { Crown, HardHat, LogOut, Moon, Settings, Sun, User } from 'lucide-svelte' import DarkModeObserver from '../DarkModeObserver.svelte' - import MenuButtonMelt from './MenuButtonMelt.svelte' + import MenuButton from './MenuButton.svelte' + import { Menu, MenuItem } from '$lib/components/meltComponents' - import { melt } from '@melt-ui/svelte' - import Menu from '$lib/components/meltComponents/Menu.svelte' let darkMode: boolean = false + export let isCollapsed: boolean = false export let lightMode: boolean = false export let createMenu: (any) => any + + const itemClass = twMerge( + 'text-secondary text-left font-normal text-xs ', + 'flex flex-row items-center gap-2 px-4 py-3 w-full', + 'data-[highlighted]:bg-surface-hover data-[highlighted]:text-primary' + ) - + - -
-
-

- {$userStore?.email} -

- - {#if $userStore?.is_admin} - Admin of this workspace - {:else if $userStore?.operator} - Operator in this workspace - {/if} - -
-
- - - Account settings - -
+
+

+ {$userStore?.email} +

+ + {#if $userStore?.is_admin} + Admin of this workspace + {:else if $userStore?.operator} + Operator in this workspace + {/if} + +
+
+ + + Account settings + -
- -
+ { + if (!document.documentElement.classList.contains('dark')) { + document.documentElement.classList.add('dark') + window.localStorage.setItem('dark-mode', 'dark') + } else { + document.documentElement.classList.remove('dark') + window.localStorage.setItem('dark-mode', 'light') + } + }} + class={itemClass} + {item} + > + {#if darkMode} + + {:else} + + {/if} + Switch theme + -
- -
+ logout()} + class={twMerge(itemClass, 'text-primary font-semibold')} + {item} + > + + Sign out + +
- {#if isCloudHosted()} + {#if isCloudHosted()} +
{#if !$isPremiumStore} -
- {$usageStore}/1000 user execs -
-
+ {$usageStore}/1000 user execs +
+
+
- {#if $workspaceStore != 'demo'} - {$workspaceUsageStore}/1000 free workspace execs -
+
+ {#if $workspaceStore != 'demo'} + {$workspaceUsageStore}/1000 free workspace execs +
+
- {/if} - {#if $userStore?.is_admin} - - {/if} -
- {:else} -
- -
+ Upgrade + + {/if} + {:else} + { + goto('/workspace_settings?tab=premium') + }} + {item} + > + Premium plan + {/if} - {/if} -
+
+ {/if}
diff --git a/frontend/src/lib/components/sidebar/WorkspaceMenu.svelte b/frontend/src/lib/components/sidebar/WorkspaceMenu.svelte index 8ce8eba2a1..bb323b4da4 100644 --- a/frontend/src/lib/components/sidebar/WorkspaceMenu.svelte +++ b/frontend/src/lib/components/sidebar/WorkspaceMenu.svelte @@ -8,8 +8,8 @@ workspaceUsageStore } from '$lib/stores' import { Building, Plus, Settings } from 'lucide-svelte' - import MenuButtonMelt from '$lib/components/sidebar/MenuButtonMelt.svelte' - import Menu from '$lib/components/meltComponents/Menu.svelte' + import MenuButton from '$lib/components/sidebar/MenuButton.svelte' + import { Menu } from '$lib/components/meltComponents' import { melt } from '@melt-ui/svelte' import { goto } from '$lib/navigation' import { base } from '$lib/base' @@ -18,7 +18,7 @@ import MultiplayerMenu from './MultiplayerMenu.svelte' import { enterpriseLicense } from '$lib/stores' import { isCloudHosted } from '$lib/cloud' - import { initAllAiWorkspace } from '../copilot/lib' + import { workspaceAIClients } from '../copilot/lib' import { twMerge } from 'tailwind-merge' export let isCollapsed: boolean = false @@ -31,7 +31,7 @@ if ($workspaceStore === id) { return } - initAllAiWorkspace(id, true) + workspaceAIClients.init(id) const editPages = [ '/scripts/edit/', '/flows/edit/', @@ -54,9 +54,9 @@ } - + - {:else if captureType === 'http'} = { - http: 'New custom HTTP route', - websocket: 'New WebSocket trigger', + http: '+ New custom HTTP route', + websocket: '+ New WebSocket trigger', webhook: 'Webhook', - kafka: 'New Kafka trigger', + kafka: '+ New Kafka trigger', email: 'Email trigger', - nats: 'NATS trigger' + nats: '+ New NATS trigger' } const { captureOn } = getContext('TriggerContext') let args: Record = {} + $: collapsed = !openForm const dispatch = createEventDispatcher() - let showCapture = false let init = false $: updateShowCapture(!!$captureOn) function updateShowCapture(show: boolean) { @@ -49,55 +52,57 @@ } -
+
-
- {#if isEditor} - - {/if} - - {#if !noSave} - {@const disabled = newItem || cloudDisabled} - + {#if !collapsed || alwaysOpened} +
+ {#if isEditor} - - {#if disabled} - {#if newItem} - Deploy the runnable to enable trigger creation - {:else if cloudDisabled} - {capitalize(triggerType)} triggers are disabled in the multi-tenant cloud + {/if} + + {#if !noSave} + {@const disabled = newItem || cloudDisabled} + + + + {#if disabled} + {#if newItem} + Deploy the runnable to enable trigger creation + {:else if cloudDisabled} + {capitalize(triggerType)} triggers are disabled in the multi-tenant cloud + {/if} + {:else} + Create new {captureTypeLabels[triggerType].toLowerCase()} {/if} - {:else} - Create new {captureTypeLabels[triggerType].toLowerCase()} - {/if} - - - {/if} -
+ + + {/if} +
+ {/if} {#if isEditor} diff --git a/frontend/src/lib/components/triggers/TriggersWrapper.svelte b/frontend/src/lib/components/triggers/TriggersWrapper.svelte index 1b958e6301..90db46c6d9 100644 --- a/frontend/src/lib/components/triggers/TriggersWrapper.svelte +++ b/frontend/src/lib/components/triggers/TriggersWrapper.svelte @@ -47,8 +47,6 @@ bind:route_path={args.route_path} bind:http_method={args.http_method} headless - {isFlow} - {path} /> {:else if triggerType === 'email'} { if (validateTimeout) { clearTimeout(validateTimeout) } validateTimeout = setTimeout(async () => { - if (!path || !method || !/^[\w-:]+(\/[\w-:]+)*$/.test(path)) { + if (!routePath || !method || !/^:?[-\w]+(\/:?[-\w]+)*$/.test(routePath)) { routeError = 'Endpoint not valid' - } else if (initialRoutePath !== path && (await routeExists(path, method))) { + } else if (await routeExists(routePath, method)) { routeError = 'Endpoint already taken' } else { routeError = '' @@ -59,14 +57,15 @@ workspace: $workspaceStore!, requestBody: { route_path, - http_method: method + http_method: method, + trigger_path: initialTriggerPath } }) } - $: captureURL = `${location.origin}${base}/api/w/${$workspaceStore}/capture_u/http/${ - isFlow ? 'flow' : 'script' - }/${path.replaceAll('/', '.')}/${route_path}` + $: validateRoute(route_path, http_method) + + $: isValid = routeError === '' function getHttpRoute(route_path: string | undefined) { return `${location.origin}${base}/api/r/${isCloudHosted() ? $workspaceStore + '/' : ''}${ @@ -74,10 +73,6 @@ }` } - $: validateRoute(route_path, http_method) - - $: isValid = routeError === '' - $: fullRoute = getHttpRoute(route_path) $: !http_method && (http_method = 'post') @@ -86,6 +81,9 @@
{#if showCapture && captureInfo} + {@const captureURL = `${location.origin}${base}/api/w/${$workspaceStore}/capture_u/http/${ + captureInfo.isFlow ? 'flow' : 'script' + }/${captureInfo.path.replaceAll('/', '.')}/${route_path}`} {#if static_asset_config} { s3Editor?.setCode(JSON.stringify(static_asset_config, null, 2)) @@ -220,102 +227,125 @@
- { - if (ev.detail === 'static_asset') { - static_asset_config = { s3: '' } - script_path = '' - initialScriptPath = '' - is_flow = false - http_method = 'get' - } else { - static_asset_config = undefined - } - }} - > - - - + {#if !isCloudHosted()} + { + if (ev.detail === 'static_asset' || ev.detail === 'static_website') { + static_asset_config = { s3: '' } + s3Editor?.setCode(JSON.stringify(static_asset_config, null, 2)) + script_path = '' + initialScriptPath = '' + is_flow = false + http_method = 'get' + is_async = false + is_static_website = ev.detail === 'static_website' + if (is_static_website) { + requires_auth = false + } + } else if (ev.detail === 'runnable') { + static_asset_config = undefined + } + }} + > + + + + + {/if} {#if static_asset_config} -
- {#if can_write} - - {/if} - {#if s3FileUploadRawMode} + {#if is_static_website} +

+ Upload or specify a folder on S3. All its files will be served under the path + above. Use this full path as the base URL of your website to ensure relative imports + work correctly. +

+ {/if} +
+
{#if can_write} - { - dispatch('focus') - }} - on:blur={(e) => { - dispatch('blur') - }} - code={JSON.stringify(static_asset_config ?? { s3: '' }, null, 2)} - bind:value={static_asset_config} - /> - {:else} - - {/if} - {#if can_write} - + options={{ left: 'Existing file' }} + disabled={!can_write} + /> {/if} - {:else} - { - static_asset_config = { - s3: evt.detail?.path ?? '', - filename: evt.detail?.filename ?? undefined - } - s3FileUploadRawMode = true - }} - on:deletion={(evt) => { - static_asset_config = { - s3: '' - } - }} - /> - {/if} + {#if s3FileUploadRawMode} + {#if can_write} + { + dispatch('focus') + }} + on:blur={(e) => { + dispatch('blur') + }} + code={JSON.stringify(static_asset_config ?? { s3: '' }, null, 2)} + bind:value={static_asset_config} + /> + {:else} + + {/if} + {#if can_write} + + {/if} + {:else} + {#key is_static_website} + { + static_asset_config = { + s3: evt.detail?.path ?? '', + filename: evt.detail?.filename ?? undefined + } + s3FileUploadRawMode = true + }} + on:deletion={(evt) => { + static_asset_config = { + s3: '' + } + }} + /> + {/key} + {/if} +
{:else} -

+

Pick a script or flow to be triggered
To handle headers, query or path parameters, add a preprocessor to your runnable.

@@ -346,48 +376,52 @@ {/if}
-
-
-
-
-
+
+ {/if}
{/if} diff --git a/frontend/src/lib/components/triggers/http/RoutesPanel.svelte b/frontend/src/lib/components/triggers/http/RoutesPanel.svelte index b05d98f1d9..4c215022d4 100644 --- a/frontend/src/lib/components/triggers/http/RoutesPanel.svelte +++ b/frontend/src/lib/components/triggers/http/RoutesPanel.svelte @@ -5,7 +5,7 @@ import { canWrite } from '$lib/utils' import { getContext, onMount } from 'svelte' import Section from '$lib/components/Section.svelte' - import { Alert, Skeleton } from '$lib/components/common' + import { Alert } from '$lib/components/common' import Description from '$lib/components/Description.svelte' import type { TriggerContext } from '$lib/components/triggers' import TriggersEditorSection from '../TriggersEditorSection.svelte' @@ -19,6 +19,8 @@ export let args: Record = {} let routeEditor: RouteEditor + let openForm = true + let dontCloseOnLoad = false $: path && loadTriggers() const { triggersCount, selectedTrigger, defaultValues } = @@ -48,6 +50,7 @@ return { canWrite: canWrite(x.path, x.extra_perms!, $userStore), ...x } }) $triggersCount = { ...($triggersCount ?? {}), http_routes_count: httpTriggers?.length } + openForm = httpTriggers?.length === 0 || dontCloseOnLoad } catch (e) { console.error('impossible to load http routes', e) } @@ -61,11 +64,42 @@ bind:this={routeEditor} /> -
+
Routes expose your scripts and flows as HTTP endpoints. Each route can be configured with a specific HTTP method and path. + {#if !newItem && httpTriggers && httpTriggers.length > 0} +
+ {#if !$userStore?.is_admin && !$userStore?.is_super_admin} + + {/if} + +
+ {#each httpTriggers as httpTriggers (httpTriggers.path)} +
+
{httpTriggers.path}
+
+ {httpTriggers.http_method.toUpperCase()} /{httpTriggers.route_path} +
+
+ +
+
+ {/each} +
+
+ {/if} + { routeEditor?.openNew(isFlow, path, e.detail.config) @@ -74,6 +108,7 @@ on:addPreprocessor on:updateSchema on:testWithArgs + bind:showCapture={dontCloseOnLoad} cloudDisabled={false} triggerType="http" {isFlow} @@ -83,43 +118,6 @@ {hasPreprocessor} {newItem} data={{ args }} + bind:openForm /> - {#if !newItem} -
- {#if !$userStore?.is_admin && !$userStore?.is_super_admin} - - {/if} - - {#if httpTriggers} - {#if httpTriggers.length == 0} -
No http routes
- {:else} -
- {#each httpTriggers as httpTriggers (httpTriggers.path)} -
-
{httpTriggers.path}
-
- {httpTriggers.http_method.toUpperCase()} /{httpTriggers.route_path} -
-
- -
-
- {/each} -
- {/if} - {:else} - - {/if} -
- {/if} -
\ No newline at end of file +
diff --git a/frontend/src/lib/components/triggers/kafka/KafkaTriggersPanel.svelte b/frontend/src/lib/components/triggers/kafka/KafkaTriggersPanel.svelte index 5e710676d1..364a976921 100644 --- a/frontend/src/lib/components/triggers/kafka/KafkaTriggersPanel.svelte +++ b/frontend/src/lib/components/triggers/kafka/KafkaTriggersPanel.svelte @@ -6,7 +6,6 @@ import KafkaTriggerEditor from './KafkaTriggerEditor.svelte' import { isCloudHosted } from '$lib/cloud' import Section from '$lib/components/Section.svelte' - import Skeleton from '$lib/components/common/skeleton/Skeleton.svelte' import Description from '$lib/components/Description.svelte' import { Alert } from '$lib/components/common' import type { TriggerContext } from '$lib/components/triggers' @@ -20,6 +19,8 @@ export let hasPreprocessor: boolean = false let kafkaTriggerEditor: KafkaTriggerEditor + let openForm = true + let dontCloseOnLoad = false $: path && loadTriggers() @@ -50,6 +51,7 @@ return { canWrite: canWrite(x.path, x.extra_perms!, $userStore), ...x } }) $triggersCount = { ...($triggersCount ?? {}), kafka_count: kafkaTriggers?.length } + openForm = kafkaTriggers?.length === 0 || dontCloseOnLoad } catch (e) { console.error('impossible to load Kafka triggers', e) } @@ -85,6 +87,32 @@ Kafka triggers execute scripts and flows in response to messages published to Kafka topics. + {#if !newItem && kafkaTriggers && kafkaTriggers.length > 0} +
+
+ {#each kafkaTriggers as kafkaTrigger (kafkaTrigger.path)} +
+
{kafkaTrigger.path}
+
+ {kafkaTrigger.kafka_resource_path} +
+
+ +
+
+ {/each} +
+
+ {/if} { saveTrigger(path, e.detail.config) @@ -102,41 +130,8 @@ {canHavePreprocessor} {hasPreprocessor} {newItem} + {openForm} + bind:showCapture={dontCloseOnLoad} /> - - {#if !newItem} - {#if kafkaTriggers} -
- {#if kafkaTriggers.length == 0} -
No Kafka triggers
- {:else} -
- {#each kafkaTriggers as kafkaTrigger (kafkaTrigger.path)} -
-
{kafkaTrigger.path}
-
- {kafkaTrigger.kafka_resource_path} -
-
- -
-
- {/each} -
- {/if} -
- {:else} - - {/if} - {/if} {/if} diff --git a/frontend/src/lib/components/triggers/nats/NatsTriggersPanel.svelte b/frontend/src/lib/components/triggers/nats/NatsTriggersPanel.svelte index c7950ebacc..12cd385e3a 100644 --- a/frontend/src/lib/components/triggers/nats/NatsTriggersPanel.svelte +++ b/frontend/src/lib/components/triggers/nats/NatsTriggersPanel.svelte @@ -6,7 +6,6 @@ import NatsTriggerEditor from './NatsTriggerEditor.svelte' import { isCloudHosted } from '$lib/cloud' import Section from '$lib/components/Section.svelte' - import Skeleton from '$lib/components/common/skeleton/Skeleton.svelte' import Alert from '$lib/components/common/alert/Alert.svelte' import Description from '$lib/components/Description.svelte' import type { TriggerContext } from '$lib/components/triggers' @@ -20,6 +19,8 @@ export let hasPreprocessor: boolean = false let natsTriggerEditor: NatsTriggerEditor + let openForm = true + let dontCloseOnLoad = false $: path && loadTriggers() @@ -50,6 +51,7 @@ return { canWrite: canWrite(x.path, x.extra_perms!, $userStore), ...x } }) $triggersCount = { ...($triggersCount ?? {}), nats_count: natsTriggers?.length } + openForm = natsTriggers?.length === 0 || dontCloseOnLoad } catch (e) { console.error('impossible to load nats triggers', e) } @@ -85,6 +87,34 @@ NATS triggers execute scripts and flows in response to messages published to NATS subjects. + + {#if !newItem && natsTriggers && natsTriggers.length > 0} +
+
+ {#each natsTriggers as natsTrigger (natsTrigger.path)} +
+
{natsTrigger.path}
+
+ {natsTrigger.nats_resource_path} +
+
+ +
+
+ {/each} +
+
+ {/if} + { saveTrigger(path, e.detail.config) @@ -100,41 +130,8 @@ {canHavePreprocessor} {hasPreprocessor} {newItem} + {openForm} + bind:showCapture={dontCloseOnLoad} /> - - {#if !newItem} - {#if natsTriggers} -
- {#if natsTriggers.length == 0} -
No nats triggers
- {:else} -
- {#each natsTriggers as natsTrigger (natsTrigger.path)} -
-
{natsTrigger.path}
-
- {natsTrigger.nats_resource_path} -
-
- -
-
- {/each} -
- {/if} -
- {:else} - - {/if} - {/if} {/if} diff --git a/frontend/src/lib/components/triggers/webhook/WebhooksPanel.svelte b/frontend/src/lib/components/triggers/webhook/WebhooksPanel.svelte index 948624646b..485a433e97 100644 --- a/frontend/src/lib/components/triggers/webhook/WebhooksPanel.svelte +++ b/frontend/src/lib/components/triggers/webhook/WebhooksPanel.svelte @@ -53,5 +53,6 @@ {canHavePreprocessor} {hasPreprocessor} {newItem} + alwaysOpened={true} /> diff --git a/frontend/src/lib/components/triggers/websocket/WebsocketTriggersPanel.svelte b/frontend/src/lib/components/triggers/websocket/WebsocketTriggersPanel.svelte index 7b633a1262..12142f8343 100644 --- a/frontend/src/lib/components/triggers/websocket/WebsocketTriggersPanel.svelte +++ b/frontend/src/lib/components/triggers/websocket/WebsocketTriggersPanel.svelte @@ -7,7 +7,7 @@ import WebsocketTriggerEditor from './WebsocketTriggerEditor.svelte' import { isCloudHosted } from '$lib/cloud' import Section from '$lib/components/Section.svelte' - import { Alert, Skeleton } from '$lib/components/common' + import { Alert } from '$lib/components/common' import Description from '$lib/components/Description.svelte' import type { TriggerContext } from '$lib/components/triggers' import TriggersEditorSection from '../TriggersEditorSection.svelte' @@ -19,6 +19,8 @@ export let hasPreprocessor: boolean = false let wsTriggerEditor: WebsocketTriggerEditor + let openForm = true + let dontCloseOnLoad = false $: path && loadTriggers() @@ -49,6 +51,7 @@ return { canWrite: canWrite(x.path, x.extra_perms!, $userStore), ...x } }) $triggersCount = { ...($triggersCount ?? {}), websocket_count: wsTriggers?.length } + openForm = wsTriggers?.length === 0 || dontCloseOnLoad } catch (e) { console.error('impossible to load WS triggers', e) } @@ -72,6 +75,36 @@ WebSocket triggers allow real-time bidirectional communication between your scripts/flows and external systems. Each trigger creates a unique WebSocket endpoint. + + {#if !newItem && wsTriggers && wsTriggers.length > 0} +
+
+
+ {#each wsTriggers as wsTriggers (wsTriggers.path)} +
+
{wsTriggers.path}
+
+ {wsTriggers.url} +
+
+ +
+
+ {/each} +
+
+
+ {/if} + { @@ -88,43 +121,7 @@ {canHavePreprocessor} {hasPreprocessor} {newItem} + {openForm} /> - - {#if !newItem} -
-
- {#if wsTriggers} - {#if wsTriggers.length == 0} -
No WebSocket triggers
- {:else} -
- {#each wsTriggers as wsTriggers (wsTriggers.path)} -
-
{wsTriggers.path}
-
- {wsTriggers.url} -
-
- -
-
- {/each} -
- {/if} - {:else} - - {/if} -
-
- {/if} {/if} diff --git a/frontend/src/lib/inferArgSig.ts b/frontend/src/lib/inferArgSig.ts index 54d63873e2..ae4ad83794 100644 --- a/frontend/src/lib/inferArgSig.ts +++ b/frontend/src/lib/inferArgSig.ts @@ -5,23 +5,23 @@ export function argSigToJsonSchemaType( | string | { resource: string | null } | { - list: - | (string | { object: { key: string; typ: any }[] }) - | { str: any } - | { object: { key: string; typ: any }[] } - | null - } + list: + | (string | { object: { key: string; typ: any }[] }) + | { str: any } + | { object: { key: string; typ: any }[] } + | null + } | { dynselect: string } | { str: string[] | null } | { object: { key: string; typ: any }[] } | { - oneof: [ - { - label: string - properties: { key: string; typ: any }[] - } - ] - }, + oneof: [ + { + label: string + properties: { key: string; typ: any }[] + } + ] + }, oldS: SchemaProperty ): void { const newS: SchemaProperty = { type: '' } @@ -105,17 +105,22 @@ export function argSigToJsonSchemaType( newS.type = 'array' if (t.list === 'int' || t.list === 'float') { newS.items = { type: 'number' } + newS.originalType = 'number[]' } else if (t.list === 'bytes') { newS.items = { type: 'string', contentEncoding: 'base64' } - } else if (t.list == 'string') { - newS.items = { type: 'string' } - } else if (t.list && typeof t.list == 'object' && 'str' in t.list) { + newS.originalType = 'bytes[]' + } else if (t.list && typeof t.list == 'object' && 'str' in t.list && t.list.str) { newS.items = { type: 'string', enum: t.list.str } + newS.originalType = 'enum[]' + } else if (t.list == 'string' || (t.list && typeof t.list == 'object' && 'str' in t.list)) { + newS.items = { type: 'string', enum: oldS.items?.enum } + newS.originalType = 'string[]' } else if (t.list && typeof t.list == 'object' && 'resource' in t.list && t.list.resource) { newS.items = { type: 'resource', resourceType: t.list.resource as string } + newS.originalType = 'resource[]' } else if ( t.list && typeof t.list == 'object' && @@ -131,8 +136,10 @@ export function argSigToJsonSchemaType( } newS.items = { type: 'object', properties: properties } + newS.originalType = 'record[]' } else { newS.items = { type: 'object' } + newS.originalType = 'object[]' } } else { newS.type = 'object' diff --git a/frontend/src/lib/stores.ts b/frontend/src/lib/stores.ts index 98886e1bc0..ec158f371a 100644 --- a/frontend/src/lib/stores.ts +++ b/frontend/src/lib/stores.ts @@ -1,13 +1,14 @@ import { BROWSER } from 'esm-env' import { derived, type Readable, writable } from 'svelte/store' +import type { IntrospectionQuery } from 'graphql' import { - type WorkspaceDefaultScripts, + type AIProvider, + type OperatorSettings, type TokenResponse, type UserWorkspaceList, - type OperatorSettings + type WorkspaceDefaultScripts } from './gen' -import type { IntrospectionQuery } from 'graphql' import { getLocalSetting } from './utils' export interface UserExt { @@ -82,13 +83,14 @@ export const userWorkspaces: Readable< } }) export const copilotInfo = writable<{ - ai_provider: string + ai_provider: AIProvider exists_ai_resource: boolean - code_completion_enabled: boolean + code_completion_model?: string + ai_models: string[] }>({ - ai_provider: '', + ai_provider: 'openai', exists_ai_resource: false, - code_completion_enabled: false + ai_models: [] }) export const codeCompletionLoading = writable(false) export const metadataCompletionEnabled = writable(true) @@ -103,6 +105,9 @@ export const vimMode = writable(getLocalSetting(VIM_MODE_SETTING_NAME) export const codeCompletionSessionEnabled = writable( getLocalSetting(CODE_COMPLETION_SETTING_NAME) != 'false' ) +export const copilotSessionModel = writable( + getLocalSetting(CODE_COMPLETION_SETTING_NAME) ?? undefined +) export const usedTriggerKinds = writable([]) type SQLBaseSchema = { diff --git a/frontend/src/lib/utils.ts b/frontend/src/lib/utils.ts index e9be17cdd9..6d2681d393 100644 --- a/frontend/src/lib/utils.ts +++ b/frontend/src/lib/utils.ts @@ -246,6 +246,8 @@ export function pointerDownOutside( options?: ClickOutsideOptions ): { destroy(): void; update(newOptions: ClickOutsideOptions): void } { const handlePointerDown = async (event: PointerEvent) => { + if (!event.isTrusted) return + if (options?.customEventName) { node.dispatchEvent( new CustomEvent(options.customEventName, { @@ -351,14 +353,13 @@ export function removeItemAll(arr: T[], value: T) { } export function emptyString(str: string | undefined | null): boolean { - return str === undefined || str === null || str === '' + return str === undefined || str === null || str === '' } export function emptyStringTrimmed(str: string | undefined | null): boolean { return str === undefined || str === null || str === '' || str.trim().length === 0 } - export function defaultIfEmptyString(str: string | undefined, dflt: string): string { return emptyString(str) ? dflt : str! } @@ -1104,3 +1105,10 @@ export function isScriptPreview(job_kind: Job['job_kind'] | undefined) { !!job_kind && (job_kind === 'preview' || job_kind === 'flowscript' || job_kind === 'appscript') ) } + +export function conditionalMelt(node: HTMLElement, meltElement: any) { + if (meltElement) { + return meltElement.action(node) + } + return { destroy: () => {} } +} diff --git a/frontend/src/routes/(root)/(logged)/+layout.svelte b/frontend/src/routes/(root)/(logged)/+layout.svelte index ab3706e4a4..85cdc7345d 100644 --- a/frontend/src/routes/(root)/(logged)/+layout.svelte +++ b/frontend/src/routes/(root)/(logged)/+layout.svelte @@ -44,14 +44,14 @@ import { syncTutorialsTodos } from '$lib/tutorialUtils' import { ArrowLeft, Search } from 'lucide-svelte' import { getUserExt } from '$lib/user' - import { initAllAiWorkspace } from '$lib/components/copilot/lib' + import { workspaceAIClients } from '$lib/components/copilot/lib' import { twMerge } from 'tailwind-merge' import OperatorMenu from '$lib/components/sidebar/OperatorMenu.svelte' import GlobalSearchModal from '$lib/components/search/GlobalSearchModal.svelte' import MenuButton from '$lib/components/sidebar/MenuButton.svelte' import { setContext } from 'svelte' import { base } from '$app/paths' - import Menubar from '$lib/components/meltComponents/Menubar.svelte' + import { Menubar } from '$lib/components/meltComponents' OpenAPI.WITH_CREDENTIALS = true let menuOpen = false @@ -241,15 +241,19 @@ let devOnly = $page.url.pathname.startsWith(base + '/scripts/dev') async function loadCopilot(workspace: string) { - initAllAiWorkspace(workspace) + workspaceAIClients.init(workspace) try { - copilotInfo.set(await WorkspaceService.getCopilotInfo({ workspace })) - } catch (err) { - console.log(err) + const info = await WorkspaceService.getCopilotInfo({ workspace }) copilotInfo.set({ - ai_provider: '', + ...info, + ai_provider: info.ai_provider ?? 'openai' + }) + } catch (err) { + copilotInfo.set({ + ai_provider: 'openai', exists_ai_resource: false, - code_completion_enabled: false + code_completion_model: undefined, + ai_models: [] }) console.error('Could not get copilot info') } @@ -380,7 +384,7 @@ >
@@ -389,10 +393,10 @@ on:click={() => { menuOpen = !menuOpen }} - class="ml-1 flex items-center justify-center h-8 w-8 rounded-full focus:outline-none focus:ring-2 focus:ring-inset focus:ring-white border border-white" + class="ml-1 flex items-center justify-center h-6 w-6 rounded-full focus:outline-none focus:ring-2 focus:ring-inset focus:ring-white border border-white" >
-
+
Windmill
-
+
@@ -468,8 +472,8 @@ {/if}
-
- +
+ @@ -515,6 +519,7 @@
{/if} +
x.path.split('/').slice(0, 2).join('/')) ?? []) ).sort() : Array.from( - new Set(filteredItems?.map((x) => x.script_path.split('/').slice(0, 2).join('/')) ?? []) + new Set( + filteredItems + ?.filter((x) => x.script_path) + .map((x) => x.script_path.split('/').slice(0, 2).join('/')) ?? [] + ) ).sort() $: items = filter !== '' ? filteredItems : preFilteredItems @@ -156,176 +160,177 @@ f={(x) => (x.summary ?? '') + ' ' + x.path + ' (' + x.script_path + ')'} /> -{#if $userStore?.operator && $workspaceStore && !$userWorkspaces.find(_ => _.id === $workspaceStore)?.operator_settings?.triggers} - -{:else} - - - {#if $userStore?.is_admin || $userStore?.is_super_admin} - - {/if} - -
-
- -
-
Filter by path of
- - - - -
- - -
- {#if $userStore?.is_super_admin && $userStore.username.includes('@')} - - {:else if $userStore?.is_admin || $userStore?.is_super_admin} - - {/if} -
-
- {#if loading} - {#each new Array(6) as _} - - {/each} - {:else if !triggers?.length} -
No routes
- {:else if items?.length} -
- {#each items.slice(0, nbDisplayed) as { path, edited_by, edited_at, script_path, route_path, is_flow, extra_perms, canWrite, marked, http_method, static_asset_config } (path)} - {@const href = `${is_flow ? '/flows/get' : '/scripts/get'}/${script_path}`} - -
-
- - - routeEditor?.openEdit(path, is_flow)} - class="min-w-0 grow hover:underline decoration-gray-400" - > -
- {#if marked} - - {@html marked} - - {:else} - {http_method.toUpperCase()} /{route_path} - {/if} -
-
- {path} -
-
- {#if static_asset_config} - file: {static_asset_config.s3} - {:else} - runnable: {script_path} - {/if} -
- - - - -
- - { - goto(href) - } - }, - { - displayName: 'Delete', - type: 'delete', - icon: Trash, - disabled: !canWrite || !($userStore?.is_admin || $userStore?.is_super_admin), - action: async () => { - await HttpTriggerService.deleteHttpTrigger({ - workspace: $workspaceStore ?? '', - path - }) - loadTriggers() - } - }, - { - displayName: canWrite ? 'Edit' : 'View', - icon: canWrite ? Pen : Eye, - action: () => { - routeEditor?.openEdit(path, is_flow) - } - }, - { - displayName: 'Audit logs', - icon: Eye, - href: `${base}/audit_logs?resource=${path}` - }, - { - displayName: canWrite ? 'Share' : 'See Permissions', - icon: Share, - action: () => { - shareModal.openDrawer(path, 'http_trigger') - } - } - ]} - /> -
-
-
-
-
edited by {edited_by}
-
at {displayDate(edited_at)}
-
-
-
- {/each} -
- {:else} - - {/if} +{#if $userStore?.operator && $workspaceStore && !$userWorkspaces.find((_) => _.id === $workspaceStore)?.operator_settings?.triggers} + - {#if items && items?.length > 15 && nbDisplayed < items.length} - {nbDisplayed} items out of {items.length} - + - {/if} - + {#if $userStore?.is_admin || $userStore?.is_super_admin} + + {/if} + +
+
+ +
+
Filter by path of
+ + + + +
+ + +
+ {#if $userStore?.is_super_admin && $userStore.username.includes('@')} + + {:else if $userStore?.is_admin || $userStore?.is_super_admin} + + {/if} +
+
+ {#if loading} + {#each new Array(6) as _} + + {/each} + {:else if !triggers?.length} +
No routes
+ {:else if items?.length} +
+ {#each items.slice(0, nbDisplayed) as { path, edited_by, edited_at, script_path, route_path, is_flow, extra_perms, canWrite, marked, http_method, static_asset_config } (path)} + {@const href = `${is_flow ? '/flows/get' : '/scripts/get'}/${script_path}`} + +
+
+ + + routeEditor?.openEdit(path, is_flow)} + class="min-w-0 grow hover:underline decoration-gray-400" + > +
+ {#if marked} + + {@html marked} + + {:else} + {http_method.toUpperCase()} /{route_path} + {/if} +
+
+ {path} +
+
+ {#if static_asset_config} + file: {static_asset_config.s3} + {:else} + runnable: {script_path} + {/if} +
+ + + + +
+ + { + goto(href) + } + }, + { + displayName: 'Delete', + type: 'delete', + icon: Trash, + disabled: + !canWrite || !($userStore?.is_admin || $userStore?.is_super_admin), + action: async () => { + await HttpTriggerService.deleteHttpTrigger({ + workspace: $workspaceStore ?? '', + path + }) + loadTriggers() + } + }, + { + displayName: canWrite ? 'Edit' : 'View', + icon: canWrite ? Pen : Eye, + action: () => { + routeEditor?.openEdit(path, is_flow) + } + }, + { + displayName: 'Audit logs', + icon: Eye, + href: `${base}/audit_logs?resource=${path}` + }, + { + displayName: canWrite ? 'Share' : 'See Permissions', + icon: Share, + action: () => { + shareModal.openDrawer(path, 'http_trigger') + } + } + ]} + /> +
+
+
+
+
edited by {edited_by}
+
at {displayDate(edited_at)}
+
+
+
+ {/each} +
+ {:else} + + {/if} +
+ {#if items && items?.length > 15 && nbDisplayed < items.length} + {nbDisplayed} items out of {items.length} + + {/if} + {/if} {/if} {#if job && job.flow_status && job.job_kind === 'script'} - + {/if} {#if persistentScriptDefinition}
@@ -230,9 +248,12 @@
- + + + +
@@ -243,7 +264,12 @@ bind:value={aiKey} on:keyup={handleKeyUp} /> - +
{#if aiKey} { - // in JS, an empty string is also falsy - aiResourceInitialPath = aiResourcePath - aiResourceInitialProvider = aiProvider + async function editCopilotConfig(): Promise { if (aiResourcePath) { await WorkspaceService.editCopilotConfig({ workspace: $workspaceStore!, @@ -203,26 +205,30 @@ path: aiResourcePath, provider: aiProvider }, - code_completion_enabled: codeCompletionEnabled + code_completion_model: codeCompletionModel, + ai_models: aiModels } }) copilotInfo.set({ ai_provider: aiProvider, exists_ai_resource: true, - code_completion_enabled: codeCompletionEnabled + code_completion_model: codeCompletionModel, + ai_models: aiModels }) } else { await WorkspaceService.editCopilotConfig({ workspace: $workspaceStore!, requestBody: { ai_resource: undefined, - code_completion_enabled: codeCompletionEnabled + code_completion_model: codeCompletionModel, + ai_models: [] } }) copilotInfo.set({ - ai_provider: '', + ai_provider: 'openai', exists_ai_resource: false, - code_completion_enabled: codeCompletionEnabled + code_completion_model: codeCompletionModel, + ai_models: [] }) } sendUserToast(`Copilot settings updated`) @@ -382,6 +388,7 @@ }, 1000 - (timeEnd - timeStart)) } + let loadedSettings = false async function loadSettings(): Promise { const settings = await WorkspaceService.getSettings({ workspace: $workspaceStore! }) team_name = settings.slack_name @@ -395,9 +402,12 @@ customer_id = settings.customer_id workspaceToDeployTo = settings.deploy_to webhook = settings.webhook - aiResourceInitialPath = settings.ai_resource?.path - aiResourceInitialProvider = settings.ai_resource?.provider - selected = (aiResourceInitialProvider as AiProviderTypes) ?? 'openai' + + aiResourcePath = settings.ai_resource?.path + aiProvider = settings.ai_resource?.provider ?? 'openai' + codeCompletionModel = settings.code_completion_model + aiModels = settings.ai_models + errorHandlerItemKind = settings.error_handler?.split('/')[0] as 'flow' | 'script' errorHandlerScriptPath = (settings.error_handler ?? '').split('/').slice(1).join('/') errorHandlerInitialScriptPath = errorHandlerScriptPath @@ -415,7 +425,6 @@ : 'custom' } errorHandlerExtraArgs = settings.error_handler_extra_args ?? {} - codeCompletionEnabled = settings.code_completion_enabled workspaceDefaultAppPath = settings.default_app s3ResourceSettings = convertBackendSettingsToFrontendSettings(settings.large_file_storage) @@ -508,6 +517,8 @@ workspace: $workspaceStore!, path: 'openai_client_credentials_oauth' }) + + loadedSettings = true } let deployUiSettings: { @@ -698,7 +709,9 @@
- {#if tab == 'users'} + {#if !loadedSettings} + + {:else if tab == 'users'} {:else if tab == 'deploy_to'}
@@ -1044,43 +1057,104 @@
- { - aiResourceInitialPath = '' - aiResourceInitialProvider = '' - }} - > - - - - -
- {#key [aiResourceInitialPath, aiResourceInitialProvider, usingOpenaiClientCredentialsOauth, selected]} - { - editCopilotConfig(ev.detail, selected) - }} - /> - - {/key} -
-
- { - editCopilotConfig(aiResourceInitialPath || '', aiResourceInitialProvider || '') + +
+ { + aiResourcePath = '' + aiModels = [] + codeCompletionModel = undefined }} - /> + > + + + + + + + + +
+ {#key aiProvider} + { + if (aiResourcePath && aiModels.length === 0) { + if (aiProvider !== 'customai') { + aiModels = AI_DEFAULT_MODELS[aiProvider].slice(0, 1) + } + } + }} + /> + + {/key} +
+ + {#if aiResourcePath} + + +
+ { + if (codeCompletionModel != undefined) { + codeCompletionModel = undefined + } else { + codeCompletionModel = AI_DEFAULT_MODELS[aiProvider][0] ?? '' + } + }} + checked={codeCompletionModel != undefined} + options={{ + right: 'Code completion' + }} + /> + + {#if codeCompletionModel != undefined} + + {/if} +
+ {/if} +
{:else if tab == 'windmill_lfs'}
diff --git a/lsp/Pipfile b/lsp/Pipfile index 940df30fff..4808e50712 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,8 +4,8 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.456.0" -wmill_pg = ">=1.456.0" +wmill = ">=1.458.4" +wmill_pg = ">=1.458.4" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index 2198173813..252d9ced7e 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.456.0 + version: 1.458.4 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index aac05daa78..4964ae3ec4 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. - ModuleVersion = '1.456.0' + ModuleVersion = '1.458.4' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index 0ee4ed7b5b..03ec4e6c26 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.456.0" +version = "1.458.4" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/python-client/wmill_pg/pyproject.toml b/python-client/wmill_pg/pyproject.toml index 2c213d557c..871e6fa5c4 100644 --- a/python-client/wmill_pg/pyproject.toml +++ b/python-client/wmill_pg/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill-pg" -version = "1.456.0" +version = "1.458.4" description = "An extension client for the wmill client library focused on pg" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index 0947f2c0ad..6d21e4d8c7 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.456.0", + "version": "1.458.4", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index 347362bc92..5357cc3dd2 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.456.0", + "version": "1.458.4", "author": "Ruben Fiszel", "license": "Apache 2.0", "devDependencies": { diff --git a/version.txt b/version.txt index aaa526a611..d22eaab6f1 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.456.0 +1.459.0