From 3535016608b48fade48d87dd77656e528ab1190d Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 17 Feb 2025 11:49:06 +0100 Subject: [PATCH 01/21] improve version detection from source --- backend/windmill-common/src/utils.rs | 7 +++++-- backend/windmill-common/src/worker.rs | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/backend/windmill-common/src/utils.rs b/backend/windmill-common/src/utils.rs index 735c3d4a27..0a6117999a 100644 --- a/backend/windmill-common/src/utils.rs +++ b/backend/windmill-common/src/utils.rs @@ -47,8 +47,11 @@ lazy_static::lazy_static! { .connect_timeout(std::time::Duration::from_secs(10)) .build().unwrap(); pub static ref GIT_SEM_VERSION: Version = Version::parse( - // skip first `v` character. - GIT_VERSION.split_at(1).1 + if GIT_VERSION.starts_with('v') { + &GIT_VERSION[1..] + } else { + GIT_VERSION + } ).unwrap_or(Version::new(0, 1, 0)); } diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 487a217d7f..eb5bd1d6f1 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -657,7 +657,7 @@ pub async fn update_min_version<'c, E: sqlx::Executor<'c, Database = sqlx::Postg let min_version = pings .iter() .filter(|x| !x.is_empty()) - .filter_map(|x| semver::Version::parse(x.split_at(1).1).ok()) + .filter_map(|x| semver::Version::parse(if x.starts_with('v') { &x[1..] } else { x }).ok()) .min() .unwrap_or_else(|| cur_version.clone()); From 3493185e2f6c66324f2de16b63cafd135c9aab4a Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 17 Feb 2025 14:34:22 +0100 Subject: [PATCH 02/21] nits --- backend/windmill-api/src/jobs.rs | 2 +- .../(logged)/workspace_settings/+page.svelte | 24 +++++++++---------- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 405a12cd1a..6b0efa485d 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -1161,7 +1161,7 @@ pub struct ListableCompletedJob { pub parent_job: Option, pub created_by: String, pub created_at: chrono::DateTime, - pub started_at: chrono::DateTime, + pub started_at: Option>, pub duration_ms: i64, pub success: bool, #[serde(skip_serializing_if = "Option::is_none")] diff --git a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte index 55726138c0..1a060debfb 100644 --- a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte @@ -447,11 +447,14 @@ if (emptyString($enterpriseLicense)) { errorHandlerSelected = 'custom' } else { - errorHandlerSelected = - emptyString(errorHandlerScriptPath) ? 'custom' : - (errorHandlerScriptPath.startsWith('hub/') && errorHandlerScriptPath.endsWith('/workspace-or-schedule-error-handler-slack')) ? 'slack' : - (errorHandlerScriptPath.endsWith('/workspace-or-schedule-error-handler-teams')) ? 'teams' : - 'custom' + errorHandlerSelected = emptyString(errorHandlerScriptPath) + ? 'custom' + : errorHandlerScriptPath.startsWith('hub/') && + errorHandlerScriptPath.endsWith('/workspace-or-schedule-error-handler-slack') + ? 'slack' + : errorHandlerScriptPath.endsWith('/workspace-or-schedule-error-handler-teams') + ? 'teams' + : 'custom' } errorHandlerExtraArgs = settings.error_handler_extra_args ?? {} workspaceDefaultAppPath = settings.default_app @@ -812,8 +815,8 @@ {#if !$enterpriseLicense}
- Workspace Teams commands is a Windmill EE feature. It enables using your current Slack / Teams - connection to run a custom script and send notifications. + Workspace Teams commands is a Windmill EE feature. It enables using your current Slack + / Teams connection to run a custom script and send notifications.
{/if} @@ -867,12 +870,7 @@
- {#if $superadmin} -

- When deleting the workspace, it will be archived for a short period of time and then - permanently deleted. -

- {:else} + {#if !$superadmin}

Only instance superadmins can delete a workspace.

{/if} {#if $workspaceStore === 'admins' || $workspaceStore === 'starter'} From 0208f53541473aa51bed0e15d938def3d4530e3f Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 17 Feb 2025 15:20:39 +0100 Subject: [PATCH 03/21] fix: windmill_admin has implicit bypass rls on v2_job even if role not set --- ...0205131519_windmill_admin_skip_bypassrls_on_v2_job.down.sql | 1 + ...250205131519_windmill_admin_skip_bypassrls_on_v2_job.up.sql | 3 +++ 2 files changed, 4 insertions(+) create mode 100644 backend/migrations/20250205131519_windmill_admin_skip_bypassrls_on_v2_job.down.sql create mode 100644 backend/migrations/20250205131519_windmill_admin_skip_bypassrls_on_v2_job.up.sql diff --git a/backend/migrations/20250205131519_windmill_admin_skip_bypassrls_on_v2_job.down.sql b/backend/migrations/20250205131519_windmill_admin_skip_bypassrls_on_v2_job.down.sql new file mode 100644 index 0000000000..d2f607c5b8 --- /dev/null +++ b/backend/migrations/20250205131519_windmill_admin_skip_bypassrls_on_v2_job.down.sql @@ -0,0 +1 @@ +-- Add down migration script here diff --git a/backend/migrations/20250205131519_windmill_admin_skip_bypassrls_on_v2_job.up.sql b/backend/migrations/20250205131519_windmill_admin_skip_bypassrls_on_v2_job.up.sql new file mode 100644 index 0000000000..f83ee7f348 --- /dev/null +++ b/backend/migrations/20250205131519_windmill_admin_skip_bypassrls_on_v2_job.up.sql @@ -0,0 +1,3 @@ +-- Add up migration script here +DROP POLICY IF EXISTS admin_policy ON v2_job; +CREATE POLICY admin_policy ON v2_job FOR ALL TO windmill_admin USING (true); From fe337293dace8c36e8b577ff3302619996fc9f1f Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 17 Feb 2025 15:41:06 +0100 Subject: [PATCH 04/21] chore(main): release 1.463.3 (#5308) * chore(main): release 1.463.3 * Apply automatic changes --------- Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com> --- CHANGELOG.md | 7 ++ backend/Cargo.lock | 66 +++++++++---------- backend/Cargo.toml | 4 +- backend/windmill-api/openapi.yaml | 2 +- benchmarks/lib.ts | 2 +- cli/main.ts | 2 +- frontend/package-lock.json | 4 +- frontend/package.json | 2 +- lsp/Pipfile | 4 +- openflow.openapi.yaml | 2 +- .../WindmillClient/WindmillClient.psd1 | 2 +- python-client/wmill/pyproject.toml | 2 +- python-client/wmill_pg/pyproject.toml | 2 +- typescript-client/jsr.json | 2 +- typescript-client/package.json | 2 +- version.txt | 2 +- 16 files changed, 57 insertions(+), 50 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 897d3205a1..9c14a34e5b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [1.463.3](https://github.com/windmill-labs/windmill/compare/v1.463.2...v1.463.3) (2025-02-17) + + +### Bug Fixes + +* windmill_admin has implicit bypass rls on v2_job even if role not set ([0208f53](https://github.com/windmill-labs/windmill/commit/0208f53541473aa51bed0e15d938def3d4530e3f)) + ## [1.463.2](https://github.com/windmill-labs/windmill/compare/v1.463.1...v1.463.2) (2025-02-16) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index f9b0598ab1..7d86e09e30 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -6795,7 +6795,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94" dependencies = [ "rand_chacha 0.9.0", - "rand_core 0.9.0", + "rand_core 0.9.1", "zerocopy 0.8.18", ] @@ -6826,7 +6826,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core 0.9.0", + "rand_core 0.9.1", ] [[package]] @@ -6849,9 +6849,9 @@ dependencies = [ [[package]] name = "rand_core" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b08f3c9802962f7e1b25113931d94f43ed9725bebc59db9d0c3e9a23b67e15ff" +checksum = "a88e0da7a2c97baa202165137c158d0a2e824ac465d13d81046727b34cb247d3" dependencies = [ "getrandom 0.3.1", "zerocopy 0.8.18", @@ -7575,9 +7575,9 @@ checksum = "6ea1a2d0a644769cc99faa24c3ad26b379b786fe7c36fd3c546254801650e6dd" [[package]] name = "ryu-js" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad97d4ce1560a5e27cec89519dc8300d1aa6035b099821261c651486a19e44d5" +checksum = "dd29631678d6fb0903b69223673e122c32e9ae559d0960a38d574695ebc0ea15" [[package]] name = "safetensors" @@ -9297,9 +9297,9 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.16.0" +version = "3.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38c246215d7d24f48ae091a2902398798e05d978b24315d6efbc00ede9a8bb91" +checksum = "a40f762a77d2afa88c2d919489e390a12bdd261ed568e60cfa7e48d4e20f0d33" dependencies = [ "cfg-if", "fastrand 2.3.0", @@ -10858,7 +10858,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill" -version = "1.463.2" +version = "1.463.3" dependencies = [ "anyhow", "axum", @@ -10901,7 +10901,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.463.2" +version = "1.463.3" dependencies = [ "anyhow", "argon2", @@ -10995,7 +10995,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.463.2" +version = "1.463.3" dependencies = [ "base64 0.22.1", "chrono", @@ -11013,7 +11013,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.463.2" +version = "1.463.3" dependencies = [ "chrono", "serde", @@ -11026,7 +11026,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.463.2" +version = "1.463.3" dependencies = [ "anyhow", "serde", @@ -11040,7 +11040,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.463.2" +version = "1.463.3" dependencies = [ "anyhow", "async-stream", @@ -11099,7 +11099,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.463.2" +version = "1.463.3" dependencies = [ "regex", "serde", @@ -11113,7 +11113,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.463.2" +version = "1.463.3" dependencies = [ "anyhow", "bytes", @@ -11136,7 +11136,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.463.2" +version = "1.463.3" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -11148,7 +11148,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.463.2" +version = "1.463.3" dependencies = [ "convert_case 0.6.0", "serde", @@ -11157,7 +11157,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.463.2" +version = "1.463.3" dependencies = [ "anyhow", "lazy_static", @@ -11169,7 +11169,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.463.2" +version = "1.463.3" dependencies = [ "anyhow", "serde_json", @@ -11181,7 +11181,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.463.2" +version = "1.463.3" dependencies = [ "anyhow", "gosyn", @@ -11193,7 +11193,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.463.2" +version = "1.463.3" dependencies = [ "anyhow", "lazy_static", @@ -11205,7 +11205,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.463.2" +version = "1.463.3" dependencies = [ "anyhow", "itertools 0.14.0", @@ -11216,7 +11216,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.463.2" +version = "1.463.3" dependencies = [ "anyhow", "itertools 0.14.0", @@ -11227,7 +11227,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.463.2" +version = "1.463.3" dependencies = [ "anyhow", "async-recursion", @@ -11247,7 +11247,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.463.2" +version = "1.463.3" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -11264,7 +11264,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.463.2" +version = "1.463.3" dependencies = [ "anyhow", "lazy_static", @@ -11276,7 +11276,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.463.2" +version = "1.463.3" dependencies = [ "anyhow", "lazy_static", @@ -11294,7 +11294,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.463.2" +version = "1.463.3" dependencies = [ "anyhow", "getrandom 0.2.15", @@ -11316,7 +11316,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.463.2" +version = "1.463.3" dependencies = [ "anyhow", "serde_json", @@ -11326,7 +11326,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.463.2" +version = "1.463.3" dependencies = [ "anyhow", "async-recursion", @@ -11359,7 +11359,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.463.2" +version = "1.463.3" dependencies = [ "wasm-bindgen", "wasm-bindgen-test", @@ -11369,7 +11369,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.463.2" +version = "1.463.3" dependencies = [ "anyhow", "async-recursion", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 69ae45d5ca..72c133e293 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.463.2" +version = "1.463.3" authors.workspace = true edition.workspace = true @@ -30,7 +30,7 @@ members = [ ] [workspace.package] -version = "1.463.2" +version = "1.463.3" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 7be948a7a0..d22a3886d3 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.463.2 + version: 1.463.3 title: Windmill API contact: diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 29292e5b21..32b434eb2f 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.463.2"; +export const VERSION = "v1.463.3"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/main.ts b/cli/main.ts index 28a6436cc6..165ce2e8c8 100644 --- a/cli/main.ts +++ b/cli/main.ts @@ -62,7 +62,7 @@ export { // } // }); -export const VERSION = "1.463.2"; +export const VERSION = "1.463.3"; const command = new Command() .name("wmill") diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 39e870f751..b4fa6c0b8a 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.463.2", + "version": "1.463.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.463.2", + "version": "1.463.3", "license": "AGPL-3.0", "dependencies": { "@anthropic-ai/sdk": "^0.32.1", diff --git a/frontend/package.json b/frontend/package.json index b9d79dccd1..a22bfb038b 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.463.2", + "version": "1.463.3", "scripts": { "dev": "vite dev", "build": "vite build", diff --git a/lsp/Pipfile b/lsp/Pipfile index c519e3a81c..5e8f57d701 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,8 +4,8 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.463.2" -wmill_pg = ">=1.463.2" +wmill = ">=1.463.3" +wmill_pg = ">=1.463.3" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index a2e58e4261..bc0eed2d61 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.463.2 + version: 1.463.3 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index f73febb17e..b1e9637ce1 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. - ModuleVersion = '1.463.2' + ModuleVersion = '1.463.3' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index fee5c46ce8..1942a92887 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.463.2" +version = "1.463.3" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/python-client/wmill_pg/pyproject.toml b/python-client/wmill_pg/pyproject.toml index 7df9b85b77..1cf8de80e6 100644 --- a/python-client/wmill_pg/pyproject.toml +++ b/python-client/wmill_pg/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill-pg" -version = "1.463.2" +version = "1.463.3" description = "An extension client for the wmill client library focused on pg" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index 908d2f3417..d513475883 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.463.2", + "version": "1.463.3", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index 37b158cd14..3c2ff4eee0 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.463.2", + "version": "1.463.3", "author": "Ruben Fiszel", "license": "Apache 2.0", "devDependencies": { diff --git a/version.txt b/version.txt index fbc30c990c..a970451822 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.463.2 +1.463.3 From 5e22690bd9257d6c515c07b9357f955c2fcbb298 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 17 Feb 2025 17:48:02 +0100 Subject: [PATCH 05/21] fix: improve que job indices for faster performances --- ...5b252449bd569df374e40ce8820fc3d75a0f0.json | 12 +++++++++ ...4ff1d599049ebefdaf97a017c9cef8d52ce20.json | 12 +++++++++ ...cfecf48305f5f4b644b3c35355074e1ccce28.json | 12 +++++++++ ...d9347206d2deaa99f9a4541101e610f84a50a.json | 12 +++++++++ ...05d394a7cbcf0038c72a78add5c7b02ef5927.json | 2 +- backend/windmill-api/src/db.rs | 17 ++++++++++++ backend/windmill-common/src/lib.rs | 26 +++++++++---------- 7 files changed, 79 insertions(+), 14 deletions(-) create mode 100644 backend/.sqlx/query-0efb16cbf130ec6e9922ecc82a95b252449bd569df374e40ce8820fc3d75a0f0.json create mode 100644 backend/.sqlx/query-3738096c29ab9d964be8a74bfd14ff1d599049ebefdaf97a017c9cef8d52ce20.json create mode 100644 backend/.sqlx/query-3bbde0fa35d935ec2dd8bd1fb14cfecf48305f5f4b644b3c35355074e1ccce28.json create mode 100644 backend/.sqlx/query-8263fe28097e094cbdbdcd16668d9347206d2deaa99f9a4541101e610f84a50a.json diff --git a/backend/.sqlx/query-0efb16cbf130ec6e9922ecc82a95b252449bd569df374e40ce8820fc3d75a0f0.json b/backend/.sqlx/query-0efb16cbf130ec6e9922ecc82a95b252449bd569df374e40ce8820fc3d75a0f0.json new file mode 100644 index 0000000000..2f17b5e8db --- /dev/null +++ b/backend/.sqlx/query-0efb16cbf130ec6e9922ecc82a95b252449bd569df374e40ce8820fc3d75a0f0.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DROP INDEX CONCURRENTLY IF EXISTS queue_sort", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "0efb16cbf130ec6e9922ecc82a95b252449bd569df374e40ce8820fc3d75a0f0" +} diff --git a/backend/.sqlx/query-3738096c29ab9d964be8a74bfd14ff1d599049ebefdaf97a017c9cef8d52ce20.json b/backend/.sqlx/query-3738096c29ab9d964be8a74bfd14ff1d599049ebefdaf97a017c9cef8d52ce20.json new file mode 100644 index 0000000000..d804949078 --- /dev/null +++ b/backend/.sqlx/query-3738096c29ab9d964be8a74bfd14ff1d599049ebefdaf97a017c9cef8d52ce20.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DROP INDEX CONCURRENTLY IF EXISTS queue_sort_2", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "3738096c29ab9d964be8a74bfd14ff1d599049ebefdaf97a017c9cef8d52ce20" +} diff --git a/backend/.sqlx/query-3bbde0fa35d935ec2dd8bd1fb14cfecf48305f5f4b644b3c35355074e1ccce28.json b/backend/.sqlx/query-3bbde0fa35d935ec2dd8bd1fb14cfecf48305f5f4b644b3c35355074e1ccce28.json new file mode 100644 index 0000000000..815f1fed70 --- /dev/null +++ b/backend/.sqlx/query-3bbde0fa35d935ec2dd8bd1fb14cfecf48305f5f4b644b3c35355074e1ccce28.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "CREATE INDEX CONCURRENTLY queue_sort_v2 ON v2_job_queue (priority DESC NULLS LAST, scheduled_for, tag) WHERE running = false", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "3bbde0fa35d935ec2dd8bd1fb14cfecf48305f5f4b644b3c35355074e1ccce28" +} diff --git a/backend/.sqlx/query-8263fe28097e094cbdbdcd16668d9347206d2deaa99f9a4541101e610f84a50a.json b/backend/.sqlx/query-8263fe28097e094cbdbdcd16668d9347206d2deaa99f9a4541101e610f84a50a.json new file mode 100644 index 0000000000..aa744f6f08 --- /dev/null +++ b/backend/.sqlx/query-8263fe28097e094cbdbdcd16668d9347206d2deaa99f9a4541101e610f84a50a.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "CREATE INDEX CONCURRENTLY queue_sort_2_v2 ON v2_job_queue (priority DESC NULLS LAST, scheduled_for) WHERE running = false", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "8263fe28097e094cbdbdcd16668d9347206d2deaa99f9a4541101e610f84a50a" +} diff --git a/backend/.sqlx/query-ddf2eccb78a310ed00c7d8b9c3f05d394a7cbcf0038c72a78add5c7b02ef5927.json b/backend/.sqlx/query-ddf2eccb78a310ed00c7d8b9c3f05d394a7cbcf0038c72a78add5c7b02ef5927.json index 5bfff47576..c2dfed73a2 100644 --- a/backend/.sqlx/query-ddf2eccb78a310ed00c7d8b9c3f05d394a7cbcf0038c72a78add5c7b02ef5927.json +++ b/backend/.sqlx/query-ddf2eccb78a310ed00c7d8b9c3f05d394a7cbcf0038c72a78add5c7b02ef5927.json @@ -15,7 +15,7 @@ ] }, "nullable": [ - true + null ] }, "hash": "ddf2eccb78a310ed00c7d8b9c3f05d394a7cbcf0038c72a78add5c7b02ef5927" diff --git a/backend/windmill-api/src/db.rs b/backend/windmill-api/src/db.rs index a034569fda..106b29a810 100644 --- a/backend/windmill-api/src/db.rs +++ b/backend/windmill-api/src/db.rs @@ -655,6 +655,23 @@ async fn fix_job_completed_index(db: &DB) -> Result<(), Error> { .await?; }); + run_windmill_migration!("v2_improve_v2_queued_jobs_indices", &db, |tx| { + sqlx::query!("CREATE INDEX CONCURRENTLY queue_sort_v2 ON v2_job_queue (priority DESC NULLS LAST, scheduled_for, tag) WHERE running = false") + .execute(db) + .await?; + + sqlx::query!("CREATE INDEX CONCURRENTLY queue_sort_2_v2 ON v2_job_queue (tag, priority DESC NULLS LAST, scheduled_for) WHERE running = false") + .execute(db) + .await?; + + sqlx::query!("DROP INDEX CONCURRENTLY IF EXISTS queue_sort") + .execute(db) + .await?; + + sqlx::query!("DROP INDEX CONCURRENTLY IF EXISTS queue_sort_2") + .execute(db) + .await?; + }); Ok(()) } diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index 802af6196b..396eafdf7f 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -281,7 +281,7 @@ pub async fn connect_db( pub async fn connect( database_url: &str, max_connections: u32, - worker_mode: bool, + _worker_mode: bool, ) -> Result, error::Error> { use std::time::Duration; @@ -289,18 +289,18 @@ pub async fn connect( .min_connections((max_connections / 5).clamp(3, max_connections)) .max_connections(max_connections) .max_lifetime(Duration::from_secs(30 * 60)) // 30 mins - .after_connect(move |conn, _| { - if worker_mode { - Box::pin(async move { - sqlx::query("SET enable_seqscan = OFF;") - .execute(conn) - .await?; - Ok(()) - }) - } else { - Box::pin(async move { Ok(()) }) - } - }) + // .after_connect(move |conn, _| { + // if worker_mode { + // Box::pin(async move { + // sqlx::query("SET enable_seqscan = OFF;") + // .execute(conn) + // .await?; + // Ok(()) + // }) + // } else { + // Box::pin(async move { Ok(()) }) + // } + // }) .connect_with( sqlx::postgres::PgConnectOptions::from_str(database_url)?.statement_cache_capacity(400), ) From 85c56e9450773c03a44a4c99821717de5aef287f Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 17 Feb 2025 19:17:30 +0100 Subject: [PATCH 06/21] fix: improve que job indices for faster performances --- ...d9347206d2deaa99f9a4541101e610f84a50a.json | 12 ---- backend/windmill-api/src/db.rs | 6 +- backend/windmill-api/src/jobs.rs | 7 +- benchmarks/benchmark_oneoff.ts | 64 ++++++++++++++----- benchmarks/benchmark_suite.ts | 2 +- 5 files changed, 58 insertions(+), 33 deletions(-) delete mode 100644 backend/.sqlx/query-8263fe28097e094cbdbdcd16668d9347206d2deaa99f9a4541101e610f84a50a.json diff --git a/backend/.sqlx/query-8263fe28097e094cbdbdcd16668d9347206d2deaa99f9a4541101e610f84a50a.json b/backend/.sqlx/query-8263fe28097e094cbdbdcd16668d9347206d2deaa99f9a4541101e610f84a50a.json deleted file mode 100644 index aa744f6f08..0000000000 --- a/backend/.sqlx/query-8263fe28097e094cbdbdcd16668d9347206d2deaa99f9a4541101e610f84a50a.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "CREATE INDEX CONCURRENTLY queue_sort_2_v2 ON v2_job_queue (priority DESC NULLS LAST, scheduled_for) WHERE running = false", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "8263fe28097e094cbdbdcd16668d9347206d2deaa99f9a4541101e610f84a50a" -} diff --git a/backend/windmill-api/src/db.rs b/backend/windmill-api/src/db.rs index 106b29a810..a1e260b7d2 100644 --- a/backend/windmill-api/src/db.rs +++ b/backend/windmill-api/src/db.rs @@ -660,9 +660,9 @@ async fn fix_job_completed_index(db: &DB) -> Result<(), Error> { .execute(db) .await?; - sqlx::query!("CREATE INDEX CONCURRENTLY queue_sort_2_v2 ON v2_job_queue (tag, priority DESC NULLS LAST, scheduled_for) WHERE running = false") - .execute(db) - .await?; + // sqlx::query!("CREATE INDEX CONCURRENTLY queue_sort_2_v2 ON v2_job_queue (tag, priority DESC NULLS LAST, scheduled_for) WHERE running = false") + // .execute(db) + // .await?; sqlx::query!("DROP INDEX CONCURRENTLY IF EXISTS queue_sort") .execute(db) diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 6b0efa485d..c678af5465 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -1800,13 +1800,13 @@ async fn list_jobs( } sqlc.unwrap().limit(per_page).offset(offset).query()? }; - let mut tx = user_db.begin(&authed).await?; + let mut tx: Transaction<'_, Postgres> = user_db.begin(&authed).await?; #[cfg(feature = "prometheus")] let start = Instant::now(); #[cfg(feature = "prometheus")] - if _api_list_jobs_query_duration.is_some() { + if _api_list_jobs_query_duration.is_some() || true { tracing::info!("list_jobs query: {}", sql); } @@ -4695,6 +4695,7 @@ struct BatchInfo { flow_value: Option, path: Option, rawscript: Option, + tag: Option, } #[tracing::instrument(level = "trace", skip_all)] @@ -4863,6 +4864,8 @@ async fn add_batch_jobs( } else { format!("{}", language.as_str()) } + } else if let Some(tag) = batch_info.tag { + tag } else { format!("{}", language.as_str()) }; diff --git a/benchmarks/benchmark_oneoff.ts b/benchmarks/benchmark_oneoff.ts index 41321b3ec6..ccfbfc09d9 100644 --- a/benchmarks/benchmark_oneoff.ts +++ b/benchmarks/benchmark_oneoff.ts @@ -152,7 +152,6 @@ export async function main({ await createBenchScript(kind, workspace); } - pastJobs = await getCompletedJobsCount(); const jobsSent = jobs; console.log(`Bulk creating ${jobsSent} jobs`); @@ -208,11 +207,43 @@ export async function main({ throw new Error("Unknown script pattern " + kind); } - const response = await fetch( - config.server + + let testOtherTag = false; + const otherTagTodo = 500000; + if (testOtherTag) { + let parsed = JSON.parse(body); + parsed.tag = "test"; + let nbody = JSON.stringify(parsed); + let response2 = await fetch( + config.server + "/api/w/" + config.workspace_id + - `/jobs/add_batch_jobs/${jobsSent}`, + `/jobs/add_batch_jobs/${otherTagTodo}`, + { + method: "POST", + headers: { + ["Authorization"]: "Bearer " + config.token, + "Content-Type": "application/json", + }, + body: nbody, + } + ); + if (!response2.ok) { + throw new Error( + "Failed to create jobs: " + + response2.statusText + + " " + + (await response2.text()) + ); + } + } + + pastJobs = await getCompletedJobsCount(); + + const response = await fetch( + config.server + + "/api/w/" + + config.workspace_id + + `/jobs/add_batch_jobs/${jobsSent}`, { method: "POST", headers: { @@ -222,20 +253,24 @@ export async function main({ body, } ); + + + + + if (!response.ok) { throw new Error( "Failed to create jobs: " + - response.statusText + - " " + - (await response.text()) + response.statusText + + " " + + (await response.text()) ); } const uuids = await response.json(); const end_create = Date.now(); const create_duration = end_create - start_create; console.log( - `Jobs successfully added to the queue in ${ - create_duration / 1000 + `Jobs successfully added to the queue in ${create_duration / 1000 }s. Windmill will start pulling them\n` ); let start = Date.now(); @@ -249,7 +284,7 @@ export async function main({ const loopStart = Date.now(); if (!didStart) { const actual_queue = await getQueueCount(); - if (actual_queue < jobsSent) { + if (actual_queue < jobsSent + otherTagTodo) { start = Date.now(); didStart = true; } @@ -263,9 +298,9 @@ export async function main({ const instThr = lastElapsed > 0 ? ( - ((completedJobs - lastCompletedJobs) / (elapsed - lastElapsed)) * - 1000 - ).toFixed(2) + ((completedJobs - lastCompletedJobs) / (elapsed - lastElapsed)) * + 1000 + ).toFixed(2) : 0; lastElapsed = elapsed; @@ -275,8 +310,7 @@ export async function main({ enc( `elapsed: ${(elapsed / 1000).toFixed( 2 - )} | jobs executed: ${completedJobs}/${jobsSent} (thr: inst ${instThr} - avg ${avgThr}) | remaining: ${ - jobsSent - completedJobs + )} | jobs executed: ${completedJobs}/${jobsSent} (thr: inst ${instThr} - avg ${avgThr}) | remaining: ${jobsSent - completedJobs } \r` ) ); diff --git a/benchmarks/benchmark_suite.ts b/benchmarks/benchmark_suite.ts index e03148ce99..7828c62d9b 100644 --- a/benchmarks/benchmark_suite.ts +++ b/benchmarks/benchmark_suite.ts @@ -27,7 +27,7 @@ async function warmUp( token, workspace, kind: "noop", - jobs: 50000, + jobs: 100000, }); } From e0d7a54a2debfad9684d7581ef9c83075a188cc5 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 17 Feb 2025 19:26:34 +0100 Subject: [PATCH 07/21] nit on bench scripts --- benchmarks/benchmark_oneoff.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/benchmark_oneoff.ts b/benchmarks/benchmark_oneoff.ts index ccfbfc09d9..285d471df8 100644 --- a/benchmarks/benchmark_oneoff.ts +++ b/benchmarks/benchmark_oneoff.ts @@ -284,7 +284,7 @@ export async function main({ const loopStart = Date.now(); if (!didStart) { const actual_queue = await getQueueCount(); - if (actual_queue < jobsSent + otherTagTodo) { + if (actual_queue < jobsSent + (testOtherTag ? otherTagTodo : 0)) { start = Date.now(); didStart = true; } From 953082681e2c4fd71d5ac1acf372265ccc72297b Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 17 Feb 2025 21:26:46 +0100 Subject: [PATCH 08/21] fix: improve que job indices for faster performances --- ...f073a79e300f3dd48f14122d1782eee663cd.json} | 7 +++--- ...cb4bdd64075939a4aa2c117e18372511ea7e0.json | 12 ---------- ...8008a9479bf4b3d7231371ebf26382ecde365.json | 12 ++++++++++ ...05d394a7cbcf0038c72a78add5c7b02ef5927.json | 2 +- backend/windmill-api/src/jobs.rs | 7 +++++- backend/windmill-common/src/lib.rs | 6 ++--- backend/windmill-queue/src/jobs.rs | 3 +++ backend/windmill-worker/src/worker.rs | 2 +- benchmarks/benchmark_oneoff.ts | 22 ++++++++++--------- benchmarks/benchmark_suite.ts | 2 +- 10 files changed, 43 insertions(+), 32 deletions(-) rename backend/.sqlx/{query-19cc8499f682ec34d54bc4f694cb281a9bd7f5431c646c6268513751fff95395.json => query-0cb0e912bc942af2b1ef784455f3f073a79e300f3dd48f14122d1782eee663cd.json} (72%) delete mode 100644 backend/.sqlx/query-47455b0ebaf999ab58b2cba3d74cb4bdd64075939a4aa2c117e18372511ea7e0.json create mode 100644 backend/.sqlx/query-b9b38d63af3670d1f11d5cbb82a8008a9479bf4b3d7231371ebf26382ecde365.json diff --git a/backend/.sqlx/query-19cc8499f682ec34d54bc4f694cb281a9bd7f5431c646c6268513751fff95395.json b/backend/.sqlx/query-0cb0e912bc942af2b1ef784455f3f073a79e300f3dd48f14122d1782eee663cd.json similarity index 72% rename from backend/.sqlx/query-19cc8499f682ec34d54bc4f694cb281a9bd7f5431c646c6268513751fff95395.json rename to backend/.sqlx/query-0cb0e912bc942af2b1ef784455f3f073a79e300f3dd48f14122d1782eee663cd.json index 0c379a7bdb..0a9e91b206 100644 --- a/backend/.sqlx/query-19cc8499f682ec34d54bc4f694cb281a9bd7f5431c646c6268513751fff95395.json +++ b/backend/.sqlx/query-0cb0e912bc942af2b1ef784455f3f073a79e300f3dd48f14122d1782eee663cd.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT coalesce(COUNT(*) FILTER(WHERE suspend = 0 AND running = false), 0) as \"database_length!\", coalesce(COUNT(*) FILTER(WHERE suspend > 0), 0) as \"suspended!\" FROM v2_as_queue WHERE (workspace_id = $1 OR $2) AND scheduled_for <= now()", + "query": "SELECT coalesce(COUNT(*) FILTER(WHERE suspend = 0 AND running = false), 0) as \"database_length!\", coalesce(COUNT(*) FILTER(WHERE suspend > 0), 0) as \"suspended!\" FROM v2_as_queue WHERE (workspace_id = $1 OR $2) AND scheduled_for <= now() AND ($3::text[] IS NULL OR tag = ANY($3))", "describe": { "columns": [ { @@ -17,7 +17,8 @@ "parameters": { "Left": [ "Text", - "Bool" + "Bool", + "TextArray" ] }, "nullable": [ @@ -25,5 +26,5 @@ null ] }, - "hash": "19cc8499f682ec34d54bc4f694cb281a9bd7f5431c646c6268513751fff95395" + "hash": "0cb0e912bc942af2b1ef784455f3f073a79e300f3dd48f14122d1782eee663cd" } diff --git a/backend/.sqlx/query-47455b0ebaf999ab58b2cba3d74cb4bdd64075939a4aa2c117e18372511ea7e0.json b/backend/.sqlx/query-47455b0ebaf999ab58b2cba3d74cb4bdd64075939a4aa2c117e18372511ea7e0.json deleted file mode 100644 index 6ccaeece1c..0000000000 --- a/backend/.sqlx/query-47455b0ebaf999ab58b2cba3d74cb4bdd64075939a4aa2c117e18372511ea7e0.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "VACUUM (skip_locked) v2_job_queue, v2_job_runtime, v2_job_status", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "47455b0ebaf999ab58b2cba3d74cb4bdd64075939a4aa2c117e18372511ea7e0" -} diff --git a/backend/.sqlx/query-b9b38d63af3670d1f11d5cbb82a8008a9479bf4b3d7231371ebf26382ecde365.json b/backend/.sqlx/query-b9b38d63af3670d1f11d5cbb82a8008a9479bf4b3d7231371ebf26382ecde365.json new file mode 100644 index 0000000000..641c94c555 --- /dev/null +++ b/backend/.sqlx/query-b9b38d63af3670d1f11d5cbb82a8008a9479bf4b3d7231371ebf26382ecde365.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "VACUUM v2_job_queue, v2_job_runtime, v2_job_status", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "b9b38d63af3670d1f11d5cbb82a8008a9479bf4b3d7231371ebf26382ecde365" +} diff --git a/backend/.sqlx/query-ddf2eccb78a310ed00c7d8b9c3f05d394a7cbcf0038c72a78add5c7b02ef5927.json b/backend/.sqlx/query-ddf2eccb78a310ed00c7d8b9c3f05d394a7cbcf0038c72a78add5c7b02ef5927.json index c2dfed73a2..5bfff47576 100644 --- a/backend/.sqlx/query-ddf2eccb78a310ed00c7d8b9c3f05d394a7cbcf0038c72a78add5c7b02ef5927.json +++ b/backend/.sqlx/query-ddf2eccb78a310ed00c7d8b9c3f05d394a7cbcf0038c72a78add5c7b02ef5927.json @@ -15,7 +15,7 @@ ] }, "nullable": [ - null + true ] }, "hash": "ddf2eccb78a310ed00c7d8b9c3f05d394a7cbcf0038c72a78add5c7b02ef5927" diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index c678af5465..b5e1bc5a08 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -1647,6 +1647,7 @@ struct QueueStats { #[derive(Deserialize)] pub struct CountQueueJobsQuery { all_workspaces: Option, + tags: Option, } async fn count_queue_jobs( @@ -1654,12 +1655,16 @@ async fn count_queue_jobs( Path(w_id): Path, Query(cq): Query, ) -> error::JsonResult { + let tags = cq + .tags + .map(|t| t.split(',').map(|s| s.to_string()).collect::>()); Ok(Json( sqlx::query_as!( QueueStats, - "SELECT coalesce(COUNT(*) FILTER(WHERE suspend = 0 AND running = false), 0) as \"database_length!\", coalesce(COUNT(*) FILTER(WHERE suspend > 0), 0) as \"suspended!\" FROM v2_as_queue WHERE (workspace_id = $1 OR $2) AND scheduled_for <= now()", + "SELECT coalesce(COUNT(*) FILTER(WHERE suspend = 0 AND running = false), 0) as \"database_length!\", coalesce(COUNT(*) FILTER(WHERE suspend > 0), 0) as \"suspended!\" FROM v2_as_queue WHERE (workspace_id = $1 OR $2) AND scheduled_for <= now() AND ($3::text[] IS NULL OR tag = ANY($3))", w_id, w_id == "admins" && cq.all_workspaces.unwrap_or(false), + tags.as_ref().map(|v| v.as_slice()) ) .fetch_one(&db) .await?, diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index 396eafdf7f..c8996b3595 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -292,9 +292,9 @@ pub async fn connect( // .after_connect(move |conn, _| { // if worker_mode { // Box::pin(async move { - // sqlx::query("SET enable_seqscan = OFF;") - // .execute(conn) - // .await?; + // // sqlx::query("SET enable_seqscan = OFF;") + // // .execute(conn) + // // .await?; // Ok(()) // }) // } else { diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 2d2025da92..5d360d3acb 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -2108,12 +2108,15 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<'c>( for query in queries.iter() { // tracing::info!("Pulling job with query: {}", query); + // let instant = std::time::Instant::now(); let r = sqlx::query_as::<_, PulledJob>(query) .bind(worker_name) .fetch_optional(db) .await?; if let Some(pulled_job) = r { + // tracing::info!("pulled job: {:?}", instant.elapsed().as_micros()); + highest_priority_job = Some(pulled_job); break; } diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 5592e0d722..590b57227c 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -1277,7 +1277,7 @@ pub async fn run_worker( tokio::task::spawn( (async move { tracing::info!(worker = %worker_name, hostname = %hostname, "vacuuming queue"); - if let Err(e) = sqlx::query!("VACUUM (skip_locked) v2_job_queue, v2_job_runtime, v2_job_status") + if let Err(e) = sqlx::query!("VACUUM v2_job_queue, v2_job_runtime, v2_job_status") .execute(&db2) .await { diff --git a/benchmarks/benchmark_oneoff.ts b/benchmarks/benchmark_oneoff.ts index 285d471df8..62a3e3d5f7 100644 --- a/benchmarks/benchmark_oneoff.ts +++ b/benchmarks/benchmark_oneoff.ts @@ -37,6 +37,7 @@ async function verifyOutputs(uuids: string[], workspace: string) { console.log(`Incorrect results: ${incorrectResults}`); } +export const NON_TEST_TAGS = ["deno", "python", "go", "bash", "dedicated", "bun", "nativets", "flow"] export async function main({ host, email, @@ -96,11 +97,11 @@ export async function main({ windmill.setClient(final_token, host); const enc = (s: string) => new TextEncoder().encode(s); - async function getQueueCount() { + async function getQueueCount(tags?: string[]) { return ( await ( await fetch( - config.server + "/api/w/" + config.workspace_id + "/jobs/queue/count", + config.server + "/api/w/" + config.workspace_id + "/jobs/queue/count" + (tags && tags.length > 0 ? "?tags=" + tags.join(",") : ""), { headers: { ["Authorization"]: "Bearer " + config.token } } ) ).json() @@ -132,11 +133,11 @@ export async function main({ } let pastJobs = 0; - async function getCompletedJobsCount(): Promise { + async function getCompletedJobsCount(tags?: string[]): Promise { const completedJobs = ( await ( await fetch( - host + "/api/w/" + config.workspace_id + "/jobs/completed/count", + host + "/api/w/" + config.workspace_id + "/jobs/completed/count" + (tags && tags.length > 0 ? "?tags=" + tags.join(",") : ""), { headers: { ["Authorization"]: "Bearer " + config.token } } ) ).json() @@ -208,8 +209,9 @@ export async function main({ } let testOtherTag = false; - const otherTagTodo = 500000; if (testOtherTag) { + const otherTagTodo = 2000000; + let parsed = JSON.parse(body); parsed.tag = "test"; let nbody = JSON.stringify(parsed); @@ -237,7 +239,7 @@ export async function main({ } } - pastJobs = await getCompletedJobsCount(); + pastJobs = await getCompletedJobsCount(NON_TEST_TAGS); const response = await fetch( config.server + @@ -283,14 +285,14 @@ export async function main({ while (completedJobs < jobsSent) { const loopStart = Date.now(); if (!didStart) { - const actual_queue = await getQueueCount(); - if (actual_queue < jobsSent + (testOtherTag ? otherTagTodo : 0)) { + const actual_queue = await getQueueCount(NON_TEST_TAGS); + if (actual_queue < jobsSent) { start = Date.now(); didStart = true; } } else { const elapsed = start ? Date.now() - start : 0; - completedJobs = await getCompletedJobsCount(); + completedJobs = await getCompletedJobsCount(NON_TEST_TAGS); if (nStepsFlow > 0) { completedJobs = Math.floor(completedJobs / (nStepsFlow + 1)); } @@ -328,7 +330,7 @@ export async function main({ console.log(`avg. throughput (jobs/time): ${jobsSent / total_duration_sec}`); console.log("completed jobs", completedJobs); - console.log("queue length:", await getQueueCount()); + console.log("queue length:", await getQueueCount(NON_TEST_TAGS)); if ( !noVerify && diff --git a/benchmarks/benchmark_suite.ts b/benchmarks/benchmark_suite.ts index 7828c62d9b..e03148ce99 100644 --- a/benchmarks/benchmark_suite.ts +++ b/benchmarks/benchmark_suite.ts @@ -27,7 +27,7 @@ async function warmUp( token, workspace, kind: "noop", - jobs: 100000, + jobs: 50000, }); } From 0dd0a795a6b5d0d58be2d32b76778ad1e455748d Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 17 Feb 2025 21:54:00 +0100 Subject: [PATCH 09/21] nit benchmarks improvement --- benchmarks/benchmark_oneoff.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/benchmark_oneoff.ts b/benchmarks/benchmark_oneoff.ts index 62a3e3d5f7..2800eadebb 100644 --- a/benchmarks/benchmark_oneoff.ts +++ b/benchmarks/benchmark_oneoff.ts @@ -26,7 +26,7 @@ async function verifyOutputs(uuids: string[], workspace: string) { incorrectResults++; } if (job.result !== uuid) { - console.log(`Job ${uuid} did not output the correct value`); + console.log(`Job ${uuid} did not output the correct value: ${JSON.stringify(job.result)}`); incorrectResults++; } } catch (_) { From c6b2e6653a6d4a8994e078d3a82fad64639714fe Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 17 Feb 2025 22:20:33 +0100 Subject: [PATCH 10/21] output incorrect jobs in benchmarks --- benchmarks/benchmark_oneoff.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/benchmark_oneoff.ts b/benchmarks/benchmark_oneoff.ts index 2800eadebb..9ec48df3a4 100644 --- a/benchmarks/benchmark_oneoff.ts +++ b/benchmarks/benchmark_oneoff.ts @@ -26,7 +26,7 @@ async function verifyOutputs(uuids: string[], workspace: string) { incorrectResults++; } if (job.result !== uuid) { - console.log(`Job ${uuid} did not output the correct value: ${JSON.stringify(job.result)}`); + console.log(`Job ${uuid} did not output the correct value: ${JSON.stringify(job)}`); incorrectResults++; } } catch (_) { From 935b5b799636c0f02597315837268d4a76f6709a Mon Sep 17 00:00:00 2001 From: Alexander Petric Date: Mon, 17 Feb 2025 16:44:26 -0500 Subject: [PATCH 11/21] fix: improve teams settings in workspace settings (#5316) --- ...8ed593004c22bb5d11170b3196e290dd1d966.json | 26 ------------------- ...4caecda6335eda5b2e97e5a7370361653ff48.json | 26 +++++++++++++++++++ ...849e72bdc197c17c0fc51777c1dc9267e2daf.json | 12 +++++++++ ...05d394a7cbcf0038c72a78add5c7b02ef5927.json | 2 +- ...a6bdb16a1be66e993a5cfadf6de2e3c8a5021.json | 12 +++++++++ ...2bcae9640cee7b936820cb46c011222a77ff0.json | 14 ++++++++++ backend/ee-repo-ref.txt | 2 +- 7 files changed, 66 insertions(+), 28 deletions(-) delete mode 100644 backend/.sqlx/query-2fa27b0a71740e4e168879cf9a58ed593004c22bb5d11170b3196e290dd1d966.json create mode 100644 backend/.sqlx/query-50c17c7848760aaf0f869acfc444caecda6335eda5b2e97e5a7370361653ff48.json create mode 100644 backend/.sqlx/query-65c339164e7669360d231d70105849e72bdc197c17c0fc51777c1dc9267e2daf.json create mode 100644 backend/.sqlx/query-df3b60c1d0fb44c97bf2611a7cda6bdb16a1be66e993a5cfadf6de2e3c8a5021.json create mode 100644 backend/.sqlx/query-e565f3b2e51059f563d18a8a9442bcae9640cee7b936820cb46c011222a77ff0.json diff --git a/backend/.sqlx/query-2fa27b0a71740e4e168879cf9a58ed593004c22bb5d11170b3196e290dd1d966.json b/backend/.sqlx/query-2fa27b0a71740e4e168879cf9a58ed593004c22bb5d11170b3196e290dd1d966.json deleted file mode 100644 index 704778d04a..0000000000 --- a/backend/.sqlx/query-2fa27b0a71740e4e168879cf9a58ed593004c22bb5d11170b3196e290dd1d966.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n WITH assigned_teams AS (\n SELECT teams_team_id\n FROM workspace_settings\n ),\n all_teams AS (\n SELECT jsonb_array_elements(value::jsonb) AS team\n FROM global_settings\n WHERE name = 'teams'\n )\n SELECT team->>'team_name' AS team_name, team->>'team_internal_id' AS team_id\n FROM all_teams\n WHERE NOT EXISTS (\n SELECT 1\n FROM assigned_teams\n WHERE assigned_teams.teams_team_id = team->>'team_internal_id'\n )\n AND team->>'team_name' IS NOT NULL\n AND team->>'team_id' IS NOT NULL\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "team_name", - "type_info": "Text" - }, - { - "ordinal": 1, - "name": "team_id", - "type_info": "Text" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - null, - null - ] - }, - "hash": "2fa27b0a71740e4e168879cf9a58ed593004c22bb5d11170b3196e290dd1d966" -} diff --git a/backend/.sqlx/query-50c17c7848760aaf0f869acfc444caecda6335eda5b2e97e5a7370361653ff48.json b/backend/.sqlx/query-50c17c7848760aaf0f869acfc444caecda6335eda5b2e97e5a7370361653ff48.json new file mode 100644 index 0000000000..f3dc153254 --- /dev/null +++ b/backend/.sqlx/query-50c17c7848760aaf0f869acfc444caecda6335eda5b2e97e5a7370361653ff48.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH assigned_teams AS (\n SELECT teams_team_id\n FROM workspace_settings\n ),\n all_teams AS (\n SELECT jsonb_array_elements(CASE\n WHEN jsonb_typeof(value::jsonb) = 'array' THEN value::jsonb\n ELSE '[]'::jsonb\n END) AS team\n FROM global_settings\n WHERE name = 'teams'\n )\n SELECT team->>'team_name' AS team_name, team->>'team_internal_id' AS team_id\n FROM all_teams\n WHERE NOT EXISTS (\n SELECT 1\n FROM assigned_teams\n WHERE assigned_teams.teams_team_id = team->>'team_internal_id'\n )\n AND team->>'team_name' IS NOT NULL\n AND team->>'team_id' IS NOT NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "team_name", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "team_id", + "type_info": "Text" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + null + ] + }, + "hash": "50c17c7848760aaf0f869acfc444caecda6335eda5b2e97e5a7370361653ff48" +} diff --git a/backend/.sqlx/query-65c339164e7669360d231d70105849e72bdc197c17c0fc51777c1dc9267e2daf.json b/backend/.sqlx/query-65c339164e7669360d231d70105849e72bdc197c17c0fc51777c1dc9267e2daf.json new file mode 100644 index 0000000000..d52d46f1dd --- /dev/null +++ b/backend/.sqlx/query-65c339164e7669360d231d70105849e72bdc197c17c0fc51777c1dc9267e2daf.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE workspace_settings\n SET teams_command_script = NULL,\n teams_team_id = NULL,\n teams_team_name = NULL\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "65c339164e7669360d231d70105849e72bdc197c17c0fc51777c1dc9267e2daf" +} diff --git a/backend/.sqlx/query-ddf2eccb78a310ed00c7d8b9c3f05d394a7cbcf0038c72a78add5c7b02ef5927.json b/backend/.sqlx/query-ddf2eccb78a310ed00c7d8b9c3f05d394a7cbcf0038c72a78add5c7b02ef5927.json index 5bfff47576..c2dfed73a2 100644 --- a/backend/.sqlx/query-ddf2eccb78a310ed00c7d8b9c3f05d394a7cbcf0038c72a78add5c7b02ef5927.json +++ b/backend/.sqlx/query-ddf2eccb78a310ed00c7d8b9c3f05d394a7cbcf0038c72a78add5c7b02ef5927.json @@ -15,7 +15,7 @@ ] }, "nullable": [ - true + null ] }, "hash": "ddf2eccb78a310ed00c7d8b9c3f05d394a7cbcf0038c72a78add5c7b02ef5927" diff --git a/backend/.sqlx/query-df3b60c1d0fb44c97bf2611a7cda6bdb16a1be66e993a5cfadf6de2e3c8a5021.json b/backend/.sqlx/query-df3b60c1d0fb44c97bf2611a7cda6bdb16a1be66e993a5cfadf6de2e3c8a5021.json new file mode 100644 index 0000000000..18f29062a6 --- /dev/null +++ b/backend/.sqlx/query-df3b60c1d0fb44c97bf2611a7cda6bdb16a1be66e993a5cfadf6de2e3c8a5021.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE global_settings\n SET value = (\n SELECT jsonb_agg(elem)\n FROM jsonb_array_elements(value) AS elem\n WHERE NOT (elem ? 'teams_channel')\n )\n WHERE name = 'critical_error_channels'\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "df3b60c1d0fb44c97bf2611a7cda6bdb16a1be66e993a5cfadf6de2e3c8a5021" +} diff --git a/backend/.sqlx/query-e565f3b2e51059f563d18a8a9442bcae9640cee7b936820cb46c011222a77ff0.json b/backend/.sqlx/query-e565f3b2e51059f563d18a8a9442bcae9640cee7b936820cb46c011222a77ff0.json new file mode 100644 index 0000000000..4c2cd96ca9 --- /dev/null +++ b/backend/.sqlx/query-e565f3b2e51059f563d18a8a9442bcae9640cee7b936820cb46c011222a77ff0.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE global_settings SET value = $1 WHERE name = 'teams'", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "e565f3b2e51059f563d18a8a9442bcae9640cee7b936820cb46c011222a77ff0" +} diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index c94a57b20f..83ad30b351 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -703a03ac430b6f603a5a189e5b4ff9a42bb2bd7f \ No newline at end of file +e507af5589efb1a8a72ae4e231130f749bde738d \ No newline at end of file From 9234701f05316b5d14a4964df1b94e212837fb4a Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 17 Feb 2025 23:43:16 +0100 Subject: [PATCH 12/21] ensure index creation of root_job_by_path --- backend/windmill-api/src/db.rs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/backend/windmill-api/src/db.rs b/backend/windmill-api/src/db.rs index a1e260b7d2..b8d15160c7 100644 --- a/backend/windmill-api/src/db.rs +++ b/backend/windmill-api/src/db.rs @@ -545,8 +545,8 @@ async fn fix_job_completed_index(db: &DB) -> Result<(), Error> { .await?; }); - run_windmill_migration!("fix_job_index_1", &db, |tx| { - let migration_job_name = "fix_job_completed_index_4"; + run_windmill_migration!("fix_job_index_1_II", &db, |tx| { + let migration_job_name = "fix_job_index_1_II"; let mut i = 1; tracing::info!("step {i} of {migration_job_name} migration"); sqlx::query!("create index concurrently if not exists ix_job_workspace_id_created_at_new_3 ON v2_job (workspace_id, created_at DESC)") @@ -579,13 +579,20 @@ async fn fix_job_completed_index(db: &DB) -> Result<(), Error> { i += 1; tracing::info!("step {i} of {migration_job_name} migration"); - sqlx::query!("create index concurrently if not exists root_job_index_by_path_2 ON v2_job (workspace_id, runnable_path, created_at desc) WHERE parent_job IS NULL") + sqlx::query!("create index concurrently if not exists ix_job_root_job_index_by_path_2 ON v2_job (workspace_id, runnable_path, created_at desc) WHERE parent_job IS NULL") .execute(db) .await?; i += 1; tracing::info!("step {i} of {migration_job_name} migration"); + sqlx::query!("DROP INDEX CONCURRENTLY IF EXISTS root_job_index_by_path_2") + .execute(db) + .await?; + + i += 1; + tracing::info!("step {i} of {migration_job_name} migration"); + sqlx::query!("create index concurrently if not exists ix_job_created_at ON v2_job (created_at DESC)") .execute(db) .await?; From 3d7882577fc01ef3cce008848e3de211e9416b41 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 18 Feb 2025 00:00:51 +0100 Subject: [PATCH 13/21] chore(main): release 1.463.4 (#5309) * chore(main): release 1.463.4 * Update CHANGELOG.md * Apply automatic changes --------- Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com> --- CHANGELOG.md | 8 +++ ...8ed593004c22bb5d11170b3196e290dd1d966.json | 26 ++++++++ ...8c6395e2440ea27553b7ccb18d7149b106728.json | 12 ++++ ...f9a71af5996bc76f328b3ba1cf68a71880462.json | 12 ++++ backend/Cargo.lock | 62 +++++++++---------- backend/Cargo.toml | 4 +- backend/windmill-api/openapi.yaml | 2 +- benchmarks/lib.ts | 2 +- cli/main.ts | 2 +- frontend/package-lock.json | 4 +- frontend/package.json | 2 +- lsp/Pipfile | 4 +- openflow.openapi.yaml | 2 +- .../WindmillClient/WindmillClient.psd1 | 2 +- python-client/wmill/pyproject.toml | 2 +- python-client/wmill_pg/pyproject.toml | 2 +- typescript-client/jsr.json | 2 +- typescript-client/package.json | 2 +- version.txt | 2 +- 19 files changed, 106 insertions(+), 48 deletions(-) create mode 100644 backend/.sqlx/query-2fa27b0a71740e4e168879cf9a58ed593004c22bb5d11170b3196e290dd1d966.json create mode 100644 backend/.sqlx/query-6536214f31e9d600e868b01385d8c6395e2440ea27553b7ccb18d7149b106728.json create mode 100644 backend/.sqlx/query-c481e5d63ebf1aa537cc4ce4e84f9a71af5996bc76f328b3ba1cf68a71880462.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c14a34e5b..8d8093eecf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [1.463.4](https://github.com/windmill-labs/windmill/compare/v1.463.3...v1.463.4) (2025-02-17) + + +### Bug Fixes + +* improve queue job indices for faster performances ([9530826](https://github.com/windmill-labs/windmill/commit/953082681e2c4fd71d5ac1acf372265ccc72297b)) +* improve teams settings in workspace settings ([#5316](https://github.com/windmill-labs/windmill/issues/5316)) ([935b5b7](https://github.com/windmill-labs/windmill/commit/935b5b799636c0f02597315837268d4a76f6709a)) + ## [1.463.3](https://github.com/windmill-labs/windmill/compare/v1.463.2...v1.463.3) (2025-02-17) diff --git a/backend/.sqlx/query-2fa27b0a71740e4e168879cf9a58ed593004c22bb5d11170b3196e290dd1d966.json b/backend/.sqlx/query-2fa27b0a71740e4e168879cf9a58ed593004c22bb5d11170b3196e290dd1d966.json new file mode 100644 index 0000000000..704778d04a --- /dev/null +++ b/backend/.sqlx/query-2fa27b0a71740e4e168879cf9a58ed593004c22bb5d11170b3196e290dd1d966.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH assigned_teams AS (\n SELECT teams_team_id\n FROM workspace_settings\n ),\n all_teams AS (\n SELECT jsonb_array_elements(value::jsonb) AS team\n FROM global_settings\n WHERE name = 'teams'\n )\n SELECT team->>'team_name' AS team_name, team->>'team_internal_id' AS team_id\n FROM all_teams\n WHERE NOT EXISTS (\n SELECT 1\n FROM assigned_teams\n WHERE assigned_teams.teams_team_id = team->>'team_internal_id'\n )\n AND team->>'team_name' IS NOT NULL\n AND team->>'team_id' IS NOT NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "team_name", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "team_id", + "type_info": "Text" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + null + ] + }, + "hash": "2fa27b0a71740e4e168879cf9a58ed593004c22bb5d11170b3196e290dd1d966" +} diff --git a/backend/.sqlx/query-6536214f31e9d600e868b01385d8c6395e2440ea27553b7ccb18d7149b106728.json b/backend/.sqlx/query-6536214f31e9d600e868b01385d8c6395e2440ea27553b7ccb18d7149b106728.json new file mode 100644 index 0000000000..fce1c4942a --- /dev/null +++ b/backend/.sqlx/query-6536214f31e9d600e868b01385d8c6395e2440ea27553b7ccb18d7149b106728.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DROP INDEX CONCURRENTLY IF EXISTS root_job_index_by_path_2", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "6536214f31e9d600e868b01385d8c6395e2440ea27553b7ccb18d7149b106728" +} diff --git a/backend/.sqlx/query-c481e5d63ebf1aa537cc4ce4e84f9a71af5996bc76f328b3ba1cf68a71880462.json b/backend/.sqlx/query-c481e5d63ebf1aa537cc4ce4e84f9a71af5996bc76f328b3ba1cf68a71880462.json new file mode 100644 index 0000000000..b3d20cccea --- /dev/null +++ b/backend/.sqlx/query-c481e5d63ebf1aa537cc4ce4e84f9a71af5996bc76f328b3ba1cf68a71880462.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "create index concurrently if not exists ix_job_root_job_index_by_path_2 ON v2_job (workspace_id, runnable_path, created_at desc) WHERE parent_job IS NULL", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "c481e5d63ebf1aa537cc4ce4e84f9a71af5996bc76f328b3ba1cf68a71880462" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 7d86e09e30..f82041c837 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -1668,9 +1668,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.29" +version = "4.5.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8acebd8ad879283633b343856142139f2da2317c96b05b4dd6181c61e2480184" +checksum = "92b7b18d71fad5313a1e320fa9897994228ce274b60faa4d694fe0ea89cd9e6d" dependencies = [ "clap_builder", "clap_derive", @@ -1678,9 +1678,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.29" +version = "4.5.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6ba32cbda51c7e1dfd49acc1457ba1a7dec5b64fe360e828acb13ca8dc9c2f9" +checksum = "a35db2071778a7344791a4fb4f95308b5673d219dee3ae348b86642574ecc90c" dependencies = [ "anstream", "anstyle", @@ -9297,9 +9297,9 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.17.0" +version = "3.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a40f762a77d2afa88c2d919489e390a12bdd261ed568e60cfa7e48d4e20f0d33" +checksum = "22e5a0acb1f3f55f65cc4a866c361b2fb2a0ff6366785ae6fbb5f85df07ba230" dependencies = [ "cfg-if", "fastrand 2.3.0", @@ -10858,7 +10858,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill" -version = "1.463.3" +version = "1.463.4" dependencies = [ "anyhow", "axum", @@ -10901,7 +10901,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.463.3" +version = "1.463.4" dependencies = [ "anyhow", "argon2", @@ -10995,7 +10995,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.463.3" +version = "1.463.4" dependencies = [ "base64 0.22.1", "chrono", @@ -11013,7 +11013,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.463.3" +version = "1.463.4" dependencies = [ "chrono", "serde", @@ -11026,7 +11026,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.463.3" +version = "1.463.4" dependencies = [ "anyhow", "serde", @@ -11040,7 +11040,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.463.3" +version = "1.463.4" dependencies = [ "anyhow", "async-stream", @@ -11099,7 +11099,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.463.3" +version = "1.463.4" dependencies = [ "regex", "serde", @@ -11113,7 +11113,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.463.3" +version = "1.463.4" dependencies = [ "anyhow", "bytes", @@ -11136,7 +11136,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.463.3" +version = "1.463.4" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -11148,7 +11148,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.463.3" +version = "1.463.4" dependencies = [ "convert_case 0.6.0", "serde", @@ -11157,7 +11157,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.463.3" +version = "1.463.4" dependencies = [ "anyhow", "lazy_static", @@ -11169,7 +11169,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.463.3" +version = "1.463.4" dependencies = [ "anyhow", "serde_json", @@ -11181,7 +11181,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.463.3" +version = "1.463.4" dependencies = [ "anyhow", "gosyn", @@ -11193,7 +11193,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.463.3" +version = "1.463.4" dependencies = [ "anyhow", "lazy_static", @@ -11205,7 +11205,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.463.3" +version = "1.463.4" dependencies = [ "anyhow", "itertools 0.14.0", @@ -11216,7 +11216,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.463.3" +version = "1.463.4" dependencies = [ "anyhow", "itertools 0.14.0", @@ -11227,7 +11227,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.463.3" +version = "1.463.4" dependencies = [ "anyhow", "async-recursion", @@ -11247,7 +11247,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.463.3" +version = "1.463.4" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -11264,7 +11264,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.463.3" +version = "1.463.4" dependencies = [ "anyhow", "lazy_static", @@ -11276,7 +11276,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.463.3" +version = "1.463.4" dependencies = [ "anyhow", "lazy_static", @@ -11294,7 +11294,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.463.3" +version = "1.463.4" dependencies = [ "anyhow", "getrandom 0.2.15", @@ -11316,7 +11316,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.463.3" +version = "1.463.4" dependencies = [ "anyhow", "serde_json", @@ -11326,7 +11326,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.463.3" +version = "1.463.4" dependencies = [ "anyhow", "async-recursion", @@ -11359,7 +11359,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.463.3" +version = "1.463.4" dependencies = [ "wasm-bindgen", "wasm-bindgen-test", @@ -11369,7 +11369,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.463.3" +version = "1.463.4" dependencies = [ "anyhow", "async-recursion", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 72c133e293..0b8c02d1fa 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.463.3" +version = "1.463.4" authors.workspace = true edition.workspace = true @@ -30,7 +30,7 @@ members = [ ] [workspace.package] -version = "1.463.3" +version = "1.463.4" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index d22a3886d3..0e9b0328d4 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.463.3 + version: 1.463.4 title: Windmill API contact: diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 32b434eb2f..fc4f611794 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.463.3"; +export const VERSION = "v1.463.4"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/main.ts b/cli/main.ts index 165ce2e8c8..51c8bd0809 100644 --- a/cli/main.ts +++ b/cli/main.ts @@ -62,7 +62,7 @@ export { // } // }); -export const VERSION = "1.463.3"; +export const VERSION = "1.463.4"; const command = new Command() .name("wmill") diff --git a/frontend/package-lock.json b/frontend/package-lock.json index b4fa6c0b8a..9a8bce2889 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.463.3", + "version": "1.463.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.463.3", + "version": "1.463.4", "license": "AGPL-3.0", "dependencies": { "@anthropic-ai/sdk": "^0.32.1", diff --git a/frontend/package.json b/frontend/package.json index a22bfb038b..1ece7bb764 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.463.3", + "version": "1.463.4", "scripts": { "dev": "vite dev", "build": "vite build", diff --git a/lsp/Pipfile b/lsp/Pipfile index 5e8f57d701..18b12dfc57 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,8 +4,8 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.463.3" -wmill_pg = ">=1.463.3" +wmill = ">=1.463.4" +wmill_pg = ">=1.463.4" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index bc0eed2d61..043c7c77fb 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.463.3 + version: 1.463.4 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index b1e9637ce1..7ab35ebe95 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. - ModuleVersion = '1.463.3' + ModuleVersion = '1.463.4' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index 1942a92887..29a80185e2 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.463.3" +version = "1.463.4" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/python-client/wmill_pg/pyproject.toml b/python-client/wmill_pg/pyproject.toml index 1cf8de80e6..ee2dac54aa 100644 --- a/python-client/wmill_pg/pyproject.toml +++ b/python-client/wmill_pg/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill-pg" -version = "1.463.3" +version = "1.463.4" description = "An extension client for the wmill client library focused on pg" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index d513475883..c4e0f34c3d 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.463.3", + "version": "1.463.4", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index 3c2ff4eee0..ba67288973 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.463.3", + "version": "1.463.4", "author": "Ruben Fiszel", "license": "Apache 2.0", "devDependencies": { diff --git a/version.txt b/version.txt index a970451822..c6fd9a4d69 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.463.3 +1.463.4 From 3b6585afdf004929be135b7c3f7cf1aa678f4d0c Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 18 Feb 2025 01:35:49 +0100 Subject: [PATCH 14/21] nit benchmarks --- ...8ed593004c22bb5d11170b3196e290dd1d966.json | 26 ------------------- ...c3030216b5a4ed669f77962509d1c2c6cb780.json | 12 --------- benchmarks/benchmark_oneoff.ts | 2 +- benchmarks/lib.ts | 4 +-- 4 files changed, 3 insertions(+), 41 deletions(-) delete mode 100644 backend/.sqlx/query-2fa27b0a71740e4e168879cf9a58ed593004c22bb5d11170b3196e290dd1d966.json delete mode 100644 backend/.sqlx/query-3bacf9cd9aa63f4bec5f983f4a0c3030216b5a4ed669f77962509d1c2c6cb780.json diff --git a/backend/.sqlx/query-2fa27b0a71740e4e168879cf9a58ed593004c22bb5d11170b3196e290dd1d966.json b/backend/.sqlx/query-2fa27b0a71740e4e168879cf9a58ed593004c22bb5d11170b3196e290dd1d966.json deleted file mode 100644 index 704778d04a..0000000000 --- a/backend/.sqlx/query-2fa27b0a71740e4e168879cf9a58ed593004c22bb5d11170b3196e290dd1d966.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n WITH assigned_teams AS (\n SELECT teams_team_id\n FROM workspace_settings\n ),\n all_teams AS (\n SELECT jsonb_array_elements(value::jsonb) AS team\n FROM global_settings\n WHERE name = 'teams'\n )\n SELECT team->>'team_name' AS team_name, team->>'team_internal_id' AS team_id\n FROM all_teams\n WHERE NOT EXISTS (\n SELECT 1\n FROM assigned_teams\n WHERE assigned_teams.teams_team_id = team->>'team_internal_id'\n )\n AND team->>'team_name' IS NOT NULL\n AND team->>'team_id' IS NOT NULL\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "team_name", - "type_info": "Text" - }, - { - "ordinal": 1, - "name": "team_id", - "type_info": "Text" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - null, - null - ] - }, - "hash": "2fa27b0a71740e4e168879cf9a58ed593004c22bb5d11170b3196e290dd1d966" -} diff --git a/backend/.sqlx/query-3bacf9cd9aa63f4bec5f983f4a0c3030216b5a4ed669f77962509d1c2c6cb780.json b/backend/.sqlx/query-3bacf9cd9aa63f4bec5f983f4a0c3030216b5a4ed669f77962509d1c2c6cb780.json deleted file mode 100644 index 200a5bf47f..0000000000 --- a/backend/.sqlx/query-3bacf9cd9aa63f4bec5f983f4a0c3030216b5a4ed669f77962509d1c2c6cb780.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "create index concurrently if not exists root_job_index_by_path_2 ON v2_job (workspace_id, runnable_path, created_at desc) WHERE parent_job IS NULL", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "3bacf9cd9aa63f4bec5f983f4a0c3030216b5a4ed669f77962509d1c2c6cb780" -} diff --git a/benchmarks/benchmark_oneoff.ts b/benchmarks/benchmark_oneoff.ts index 9ec48df3a4..6685eae1c5 100644 --- a/benchmarks/benchmark_oneoff.ts +++ b/benchmarks/benchmark_oneoff.ts @@ -201,7 +201,7 @@ export async function main({ kind: "rawscript", rawscript: { language: api.RawScript.language.BASH, - content: "# let's bloat that bash script, 3.. 2.. 1.. BOOM\n".repeat(25000) + "echo \"$WM_FLOW_JOB_ID\"\n", + content: "# let's bloat that bash script, 3.. 2.. 1.. BOOM\n".repeat(100) + "echo \"$WM_FLOW_JOB_ID\"\n", }, }); } else { diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index fc4f611794..d7543a3d55 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -25,7 +25,7 @@ async function waitForDeployment(workspace: string, hash: string) { if (resp.lock !== null) { return; } - } catch (err) {} + } catch (err) { } await sleep(0.5); } throw new Error("Script did not deploy in time"); @@ -246,7 +246,7 @@ export const getFlowPayload = (flowPattern: string): api.FlowPreview => { input_transforms: {}, language: api.RawScript.language.BASH, type: "rawscript", - content: "# let's bloat that bash script, 3.. 2.. 1.. BOOM\n".repeat(25000) + "echo \"$WM_FLOW_JOB_ID\"\n", + content: "# let's bloat that bash script, 3.. 2.. 1.. BOOM\n".repeat(100) + `if [[ -z $\{WM_FLOW_JOB_ID+x\} ]]; then\necho "not set"\nelif [[ -z "$WM_FLOW_JOB_ID" ]]; then\necho "empty"\nelse\necho "$WM_FLOW_JOB_ID"\nfi`, }, } ], From 588ff79364c632563f9977000cff8682bf94b4f3 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 18 Feb 2025 08:43:03 +0100 Subject: [PATCH 15/21] make benchmarks even more stable --- ...9c2e1e9ca0ed33077ade8d7560e7cc21fa06.json} | 5 ++-- ...05d394a7cbcf0038c72a78add5c7b02ef5927.json | 2 +- backend/pg_log_tail | 0 backend/windmill-common/src/lib.rs | 26 +++++++++---------- backend/windmill-queue/src/jobs.rs | 20 +++++++------- 5 files changed, 26 insertions(+), 27 deletions(-) rename backend/.sqlx/{query-d25c58d2722ad3dcd91101ce6f66e1d802dd5d82e1cd5f5ed3a15cbc75eb6745.json => query-c92cc71e6d10c41368f7aa75b0799c2e1e9ca0ed33077ade8d7560e7cc21fa06.json} (60%) create mode 100644 backend/pg_log_tail diff --git a/backend/.sqlx/query-d25c58d2722ad3dcd91101ce6f66e1d802dd5d82e1cd5f5ed3a15cbc75eb6745.json b/backend/.sqlx/query-c92cc71e6d10c41368f7aa75b0799c2e1e9ca0ed33077ade8d7560e7cc21fa06.json similarity index 60% rename from backend/.sqlx/query-d25c58d2722ad3dcd91101ce6f66e1d802dd5d82e1cd5f5ed3a15cbc75eb6745.json rename to backend/.sqlx/query-c92cc71e6d10c41368f7aa75b0799c2e1e9ca0ed33077ade8d7560e7cc21fa06.json index 2872c1655b..8a4b957c2a 100644 --- a/backend/.sqlx/query-d25c58d2722ad3dcd91101ce6f66e1d802dd5d82e1cd5f5ed3a15cbc75eb6745.json +++ b/backend/.sqlx/query-c92cc71e6d10c41368f7aa75b0799c2e1e9ca0ed33077ade8d7560e7cc21fa06.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "DELETE FROM v2_job_queue WHERE workspace_id = $1 AND id = $2 RETURNING 1", + "query": "DELETE FROM v2_job_queue WHERE id = $1 RETURNING 1", "describe": { "columns": [ { @@ -11,7 +11,6 @@ ], "parameters": { "Left": [ - "Text", "Uuid" ] }, @@ -19,5 +18,5 @@ null ] }, - "hash": "d25c58d2722ad3dcd91101ce6f66e1d802dd5d82e1cd5f5ed3a15cbc75eb6745" + "hash": "c92cc71e6d10c41368f7aa75b0799c2e1e9ca0ed33077ade8d7560e7cc21fa06" } diff --git a/backend/.sqlx/query-ddf2eccb78a310ed00c7d8b9c3f05d394a7cbcf0038c72a78add5c7b02ef5927.json b/backend/.sqlx/query-ddf2eccb78a310ed00c7d8b9c3f05d394a7cbcf0038c72a78add5c7b02ef5927.json index c2dfed73a2..5bfff47576 100644 --- a/backend/.sqlx/query-ddf2eccb78a310ed00c7d8b9c3f05d394a7cbcf0038c72a78add5c7b02ef5927.json +++ b/backend/.sqlx/query-ddf2eccb78a310ed00c7d8b9c3f05d394a7cbcf0038c72a78add5c7b02ef5927.json @@ -15,7 +15,7 @@ ] }, "nullable": [ - null + true ] }, "hash": "ddf2eccb78a310ed00c7d8b9c3f05d394a7cbcf0038c72a78add5c7b02ef5927" diff --git a/backend/pg_log_tail b/backend/pg_log_tail new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index c8996b3595..802af6196b 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -281,7 +281,7 @@ pub async fn connect_db( pub async fn connect( database_url: &str, max_connections: u32, - _worker_mode: bool, + worker_mode: bool, ) -> Result, error::Error> { use std::time::Duration; @@ -289,18 +289,18 @@ pub async fn connect( .min_connections((max_connections / 5).clamp(3, max_connections)) .max_connections(max_connections) .max_lifetime(Duration::from_secs(30 * 60)) // 30 mins - // .after_connect(move |conn, _| { - // if worker_mode { - // Box::pin(async move { - // // sqlx::query("SET enable_seqscan = OFF;") - // // .execute(conn) - // // .await?; - // Ok(()) - // }) - // } else { - // Box::pin(async move { Ok(()) }) - // } - // }) + .after_connect(move |conn, _| { + if worker_mode { + Box::pin(async move { + sqlx::query("SET enable_seqscan = OFF;") + .execute(conn) + .await?; + Ok(()) + }) + } else { + Box::pin(async move { Ok(()) }) + } + }) .connect_with( sqlx::postgres::PgConnectOptions::from_str(database_url)?.statement_cache_capacity(400), ) diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 5d360d3acb..e582fa39dc 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -566,6 +566,9 @@ pub async fn add_completed_job( let result_columns = result_columns.as_ref(); let _job_id = queued_job.id; let (opt_uuid, _duration, _skip_downstream_error_handlers) = (|| async { + + // let start = std::time::Instant::now(); + let mut tx = db.begin().await?; let job_id = queued_job.id; @@ -663,7 +666,7 @@ pub async fn add_completed_job( // tracing::error!("Added completed job {:#?}", queued_job); let mut _skip_downstream_error_handlers = false; - tx = delete_job(tx, &queued_job.workspace_id, job_id).await?; + tx = delete_job(tx, &job_id).await?; // tracing::error!("3 {:?}", start.elapsed()); if queued_job.is_flow_step { @@ -858,6 +861,7 @@ pub async fn add_completed_job( "inserted completed job: {} (success: {success})", queued_job.id ); + // tracing::info!("completed job: {:?}", start.elapsed().as_micros()); Ok((None, _duration, _skip_downstream_error_handlers)) as windmill_common::error::Result<(Option, i64, bool)> }) .retry( @@ -2597,21 +2601,17 @@ async fn extract_result_from_job_result( pub async fn delete_job<'c>( mut tx: Transaction<'c, Postgres>, - w_id: &str, - job_id: Uuid, + job_id: &Uuid, ) -> windmill_common::error::Result> { #[cfg(feature = "prometheus")] if METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed) { QUEUE_DELETE_COUNT.inc(); } - let job_removed = sqlx::query_scalar!( - "DELETE FROM v2_job_queue WHERE workspace_id = $1 AND id = $2 RETURNING 1", - w_id, - job_id - ) - .fetch_optional(&mut *tx) - .await; + let job_removed = + sqlx::query_scalar!("DELETE FROM v2_job_queue WHERE id = $1 RETURNING 1", job_id,) + .fetch_optional(&mut *tx) + .await; if let Err(job_removed) = job_removed { tracing::error!( From 0c391e92a22b89221f7754a1ed360c6ee6e80965 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 18 Feb 2025 09:51:21 +0100 Subject: [PATCH 16/21] improve benchmarks --- .github/workflows/benchmark.yml | 2 ++ benchmarks/benchmark_suite.ts | 7 ++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index f95dee00da..b48ad1c990 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -162,6 +162,7 @@ jobs: -c https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/suite_config.json --workers 4 + --factor 3 - name: Save benchmark results uses: actions/upload-artifact@v4 with: @@ -281,6 +282,7 @@ jobs: -c https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/suite_config.json --workers 8 + --factor 3 - name: Save benchmark results uses: actions/upload-artifact@v4 with: diff --git a/benchmarks/benchmark_suite.ts b/benchmarks/benchmark_suite.ts index e03148ce99..f4840dda05 100644 --- a/benchmarks/benchmark_suite.ts +++ b/benchmarks/benchmark_suite.ts @@ -39,6 +39,7 @@ async function main({ workspace, configPath, workers, + factor }: { host: string; email?: string; @@ -47,6 +48,7 @@ async function main({ workspace: string; configPath: string; workers: number; + factor?: number; }) { async function getConfig(configPath: string): Promise { if (configPath.startsWith("http")) { @@ -77,7 +79,7 @@ async function main({ token, workspace, kind: benchmark.kind, - jobs: benchmark.jobs, + jobs: benchmark.jobs * (factor ?? 1), }); if (benchmark.noSave) { @@ -153,6 +155,9 @@ await new Command() "Number of workers that are used to run the benchmarks (only affect graph title)", { default: 1 } ) + .option("--factor ", "Factor to multiply the number of jobs by.", { + default: 1, + }) .action(main) .command( "upgrade", From 1b46e0f08426497d549cf5007c93981df9ab41e5 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 18 Feb 2025 10:25:42 +0100 Subject: [PATCH 17/21] fix: fix teams cleanup preventing start --- ...05ba5c7c14aa7614f2610e79a8c7302eaa63fb74984d.json | 12 ++++++++++++ ...d8b9c3f05d394a7cbcf0038c72a78add5c7b02ef5927.json | 2 +- ...611a7cda6bdb16a1be66e993a5cfadf6de2e3c8a5021.json | 12 ------------ backend/ee-repo-ref.txt | 2 +- 4 files changed, 14 insertions(+), 14 deletions(-) create mode 100644 backend/.sqlx/query-81b06122c7a12a314d8905ba5c7c14aa7614f2610e79a8c7302eaa63fb74984d.json delete mode 100644 backend/.sqlx/query-df3b60c1d0fb44c97bf2611a7cda6bdb16a1be66e993a5cfadf6de2e3c8a5021.json diff --git a/backend/.sqlx/query-81b06122c7a12a314d8905ba5c7c14aa7614f2610e79a8c7302eaa63fb74984d.json b/backend/.sqlx/query-81b06122c7a12a314d8905ba5c7c14aa7614f2610e79a8c7302eaa63fb74984d.json new file mode 100644 index 0000000000..91718513bb --- /dev/null +++ b/backend/.sqlx/query-81b06122c7a12a314d8905ba5c7c14aa7614f2610e79a8c7302eaa63fb74984d.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE global_settings\n SET value = (\n SELECT COALESCE(jsonb_agg(elem), '[]'::jsonb)\n FROM jsonb_array_elements(value) AS elem\n WHERE NOT (elem ? 'teams_channel')\n )\n WHERE name = 'critical_error_channels'\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "81b06122c7a12a314d8905ba5c7c14aa7614f2610e79a8c7302eaa63fb74984d" +} diff --git a/backend/.sqlx/query-ddf2eccb78a310ed00c7d8b9c3f05d394a7cbcf0038c72a78add5c7b02ef5927.json b/backend/.sqlx/query-ddf2eccb78a310ed00c7d8b9c3f05d394a7cbcf0038c72a78add5c7b02ef5927.json index 5bfff47576..c2dfed73a2 100644 --- a/backend/.sqlx/query-ddf2eccb78a310ed00c7d8b9c3f05d394a7cbcf0038c72a78add5c7b02ef5927.json +++ b/backend/.sqlx/query-ddf2eccb78a310ed00c7d8b9c3f05d394a7cbcf0038c72a78add5c7b02ef5927.json @@ -15,7 +15,7 @@ ] }, "nullable": [ - true + null ] }, "hash": "ddf2eccb78a310ed00c7d8b9c3f05d394a7cbcf0038c72a78add5c7b02ef5927" diff --git a/backend/.sqlx/query-df3b60c1d0fb44c97bf2611a7cda6bdb16a1be66e993a5cfadf6de2e3c8a5021.json b/backend/.sqlx/query-df3b60c1d0fb44c97bf2611a7cda6bdb16a1be66e993a5cfadf6de2e3c8a5021.json deleted file mode 100644 index 18f29062a6..0000000000 --- a/backend/.sqlx/query-df3b60c1d0fb44c97bf2611a7cda6bdb16a1be66e993a5cfadf6de2e3c8a5021.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE global_settings\n SET value = (\n SELECT jsonb_agg(elem)\n FROM jsonb_array_elements(value) AS elem\n WHERE NOT (elem ? 'teams_channel')\n )\n WHERE name = 'critical_error_channels'\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "df3b60c1d0fb44c97bf2611a7cda6bdb16a1be66e993a5cfadf6de2e3c8a5021" -} diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 83ad30b351..23791d63cd 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -e507af5589efb1a8a72ae4e231130f749bde738d \ No newline at end of file +d6aeb430a172cb8e969a4d29bd0e0727c2a25b9d \ No newline at end of file From 112361adbaa24780913f774a8f3cb3e5ffbfec91 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 18 Feb 2025 10:32:38 +0100 Subject: [PATCH 18/21] chore(main): release 1.463.5 (#5318) * chore(main): release 1.463.4 * update * Update CHANGELOG.md --- CHANGELOG.md | 8 ++ backend/Cargo.lock | 109 +++++++----------- backend/Cargo.toml | 4 +- backend/windmill-api/openapi.yaml | 2 +- benchmarks/lib.ts | 2 +- cli/main.ts | 2 +- frontend/package-lock.json | 4 +- frontend/package.json | 2 +- lsp/Pipfile | 4 +- openflow.openapi.yaml | 2 +- .../WindmillClient/WindmillClient.psd1 | 2 +- python-client/wmill/pyproject.toml | 2 +- python-client/wmill_pg/pyproject.toml | 2 +- typescript-client/jsr.json | 2 +- typescript-client/package.json | 2 +- version.txt | 2 +- 16 files changed, 69 insertions(+), 82 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d8093eecf..458b3a19ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [1.463.5](https://github.com/windmill-labs/windmill/compare/v1.463.4...v1.463.5) (2025-02-18) + + +### Bug Fixes + +* fix teams cleanup preventing start ([1b46e0f](https://github.com/windmill-labs/windmill/commit/1b46e0f08426497d549cf5007c93981df9ab41e5)) + + ## [1.463.4](https://github.com/windmill-labs/windmill/compare/v1.463.3...v1.463.4) (2025-02-17) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index f82041c837..53223e672c 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -263,7 +263,7 @@ dependencies = [ "arrow-data", "arrow-schema", "chrono", - "chrono-tz 0.9.0", + "chrono-tz", "half", "hashbrown 0.14.5", "num", @@ -1610,18 +1610,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93698b29de5e97ad0ae26447b344c482a7284c737d9ddc5f9e52b74a336671bb" dependencies = [ "chrono", - "chrono-tz-build 0.3.0", - "phf", -] - -[[package]] -name = "chrono-tz" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c6ac4f2c0bf0f44e9161aec9675e1050aa4a530663c4a9e37e108fa948bca9f" -dependencies = [ - "chrono", - "chrono-tz-build 0.4.0", + "chrono-tz-build", "phf", ] @@ -1636,16 +1625,6 @@ dependencies = [ "phf_codegen", ] -[[package]] -name = "chrono-tz-build" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94fea34d77a245229e7746bd2beb786cd2a896f306ff491fb8cecb3074b10a7" -dependencies = [ - "parse-zoneinfo", - "phf_codegen", -] - [[package]] name = "cipher" version = "0.3.0" @@ -10858,7 +10837,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill" -version = "1.463.4" +version = "1.463.5" dependencies = [ "anyhow", "axum", @@ -10901,7 +10880,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.463.4" +version = "1.463.5" dependencies = [ "anyhow", "argon2", @@ -10913,14 +10892,14 @@ dependencies = [ "async_zip", "axum", "base32", - "base64 0.22.1", + "base64 0.13.1", "byteorder", "bytes", "candle-core", "candle-nn", "candle-transformers", "chrono", - "chrono-tz 0.10.1", + "chrono-tz", "const_format", "cookie 0.17.0", "cron", @@ -10932,7 +10911,7 @@ dependencies = [ "hmac", "http 1.2.0", "hyper 1.6.0", - "itertools 0.14.0", + "itertools 0.10.5", "jsonwebtoken", "lazy_static", "magic-crypt", @@ -10974,7 +10953,7 @@ dependencies = [ "tokio-tar", "tokio-tungstenite", "tokio-util", - "tower 0.5.2", + "tower 0.4.13", "tower-cookies", "tower-http", "tracing", @@ -10995,9 +10974,9 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.463.4" +version = "1.463.5" dependencies = [ - "base64 0.22.1", + "base64 0.13.1", "chrono", "openapiv3", "prettyplease 0.1.25", @@ -11013,7 +10992,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.463.4" +version = "1.463.5" dependencies = [ "chrono", "serde", @@ -11026,7 +11005,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.463.4" +version = "1.463.5" dependencies = [ "anyhow", "serde", @@ -11040,7 +11019,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.463.4" +version = "1.463.5" dependencies = [ "anyhow", "async-stream", @@ -11049,7 +11028,7 @@ dependencies = [ "axum", "bytes", "chrono", - "chrono-tz 0.10.1", + "chrono-tz", "const_format", "crc", "cron", @@ -11062,7 +11041,7 @@ dependencies = [ "hmac", "hyper 1.6.0", "indexmap 2.7.1", - "itertools 0.14.0", + "itertools 0.10.5", "lazy_static", "magic-crypt", "mail-send", @@ -11099,7 +11078,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.463.4" +version = "1.463.5" dependencies = [ "regex", "serde", @@ -11113,7 +11092,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.463.4" +version = "1.463.5" dependencies = [ "anyhow", "bytes", @@ -11136,9 +11115,9 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.463.4" +version = "1.463.5" dependencies = [ - "itertools 0.14.0", + "itertools 0.10.5", "lazy_static", "proc-macro2", "quote", @@ -11148,7 +11127,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.463.4" +version = "1.463.5" dependencies = [ "convert_case 0.6.0", "serde", @@ -11157,7 +11136,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.463.4" +version = "1.463.5" dependencies = [ "anyhow", "lazy_static", @@ -11169,7 +11148,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.463.4" +version = "1.463.5" dependencies = [ "anyhow", "serde_json", @@ -11181,11 +11160,11 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.463.4" +version = "1.463.5" dependencies = [ "anyhow", "gosyn", - "itertools 0.14.0", + "itertools 0.10.5", "lazy_static", "regex", "windmill-parser", @@ -11193,7 +11172,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.463.4" +version = "1.463.5" dependencies = [ "anyhow", "lazy_static", @@ -11205,10 +11184,10 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.463.4" +version = "1.463.5" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.10.5", "php-parser-rs", "serde_json", "windmill-parser", @@ -11216,10 +11195,10 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.463.4" +version = "1.463.5" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.10.5", "rustpython-parser", "serde_json", "windmill-parser", @@ -11227,11 +11206,11 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.463.4" +version = "1.463.5" dependencies = [ "anyhow", "async-recursion", - "itertools 0.14.0", + "itertools 0.10.5", "lazy_static", "malachite", "malachite-bigint", @@ -11247,11 +11226,11 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.463.4" +version = "1.463.5" dependencies = [ "anyhow", "convert_case 0.6.0", - "itertools 0.14.0", + "itertools 0.10.5", "lazy_static", "pulldown-cmark", "quote", @@ -11264,7 +11243,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.463.4" +version = "1.463.5" dependencies = [ "anyhow", "lazy_static", @@ -11276,7 +11255,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.463.4" +version = "1.463.5" dependencies = [ "anyhow", "lazy_static", @@ -11294,7 +11273,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.463.4" +version = "1.463.5" dependencies = [ "anyhow", "getrandom 0.2.15", @@ -11316,7 +11295,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.463.4" +version = "1.463.5" dependencies = [ "anyhow", "serde_json", @@ -11326,20 +11305,20 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.463.4" +version = "1.463.5" dependencies = [ "anyhow", "async-recursion", "axum", "backon", "chrono", - "chrono-tz 0.10.1", + "chrono-tz", "cron", "futures", "futures-core", "hex", "hmac", - "itertools 0.14.0", + "itertools 0.10.5", "lazy_static", "prometheus", "regex", @@ -11359,7 +11338,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.463.4" +version = "1.463.5" dependencies = [ "wasm-bindgen", "wasm-bindgen-test", @@ -11369,12 +11348,12 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.463.4" +version = "1.463.5" dependencies = [ "anyhow", "async-recursion", "backon", - "base64 0.22.1", + "base64 0.13.1", "bit-vec", "bollard", "bytes", @@ -11397,7 +11376,7 @@ dependencies = [ "gcp_auth", "git-version", "hex", - "itertools 0.14.0", + "itertools 0.10.5", "jsonwebtoken", "lazy_static", "mappable-rc", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 0b8c02d1fa..2cb3e25aeb 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.463.4" +version = "1.463.5" authors.workspace = true edition.workspace = true @@ -30,7 +30,7 @@ members = [ ] [workspace.package] -version = "1.463.4" +version = "1.463.5" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 0e9b0328d4..c41a002fc8 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.463.4 + version: 1.463.5 title: Windmill API contact: diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index d7543a3d55..311c804d6c 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.463.4"; +export const VERSION = "v1.463.5"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/main.ts b/cli/main.ts index 51c8bd0809..34d7133577 100644 --- a/cli/main.ts +++ b/cli/main.ts @@ -62,7 +62,7 @@ export { // } // }); -export const VERSION = "1.463.4"; +export const VERSION = "1.463.5"; const command = new Command() .name("wmill") diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 9a8bce2889..22ed0a5141 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.463.4", + "version": "1.463.5", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.463.4", + "version": "1.463.5", "license": "AGPL-3.0", "dependencies": { "@anthropic-ai/sdk": "^0.32.1", diff --git a/frontend/package.json b/frontend/package.json index 1ece7bb764..84c3029eaf 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.463.4", + "version": "1.463.5", "scripts": { "dev": "vite dev", "build": "vite build", diff --git a/lsp/Pipfile b/lsp/Pipfile index 18b12dfc57..bb8f223315 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,8 +4,8 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.463.4" -wmill_pg = ">=1.463.4" +wmill = ">=1.463.5" +wmill_pg = ">=1.463.5" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index 043c7c77fb..ebad73b1a5 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.463.4 + version: 1.463.5 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index 7ab35ebe95..1f5f2a972f 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. - ModuleVersion = '1.463.4' + ModuleVersion = '1.463.5' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index 29a80185e2..e41d4d318d 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.463.4" +version = "1.463.5" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/python-client/wmill_pg/pyproject.toml b/python-client/wmill_pg/pyproject.toml index ee2dac54aa..ba4af4d219 100644 --- a/python-client/wmill_pg/pyproject.toml +++ b/python-client/wmill_pg/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill-pg" -version = "1.463.4" +version = "1.463.5" description = "An extension client for the wmill client library focused on pg" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index c4e0f34c3d..c5958e2f68 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.463.4", + "version": "1.463.5", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index ba67288973..6c750f8317 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.463.4", + "version": "1.463.5", "author": "Ruben Fiszel", "license": "Apache 2.0", "devDependencies": { diff --git a/version.txt b/version.txt index c6fd9a4d69..5d003b1aa7 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.463.4 +1.463.5 From 24ff5a6261e82eda06f84f43c47749e7aa609bdb Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 18 Feb 2025 10:40:17 +0100 Subject: [PATCH 19/21] fix: make teams cleanup non critical --- backend/ee-repo-ref.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 23791d63cd..562c984114 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -d6aeb430a172cb8e969a4d29bd0e0727c2a25b9d \ No newline at end of file +5d25cf2cd15c1953794045fd7debea14a33c7519 \ No newline at end of file From b4088faae1998f4a9efbae903cbdb85d5c6099af Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 18 Feb 2025 10:50:03 +0100 Subject: [PATCH 20/21] fix: pin chrono tz version to 0.10.1 --- backend/Cargo.lock | 128 ++++++++++++++++++++++++++------------------- backend/Cargo.toml | 2 +- 2 files changed, 76 insertions(+), 54 deletions(-) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 53223e672c..d0053f30fe 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -263,7 +263,7 @@ dependencies = [ "arrow-data", "arrow-schema", "chrono", - "chrono-tz", + "chrono-tz 0.9.0", "half", "hashbrown 0.14.5", "num", @@ -713,7 +713,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "tracing", - "uuid 1.13.1", + "uuid 1.13.2", ] [[package]] @@ -1222,15 +1222,16 @@ dependencies = [ [[package]] name = "blake3" -version = "1.5.5" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8ee0c1824c4dea5b5f81736aff91bae041d2c07ee1192bec91054e10e3e601e" +checksum = "1230237285e3e10cde447185e8975408ae24deaa67205ce684805c25bc0c7937" dependencies = [ "arrayref", "arrayvec", "cc", "cfg-if", "constant_time_eq", + "memmap2", ] [[package]] @@ -1610,7 +1611,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93698b29de5e97ad0ae26447b344c482a7284c737d9ddc5f9e52b74a336671bb" dependencies = [ "chrono", - "chrono-tz-build", + "chrono-tz-build 0.3.0", + "phf", +] + +[[package]] +name = "chrono-tz" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c6ac4f2c0bf0f44e9161aec9675e1050aa4a530663c4a9e37e108fa948bca9f" +dependencies = [ + "chrono", + "chrono-tz-build 0.4.0", "phf", ] @@ -1625,6 +1637,16 @@ dependencies = [ "phf_codegen", ] +[[package]] +name = "chrono-tz-build" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e94fea34d77a245229e7746bd2beb786cd2a896f306ff491fb8cecb3074b10a7" +dependencies = [ + "parse-zoneinfo", + "phf_codegen", +] + [[package]] name = "cipher" version = "0.3.0" @@ -2242,7 +2264,7 @@ dependencies = [ "tokio", "tokio-util", "url", - "uuid 1.13.1", + "uuid 1.13.2", "xz2", "zstd", ] @@ -2342,7 +2364,7 @@ dependencies = [ "regex", "sha2 0.10.8", "unicode-segmentation", - "uuid 1.13.1", + "uuid 1.13.2", ] [[package]] @@ -2509,7 +2531,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d" dependencies = [ "serde", - "uuid 1.13.1", + "uuid 1.13.2", ] [[package]] @@ -2804,7 +2826,7 @@ dependencies = [ "serde", "thiserror 1.0.69", "tokio", - "uuid 1.13.1", + "uuid 1.13.2", ] [[package]] @@ -5384,7 +5406,7 @@ dependencies = [ "sha2 0.10.8", "subprocess", "thiserror 1.0.69", - "uuid 1.13.1", + "uuid 1.13.2", "zstd", ] @@ -6345,7 +6367,7 @@ dependencies = [ "postgres-protocol 0.6.8", "serde", "serde_json", - "uuid 1.13.1", + "uuid 1.13.2", ] [[package]] @@ -7201,7 +7223,7 @@ dependencies = [ "rkyv_derive", "seahash", "tinyvec", - "uuid 1.13.1", + "uuid 1.13.2", ] [[package]] @@ -7592,7 +7614,7 @@ dependencies = [ "serde", "thiserror 1.0.69", "url", - "uuid 1.13.1", + "uuid 1.13.2", ] [[package]] @@ -7630,7 +7652,7 @@ dependencies = [ "schemars_derive", "serde", "serde_json", - "uuid 1.13.1", + "uuid 1.13.2", ] [[package]] @@ -8365,7 +8387,7 @@ dependencies = [ "tokio-stream", "tracing", "url", - "uuid 1.13.1", + "uuid 1.13.2", "webpki-roots", ] @@ -8449,7 +8471,7 @@ dependencies = [ "stringprep", "thiserror 2.0.11", "tracing", - "uuid 1.13.1", + "uuid 1.13.2", "whoami", ] @@ -8490,7 +8512,7 @@ dependencies = [ "stringprep", "thiserror 2.0.11", "tracing", - "uuid 1.13.1", + "uuid 1.13.2", "whoami", ] @@ -8516,7 +8538,7 @@ dependencies = [ "sqlx-core", "tracing", "url", - "uuid 1.13.1", + "uuid 1.13.2", ] [[package]] @@ -9163,7 +9185,7 @@ dependencies = [ "tempfile", "thiserror 1.0.69", "time", - "uuid 1.13.1", + "uuid 1.13.2", "winapi", ] @@ -9393,7 +9415,7 @@ dependencies = [ "tokio-rustls 0.24.1", "tokio-util", "tracing", - "uuid 1.13.1", + "uuid 1.13.2", ] [[package]] @@ -10100,9 +10122,9 @@ dependencies = [ [[package]] name = "tree-sitter-language" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38eee4db33814de3d004de9d8d825627ed3320d0989cce0dea30efaf5be4736c" +checksum = "c4013970217383f67b18aef68f6fb2e8d409bc5755227092d32efb0422ba24b8" [[package]] name = "triomphe" @@ -10174,9 +10196,9 @@ checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a" [[package]] name = "typenum" -version = "1.17.0" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" +checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" [[package]] name = "typify" @@ -10229,7 +10251,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab82fc73182c29b02e2926a6df32f2241dbadb5cfc111fd595515b3598f46bb3" dependencies = [ "rand 0.9.0", - "uuid 1.13.1", + "uuid 1.13.2", "web-time", ] @@ -10511,9 +10533,9 @@ dependencies = [ [[package]] name = "uuid" -version = "1.13.1" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ced87ca4be083373936a67f8de945faa23b6b42384bd5b64434850802c6dccd0" +checksum = "8c1f41ffb7cf259f1ecc2876861a17e7142e63ead296f671f81f6ae85903e0d6" dependencies = [ "getrandom 0.3.1", "serde", @@ -10866,7 +10888,7 @@ dependencies = [ "tokio", "tracing", "url", - "uuid 1.13.1", + "uuid 1.13.2", "v8", "windmill-api", "windmill-api-client", @@ -10892,14 +10914,14 @@ dependencies = [ "async_zip", "axum", "base32", - "base64 0.13.1", + "base64 0.22.1", "byteorder", "bytes", "candle-core", "candle-nn", "candle-transformers", "chrono", - "chrono-tz", + "chrono-tz 0.10.1", "const_format", "cookie 0.17.0", "cron", @@ -10911,7 +10933,7 @@ dependencies = [ "hmac", "http 1.2.0", "hyper 1.6.0", - "itertools 0.10.5", + "itertools 0.14.0", "jsonwebtoken", "lazy_static", "magic-crypt", @@ -10953,7 +10975,7 @@ dependencies = [ "tokio-tar", "tokio-tungstenite", "tokio-util", - "tower 0.4.13", + "tower 0.5.2", "tower-cookies", "tower-http", "tracing", @@ -10961,7 +10983,7 @@ dependencies = [ "ulid", "url", "urlencoding", - "uuid 1.13.1", + "uuid 1.13.2", "windmill-audit", "windmill-common", "windmill-git-sync", @@ -10976,7 +10998,7 @@ dependencies = [ name = "windmill-api-client" version = "1.463.5" dependencies = [ - "base64 0.13.1", + "base64 0.22.1", "chrono", "openapiv3", "prettyplease 0.1.25", @@ -10987,7 +11009,7 @@ dependencies = [ "serde", "serde_json", "syn 1.0.109", - "uuid 1.13.1", + "uuid 1.13.2", ] [[package]] @@ -11012,7 +11034,7 @@ dependencies = [ "serde_json", "sqlx", "tracing", - "uuid 1.13.1", + "uuid 1.13.2", "windmill-common", "windmill-queue", ] @@ -11028,7 +11050,7 @@ dependencies = [ "axum", "bytes", "chrono", - "chrono-tz", + "chrono-tz 0.10.1", "const_format", "crc", "cron", @@ -11041,7 +11063,7 @@ dependencies = [ "hmac", "hyper 1.6.0", "indexmap 2.7.1", - "itertools 0.10.5", + "itertools 0.14.0", "lazy_static", "magic-crypt", "mail-send", @@ -11072,7 +11094,7 @@ dependencies = [ "tracing-loki", "tracing-opentelemetry", "tracing-subscriber", - "uuid 1.13.1", + "uuid 1.13.2", "windmill-macros", ] @@ -11085,7 +11107,7 @@ dependencies = [ "serde_json", "sqlx", "tracing", - "uuid 1.13.1", + "uuid 1.13.2", "windmill-common", "windmill-queue", ] @@ -11109,7 +11131,7 @@ dependencies = [ "tokio", "tokio-tar", "tracing", - "uuid 1.13.1", + "uuid 1.13.2", "windmill-common", ] @@ -11117,7 +11139,7 @@ dependencies = [ name = "windmill-macros" version = "1.463.5" dependencies = [ - "itertools 0.10.5", + "itertools 0.14.0", "lazy_static", "proc-macro2", "quote", @@ -11164,7 +11186,7 @@ version = "1.463.5" dependencies = [ "anyhow", "gosyn", - "itertools 0.10.5", + "itertools 0.14.0", "lazy_static", "regex", "windmill-parser", @@ -11187,7 +11209,7 @@ name = "windmill-parser-php" version = "1.463.5" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.14.0", "php-parser-rs", "serde_json", "windmill-parser", @@ -11198,7 +11220,7 @@ name = "windmill-parser-py" version = "1.463.5" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.14.0", "rustpython-parser", "serde_json", "windmill-parser", @@ -11210,7 +11232,7 @@ version = "1.463.5" dependencies = [ "anyhow", "async-recursion", - "itertools 0.10.5", + "itertools 0.14.0", "lazy_static", "malachite", "malachite-bigint", @@ -11230,7 +11252,7 @@ version = "1.463.5" dependencies = [ "anyhow", "convert_case 0.6.0", - "itertools 0.10.5", + "itertools 0.14.0", "lazy_static", "pulldown-cmark", "quote", @@ -11312,13 +11334,13 @@ dependencies = [ "axum", "backon", "chrono", - "chrono-tz", + "chrono-tz 0.10.1", "cron", "futures", "futures-core", "hex", "hmac", - "itertools 0.10.5", + "itertools 0.14.0", "lazy_static", "prometheus", "regex", @@ -11331,7 +11353,7 @@ dependencies = [ "tokio", "tracing", "ulid", - "uuid 1.13.1", + "uuid 1.13.2", "windmill-audit", "windmill-common", ] @@ -11353,7 +11375,7 @@ dependencies = [ "anyhow", "async-recursion", "backon", - "base64 0.13.1", + "base64 0.22.1", "bit-vec", "bollard", "bytes", @@ -11376,7 +11398,7 @@ dependencies = [ "gcp_auth", "git-version", "hex", - "itertools 0.10.5", + "itertools 0.14.0", "jsonwebtoken", "lazy_static", "mappable-rc", @@ -11405,7 +11427,7 @@ dependencies = [ "tokio-util", "tracing", "urlencoding", - "uuid 1.13.1", + "uuid 1.13.2", "windmill-audit", "windmill-common", "windmill-git-sync", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 2cb3e25aeb..f3b95d4c78 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -174,7 +174,7 @@ uuid = { version = "^1", features = ["serde", "v4"] } thiserror = "^2" anyhow = "^1" chrono = { version = "0.4.35", features = ["serde"] } -chrono-tz = "^0" +chrono-tz = "^0.10.1" tracing = "^0" tracing-subscriber = { version = "^0", features = ["env-filter", "json"] } tracing-appender = "^0" From 1069ad39992940e32e5d8566ef2283970525be1a Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 18 Feb 2025 13:17:10 +0100 Subject: [PATCH 21/21] fix: improve v2 migration finalizer to avoid deadlocks --- backend/src/main.rs | 13 +++- backend/windmill-api/src/db.rs | 104 ++++++++++++++++++++++++++------ backend/windmill-api/src/lib.rs | 8 ++- 3 files changed, 101 insertions(+), 24 deletions(-) diff --git a/backend/src/main.rs b/backend/src/main.rs index 432095580d..58d22b6eee 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -21,7 +21,7 @@ use std::{ net::{IpAddr, Ipv4Addr, SocketAddr}, time::Duration, }; -use tokio::{fs::File, io::AsyncReadExt}; +use tokio::{fs::File, io::AsyncReadExt, task::JoinHandle}; use uuid::Uuid; use windmill_api::HTTP_CLIENT; @@ -372,6 +372,7 @@ async fn windmill_main() -> anyhow::Result<()> { let is_agent = mode == Mode::Agent; + let mut migration_handle: Option> = None; #[cfg(feature = "parquet")] let disable_s3_store = std::env::var("DISABLE_S3_STORE") .ok() @@ -384,7 +385,7 @@ async fn windmill_main() -> anyhow::Result<()> { if !skip_migration { // migration code to avoid break - windmill_api::migrate_db(&db).await?; + migration_handle = windmill_api::migrate_db(&db).await?; } else { tracing::info!("SKIP_MIGRATION set, skipping db migration...") } @@ -682,6 +683,14 @@ Windmill Community Edition {GIT_VERSION} loop { tokio::select! { biased; + Some(_) = async { if let Some(jh) = migration_handle.take() { + tracing::info!("migration job finished"); + Some(jh.await) + } else { + None + }} => { + continue; + }, _ = monitor_killpill_rx.recv() => { tracing::info!("received killpill for monitor job"); break; diff --git a/backend/windmill-api/src/db.rs b/backend/windmill-api/src/db.rs index b8d15160c7..9b694ad3bf 100644 --- a/backend/windmill-api/src/db.rs +++ b/backend/windmill-api/src/db.rs @@ -15,6 +15,7 @@ use sqlx::{ Executor, PgConnection, Pool, Postgres, }; +use tokio::task::JoinHandle; use windmill_audit::audit_ee::{AuditAuthor, AuditAuthorable}; use windmill_common::{ db::{Authable, Authed}, @@ -170,7 +171,7 @@ impl Migrate for CustomMigrator { } } -pub async fn migrate(db: &DB) -> Result<(), Error> { +pub async fn migrate(db: &DB) -> Result>, Error> { let migrator = db.acquire().await?; let mut custom_migrator = CustomMigrator { inner: migrator }; @@ -225,9 +226,10 @@ pub async fn migrate(db: &DB) -> Result<(), Error> { } }); - if !has_done_migration(db, "v2_finalize_disable_sync_III").await { + let mut jh = None; + if !has_done_migration(db, "v2_finalize_job_completed").await { let db2 = db.clone(); - let _ = tokio::task::spawn(async move { + let v2jh = tokio::task::spawn(async move { loop { if !*MIN_VERSION_IS_AT_LEAST_1_461.read().await { tracing::info!("Waiting for all workers to be at least version 1.461 before applying v2 finalize migration, sleeping for 5s..."); @@ -245,9 +247,10 @@ pub async fn migrate(db: &DB) -> Result<(), Error> { break; } }); + jh = Some(v2jh) } - Ok(()) + Ok(jh) } async fn fix_flow_versioning_migration( @@ -373,29 +376,91 @@ async fn v2_finalize(db: &DB) -> Result<(), Error> { run_windmill_migration!("v2_finalize_disable_sync_III", db, |tx| { tx.execute( r#" + LOCK TABLE v2_job_queue IN ACCESS EXCLUSIVE MODE; ALTER TABLE v2_job_queue DISABLE ROW LEVEL SECURITY; - ALTER TABLE v2_job_completed DISABLE ROW LEVEL SECURITY; - - DROP FUNCTION IF EXISTS v2_job_after_update CASCADE; - DROP FUNCTION IF EXISTS v2_job_completed_before_insert CASCADE; - DROP FUNCTION IF EXISTS v2_job_completed_before_update CASCADE; - DROP FUNCTION IF EXISTS v2_job_queue_after_insert CASCADE; - DROP FUNCTION IF EXISTS v2_job_queue_before_insert CASCADE; - DROP FUNCTION IF EXISTS v2_job_queue_before_update CASCADE; - DROP FUNCTION IF EXISTS v2_job_runtime_before_insert CASCADE; - DROP FUNCTION IF EXISTS v2_job_runtime_before_update CASCADE; - DROP FUNCTION IF EXISTS v2_job_status_before_insert CASCADE; - DROP FUNCTION IF EXISTS v2_job_status_before_update CASCADE; - - DROP VIEW IF EXISTS completed_job, completed_job_view, job, queue, queue_view CASCADE; - "#, ) .await?; }); + + run_windmill_migration!("v2_finalize_disable_sync_III_2", db, |tx| { + tx.execute( + r#" + LOCK TABLE v2_job_completed IN ACCESS EXCLUSIVE MODE; + ALTER TABLE v2_job_completed DISABLE ROW LEVEL SECURITY; + "#, + ) + .await?; + }); + + run_windmill_migration!("v2_finalize_disable_sync_III_3", db, |tx| { + tx.execute( + r#" + LOCK TABLE v2_job IN ACCESS EXCLUSIVE MODE; + DROP FUNCTION IF EXISTS v2_job_after_update CASCADE; + "#, + ) + .await?; + }); + + run_windmill_migration!("v2_finalize_disable_sync_III_4", db, |tx| { + tx.execute( + r#" + LOCK TABLE v2_job_completed IN ACCESS EXCLUSIVE MODE; + DROP FUNCTION IF EXISTS v2_job_completed_before_insert CASCADE; + DROP FUNCTION IF EXISTS v2_job_completed_before_update CASCADE; + "#, + ) + .await?; + }); + + run_windmill_migration!("v2_finalize_disable_sync_III_5", db, |tx| { + tx.execute( + r#" + LOCK TABLE v2_job_queue IN ACCESS EXCLUSIVE MODE; + DROP FUNCTION IF EXISTS v2_job_queue_after_insert CASCADE; + DROP FUNCTION IF EXISTS v2_job_queue_before_insert CASCADE; + DROP FUNCTION IF EXISTS v2_job_queue_before_update CASCADE; + "#, + ) + .await?; + }); + + run_windmill_migration!("v2_finalize_disable_sync_III_6", db, |tx| { + tx.execute( + r#" + LOCK TABLE v2_job_runtime IN ACCESS EXCLUSIVE MODE; + DROP FUNCTION IF EXISTS v2_job_runtime_before_insert CASCADE; + DROP FUNCTION IF EXISTS v2_job_runtime_before_update CASCADE; + "#, + ) + .await?; + }); + + run_windmill_migration!("v2_finalize_disable_sync_III_7", db, |tx| { + tx.execute( + r#" + LOCK TABLE v2_job_status IN ACCESS EXCLUSIVE MODE; + DROP FUNCTION IF EXISTS v2_job_status_before_insert CASCADE; + DROP FUNCTION IF EXISTS v2_job_status_before_update CASCADE; + "#, + ) + .await?; + }); + + run_windmill_migration!("v2_finalize_disable_sync_III_8", db, |tx| { + tx.execute( + r#" + DROP VIEW IF EXISTS completed_job, completed_job_view, job, queue, queue_view CASCADE; + "#, + ) + .await?; + }); + run_windmill_migration!("v2_finalize_job_queue", db, |tx| { tx.execute( r#" + LOCK TABLE v2_job_queue IN ACCESS EXCLUSIVE MODE; ALTER TABLE v2_job_queue DROP COLUMN IF EXISTS __parent_job CASCADE, DROP COLUMN IF EXISTS __created_by CASCADE, @@ -434,6 +499,7 @@ async fn v2_finalize(db: &DB) -> Result<(), Error> { run_windmill_migration!("v2_finalize_job_completed", db, |tx| { tx.execute( r#" + LOCK TABLE v2_job_completed IN ACCESS EXCLUSIVE MODE; ALTER TABLE v2_job_completed DROP COLUMN IF EXISTS __parent_job CASCADE, DROP COLUMN IF EXISTS __created_by CASCADE, diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index b4c262b9eb..4199494ed4 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -34,6 +34,7 @@ use http::HeaderValue; use reqwest::Client; #[cfg(feature = "oauth2")] use std::collections::HashMap; +use tokio::task::JoinHandle; use windmill_common::global_settings::load_value_from_global_settings; use windmill_common::global_settings::EMAIL_DOMAIN_SETTING; use windmill_common::worker::HUB_CACHE_DIR; @@ -641,7 +642,8 @@ async fn openapi_json() -> &'static str { include_str!("../openapi-deref.json") } -pub async fn migrate_db(db: &DB) -> anyhow::Result<()> { - db::migrate(db).await?; - Ok(()) +pub async fn migrate_db(db: &DB) -> anyhow::Result>> { + db::migrate(db) + .await + .map_err(|e| anyhow::anyhow!("Error migrating db: {e:#}")) }