diff --git a/.github/workflows/backend-check.yml b/.github/workflows/backend-check.yml new file mode 100644 index 0000000000..c26abd93d4 --- /dev/null +++ b/.github/workflows/backend-check.yml @@ -0,0 +1,130 @@ +name: Backend check +on: + push: + paths: + - "backend/**" + - ".github/workflows/backend-check.yml" + +jobs: + check_oss: + runs-on: ubicloud-standard-8 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + cache-workspaces: backend + toolchain: 1.82.0 + - uses: Swatinem/rust-cache@v2 + with: + workspaces: backend + - name: cargo check + working-directory: ./backend + timeout-minutes: 16 + run: SQLX_OFFLINE=true cargo check + + check_oss_full: + runs-on: ubicloud-standard-8 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + + - name: install xmlsec1 + run: | + sudo apt-get update + sudo apt-get install -y libxml2-dev libxmlsec1-dev + + - uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + cache-workspaces: backend + toolchain: 1.82.0 + - uses: Swatinem/rust-cache@v2 + with: + workspaces: backend + - name: cargo check + working-directory: ./backend + timeout-minutes: 16 + run: SQLX_OFFLINE=true cargo check --all-features + + check_ee: + runs-on: ubicloud-standard-8 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Read EE repo commit hash + run: | + echo "ee_repo_ref=$(cat ./backend/ee-repo-ref.txt)" >> "$GITHUB_ENV" + + - uses: actions/checkout@v4 + with: + repository: windmill-labs/windmill-ee-private + path: ./windmill-ee-private + ref: ${{ env.ee_repo_ref }} + token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }} + fetch-depth: 0 + + + - name: Substitute EE code (EE logic is behind feature flag) + run: | + ./backend/substitute_ee_code.sh --copy --dir ./windmill-ee-private + + + - uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + cache-workspaces: backend + toolchain: 1.82.0 + - uses: Swatinem/rust-cache@v2 + with: + workspaces: backend + - name: cargo check + working-directory: ./backend + timeout-minutes: 16 + run: SQLX_OFFLINE=true cargo check + + check_ee_full: + runs-on: ubicloud-standard-8 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Read EE repo commit hash + run: | + echo "ee_repo_ref=$(cat ./backend/ee-repo-ref.txt)" >> "$GITHUB_ENV" + + - uses: actions/checkout@v4 + with: + repository: windmill-labs/windmill-ee-private + path: ./windmill-ee-private + ref: ${{ env.ee_repo_ref }} + token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }} + fetch-depth: 0 + + - name: install xmlsec1 + run: | + sudo apt-get update + sudo apt-get install -y libxml2-dev libxmlsec1-dev + + - name: Substitute EE code (EE logic is behind feature flag) + run: | + ./backend/substitute_ee_code.sh --copy --dir ./windmill-ee-private + + - uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + cache-workspaces: backend + toolchain: 1.82.0 + - uses: Swatinem/rust-cache@v2 + with: + workspaces: backend + - name: cargo check + timeout-minutes: 16 + working-directory: ./backend + run: SQLX_OFFLINE=true cargo check --all-features + + \ No newline at end of file diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 0c4955c925..fbff0f8254 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -36,7 +36,7 @@ permissions: write-all jobs: build: - runs-on: ubicloud-standard-30 + runs-on: ubicloud-standard-8 if: (github.event_name != 'workflow_dispatch') || (inputs.ee == false) steps: - uses: actions/checkout@v4 @@ -99,7 +99,7 @@ jobs: cache-to: type=gha build_arm: - runs-on: ubicloud-standard-30-arm + runs-on: ubicloud-standard-8-arm if: (github.event_name != 'workflow_dispatch') || (inputs.ee == false) steps: - uses: actions/checkout@v4 diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ad3331073..a5beb3719e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,25 @@ # Changelog +## [1.440.1](https://github.com/windmill-labs/windmill/compare/v1.440.0...v1.440.1) (2024-12-18) + + +### Bug Fixes + +* **internal:** updating rust to 1.82 ([02a8f1f](https://github.com/windmill-labs/windmill/commit/02a8f1f86453a5f8769364ba3798998b8830d086)) + +## [1.440.0](https://github.com/windmill-labs/windmill/compare/v1.439.0...v1.440.0) (2024-12-18) + + +### Features + +* **cache:** remove persistent raw values from queue ([#4866](https://github.com/windmill-labs/windmill/issues/4866)) ([977ac5c](https://github.com/windmill-labs/windmill/commit/977ac5c3f3c2e8224f4915e483ae60f28ce008fc)) + + +### Bug Fixes + +* add workspace selector and fix css for create webhook page ([#4939](https://github.com/windmill-labs/windmill/issues/4939)) ([d880655](https://github.com/windmill-labs/windmill/commit/d8806555f1d78a5fae7ae77acbcdad402e89951d)) +* fix relative imports in cached flow scripts ([13be0cd](https://github.com/windmill-labs/windmill/commit/13be0cd1c822d7a809dc96914dd1286510b9f9eb)) + ## [1.439.0](https://github.com/windmill-labs/windmill/compare/v1.438.0...v1.439.0) (2024-12-15) diff --git a/Dockerfile b/Dockerfile index ccc8643223..df8e493e66 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ ARG DEBIAN_IMAGE=debian:bookworm-slim -ARG RUST_IMAGE=rust:1.80-slim-bookworm +ARG RUST_IMAGE=rust:1.82-slim-bookworm ARG PYTHON_IMAGE=python:3.11.10-slim-bookworm FROM ${RUST_IMAGE} AS rust_base diff --git a/backend/.sqlx/query-7a9a711c7cb05ed8a61116586fb68ef270567ebd70266f07139f3e3e34940699.json b/backend/.sqlx/query-7a9a711c7cb05ed8a61116586fb68ef270567ebd70266f07139f3e3e34940699.json new file mode 100644 index 0000000000..4a849599fe --- /dev/null +++ b/backend/.sqlx/query-7a9a711c7cb05ed8a61116586fb68ef270567ebd70266f07139f3e3e34940699.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT value AS \"value!: Json>\"\n FROM flow_version WHERE id = $1 LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "value!: Json>", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + false + ] + }, + "hash": "7a9a711c7cb05ed8a61116586fb68ef270567ebd70266f07139f3e3e34940699" +} diff --git a/backend/.sqlx/query-c7de95ef4934752b62d6d01bf663ab6a104bbcfb827a4a9e4e03e38732375d55.json b/backend/.sqlx/query-c7de95ef4934752b62d6d01bf663ab6a104bbcfb827a4a9e4e03e38732375d55.json new file mode 100644 index 0000000000..c044973cc0 --- /dev/null +++ b/backend/.sqlx/query-c7de95ef4934752b62d6d01bf663ab6a104bbcfb827a4a9e4e03e38732375d55.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT value AS \"value!: Json>\"\n FROM flow_version_lite WHERE id = $1 LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "value!: Json>", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + true + ] + }, + "hash": "c7de95ef4934752b62d6d01bf663ab6a104bbcfb827a4a9e4e03e38732375d55" +} diff --git a/backend/.sqlx/query-d061e7ca73987036928e17245360fd7f2969aab0c422a00bbdade6a7236aa75d.json b/backend/.sqlx/query-d061e7ca73987036928e17245360fd7f2969aab0c422a00bbdade6a7236aa75d.json new file mode 100644 index 0000000000..d6d6e77aa4 --- /dev/null +++ b/backend/.sqlx/query-d061e7ca73987036928e17245360fd7f2969aab0c422a00bbdade6a7236aa75d.json @@ -0,0 +1,72 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT lock AS \"lock: String\", content AS \"code!: String\",\n language AS \"language: Option\", envs AS \"envs: Vec\", codebase AS \"codebase: String\" FROM script WHERE hash = $1 LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "lock: String", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "code!: String", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "language: Option", + "type_info": { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp" + ] + } + } + } + }, + { + "ordinal": 3, + "name": "envs: Vec", + "type_info": "VarcharArray" + }, + { + "ordinal": 4, + "name": "codebase: String", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + true, + false, + false, + true, + true + ] + }, + "hash": "d061e7ca73987036928e17245360fd7f2969aab0c422a00bbdade6a7236aa75d" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 445350f510..09049c2df1 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -537,7 +537,7 @@ dependencies = [ "hex", "hmac", "http-types", - "hyper 0.14.31", + "hyper 0.14.32", "hyper-tls 0.5.0", "serde", "serde_json", @@ -771,9 +771,9 @@ dependencies = [ [[package]] name = "aws-smithy-async" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62220bc6e97f946ddd51b5f1361f78996e704677afc518a4ff66b7a72ea1378c" +checksum = "8aa8ff1492fd9fb99ae28e8467af0dbbb7c31512b16fabf1a0f10d7bb6ef78bb" dependencies = [ "futures-util", "pin-project-lite", @@ -830,9 +830,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.7.4" +version = "1.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f20685047ca9d6f17b994a07f629c813f08b5bce65523e47124879e60103d45" +checksum = "431a10d0e07e09091284ef04453dae4069283aa108d209974d67e77ae1caa658" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -845,7 +845,7 @@ dependencies = [ "http-body 0.4.6", "http-body 1.0.1", "httparse", - "hyper 0.14.31", + "hyper 0.14.32", "hyper-rustls 0.24.2", "once_cell", "pin-project-lite", @@ -874,9 +874,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.2.9" +version = "1.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fbd94a32b3a7d55d3806fe27d98d3ad393050439dd05eb53ece36ec5e3d3510" +checksum = "8ecbf4d5dfb169812e2b240a4350f15ad3c6b03a54074e5712818801615f2dc5" dependencies = [ "base64-simd 0.8.0", "bytes", @@ -934,7 +934,7 @@ dependencies = [ "http 1.2.0", "http-body 1.0.1", "http-body-util", - "hyper 1.5.1", + "hyper 1.5.2", "hyper-util", "itoa", "matchit", @@ -997,7 +997,7 @@ dependencies = [ "addr2line", "cfg-if", "libc", - "miniz_oxide 0.8.0", + "miniz_oxide 0.8.2", "object", "rustc-demangle", "windows-targets 0.52.6", @@ -1272,7 +1272,7 @@ dependencies = [ "hex", "http 1.2.0", "http-body-util", - "hyper 1.5.1", + "hyper 1.5.2", "hyper-named-pipe", "hyper-util", "hyperlocal", @@ -1283,7 +1283,7 @@ dependencies = [ "serde_json", "serde_repr", "serde_urlencoded", - "thiserror 2.0.7", + "thiserror 2.0.8", "tokio", "tokio-util", "tower-service", @@ -1736,15 +1736,15 @@ checksum = "510ca239cf13b7f8d16a2b48f263de7b4f8c566f0af58d901031473c76afb1e3" [[package]] name = "console" -version = "0.15.8" +version = "0.15.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb" +checksum = "ea3c6ecd8059b57859df5c69830340ed3c41d30e3da0c1cbed90a96ac853041b" dependencies = [ "encode_unicode", - "lazy_static", "libc", - "unicode-width 0.1.14", - "windows-sys 0.52.0", + "once_cell", + "unicode-width 0.2.0", + "windows-sys 0.59.0", ] [[package]] @@ -2593,8 +2593,8 @@ dependencies = [ "hickory-resolver", "http 1.2.0", "http-body-util", - "hyper 1.5.1", - "hyper-rustls 0.27.3", + "hyper 1.5.2", + "hyper-rustls 0.27.4", "hyper-util", "ipnet", "percent-encoding", @@ -3114,9 +3114,9 @@ dependencies = [ [[package]] name = "encode_unicode" -version = "0.3.6" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" [[package]] name = "encoding_rs" @@ -3297,7 +3297,7 @@ checksum = "c936bfdafb507ebbf50b8074c54fa31c5be9a1e7e5f467dd659697041407d07c" dependencies = [ "crc32fast", "libz-sys", - "miniz_oxide 0.8.0", + "miniz_oxide 0.8.2", ] [[package]] @@ -3583,7 +3583,7 @@ dependencies = [ "async-trait", "base64 0.21.7", "dirs-next", - "hyper 0.14.31", + "hyper 0.14.32", "hyper-rustls 0.24.2", "ring 0.16.20", "rustls 0.21.12", @@ -4068,11 +4068,11 @@ dependencies = [ [[package]] name = "home" -version = "0.5.9" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -4203,9 +4203,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.31" +version = "0.14.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c08302e8fa335b151b788c775ff56e7a03ae64ff85c548ee820fecb70356e85" +checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" dependencies = [ "bytes", "futures-channel", @@ -4227,9 +4227,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.5.1" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97818827ef4f364230e16705d4706e2897df2bb60617d6ca15d598025a3c481f" +checksum = "256fb8d4bd6413123cc9d91832d78325c48ff41677595be797d90f42969beae0" dependencies = [ "bytes", "futures-channel", @@ -4253,7 +4253,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73b7d8abf35697b81a825e386fc151e0d503e8cb5fcb93cc8669c376dfd6f278" dependencies = [ "hex", - "hyper 1.5.1", + "hyper 1.5.2", "hyper-util", "pin-project-lite", "tokio", @@ -4269,7 +4269,7 @@ checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", "http 0.2.12", - "hyper 0.14.31", + "hyper 0.14.32", "log", "rustls 0.21.12", "rustls-native-certs 0.6.3", @@ -4279,13 +4279,13 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.3" +version = "0.27.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" +checksum = "f6884a48c6826ec44f524c7456b163cebe9e55a18d7b5e307cb4f100371cc767" dependencies = [ "futures-util", "http 1.2.0", - "hyper 1.5.1", + "hyper 1.5.2", "hyper-util", "rustls 0.23.20", "rustls-native-certs 0.8.1", @@ -4302,7 +4302,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3203a961e5c83b6f5498933e78b6b263e208c197b63e9c6c53cc82ffd3f63793" dependencies = [ - "hyper 1.5.1", + "hyper 1.5.2", "hyper-util", "pin-project-lite", "tokio", @@ -4316,7 +4316,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ "bytes", - "hyper 0.14.31", + "hyper 0.14.32", "native-tls", "tokio", "tokio-native-tls", @@ -4330,7 +4330,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", "http-body-util", - "hyper 1.5.1", + "hyper 1.5.2", "hyper-util", "native-tls", "tokio", @@ -4349,7 +4349,7 @@ dependencies = [ "futures-util", "http 1.2.0", "http-body 1.0.1", - "hyper 1.5.1", + "hyper 1.5.2", "pin-project-lite", "socket2", "tokio", @@ -4366,7 +4366,7 @@ checksum = "986c5ce3b994526b3cd75578e62554abd09f0899d6206de48b3e96ab34ccc8c7" dependencies = [ "hex", "http-body-util", - "hyper 1.5.1", + "hyper 1.5.2", "hyper-util", "pin-project-lite", "tokio", @@ -5241,9 +5241,9 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.8.0" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" +checksum = "4ffbe83022cedc1d264172192511ae958937694cd57ce297164951b8b3568394" dependencies = [ "adler2", ] @@ -5623,7 +5623,7 @@ dependencies = [ "chrono", "futures", "humantime", - "hyper 1.5.1", + "hyper 1.5.2", "itertools 0.13.0", "md-5 0.10.6", "parking_lot", @@ -6609,7 +6609,7 @@ dependencies = [ "rustc-hash 2.1.0", "rustls 0.23.20", "socket2", - "thiserror 2.0.7", + "thiserror 2.0.8", "tokio", "tracing", ] @@ -6628,7 +6628,7 @@ dependencies = [ "rustls 0.23.20", "rustls-pki-types", "slab", - "thiserror 2.0.7", + "thiserror 2.0.8", "tinyvec", "tracing", "web-time", @@ -6636,9 +6636,9 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.8" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52cd4b1eff68bf27940dd39811292c49e007f4d0b4c357358dc9b0197be6b527" +checksum = "1c40286217b4ba3a71d644d752e6a0b71f13f1b6a2c5311acfcbe0c2418ed904" dependencies = [ "cfg_aliases", "libc", @@ -6940,7 +6940,7 @@ dependencies = [ "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", - "hyper 0.14.31", + "hyper 0.14.32", "hyper-tls 0.5.0", "ipnet", "js-sys", @@ -6984,8 +6984,8 @@ dependencies = [ "http 1.2.0", "http-body 1.0.1", "http-body-util", - "hyper 1.5.1", - "hyper-rustls 0.27.3", + "hyper 1.5.2", + "hyper-rustls 0.27.4", "hyper-tls 0.6.0", "hyper-util", "ipnet", @@ -7310,7 +7310,7 @@ dependencies = [ "openssl-probe", "rustls-pki-types", "schannel", - "security-framework 3.0.1", + "security-framework 3.1.0", ] [[package]] @@ -7592,9 +7592,9 @@ dependencies = [ [[package]] name = "security-framework" -version = "3.0.1" +version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1415a607e92bec364ea2cf9264646dcce0f91e6d65281bd6f2819cca3bf39c8" +checksum = "81d3f8c9bfcc3cbb6b0179eb57042d75b1582bdc65c3cb95f3fa999509c03cbc" dependencies = [ "bitflags 2.6.0", "core-foundation 0.10.0", @@ -7605,9 +7605,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.12.1" +version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa39c7303dc58b5543c94d22c1766b0d31f2ee58306363ea622b10bbc075eaa2" +checksum = "1863fd3768cd83c56a7f60faa4dc0d403f1b6df0a38c3c25f44b7894e45370d5" dependencies = [ "core-foundation-sys", "libc", @@ -9195,11 +9195,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.7" +version = "2.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93605438cbd668185516ab499d589afb7ee1859ea3d5fc8f6b0755e1c7443767" +checksum = "08f5383f3e0071702bf93ab5ee99b52d26936be9dedd9413067cbdcddcb6141a" dependencies = [ - "thiserror-impl 2.0.7", + "thiserror-impl 2.0.8", ] [[package]] @@ -9215,9 +9215,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.7" +version = "2.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1d8749b4531af2117677a5fcd12b1348a3fe2b81e36e61ffeac5c4aa3273e36" +checksum = "f2f357fcec90b3caef6623a099691be676d033b40a058ac95d2a6ade6fa0c943" dependencies = [ "proc-macro2", "quote", @@ -9655,7 +9655,7 @@ dependencies = [ "http 1.2.0", "http-body 1.0.1", "http-body-util", - "hyper 1.5.1", + "hyper 1.5.2", "hyper-timeout", "hyper-util", "percent-encoding", @@ -10143,9 +10143,9 @@ checksum = "7e51b68083f157f853b6379db119d1c1be0e6e4dec98101079dec41f6f5cf6df" [[package]] name = "unicode-bidi" -version = "0.3.17" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ab17db44d7388991a428b2ee655ce0c212e862eff1768a455c58f9aad6e7893" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" [[package]] name = "unicode-id" @@ -10666,7 +10666,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill" -version = "1.439.0" +version = "1.440.1" dependencies = [ "anyhow", "axum", @@ -10707,7 +10707,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.439.0" +version = "1.440.1" dependencies = [ "anyhow", "argon2", @@ -10735,7 +10735,7 @@ dependencies = [ "hf-hub", "hmac", "http 1.2.0", - "hyper 1.5.1", + "hyper 1.5.2", "itertools 0.13.0", "jsonwebtoken", "lazy_static", @@ -10792,7 +10792,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.439.0" +version = "1.440.1" dependencies = [ "base64 0.22.1", "chrono", @@ -10810,7 +10810,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.439.0" +version = "1.440.1" dependencies = [ "chrono", "serde", @@ -10823,7 +10823,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.439.0" +version = "1.440.1" dependencies = [ "anyhow", "serde", @@ -10837,7 +10837,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.439.0" +version = "1.440.1" dependencies = [ "anyhow", "async-stream", @@ -10857,7 +10857,7 @@ dependencies = [ "git-version", "hex", "hmac", - "hyper 1.5.1", + "hyper 1.5.2", "indexmap 2.7.0", "itertools 0.13.0", "lazy_static", @@ -10880,7 +10880,7 @@ dependencies = [ "serde_json", "sha2 0.10.8", "sqlx", - "thiserror 2.0.7", + "thiserror 2.0.8", "tikv-jemalloc-ctl", "tokio", "tonic", @@ -10895,7 +10895,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.439.0" +version = "1.440.1" dependencies = [ "regex", "serde", @@ -10909,7 +10909,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.439.0" +version = "1.440.1" dependencies = [ "anyhow", "bytes", @@ -10932,7 +10932,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.439.0" +version = "1.440.1" dependencies = [ "itertools 0.13.0", "lazy_static", @@ -10944,7 +10944,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.439.0" +version = "1.440.1" dependencies = [ "convert_case 0.6.0", "serde", @@ -10953,7 +10953,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.439.0" +version = "1.440.1" dependencies = [ "anyhow", "lazy_static", @@ -10965,7 +10965,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.439.0" +version = "1.440.1" dependencies = [ "anyhow", "serde_json", @@ -10977,7 +10977,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.439.0" +version = "1.440.1" dependencies = [ "anyhow", "gosyn", @@ -10989,7 +10989,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.439.0" +version = "1.440.1" dependencies = [ "anyhow", "lazy_static", @@ -11001,7 +11001,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.439.0" +version = "1.440.1" dependencies = [ "anyhow", "itertools 0.13.0", @@ -11012,7 +11012,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.439.0" +version = "1.440.1" dependencies = [ "anyhow", "itertools 0.13.0", @@ -11023,7 +11023,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.439.0" +version = "1.440.1" dependencies = [ "anyhow", "async-recursion", @@ -11041,7 +11041,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.439.0" +version = "1.440.1" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -11058,7 +11058,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.439.0" +version = "1.440.1" dependencies = [ "anyhow", "lazy_static", @@ -11070,7 +11070,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.439.0" +version = "1.440.1" dependencies = [ "anyhow", "lazy_static", @@ -11088,7 +11088,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.439.0" +version = "1.440.1" dependencies = [ "anyhow", "getrandom 0.2.15", @@ -11110,7 +11110,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.439.0" +version = "1.440.1" dependencies = [ "anyhow", "serde_json", @@ -11120,7 +11120,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.439.0" +version = "1.440.1" dependencies = [ "anyhow", "async-recursion", @@ -11154,7 +11154,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.439.0" +version = "1.440.1" dependencies = [ "wasm-bindgen", "wasm-bindgen-test", @@ -11164,7 +11164,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.439.0" +version = "1.440.1" dependencies = [ "anyhow", "async-recursion", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index dcd9dcdf72..2c2dd24c09 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.439.0" +version = "1.440.1" authors.workspace = true edition.workspace = true @@ -30,7 +30,7 @@ members = [ ] [workspace.package] -version = "1.439.0" +version = "1.440.1" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index 559f82ecc7..c9a162635b 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -3848,6 +3848,7 @@ mod job_payload { concurrency_time_window_s: None, cache_ttl: None, dedicated_worker: None, + path: "f/system/hello/test-0".into(), }) .arg("world", json!("foo")) .run_until_complete(&db, port) @@ -3867,6 +3868,7 @@ mod job_payload { concurrency_time_window_s: None, cache_ttl: None, dedicated_worker: None, + path: "f/system/hello/test-0".into(), }) .arg("hello", json!("You know nothing Jean Neige")) .run_until_complete(&db, port) diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index f962affd08..72c92db707 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.439.0 + version: 1.440.1 title: Windmill API contact: diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index f9228a6693..814fd58cb0 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -1324,7 +1324,7 @@ async fn execute_component( let cache = cache::anon!({ u64 => Arc } in "policy" <= 1000); arc_policy = policy_fut .map_ok(sqlx::types::Json) // cache as json. - .cached(cache, &(id as u64), |sqlx::types::Json(x)| Arc::new(x)) + .cached(cache, id as u64, |sqlx::types::Json(x)| Arc::new(x)) .await?; &*arc_policy } else { @@ -1353,7 +1353,7 @@ async fn execute_component( .fetch_one(&db) .map_err(Into::::into) .map_ok(sqlx::types::Json) // cache as json. - .cached(cache, &(*id as u64), |sqlx::types::Json(x)| Arc::new(x)) + .cached(cache, *id as u64, |sqlx::types::Json(x)| Arc::new(x)) .await? } _ => unreachable!(), diff --git a/backend/windmill-common/src/cache.rs b/backend/windmill-common/src/cache.rs index 9f3ec024ae..81043025c2 100644 --- a/backend/windmill-common/src/cache.rs +++ b/backend/windmill-common/src/cache.rs @@ -7,11 +7,10 @@ use crate::scripts::ScriptLang; use std::future::Future; use std::hash::Hash; +use std::panic::Location; use std::path::{Path, PathBuf}; use std::sync::Arc; -use futures::future::TryFutureExt; -use quick_cache::Equivalent; use serde::{Deserialize, Serialize}; use sqlx::types::{Json, JsonRawValue as RawValue}; use sqlx::PgExecutor; @@ -48,19 +47,23 @@ impl FsBackedCache { } /// Gets or inserts an item in the cache with key `key`. - pub async fn get_or_insert_async<'a, T: fs::Bundle, Q, F>( + pub async fn get_or_insert_async<'a, T: fs::Bundle, F>( &'a self, - key: &Q, + key: Key, map: impl Fn(T) -> Val, with: F, ) -> error::Result where - Q: Hash + Equivalent + ToOwned, + Key: Clone, F: Future>, { + if cfg!(test) { + // Disable caching in tests: since `#[sqlx::test]` spawn a database per test, the cache + // could yield unexpected results. + return with.await.map(map); + } self.cache - .get_or_insert_async(key, async { - let key = key.to_owned(); + .get_or_insert_async(&key, async { fs::import_or_insert_with(self.path(&key), with) .await .map(map) @@ -138,22 +141,22 @@ pub mod future { /// #[allow(unused)] /// async { /// let result = std::future::ready(Ok(Json(42))) - /// .cached(cache::anon!({ u64 => Json } in "test" <= 1), &42, |x| x) + /// .cached(cache::anon!({ u64 => Json } in "test" <= 1), 42, |x| x) /// .await; /// /// assert_eq!(result.unwrap(), Json(42)); /// }; /// ``` - fn cached( + fn cached( self, cache: &FsBackedCache, - key: &Q, + key: Key, map: impl Fn(T) -> Val, ) -> impl Future> where - Q: Hash + Equivalent + ToOwned, + Key: Clone, { - cache.get_or_insert_async(key, map, self) + cache.get_or_insert_async(key.to_owned(), map, self) } } @@ -221,6 +224,17 @@ pub struct ScriptMetadata { pub codebase: Option, } +fn unwrap_or_error( + at: &'static Location, + entity: &'static str, + key: Key, +) -> impl FnOnce(Option) -> error::Result { + move |optional| { + optional + .ok_or_else(|| error::Error::InternalErrAt(at, format!("{key:?}: {entity} not found"))) + } +} + const _: () = { impl fs::Bundle for FlowData { type Item = &'static str; @@ -302,110 +316,124 @@ pub mod flow { /// If not present, import from the file-system cache or fetch it from the database and write /// it to the file system and cache. /// This should be preferred over fetching the database directly. - pub async fn fetch_script( - e: impl PgExecutor<'_>, + #[track_caller] + pub fn fetch_script<'c>( + e: impl PgExecutor<'c>, node: FlowNodeId, - ) -> error::Result> { - fetch_node(e, node).await.and_then(|data| match data { - RawData::Script(data) => Ok(data), - RawData::Flow(_) => Err(error::Error::InternalErr(format!( - "Flow node ({:x}) isn't a script node.", - node.0 - ))), - }) + ) -> impl Future>> { + let fetch_node = fetch_node(e, node); + async move { + fetch_node.await.and_then(|data| match data { + RawData::Script(data) => Ok(data), + RawData::Flow(_) => Err(error::Error::InternalErr(format!( + "Flow node ({:x}) isn't a script node.", + node.0 + ))), + }) + } } /// Fetch the flow node flow value referenced by `node` from the cache. /// If not present, import from the file-system cache or fetch it from the database and write /// it to the file system and cache. /// This should be preferred over fetching the database directly. - pub async fn fetch_flow( - e: impl PgExecutor<'_>, + #[track_caller] + pub fn fetch_flow<'c>( + e: impl PgExecutor<'c>, node: FlowNodeId, - ) -> error::Result> { - fetch_node(e, node).await.and_then(|data| match data { - RawData::Flow(data) => Ok(data), - RawData::Script(_) => Err(error::Error::InternalErr(format!( - "Flow node ({:x}) isn't a flow node.", - node.0 - ))), - }) + ) -> impl Future>> { + let fetch_node = fetch_node(e, node); + async move { + fetch_node.await.and_then(|data| match data { + RawData::Flow(data) => Ok(data), + RawData::Script(_) => Err(error::Error::InternalErr(format!( + "Flow node ({:x}) isn't a flow node.", + node.0 + ))), + }) + } } /// Fetch the flow node referenced by `node` from the cache. /// If not present, import from the file-system cache or fetch it from the database and write /// it to the file system and cache. /// This should be preferred over fetching the database directly. - pub(super) async fn fetch_node( - e: impl PgExecutor<'_>, + #[track_caller] + pub(super) fn fetch_node<'c>( + e: impl PgExecutor<'c>, node: FlowNodeId, - ) -> error::Result { + ) -> impl Future> { + let loc = Location::caller(); // If not present, `get_or_insert_async` will lock the key until the future completes, // so only one thread will be able to fetch the data from the database and write it to // the file system and cache, hence no race on the file system. - NODES - .get_or_insert_async( - &node, - |(script, flow)| match flow { - Some(flow) => RawData::Flow(Arc::new(flow)), - _ => RawData::Script(Arc::new(script)), - }, - async { - sqlx::query!( - "SELECT \ - lock AS \"lock: String\", \ - code AS \"code: String\", \ - flow AS \"flow: Json>\" \ - FROM flow_node WHERE id = $1 LIMIT 1", - node.0, + NODES.get_or_insert_async( + node, + |(script, flow)| match flow { + Some(flow) => RawData::Flow(Arc::new(flow)), + _ => RawData::Script(Arc::new(script)), + }, + async move { + sqlx::query!( + "SELECT \ + lock AS \"lock: String\", \ + code AS \"code: String\", \ + flow AS \"flow: Json>\" \ + FROM flow_node WHERE id = $1 LIMIT 1", + node.0, + ) + .fetch_optional(e) + .await + .map_err(Into::into) + .and_then(unwrap_or_error(&loc, "Flow node", node)) + .map(|r| { + ( + ScriptData::from_raw(r.lock, r.code), + r.flow.map(|Json(raw_flow)| FlowData::from_raw(raw_flow)), ) - .fetch_one(e) - .await - .map_err(Into::into) - .map(|r| { - ( - ScriptData::from_raw(r.lock, r.code), - r.flow.map(|Json(raw_flow)| FlowData::from_raw(raw_flow)), - ) - }) - }, - ) - .await + }) + }, + ) } - pub async fn fetch_version(e: impl PgExecutor<'_>, id: i64) -> error::Result> { - FLOWS - .get_or_insert_async(&id, Arc::new, async { - sqlx::query_scalar!( - "SELECT value AS \"value!: Json>\" - FROM flow_version WHERE id = $1 LIMIT 1", - id, - ) - .fetch_one(e) - .await - .map_err(Into::into) - .map(|Json(raw_flow)| FlowData::from_raw(raw_flow)) - }) - .await - } - - pub async fn fetch_version_lite( - e: impl PgExecutor<'_>, + #[track_caller] + pub fn fetch_version<'c>( + e: impl PgExecutor<'c>, id: i64, - ) -> error::Result> { - FLOWS_LITE - .get_or_insert_async(&id, Arc::new, async { - sqlx::query_scalar!( - "SELECT value AS \"value!: Json>\" - FROM flow_version_lite WHERE id = $1 LIMIT 1", - id, - ) - .fetch_one(e) - .await - .map_err(Into::into) - .map(|Json(raw_flow)| FlowData::from_raw(raw_flow)) - }) + ) -> impl Future>> { + let loc = Location::caller(); + FLOWS.get_or_insert_async(id, Arc::new, async move { + sqlx::query_scalar!( + "SELECT value AS \"value!: Json>\" + FROM flow_version WHERE id = $1 LIMIT 1", + id, + ) + .fetch_optional(e) .await + .map_err(Into::into) + .and_then(unwrap_or_error(&loc, "Flow version", id)) + .map(|Json(raw_flow)| FlowData::from_raw(raw_flow)) + }) + } + + #[track_caller] + pub fn fetch_version_lite<'c>( + e: impl PgExecutor<'c>, + id: i64, + ) -> impl Future>> { + let loc = Location::caller(); + FLOWS_LITE.get_or_insert_async(id, Arc::new, async move { + sqlx::query_scalar!( + "SELECT value AS \"value!: Json>\" + FROM flow_version_lite WHERE id = $1 LIMIT 1", + id, + ) + .fetch_optional(e) + .await + .map_err(Into::into) + .and_then(unwrap_or_error(&loc, "Flow version \"lite\"", id)) + .map(|Json(raw_flow)| FlowData::from_raw(raw_flow)) + }) } } @@ -422,44 +450,41 @@ pub mod script { /// If not present, import from the file-system cache or fetch it from the database and write /// it to the file system and cache. /// This should be preferred over fetching the database directly. - pub async fn fetch( - e: impl PgExecutor<'_>, + #[track_caller] + pub fn fetch<'c>( + e: impl PgExecutor<'c>, hash: ScriptHash, - ) -> error::Result<(Arc, Arc)> { + ) -> impl Future, Arc)>> { // If not present, `get_or_insert_async` will lock the key until the future completes, // so only one thread will be able to fetch the data from the database and write it to // the file system and cache, hence no race on the file system. - CACHE - .get_or_insert_async( - &hash, - |(data, metadata)| (Arc::new(data), Arc::new(metadata)), - async { - sqlx::query!( - "SELECT \ - lock AS \"lock: String\", \ - content AS \"code!: String\", - language AS \"language: Option\", \ - envs AS \"envs: Vec\", \ - codebase AS \"codebase: String\" \ - FROM script WHERE hash = $1 LIMIT 1", - hash.0 + let loc = Location::caller(); + CACHE.get_or_insert_async( + hash, + |(data, metadata)| (Arc::new(data), Arc::new(metadata)), + async move { + sqlx::query!( + "SELECT \ + lock AS \"lock: String\", \ + content AS \"code!: String\", + language AS \"language: Option\", \ + envs AS \"envs: Vec\", \ + codebase AS \"codebase: String\" \ + FROM script WHERE hash = $1 LIMIT 1", + hash.0 + ) + .fetch_optional(e) + .await + .map_err(Into::into) + .and_then(unwrap_or_error(&loc, "Script", hash)) + .map(|r| { + ( + ScriptData::from_raw(r.lock, Some(r.code)), + ScriptMetadata { language: r.language, envs: r.envs, codebase: r.codebase }, ) - .fetch_one(e) - .await - .map_err(Into::into) - .map(|r| { - ( - ScriptData::from_raw(r.lock, Some(r.code)), - ScriptMetadata { - language: r.language, - envs: r.envs, - codebase: r.codebase, - }, - ) - }) - }, - ) - .await + }) + }, + ) } /// Invalidate the script cache for the given `hash`. @@ -480,25 +505,26 @@ pub mod app { /// If not present, import from the file-system cache or fetch it from the database and write /// it to the file system and cache. /// This should be preferred over fetching the database directly. - pub async fn fetch_script( - e: impl PgExecutor<'_>, + #[track_caller] + pub fn fetch_script<'c>( + e: impl PgExecutor<'c>, id: AppScriptId, - ) -> error::Result> { + ) -> impl Future>> { // If not present, `get_or_insert_async` will lock the key until the future completes, // so only one thread will be able to fetch the data from the database and write it to // the file system and cache, hence no race on the file system. - CACHE - .get_or_insert_async(&id, Arc::new, async { - sqlx::query!( - "SELECT lock, code FROM app_script WHERE id = $1 LIMIT 1", - id.0, - ) - .fetch_one(e) - .await - .map_err(Into::into) - .map(|r| ScriptData::from_raw(r.lock, Some(r.code))) - }) + let loc = Location::caller(); + CACHE.get_or_insert_async(id, Arc::new, async move { + sqlx::query!( + "SELECT lock, code FROM app_script WHERE id = $1 LIMIT 1", + id.0, + ) + .fetch_optional(e) .await + .map_err(Into::into) + .and_then(unwrap_or_error(&loc, "Application script", id)) + .map(|r| ScriptData::from_raw(r.lock, Some(r.code))) + }) } } @@ -513,108 +539,129 @@ pub mod job { static ref PREVIEWS: Cache = Cache::new(50); } - pub async fn fetch_preview_flow( - e: impl PgExecutor<'_>, - job: &Uuid, + #[track_caller] + pub fn fetch_preview_flow<'a, 'c>( + e: impl PgExecutor<'c> + 'a, + job: &'a Uuid, // original raw values from `queue` or `completed_job` tables: // kept for backward compatibility. raw_flow: Option>>, - ) -> error::Result> { - fetch_preview(e, job, None, None, raw_flow) - .await - .and_then(|data| match data { + ) -> impl Future>> + 'a { + let fetch_preview = fetch_preview(e, job, None, None, raw_flow); + async move { + fetch_preview.await.and_then(|data| match data { RawData::Flow(data) => Ok(data), RawData::Script(_) => Err(error::Error::InternalErr(format!( "Job ({job}) isn't a flow job." ))), }) + } } - pub async fn fetch_preview_script( - e: impl PgExecutor<'_>, - job: &Uuid, + #[track_caller] + pub fn fetch_preview_script<'a, 'c>( + e: impl PgExecutor<'c> + 'a, + job: &'a Uuid, // original raw values from `queue` or `completed_job` tables: // kept for backward compatibility. raw_lock: Option, raw_code: Option, - ) -> error::Result> { - fetch_preview(e, job, raw_lock, raw_code, None) - .await - .and_then(|data| match data { + ) -> impl Future>> + 'a { + let fetch_preview = fetch_preview(e, job, raw_lock, raw_code, None); + async move { + fetch_preview.await.and_then(|data| match data { RawData::Script(data) => Ok(data), RawData::Flow(_) => Err(error::Error::InternalErr(format!( "Job ({job}) isn't a script job." ))), }) + } } - pub async fn fetch_preview( - e: impl PgExecutor<'_>, - job: &Uuid, + #[track_caller] + pub fn fetch_preview<'a, 'c>( + e: impl PgExecutor<'c> + 'a, + job: &'a Uuid, // original raw values from `queue` or `completed_job` tables: // kept for backward compatibility. raw_lock: Option, raw_code: Option, raw_flow: Option>>, - ) -> error::Result { - PREVIEWS - .get_or_insert_async(job, async { - match (raw_lock, raw_code, raw_flow) { - (None, None, None) => sqlx::query!( - "SELECT raw_code, raw_lock, raw_flow AS \"raw_flow: Json>\" \ - FROM job WHERE id = $1 LIMIT 1", - job - ) - .fetch_one(e) - .map_err(Into::into) - .await - .map(|r| (r.raw_lock, r.raw_code, r.raw_flow)), - (lock, code, flow) => Ok((lock, code, flow)), - } - .map(|(lock, code, flow)| match flow { - Some(Json(flow)) => RawData::Flow(Arc::new(FlowData::from_raw(flow))), - _ => RawData::Script(Arc::new(ScriptData::from_raw(lock, code))), - }) + ) -> impl Future> + 'a { + let loc = Location::caller(); + let fetch = async move { + match (raw_lock, raw_code, raw_flow) { + (None, None, None) => sqlx::query!( + "SELECT raw_code, raw_lock, raw_flow AS \"raw_flow: Json>\" \ + FROM job WHERE id = $1 LIMIT 1", + job + ) + .fetch_optional(e) + .await + .map_err(Into::into) + .and_then(unwrap_or_error(&loc, "Preview", job)) + .map(|r| (r.raw_lock, r.raw_code, r.raw_flow)), + (lock, code, flow) => Ok((lock, code, flow)), + } + .map(|(lock, code, flow)| match flow { + Some(Json(flow)) => RawData::Flow(Arc::new(FlowData::from_raw(flow))), + _ => RawData::Script(Arc::new(ScriptData::from_raw(lock, code))), }) - .await + }; + // Disable caching in tests: as `#[sqlx::test]` spawn a database per test, the cache + // could yield unexpected results. + #[cfg(test)] + return fetch; + #[cfg(not(test))] + PREVIEWS.get_or_insert_async(job, fetch) } - pub async fn fetch_script( - e: impl PgExecutor<'_>, + #[track_caller] + pub fn fetch_script<'c>( + e: impl PgExecutor<'c>, kind: JobKind, hash: Option, - ) -> error::Result> { + ) -> impl Future>> { use JobKind::*; - match (kind, hash.map(|ScriptHash(id)| id)) { - (FlowScript, Some(id)) => flow::fetch_script(e, FlowNodeId(id)).await, - (Script | Dependencies, Some(hash)) => script::fetch(e, ScriptHash(hash)) - .await - .map(|(raw_script, _metadata)| raw_script), - (AppScript, Some(id)) => app::fetch_script(e, AppScriptId(id)).await, - _ => Err(error::Error::InternalErr(format!( - "Isn't a script job: {:?}", - kind - ))), + let loc = Location::caller(); + async move { + match (kind, hash.map(|ScriptHash(id)| id)) { + (FlowScript, Some(id)) => flow::fetch_script(e, FlowNodeId(id)).await, + (Script | Dependencies, Some(hash)) => script::fetch(e, ScriptHash(hash)) + .await + .map(|(raw_script, _metadata)| raw_script), + (AppScript, Some(id)) => app::fetch_script(e, AppScriptId(id)).await, + _ => Err(error::Error::InternalErr(format!( + "Isn't a script job: {:?}", + kind + ))), + } + .map_err(error::relocate_internal(loc)) } } - pub async fn fetch_flow( - e: impl PgExecutor<'_> + Copy, + #[track_caller] + pub fn fetch_flow<'c>( + e: impl PgExecutor<'c> + Copy, kind: JobKind, hash: Option, - ) -> error::Result> { + ) -> impl Future>> { use JobKind::*; - match (kind, hash.map(|ScriptHash(id)| id)) { - (FlowDependencies, Some(id)) => flow::fetch_version(e, id).await, - (FlowNode, Some(id)) => flow::fetch_flow(e, FlowNodeId(id)).await, - (Flow, Some(id)) => match flow::fetch_version_lite(e, id).await { - Ok(raw_flow) => Ok(raw_flow), - Err(_) => flow::fetch_version(e, id).await, - }, - _ => Err(error::Error::InternalErr(format!( - "Isn't a flow job {:?}", - kind - ))), + let loc = Location::caller(); + async move { + match (kind, hash.map(|ScriptHash(id)| id)) { + (FlowDependencies, Some(id)) => flow::fetch_version(e, id).await, + (FlowNode, Some(id)) => flow::fetch_flow(e, FlowNodeId(id)).await, + (Flow, Some(id)) => match flow::fetch_version_lite(e, id).await { + Ok(raw_flow) => Ok(raw_flow), + Err(_) => flow::fetch_version(e, id).await, + }, + _ => Err(error::Error::InternalErr(format!( + "Isn't a flow job {:?}", + kind + ))), + } + .map_err(error::relocate_internal(loc)) } } } diff --git a/backend/windmill-common/src/error.rs b/backend/windmill-common/src/error.rs index 7b959ebed1..0080568f5e 100644 --- a/backend/windmill-common/src/error.rs +++ b/backend/windmill-common/src/error.rs @@ -6,6 +6,8 @@ * LICENSE-AGPL for a copy of the license. */ +use std::panic::Location; + use axum::body::Body; use axum::response::Response; use axum::{response::IntoResponse, response::Json}; @@ -48,6 +50,8 @@ pub enum Error { QuotaExceeded(String), #[error("Internal: {0}")] InternalErr(String), + #[error("Internal: {0}: {1}")] + InternalErrAt(&'static Location<'static>, String), #[error("Hexadecimal decoding error: {0}")] HexErr(#[from] hex::FromHexError), #[error("Migrating database: {0}")] @@ -77,6 +81,17 @@ impl Error { pub fn dbg(&self) -> String { format!("{:?}", self) } + + pub fn relocate_internal(self, loc: &'static Location<'static>) -> Self { + match self { + Self::InternalErr(s) | Self::InternalErrAt(_, s) => Self::InternalErrAt(loc, s), + _ => self, + } + } +} + +pub fn relocate_internal(loc: &'static Location<'static>) -> impl FnOnce(Error) -> Error { + move |e| e.relocate_internal(loc) } pub fn to_anyhow(e: T) -> anyhow::Error { @@ -92,10 +107,9 @@ impl IntoResponse for Error { Self::NotFound(_) => axum::http::StatusCode::NOT_FOUND, Self::NotAuthorized(_) => axum::http::StatusCode::UNAUTHORIZED, Self::RequireAdmin(_) => axum::http::StatusCode::FORBIDDEN, - Self::SqlErr(_) - | Self::BadRequest(_) - | Self::AiError(_) - | Self::QuotaExceeded(_) => axum::http::StatusCode::BAD_REQUEST, + Self::SqlErr(_) | Self::BadRequest(_) | Self::AiError(_) | Self::QuotaExceeded(_) => { + axum::http::StatusCode::BAD_REQUEST + } _ => axum::http::StatusCode::INTERNAL_SERVER_ERROR, }; diff --git a/backend/windmill-common/src/jobs.rs b/backend/windmill-common/src/jobs.rs index 086976e701..dc371d319f 100644 --- a/backend/windmill-common/src/jobs.rs +++ b/backend/windmill-common/src/jobs.rs @@ -281,6 +281,7 @@ pub enum JobPayload { concurrency_time_window_s: Option, cache_ttl: Option, dedicated_worker: Option, + path: String, }, FlowNode { id: FlowNodeId, // flow_node(id). diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 9df0b70d89..ffedbda163 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -2852,9 +2852,10 @@ pub async fn push<'c, 'd>( concurrency_time_window_s, cache_ttl, dedicated_worker, + path, } => ( Some(id.0), - None, + Some(path), None, JobKind::FlowScript, None, diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index e8cd67108b..9f71a7d807 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -2978,6 +2978,18 @@ fn payload_from_modules<'a>( }) } +fn get_path(flow_job: &QueuedJob, status: &FlowStatus, module: &FlowModule) -> String { + if status + .preprocessor_module + .as_ref() + .is_some_and(|x| x.id() == module.id) + { + format!("{}/preprocessor", flow_job.script_path()) + } else { + format!("{}/step-{}", flow_job.script_path(), status.step) + } +} + async fn compute_next_flow_transform( arc_flow_job_args: Marc>>, arc_last_job_result: Arc>, @@ -3023,6 +3035,7 @@ async fn compute_next_flow_transform( if is_skipped { return trivial_next_job(JobPayload::Identity); } + match module.get_value()? { FlowModuleValue::Identity => trivial_next_job(JobPayload::Identity), FlowModuleValue::Flow { path, .. } => { @@ -3052,17 +3065,8 @@ async fn compute_next_flow_transform( concurrency_time_window_s, .. } => { - let path = path.clone().or_else(|| { - if status - .preprocessor_module - .as_ref() - .is_some_and(|x| x.id() == module.id) - { - Some(format!("{}/preprocessor", flow_job.script_path())) - } else { - Some(format!("{}/step-{}", flow_job.script_path(), status.step)) - } - }); + let path = path.unwrap_or_else(|| get_path(flow_job, status, module)); + let payload = raw_script_to_payload( path, content, @@ -3089,6 +3093,8 @@ async fn compute_next_flow_transform( concurrency_time_window_s, .. } => { + let path = get_path(flow_job, status, module); + let payload = JobPayloadWithTag { payload: JobPayload::FlowScript { id, @@ -3098,6 +3104,7 @@ async fn compute_next_flow_transform( concurrency_time_window_s, cache_ttl: module.cache_ttl.map(|x| x as i32), dedicated_worker: None, + path, }, tag: tag.clone(), delete_after_use, @@ -3147,7 +3154,9 @@ async fn compute_next_flow_transform( /* forloop modules are expected set `iter: { value: Value, index: usize }` as job arguments */ FlowModuleValue::ForloopFlow { modules, modules_node, iterator, parallel, .. } => { // if it's a simple single step flow, we will collapse it as an optimization and need to pass flow_input as an arg - let is_simple = !parallel && is_simple_modules(&modules, flow.failure_module.as_ref()); + let is_simple = !matches!(flow_job.job_kind, JobKind::FlowPreview) + && !parallel + && is_simple_modules(&modules, flow.failure_module.as_ref()); // if is_simple { // match value { @@ -3459,7 +3468,7 @@ async fn next_loop_iteration( }; return Ok(NextFlowTransform::Continue( ContinuePayload::SingleJob( - payload_from_simple_module(value, db, flow_job, module, Some(inner_path())).await?, + payload_from_simple_module(value, db, flow_job, module, inner_path()).await?, ), NextStatus::NextLoopIteration { next: ns, simple_input_transforms }, )); @@ -3650,7 +3659,7 @@ async fn payload_from_simple_module( db: &sqlx::Pool, flow_job: &QueuedJob, module: &FlowModule, - inner_path: Option, + inner_path: String, ) -> Result { let delete_after_use = module.delete_after_use.unwrap_or(false); Ok(match value { @@ -3669,7 +3678,7 @@ async fn payload_from_simple_module( concurrency_time_window_s, .. } => raw_script_to_payload( - path.or(inner_path), + path.unwrap_or_else(|| inner_path), content, language, lock, @@ -3697,6 +3706,7 @@ async fn payload_from_simple_module( concurrency_time_window_s, cache_ttl: module.cache_ttl.map(|x| x as i32), dedicated_worker: None, + path: inner_path, }, tag, delete_after_use, @@ -3707,7 +3717,7 @@ async fn payload_from_simple_module( } fn raw_script_to_payload( - path: Option, + path: String, content: String, language: windmill_common::scripts::ScriptLang, lock: Option, @@ -3721,7 +3731,7 @@ fn raw_script_to_payload( JobPayloadWithTag { payload: JobPayload::Code(RawCode { hash: None, - path, + path: Some(path), content, language, lock, diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 35d4312382..887644c2ff 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.439.0"; +export const VERSION = "v1.440.1"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/main.ts b/cli/main.ts index c2bfd62d4b..cff86d6365 100644 --- a/cli/main.ts +++ b/cli/main.ts @@ -60,7 +60,7 @@ export { // } // }); -export const VERSION = "1.439.0"; +export const VERSION = "1.440.1"; const command = new Command() .name("wmill") diff --git a/frontend/package-lock.json b/frontend/package-lock.json index d65a848fa8..02632ec441 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.439.0", + "version": "1.440.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.439.0", + "version": "1.440.1", "license": "AGPL-3.0", "dependencies": { "@anthropic-ai/sdk": "^0.32.1", diff --git a/frontend/package.json b/frontend/package.json index 3f4295e107..e70039c21b 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.439.0", + "version": "1.440.1", "scripts": { "dev": "vite dev", "build": "vite build", diff --git a/frontend/src/lib/components/Dev.svelte b/frontend/src/lib/components/Dev.svelte index 90659e3103..02967efaeb 100644 --- a/frontend/src/lib/components/Dev.svelte +++ b/frontend/src/lib/components/Dev.svelte @@ -504,7 +504,8 @@ initialPath: '', flowInputsStore: writable({}), customUi: {}, - insertButtonOpen: writable(false) + insertButtonOpen: writable(false), + executionCount: writable(0), }) setContext('PropPickerContext', { flowPropPickerConfig: writable(undefined), diff --git a/frontend/src/lib/components/FlowBuilder.svelte b/frontend/src/lib/components/FlowBuilder.svelte index 4ada01aea7..156954d8e1 100644 --- a/frontend/src/lib/components/FlowBuilder.svelte +++ b/frontend/src/lib/components/FlowBuilder.svelte @@ -107,7 +107,7 @@ export let disabledFlowInputs = false export let savedPrimarySchedule: ScheduleTrigger | undefined = undefined export let version: number | undefined = undefined - export let setSavedraftCb: ((cb: () => void) => void) | undefined = undefined + export let setSavedraftCb: ((cb: () => void) => void) | undefined = undefined // Used by multiplayer deploy collision warning let deployedValue: Value | undefined = undefined // Value to diff against @@ -528,7 +528,8 @@ initialPath, flowInputsStore: writable({}), customUi, - insertButtonOpen + insertButtonOpen, + executionCount: writable(0) }) setContext('TriggerContext', { diff --git a/frontend/src/lib/components/FlowPreviewContent.svelte b/frontend/src/lib/components/FlowPreviewContent.svelte index a336fd42d3..88b22c8996 100644 --- a/frontend/src/lib/components/FlowPreviewContent.svelte +++ b/frontend/src/lib/components/FlowPreviewContent.svelte @@ -36,8 +36,16 @@ runPreview($previewArgs, undefined) } - const { selectedId, previewArgs, flowStateStore, flowStore, pathStore, initialPath, customUi } = - getContext('FlowEditorContext') + const { + selectedId, + previewArgs, + flowStateStore, + flowStore, + pathStore, + initialPath, + customUi, + executionCount + } = getContext('FlowEditorContext') const dispatch = createEventDispatcher() function extractFlow(previewMode: 'upTo' | 'whole'): OpenFlow { @@ -330,6 +338,10 @@ wideResults {flowStateStore} {jobId} + on:done={() => { + console.log('done') + $executionCount = $executionCount + 1 + }} on:jobsLoaded={({ detail }) => { job = detail }} diff --git a/frontend/src/lib/components/FlowStatusViewerInner.svelte b/frontend/src/lib/components/FlowStatusViewerInner.svelte index 35803d53e3..5ad9145c5d 100644 --- a/frontend/src/lib/components/FlowStatusViewerInner.svelte +++ b/frontend/src/lib/components/FlowStatusViewerInner.svelte @@ -627,7 +627,6 @@ j != state?.selectedForloopIndex || setManually != state?.selectedForLoopSetManually if (selectedNotEqual) { - console.log('not equal') globalState?.update((topLevelModuleStates) => { topLevelModuleStates[modId] = { type: 'WaitingForPriorSteps', diff --git a/frontend/src/lib/components/apps/components/inputs/AppDateInput.svelte b/frontend/src/lib/components/apps/components/inputs/AppDateInput.svelte index dd78968a70..a17cb29ce2 100644 --- a/frontend/src/lib/components/apps/components/inputs/AppDateInput.svelte +++ b/frontend/src/lib/components/apps/components/inputs/AppDateInput.svelte @@ -30,7 +30,12 @@ $componentControl[id] = { setValue(nvalue: string) { - value = nvalue + if (typeof nvalue === 'string') { + value = nvalue?.split('T')?.[0] + } else { + console.error('Invalid value', nvalue) + value = undefined + } } } diff --git a/frontend/src/lib/components/flows/content/FlowModuleComponent.svelte b/frontend/src/lib/components/flows/content/FlowModuleComponent.svelte index 7737c9e173..95c549e640 100644 --- a/frontend/src/lib/components/flows/content/FlowModuleComponent.svelte +++ b/frontend/src/lib/components/flows/content/FlowModuleComponent.svelte @@ -57,7 +57,8 @@ pathStore, saveDraft, flowInputsStore, - customUi + customUi, + executionCount } = getContext('FlowEditorContext') export let flowModule: FlowModule @@ -107,17 +108,18 @@ $: editor !== undefined && setCopilotModuleEditor() - $: stepPropPicker = failureModule - ? getFailureStepPropPicker($flowStateStore, $flowStore, $previewArgs) - : getStepPropPicker( - $flowStateStore, - parentModule, - previousModule, - flowModule.id, - $flowStore, - $previewArgs, - false - ) + $: stepPropPicker = + $executionCount != undefined && failureModule + ? getFailureStepPropPicker($flowStateStore, $flowStore, $previewArgs) + : getStepPropPicker( + $flowStateStore, + parentModule, + previousModule, + flowModule.id, + $flowStore, + $previewArgs, + false + ) function onKeyDown(event: KeyboardEvent) { if ((event.ctrlKey || event.metaKey) && event.key == 'Enter') { diff --git a/frontend/src/lib/components/flows/previousResults.ts b/frontend/src/lib/components/flows/previousResults.ts index 3e2a297390..6ca5481dd4 100644 --- a/frontend/src/lib/components/flows/previousResults.ts +++ b/frontend/src/lib/components/flows/previousResults.ts @@ -74,12 +74,18 @@ function getFlowInput( if (parentState && parentModule) { if ( parentState.previewArgs && - !(parentModule.value?.type === 'forloopflow' && parentModule.value?.modules?.length === 1) && parentState && typeof parentState.previewArgs == 'object' && `iter` in parentState.previewArgs ) { - return { ...topFlowInput, ...parentState.previewArgs } + return { + iter: { + value: "Iteration's value", + index: "Iteration's index" + }, + ...topFlowInput, + ...parentState.previewArgs + } } else { let parentFlowInput = getFlowInput(parentModules, flowState, args, schema) if (parentModule.value.type === 'forloopflow') { diff --git a/frontend/src/lib/components/flows/types.ts b/frontend/src/lib/components/flows/types.ts index 2cf7127018..633834da0b 100644 --- a/frontend/src/lib/components/flows/types.ts +++ b/frontend/src/lib/components/flows/types.ts @@ -40,4 +40,5 @@ export type FlowEditorContext = { flowInputsStore: Writable customUi: FlowBuilderWhitelabelCustomUi insertButtonOpen: Writable + executionCount: Writable } diff --git a/frontend/src/lib/components/propertyPicker/PropPicker.svelte b/frontend/src/lib/components/propertyPicker/PropPicker.svelte index 66b85816a3..7e62990eb6 100644 --- a/frontend/src/lib/components/propertyPicker/PropPicker.svelte +++ b/frontend/src/lib/components/propertyPicker/PropPicker.svelte @@ -94,8 +94,10 @@ let filteringFlowInputsOrResult = '' async function filterPickableProperties() { if (!$propPickerConfig || !filterActive) { - flowInputsFiltered = pickableProperties.flow_input - resultByIdFiltered = pickableProperties.priorIds + if (search === EMPTY_STRING) { + flowInputsFiltered = pickableProperties.flow_input + resultByIdFiltered = pickableProperties.priorIds + } filteringFlowInputsOrResult = '' return } @@ -189,7 +191,7 @@ await updateCollapsable() } - $: search, $inputMatches, $propPickerConfig, updateState() + $: search, $inputMatches, $propPickerConfig, pickableProperties, updateState()
diff --git a/frontend/src/lib/components/sidebar/WorkspaceMenu.svelte b/frontend/src/lib/components/sidebar/WorkspaceMenu.svelte index 515d3514a6..bfa45d8d86 100644 --- a/frontend/src/lib/components/sidebar/WorkspaceMenu.svelte +++ b/frontend/src/lib/components/sidebar/WorkspaceMenu.svelte @@ -23,6 +23,9 @@ export let isCollapsed: boolean = false + // When used outside of the side bar, where links to workspace settings and such don't make as much sense. + export let strictWorkspaceSelect = false + async function toggleSwitchWorkspace(id: string) { if ($workspaceStore === id) { return @@ -76,7 +79,7 @@ {/each}
- {#if isCloudHosted() || $superadmin} + {#if (isCloudHosted() || $superadmin) && !strictWorkspaceSelect}
{/if} + {#if !strictWorkspaceSelect}
- {#if $userStore?.is_admin || $superadmin} + {/if} + {#if ($userStore?.is_admin || $superadmin) && !strictWorkspaceSelect}
{/if}
- {#if isCloudHosted() && !$isPremiumStore} + {#if isCloudHosted() && !$isPremiumStore && !strictWorkspaceSelect}
{#if $workspaceStore != 'demo'} {/if} - {#if $enterpriseLicense} + {#if $enterpriseLicense && !strictWorkspaceSelect} {/if} diff --git a/frontend/src/routes/(root)/(logged)/svix/create-webhook/+page@(root).svelte b/frontend/src/routes/(root)/(logged)/svix/create-webhook/+page@(root).svelte index 500e1047b5..e68a192eaf 100644 --- a/frontend/src/routes/(root)/(logged)/svix/create-webhook/+page@(root).svelte +++ b/frontend/src/routes/(root)/(logged)/svix/create-webhook/+page@(root).svelte @@ -1,7 +1,5 @@