Compare commits

..

3 Commits

Author SHA1 Message Date
Ruben Fiszel
2f46459844 all 2024-04-14 19:37:12 +02:00
Ruben Fiszel
f82dfa7e97 all 2024-04-14 19:37:03 +02:00
Ruben Fiszel
a605d79f21 all 2024-04-14 19:24:54 +02:00
325 changed files with 5700 additions and 11605 deletions

View File

@@ -16,7 +16,6 @@ sed -i '' -e "/\"version\": /s/: .*,/: \"$VERSION\",/" ${root_dirpath}/frontend/
sed -i '' -e "/^version =/s/= .*/= \"$VERSION\"/" ${root_dirpath}/python-client/wmill/pyproject.toml
sed -i '' -e "/^windmill-api =/s/= .*/= \"\\^$VERSION\"/" ${root_dirpath}/python-client/wmill/pyproject.toml
sed -i '' -e "/^version =/s/= .*/= \"$VERSION\"/" ${root_dirpath}/python-client/wmill_pg/pyproject.toml
sed -i '' -e "/^ModuleVersion =/s/= .*/= '$VERSION'/" ${root_dirpath}/powershell-client/WindmillClient/WindmillClient.psd1
# sed -i '' -e "/^wmill =/s/= .*/= \"\\^$VERSION\"/" python-client/wmill_pg/pyproject.toml
sed -i '' -e "/^wmill =/s/= .*/= \">=$VERSION\"/" ${root_dirpath}/lsp/Pipfile
sed -i '' -e "/^wmill_pg =/s/= .*/= \">=$VERSION\"/" ${root_dirpath}/lsp/Pipfile

View File

@@ -12,12 +12,10 @@ sed -i -e "/^export const VERSION =/s/= .*/= \"v$VERSION\";/" ${root_dirpath}/be
sed -i -e "/version: /s/: .*/: $VERSION/" ${root_dirpath}/backend/windmill-api/openapi.yaml
sed -i -e "/version: /s/: .*/: $VERSION/" ${root_dirpath}/openflow.openapi.yaml
sed -i -e "/\"version\": /s/: .*,/: \"$VERSION\",/" ${root_dirpath}/typescript-client/package.json
sed -i -e "/\"version\": /s/: .*,/: \"$VERSION\",/" ${root_dirpath}/typescript-client/jsr.json
sed -i -e "/\"version\": /s/: .*,/: \"$VERSION\",/" ${root_dirpath}/frontend/package.json
sed -i -e "/^version =/s/= .*/= \"$VERSION\"/" ${root_dirpath}/python-client/wmill/pyproject.toml
sed -i -e "/^windmill-api =/s/= .*/= \"\\^$VERSION\"/" ${root_dirpath}/python-client/wmill/pyproject.toml
sed -i -e "/^version =/s/= .*/= \"$VERSION\"/" ${root_dirpath}/python-client/wmill_pg/pyproject.toml
sed -i -e "/^ModuleVersion =/s/= .*/= '$VERSION'/" ${root_dirpath}/powershell-client/WindmillClient/WindmillClient.psd1
# sed -i -e "/^wmill =/s/= .*/= \"\\^$VERSION\"/" ${root_dirpath}/python-client/wmill_pg/pyproject.toml
sed -i -e "/^wmill =/s/= .*/= \">=$VERSION\"/" ${root_dirpath}/lsp/Pipfile
sed -i -e "/^wmill_pg =/s/= .*/= \">=$VERSION\"/" ${root_dirpath}/lsp/Pipfile

View File

@@ -1,16 +0,0 @@
name: Publish powershell-client
on:
push:
tags:
- "v*"
workflow_dispatch:
jobs:
publish_gallery:
runs-on: ubicloud-standard-8
steps:
- uses: actions/checkout@v4
- run: . ./powershell-client/publish.ps1
shell: pwsh
env:
NUGET_API_KEY: ${{ secrets.NUGET_API_KEY }}

View File

@@ -1,16 +0,0 @@
name: Publish typescript-client on JSR
on:
push:
tags:
- "v*"
jobs:
publish:
runs-on: ubuntu-latest
permissions:
contents: read
id-token: ${{ secrets.JSR_OIDC_ID_TOKEN }}
steps:
- uses: actions/checkout@v4
- run: cd typescript-client && ./publish.jsr.sh

File diff suppressed because it is too large Load Diff

View File

@@ -12,14 +12,12 @@ RUN apt-get -y update \
RUN rustup component add rustfmt
RUN CARGO_NET_GIT_FETCH_WITH_CLI=true cargo install cargo-chef --version ^0.1
RUN cargo install sccache --version ^0.8
ENV RUSTC_WRAPPER=sccache SCCACHE_DIR=/backend/sccache
RUN CARGO_NET_GIT_FETCH_WITH_CLI=true cargo install cargo-chef
WORKDIR /windmill
ENV SQLX_OFFLINE=true
# ENV CARGO_INCREMENTAL=1
ENV CARGO_INCREMENTAL=1
FROM node:20-alpine as frontend
@@ -48,9 +46,7 @@ FROM rust_base AS planner
COPY ./openflow.openapi.yaml /openflow.openapi.yaml
COPY ./backend ./
RUN --mount=type=cache,target=/usr/local/cargo/registry \
--mount=type=cache,target=$SCCACHE_DIR,sharing=locked \
CARGO_NET_GIT_FETCH_WITH_CLI=true cargo chef prepare --recipe-path recipe.json
RUN CARGO_NET_GIT_FETCH_WITH_CLI=true cargo chef prepare --recipe-path recipe.json
FROM rust_base AS builder
ARG features=""
@@ -59,9 +55,7 @@ COPY --from=planner /windmill/recipe.json recipe.json
RUN apt-get update && apt-get install -y libxml2-dev libxmlsec1-dev clang libclang-dev cmake
RUN --mount=type=cache,target=/usr/local/cargo/registry \
--mount=type=cache,target=$SCCACHE_DIR,sharing=locked \
CARGO_NET_GIT_FETCH_WITH_CLI=true RUST_BACKTRACE=1 cargo chef cook --release --features "$features" --recipe-path recipe.json
RUN CARGO_NET_GIT_FETCH_WITH_CLI=true RUST_BACKTRACE=1 cargo chef cook --release --features "$features" --recipe-path recipe.json
COPY ./openflow.openapi.yaml /openflow.openapi.yaml
COPY ./backend ./
@@ -70,9 +64,7 @@ COPY --from=frontend /frontend /frontend
COPY --from=frontend /backend/windmill-api/openapi-deref.yaml ./windmill-api/openapi-deref.yaml
COPY .git/ .git/
RUN --mount=type=cache,target=/usr/local/cargo/registry \
--mount=type=cache,target=$SCCACHE_DIR,sharing=locked \
CARGO_NET_GIT_FETCH_WITH_CLI=true cargo build --release --features "$features"
RUN CARGO_NET_GIT_FETCH_WITH_CLI=true cargo build --release --features "$features"
FROM ${DEBIAN_IMAGE} as downloader
@@ -171,7 +163,7 @@ COPY --from=builder /windmill/target/release/windmill ${APP}/windmill
COPY --from=downloader --chmod=755 /deno /usr/bin/deno
COPY --from=oven/bun:1.1.5 /usr/local/bin/bun /usr/bin/bun
COPY --from=oven/bun:1.1.0 /usr/local/bin/bun /usr/bin/bun
# add the docker client to call docker from a worker if enabled
COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/

View File

@@ -13,7 +13,7 @@ any snippets of code that require a positive license check to be activated.
Those snippets and files are under a proprietary and commercial license. Private
and public forks MUST not include any of the above proprietary and commercial
code. Windmill Labs, Inc. provide tools to clean the codebase from those
snippets upon demand. The files under python-client/ deno-client/ go-client/ powershell-client/ are
snippets upon demand. The files under python-client/ deno-client/ go-client/ are
Apache 2.0 Licensed.
The openapi files, including the OpenFlow spec is Apache 2.0 Licensed.

View File

@@ -353,43 +353,6 @@ you to have it being synced automatically everyday.
| QUEUE_LIMIT_WAIT_RESULT | None | The number of max jobs in the queue before rejecting immediately the request in 'run_wait_result' endpoint. Takes precedence on the query arg. If none is specified, there are no limit. | Worker |
| DENO_AUTH_TOKENS | None | Custom DENO_AUTH_TOKENS to pass to worker to allow the use of private modules | Worker |
## Run a local dev setup
### only Frontend
This will use the backend of <https://app.windmill.dev> but your own frontend
with hot-code reloading.
1. Install [caddy](https://caddyserver.com)
2. Go to `frontend/`:
1. `npm install`, `npm run generate-backend-client` then `npm run dev`
2. In another shell `sudo caddy run --config CaddyfileRemote`
3. Et voilà, windmill should be available at `http://localhost/`
### Backend + Frontend
See the [./frontend/README_DEV.md](./frontend/README_DEV.md) file for all
running options.
1. Create a Postgres Database for Windmill and create an admin role inside your
Postgres setup.
The easiest way to get a working db is to run
```
cargo install sqlx-cli
env DATABASE_URL=<YOUR_DATABASE_URL> sqlx migrate run
```
This will also avoid compile time issue with sqlx's `query!` macro
2. Install [nsjail](https://github.com/google/nsjail) and have it accessible in
your PATH
3. Install deno and python3, have the bins at `/usr/bin/deno` and
`/usr/local/bin/python3`
4. Install [caddy](https://caddyserver.com)
5. Install the [lld linker](https://lld.llvm.org/)
6. Go to `frontend/`:
1. `npm install`, `npm run generate-backend-client` then `npm run dev`
2. You might need to set some extra heap space for the node runtime `export NODE_OPTIONS="--max-old-space-size=4096"`
3. In another shell `npm run build` otherwise the backend will not find the `frontend/build` folder and will not compile.
4. In another shell `sudo caddy run --config Caddyfile`
7. Go to `backend/`:
`env DATABASE_URL=<DATABASE_URL_TO_YOUR_WINDMILL_DB> RUST_LOG=info cargo run`
8. Et voilà, windmill should be available at `http://localhost/`
## Contributors
<a href="https://github.com/windmill-labs/windmill/graphs/contributors">

View File

@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29)",
"query": "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28)",
"describe": {
"columns": [],
"parameters": {
@@ -66,11 +66,10 @@
"Bool",
"Int4",
"Varchar",
"Bool",
"Bool"
]
},
"nullable": []
},
"hash": "92d00c6a1f4c40f2a23c9ba758a59597deabd7ca93653ad72d2cdc37efefc9d4"
"hash": "020d33ed5d47350b456783fd548422ea8dcf2d786d0e9fa849754db82c9fa378"
}

View File

@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT id, running, is_flow_step FROM queue WHERE scheduled_for < now() AND workspace_id = $1 AND schedule_path IS NULL",
"query": "UPDATE queue SET canceled = true, canceled_by = $2, scheduled_for = now(), suspend = 0 WHERE scheduled_for < now() AND workspace_id = $1 AND schedule_path IS NULL RETURNING id, running, is_flow_step",
"describe": {
"columns": [
{
@@ -21,7 +21,8 @@
],
"parameters": {
"Left": [
"Text"
"Text",
"Varchar"
]
},
"nullable": [
@@ -30,5 +31,5 @@
true
]
},
"hash": "caeb49629b8673c1f1c84a6e40c3e2d2c3bc3fdbde530a0a6b6fd68a22b867c3"
"hash": "18699cb0eca25b6bde05d81571dfdea8cafd0043634f61b0f652a93767c9c30a"
}

View File

@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM workspace_invite WHERE workspace_id = $1 AND email = $2",
"query": "DELETE FROM folder WHERE name = $1 AND workspace_id = $2",
"describe": {
"columns": [],
"parameters": {
@@ -11,5 +11,5 @@
},
"nullable": []
},
"hash": "0d7ba88a9810e434aa00fd63bbf416cbe222f2c67ccc8aa92e651c2bea4c2d7b"
"hash": "26e4ec75366d1e46a98710f29066b40e66a802f98eeabbb3ae5bebe3aeb6b3f8"
}

View File

@@ -1,23 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT 1 FROM schedule WHERE path = $1 AND workspace_id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "?column?",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
null
]
},
"hash": "4613382f7b031a2b667f86d9af995065a99a63c407fd58aa293fb269e032121d"
}

View File

@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM schedule WHERE path = $1 AND workspace_id = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": []
},
"hash": "61e9662fe42506131222412ab3de48cf6485dea10aa3a2f97c0fd6322a0cb17f"
}

View File

@@ -1,12 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "DROP INDEX CONCURRENTLY IF EXISTS labeled_jobs_on_jobs",
"describe": {
"columns": [],
"parameters": {
"Left": []
},
"nullable": []
},
"hash": "6f12be65a4fe3eb39292164363f557de9cef7017dcfbcd40370b849a288c52e3"
}

View File

@@ -1,23 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT EXISTS (SELECT 1 FROM usr WHERE workspace_id = $1 AND email = $2)",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "exists",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
null
]
},
"hash": "7142222bee1f60bc56752e377b96ac6e4e981ea219c173146088faf1ad9f3822"
}

View File

@@ -1,23 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM folder WHERE name = $1 AND workspace_id = $2 RETURNING 1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "?column?",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
null
]
},
"hash": "748904c35cdbb6c7b8a8e0024b341278bf2bb727f2fe0427847565fb9c774abc"
}

View File

@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT usage.usage + 1 FROM usage \n WHERE is_workspace IS FALSE AND\n month_ = EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date)\n AND id = $1",
"query": "SELECT usage.usage + 1 FROM usage \n WHERE is_workspace IS FALSE AND\n month_ = EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date)\n AND id = $1",
"describe": {
"columns": [
{
@@ -18,5 +18,5 @@
null
]
},
"hash": "94ff696b4d3904e3823ef637fa8f1f0d0bdac01040c81b31514326417eb58cee"
"hash": "7d93eb90163516718c85f28f8f05093133c5cbc96414000ac364583114d0ce77"
}

View File

@@ -1,12 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "CREATE INDEX CONCURRENTLY labeled_jobs_on_jobs ON completed_job USING GIN ((result -> 'wm_labels')) WHERE result ? 'wm_label';",
"describe": {
"columns": [],
"parameters": {
"Left": []
},
"nullable": []
},
"hash": "89e72f997e27b9298bd923e7f2b546c5ff061bd9ed63798024306456ca148aec"
}

View File

@@ -1,16 +1,11 @@
{
"db_name": "PostgreSQL",
"query": "SELECT COALESCE((SELECT MIN(started_at) as min_started_at\n FROM queue\n WHERE script_path = $1 AND job_kind != 'dependencies' AND running = true AND workspace_id = $2 AND canceled = false\n GROUP BY script_path), $3) as min_started_at, now() AS now",
"query": "SELECT COALESCE((SELECT MIN(started_at) as min_started_at\n FROM queue\n WHERE script_path = $1 AND job_kind != 'dependencies' AND running = true AND workspace_id = $2 AND canceled = false\n GROUP BY script_path), $3)",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "min_started_at",
"type_info": "Timestamptz"
},
{
"ordinal": 1,
"name": "now",
"name": "coalesce",
"type_info": "Timestamptz"
}
],
@@ -22,9 +17,8 @@
]
},
"nullable": [
null,
null
]
},
"hash": "3901cce744c9b246b661c817e068bdb3b1ab504ff8070fcccf6c909ad75f1f6d"
"hash": "abc7c72dfe9b01cde6f5b206300ed33e3d15b16bce2510160166a66fb7598e61"
}

View File

@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM queue WHERE schedule_path = $1 AND running = false AND workspace_id = $2 AND is_flow_step = false",
"query": "DELETE FROM queue WHERE schedule_path = $1 AND running = false AND workspace_id = $2",
"describe": {
"columns": [],
"parameters": {
@@ -11,5 +11,5 @@
},
"nullable": []
},
"hash": "8d4235984f27d8b939ffd5c660d5b57dc38dd3b2643361ed3b7cdcd1534d2e21"
"hash": "ade89de6e8527c543b182229f1febeb2513ad58b03ab526df148582264fb3a44"
}

View File

@@ -1,23 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM schedule WHERE path = $1 AND workspace_id = $2 RETURNING 1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "?column?",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
null
]
},
"hash": "d1ded8b38e50eb01fa5e5e122dae48ec21856a0041f4aeb244349ab7648d306f"
}

View File

@@ -1,23 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE queue SET canceled = true, canceled_by = $1, scheduled_for = now(), suspend = 0 WHERE id = $2 RETURNING 1 as one",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "one",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Varchar",
"Uuid"
]
},
"nullable": [
null
]
},
"hash": "d4fb94ee8198592c24e85d29078feb5220ab367e339510ee0e83bb7b5abfd184"
}

784
backend/Cargo.lock generated

File diff suppressed because it is too large Load Diff

View File

@@ -1,6 +1,6 @@
[package]
name = "windmill"
version = "1.322.0"
version = "1.306.2"
authors.workspace = true
edition.workspace = true
@@ -24,7 +24,7 @@ members = [
]
[workspace.package]
version = "1.322.0"
version = "1.306.2"
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
edition = "2021"
@@ -224,7 +224,7 @@ tokenizers = "0.14.1"
candle-core = "0.3.0"
candle-transformers = "0.3.0"
candle-nn = "0.3.0"
tiberius = { git = "https://github.com/prisma/tiberius", rev = "8f66a699dfa041e7b5f736c7e94f92c945453c9e", default-features = false, features = ["rustls", "tds73", "chrono", "sql-browser-tokio"]}
tiberius = { version = "0.12.2", default-features = false, features = ["rustls", "tds73", "chrono", "sql-browser-tokio"] }
pin-project = "1"
indexmap = { version = "2.2.5", features = ["serde"]}

View File

@@ -1 +0,0 @@
-- Add down migration script here

View File

@@ -1,228 +0,0 @@
DO
$do$
DECLARE
i text;
arr text[] := array['resource', 'script', 'variable', 'schedule', 'flow', 'app', 'raw_app'];
BEGIN
FOREACH i IN ARRAY arr
LOOP
EXECUTE FORMAT(
$$
DROP POLICY IF EXISTS see_folder_extra_perms_user ON %1$I;
DROP POLICY IF EXISTS see_folder_extra_perms_user_delete ON %1$I;
DROP POLICY IF EXISTS see_extra_perms_user ON %1$I;
DROP POLICY IF EXISTS see_member ON %1$I;
DROP POLICY IF EXISTS see_own ON %1$I;
DROP POLICY IF EXISTS see_extra_perms_user_delete ON %1$I;
DROP POLICY IF EXISTS see_extra_perms_groups ON %1$I;
DROP POLICY IF EXISTS see_extra_perms_groups_delete ON %1$I;
-- New policies for select, insert, update
DROP POLICY IF EXISTS see_folder_extra_perms_user_select ON %1$I;
DROP POLICY IF EXISTS see_folder_extra_perms_user_insert ON %1$I;
DROP POLICY IF EXISTS see_folder_extra_perms_user_update ON %1$I;
DROP POLICY IF EXISTS see_own ON %1$I;
DROP POLICY IF EXISTS see_member ON %1$I;
DROP POLICY IF EXISTS see_extra_perms_user_select ON %1$I;
DROP POLICY IF EXISTS see_extra_perms_user_insert ON %1$I;
DROP POLICY IF EXISTS see_extra_perms_user_update ON %1$I;
DROP POLICY IF EXISTS see_extra_perms_groups_select ON %1$I;
DROP POLICY IF EXISTS see_extra_perms_groups_insert ON %1$I;
DROP POLICY IF EXISTS see_extra_perms_groups_update ON %1$I;
-- Folder permissions split into select, insert, and update
CREATE POLICY see_folder_extra_perms_user_select ON %1$I FOR SELECT TO windmill_user
USING (SPLIT_PART(%1$I.path, '/', 1) = 'f' AND SPLIT_PART(%1$I.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_read'), ',')::text[]));
CREATE POLICY see_folder_extra_perms_user_insert ON %1$I FOR INSERT TO windmill_user
WITH CHECK (SPLIT_PART(%1$I.path, '/', 1) = 'f' AND SPLIT_PART(%1$I.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_write'), ',')::text[]));
CREATE POLICY see_folder_extra_perms_user_update ON %1$I FOR UPDATE TO windmill_user
USING (SPLIT_PART(%1$I.path, '/', 1) = 'f' AND SPLIT_PART(%1$I.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_write'), ',')::text[]));
CREATE POLICY see_folder_extra_perms_user_delete ON %1$I FOR UPDATE TO windmill_user
USING (SPLIT_PART(%1$I.path, '/', 1) = 'f' AND SPLIT_PART(%1$I.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_write'), ',')::text[]));
CREATE POLICY see_own ON %1$I FOR ALL TO windmill_user
USING (SPLIT_PART(%1$I.path, '/', 1) = 'u' AND SPLIT_PART(%1$I.path, '/', 2) = current_setting('session.user'));
CREATE POLICY see_member ON %1$I FOR ALL TO windmill_user
USING (SPLIT_PART(%1$I.path, '/', 1) = 'g' AND SPLIT_PART(%1$I.path, '/', 2) = any(regexp_split_to_array(current_setting('session.groups'), ',')::text[]));
CREATE POLICY see_extra_perms_user_select ON %1$I FOR SELECT TO windmill_user
USING (extra_perms ? CONCAT('u/', current_setting('session.user')));
CREATE POLICY see_extra_perms_user_insert ON %1$I FOR INSERT TO windmill_user
WITH CHECK ((extra_perms ->> CONCAT('u/', current_setting('session.user')))::boolean);
CREATE POLICY see_extra_perms_user_update ON %1$I FOR UPDATE TO windmill_user
USING ((extra_perms ->> CONCAT('u/', current_setting('session.user')))::boolean);
CREATE POLICY see_extra_perms_user_delete ON %1$I FOR DELETE TO windmill_user
USING ((extra_perms ->> CONCAT('u/', current_setting('session.user')))::boolean);
CREATE POLICY see_extra_perms_groups_select ON %1$I FOR SELECT TO windmill_user
USING (extra_perms ?| regexp_split_to_array(current_setting('session.pgroups'), ',')::text[]);
CREATE POLICY see_extra_perms_groups_insert ON %1$I FOR INSERT TO windmill_user
WITH CHECK (exists(
SELECT key, value FROM jsonb_each_text(extra_perms)
WHERE SPLIT_PART(key, '/', 1) = 'g' AND key = ANY(regexp_split_to_array(current_setting('session.pgroups'), ',')::text[])
AND value::boolean));
CREATE POLICY see_extra_perms_groups_update ON %1$I FOR UPDATE TO windmill_user
USING (exists(
SELECT key, value FROM jsonb_each_text(extra_perms)
WHERE SPLIT_PART(key, '/', 1) = 'g' AND key = ANY(regexp_split_to_array(current_setting('session.pgroups'), ',')::text[])
AND value::boolean));
CREATE POLICY see_extra_perms_groups_delete ON %1$I FOR DELETE TO windmill_user
USING (exists(
SELECT key, value FROM jsonb_each_text(extra_perms)
WHERE SPLIT_PART(key, '/', 1) = 'g' AND key = ANY(regexp_split_to_array(current_setting('session.pgroups'), ',')::text[])
AND value::boolean));
$$,
i
);
END LOOP;
END
$do$;
DROP POLICY IF EXISTS see_extra_perms_user ON folder;
DROP POLICY IF EXISTS see_extra_perms_user_select ON folder;
DROP POLICY IF EXISTS see_extra_perms_user_insert ON folder;
DROP POLICY IF EXISTS see_extra_perms_user_update ON folder;
DROP POLICY IF EXISTS see_extra_perms_user_delete ON folder;
DROP POLICY IF EXISTS see_extra_perms_groups ON folder;
DROP POLICY IF EXISTS see_extra_perms_groups_select ON folder;
DROP POLICY IF EXISTS see_extra_perms_groups_insert ON folder;
DROP POLICY IF EXISTS see_extra_perms_groups_update ON folder;
DROP POLICY IF EXISTS see_extra_perms_groups_delete ON folder;
-- Existing CREATE POLICY statements updated to reflect policy splitting for 'folder' table
CREATE POLICY see_extra_perms_user_select ON folder FOR SELECT TO windmill_user
USING (extra_perms ? CONCAT('u/', current_setting('session.user')) OR CONCAT('u/', current_setting('session.user')) = ANY(owners));
CREATE POLICY see_extra_perms_user_insert ON folder FOR INSERT TO windmill_user
WITH CHECK ((CONCAT('u/', current_setting('session.user')) = ANY(owners)));
CREATE POLICY see_extra_perms_user_update ON folder FOR UPDATE TO windmill_user
USING ((CONCAT('u/', current_setting('session.user')) = ANY(owners)));
CREATE POLICY see_extra_perms_user_delete ON folder FOR DELETE TO windmill_user
USING ((CONCAT('u/', current_setting('session.user')) = ANY(owners)));
CREATE POLICY see_extra_perms_groups_select ON folder FOR SELECT TO windmill_user
USING (extra_perms ?| regexp_split_to_array(current_setting('session.pgroups'), ',')::text[] OR EXISTS (
SELECT o FROM unnest(owners) AS o
WHERE o = ANY(regexp_split_to_array(current_setting('session.pgroups'), ',')::text[])));
CREATE POLICY see_extra_perms_groups_insert ON folder FOR INSERT TO windmill_user
WITH CHECK (EXISTS (
SELECT o FROM unnest(owners) AS o
WHERE o = ANY(regexp_split_to_array(current_setting('session.pgroups'), ',')::text[])));
CREATE POLICY see_extra_perms_groups_update ON folder FOR UPDATE TO windmill_user
USING (EXISTS (
SELECT o FROM unnest(owners) AS o
WHERE o = ANY(regexp_split_to_array(current_setting('session.pgroups'), ',')::text[])));
CREATE POLICY see_extra_perms_groups_delete ON folder FOR DELETE TO windmill_user
USING (EXISTS (
SELECT o FROM unnest(owners) AS o
WHERE o = ANY(regexp_split_to_array(current_setting('session.pgroups'), ',')::text[])));
-- DO
-- $do$
-- DECLARE
-- i text;
-- arr text[] := array['resource', 'script', 'variable', 'schedule', 'flow', 'app', 'raw_app'];
-- BEGIN
-- FOREACH i IN ARRAY arr
-- LOOP
-- EXECUTE FORMAT(
-- $$
-- CREATE POLICY see_folder_extra_perms_user ON %1$I FOR ALL TO windmill_user
-- USING (SPLIT_PART(%1$I.path, '/', 1) = 'f' AND SPLIT_PART(%1$I.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_read'), ',')::text[]))
-- WITH CHECK (SPLIT_PART(%1$I.path, '/', 1) = 'f' AND SPLIT_PART(%1$I.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_write'), ',')::text[]));
-- CREATE POLICY see_folder_extra_perms_user_delete ON %1$I AS RESTRICTIVE FOR DELETE TO windmill_user
-- USING (SPLIT_PART(%1$I.path, '/', 1) = 'f' AND SPLIT_PART(%1$I.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_write'), ',')::text[]));
-- CREATE POLICY see_own ON %1$I FOR ALL TO windmill_user
-- USING (SPLIT_PART(%1$I.path, '/', 1) = 'u' AND SPLIT_PART(%1$I.path, '/', 2) = current_setting('session.user'));
-- CREATE POLICY see_member ON %1$I FOR ALL TO windmill_user
-- USING (SPLIT_PART(%1$I.path, '/', 1) = 'g' AND SPLIT_PART(%1$I.path, '/', 2) = any(regexp_split_to_array(current_setting('session.groups'), ',')::text[]));
-- CREATE POLICY see_extra_perms_user ON %1$I FOR ALL TO windmill_user
-- USING (extra_perms ? CONCAT('u/', current_setting('session.user')))
-- WITH CHECK ((extra_perms ->> CONCAT('u/', current_setting('session.user')))::boolean);
-- CREATE POLICY see_extra_perms_user_delete ON %1$I FOR DELETE TO windmill_user
-- USING ((extra_perms ->> CONCAT('u/', current_setting('session.user')))::boolean);
-- CREATE POLICY see_extra_perms_groups ON %1$I FOR ALL TO windmill_user
-- USING (extra_perms ?| regexp_split_to_array(current_setting('session.pgroups'), ',')::text[])
-- WITH CHECK (exists(
-- SELECT key, value FROM jsonb_each_text(extra_perms)
-- WHERE SPLIT_PART(key, '/', 1) = 'g' AND key = ANY(regexp_split_to_array(current_setting('session.pgroups'), ',')::text[])
-- AND value::boolean));
-- CREATE POLICY see_extra_perms_groups_delete ON %1$I FOR DELETE TO windmill_user
-- USING (exists(
-- SELECT key, value FROM jsonb_each_text(extra_perms)
-- WHERE SPLIT_PART(key, '/', 1) = 'g' AND key = ANY(regexp_split_to_array(current_setting('session.pgroups'), ',')::text[])
-- AND value::boolean));
-- $$,
-- i
-- );
-- END LOOP;
-- END
-- $do$;
-- DROP POLICY see_extra_perms_user ON folder;
-- DROP POLICY see_extra_perms_groups ON folder;
-- CREATE POLICY see_extra_perms_user ON folder FOR ALL to windmill_user
-- USING (extra_perms ? CONCAT('u/', current_setting('session.user')) or (CONCAT('u/', current_setting('session.user')) = ANY(owners)))
-- WITH CHECK ((CONCAT('u/', current_setting('session.user')) = ANY(owners)));
-- DROP POLICY IF EXISTS see_extra_perms_user_delete ON folder;
-- CREATE POLICY see_extra_perms_user_delete ON folder AS RESTRICTIVE FOR DELETE to windmill_user
-- USING ((CONCAT('u/', current_setting('session.user')) = ANY(owners)));
-- CREATE POLICY see_extra_perms_groups ON folder FOR ALL to windmill_user
-- USING (extra_perms ?| regexp_split_to_array(current_setting('session.pgroups'), ',')::text[] or (exists(
-- SELECT o FROM unnest(owners) as o
-- WHERE o = ANY(regexp_split_to_array(current_setting('session.pgroups'), ',')::text[]))))
-- WITH CHECK (exists(
-- SELECT o FROM unnest(owners) as o
-- WHERE o = ANY(regexp_split_to_array(current_setting('session.pgroups'), ',')::text[])));
-- DROP POLICY IF EXISTS see_extra_perms_groups_delete ON folder;
-- CREATE POLICY see_extra_perms_groups_delete ON folder AS RESTRICTIVE FOR DELETE to windmill_user
-- USING (exists(
-- SELECT o FROM unnest(owners) as o
-- WHERE o = ANY(regexp_split_to_array(current_setting('session.pgroups'), ',')::text[])));

View File

@@ -1 +0,0 @@
-- Add down migration script here

View File

@@ -1,22 +0,0 @@
DO
$do$
DECLARE
i text;
arr text[] := array['resource', 'script', 'variable', 'schedule', 'flow', 'app', 'raw_app'];
BEGIN
FOREACH i IN ARRAY arr
LOOP
EXECUTE FORMAT(
$$
DROP POLICY IF EXISTS see_folder_extra_perms_user_delete ON %1$I;
CREATE POLICY see_folder_extra_perms_user_delete ON %1$I FOR DELETE TO windmill_user
USING (SPLIT_PART(%1$I.path, '/', 1) = 'f' AND SPLIT_PART(%1$I.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_write'), ',')::text[]));
$$,
i
);
END LOOP;
END
$do$;-- Add up migration script here

View File

@@ -1 +0,0 @@
-- Add down migration script here

View File

@@ -1,2 +0,0 @@
-- Add up migration script here
ALTER TABLE script ADD COLUMN no_main_func BOOLEAN;

View File

@@ -1,2 +0,0 @@
-- Add down migration script here
DROP INDEX IF EXISTS labeled_jobs_on_completed_jobs;

View File

@@ -1 +0,0 @@
-- Add up migration script here

View File

@@ -1 +0,0 @@
-- Add down migration script here

View File

@@ -1 +0,0 @@
-- Add up migration script here

View File

@@ -141,16 +141,5 @@
"scopes": [
"com.intuit.quickbooks.accounting"
]
},
"visma": {
"auth_url": "https://connect.visma.com/connect/authorize",
"token_url": "https://connect.visma.com/connect/token",
"scopes": [
"offline_access",
"vismanet_erp_interactive_api:create",
"vismanet_erp_interactive_api:delete",
"vismanet_erp_interactive_api:read",
"vismanet_erp_interactive_api:update"
]
}
}

View File

@@ -50,8 +50,6 @@ static PYTHON_IMPORTS_REPLACEMENT: phf::Map<&'static str, &'static str> = phf_ma
"mysql" => "mysql-connector-python",
"tenable" => "pytenable",
"ns1" => "ns1-python",
"pymsql" => "PyMySQL",
"haystack" => "haystack-ai",
};
fn replace_import(x: String) -> String {
@@ -63,7 +61,7 @@ fn replace_import(x: String) -> String {
}
lazy_static! {
static ref RE: Regex = Regex::new(r"^\#\s?(\S+)\s*$").unwrap();
static ref RE: Regex = Regex::new(r"^\#\s?(\S+)$").unwrap();
}
fn process_import(module: Option<String>, path: &str, level: usize) -> Vec<String> {

View File

@@ -292,30 +292,22 @@ pub fn parse_pg_typ(typ: &str) -> Typ {
Typ::List(Box::new(base_typ))
} else {
match typ {
"varchar" | "character varying" => Typ::Str(None),
"varchar" => Typ::Str(None),
"text" => Typ::Str(None),
"int" | "integer" | "int4" => Typ::Int,
"int" => Typ::Int,
"bigint" => Typ::Int,
"bool" | "boolean" => Typ::Bool,
"char" | "character" => Typ::Str(None),
"json" | "jsonb" => Typ::Object(vec![]),
"smallint" | "int2" => Typ::Int,
"smallserial" | "serial2" => Typ::Int,
"serial" | "serial4" => Typ::Int,
"bigserial" | "serial8" => Typ::Int,
"real" | "float4" => Typ::Float,
"double" | "double precision" | "float8" => Typ::Float,
"numeric" | "decimal" => Typ::Float,
"bool" => Typ::Bool,
"char" => Typ::Str(None),
"smallint" => Typ::Int,
"smallserial" => Typ::Int,
"serial" => Typ::Int,
"bigserial" => Typ::Int,
"real" => Typ::Float,
"double precision" => Typ::Float,
"numeric" => Typ::Float,
"decimal" => Typ::Float,
"oid" => Typ::Int,
"date"
| "time"
| "timetz"
| "time with time zone"
| "time without time zone"
| "timestamp"
| "timestamptz"
| "timestamp with time zone"
| "timestamp without time zone" => Typ::Datetime,
"date" | "time" | "timestamp" | "timestamptz" => Typ::Datetime,
_ => Typ::Str(None),
}
}

View File

@@ -3,7 +3,7 @@
"collaborators": [
"Ruben Fiszel <ruben@windmill.dev>"
],
"version": "1.318.0",
"version": "1.286.2",
"files": [
"windmill_parser_wasm_bg.wasm",
"windmill_parser_wasm.js",

View File

@@ -561,6 +561,10 @@ async function __wbg_load(module, imports) {
function __wbg_get_imports() {
const imports = {};
imports.wbg = {};
imports.wbg.__wbg_eval_89aaea39f7e976e8 = function(arg0, arg1) {
const ret = eval(getStringFromWasm0(arg0, arg1));
return addHeapObject(ret);
};
imports.wbg.__wbindgen_object_drop_ref = function(arg0) {
takeObject(arg0);
};
@@ -612,10 +616,6 @@ function __wbg_get_imports() {
const ret = getObject(arg0) in getObject(arg1);
return ret;
};
imports.wbg.__wbg_eval_33c4985197d1feaf = function(arg0, arg1) {
const ret = eval(getStringFromWasm0(arg0, arg1));
return addHeapObject(ret);
};
imports.wbg.__wbindgen_jsval_loose_eq = function(arg0, arg1) {
const ret = getObject(arg0) == getObject(arg1);
return ret;

View File

@@ -1026,7 +1026,7 @@ async fn listen_for_uuid_on(
async fn completed_job(uuid: Uuid, db: &Pool<Postgres>) -> CompletedJob {
sqlx::query_as::<_, CompletedJob>("SELECT *, result->'wm_labels' as labels FROM completed_job WHERE id = $1").bind(uuid)
sqlx::query_as::<_, CompletedJob>("SELECT * FROM completed_job WHERE id = $1").bind(uuid)
.fetch_one(db)
.await
.unwrap()
@@ -3164,7 +3164,6 @@ async fn run_deployed_relative_imports(db: &Pool<Postgres>, script_content: Stri
deployment_message: None,
concurrency_key: None,
visible_to_runner_only: None,
no_main_func: None,
},
).await.unwrap();

View File

@@ -1,7 +1,7 @@
openapi: "3.0.3"
info:
version: 1.322.0
version: 1.306.2
title: Windmill API
contact:
@@ -3401,13 +3401,6 @@ paths:
in: query
schema:
type: boolean
- name: hide_without_main
description: |
(default false)
hide the scripts without an exported main function
in: query
schema:
type: boolean
- name: is_template
description: |
(default regardless)
@@ -4907,8 +4900,6 @@ paths:
- language
force_viewer_static_fields:
type: object
force_viewer_one_of_fields:
type: object
required:
- args
- component
@@ -5250,11 +5241,6 @@ paths:
in: query
schema:
type: boolean
- name: is_not_schedule
description: is not a scheduled job
in: query
schema:
type: boolean
responses:
"200":
description: All queued jobs
@@ -5340,7 +5326,6 @@ paths:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/OrderDesc"
- $ref: "#/components/parameters/CreatedBy"
- $ref: "#/components/parameters/Label"
- $ref: "#/components/parameters/ParentJob"
- $ref: "#/components/parameters/ScriptExactPath"
- $ref: "#/components/parameters/ScriptStartPath"
@@ -5370,11 +5355,6 @@ paths:
in: query
schema:
type: boolean
- name: is_not_schedule
description: is not a scheduled job
in: query
schema:
type: boolean
responses:
"200":
description: All completed jobs
@@ -5394,7 +5374,6 @@ paths:
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/CreatedBy"
- $ref: "#/components/parameters/Label"
- $ref: "#/components/parameters/ParentJob"
- $ref: "#/components/parameters/ScriptExactPath"
- $ref: "#/components/parameters/ScriptStartPath"
@@ -5437,11 +5416,6 @@ paths:
in: query
schema:
type: boolean
- name: is_not_schedule
description: is not a scheduled job
in: query
schema:
type: boolean
responses:
"200":
description: All jobs
@@ -5560,26 +5534,6 @@ paths:
flow_status:
$ref: "#/components/schemas/WorkflowStatusRecord"
/w/{workspace}/jobs_u/get_log_file/{path}:
get:
summary: get log file from object store
operationId: getLogFileFromStore
tags:
- job
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- name: path
in: path
required: true
schema:
type: string
responses:
"200":
description: job log
content:
text/plain:
type: string
/w/{workspace}/jobs_u/get_flow_debug_info/{id}:
get:
summary: get flow debug info
@@ -8051,12 +8005,6 @@ components:
in: query
schema:
type: string
Label:
name: label
description: mask to filter exact matching job's label (job labels are completed jobs with as a result an object containing a string in the array at key 'wm_labels')
in: query
schema:
type: string
ParentJob:
name: parent_job
description:
@@ -8388,8 +8336,6 @@ components:
type: boolean
visible_to_runner_only:
type: boolean
no_main_func:
type: boolean
required:
- hash
- path
@@ -8405,7 +8351,6 @@ components:
- language
- kind
- starred
- no_main_func
NewScript:
type: object
@@ -8479,8 +8424,6 @@ components:
type: string
visible_to_runner_only:
type: boolean
no_main_func:
type: boolean
required:
- path
- summary
@@ -8782,10 +8725,6 @@ components:
type: string
priority:
type: integer
labels:
type: array
items:
type: string
required:
- id
- created_by
@@ -8803,21 +8742,15 @@ components:
- tag
Job:
oneOf:
- allOf:
allOf:
- oneOf:
- $ref: "#/components/schemas/CompletedJob"
- type: object
properties:
type:
type: string
enum: [CompletedJob]
- allOf:
- $ref: "#/components/schemas/QueuedJob"
- type: object
properties:
type:
type: string
enum: [QueuedJob]
- type: object
properties:
type:
type: string
enum: [CompletedJob, QueuedJob]
discriminator:
propertyName: type
@@ -9787,11 +9720,6 @@ components:
- $ref: "../../openflow.openapi.yaml#/components/schemas/OpenFlow"
- $ref: "#/components/schemas/FlowMetadata"
ExtraPerms:
type: object
additionalProperties:
type: boolean
FlowMetadata:
type: object
properties:
@@ -9807,7 +9735,9 @@ components:
archived:
type: boolean
extra_perms:
$ref: "#/components/schemas/ExtraPerms"
type: object
additionalProperties:
type: boolean
starred:
type: boolean
draft_only:
@@ -9890,10 +9820,6 @@ components:
type: object
additionalProperties:
type: object
triggerables_v2:
type: object
additionalProperties:
type: object
execution_mode:
type: string
enum: [viewer, publisher, anonymous]
@@ -9985,8 +9911,7 @@ components:
created_at:
type: string
format: date-time
value:
type: object
value: {}
policy:
$ref: "#/components/schemas/Policy"
execution_mode:

View File

@@ -143,7 +143,6 @@ pub struct AppHistoryUpdate {
}
pub type StaticFields = HashMap<String, Box<RawValue>>;
pub type OneOfFields = HashMap<String, Vec<Box<RawValue>>>;
#[derive(Serialize, Deserialize, Debug, PartialEq, Clone)]
#[serde(rename_all = "lowercase")]
@@ -153,12 +152,6 @@ pub enum ExecutionMode {
Viewer,
}
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct PolicyTriggerableInputs {
static_inputs: StaticFields,
one_of_inputs: OneOfFields,
}
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct Policy {
pub on_behalf_of: Option<String>,
@@ -167,10 +160,7 @@ pub struct Policy {
// - script/<path>
// - flow/<path>
// - rawscript/<sha256>
#[serde(skip_serializing_if = "Option::is_none")]
pub triggerables: Option<HashMap<String, StaticFields>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub triggerables_v2: Option<HashMap<String, PolicyTriggerableInputs>>,
pub triggerables: HashMap<String, StaticFields>,
pub execution_mode: ExecutionMode,
}
@@ -944,7 +934,6 @@ pub struct ExecuteApp {
pub raw_code: Option<RawCode>,
// if set, the app is executed as viewer with the given static fields
pub force_viewer_static_fields: Option<StaticFields>,
pub force_viewer_one_of_fields: Option<OneOfFields>,
}
fn digest(code: &str) -> String {
@@ -977,56 +966,39 @@ async fn execute_component(
let path = path.to_path();
let policy = match payload.clone() {
ExecuteApp {
force_viewer_static_fields: Some(static_fields),
force_viewer_one_of_fields: Some(one_of_fields),
..
} => {
let mut hm = HashMap::new();
let policy = if let Some(static_fields) = payload.clone().force_viewer_static_fields {
let mut hm = HashMap::new();
if let Some(path) = payload.path.clone() {
hm.insert(
format!("{}:{path}", payload.component),
PolicyTriggerableInputs {
static_inputs: static_fields,
one_of_inputs: one_of_fields,
},
);
} else {
hm.insert(
format!(
"{}:{}",
payload.component,
digest(payload.raw_code.clone().unwrap().content.as_str())
),
PolicyTriggerableInputs {
static_inputs: static_fields,
one_of_inputs: one_of_fields,
},
);
}
Policy {
execution_mode: ExecutionMode::Viewer,
triggerables: None,
triggerables_v2: Some(hm),
on_behalf_of: None,
on_behalf_of_email: None,
}
if let Some(path) = payload.path.clone() {
hm.insert(format!("{}:{path}", payload.component), static_fields);
} else {
hm.insert(
format!(
"{}:{}",
payload.component,
digest(payload.raw_code.clone().unwrap().content.as_str())
),
static_fields,
);
}
_ => {
let policy_o = sqlx::query_scalar!(
"SELECT policy from app WHERE path = $1 AND workspace_id = $2",
path,
&w_id
)
.fetch_optional(&db)
.await?;
let policy = not_found_if_none(policy_o, "App", path)?;
serde_json::from_value::<Policy>(policy).map_err(to_anyhow)?
Policy {
execution_mode: ExecutionMode::Viewer,
triggerables: hm,
on_behalf_of: None,
on_behalf_of_email: None,
}
} else {
let policy_o = sqlx::query_scalar!(
"SELECT policy from app WHERE path = $1 AND workspace_id = $2",
path,
&w_id
)
.fetch_optional(&db)
.await?;
let policy = not_found_if_none(policy_o, "App", path)?;
serde_json::from_value::<Policy>(policy).map_err(to_anyhow)?
};
let (username, permissioned_as, email) = match policy.execution_mode {
@@ -1161,106 +1133,22 @@ fn build_args(
path: String,
args: HashMap<String, Box<RawValue>>,
) -> Result<PushArgs<HashMap<String, Box<RawValue>>>> {
let key = format!("{}:{}", component, &path);
let (static_inputs, one_of_inputs) = match policy {
Policy { triggerables_v2: Some(t), .. } => {
let PolicyTriggerableInputs { static_inputs, one_of_inputs } = t
.get(&key)
.or_else(|| t.get(&path))
.map(|x| x.clone())
.or_else(|| {
if matches!(policy.execution_mode, ExecutionMode::Viewer) {
Some(PolicyTriggerableInputs {
static_inputs: HashMap::new(),
one_of_inputs: HashMap::new(),
})
} else {
None
}
})
.ok_or_else(|| {
Error::BadRequest(format!("path {} is not allowed in the app policy", path))
})?;
(static_inputs, one_of_inputs)
}
Policy { triggerables: Some(t), .. } => {
let static_inputs = t
.get(&key)
.or_else(|| t.get(&path))
.map(|x| x.clone())
.or_else(|| {
if matches!(policy.execution_mode, ExecutionMode::Viewer) {
Some(HashMap::new())
} else {
None
}
})
.ok_or_else(|| {
Error::BadRequest(format!("path {} is not allowed in the app policy", path))
})?;
(static_inputs, HashMap::new())
}
_ => Err(Error::BadRequest(format!(
"Policy is missing triggerables for {}",
key
)))?,
};
let mut args = args.clone();
let mut safe_args = HashMap::<String, Box<RawValue>>::new();
for (k, v) in one_of_inputs {
if let Some(arg_val) = args.get(&k) {
let arg_str = arg_val.get();
let options_str_vec = v.iter().map(|x| x.get()).collect::<Vec<&str>>();
if options_str_vec.contains(&arg_str) {
safe_args.insert(k.to_string(), arg_val.clone());
args.remove(&k);
continue;
}
// check if multiselect
if let Ok(args_str_vec) = serde_json::from_str::<Vec<Box<RawValue>>>(arg_val.get()) {
if args_str_vec
.iter()
.all(|x| options_str_vec.contains(&x.get()))
{
safe_args.insert(k.to_string(), arg_val.clone());
args.remove(&k);
continue;
}
}
return Err(Error::BadRequest(format!(
"argument {} with value {} must be one of [{}]",
k,
arg_str,
options_str_vec.join(",")
)));
}
}
// disallow var and res access in args coming from the user for security reasons
let mut safe_args: HashMap<String, Box<RawValue>> = args.clone();
for (k, v) in args {
let arg_str = serde_json::to_string(&v).unwrap_or_else(|_| "".to_string());
if !arg_str.contains("$var:") && !arg_str.contains("$res:") {
safe_args.insert(k.to_string(), v);
} else {
let args_str = serde_json::to_string(&v).unwrap_or_else(|_| "".to_string());
if args_str.contains("$var:") || args_str.contains("$res:") {
safe_args.insert(
k.to_string(),
RawValue::from_string(
arg_str
args_str
.replace(
"$var:",
"The following variable has been omitted for security reasons: ",
"The following variable has been ommited for security reasons: ",
)
.replace(
"$res:",
"The following resource has been omitted for security reasons: ",
"The following resource has been ommited for security reasons: ",
),
)
.map_err(|e| {
@@ -1272,8 +1160,24 @@ fn build_args(
);
}
}
let key = format!("{}:{}", component, &path);
let static_args = policy
.triggerables
.get(&key)
.or_else(|| policy.triggerables.get(&path))
.map(|x| x.clone())
.or_else(|| {
if matches!(policy.execution_mode, ExecutionMode::Viewer) {
Some(HashMap::new())
} else {
None
}
})
.ok_or_else(|| {
Error::BadRequest(format!("path {} is not allowed in the app policy", path))
})?;
let mut extra = HashMap::new();
for (k, v) in static_inputs {
for (k, v) in static_args {
extra.insert(k.to_string(), v.to_owned());
}
Ok(PushArgs { extra, args: sqlx::types::Json(safe_args) })

View File

@@ -136,15 +136,6 @@ impl Migrate for CustomMigrator {
migration.version,
migration.description
);
if migration.version == 20240424083501 {
tracing::info!("Special migration to add index concurrently on job labels 2");
sqlx::query!(
"DROP INDEX CONCURRENTLY IF EXISTS labeled_jobs_on_jobs"
).execute(&mut *self.inner).await?;
sqlx::query!(
"CREATE INDEX CONCURRENTLY labeled_jobs_on_jobs ON completed_job USING GIN ((result -> 'wm_labels')) WHERE result ? 'wm_label';"
).execute(&mut *self.inner).await?;
}
let r = self.inner.apply(migration).await;
tracing::info!("Finished applying migration {}", migration.version);
r

View File

@@ -271,7 +271,7 @@ async fn update_folder(
Extension(user_db): Extension<UserDB>,
Extension(webhook): Extension<WebhookShared>,
Path((w_id, name)): Path<(String, String)>,
Json(mut ng): Json<UpdateFolder>,
Json(ng): Json<UpdateFolder>,
) -> Result<String> {
use sql_builder::prelude::*;
@@ -282,25 +282,6 @@ async fn update_folder(
if let Some(display_name) = ng.display_name {
sqlb.set("display_name", "?".bind(&display_name));
}
if !authed.is_admin {
let prefixed_username = format!("u/{}", authed.username);
if ng.owners.as_ref().is_some_and(|x| {
!x.contains(&prefixed_username)
&& !authed.groups.iter().any(|g| x.contains(&format!("g/{g}")))
}) {
ng.owners.as_mut().unwrap().push(prefixed_username.clone());
if ng.extra_perms.is_none() {
ng.extra_perms = Some(serde_json::Value::Object(serde_json::Map::new()));
}
ng.extra_perms
.as_mut()
.unwrap()
.as_object_mut()
.unwrap()
.insert(prefixed_username, serde_json::json!(true));
}
}
if let Some(owners) = ng.owners {
sqlb.set(
"owners",
@@ -329,16 +310,9 @@ async fn update_folder(
.sql()
.map_err(|e| error::Error::InternalErr(e.to_string()))?;
let nfolder = sqlx::query_as::<_, Folder>(&sql)
.fetch_optional(&mut *tx)
.fetch_one(&mut *tx)
.await?;
let nfolder = nfolder.ok_or_else(|| {
windmill_common::error::Error::NotAuthorized(format!(
"You are not an owner of {} and hence cannot modify it",
name
))
})?;
if let Some(extra_perms) = nfolder.extra_perms.as_object() {
for o in nfolder.owners {
if !extra_perms
@@ -504,22 +478,13 @@ async fn delete_folder(
not_found_if_none(get_folderopt(&mut tx, &w_id, &name).await?, "Folder", &name)?;
let del = sqlx::query_scalar!(
"DELETE FROM folder WHERE name = $1 AND workspace_id = $2 RETURNING 1",
sqlx::query!(
"DELETE FROM folder WHERE name = $1 AND workspace_id = $2",
name,
w_id
)
.fetch_optional(&mut *tx)
.await?
.flatten();
if del.is_none() {
return Err(windmill_common::error::Error::NotAuthorized(format!(
"Not authorized to delete folder {}",
name
)));
}
.execute(&mut *tx)
.await?;
audit_log(
&mut *tx,
&authed.username,

View File

@@ -251,7 +251,6 @@ pub fn global_service() -> Router {
get(get_completed_job_result_maybe),
)
.route("/getupdate/:id", get(get_job_update))
.route("/get_log_file/*file_path", get(get_log_file))
.route("/queue/cancel/:id", post(cancel_job_api))
.route(
"/queue/cancel_persistent/*script_path",
@@ -585,7 +584,6 @@ fn generate_get_job_query(no_logs: bool, table: &str) -> String {
result,
deleted,
is_skipped,
result->'wm_labels' as labels,
CASE WHEN result is null or pg_column_size(result) < 2000000 THEN result ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as result"
} else {
"scheduled_for,
@@ -780,8 +778,6 @@ pub struct ListableCompletedJob {
pub tag: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub priority: Option<i16>,
#[serde(skip_serializing_if = "Option::is_none")]
pub labels: Option<serde_json::Value>,
}
#[derive(Deserialize, Clone)]
@@ -842,7 +838,6 @@ pub struct ListQueueQuery {
pub all_workspaces: Option<bool>,
pub is_flow_step: Option<bool>,
pub has_null_parent: Option<bool>,
pub is_not_schedule: Option<bool>,
}
fn list_queue_jobs_query(w_id: &str, lq: &ListQueueQuery, fields: &[&str]) -> SqlBuilder {
@@ -947,10 +942,6 @@ fn list_queue_jobs_query(w_id: &str, lq: &ListQueueQuery, fields: &[&str]) -> Sq
sqlb.and_where_le("scheduled_for", "now()");
}
if lq.is_not_schedule.unwrap_or(false) {
sqlb.and_where("schedule_path IS null");
}
sqlb
}
@@ -1021,62 +1012,49 @@ async fn cancel_all(
) -> error::JsonResult<Vec<Uuid>> {
require_admin(authed.is_admin, &authed.username)?;
let jobs = sqlx::query!(
"SELECT id, running, is_flow_step FROM queue WHERE scheduled_for < now() AND workspace_id = $1 AND schedule_path IS NULL",
let mut jobs = sqlx::query!(
"UPDATE queue SET canceled = true, canceled_by = $2, scheduled_for = now(), suspend = 0 WHERE scheduled_for < now() AND workspace_id = $1 AND schedule_path IS NULL RETURNING id, running, is_flow_step",
w_id,
authed.username
)
.fetch_all(&db)
.await?;
let username = authed.username;
let mut uuids = vec![];
for j in jobs.iter() {
let r = sqlx::query!(
"UPDATE queue SET canceled = true, canceled_by = $1, scheduled_for = now(), suspend = 0 WHERE id = $2 RETURNING 1 as one",
username,
j.id,
)
.fetch_optional(&db)
.await;
if !j.running && !j.is_flow_step.unwrap_or(false) {
let e = serde_json::json!({"message": format!("Job canceled: cancel_all by {username}"), "name": "Canceled", "reason": "cancel_all", "canceler": username});
let job_running = get_queued_job(&j.id, &w_id, &db).await?;
if r.as_ref().is_ok_and(|x| x.is_some()) {
uuids.push(j.id);
if !j.running && !j.is_flow_step.unwrap_or(false) {
let e = serde_json::json!({"message": format!("Job canceled: cancel_all by {username}"), "name": "Canceled", "reason": "cancel_all", "canceler": username});
let job_running = get_queued_job(&j.id, &w_id, &db).await?;
if let Some(job_running) = job_running {
append_logs(
j.id,
w_id.clone(),
format!("canceled by {username}: cancel_all"),
db.clone(),
)
.await;
let add_job = add_completed_job_error(
&db,
&job_running,
job_running.mem_peak.unwrap_or(0),
Some(CanceledBy {
username: Some(username.to_string()),
reason: Some("cancel_all".to_string()),
}),
e,
rsmq.clone(),
"server",
true,
)
.await;
if let Err(e) = add_job {
tracing::error!("Failed to add canceled job: {}", e);
}
if let Some(job_running) = job_running {
append_logs(
j.id,
w_id.clone(),
format!("canceled by {username}: cancel_all"),
db.clone(),
)
.await;
let add_job = add_completed_job_error(
&db,
&job_running,
job_running.mem_peak.unwrap_or(0),
Some(CanceledBy {
username: Some(username.to_string()),
reason: Some("cancel_all".to_string()),
}),
e,
rsmq.clone(),
"server",
true,
)
.await;
if let Err(e) = add_job {
tracing::error!("Failed to add canceled job: {}", e);
}
}
} else {
tracing::error!("Failed to cancel job: {:?} {:?}", j.id, r.err());
}
}
let uuids = jobs.iter_mut().map(|j| j.id).collect::<Vec<_>>();
Ok(Json(uuids))
}
@@ -1179,14 +1157,13 @@ async fn list_jobs(
"null as concurrent_limit",
"null as concurrency_time_window_s",
"priority",
"result->'wm_labels' as labels",
],
))
} else {
None
};
let sql = if lq.success.is_none() && lq.label.is_none() {
let sql = if lq.success.is_none() {
let sqlq = list_queue_jobs_query(
&w_id,
&ListQueueQuery {
@@ -1212,7 +1189,6 @@ async fn list_jobs(
all_workspaces: lq.all_workspaces,
is_flow_step: lq.is_flow_step,
has_null_parent: lq.has_null_parent,
is_not_schedule: lq.is_not_schedule,
},
&[
"'QueuedJob' as typ",
@@ -1246,7 +1222,6 @@ async fn list_jobs(
"concurrent_limit",
"concurrency_time_window_s",
"priority",
"null as labels",
],
);
@@ -1979,7 +1954,6 @@ struct UnifiedJob {
concurrent_limit: Option<i32>,
concurrency_time_window_s: Option<i32>,
priority: Option<i16>,
labels: Option<serde_json::Value>,
}
impl<'a> From<UnifiedJob> for Job {
@@ -2017,7 +1991,6 @@ impl<'a> From<UnifiedJob> for Job {
mem_peak: uj.mem_peak,
tag: uj.tag,
priority: uj.priority,
labels: uj.labels,
}),
"QueuedJob" => Job::QueuedJob(QueuedJob {
workspace_id: uj.workspace_id,
@@ -2298,7 +2271,7 @@ pub async fn restart_flow(
check_license_key_valid().await?;
let completed_job = sqlx::query_as::<_, CompletedJob>(
"SELECT *, result->'wm_labels' as labels from completed_job WHERE id = $1 and workspace_id = $2",
"SELECT * from completed_job WHERE id = $1 and workspace_id = $2",
)
.bind(job_id)
.bind(&w_id)
@@ -3511,84 +3484,6 @@ pub struct JobUpdate {
pub flow_status: Option<serde_json::Value>,
}
// #[cfg(all(feature = "enterprise", feature = "parquet"))]
// async fn get_logs_from_store(
// log_offset: i32,
// logs: &str,
// log_file_index: Option<Vec<String>>,
// ) -> Option<error::Result<Body>> {
// if log_offset > 0 {
// if let Some(file_index) = log_file_index {
// tracing::debug!("Getting logs from store: {file_index:?}");
// if let Some(os) = OBJECT_STORE_CACHE_SETTINGS.read().await.clone() {
// tracing::debug!("object store client present, streaming from there");
// let logs = logs.to_string();
// let stream = async_stream::stream! {
// for file_p in file_index {
// let file_p_2 = file_p.clone();
// let file = os.get(&object_store::path::Path::from(file_p)).await;
// if let Ok(file) = file {
// if let Ok(bytes) = file.bytes().await {
// yield Ok(bytes::Bytes::from(bytes)) as object_store::Result<bytes::Bytes>;
// }
// } else {
// tracing::debug!("error getting file from store: {file_p_2}: {}", file.err().unwrap());
// }
// }
// yield Ok(bytes::Bytes::from(logs))
// };
// return Some(Ok(Body::from_stream(stream)));
// } else {
// tracing::debug!("object store client not present, cannot stream logs from store");
// }
// }
// }
// return None;
// }
#[cfg(all(feature = "enterprise", feature = "parquet"))]
async fn get_log_file(Path((_w_id, file_p)): Path<(String, String)>) -> error::Result<Response> {
if let Some(os) = OBJECT_STORE_CACHE_SETTINGS.read().await.clone() {
let file = os
.get(&object_store::path::Path::from(format!("logs/{file_p}")))
.await;
if let Ok(file) = file {
if let Ok(bytes) = file.bytes().await {
use axum::http::header;
let res = Response::builder()
.header(header::CONTENT_TYPE, "text/plain")
.body(Body::from(bytes::Bytes::from(bytes)))
.unwrap();
return Ok(res);
} else {
return Err(error::Error::InternalErr(format!(
"Error getting bytes from file: {}",
file_p
)));
}
} else {
return Err(error::Error::NotFound(format!(
"File not found: {}",
file_p
)));
}
} else {
return Err(error::Error::InternalErr(
"Object store client not present, cannot stream logs from store".to_string(),
));
}
}
#[cfg(not(all(feature = "enterprise", feature = "parquet")))]
async fn get_log_file(Path((_w_id, file_p)): Path<(String, String)>) -> error::Result<Response> {
return Err(error::Error::NotFound(format!(
"Get log file is an EE feature: {}",
file_p
)));
}
async fn get_job_update(
Extension(db): Extension<DB>,
Path((w_id, job_id)): Path<(String, Uuid)>,
@@ -3745,17 +3640,6 @@ fn list_completed_jobs_query(
sqlb.and_where("result @> ?".bind(&result.replace("'", "''")));
}
if let Some(label) = &lq.label {
let mut wh = format!("result->'wm_labels' ? ");
wh.push_str(&format!("'{}'", &label.replace("'", "''")));
sqlb.and_where(&wh);
sqlb.and_where("result ? 'wm_labels'");
}
if lq.is_not_schedule.unwrap_or(false) {
sqlb.and_where("schedule_path IS null");
}
sqlb
}
#[derive(Deserialize, Clone)]
@@ -3787,8 +3671,6 @@ pub struct ListCompletedQuery {
pub scheduled_for_before_now: Option<bool>,
pub all_workspaces: Option<bool>,
pub has_null_parent: Option<bool>,
pub label: Option<String>,
pub is_not_schedule: Option<bool>,
}
async fn list_completed_jobs(
@@ -3836,7 +3718,6 @@ async fn list_completed_jobs(
"mem_peak",
"tag",
"priority",
"result->'wm_labels' as labels",
"'CompletedJob' as type",
],
)
@@ -3854,7 +3735,7 @@ async fn get_completed_job<'a>(
let job_o = sqlx::query("SELECT id, workspace_id, parent_job, created_by, created_at, duration_ms, success, script_hash, script_path,
CASE WHEN args is null or pg_column_size(args) < 2000000 THEN args ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as args, CASE WHEN result is null or pg_column_size(result) < 2000000 THEN result ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as result, logs, deleted, raw_code, canceled, canceled_by, canceled_reason, job_kind, env_id,
schedule_path, permissioned_as, flow_status, raw_flow, is_flow_step, language, started_at, is_skipped,
raw_lock, email, visible_to_owner, mem_peak, tag, priority, result->'wm_labels' as labels FROM completed_job WHERE id = $1 AND workspace_id = $2")
raw_lock, email, visible_to_owner, mem_peak, tag, priority FROM completed_job WHERE id = $1 AND workspace_id = $2")
.bind(id)
.bind(w_id)
.fetch_optional(&db)

View File

@@ -581,37 +581,14 @@ async fn delete_schedule(
let path = path.to_path();
clear_schedule(&mut tx, path, &w_id).await?;
let exists = sqlx::query_scalar!(
"SELECT 1 FROM schedule WHERE path = $1 AND workspace_id = $2",
sqlx::query!(
"DELETE FROM schedule WHERE path = $1 AND workspace_id = $2",
path,
w_id
)
.fetch_optional(&mut *tx)
.await?
.flatten();
if exists.is_none() {
return Err(windmill_common::error::Error::NotFound(format!(
"Schedule {} not found",
path
)));
}
let del = sqlx::query_scalar!(
"DELETE FROM schedule WHERE path = $1 AND workspace_id = $2 RETURNING 1",
path,
w_id
)
.fetch_optional(&mut *tx)
.await?
.flatten();
if del.is_none() {
return Err(windmill_common::error::Error::NotAuthorized(format!(
"Not authorized to delete schedule {}",
path
)));
}
.execute(&mut *tx)
.await?;
handle_deployment_metadata(
&authed.email,
@@ -800,7 +777,7 @@ pub async fn clear_schedule<'c>(
w_id: &str,
) -> Result<()> {
sqlx::query!(
"DELETE FROM queue WHERE schedule_path = $1 AND running = false AND workspace_id = $2 AND is_flow_step = false",
"DELETE FROM queue WHERE schedule_path = $1 AND running = false AND workspace_id = $2",
path,
w_id
)

View File

@@ -91,8 +91,6 @@ pub struct ScriptWDraft {
pub concurrency_key: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub visible_to_runner_only: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub no_main_func: Option<bool>,
}
pub fn global_service() -> Router {
@@ -194,8 +192,7 @@ async fn list_scripts(
"tag",
"draft.path IS NOT NULL as has_draft",
"draft_only",
"ws_error_handler_muted",
"no_main_func",
"ws_error_handler_muted"
])
.left()
.join("favorite")
@@ -215,10 +212,6 @@ async fn list_scripts(
.limit(per_page)
.clone();
if authed.is_operator || lq.hide_without_main.unwrap_or(false) {
sqlb.and_where("o.no_main_func IS NOT TRUE");
}
if lq.show_archived.unwrap_or(false) {
sqlb.and_where_eq(
"o.created_at",
@@ -482,8 +475,8 @@ async fn create_script(
content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, \
draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, \
dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, \
delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func) \
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29)",
delete_after_use, timeout, concurrency_key, visible_to_runner_only) \
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28)",
&w_id,
&hash.0,
ns.path,
@@ -512,7 +505,6 @@ async fn create_script(
ns.timeout,
ns.concurrency_key,
ns.visible_to_runner_only,
ns.no_main_func
)
.execute(&mut tx)
.await?;
@@ -732,7 +724,7 @@ async fn get_script_by_path_w_draft(
let mut tx = user_db.begin(&authed).await?;
let script_o = sqlx::query_as::<_, ScriptWDraft>(
"SELECT hash, script.path, summary, description, content, language, kind, tag, schema, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, ws_error_handler_muted, draft.value as draft, dedicated_worker, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func FROM script LEFT JOIN draft ON
"SELECT hash, script.path, summary, description, content, language, kind, tag, schema, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, ws_error_handler_muted, draft.value as draft, dedicated_worker, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only FROM script LEFT JOIN draft ON
script.path = draft.path AND script.workspace_id = draft.workspace_id AND draft.typ = 'script'
WHERE script.path = $1 AND script.workspace_id = $2 \
AND script.created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND \

View File

@@ -1337,25 +1337,6 @@ async fn accept_invite(
.await?;
if let Some(r) = r {
let already_in_workspace = sqlx::query_scalar!(
"SELECT EXISTS(SELECT 1 FROM usr WHERE workspace_id = $1 AND email = $2)",
&nu.workspace_id,
&email,
)
.fetch_one(&mut *tx)
.await?
.unwrap_or(false);
if already_in_workspace {
tx.commit().await?;
return Ok((
StatusCode::CREATED,
format!(
"user {} accepted invite to workspace {}",
&email, nu.workspace_id
),
));
}
let username;
(tx, username) = add_user_to_workspace(
&nu.workspace_id,

View File

@@ -722,16 +722,6 @@ async fn edit_auto_invite(
for user in users_to_auto_add.as_ref().unwrap() {
auto_add_user(&user.email, &w_id, &operator, &mut tx).await?;
send_email_if_possible(
&format!("Added to Windmill's workspace: {w_id}"),
&format!(
"You have been granted access to Windmill's workspace {w_id} by {email}.
Access the workspace at {}/?workspace={w_id}",
BASE_URL.read().await.clone()
),
&user.email,
);
}
} else {
sqlx::query!(
@@ -1946,22 +1936,6 @@ async fn invite_user(
let mut tx = db.begin().await?;
let already_in_workspace = sqlx::query_scalar!(
"SELECT EXISTS (SELECT 1 FROM usr WHERE workspace_id = $1 AND email = $2)",
&w_id,
nu.email
)
.fetch_one(&mut *tx)
.await?
.unwrap_or(false);
if already_in_workspace {
return Err(Error::BadRequest(format!(
"user with email {} already exists in workspace {}",
nu.email, w_id
)));
}
sqlx::query!(
"INSERT INTO workspace_invite
(workspace_id, email, is_admin, operator)
@@ -2072,14 +2046,6 @@ async fn add_user(
.execute(&mut *tx)
.await?;
sqlx::query!(
"DELETE FROM workspace_invite WHERE workspace_id = $1 AND email = $2",
&w_id,
nu.email
)
.execute(&mut *tx)
.await?;
sqlx::query_as!(
Group,
"INSERT INTO usr_to_group (workspace_id, usr, group_) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING",
@@ -2217,8 +2183,6 @@ struct ScriptMetadata {
pub restart_unless_cancelled: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub visible_to_runner_only: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub no_main_func: Option<bool>,
}
pub fn is_none_or_false(val: &Option<bool>) -> bool {
@@ -2495,7 +2459,6 @@ async fn tarball_workspace(
delete_after_use: script.delete_after_use,
restart_unless_cancelled: script.restart_unless_cancelled,
visible_to_runner_only: script.visible_to_runner_only,
no_main_func: script.no_main_func,
};
let metadata_str = serde_json::to_string_pretty(&metadata).unwrap();
archive

View File

@@ -243,8 +243,6 @@ pub struct CompletedJob {
pub tag: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub priority: Option<i16>,
#[serde(skip_serializing_if = "Option::is_none")]
pub labels: Option<serde_json::Value>,
}
impl CompletedJob {

View File

@@ -175,8 +175,6 @@ pub struct Script {
pub concurrency_key: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub visible_to_runner_only: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub no_main_func: Option<bool>,
}
#[derive(Serialize, sqlx::FromRow)]
@@ -196,8 +194,6 @@ pub struct ListableScript {
pub draft_only: Option<bool>,
pub has_deploy_errors: bool,
pub ws_error_handler_muted: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub no_main_func: Option<bool>,
}
#[derive(Serialize)]
@@ -255,7 +251,6 @@ pub struct NewScript {
#[serde(skip_serializing_if = "Option::is_none")]
pub concurrency_key: Option<String>,
pub visible_to_runner_only: Option<bool>,
pub no_main_func: Option<bool>,
}
fn lock_deserialize<'de, D>(deserializer: D) -> Result<Option<String>, D::Error>
@@ -325,7 +320,6 @@ pub struct ListScriptQuery {
pub is_template: Option<bool>,
pub kinds: Option<String>,
pub starred_only: Option<bool>,
pub hide_without_main: Option<bool>,
}
pub fn to_i64(s: &str) -> crate::error::Result<i64> {

View File

@@ -53,13 +53,7 @@ pub fn initialize_tracing() {
match json_fmt {
true => ts_base.with(json_layer().flatten_event(true)).init(),
false => ts_base
.with(
compact_layer()
.with_ansi(style.to_lowercase() != "never")
.with_file(true)
.with_line_number(true)
.with_target(false),
)
.with(compact_layer().with_ansi(style.to_lowercase() != "never"))
.init(),
}
}

View File

@@ -60,7 +60,7 @@ use windmill_common::{
oauth2::WORKSPACE_SLACK_BOT_TOKEN_PATH,
schedule::Schedule,
scripts::{ScriptHash, ScriptLang},
users::{SUPERADMIN_NOTIFICATION_EMAIL, SUPERADMIN_SECRET_EMAIL, SUPERADMIN_SYNC_EMAIL},
users::{SUPERADMIN_NOTIFICATION_EMAIL, SUPERADMIN_SECRET_EMAIL},
worker::{to_raw_value, DEFAULT_TAGS_PER_WORKSPACE, NO_LOGS, WORKER_CONFIG},
DB, METRICS_ENABLED,
};
@@ -612,10 +612,8 @@ pub async fn add_completed_job<
}
} else {
if queued_job.schedule_path.is_some() && queued_job.script_path.is_some() {
let schedule_handlers_tx: QueueTransaction<'_, R> =
(rsmq.clone(), db.begin().await?).into();
match apply_schedule_handlers(
schedule_handlers_tx,
(skip_downstream_error_handlers, tx) = apply_schedule_handlers(
tx,
db,
queued_job.schedule_path.as_ref().unwrap(),
queued_job.script_path.as_ref().unwrap(),
@@ -626,30 +624,21 @@ pub async fn add_completed_job<
queued_job.started_at.unwrap_or(chrono::Utc::now()),
queued_job.priority,
)
.await
{
Ok((skip, mut schedule_handlers_tx)) => {
skip_downstream_error_handlers = skip;
if !queued_job.is_flow() {
// script only
schedule_handlers_tx = handle_maybe_scheduled_job(
schedule_handlers_tx,
db,
queued_job.schedule_path.as_ref().unwrap(),
queued_job.script_path.as_ref().unwrap(),
&queued_job.workspace_id,
)
.await?;
}
schedule_handlers_tx.commit().await?;
}
Err(err) => {
skip_downstream_error_handlers = true;
tracing::error!("Could not apply schedule handlers with error: {}", err);
}
};
.await?;
}
if !queued_job.is_flow()
&& queued_job.schedule_path.is_some()
&& queued_job.script_path.is_some()
{
// script only
tx = handle_maybe_scheduled_job(
tx,
db,
queued_job.schedule_path.as_ref().unwrap(),
queued_job.script_path.as_ref().unwrap(),
&queued_job.workspace_id,
)
.await?;
}
}
if queued_job.concurrent_limit.is_some() {
@@ -1231,13 +1220,13 @@ async fn apply_schedule_handlers<
}
Err(err) => {
sqlx::query!(
"UPDATE schedule SET enabled = false, error = $1 WHERE workspace_id = $2 AND path = $3",
format!("Could not trigger error handler: {err}"),
&schedule.workspace_id,
&schedule.path
)
.execute(db)
.await?;
"UPDATE schedule SET enabled = false, error = $1 WHERE workspace_id = $2 AND path = $3",
format!("Could not trigger error handler: {err}"),
&schedule.workspace_id,
&schedule.path
)
.execute(db)
.await?;
tracing::warn!(
"Could not trigger error handler for {}: {}",
schedule_path,
@@ -1603,11 +1592,11 @@ pub async fn pull<R: rsmq_async::RsmqConnection + Send + Clone>(
))
})?;
let min_started_at = sqlx::query!(
let min_started_at = sqlx::query_scalar!(
"SELECT COALESCE((SELECT MIN(started_at) as min_started_at
FROM queue
WHERE script_path = $1 AND job_kind != 'dependencies' AND running = true AND workspace_id = $2 AND canceled = false
GROUP BY script_path), $3) as min_started_at, now() AS now",
GROUP BY script_path), $3)",
job_script_path,
&pulled_job.workspace_id,
completed_count.max_ended_at
@@ -1662,17 +1651,11 @@ pub async fn pull<R: rsmq_async::RsmqConnection + Send + Clone>(
.await?;
// optimal scheduling is: 'older_job_in_concurrency_time_window_started_timestamp + script_avg_duration + concurrency_time_window_s'
let estimated_next_schedule_timestamp = ((min_started_at
.min_started_at
.unwrap_or(min_started_at.now.unwrap()))
let estimated_next_schedule_timestamp = min_started_at.unwrap_or(pulled_job.scheduled_for)
+ Duration::try_seconds(avg_script_duration.map(i64::from).unwrap_or(0))
.unwrap_or_default()
.max(Duration::try_seconds(5).unwrap_or_default())
+ Duration::try_seconds(i64::from(job_custom_concurrency_time_window_s))
.unwrap_or_default()
.max(Duration::try_seconds(5).unwrap_or_default()))
.max(min_started_at.now.unwrap() + Duration::try_seconds(10).unwrap_or_default());
.unwrap_or_default();
tracing::info!("Job '{}' from path '{}' with concurrency key '{}' has reached its concurrency limit of {} jobs run in the last {} seconds. This job will be re-queued for next execution at {}",
job_uuid, job_script_path, job_concurrency_key, job_custom_concurrent_limit, job_custom_concurrency_time_window_s, estimated_next_schedule_timestamp);
@@ -2076,7 +2059,7 @@ async fn get_result_by_id_from_original_flow(
json_path: Option<String>,
) -> error::Result<Box<RawValue>> {
let flow_job = sqlx::query_as::<_, CompletedJob>(
"SELECT *, null as labels FROM completed_job WHERE id = $1 AND workspace_id = $2",
"SELECT * FROM completed_job WHERE id = $1 AND workspace_id = $2",
)
.bind(completed_flow_id)
.bind(w_id)
@@ -2671,70 +2654,26 @@ pub async fn push<'c, T: Serialize + Send + Sync, R: rsmq_async::RsmqConnection
.unwrap_or(false);
if !is_super_admin {
if email != ERROR_HANDLER_USER_EMAIL
&& email != "worker@windmill.dev"
&& email != SUPERADMIN_SECRET_EMAIL
&& email != SUPERADMIN_SYNC_EMAIL
&& email != SUPERADMIN_NOTIFICATION_EMAIL
{
let user_usage = if let Some(user_usage) = user_usage {
user_usage
} else {
sqlx::query_scalar!(
"SELECT usage.usage + 1 FROM usage
WHERE is_workspace IS FALSE AND
month_ = EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date)
AND id = $1",
email
)
.fetch_optional(_db)
.await?
.flatten()
.unwrap_or(1)
};
if user_usage > MAX_FREE_EXECS
&& !matches!(job_payload, JobPayload::Dependencies { .. })
&& !matches!(job_payload, JobPayload::FlowDependencies { .. })
&& !matches!(job_payload, JobPayload::AppDependencies { .. })
{
return Err(error::Error::BadRequest(format!(
"User {email} has exceeded the free usage limit of {MAX_FREE_EXECS} that applies outside of premium workspaces."
)));
}
let in_queue =
sqlx::query_scalar!("SELECT COUNT(id) FROM queue WHERE email = $1", email)
.fetch_one(_db)
.await?
.unwrap_or(0);
if in_queue > MAX_FREE_EXECS.into() {
return Err(error::Error::BadRequest(format!(
"User {email} has exceeded the jobs in queue limit of {MAX_FREE_EXECS} that applies outside of premium workspaces."
)));
}
let concurrent_runs = sqlx::query_scalar!(
"SELECT COUNT(id) FROM queue WHERE running = true AND email = $1",
let user_usage = if let Some(user_usage) = user_usage {
user_usage
} else {
sqlx::query_scalar!(
"SELECT usage.usage + 1 FROM usage
WHERE is_workspace IS FALSE AND
month_ = EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date)
AND id = $1",
email
)
.fetch_one(_db)
.fetch_optional(_db)
.await?
.unwrap_or(0);
.flatten()
.unwrap_or(1)
};
if concurrent_runs > MAX_FREE_CONCURRENT_RUNS.into() {
return Err(error::Error::BadRequest(format!(
"User {email} has exceeded the concurrent runs limit of {MAX_FREE_CONCURRENT_RUNS} that applies outside of premium workspaces."
)));
}
}
if workspace_id != "demo" {
let workspace_usage = if let Some(workspace_usage) = workspace_usage {
workspace_usage
} else {
sqlx::query_scalar!(
let workspace_usage = if let Some(workspace_usage) = workspace_usage {
workspace_usage
} else {
sqlx::query_scalar!(
"SELECT usage.usage + 1 FROM usage
WHERE is_workspace IS TRUE AND
month_ = EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date)
@@ -2745,45 +2684,79 @@ pub async fn push<'c, T: Serialize + Send + Sync, R: rsmq_async::RsmqConnection
.await?
.flatten()
.unwrap_or(1)
};
};
if workspace_usage > MAX_FREE_EXECS
&& !matches!(job_payload, JobPayload::Dependencies { .. })
&& !matches!(job_payload, JobPayload::FlowDependencies { .. })
&& !matches!(job_payload, JobPayload::AppDependencies { .. })
{
return Err(error::Error::BadRequest(format!(
"Workspace {workspace_id} has exceeded the free usage limit of {MAX_FREE_EXECS} that applies outside of premium workspaces."
)));
}
if user_usage > MAX_FREE_EXECS
&& !matches!(job_payload, JobPayload::Dependencies { .. })
&& !matches!(job_payload, JobPayload::FlowDependencies { .. })
&& !matches!(job_payload, JobPayload::AppDependencies { .. })
{
return Err(error::Error::BadRequest(format!(
"User {email} has exceeded the free usage limit of {MAX_FREE_EXECS} that applies outside of premium workspaces."
)));
}
if workspace_id != "demo"
&& workspace_usage > MAX_FREE_EXECS
&& !matches!(job_payload, JobPayload::Dependencies { .. })
&& !matches!(job_payload, JobPayload::FlowDependencies { .. })
&& !matches!(job_payload, JobPayload::AppDependencies { .. })
{
return Err(error::Error::BadRequest(format!(
"Workspace {workspace_id} has exceeded the free usage limit of {MAX_FREE_EXECS} that applies outside of premium workspaces."
)));
}
let in_queue =
sqlx::query_scalar!("SELECT COUNT(id) FROM queue WHERE email = $1", email)
.fetch_one(_db)
.await?
.unwrap_or(0);
let in_queue_workspace = sqlx::query_scalar!(
"SELECT COUNT(id) FROM queue WHERE workspace_id = $1",
workspace_id
)
.fetch_one(_db)
.await?
.unwrap_or(0);
if in_queue > MAX_FREE_EXECS.into() {
return Err(error::Error::BadRequest(format!(
"User {email} has exceeded the jobs in queue limit of {MAX_FREE_EXECS} that applies outside of premium workspaces."
)));
}
if in_queue_workspace > MAX_FREE_EXECS.into() {
return Err(error::Error::BadRequest(format!(
"Workspace {workspace_id} has exceeded the jobs in queue limit of {MAX_FREE_EXECS} that applies outside of premium workspaces."
)));
}
let in_queue_workspace = sqlx::query_scalar!(
"SELECT COUNT(id) FROM queue WHERE workspace_id = $1",
workspace_id
)
.fetch_one(_db)
.await?
.unwrap_or(0);
let concurrent_runs_workspace = sqlx::query_scalar!(
"SELECT COUNT(id) FROM queue WHERE running = true AND workspace_id = $1",
workspace_id
)
.fetch_one(_db)
.await?
.unwrap_or(0);
if in_queue_workspace > MAX_FREE_EXECS.into() {
return Err(error::Error::BadRequest(format!(
"Workspace {workspace_id} has exceeded the jobs in queue limit of {MAX_FREE_EXECS} that applies outside of premium workspaces."
)));
}
if concurrent_runs_workspace > MAX_FREE_CONCURRENT_RUNS.into() {
return Err(error::Error::BadRequest(format!(
"Workspace {workspace_id} has exceeded the concurrent runs limit of {MAX_FREE_CONCURRENT_RUNS} that applies outside of premium workspaces."
)));
}
let concurrent_runs = sqlx::query_scalar!(
"SELECT COUNT(id) FROM queue WHERE running = true AND email = $1",
email
)
.fetch_one(_db)
.await?
.unwrap_or(0);
if concurrent_runs > MAX_FREE_CONCURRENT_RUNS.into() {
return Err(error::Error::BadRequest(format!(
"User {email} has exceeded the concurrent runs limit of {MAX_FREE_CONCURRENT_RUNS} that applies outside of premium workspaces."
)));
}
let concurrent_runs_workspace = sqlx::query_scalar!(
"SELECT COUNT(id) FROM queue WHERE running = true AND workspace_id = $1",
workspace_id
)
.fetch_one(_db)
.await?
.unwrap_or(0);
if concurrent_runs_workspace > MAX_FREE_CONCURRENT_RUNS.into() {
return Err(error::Error::BadRequest(format!(
"Workspace {workspace_id} has exceeded the concurrent runs limit of {MAX_FREE_CONCURRENT_RUNS} that applies outside of premium workspaces."
)));
}
}
}
@@ -3469,7 +3442,7 @@ async fn restarted_flows_resolution(
Error,
> {
let completed_job = sqlx::query_as::<_, CompletedJob>(
"SELECT *, null as labels FROM completed_job WHERE id = $1 and workspace_id = $2",
"SELECT * FROM completed_job WHERE id = $1 and workspace_id = $2",
)
.bind(completed_flow_id)
.bind(workspace_id)

View File

@@ -983,7 +983,6 @@ plugin(p)
jobs_rx,
worker_name,
db,
script_path,
)
.await
}

View File

@@ -453,26 +453,6 @@ async fn get_mem_peak(pid: Option<u32>, nsjail: bool) -> i32 {
}
}
pub fn sizeof_val(v: &serde_json::Value) -> usize {
std::mem::size_of::<serde_json::Value>()
+ match v {
serde_json::Value::Null => 0,
serde_json::Value::Bool(_) => 0,
serde_json::Value::Number(_) => 4, // Incorrect if arbitrary_precision is enabled. oh well
serde_json::Value::String(s) => s.capacity(),
serde_json::Value::Array(a) => a.iter().map(sizeof_val).sum(),
serde_json::Value::Object(o) => o
.iter()
.map(|(k, v)| {
std::mem::size_of::<String>()
+ k.capacity()
+ sizeof_val(v)
+ std::mem::size_of::<usize>() * 3
})
.sum(),
}
}
pub async fn run_future_with_polling_update_job_poller<Fut, T>(
job_id: Uuid,
timeout: Option<i32>,
@@ -665,20 +645,12 @@ async fn compact_logs(
let (excess_prev_logs, current_logs) = if extra_split {
let split_idx = nlogs
.char_indices()
.nth(excess_size)
.nth(excess_size_modulo)
.map(|(i, _)| i)
.unwrap_or(0);
let (excess_prev_logs, current_logs) = nlogs.split_at(split_idx);
// tracing::error!(
// "{:?} {:?} {} {}",
// excess_prev_logs.lines().last(),
// current_logs.lines().next(),
// split_idx,
// excess_size_modulo
// );
(excess_prev_logs, current_logs.to_string())
} else {
// tracing::error!("{:?}", nlogs.lines().last());
("", nlogs.to_string())
};
@@ -696,7 +668,7 @@ async fn compact_logs(
let mut new_current_logs = match compact_kind {
CompactLogs::NoS3 => format!("[windmill] worker {worker_name}: Logs length has exceeded a threshold\n[windmill] Previous logs have been saved to disk at {path}, add object storage in the instance settings to save it on distributed storage and allow direct download from Windmill\n"),
CompactLogs::S3 => format!("[windmill] Previous logs have been saved to object storage at {path}\n"),
CompactLogs::S3 => format!("[windmill] worker {worker_name}: Logs length has exceeded a threshold\n[windmill] Previous logs have been saved to object storage at {path}\n[windmill] Download logs in expanded drawer to get full logs.\n"),
CompactLogs::NotEE => format!("[windmill] worker {worker_name}: Logs length has exceeded a threshold\n[windmill] Previous logs have been saved to disk at {path}\n[windmill] Upgrade to EE and add object storage to save it persistentely on distributed storage and allow direct download from Windmill\n"),
};
new_current_logs.push_str(&current_logs);

View File

@@ -58,7 +58,6 @@ pub async fn handle_dedicated_process(
mut jobs_rx: Receiver<Arc<QueuedJob>>,
worker_name: &str,
db: &DB,
script_path: &str,
) -> std::result::Result<(), error::Error> {
//do not cache local dependencies
let mut child = {
@@ -129,7 +128,6 @@ pub async fn handle_dedicated_process(
},
line = err_reader.next_line() => {
if let Some(line) = line.expect("line is ok") {
tracing::debug!("stderr dedicated worker: {line}");
logs.push_str("[stderr] ");
logs.push_str(&line);
logs.push_str("\n");
@@ -149,7 +147,6 @@ pub async fn handle_dedicated_process(
tracing::debug!("processed job: {line}");
if line.starts_with("wm_res[") {
let job: Arc<QueuedJob> = jobs.pop_front().expect("pop");
tracing::info!("job completed on dedicated worker {script_path}: {}", job.id);
match serde_json::from_str::<Box<serde_json::value::RawValue>>(&line.replace("wm_res[success]:", "").replace("wm_res[error]:", "")) {
Ok(result) => {
append_logs(job.id, job.workspace_id.clone(), logs.clone(), db).await;
@@ -177,9 +174,8 @@ pub async fn handle_dedicated_process(
job = conditional_polling(jobs_rx.recv(), alive && jobs.len() < MAX_BUFFERED_DEDICATED_JOBS) => {
// i += 1;
if let Some(job) = job {
tracing::debug!("received job");
jobs.push_back(job.clone());
tracing::info!("received job and adding to queue on dedicated worker for {script_path}: {} (queue_size: {})", job.id, jobs.len());
// write_stdin(&mut stdin, &serde_json::to_string(&job.args.unwrap_or_else(|| serde_json::json!({"x": job.id}))).expect("serialize")).await?;
write_stdin(&mut stdin, &serde_json::to_string(&job.args).expect("serialize")).await?;
stdin.flush().await.context("stdin flush")?;

View File

@@ -527,7 +527,6 @@ for await (const chunk of Deno.stdin.readable) {{
jobs_rx,
worker_name,
db,
script_path,
)
.await
}

View File

@@ -71,58 +71,46 @@ pub async fn build_tar_and_push(
Ok(())
}
#[cfg(all(feature = "enterprise", feature = "parquet"))]
pub async fn attempt_fetch_bytes(
client: Arc<dyn ObjectStore>,
path: &str,
) -> error::Result<bytes::Bytes> {
use object_store::path::Path;
let object = client.get(&Path::from(path.clone())).await;
if let Err(e) = object {
tracing::info!(
"Failed to pull bytes from object store at path {path}. Error: {:?}",
e
);
return Err(error::Error::ExecutionErr(format!(
"Failed to pull bytes from object store: {path}"
)));
}
let bytes = object.unwrap().bytes().await;
if bytes.is_err() {
tracing::info!(
"Failed to read bytes from object store: {path}. Error: {:?}",
bytes.err()
);
return Err(error::Error::ExecutionErr(format!(
"Failed to read bytes from object store: {path}"
)));
}
let bytes = bytes.unwrap();
tracing::info!("{path} len: {}", bytes.len());
if bytes.len() == 0 {
tracing::info!("object {path} not found in bucket, bytes empty",);
return Err(error::Error::ExecutionErr(format!(
"object {path} does not exist in bucket"
)));
}
return Ok(bytes);
}
#[cfg(all(feature = "enterprise", feature = "parquet"))]
pub async fn pull_from_tar(client: Arc<dyn ObjectStore>, folder: String) -> error::Result<()> {
use object_store::path::Path;
let folder_name = folder.split("/").last().unwrap();
tracing::info!("Attempting to pull piptar {folder_name} from bucket");
let start = Instant::now();
let tar_path = format!("tar/pip/{folder_name}.tar");
let bytes = attempt_fetch_bytes(client, &tar_path).await?;
let object = client.get(&Path::from(tar_path.clone())).await;
if let Err(e) = object {
tracing::info!("Failed to pull tar from s3: {tar_path}. Error: {:?}", e);
return Err(error::Error::ExecutionErr(format!(
"Failed to pull tar from s3: {tar_path}"
)));
}
let bytes = object.unwrap().bytes().await;
if bytes.is_err() {
tracing::info!(
"Failed to read tar from s3: {tar_path}. Error: {:?}",
bytes.err()
);
return Err(error::Error::ExecutionErr(format!(
"Failed to read tar from s3: {tar_path}"
)));
}
let bytes = bytes.unwrap();
tracing::info!("{tar_path} len: {}", bytes.len());
if bytes.len() == 0 {
tracing::info!(
"piptar {folder_name} not found in bucket. Took {:?}ms",
start.elapsed().as_millis()
);
return Err(error::Error::ExecutionErr(format!(
"tar {folder_name} does not exist in bucket"
)));
}
// tracing::info!("B: {target} {folder}");
extract_pip_tar(bytes, &folder).await.map_err(|e| {

View File

@@ -33,111 +33,6 @@ lazy_static::lazy_static! {
static ref GO_PATH: String = std::env::var("GO_PATH").unwrap_or_else(|_| "/usr/bin/go".to_string());
}
pub async fn save_cache(
bin_path: &str,
job_dir: &str,
_hash: &str,
job: &QueuedJob,
db: &sqlx::Pool<sqlx::Postgres>,
) -> windmill_common::error::Result<()> {
let job_main_path = format!("{job_dir}/main");
let mut _cached_to_s3 = false;
#[cfg(all(feature = "enterprise", feature = "parquet"))]
if let Some(os) = windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS
.read()
.await
.clone()
{
use object_store::path::Path;
let hash_path = hash_to_os_path(_hash);
if let Err(e) = os
.put(
&Path::from(hash_path.clone()),
bytes::Bytes::from(std::fs::read(&job_main_path)?),
)
.await
{
tracing::error!(
"Failed to put go bin to object store: {hash_path}. Error: {:?}",
e
);
} else {
_cached_to_s3 = true;
}
}
if !*CLOUD_HOSTED {
tokio::fs::copy(&job_main_path, bin_path).await?;
append_logs(
job.id.clone(),
job.workspace_id.to_string(),
format!(
"\nwrite cached binary: {} (backed by object store: {_cached_to_s3})\n",
bin_path
),
db,
)
.await;
} else if _cached_to_s3 {
append_logs(
job.id.clone(),
job.workspace_id.to_string(),
format!("write cached binary to object store {}\n", bin_path),
db,
)
.await;
}
Ok(())
}
#[cfg(all(feature = "enterprise", feature = "parquet"))]
async fn write_binary_file(main_path: &str, byts: &mut bytes::Bytes) -> error::Result<()> {
use std::fs::Permissions;
use std::os::unix::fs::PermissionsExt;
use tokio::io::AsyncWriteExt;
let mut file = File::create(main_path).await?;
file.write_buf(byts).await?;
file.set_permissions(Permissions::from_mode(0o755)).await?;
file.flush().await?;
Ok(())
}
#[cfg(all(feature = "enterprise", feature = "parquet"))]
fn hash_to_os_path(hash: &str) -> String {
format!("gobin/{hash}")
}
async fn load_cache(bin_path: &str, _hash: &str) -> (bool, String) {
if tokio::fs::metadata(&bin_path).await.is_ok() {
(true, format!("loaded bin from local cache: {}\n", bin_path))
} else {
#[cfg(all(feature = "enterprise", feature = "parquet"))]
if let Some(os) = windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS
.read()
.await
.clone()
{
use crate::global_cache::attempt_fetch_bytes;
if let Ok(mut x) = attempt_fetch_bytes(os, &hash_to_os_path(_hash)).await {
if let Err(e) = write_binary_file(bin_path, &mut x).await {
tracing::error!("could not write binary file: {e:?}");
return (
false,
"error writing binary file from object store".to_string(),
);
}
tracing::info!("loaded bin from object store {}", bin_path);
return (true, format!("loaded bin from object store {}", bin_path));
}
}
(false, "".to_string())
}
}
#[tracing::instrument(level = "trace", skip_all)]
pub async fn handle_go_job(
mem_peak: &mut i32,
@@ -155,19 +50,21 @@ pub async fn handle_go_job(
) -> Result<Box<RawValue>, Error> {
//go does not like executing modules at temp root
let job_dir = &format!("{job_dir}/go");
let hash = calculate_hash(&format!(
"{}{}v2",
inner_content,
requirements_o
.as_ref()
.map(|x| x.to_string())
.unwrap_or_default()
));
let bin_path = format!("{}/{hash}", GO_BIN_CACHE_DIR,);
let bin_path = format!(
"{}/{}",
GO_BIN_CACHE_DIR,
calculate_hash(&format!(
"{}{}",
inner_content,
requirements_o
.as_ref()
.map(|x| x.to_string())
.unwrap_or_default()
))
);
let bin_exists = !*CLOUD_HOSTED && tokio::fs::metadata(&bin_path).await.is_ok();
let (cache, cache_logs) = load_cache(&bin_path, &hash).await;
let (skip_go_mod, skip_tidy) = if cache {
let (skip_go_mod, skip_tidy) = if bin_exists {
create_dir(job_dir).await?;
(true, true)
} else if let Some(requirements) = requirements_o {
@@ -176,8 +73,8 @@ pub async fn handle_go_job(
(false, false)
};
let cache_logs = if !cache {
let logs1 = format!("{cache_logs}\n\n--- GO DEPENDENCIES SETUP ---\n");
if !bin_exists {
let logs1 = "\n\n--- GO DEPENDENCIES SETUP ---\n".to_string();
append_logs(job.id.clone(), job.workspace_id.to_string(), logs1, db).await;
install_go_dependencies(
@@ -195,6 +92,9 @@ pub async fn handle_go_job(
)
.await?;
let logs2 = "\n\n--- GO CODE EXECUTION ---\n".to_string();
append_logs(job.id.clone(), job.workspace_id.to_string(), logs2, db).await;
create_args_and_out_file(client, job, job_dir, db).await?;
{
let sig = windmill_parser_go::parse_go_sig(&inner_content)?;
@@ -309,24 +209,33 @@ func Run(req Req) (interface{{}}, error){{
)
.await?;
if let Err(e) = save_cache(&bin_path, &job_dir, &hash, &job, db).await {
tracing::error!("could not save {bin_path} to go cache: {e:?}");
if !*CLOUD_HOSTED {
create_dir(&bin_path).await?;
let target = format!("{bin_path}/main");
tokio::fs::copy(format!("{job_dir}/main"), &target).await?;
append_logs(
job.id.clone(),
job.workspace_id.to_string(),
format!("write cached binary: {}\n", bin_path),
db,
)
.await;
}
"".to_string()
} else {
let path = format!("{bin_path}/main");
let mut logs2 = "".to_string();
logs2.push_str(&format!("found cached binary: {path}\n"));
let target = format!("{job_dir}/main");
tokio::fs::symlink(&bin_path, &target).await.map_err(|e| {
tokio::fs::symlink(&path, &target).await.map_err(|e| {
Error::ExecutionErr(format!(
"could not copy cached binary from {bin_path} to {job_dir}/main: {e:?}"
"could not copy cached binary from {path} to {job_dir}/main: {e:?}"
))
})?;
logs2.push_str("\n\n--- GO CODE EXECUTION ---\n");
append_logs(job.id.clone(), job.workspace_id.to_string(), logs2, db).await;
create_args_and_out_file(client, job, job_dir, db).await?;
cache_logs
};
let logs2 = format!("{cache_logs}\n\n--- GO CODE EXECUTION ---\n");
append_logs(job.id.clone(), job.workspace_id.to_string(), logs2, db).await;
}
let client = &client.get_authed().await;
@@ -394,13 +303,6 @@ func Run(req Req) (interface{{}}, error){{
false,
)
.await?;
if cache && *CLOUD_HOSTED {
//do not keep the binary in the cache if it is cloud hosted to avoid filling up the disk
if let Err(e) = tokio::fs::remove_file(&bin_path).await {
tracing::error!("could not remove {bin_path} from go cache: {e:?}");
}
}
read_result(job_dir).await
}

View File

@@ -774,7 +774,6 @@ fn op_get_static_args(op_state: Rc<RefCell<OpState>>) -> Vec<Option<String>> {
#[op2(fast)]
fn op_log(op_state: Rc<RefCell<OpState>>, #[string] log: &str) {
// tracing::error!("log: |{}|", log);
op_state
.borrow_mut()
.borrow_mut::<LogString>()

View File

@@ -1,7 +1,6 @@
use base64::{engine::general_purpose, Engine as _};
use chrono::{DateTime, NaiveDate, NaiveDateTime, NaiveTime, Utc};
use futures::TryFutureExt;
use regex::Regex;
use serde::Deserialize;
use serde_json::value::RawValue;
use serde_json::{Map, Value};
@@ -13,7 +12,7 @@ use windmill_common::error::{self, Error};
use windmill_common::worker::to_raw_value;
use windmill_common::{error::to_anyhow, jobs::QueuedJob};
use windmill_parser_sql::{parse_db_resource, parse_mssql_sig};
use windmill_queue::{append_logs, CanceledBy};
use windmill_queue::CanceledBy;
use crate::common::{build_args_values, run_future_with_polling_update_job_poller};
use crate::AuthedClientBackgroundTask;
@@ -28,10 +27,6 @@ struct MssqlDatabase {
instance_name: Option<String>,
}
lazy_static::lazy_static! {
static ref RE_MSSQL_READONLY_INTENT: Regex = Regex::new(r#"(?mi)^-- ApplicationIntent=ReadOnly *(?:\r|\n|$)"#).unwrap();
}
pub async fn do_mssql(
job: &QueuedJob,
client: &AuthedClientBackgroundTask,
@@ -79,14 +74,6 @@ pub async fn do_mssql(
config.port(port);
}
let readonly_intent = RE_MSSQL_READONLY_INTENT.is_match(query);
config.readonly(readonly_intent);
if readonly_intent {
let logs = format!("\nSetting ApplicationIntent to ReadOnly");
append_logs(job.id, job.workspace_id.clone(), logs, db).await;
}
// Using SQL Server authentication.
config.authentication(AuthMethod::sql_server(database.user, database.password));
config.trust_cert(); // on production, it is not a good idea to do this

View File

@@ -145,16 +145,7 @@ pub async fn do_mysql(
Value::Number(n) if n.is_u64() && arg_t == "uint" => {
mysql_async::Value::UInt(n.as_u64().unwrap())
}
Value::Number(n)
if n.is_f64() && (arg_t == "real" || arg_t == "dec" || arg_t == "fixed") =>
{
n.as_f64().unwrap().into()
}
Value::Number(n)
if n.is_i64() && (arg_t == "real" || arg_t == "dec" || arg_t == "fixed") =>
{
(n.as_i64().unwrap() as f64).into()
}
Value::Number(n) if n.is_f64() && arg_t == "real" => n.as_f64().into(),
value @ _ => {
return Err(Error::ExecutionErr(format!(
"Unsupported type in query: {:?} and signature {arg_t:?}",
@@ -281,9 +272,6 @@ fn convert_mysql_value_to_json(v: mysql_async::Value, c: ColumnType) -> serde_js
ColumnType::MYSQL_TYPE_FLOAT | ColumnType::MYSQL_TYPE_DOUBLE => {
conversion_error(f64::from_value_opt(v))
}
ColumnType::MYSQL_TYPE_DECIMAL | ColumnType::MYSQL_TYPE_NEWDECIMAL => {
conversion_error(rust_decimal::Decimal::from_value_opt(v))
}
ColumnType::MYSQL_TYPE_TINY
| ColumnType::MYSQL_TYPE_SHORT
| ColumnType::MYSQL_TYPE_LONG

View File

@@ -31,8 +31,8 @@ use windmill_parser::Typ;
use windmill_parser_sql::{parse_db_resource, parse_pgsql_sig};
use windmill_queue::CanceledBy;
use crate::common::{build_args_values, run_future_with_polling_update_job_poller, sizeof_val};
use crate::{AuthedClientBackgroundTask, MAX_RESULT_SIZE};
use crate::common::{build_args_values, run_future_with_polling_update_job_poller};
use crate::AuthedClientBackgroundTask;
use bytes::{Buf, BytesMut};
use lazy_static::lazy_static;
use urlencoding::encode;
@@ -230,32 +230,15 @@ pub async fn do_postgresql(
.unwrap_or_default(),
);
let mut siz = 0;
let mut res: Vec<serde_json::Value> = vec![];
for row in rows.into_iter() {
let r = postgres_row_to_json_value(row);
if let Ok(v) = r.as_ref() {
let size = sizeof_val(v);
siz += size;
}
if *CLOUD_HOSTED && siz > MAX_RESULT_SIZE * 4 {
return Err(anyhow::anyhow!(
"Query result too large for cloud (size = {} > {})",
siz,
MAX_RESULT_SIZE & 4
));
}
if let Ok(v) = r {
res.push(v);
} else {
return Err(to_anyhow(r.err().unwrap()));
}
}
let result = rows
.into_iter()
.map(|x: Row| postgres_row_to_json_value(x))
.collect::<Result<Vec<_>, _>>()?;
Ok((res, siz))
Ok(result)
};
let (result, size) = run_future_with_polling_update_job_poller(
let result = run_future_with_polling_update_job_poller(
job.id,
job.timeout,
db,
@@ -267,8 +250,6 @@ pub async fn do_postgresql(
)
.await?;
*mem_peak = size as i32;
RUNNING.store(false, std::sync::atomic::Ordering::Relaxed);
if let Some(handle) = handle {
@@ -342,7 +323,6 @@ enum PgType {
Timestamp(chrono::NaiveDateTime),
None(Option<bool>),
Array(Vec<PgType>),
Json(serde_json::Value),
}
impl ToSql for PgType {
@@ -368,7 +348,6 @@ impl ToSql for PgType {
PgType::Timestamp(ref val) => val.to_sql(ty, out),
PgType::None(ref val) => val.to_sql(ty, out),
PgType::Array(ref val) => val.to_sql(ty, out),
PgType::Json(ref val) => val.to_sql(ty, out),
}
}
@@ -431,7 +410,7 @@ fn convert_val(value: &Value, arg_t: &String, typ: &Typ) -> windmill_common::err
chrono::NaiveDate::parse_from_str(s, "%Y-%m-%dT%H:%M:%S.%3fZ").unwrap_or_default();
Ok(PgType::Date(date))
}
Value::String(s) if arg_t == "time" || arg_t == "timetz" => {
Value::String(s) if arg_t == "time" => {
let time =
chrono::NaiveTime::parse_from_str(s, "%Y-%m-%dT%H:%M:%S.%3fZ").unwrap_or_default();
Ok(PgType::Time(time))
@@ -441,7 +420,6 @@ fn convert_val(value: &Value, arg_t: &String, typ: &Typ) -> windmill_common::err
.unwrap_or_default();
Ok(PgType::Timestamp(datetime))
}
Value::Object(_) => Ok(PgType::Json(value.clone())),
Value::String(s) => Ok(PgType::String(s.clone())),
_ => Err(Error::ExecutionErr(format!(
"Unsupported type in query: {:?} and signature {arg_t:?}",

View File

@@ -391,6 +391,8 @@ mount {{
}
tracing::info!(
worker_name = %worker_name,
job_id = %job.id,
workspace_id = %job.workspace_id,
"started python code execution {}",
job.id
@@ -472,13 +474,7 @@ async fn prepare_wrapper(
let relative_imports = RELATIVE_IMPORT_REGEX.is_match(&inner_content);
let script_path_splitted = script_path.split("/").map(|x| {
if x.starts_with(|x: char| x.is_ascii_digit()) {
format!("_{}", x)
} else {
x.to_string()
}
});
let script_path_splitted = script_path.split("/");
let dirs_full = script_path_splitted
.clone()
.take(script_path_splitted.clone().count() - 1)
@@ -503,6 +499,12 @@ async fn prepare_wrapper(
let module_dir = format!("{}/{}", job_dir, dirs);
tokio::fs::create_dir_all(format!("{module_dir}/")).await?;
let last = if last.starts_with(|x: char| x.is_ascii_digit()) {
format!("_{}", last)
} else {
last
};
let _ = write_file(&module_dir, &format!("{last}.py"), inner_content).await?;
if relative_imports {
let _ = write_file(job_dir, "loader.py", RELATIVE_PYTHON_LOADER).await?;
@@ -612,7 +614,7 @@ async fn replace_pip_secret(
}
let secret = get_secret_value_as_admin(db, w_id, variable).await?;
tracing::info!(
worker = %worker_name,
worker_name = %worker_name,
job_id = %job_id,
workspace_id = %w_id,
"found secret variable in pip requirements: {}",
@@ -876,12 +878,16 @@ pub async fn handle_python_reqs(
append_logs(job_id.clone(), w_id.to_string(), logs1, db).await;
tracing::info!(
worker_name = %worker_name,
job_id = %job_id,
workspace_id = %w_id,
"started setup python dependencies"
);
let child = if !*DISABLE_NSJAIL {
tracing::info!(
worker_name = %worker_name,
job_id = %job_id,
workspace_id = %w_id,
"starting nsjail"
);
@@ -986,6 +992,8 @@ pub async fn handle_python_reqs(
)
.await;
tracing::info!(
worker_name = %worker_name,
job_id = %job_id,
workspace_id = %w_id,
is_ok = child.is_ok(),
"finished setting up python dependencies {}",
@@ -1193,7 +1201,6 @@ for line in sys.stdin:
jobs_rx,
worker_name,
db,
script_path,
)
.await
}

View File

@@ -34,7 +34,7 @@ globalThis.URLSearchParams = url.URLSearchParams;
globalThis.Headers = headers.Headers;
globalThis.FileReader = fileReader.FileReader;
globalThis.console = new console.Console((msg, level) =>
globalThis.Deno.core.ops.op_log(msg)
globalThis.Deno.core.ops.op_log([msg])
);
// Object.assign(globalThis, {
// console: nonEnumerable(

View File

@@ -13,7 +13,6 @@ use prometheus::{
core::{AtomicI64, GenericGauge},
IntCounter,
};
use tracing::Instrument;
#[cfg(feature = "prometheus")]
use windmill_common::METRICS_DEBUG_ENABLED;
#[cfg(feature = "prometheus")]
@@ -89,8 +88,8 @@ use crate::{
bash_executor::{handle_bash_job, handle_powershell_job, ANSI_ESCAPE_RE},
bun_executor::{gen_lockfile, get_trusted_deps, handle_bun_job},
common::{
build_args_map, get_cached_resource_value_if_valid, get_reserved_variables, hash_args,
read_result, save_in_cache, write_file, NO_LOGS_AT_ALL, SLOW_LOGS,
build_args_map, get_cached_resource_value_if_valid, hash_args, read_result, save_in_cache,
write_file, NO_LOGS_AT_ALL, SLOW_LOGS,
},
deno_executor::{generate_deno_lock, handle_deno_job},
go_executor::{handle_go_job, install_go_dependencies},
@@ -585,7 +584,6 @@ impl JobCompletedSender {
}
}
#[tracing::instrument(name = "worker", level = "info", skip_all, fields(worker = %worker_name))]
pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 'static>(
db: &Pool<Postgres>,
worker_instance: &str,
@@ -610,7 +608,7 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
let start_time = Instant::now();
let worker_dir = format!("{TMP_DIR}/{worker_name}");
tracing::debug!(worker_dir = %worker_dir, "Creating worker dir");
tracing::debug!(worker_dir = %worker_dir, worker_name = %worker_name, "Creating worker dir");
if let Some(ref netrc) = *NETRC {
tracing::info!("Writing netrc at {}/.netrc", HOME_ENV.as_str());
@@ -1024,7 +1022,7 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
let worker_name2 = worker_name.clone();
let killpill_tx2 = killpill_tx.clone();
let job_completed_sender = job_completed_tx.0.clone();
let send_result = tokio::spawn((async move {
let send_result = tokio::spawn(async move {
while let Some(sr) = job_completed_rx.recv().await {
match sr {
SendResult::JobCompleted(jc) => {
@@ -1202,7 +1200,7 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
tokio::time::sleep(Duration::from_millis(50)).await;
}
tracing::info!("finished processing all completed jobs");
}).instrument(tracing::Span::current()));
});
let mut last_executed_job: Option<Instant> = None;
let mut last_checked_suspended = Instant::now();
@@ -1216,11 +1214,7 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
let vacuum_shift = rand::thread_rng().gen_range(0..VACUUM_PERIOD);
IS_READY.store(true, Ordering::Relaxed);
tracing::info!(
"listening for jobs, WORKER_GROUP: {}, config: {:?}",
*WORKER_GROUP,
WORKER_CONFIG.read().await
);
tracing::info!(worker = %worker_name, "listening for jobs, WORKER_GROUP: {}, config: {:?}", *WORKER_GROUP, WORKER_CONFIG.read().await);
// (dedi_path, dedicated_worker_tx, dedicated_worker_handle)
// Option<Sender<Arc<QueuedJob>>>,
@@ -1384,27 +1378,24 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
tracing::error!("failed to update worker ping, exiting: {}", e);
killpill_tx.send(()).unwrap_or_default();
}
tracing::info!("updating last ping");
tracing::debug!("set last ping");
last_ping = Instant::now();
}
if (jobs_executed as u32 + vacuum_shift) % VACUUM_PERIOD == 0 {
let db2 = db.clone();
let current_span = tracing::Span::current();
tokio::task::spawn(
(async move {
tracing::info!("vacuuming queue and completed_job");
if let Err(e) = sqlx::query!("VACUUM (skip_locked) queue")
.execute(&db2)
.await
{
tracing::error!("failed to vacuum queue: {}", e);
}
tracing::info!("vacuumed queue and completed_job");
})
.instrument(current_span),
);
let worker_name2 = worker_name.clone();
tokio::task::spawn(async move {
tracing::info!(worker = %worker_name2, "vacuuming queue and completed_job");
if let Err(e) = sqlx::query!("VACUUM (skip_locked) queue")
.execute(&db2)
.await
{
tracing::error!(worker = %worker_name2, "failed to vacuum queue: {}", e);
}
tracing::info!(worker = %worker_name2, "vacuumed queue and completed_job");
});
jobs_executed += 1;
}
@@ -1508,7 +1499,7 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
last_executed_job = None;
jobs_executed += 1;
tracing::debug!("started handling of job {}", job.id);
tracing::debug!(worker = %worker_name, "started handling of job {}", job.id);
if matches!(job.job_kind, JobKind::Script | JobKind::Preview) {
if !dedicated_workers.is_empty() {
@@ -1620,12 +1611,10 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
.unwrap_or_else(|| "none".to_string());
if job.id == Uuid::nil() {
tracing::info!("running warmup job");
tracing::info!(worker = %worker_name, "running warmup job");
} else {
tracing::info!(workspace_id = %job.workspace_id, job_id = %job.id, root_id = %job_root, "fetched job {}, root job: {}", job.id, job_root);
} // Here we can't remove the job id, but maybe with the
// fields macro we can make a job id that only appears when
// the job is defined?
tracing::info!(worker = %worker_name, workspace_id = %job.workspace_id, id = %job.id, root_id = %job_root, "fetched job {}, root job: {}", job.id, job_root);
}
let job_dir = format!("{worker_dir}/{}", job.id);
@@ -1750,7 +1739,7 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
if let Some(secs) = *EXIT_AFTER_NO_JOB_FOR_SECS {
if let Some(lj) = last_executed_job {
if lj.elapsed().as_secs() > secs {
tracing::info!("no job for {} seconds, exiting", secs);
tracing::info!(worker = %worker_name, "no job for {} seconds, exiting", secs);
break;
}
} else {
@@ -1779,7 +1768,7 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
});
}
Err(err) => {
tracing::error!("Failed to pull jobs: {}", err);
tracing::error!(worker = %worker_name, "Failed to pull jobs: {}", err);
}
};
}
@@ -1808,6 +1797,7 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
send_result.await.expect("send result failed");
tracing::info!("worker {} exited", worker_name);
println!("worker {} exited", worker_name);
}
type DedicatedWorker = (String, Sender<Arc<QueuedJob>>, Option<JoinHandle<()>>);
@@ -2155,9 +2145,6 @@ async fn spawn_dedicated_worker(
} {
tracing::error!("error in dedicated worker: {:?}", e);
};
if let Err(e) = killpill_tx.clone().send(()) {
tracing::error!("failed to send final killpill to dedicated worker: {:?}", e);
}
});
return Some((node_id.unwrap_or(path2), dedicated_worker_tx, Some(handle)));
// (Some(dedi_path), Some(dedicated_worker_tx), Some(handle))
@@ -2231,7 +2218,6 @@ async fn queue_init_bash_maybe<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
// logs: String,
// ) -> error::Result<()> {
#[tracing::instrument(name = "completed_job", level = "info", skip_all, fields(job_id = %job.id))]
pub async fn process_completed_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
JobCompleted { job, result, mem_peak, success, cached_res_path, canceled_by, .. }: JobCompleted,
client: &AuthedClient,
@@ -2373,8 +2359,6 @@ pub async fn process_completed_job<R: rsmq_async::RsmqConnection + Send + Sync +
// *barrier = None;
// tracing::debug!("leader worker done waiting for");
// }
#[tracing::instrument(name = "job_error", level = "info", skip_all, fields(job_id = %job.id))]
pub async fn handle_job_error<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
db: &Pool<Postgres>,
client: &AuthedClient,
@@ -2570,7 +2554,7 @@ pub struct PreviousResult<'a> {
pub previous_result: Option<&'a RawValue>,
}
#[tracing::instrument(name = "job", level = "info", skip_all, fields(job_id = %job.id))]
#[tracing::instrument(level = "trace", skip_all)]
async fn handle_queued_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
job: Arc<QueuedJob>,
db: &DB,
@@ -2747,6 +2731,8 @@ async fn handle_queued_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
}
tracing::debug!(
worker = %worker_name,
job_id = %job.id,
workspace_id = %job.workspace_id,
"handling job {}",
job.id
@@ -3206,16 +3192,9 @@ async fn handle_code_execution_job(
db,
)
.await;
let reserved_variables = get_reserved_variables(job, &client.get_token().await, db).await?;
let code = format!(
"const BASE_URL = '{base_internal_url}';\nconst BASE_INTERNAL_URL = '{base_internal_url}';\n{}\n{}",
reserved_variables
.iter()
.map(|(k, v)| format!("const {} = '{}';\n", k, v))
.collect::<Vec<String>>()
.join("\n"),
"const BASE_URL = '{base_internal_url}';\nconst WM_TOKEN = '{}';\n{}",
&client.get_token().await,
inner_content
);
let (result, ts_logs) =
@@ -3231,6 +3210,8 @@ async fn handle_code_execution_job(
.unwrap_or_else(|| "NO_LANG".to_string());
tracing::debug!(
worker_name = %worker_name,
job_id = %job.id,
workspace_id = %job.workspace_id,
"started {} job {}",
&lang_str,
@@ -3365,6 +3346,8 @@ mount {{
_ => panic!("unreachable, language is not supported: {language:#?}"),
};
tracing::info!(
worker_name = %worker_name,
job_id = %job.id,
workspace_id = %job.workspace_id,
is_ok = result.is_ok(),
"finished {} job {}",
@@ -3948,14 +3931,6 @@ async fn lock_modules_app(
.unwrap_or_default()
.to_string();
let mut logs = "".to_string();
if v.get("lock")
.is_some_and(|x| !x.as_str().unwrap().trim().is_empty())
{
logs.push_str(
"Found already locked inline script. Skipping lock...\n",
);
return Ok(Value::Object(m.clone()));
}
logs.push_str("Found lockable inline script. Generating lock...\n");
let new_lock = capture_dependency_job(
&job.id,

View File

@@ -671,7 +671,7 @@ pub async fn update_flow_status_after_job_completion_internal<
.root_job
.map(|x| x.to_string())
.unwrap_or_else(|| "none".to_string());
tracing::info!(id = %flow_job.id, root_id = %job_root, "update flow status");
tracing::info!(id = %flow_job.id, root_id = %job_root, worker_name = %worker_name, "update flow status");
let module = get_module(&flow_job, module_index);
@@ -869,7 +869,7 @@ pub async fn update_flow_status_after_job_completion_internal<
}
if let Some(parent_job) = flow_job.parent_job {
tracing::info!(subflow_id = %flow_job.id, parent_id = %parent_job, "subflow is finished, updating parent flow status");
tracing::info!(subflow_id = %flow_job.id, parent_id = %parent_job, worker_name = %worker_name, "subflow is finished, updating parent flow status");
return Ok(Some(RecUpdateFlowStatusAfterJobCompletion {
flow: parent_job,

View File

@@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts";
import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts";
import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts";
export const VERSION = "v1.322.0";
export const VERSION = "v1.306.2";
export async function login(email: string, password: string): Promise<string> {
return await windmill.UserService.login({

View File

@@ -5,13 +5,15 @@ import {
colors,
Command,
ListableApp,
log,
Policy,
SEP,
Table,
yamlParse,
} from "./deps.ts";
import { GlobalOptions, isSuperset } from "./types.ts";
import {
GlobalOptions,
isSuperset,
parseFromFile,
removeType,
} from "./types.ts";
export interface AppFile {
value: any;
@@ -19,20 +21,14 @@ export interface AppFile {
policy: Policy;
}
const alreadySynced: string[] = [];
export async function pushApp(
workspace: string,
remotePath: string,
localPath: string,
filePath: string,
app: AppFile | undefined,
newApp: AppFile,
message?: string
): Promise<void> {
if (alreadySynced.includes(localPath)) {
return;
}
alreadySynced.push(localPath);
let app: any = undefined;
const remotePath = removeType(filePath, "app");
// deleting old app if it exists in raw mode
try {
app = await AppService.getAppByPath({
@@ -43,61 +39,28 @@ export async function pushApp(
//ignore
}
if (!localPath.endsWith(SEP)) {
localPath += SEP;
}
const localAppRaw = await Deno.readTextFile(localPath + "app.yaml");
const localApp = yamlParse(localAppRaw) as AppFile;
function replaceInlineScripts(rec: any) {
if (!rec) {
return;
}
if (typeof rec == "object") {
return Object.entries(rec).flatMap(([k, v]) => {
if (k == "inlineScript" && typeof v == "object") {
const o: Record<string, any> = v as any;
if (o["content"] && o["content"].startsWith("!inline")) {
const basePath = localPath + o["content"].split(" ")[1];
o["content"] = Deno.readTextFileSync(basePath);
}
if (o["lock"] && o["lock"].startsWith("!inline")) {
const basePath = localPath + o["lock"].split(" ")[1];
o["lock"] = Deno.readTextFileSync(basePath);
}
} else {
replaceInlineScripts(v);
}
});
}
return [];
}
replaceInlineScripts(localApp.value);
if (app) {
if (isSuperset(localApp, app)) {
log.info(colors.green(`App ${remotePath} is up to date`));
if (isSuperset(newApp, app)) {
return;
}
log.info(colors.bold.yellow(`Updating app ${remotePath}...`));
await AppService.updateApp({
workspace,
path: remotePath.replaceAll("\\", "/"),
requestBody: {
deployment_message: message,
...localApp,
...newApp,
},
});
} else {
log.info(colors.yellow.bold("Creating new app..."));
console.log(colors.yellow.bold("Creating new app..."));
console.log(message);
await AppService.createApp({
workspace,
requestBody: {
path: remotePath.replaceAll("\\", "/"),
deployment_message: message,
...localApp,
...newApp,
},
});
}
@@ -131,22 +94,32 @@ async function list(opts: GlobalOptions) {
.render();
}
async function push(opts: GlobalOptions, filePath: string, remotePath: string) {
async function push(opts: GlobalOptions, filePath: string) {
const remotePath = filePath.split(".")[0];
if (!validatePath(remotePath)) {
return;
}
const workspace = await resolveWorkspace(opts);
await requireLogin(opts);
await pushApp(workspace.workspaceId, remotePath, filePath);
log.info(colors.bold.underline.green("Flow pushed"));
<<<<<<< Updated upstream
await pushApp(
workspace.workspaceId,
filePath,
undefined,
parseFromFile(filePath)
);
=======
await pushApp(workspace.workspaceId, filePath, parseFromFile(filePath));
>>>>>>> Stashed changes
console.log(colors.bold.underline.green("App pushed"));
}
const command = new Command()
.description("app related commands")
.action(list as any)
.command("push", "push a local app ")
.arguments("<file_path:string> <remote_path:string>")
.arguments("<file_path:file>")
.action(push as any);
export default command;

View File

@@ -1,5 +1,6 @@
// windmill
export * from "npm:windmill-client@1.319.1";
export { setClient } from "npm:windmill-client@1.304.2";
export * from "npm:windmill-client@1.304.2";
// cliffy
export { Command } from "https://deno.land/x/cliffy@v1.0.0-rc.4/command/mod.ts";
@@ -27,11 +28,6 @@ export { DelimiterStream } from "https://deno.land/std@0.207.0/streams/mod.ts";
export { iterateReader } from "https://deno.land/std@0.207.0/streams/iterate_reader.ts";
export { writeAllSync } from "https://deno.land/std@0.207.0/streams/mod.ts";
export { encodeHex } from "https://deno.land/std@0.207.0/encoding/hex.ts";
export * as log from "https://deno.land/std@0.207.0/log/mod.ts";
export {
stringify as yamlStringify,
parse as yamlParse,
} from "https://deno.land/std@0.207.0/yaml/mod.ts";
// other
export { Application, Router } from "https://deno.land/x/oak@v12.5.0/mod.ts";
@@ -47,6 +43,10 @@ export { default as microdiff } from "https://deno.land/x/microdiff@v1.3.1/index
export { default as objectHash } from "https://deno.land/x/object_hash@2.0.3.1/mod.ts";
export { minimatch } from "npm:minimatch";
export { default as JSZip } from "npm:jszip@3.7.1";
export * as log from "https://deno.land/std@0.186.0/log/mod.ts";
export {
stringify as yamlStringify,
parse as yamlParse,
} from "https://deno.land/std@0.184.0/yaml/mod.ts";
export { open } from "https://deno.land/x/open@v0.0.5/index.ts";
export { default as gitignore_parser } from "npm:gitignore-parser";

View File

@@ -27,13 +27,13 @@ const alreadySynced: string[] = [];
export async function pushFlow(
workspace: string,
remotePath: string,
localPath: string,
localFlowPath: string,
message?: string
): Promise<void> {
if (alreadySynced.includes(localPath)) {
if (alreadySynced.includes(localFlowPath)) {
return;
}
alreadySynced.push(localPath);
alreadySynced.push(localFlowPath);
let flow: Flow | undefined = undefined;
try {
flow = await FlowService.getFlowByPath({
@@ -44,31 +44,17 @@ export async function pushFlow(
// flow doesn't exist
}
if (!localPath.endsWith(SEP)) {
localPath += SEP;
if (!localFlowPath.endsWith(SEP)) {
localFlowPath += SEP;
}
const localFlowRaw = await Deno.readTextFile(localPath + "flow.yaml");
const localFlowRaw = await Deno.readTextFile(localFlowPath + "flow.yaml");
const localFlow = yamlParse(localFlowRaw) as FlowFile;
function replaceInlineScripts(modules: FlowModule[]) {
modules.forEach((m) => {
if (m.value.type == "rawscript") {
const path = m.value.content.split(" ")[1];
m.value.content = Deno.readTextFileSync(localPath + path);
const lock = m.value.lock;
if (
lock &&
typeof lock == "string" &&
lock.trimStart().startsWith("!inline ")
) {
const path = lock.split(" ")[1];
try {
m.value.lock = Deno.readTextFileSync(localPath + path);
} catch {
log.error(`Lock file ${path} not found`);
}
}
m.value.content = Deno.readTextFileSync(localFlowPath + path);
} else if (m.value.type == "forloopflow") {
replaceInlineScripts(m.value.modules);
} else if (m.value.type == "branchall") {
@@ -208,15 +194,12 @@ async function run(
log.info(jobInfo.result ?? {});
}
export function bootstrap(
opts: GlobalOptions & { summary: string; description: string },
flowPath: string
) {
export function bootstrap(opts: GlobalOptions & {summary: string, description: string}, flowPath: string) {
if (!validatePath(flowPath)) {
return;
}
const flowDirFullPath = `${flowPath}.flow`;
const flowDirFullPath = `${flowPath}.flow`
Deno.mkdirSync(flowDirFullPath, { recursive: false });
const newFlowDefinition = defaultFlowDefinition();
@@ -227,12 +210,13 @@ export function bootstrap(
newFlowDefinition.description = opts.description;
}
const newFlowDefinitionYaml = yamlStringify(
newFlowDefinition as Record<string, any>
);
const newFlowDefinitionYaml = yamlStringify(newFlowDefinition as Record<string, any>);
const flowYamlPath = `${flowDirFullPath}/flow.yaml`;
Deno.writeTextFile(flowYamlPath, newFlowDefinitionYaml, { createNew: true });
Deno.writeTextFile(
flowYamlPath,
newFlowDefinitionYaml,
{ createNew: true });
}
const command = new Command()

View File

@@ -17,16 +17,16 @@ async function pull(opts: GlobalOptions) {
const userInfo = await requireLogin(opts);
const uid = (await SettingService.getGlobal({
const uid = await SettingService.getGlobal({
key: "uid",
})) as string;
});
const hubBaseUrl =
(await SettingService.getGlobal({
key: "hubBaseUrl",
})) ?? "https://hub.windmill.dev";
const headers: Record<string, string> = {
const headers = {
Accept: "application/json",
"X-email": userInfo.email,
};
@@ -99,7 +99,8 @@ async function pull(opts: GlobalOptions) {
workspace.workspaceId,
x.name + ".resource-type.json",
undefined,
x
x,
true
);
}
}

View File

@@ -31,7 +31,7 @@ addEventListener("error", (event) => {
}
});
export const VERSION = "v1.322.0";
export const VERSION = "v1.306.2";
let command: any = new Command()
.name("wmill")

View File

@@ -178,16 +178,6 @@ async function updateScriptLock(
metadataContent: Record<string, any>,
rawDeps: string | undefined
): Promise<void> {
if (
!(
language == "bun" ||
language == "python3" ||
language == "go" ||
language == "deno"
)
) {
return;
}
// generate the script lock running a dependency job in Windmill and update it inplace
// TODO: update this once the client is released
const rawResponse = await fetch(
@@ -224,9 +214,7 @@ async function updateScriptLock(
)}`
);
}
const lockPath = remotePath + ".script.lock";
await Deno.writeTextFile(lockPath, lock);
metadataContent.lock = "!inline " + lockPath;
metadataContent.lock = lock;
} catch (e) {
throw new Error(
`Failed to generate lockfile. Status was: ${rawResponse.statusText}, ${responseText}, ${e}`
@@ -458,19 +446,6 @@ export async function parseMetadataFile(
metadataFilePath = scriptPath + ".script.yaml";
await Deno.stat(metadataFilePath);
const payload: any = yamlParse(await Deno.readTextFile(metadataFilePath));
if (payload?.["lock"].startsWith("!inline ")) {
try {
const lockPath = payload["lock"].split(" ")[1];
payload["lock"] = await Deno.readTextFile(lockPath);
} catch (e) {
log.info(
colors.yellow(
`Failed to read lockfile, doing as if it was empty: ${e}`
)
);
payload["lock"] = "";
}
}
if (Array.isArray(payload?.["lock"])) {
payload["lock"] = payload["lock"].join("\n");
}

View File

@@ -16,8 +16,7 @@ export async function downloadZip(
includeSettings?: boolean,
defaultTs?: "bun" | "deno"
): Promise<JSZip | undefined> {
const requestHeaders: HeadersInit & { set(x: string, y: string): void } =
new Headers();
const requestHeaders: HeadersInit = new Headers();
requestHeaders.set("Authorization", "Bearer " + workspace.token);
requestHeaders.set("Content-Type", "application/octet-stream");

View File

@@ -1,8 +1,8 @@
// deno-lint-ignore-file no-explicit-any
import { colors, Command, log } from "./deps.ts";
import { colors, Command } from "./deps.ts";
import { GlobalOptions } from "./types.ts";
function stub(_opts: GlobalOptions, _dir?: string) {
async function stub(_opts: GlobalOptions, _dir?: string) {
log.info(
colors.red.underline(
'Push is deprecated. Use "sync push --raw" instead. See <TODO_LINK_HERE> for more information.'

View File

@@ -83,11 +83,7 @@ export async function handleScriptMetadata(
message: string | undefined,
globalDeps: GlobalDeps
): Promise<boolean> {
if (
path.endsWith(".script.json") ||
path.endsWith(".script.yaml") ||
path.endsWith(".script.lock")
) {
if (path.endsWith(".script.json") || path.endsWith(".script.yaml")) {
const contentPath = await findContentFile(path);
return handleFile(
contentPath,
@@ -171,7 +167,6 @@ export async function handleFile(
Boolean(remote.restart_unless_cancelled) &&
Boolean(typed.visible_to_runner_only) ==
Boolean(remote.visible_to_runner_only) &&
Boolean(typed.no_main_func) == Boolean(remote.no_main_func) &&
typed.priority == Boolean(remote.priority))
) {
log.info(colors.green(`Script ${remotePath} is up to date`));
@@ -187,7 +182,7 @@ export async function handleFile(
requestBody: {
content,
description: typed?.description ?? "",
language: language,
language: language as NewScript.language,
path: remotePath.replaceAll("\\", "/"),
summary: typed?.summary ?? "",
kind: typed?.kind,
@@ -203,7 +198,6 @@ export async function handleFile(
deployment_message: message,
restart_unless_cancelled: typed?.restart_unless_cancelled,
visible_to_runner_only: typed?.visible_to_runner_only,
no_main_func: typed?.no_main_func,
priority: typed?.priority,
},
});
@@ -217,7 +211,7 @@ export async function handleFile(
requestBody: {
content,
description: typed?.description ?? "",
language: language,
language: language as NewScript.language,
path: remotePath.replaceAll("\\", "/"),
summary: typed?.summary ?? "",
kind: typed?.kind,
@@ -233,7 +227,6 @@ export async function handleFile(
deployment_message: message,
restart_unless_cancelled: typed?.restart_unless_cancelled,
visible_to_runner_only: typed?.visible_to_runner_only,
no_main_func: typed?.no_main_func,
priority: typed?.priority,
},
});
@@ -246,8 +239,6 @@ export async function handleFile(
export async function findContentFile(filePath: string) {
const candidates = filePath.endsWith("script.json")
? exts.map((x) => filePath.replace(".script.json", x))
: filePath.endsWith("script.lock")
? exts.map((x) => filePath.replace(".script.lock", x))
: exts.map((x) => filePath.replace(".script.yaml", x));
const validCandidates = (
@@ -646,8 +637,7 @@ async function generateMetadata(
return (
(!isD && !exts.some((ext) => p.endsWith(ext))) ||
ignore(p, isD) ||
p.includes(".flow/") ||
p.includes(".app/")
p.includes(".flow/")
);
},
false,

View File

@@ -108,19 +108,17 @@ export const yamlOptions = {
};
function ZipFSElement(zip: JSZip, useYaml: boolean): DynFSElement {
async function _internal_file(
p: string,
f: JSZip.JSZipObject
): Promise<DynFSElement[]> {
const kind: "flow" | "app" | "script" | "other" = p.endsWith("flow.json")
? "flow"
: p.endsWith("app.json")
? "app"
: p.endsWith("script.json")
? "script"
: "other";
const isJson = p.endsWith(".json");
function _internal_file(p: string, f: JSZip.JSZipObject): DynFSElement {
const isFlow = p.endsWith("flow.json");
function transformPath() {
if (isFlow) {
return p.replace("flow.json", "flow");
} else {
return useYaml && p.endsWith(".json")
? p.replaceAll(".json", ".yaml")
: p;
}
}
interface InlineScript {
path: string;
@@ -131,8 +129,8 @@ function ZipFSElement(zip: JSZip, useYaml: boolean): DynFSElement {
const seen_names = new Set<string>();
function assignPath(
summary: string | undefined,
language: RawScript["language"]
): [string, string] {
language: RawScript.language
): string {
let name;
const INLINE_SCRIPT = "inline_script";
@@ -165,43 +163,27 @@ function ZipFSElement(zip: JSZip, useYaml: boolean): DynFSElement {
else if (language == "graphql") ext = "gql";
else if (language == "bun") ext = "bun.ts";
else if (language == "nativets") ext = "native.ts";
else if (language == "frontend") ext = "frontend.js";
else ext = "no_ext";
return [`${name}.inline_script.`, ext];
return `${name}.inline_script.${ext}`;
}
function extractInlineScriptsForFlows(
modules: FlowModule[]
): InlineScript[] {
function extractInlineScripts(modules: FlowModule[]): InlineScript[] {
return modules.flatMap((m) => {
if (m.value.type == "rawscript") {
const [basePath, ext] = assignPath(m.summary, m.value.language);
const path = basePath + ext;
const path = assignPath(m.summary, m.value.language);
const content = m.value.content;
const r = [{ path: path, content: content }];
m.value.content = "!inline " + path;
const lock = m.value.lock;
if (lock) {
const lockPath = basePath + "lock";
m.value.lock = "!inline " + lockPath;
r.push({ path: lockPath, content: lock });
}
return r;
return [{ path: path, content: content }];
} else if (m.value.type == "forloopflow") {
return extractInlineScriptsForFlows(m.value.modules);
return extractInlineScripts(m.value.modules);
} else if (m.value.type == "branchall") {
return m.value.branches.flatMap((b) =>
extractInlineScriptsForFlows(b.modules)
extractInlineScripts(b.modules)
);
} else if (m.value.type == "whileloopflow") {
return extractInlineScriptsForFlows(m.value.modules);
} else if (m.value.type == "branchone") {
return [
...m.value.branches.flatMap((b) =>
extractInlineScriptsForFlows(b.modules)
),
...extractInlineScriptsForFlows(m.value.default),
...m.value.branches.flatMap((b) => extractInlineScripts(b.modules)),
...extractInlineScripts(m.value.default),
];
} else {
return [];
@@ -209,147 +191,47 @@ function ZipFSElement(zip: JSZip, useYaml: boolean): DynFSElement {
});
}
function extractInlineScriptsForApps(rec: any): InlineScript[] {
if (!rec) {
return [];
}
if (typeof rec == "object") {
return Object.entries(rec).flatMap(([k, v]) => {
if (k == "inlineScript" && typeof v == "object") {
const o: Record<string, any> = v as any;
const name = rec["name"];
const [basePath, ext] = assignPath(name, o["language"]);
const r = [];
if (o["content"]) {
const content = o["content"];
o["content"] = "!inline " + basePath + ext;
r.push({
path: basePath + ext,
content: content,
});
}
if (o["lock"]) {
const lock = o["lock"];
o["lock"] = "!inline " + basePath + "lock";
r.push({
path: basePath + "lock",
content: lock,
});
}
return r;
} else {
return extractInlineScriptsForApps(v);
}
});
}
return [];
}
function transformPath() {
if (kind == "flow") {
return p.replace("flow.json", "flow");
} else if (kind == "app") {
return p.replace("app.json", "app");
} else {
return useYaml && isJson ? p.replaceAll(".json", ".yaml") : p;
}
}
const finalPath = transformPath();
const r = [
{
isDirectory: kind == "flow" || kind == "app",
path: finalPath,
async *getChildren(): AsyncIterable<DynFSElement> {
if (kind == "flow") {
const flow: OpenFlow = JSON.parse(await f.async("text"));
const inlineScripts = extractInlineScriptsForFlows(
flow.value.modules
);
for (const s of inlineScripts) {
yield {
isDirectory: false,
path: path.join(finalPath, s.path),
async *getChildren() {},
// deno-lint-ignore require-await
async getContentText() {
return s.content;
},
};
}
const flowPath = transformPath();
return {
isDirectory: isFlow,
path: flowPath,
async *getChildren(): AsyncIterable<DynFSElement> {
if (isFlow) {
const flow: OpenFlow = JSON.parse(await f.async("text"));
const inlineScripts = extractInlineScripts(flow.value.modules);
for (const s of inlineScripts) {
yield {
isDirectory: false,
path: path.join(finalPath, "flow.yaml"),
path: path.join(flowPath, s.path),
async *getChildren() {},
// deno-lint-ignore require-await
async getContentText() {
return yamlStringify(flow, yamlOptions);
},
};
} else if (kind == "app") {
const app = JSON.parse(await f.async("text"));
const inlineScripts = extractInlineScriptsForApps(app?.["value"]);
for (const s of inlineScripts) {
yield {
isDirectory: false,
path: path.join(finalPath, s.path),
async *getChildren() {},
// deno-lint-ignore require-await
async getContentText() {
return s.content;
},
};
}
yield {
isDirectory: false,
path: path.join(finalPath, "app.yaml"),
async *getChildren() {},
// deno-lint-ignore require-await
async getContentText() {
return yamlStringify(app, yamlOptions);
return s.content;
},
};
}
},
async getContentText(): Promise<string> {
const content = await f.async("text");
if (kind == "script") {
const parsed = JSON.parse(content);
if (parsed["lock"]) {
parsed["lock"] = "!inline " + removeSuffix(p, ".json") + ".lock";
}
return useYaml
? yamlStringify(parsed, yamlOptions)
: JSON.stringify(parsed, null, 2);
}
return useYaml && isJson
? yamlStringify(JSON.parse(content), yamlOptions)
: content;
},
yield {
isDirectory: false,
path: path.join(flowPath, "flow.yaml"),
async *getChildren() {},
// deno-lint-ignore require-await
async getContentText() {
return yamlStringify(flow, yamlOptions);
},
};
}
},
];
if (kind == "script") {
const content = await f.async("text");
const parsed = JSON.parse(content);
const lock = parsed["lock"];
if (lock) {
r.push({
isDirectory: false,
path: removeSuffix(finalPath, ".json") + ".lock",
async *getChildren() {},
// deno-lint-ignore require-await
async getContentText() {
return lock;
},
});
}
}
return r;
// async getContentBytes(): Promise<Uint8Array> {
// return await f.async("uint8array");
// },
async getContentText(): Promise<string> {
const content = await f.async("text");
return useYaml && p.endsWith(".json")
? yamlStringify(JSON.parse(content), yamlOptions)
: content;
},
};
}
function _internal_folder(p: string, zip: JSZip): DynFSElement {
return {
@@ -363,10 +245,7 @@ function ZipFSElement(zip: JSZip, useYaml: boolean): DynFSElement {
const e = zip.folder(file.name)!;
yield _internal_folder(totalPath, e);
} else {
const fs = await _internal_file(totalPath, file);
for (const f of fs) {
yield f;
}
yield _internal_file(totalPath, file);
}
}
},
@@ -460,19 +339,9 @@ export async function elementsToMap(
if (skips.skipResources && path.endsWith(".resource" + ext)) continue;
if (
![
"json",
"yaml",
"go",
"sh",
"ts",
"py",
"sql",
"gql",
"ps1",
"js",
"lock",
].includes(path.split(".").pop() ?? "")
!["json", "yaml", "go", "sh", "ts", "py", "sql", "gql", "ps1"].includes(
path.split(".").pop() ?? ""
)
)
continue;
const content = await entry.getContentText();
@@ -861,7 +730,7 @@ async function pull(opts: GlobalOptions & SyncOptions) {
showConflict(conflict.path, conflict.local, conflict.change.after);
}
log.info(
colors.red(`Please resolve these conflicts manually by either:
colors.red(`Please resolve theses conflicts manually by either:
- reverting the content back to its remote (\`wmill pull\` and refuse to preserve local when prompted)
- pushing the changes with \`wmill push --skip-pull\` to override wmill with all your local changes
`)
@@ -1056,8 +925,7 @@ async function push(opts: GlobalOptions & SyncOptions) {
} else if (change.name === "added") {
if (
change.path.endsWith(".script.json") ||
change.path.endsWith(".script.yaml") ||
change.path.endsWith(".lock")
change.path.endsWith(".script.yaml")
) {
continue;
} else if (
@@ -1090,9 +958,6 @@ async function push(opts: GlobalOptions & SyncOptions) {
await Deno.writeTextFile(stateTarget, change.content);
}
} else if (change.name === "deleted") {
if (change.path.endsWith(".lock")) {
continue;
}
const typ = getTypeStrFromPath(change.path);
if (typ == "script") {
@@ -1140,7 +1005,7 @@ async function push(opts: GlobalOptions & SyncOptions) {
case "app":
await AppService.deleteApp({
workspace: workspaceId,
path: removeSuffix(change.path, ".app/app.json"),
path: removeSuffix(change.path, ".app.json"),
});
break;
case "schedule":

View File

@@ -111,8 +111,7 @@ export async function pushObj(
const typeEnding = getTypeStrFromPath(p);
if (typeEnding === "app") {
const appName = p.split(".app" + path.sep)[0];
await pushApp(workspace, appName, appName + ".app", message);
await pushApp(workspace, p, befObj, newObj, message);
} else if (typeEnding === "folder") {
await pushFolder(workspace, p, befObj, newObj);
} else if (typeEnding === "variable") {
@@ -172,9 +171,6 @@ export function getTypeStrFromPath(
if (p.includes(".flow" + path.sep)) {
return "flow";
}
if (p.includes(".app" + path.sep)) {
return "app";
}
const parsed = path.parse(p);
if (
parsed.ext == ".go" ||
@@ -183,8 +179,7 @@ export function getTypeStrFromPath(
parsed.ext == ".py" ||
parsed.ext == ".sql" ||
parsed.ext == ".gql" ||
parsed.ext == ".ps1" ||
parsed.ext == ".js"
parsed.ext == ".ps1"
) {
return "script";
}

View File

@@ -242,9 +242,9 @@ export async function add(
)
);
const automateUsernameCreation: boolean =
((await SettingService.getGlobal({
(await SettingService.getGlobal({
key: "automate_username_creation",
})) as any) ?? false;
})) ?? false;
await WorkspaceService.createWorkspace({
requestBody: {
id: workspaceId,

File diff suppressed because it is too large Load Diff

View File

@@ -1,6 +1,6 @@
{
"name": "windmill-components",
"version": "1.322.0",
"version": "1.306.9",
"scripts": {
"dev": "vite dev",
"build": "vite build",
@@ -10,15 +10,15 @@
"lint": "prettier --ignore-path .gitignore --check --plugin-search-dir=. . && eslint --ignore-path .gitignore .",
"format": "prettier --ignore-path .gitignore --write --plugin-search-dir=. .",
"package": "svelte-package -o package",
"generate-backend-client": "openapi-ts --input ../backend/windmill-api/openapi.yaml --output ./src/lib/gen --useOptions --enums javascript --format false",
"generate-backend-client-mac": "openapi-ts --input ../backend/windmill-api/openapi.yaml --output ./src/lib/gen --useOptions --enums javascript",
"generate-backend-client": "openapi --input ../backend/windmill-api/openapi.yaml --output ./src/lib/gen --useOptions && sed -i '213 i \\ request.referrerPolicy = \"no-referrer\"\n' src/lib/gen/core/request.ts",
"generate-backend-client-mac": "openapi --input ../backend/windmill-api/openapi.yaml --output ./src/lib/gen --useOptions",
"pretest": "tsc --incremental -p tests/tsconfig.json",
"test": "playwright test --config=tests-out/playwright.config.js",
"filter-classes": "node filterTailwindClasses.js"
},
"devDependencies": {
"@codingame/esbuild-import-meta-url-plugin": "^1.0.2",
"@floating-ui/core": "^1.3.1",
"@hey-api/openapi-ts": "^0.40.0",
"@playwright/test": "^1.34.3",
"@rgossiaux/svelte-headlessui": "^2.0.0",
"@sveltejs/adapter-static": "^3.0.0",
@@ -41,7 +41,10 @@
"eslint": "^8.47.0",
"eslint-config-prettier": "^8.6.0",
"eslint-plugin-svelte": "^2.33.1",
"ol": "^7.4.0",
"openapi-typescript-codegen": "^0.25.0",
"path-browserify": "^1.0.1",
"pdfjs-dist": "^3.8.162",
"postcss": "^8.4.24",
"postcss-load-config": "^4.0.1",
"prettier": "^2.8.8",
@@ -79,17 +82,24 @@
},
"svelte-timezone-picker": {
"svelte": "$svelte"
},
"monaco-editor": "$monaco-editor",
"vscode": "$vscode"
},
"resolutions": {
"monaco-editor": "npm:@codingame/monaco-editor-treemended@>=1.83.5 <1.84.0",
"vscode": "npm:@codingame/monaco-vscode-api@>=1.83.5 <1.84.0"
}
},
"type": "module",
"dependencies": {
"@aws-crypto/sha256-js": "^4.0.0",
"@codingame/monaco-vscode-go-default-extension": "^4.2.1",
"@codingame/monaco-vscode-javascript-default-extension": "^4.2.1",
"@codingame/monaco-vscode-json-default-extension": "^4.2.1",
"@codingame/monaco-vscode-powershell-default-extension": "^4.2.1",
"@codingame/monaco-vscode-python-default-extension": "^4.2.1",
"@codingame/monaco-vscode-shellscript-default-extension": "^4.2.1",
"@codingame/monaco-vscode-sql-default-extension": "^4.2.1",
"@codingame/monaco-vscode-standalone-css-language-features": "^4.2.1",
"@codingame/monaco-vscode-standalone-html-language-features": "^4.2.1",
"@codingame/monaco-vscode-standalone-json-language-features": "^4.2.1",
"@codingame/monaco-vscode-standalone-typescript-language-features": "^4.2.1",
"@codingame/monaco-vscode-typescript-basics-default-extension": "^4.2.1",
"@codingame/monaco-vscode-typescript-language-features-default-extension": "^4.2.1",
"@json2csv/plainjs": "^7.0.6",
"@leeoniya/ufuzzy": "^1.0.8",
"@popperjs/core": "^2.11.6",
@@ -114,12 +124,15 @@
"highlight.js": "^11.8.0",
"lodash": "^4.17.21",
"lucide-svelte": "^0.293.0",
"monaco-editor": "npm:@codingame/monaco-editor-treemended@>=1.83.5 <1.84.0",
"monaco-editor": "npm:@codingame/monaco-vscode-editor-api@~4.2.1",
"vscode": "npm:@codingame/monaco-vscode-api@~4.2.1",
"vscode-languageclient": "~9.0.1",
"vscode-uri": "~3.0.8",
"vscode-ws-jsonrpc": "~3.3.1",
"monaco-editor-wrapper": "^4.2.0",
"monaco-graphql": "^1.5.1",
"monaco-languageclient": "~7.0.1",
"ol": "^7.4.0",
"monaco-languageclient": "~8.3.0",
"openai": "^4.3.0",
"pdfjs-dist": "^3.8.162",
"quill": "^1.3.7",
"svelte-carousel": "^1.0.25",
"svelte-chartjs": "^3.1.5",
@@ -129,12 +142,8 @@
"svelte-portal": "^2.2.1",
"svelte-tiny-virtual-list": "^2.0.5",
"tailwind-merge": "^1.13.2",
"vscode": "npm:@codingame/monaco-vscode-api@>=1.83.5 <1.84.0",
"vscode-languageclient": "~9.0.1",
"vscode-uri": "~3.0.8",
"vscode-ws-jsonrpc": "~3.1.0",
"windmill-parser-wasm": "^1.318.0",
"windmill-sql-datatype-parser-wasm": "^1.318.0",
"windmill-parser-wasm": "^1.286.2",
"windmill-sql-datatype-parser-wasm": "1.305.0",
"y-monaco": "^0.1.4",
"y-websocket": "^1.5.0",
"yaml": "^2.3.4",
@@ -225,11 +234,6 @@
"svelte": "./package/components/FlowBuilder.svelte",
"default": "./package/components/FlowBuilder.svelte"
},
"./components/ScriptBuilder.svelte": {
"types": "./package/components/ScriptBuilder.svelte.d.ts",
"svelte": "./package/components/ScriptBuilder.svelte",
"default": "./package/components/ScriptBuilder.svelte"
},
"./components/FlowEditor.svelte": {
"types": "./package/components/flows/FlowEditor.svelte.d.ts",
"svelte": "./package/components/flows/FlowEditor.svelte",
@@ -271,10 +275,6 @@
"types": "./package/stores.d.ts",
"default": "./package/stores.js"
},
"./gen": {
"types": "./package/gen/index.d.ts",
"default": "./package/gen/index.js"
},
"./components/flows/flowStore": {
"types": "./package/components/flows/flowStore.d.ts",
"default": "./package/components/flows/flowStore.js"
@@ -300,6 +300,10 @@
"types": "./package/gen/core/OpenAPI.d.ts",
"default": "./package/gen/core/OpenAPI.js"
},
"./gen": {
"default": "./package/gen/index.js",
"types": "./package/gen/index.d.ts"
},
"./components/DropdownV2.svelte": {
"types": "./package/components/DropdownV2.svelte.d.ts",
"svelte": "./package/components/DropdownV2.svelte",
@@ -365,9 +369,6 @@
"components/FlowBuilder.svelte": [
"./package/components/FlowBuilder.svelte.d.ts"
],
"components/ScriptBuilder.svelte": [
"./package/components/ScriptBuilder.svelte.d.ts"
],
"components/FlowEditor.svelte": [
"./package/components/flows/FlowEditor.svelte.d.ts"
],
@@ -418,4 +419,4 @@
"optionalDependencies": {
"fsevents": "^2.3.3"
}
}
}

View File

@@ -17,17 +17,6 @@
font-display: swap;
}
.prose-xs ul {
margin-top: 0.5rem;
list-style-type: '- ';
padding-left: 1.5rem;
}
.prose ul {
margin-top: 1.5rem;
list-style-type: '- ';
padding-left: 3rem;
}
.splitpanes--vertical > .splitpanes__pane {
transition: none !important;
}
@@ -146,11 +135,6 @@
@apply flex flex-col justify-start items-end;
}
.ol-control button {
@apply w-7 h-7 center-center bg-surface border text-secondary
@apply w-7 h-7 center-center bg-surface border text-secondary
rounded mt-1 mr-1 shadow duration-200 hover:bg-surface-hover focus:bg-surface-hover;
}
/* Components */
.component-wrapper {
@apply rounded-md border overflow-hidden border-gray-300 dark:border-gray-600;
}

View File

@@ -17,13 +17,6 @@ export const action = (node) => {
const setInitialHeight = () => {
let height = 0
const style = window.getComputedStyle(node)
const visible = style?.getPropertyValue('visibility') === 'hidden'
if (visible === false) {
return
}
if (node.value) {
height = node.scrollHeight
} else {
@@ -44,7 +37,7 @@ export const action = (node) => {
const setHeight = () => {
node.style.height = '0px'
node.style.height = Math.max(node.scrollHeight ?? 0, 40) + 2 + 'px'
node.style.height = Math.max(node.scrollHeight, 40) + 5 + 'px'
}
const addStyles = () => {

View File

@@ -4,7 +4,7 @@ export type OwnerKind = 'group' | 'user' | 'folder'
export type ActionKind = 'Create' | 'Update' | 'Delete' | 'Execute'
export type SupportedLanguage = Script['language']
export type SupportedLanguage = Script.language
export interface PropertyDisplayInfo {
property: SchemaProperty

View File

@@ -26,7 +26,7 @@
let automateUsernameCreation = false
async function getAutomateUsernameCreationSetting() {
automateUsernameCreation =
((await SettingService.getGlobal({ key: 'automate_username_creation' })) as any) ?? false
(await SettingService.getGlobal({ key: 'automate_username_creation' })) ?? false
}
getAutomateUsernameCreationSetting()

View File

@@ -23,15 +23,14 @@
let supabaseWizard = false
async function isSupabaseAvailable() {
supabaseWizard =
((await OauthService.listOauthConnects()) ?? {})['supabase_wizard'] != undefined
supabaseWizard = (await OauthService.listOAuthConnects())['supabase_wizard'] != undefined
}
async function loadSchema() {
if (!resourceTypeInfo) return
rawCode = '{}'
viewJsonSchema = false
try {
schema = resourceTypeInfo.schema as any
schema = resourceTypeInfo.schema
notFound = false
} catch (e) {
notFound = true

View File

@@ -191,7 +191,7 @@
}
async function loadConnects() {
const nconnects = (await OauthService.listOauthConnects()) as any
const nconnects = await OauthService.listOAuthConnects()
if (nconnects['supabase_wizard']) {
delete nconnects['supabase_wizard']
}

View File

@@ -1,5 +1,5 @@
<script lang="ts">
import { ResourceService, VariableService } from '$lib/gen'
import { ResourceService } from '$lib/gen'
import { workspaceStore } from '$lib/stores'
import { copyToClipboard, truncate } from '$lib/utils'
import { ClipboardCopy, Expand } from 'lucide-svelte'
@@ -17,17 +17,8 @@
}
async function getResource(path: string) {
jsonViewerContent = await ResourceService.getResourceValue({
workspace: $workspaceStore!,
path
})
}
async function getVariable(path: string) {
jsonViewerContent = await VariableService.getVariableValue({
workspace: $workspaceStore!,
path
})
jsonViewerContent = (await ResourceService.getResource({ workspace: $workspaceStore!, path }))
.value
}
</script>
@@ -66,14 +57,6 @@
jsonViewer.toggleDrawer()
}}>{value}</button
>
{:else if isString(value) && value.startsWith('$var:')}
<button
class="text-xs text-blue-500"
on:click={async () => {
await getVariable(value.substring('$res:'.length))
jsonViewer.toggleDrawer()
}}>{value}</button
>
{:else if typeof value !== 'object'}
{truncate(JSON.stringify(value), 100)}
{#if JSON.stringify(value).length > 100}

View File

@@ -12,6 +12,7 @@
import NumberTypeNarrowing from './NumberTypeNarrowing.svelte'
import ObjectResourceInput from './ObjectResourceInput.svelte'
import ObjectTypeNarrowing from './ObjectTypeNarrowing.svelte'
import Password from './Password.svelte'
import Range from './Range.svelte'
import ResourcePicker from './ResourcePicker.svelte'
import SchemaForm from './SchemaForm.svelte'
@@ -27,7 +28,6 @@
import CurrencyInput from './apps/components/inputs/currency/CurrencyInput.svelte'
import FileUpload from './common/fileUpload/FileUpload.svelte'
import autosize from '$lib/autosize'
import PasswordArgInput from './PasswordArgInput.svelte'
export let label: string = ''
export let value: any
@@ -284,7 +284,6 @@
bind:pattern
bind:enum_
bind:contentEncoding
bind:password={extra['password']}
bind:minRows={extra['minRows']}
bind:disableCreate={extra['disableCreate']}
bind:disableVariablePicker={extra['disableVariablePicker']}
@@ -307,7 +306,7 @@
{/if}
{#if description}
<div class="text-xs italic pb-1 text-secondary">
<pre class="font-main whitespace-normal">{description}</pre>
<pre class="font-main">{description}</pre>
</div>
{/if}
<div class="flex space-x-1">
@@ -651,7 +650,7 @@
<div class="flex flex-col w-full">
<div class="flex flex-row w-full items-center justify-between relative">
{#if password || extra?.['password'] == true}
<PasswordArgInput {disabled} bind:value />
<Password {disabled} bind:password={value} />
{:else}
{#key extra?.['minRows']}
<textarea

View File

@@ -5,16 +5,19 @@
import Drawer from './common/drawer/Drawer.svelte'
import DrawerContent from './common/drawer/DrawerContent.svelte'
import DefaultScriptsInner from './DefaultScriptsInner.svelte'
import Portal from 'svelte-portal'
let drawer: Drawer
</script>
{#if $userStore?.is_admin || $userStore?.is_super_admin}
<Drawer bind:this={drawer} placement="left">
<DrawerContent title="Edit Default Scripts" on:close={drawer.closeDrawer}>
<DefaultScriptsInner />
</DrawerContent>
</Drawer>
<Portal>
<Drawer bind:this={drawer} placement="left">
<DrawerContent title="Edit Default Scripts" on:close={drawer.closeDrawer}>
<DefaultScriptsInner />
</DrawerContent>
</Drawer>
</Portal>
<Button
on:click={drawer?.openDrawer}
startIcon={{ icon: SettingsIcon }}

View File

@@ -5,8 +5,6 @@
import DefaultTagsInner from './DefaultTagsInner.svelte'
export let defaultTagPerWorkspace: boolean | undefined = undefined
let placement: 'bottom-end' | 'top-end' = 'bottom-end'
</script>
@@ -24,5 +22,5 @@
>
</Button>
</svelte:fragment>
<DefaultTagsInner bind:defaultTagPerWorkspace />
</Popup>
<DefaultTagsInner />
</Popup>

View File

@@ -9,7 +9,15 @@
import Toggle from './Toggle.svelte'
let defaultTags: string[] | undefined = undefined
export let defaultTagPerWorkspace: boolean | undefined = undefined
let defaultTagPerWorkspace: boolean | undefined = undefined
async function loadDefaultTagsPerWorkspace() {
try {
defaultTagPerWorkspace = await WorkerService.isDefaultTagsPerWorkspace()
} catch (err) {
sendUserToast(`Could not load default tag per workspace setting: ${err}`, true)
}
}
async function loadDefaultTags() {
try {
@@ -20,6 +28,7 @@
}
loadDefaultTags()
loadDefaultTagsPerWorkspace()
</script>
<div class="flex flex-col w-80 p-2 gap-2">

View File

@@ -517,8 +517,8 @@
])
diffDrawer.setDiff({
mode: 'simple',
original: values?.[0] as any,
current: values?.[1] as any,
original: values[0],
current: values[1],
title: 'Staging/prod <> Dev'
})
}

View File

@@ -5,16 +5,16 @@
import { WindmillIcon } from '$lib/components/icons'
import LogPanel from '$lib/components/scriptEditor/LogPanel.svelte'
import {
type CompletedJob,
type Job,
CompletedJob,
Job,
JobService,
OpenAPI,
type Preview,
Preview,
type OpenFlow,
type FlowModule,
WorkspaceService,
type InputTransform,
type RawScript,
RawScript,
type PathScript
} from '$lib/gen'
import { inferArgs } from '$lib/infer'
@@ -157,7 +157,7 @@
type LastEditScript = {
content: string
path: string
language: Preview['language']
language: Preview.language
lock?: string
}
@@ -309,7 +309,6 @@
} catch (e) {
console.error(e)
validCode = false
schema = emptySchema()
}
}
@@ -400,7 +399,6 @@
if (selectedIdStore == '') {
return
}
//@ts-ignore
dfs($flowStore.value.modules, async (mod) => {
if (mod.id == selectedIdStore) {
if (

View File

@@ -63,7 +63,7 @@
metadata['content'] = 'check content diff'
}
return {
lang: data.language ? scriptLangToEditorLang(data.language as Script['language']) : undefined,
lang: data.language ? scriptLangToEditorLang(data.language as Script.language) : undefined,
content,
metadata: orderedYamlStringify(metadata)
}

View File

@@ -3,17 +3,27 @@
import { onMount } from 'svelte'
import { editor as meditor } from 'monaco-editor'
import 'monaco-editor/esm/vs/basic-languages/python/python.contribution'
import 'monaco-editor/esm/vs/basic-languages/go/go.contribution'
import 'monaco-editor/esm/vs/basic-languages/shell/shell.contribution'
import 'monaco-editor/esm/vs/basic-languages/typescript/typescript.contribution'
import 'monaco-editor/esm/vs/basic-languages/sql/sql.contribution'
import 'monaco-editor/esm/vs/language/typescript/monaco.contribution'
import '@codingame/monaco-vscode-theme-defaults-default-extension'
import '@codingame/monaco-vscode-json-default-extension'
import '@codingame/monaco-vscode-standalone-json-language-features'
import '@codingame/monaco-vscode-standalone-css-language-features'
import '@codingame/monaco-vscode-standalone-html-language-features'
import '@codingame/monaco-vscode-standalone-typescript-language-features'
import '@codingame/monaco-vscode-typescript-basics-default-extension'
import '@codingame/monaco-vscode-typescript-language-features-default-extension'
import '@codingame/monaco-vscode-go-default-extension'
import '@codingame/monaco-vscode-javascript-default-extension'
import '@codingame/monaco-vscode-powershell-default-extension'
import '@codingame/monaco-vscode-python-default-extension'
import '@codingame/monaco-vscode-shellscript-default-extension'
import '@codingame/monaco-vscode-sql-default-extension'
import { initializeVscode } from './vscode'
import EditorTheme from './EditorTheme.svelte'
import { buildWorkerDefinition } from './build_workers'
import { configureMonacoWorkers } from './build_workers'
buildWorkerDefinition('../../../workers', import.meta.url, false)
configureMonacoWorkers()
const SIDE_BY_SIDE_MIN_WIDTH = 700
@@ -32,10 +42,6 @@
async function loadDiffEditor() {
await initializeVscode()
if (!diffDivEl) {
return
}
diffEditor = meditor.createDiffEditor(diffDivEl!, {
automaticLayout,
renderSideBySide: editorWidth >= SIDE_BY_SIDE_MIN_WIDTH,

View File

@@ -26,7 +26,6 @@
import ToggleButtonGroup from './common/toggleButton-v2/ToggleButtonGroup.svelte'
import ToggleButton from './common/toggleButton-v2/ToggleButton.svelte'
import MapResult from './MapResult.svelte'
import Tooltip from '$lib/components/Tooltip.svelte'
export let result: any
export let requireHtmlApproval = false
@@ -111,11 +110,11 @@
keys.length == 1 && keys.includes('render_all') && Array.isArray(result['render_all'])
// Check if the result is an image
if (['png', 'svg', 'jpeg', 'html', 'gif'].includes(keys[0]) && keys.length == 1) {
if (['png', 'svg', 'jpeg', 'html'].includes(keys[0]) && keys.length == 1) {
// Check if the image is too large (10mb)
largeObject = roughSizeOfObject(result) > 10000000
return keys[0] as 'png' | 'svg' | 'jpeg' | 'html' | 'gif'
return keys[0] as 'png' | 'svg' | 'jpeg' | 'html'
}
let length = roughSizeOfObject(result)
@@ -304,19 +303,10 @@
<div class="text-tertiary text-xs flex gap-2 z-10 items-center">
<slot name="copilot-fix" />
{#if !disableExpand && !noControls}
<Tooltip
documentationLink="https://www.windmill.dev/docs/core_concepts/rich_display_rendering"
customSize="115%"
<button on:click={() => copyToClipboard(toJsonStr(result))}
><ClipboardCopy size={16} /></button
>
The result renderer in Windmill supports rich display rendering, allowing you to
customize the display format of your results.
</Tooltip>
<button on:click={() => copyToClipboard(toJsonStr(result))} class="-mt-1">
<ClipboardCopy size={16} />
</button>
<button on:click={jsonViewer.openDrawer} class="-mt-1">
<Expand size={16} />
</button>
<button on:click={jsonViewer.openDrawer}><Expand size={16} /></button>
{/if}
</div>
</div>{#if !forceJson && resultKind == 'table-col'}
@@ -528,7 +518,7 @@
</div>
</div>
{:else if !forceJson && resultKind == 'markdown'}
<div class="prose-xs dark:prose-invert !list-disc !list-outside">
<div class="prose-xs dark:prose-invert">
<Markdown md={result?.md ?? result?.markdown} />
</div>
{:else if !forceJson && isTableDisplay && richRender}
@@ -598,44 +588,46 @@
</div>
{#if !disableExpand && !noControls}
<Drawer bind:this={jsonViewer} size="900px">
<DrawerContent title="Expanded Result" on:close={jsonViewer.closeDrawer}>
<svelte:fragment slot="actions">
<Button
download="{filename ?? 'result'}.json"
href={workspaceId && jobId
? `/api/w/${workspaceId}/jobs_u/completed/get_result/${jobId}`
: `data:text/json;charset=utf-8,${encodeURIComponent(toJsonStr(result))}`}
startIcon={{ icon: Download }}
color="light"
size="xs"
>
Download
</Button>
<Button
on:click={() => copyToClipboard(toJsonStr(result))}
color="light"
size="xs"
startIcon={{
icon: ClipboardCopy
}}
>
Copy to clipboard
</Button>
</svelte:fragment>
<svelte:self
{noControls}
{result}
{requireHtmlApproval}
{filename}
{jobId}
{workspaceId}
{hideAsJson}
{forceJson}
disableExpand={true}
/>
</DrawerContent>
</Drawer>
<Portal>
<Drawer bind:this={jsonViewer} size="900px">
<DrawerContent title="Expanded Result" on:close={jsonViewer.closeDrawer}>
<svelte:fragment slot="actions">
<Button
download="{filename ?? 'result'}.json"
href={workspaceId && jobId
? `/api/w/${workspaceId}/jobs_u/completed/get_result/${jobId}`
: `data:text/json;charset=utf-8,${encodeURIComponent(toJsonStr(result))}`}
startIcon={{ icon: Download }}
color="light"
size="xs"
>
Download
</Button>
<Button
on:click={() => copyToClipboard(toJsonStr(result))}
color="light"
size="xs"
startIcon={{
icon: ClipboardCopy
}}
>
Copy to clipboard
</Button>
</svelte:fragment>
<svelte:self
{noControls}
{result}
{requireHtmlApproval}
{filename}
{jobId}
{workspaceId}
{hideAsJson}
{forceJson}
disableExpand={true}
/>
</DrawerContent>
</Drawer>
</Portal>
<Portal>
<S3FilePicker bind:this={s3FileViewer} readOnlyMode={true} />

View File

@@ -47,10 +47,7 @@
class={twMerge(
'px-4 py-2 text-primary hover:bg-surface-hover hover:text-primary cursor-pointer text-xs transition-all',
'flex flex-row gap-2 items-center',
item?.disabled && 'text-gray-400 cursor-not-allowed',
item?.type === 'delete' &&
!item?.disabled &&
'text-red-500 hover:bg-red-100 hover:text-red-500'
item?.type === 'delete' && 'text-red-500 hover:bg-red-100 hover:text-red-500'
)}
>
{#if item.icon}

Some files were not shown because too many files have changed in this diff Show More