feat(frontend): add a dev docker-compose file + update readme

* display previous script inputs on script run page

* parallelise loading

* parallelise loading

* also working for flows

* separate endpoints for scripts and flows

* Splitpanes and Saved Inputs (UI)

* Saved inputs API endpoints

* Editable Input name

* Narrow width styling

* feat(frontend): Add a toggle to open the saved inputs (#1401)

* feat(frontend): Add a toggle to open the saved inputs

* feat(frontend): Add a toggle to open the saved inputs

* feat(frontend): Move toggle

* update all

---------

Co-authored-by: Ruben Fiszel <ruben@rubenfiszel.com>
Co-authored-by: Faton Ramadani <faton.ramadani14@gmail.com>

.env

@@ -1,2 +1,7 @@
 DB_PASSWORD=changeme
-WM_BASE_URL=localhost
+
+# this is the url that your instance is publicly exposed to
+WM_BASE_URL=http://localhost
+
+# this is the url that caddy will reverse proxy from. It might be different than WM_BASE_URL if you re using a second proxy/load balancer in front
+CADDY_REVERSE_PROXY=http://localhost

.github/workflows/deploy-to-s3.yml

@@ -1,32 +0,0 @@
-name: Deploy frontend to s3
-on:
-  push:
-    branches: [main]
-    tags: ["*"]
-
-concurrency:
-  group: ${{ github.ref }}-npmbuild
-  cancel-in-progress: true
-
-jobs:
-  npm_build:
-    runs-on: ubuntu-latest
-    env:
-      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-node@v3
-        with:
-          node-version: 18
-      - name: "npm build"
-        timeout-minutes: 15
-        run:
-          cd frontend && npm ci && npm run generate-backend-client &&
-          NODE_OPTIONS="--max-old-space-size=8192" npm run build
-
-      - uses: reggionick/s3-deploy@v3
-        with:
-          folder: frontend/build
-          bucket: windmill-frontend
-          bucket-region: us-east-1

.github/workflows/docker-image.yml

@@ -18,7 +18,7 @@ permissions:
   contents: read
   id-token: write
   packages: write
-  
+
 jobs:
   build:
     runs-on: ubuntu-22.04
@@ -50,7 +50,6 @@ jobs:
             type=semver,pattern={{version}}
             type=semver,pattern={{major}}.{{minor}}
 
-
       - name: Build and push publicly
         uses: depot/build-push-action@v1
         with:
@@ -63,7 +62,7 @@ jobs:
           labels: |
             ${{ steps.meta-public.outputs.labels }}
             org.opencontainers.image.licenses=AGPLv3
-   
+
   build_ee:
     runs-on: ubuntu-22.04
     steps:
@@ -142,7 +141,28 @@ jobs:
   #       run: docker kill ${{ steps.docker-container.outputs.id }}
   #       if: always()
 
+  deploy_s3:
+    needs: [build_ee]
+    runs-on: ubuntu-latest
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 18
+      - uses: shrink/actions-docker-extract@v2
+        id: extract
+        with:
+          image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest
+          path: "/static_frontend/."
 
+      - uses: reggionick/s3-deploy@v3
+        with:
+          folder: ${{ steps.extract.outputs.destination }}
+          bucket: windmill-frontend
+          bucket-region: us-east-1
   publish_privately_heavy:
     needs: [build_ee]
     runs-on: [self-hosted, new]
@@ -191,8 +211,12 @@ jobs:
           tags: |
             ${{ steps.meta-heavy.outputs.tags }}
           labels: ${{ steps.meta-heavy.outputs.labels }}
-          cache-from: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}-heavy:buildcache
-          cache-to: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}-heavy:buildcache,mode=max
+          cache-from:
+            type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME
+            }}-heavy:buildcache
+          cache-to:
+            type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME
+            }}-heavy:buildcache,mode=max
 
   publish_privately_helm:
     runs-on: [self-hosted, new]
@@ -204,7 +228,6 @@ jobs:
           fetch-depth: 0
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
- 
 
       - name: Login to registry
         uses: docker/login-action@v2
@@ -220,7 +243,7 @@ jobs:
           registry: ${{ env.ECR_REGISTRY }}
           username: ${{ secrets.AWS_ACCESS_KEY_ID }}
           password: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          
+
       - name: Build and push privately
         uses: docker/build-push-action@v4
         if: github.event_name != 'pull_request'
@@ -230,5 +253,9 @@ jobs:
           file: ./docker/DockerfileHelm
           tags: |
             ${{ env.ECR_REGISTRY }}/${{ env.IMAGE_NAME }}:helm
-          cache-from: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}-helm:buildcache
-          cache-to: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}-helm:buildcache,mode=max
+          cache-from:
+            type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME
+            }}-helm:buildcache
+          cache-to:
+            type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME
+            }}-helm:buildcache,mode=max

.github/workflows/go_on_release.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: actions/setup-go@v3
+      - uses: actions/setup-go@v4
       - name: generate_go
         run: |
           go install github.com/deepmap/oapi-codegen/cmd/oapi-codegen@v1.11.0

CHANGELOG.md

@@ -1,6 +1,89 @@
 # Changelog
 
 
+## [1.87.0](https://github.com/windmill-labs/windmill/compare/v1.86.0...v2.0.0) (2023-04-11)
+
+
+### ⚠ BREAKING CHANGES
+
+* **frontend:** Add option to return file names ([#1380](https://github.com/windmill-labs/windmill/issues/1380))
+
+### Features
+
+* **backend:** add instance events webhook ([f2d3c82](https://github.com/windmill-labs/windmill/commit/f2d3c8208b6daa49f304f355752145de47138a3c))
+* **backend:** extend cached resolution for go ([dac61d1](https://github.com/windmill-labs/windmill/commit/dac61d1c982576d7589e16ab01c8cc8bad6e1686))
+* **backend:** Redis based queue ([#1324](https://github.com/windmill-labs/windmill/issues/1324)) ([d45e6c9](https://github.com/windmill-labs/windmill/commit/d45e6c94abed609357b18d4daa7de6b2ea0ba978))
+* **frontend:** Add option to return file names ([#1380](https://github.com/windmill-labs/windmill/issues/1380)) ([3dabac1](https://github.com/windmill-labs/windmill/commit/3dabac153f302f48210d15ebaec514e72717300f))
+* **python:** cache dependency resolution ([facb670](https://github.com/windmill-labs/windmill/commit/facb67093ce7d3b0874d0d559fb272ed822ce360))
+
+
+### Bug Fixes
+
+* **backend:** nested deno relative imports ([955a213](https://github.com/windmill-labs/windmill/commit/955a213a504c1f3b8811c930823e87fe7dba101a))
+* **cli:** overwrite archived scripts ([1f705ca](https://github.com/windmill-labs/windmill/commit/1f705cab2ce8c79829f22fc6af9e06ecba7450b1))
+* **frontend:** Add missing stopPropagation ([#1394](https://github.com/windmill-labs/windmill/issues/1394)) ([58d4b55](https://github.com/windmill-labs/windmill/commit/58d4b556ebbd76c6f07f1a16d601a9d824b99f7e))
+* **frontend:** fix app init issue ([d0e0e1f](https://github.com/windmill-labs/windmill/commit/d0e0e1fdf27d9a7fb86c66e43398786b64d8b6b7))
+* **frontend:** Fix frontend dependencies ([#1379](https://github.com/windmill-labs/windmill/issues/1379)) ([8e9c491](https://github.com/windmill-labs/windmill/commit/8e9c49165060a4a7f831b8be075593f89d867784))
+* **frontend:** Fix icon picker input ([#1389](https://github.com/windmill-labs/windmill/issues/1389)) ([8a44f8e](https://github.com/windmill-labs/windmill/commit/8a44f8e7796f13698e2a99af9f3772f5e676604b))
+* **frontend:** Fix mac shortcuts ([#1381](https://github.com/windmill-labs/windmill/issues/1381)) ([41831d5](https://github.com/windmill-labs/windmill/commit/41831d58ed593bb283600b76170f6e76783e0eae))
+* **frontend:** fix popover configuration to avoid content shift ([#1377](https://github.com/windmill-labs/windmill/issues/1377)) ([2031e1e](https://github.com/windmill-labs/windmill/commit/2031e1ebd0dc020da104ee84a0294c86babcefaf))
+* **frontend:** remove stopPropagation that was preventing components dnd ([#1378](https://github.com/windmill-labs/windmill/issues/1378)) ([de8dc1e](https://github.com/windmill-labs/windmill/commit/de8dc1e9cd7beea2ce62656e9e7676214f77a110))
+
+
+### Performance Improvements
+
+* parallelize more operations for deno jobs ([e911869](https://github.com/windmill-labs/windmill/commit/e911869d990956463834ac9ff35c52ba8236e362))
+
+## [1.86.0](https://github.com/windmill-labs/windmill/compare/v1.85.0...v1.86.0) (2023-04-08)
+
+
+### Features
+
+* **backend:** add /ready endpoint for workers ([94eecea](https://github.com/windmill-labs/windmill/commit/94eecea02b6295ad5674db4b010bf6ab7984fa17))
+* **backend:** add GET endpoint to trigger scripts ([15c75d9](https://github.com/windmill-labs/windmill/commit/15c75d9d00a69ae97123ed371b9657e298345bdb))
+* **backend:** lowercase all emails in relevant endpoints ([#1361](https://github.com/windmill-labs/windmill/issues/1361)) ([7f9050b](https://github.com/windmill-labs/windmill/commit/7f9050b285cf8f7f6baf05452b673f58988c452c))
+* **cli:** add getFullResource ([3a232db](https://github.com/windmill-labs/windmill/commit/3a232dbb5792c28b26747e1ba260fffcdd4a8416))
+* do cache bucket syncing in background + check tar before pushing it ([#1360](https://github.com/windmill-labs/windmill/issues/1360)) ([3e5ff86](https://github.com/windmill-labs/windmill/commit/3e5ff8682a298ba9e59b2662c4c04c5698447204))
+* **frontend:** add flow expand button ([34a8b01](https://github.com/windmill-labs/windmill/commit/34a8b01b762c0b210d76101e7da7bd2397258e8d))
+* **frontend:** add impersonate api + local resolution of import by lsp v0 ([7675f08](https://github.com/windmill-labs/windmill/commit/7675f08b7bfe319e496a86a7ef1ab7cc8c1d12d2))
+* **frontend:** add workspace to ctx ([8f7a11b](https://github.com/windmill-labs/windmill/commit/8f7a11b8964e2c3405ce3689f9cf2298f9e71c75))
+* **frontend:** Improve login + toasts ([#1363](https://github.com/windmill-labs/windmill/issues/1363)) ([92be102](https://github.com/windmill-labs/windmill/commit/92be102a070b1f17b9d3e40524cd21b54301b5a7))
+* **frontend:** make script editor a single page ([b84be60](https://github.com/windmill-labs/windmill/commit/b84be60c53ca1ef65826123f39099d33c1f549c0))
+* **frontend:** Tone down text + display whole text ([#1366](https://github.com/windmill-labs/windmill/issues/1366)) ([f214d5f](https://github.com/windmill-labs/windmill/commit/f214d5f96b6ac26cd3ef90a6ab696a6dfe02b3f0))
+* improved cron/schedule editor ([#1362](https://github.com/windmill-labs/windmill/issues/1362)) ([17176bb](https://github.com/windmill-labs/windmill/commit/17176bb8d112b35228ce9183f4b2f81abe9e5b6e))
+
+
+### Bug Fixes
+
+* **backend:** allow cors ([8a594a8](https://github.com/windmill-labs/windmill/commit/8a594a89adba9915508884f900f58c4ab53cdfec))
+* **backend:** allow longer name/company ([eff61bb](https://github.com/windmill-labs/windmill/commit/eff61bb8d3496bc1c5be4b1051f99ed4470a47ff))
+* **backend:** always flush bash output ([517b2c9](https://github.com/windmill-labs/windmill/commit/517b2c9cca54628c8ee692d65c05bc2513eaaf22))
+* **backend:** always flush bash output ([7a9091f](https://github.com/windmill-labs/windmill/commit/7a9091fed6aa99201b75bab88d4faddbe041eee4))
+* **backend:** inline script app python fix ([8c72722](https://github.com/windmill-labs/windmill/commit/8c72722710db8e3720b01180b504cbc66e79f5ca))
+* **frontend:** Add FlowGraph display on Safari ([#1351](https://github.com/windmill-labs/windmill/issues/1351)) ([2819b09](https://github.com/windmill-labs/windmill/commit/2819b09ce5011a467e994ee8b1f09cf33145003d))
+* **frontend:** Fix button poppup ([#1368](https://github.com/windmill-labs/windmill/issues/1368)) ([a344928](https://github.com/windmill-labs/windmill/commit/a344928f251d697f53e40c517b0b86bd90e0ad52))
+* **frontend:** Fix connected property ([#1371](https://github.com/windmill-labs/windmill/issues/1371)) ([4af39f0](https://github.com/windmill-labs/windmill/commit/4af39f081bf3d07aaade39e5a5a221741fe8f973))
+* **frontend:** Fix flow templateEditor ([#1367](https://github.com/windmill-labs/windmill/issues/1367)) ([51fc436](https://github.com/windmill-labs/windmill/commit/51fc436456104c2d6a3cd6f6d62f08929e40d450))
+* **frontend:** make croninput a builder rather than a tab ([266b5b0](https://github.com/windmill-labs/windmill/commit/266b5b00da3bd7643eaa5dba1b8c1456f11c5e30))
+* **frontend:** Minor fixes ([#1374](https://github.com/windmill-labs/windmill/issues/1374)) ([76a2a1d](https://github.com/windmill-labs/windmill/commit/76a2a1db363facbaf9a0e9618f169d6cc66e946f))
+* no need to map internal ports to hosts ([#1365](https://github.com/windmill-labs/windmill/issues/1365)) ([4ec035b](https://github.com/windmill-labs/windmill/commit/4ec035b09a58f8859bc576b03c24cc73f335f32d))
+
+## [1.85.0](https://github.com/windmill-labs/windmill/compare/v1.84.1...v1.85.0) (2023-04-03)
+
+
+### Features
+
+* add local cache for folder path used + invalidate cache on folder creation ([018b051](https://github.com/windmill-labs/windmill/commit/018b051781e3f40b9d1da8ccdd5edb1cd49877ba))
+* **frontend:** add agGrid api hooks + ready ([de1e294](https://github.com/windmill-labs/windmill/commit/de1e29492c9aefdfc59f605ba81f7c51a96bf2f3))
+* **frontend:** Add ID renaming popup ([#1344](https://github.com/windmill-labs/windmill/issues/1344)) ([0b8a08c](https://github.com/windmill-labs/windmill/commit/0b8a08cb49644da7c354c3631751e925ac5353b9))
+
+
+### Bug Fixes
+
+* **backend:** improve handling subflow with many depth using tailrec ([8c53598](https://github.com/windmill-labs/windmill/commit/8c53598aba3fb89f4174d1c0ab3912096ac07c96))
+* **backend:** improve subflow processing ([390a988](https://github.com/windmill-labs/windmill/commit/390a988d4c96256a4fbd6a9302fc47a5648c2c43))
+* **frontend:** PDF reader header positioning ([#1350](https://github.com/windmill-labs/windmill/issues/1350)) ([daf8276](https://github.com/windmill-labs/windmill/commit/daf827666b13917f8c9abeab5bb2b072bd0fef0b))
+
 ## [1.84.1](https://github.com/windmill-labs/windmill/compare/v1.84.0...v1.84.1) (2023-03-31)
 
 

Caddyfile

@@ -1,15 +1,5 @@
-{
-        auto_https off
-}
-
-http://{$BASE_URL} {
-        bind {$ADDRESS}
+{$BASE_URL} {
         bind {$ADDRESS}
         reverse_proxy /ws/* http://lsp:3001
         reverse_proxy /* http://windmill_server:8000
-
-https://{$BASE_URL}  {  
-  bind {$ADDRESS}
-  reverse_proxy /ws/* http://lsp:3001
-}
-}
+}

Dockerfile

@@ -85,7 +85,7 @@ COPY .git/ .git/
 RUN CARGO_NET_GIT_FETCH_WITH_CLI=true cargo build --release --features "$features"
 
 
-FROM python:3.11.2-slim-buster
+FROM python:3.11.3-slim-buster
 ARG TARGETPLATFORM
 
 ARG APP=/usr/src/app
@@ -124,6 +124,7 @@ ENV TZ=Etc/UTC
 
 RUN /usr/local/bin/python3 -m pip install pip-tools
 
+COPY --from=frontend /frontend/build /static_frontend
 COPY --from=builder /windmill/target/release/windmill ${APP}/windmill
 
 COPY --from=nsjail /nsjail/nsjail /bin/nsjail

README.md

@@ -70,10 +70,10 @@ https://user-images.githubusercontent.com/275584/218350457-bc2fdc3b-e667-4da5-a2
 
 3. Make it flow! You can chain your scripts or scripts made by the community
    shared on [WindmillHub](https://hub.windmill.dev).
-   ![Step 4](./imgs/windmill-flow.png)
+   ![Step 3](./imgs/windmill-flow.png)
 
 4. Build complex UI on top of your scripts and flows.
-   ![Step 5](./imgs/windmill-builder.png)
+   ![Step 4](./imgs/windmill-builder.png)
 
 Scripts and flows can also be triggered by a cron schedule '_/5 _ \* \* \*' or
 through webhooks.
@@ -123,7 +123,7 @@ instance from local commands. See
 ### Running scripts locally
 
 You can run your script locally easily, you simply need to pass the right
-environment variables for the `wmill` client library to fetch resource and
+environment variables for the `wmill` client library to fetch resources and
 variables from your instance if necessary. See more:
 <https://docs.windmill.dev/docs/advanced/local_development/>
 
@@ -186,14 +186,19 @@ compiling from source or using without a postgres super user, see
 
 ### Docker compose
 
-`docker compose up` with the following docker-compose is sufficient:
-<https://github.com/windmill-labs/windmill/blob/main/docker-compose.yml>
+```
+curl https://github.com/windmill-labs/windmill/blob/main/docker-compose.yml -o docker-compose.yml
+curl https://github.com/windmill-labs/windmill/blob/main/CaddyFile -o Caddyfile
+curl https://github.com/windmill-labs/windmill/blob/main/.env -o .env
+
+docker compose up -d --pull always
+```
 
 Go to http://localhost et voilà :)
 
 The default super-admin user is: admin@windmill.dev / changeme
 
-From there, you can create other users (do not forget to change the password!)
+From there, you can follow the setup app and creat other users.
 
 ### Kubernetes (k8s) and Helm charts
 
@@ -310,6 +315,8 @@ it being synced automatically everyday.
 | DENO_PATH                 | /usr/bin/deno          | The path to the deno binary.                                                                                                                                                                       | Worker                |
 | PYTHON_PATH               | /usr/local/bin/python3 | The path to the python binary.                                                                                                                                                                     | Worker                |
 | GO_PATH                   | /usr/bin/go            | The path to the go binary.                                                                                                                                                                         | Worker                |
+| GOPRIVATE                 |                        | The GOPRIVATE env variable to use private go modules                                                                                                                                               | Worker                |
+| NETRC                     |                        | The netrc content to use a private go registry                                                                                                                                                     | Worker                |
 | PIP_INDEX_URL             | None                   | The index url to pass for pip.                                                                                                                                                                     | Worker                |
 | PIP_EXTRA_INDEX_URL       | None                   | The extra index url to pass to pip.                                                                                                                                                                | Worker                |
 | PIP_TRUSTED_HOST          | None                   | The trusted host to pass to pip.                                                                                                                                                                   | Worker                |
@@ -327,7 +334,7 @@ it being synced automatically everyday.
 | INCLUDE_HEADERS           | None                   | Whitelist of headers that are passed to jobs as args (separated by a comma)                                                                                                                        | Server                |
 | WHITELIST_WORKSPACES      | None                   | Whitelist of workspaces this worker takes job from                                                                                                                                                 | Worker                |
 | BLACKLIST_WORKSPACES      | None                   | Blacklist of workspaces this worker takes job from                                                                                                                                                 | Worker                |
-| NEW_USER_WEBHOOK          | None                   | Webhook to notify of a new user added, signup/invite. Can hook back to windmill to send emails                                                                                                     | Server                |
+| INSTANCE_EVENTS_WEBHOOK   | None                   | Webhook to notify of events such as new user added, signup/invite. Can hook back to windmill to send emails                                                                                        | Server                |
 
 ## Run a local dev setup
 

backend/.cargo/config.toml

@@ -18,4 +18,4 @@ rustflags = [
 rustflags = [
     "-C", "link-arg=-undefined",
     "-C", "link-arg=dynamic_lookup",
-]
+]

backend/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "windmill"
-version = "1.84.1"
+version = "1.87.0"
 authors.workspace = true
 edition.workspace = true
 
@@ -19,7 +19,7 @@ members = [
 ]
 
 [workspace.package]
-version = "1.84.1"
+version = "1.87.0"
 authors = ["Ruben Fiszel <ruben@windmill.dev>"]
 edition = "2021"
 
@@ -28,11 +28,7 @@ name = "windmill"
 path = "./src/main.rs"
 
 [features]
-enterprise = [
-    "windmill-worker/enterprise",
-    "windmill-queue/enterprise",
-    "windmill-api/enterprise",
-]
+enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmill-api/enterprise"]
 
 [dependencies]
 anyhow.workspace = true
@@ -52,12 +48,16 @@ git-version.workspace = true
 rsa.workspace = true
 base64.workspace = true
 sha2.workspace = true
+rsmq_async.workspace = true
+url.workspace = true
+
 
 [dev-dependencies]
 serde_json.workspace = true
 reqwest.workspace = true
 windmill-queue.workspace = true
 axum.workspace = true
+serde.workspace = true
 
 [workspace.dependencies]
 windmill-api = { path = "./windmill-api" }
@@ -76,7 +76,7 @@ headers = "^0"
 hyper = { version = "^0", features = ["full"] }
 tokio = { version = "^1", features = ["full", "tracing"] }
 tower = "^0"
-tower-http = { version = "^0", features = ["trace"] }
+tower-http = { version = "^0", features = ["trace", "cors"] }
 tower-cookies = "^0"
 serde = "^1"
 serde_json = { version = "^1", features = ["preserve_order"] }
@@ -84,6 +84,7 @@ uuid = { version = "^1", features = ["serde", "v4"] }
 thiserror = "^1"
 anyhow = "^1"
 chrono = { version = "^0", features = ["serde"] }
+chrono-tz = "^0"
 tracing = "^0"
 tracing-subscriber = { version = "^0", features = ["env-filter", "json"] }
 prometheus = { version = "^0", default-features = false }
@@ -142,6 +143,7 @@ sqlx = { version = "^0", features = [
 dotenv = "^0"
 ulid = { version = "^1", features = ["uuid"] }
 futures = "^0"
+futures-core = "^0"
 tokio-metrics = "0.1.0"
 lazy_static = "1.4.0"
 serde_derive = "1.0.147"
@@ -153,4 +155,8 @@ async-stripe = { version = "0.14", features = [
     "checkout",
 ] }
 async_zip = { version = "0.0.11", features = ["full"] }
-once_cell = "1.17.1"
+once_cell = "1.17.1"
+rsmq_async = { version = "5.1.5" }
+gosyn = "0.2.2"
+
+[patch.crates-io]

backend/migrations/20230405071524_schedule_timezone.down.sql

@@ -0,0 +1,2 @@
+ALTER TABLE schedule DROP COLUMN timezone;
+ALTER TABLE schedule ADD COLUMN offset_ INTEGER NOT NULL DEFAULT 0;

backend/migrations/20230405071524_schedule_timezone.up.sql

@@ -0,0 +1,26 @@
+ALTER TABLE schedule ADD COLUMN timezone VARCHAR(255) NOT NULL DEFAULT 'UTC';
+
+-- INSERT the correct IANA timezone string for each offset value
+
+UPDATE schedule SET timezone = 'Pacific/Honolulu' WHERE offset_ = 600;
+UPDATE schedule SET timezone = 'America/Anchorage' WHERE offset_ = 540;
+UPDATE schedule SET timezone = 'America/Los_Angeles' WHERE offset_ = 480;
+UPDATE schedule SET timezone = 'America/Chicago' WHERE offset_ = 360;
+UPDATE schedule SET timezone = 'America/New_York' WHERE offset_ = 300;
+UPDATE schedule SET timezone = 'America/Halifax' WHERE offset_ = 240;
+UPDATE schedule SET timezone = 'America/Sao_Paulo' WHERE offset_ = 180;
+UPDATE schedule SET timezone = 'Atlantic/South_Georgia' WHERE offset_ = 120;
+UPDATE schedule SET timezone = 'Atlantic/Cape_Verde' WHERE offset_ = 60;
+UPDATE schedule SET timezone = 'Europe/London' WHERE offset_ = 0;
+UPDATE schedule SET timezone = 'Europe/Berlin' WHERE offset_ = -60;
+UPDATE schedule SET timezone = 'Europe/Athens' WHERE offset_ = -120;
+UPDATE schedule SET timezone = 'Europe/Moscow' WHERE offset_ = -180;
+UPDATE schedule SET timezone = 'Asia/Dubai' WHERE offset_ = -240;
+UPDATE schedule SET timezone = 'Asia/Aqtau' WHERE offset_ = -300;
+UPDATE schedule SET timezone = 'Asia/Almaty' WHERE offset_ = -360;
+UPDATE schedule SET timezone = 'Asia/Bangkok' WHERE offset_ = -420;
+UPDATE schedule SET timezone = 'Asia/Hong_Kong' WHERE offset_ = -480;
+UPDATE schedule SET timezone = 'Asia/Tokyo' WHERE offset_ = -540;
+UPDATE schedule SET timezone = 'Australia/Sydney' WHERE offset_ = -600;
+
+ALTER TABLE schedule DROP COLUMN offset_;

backend/migrations/20230405195753_relax_varchar_restrictions.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230405195753_relax_varchar_restrictions.up.sql

@@ -0,0 +1,3 @@
+-- Add up migration script here
+ALTER TABLE password ALTER COLUMN company TYPE VARCHAR(255);
+ALTER TABLE password ALTER COLUMN name TYPE VARCHAR(255);

backend/migrations/20230408203759_cache_pip_resolution.down.sql

@@ -0,0 +1,2 @@
+-- Add down migration script here
+DROP TABLE pip_resolution_cache;

backend/migrations/20230408203759_cache_pip_resolution.up.sql

@@ -0,0 +1,6 @@
+-- Add up migration script here
+CREATE TABLE pip_resolution_cache(
+    hash VARCHAR(255) PRIMARY KEY,
+    expiration TIMESTAMP NOT NULL,
+    lockfile TEXT NOT NULL
+);

backend/migrations/20230410122342_saved_inputs.down.sql

@@ -0,0 +1,2 @@
+DROP TABLE IF EXISTS input;
+DROP TYPE RUNNABLE_TYPE;

backend/migrations/20230410122342_saved_inputs.up.sql

@@ -0,0 +1,13 @@
+CREATE TYPE RUNNABLE_TYPE AS ENUM ('ScriptHash', 'ScriptPath', 'FlowPath');
+
+CREATE TABLE IF NOT EXISTS input (
+  id UUID PRIMARY KEY,
+  workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id),
+  runnable_id VARCHAR(255) NOT NULL,
+  runnable_type RUNNABLE_TYPE NOT NULL,
+  name TEXT NOT NULL,
+  args JSONB NOT NULL,
+  created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
+  created_by VARCHAR(50) NOT NULL,
+  is_public BOOLEAN NOT NULL DEFAULT FALSE
+);

backend/parsers/windmill-parser-go/Cargo.toml

@@ -15,3 +15,4 @@ phf.workspace = true
 unicode-general-category.workspace = true
 itertools.workspace = true
 anyhow.workspace = true
+gosyn.workspace = true

backend/parsers/windmill-parser-go/src/parser_go_ast.rs

@@ -1,347 +0,0 @@
-#![allow(clippy::large_enum_variant)] // TODO: we allow large enum variant for now, let's profile properly to see if we want to box.
-
-use crate::parser_go_token::{Position, Token};
-use std::collections::BTreeMap;
-
-// https://pkg.go.dev/go/ast#CommentGroup
-#[derive(Debug)]
-pub struct CommentGroup {
-    // List []*Comment // len(List) > 0
-}
-
-// https://pkg.go.dev/go/ast#FieldList
-#[derive(Debug)]
-pub struct FieldList<'a> {
-    pub opening: Option<Position<'a>>, // position of opening parenthesis/brace, if any
-    pub list: Vec<Field<'a>>,          // field list; or nil
-    pub closing: Option<Position<'a>>, // position of closing parenthesis/brace, if any
-}
-
-// https://pkg.go.dev/go/ast#Field
-#[derive(Debug)]
-pub struct Field<'a> {
-    pub doc: Option<CommentGroup>,     // associated documentation; or nil
-    pub names: Option<Vec<Ident<'a>>>, // field/method/(type) parameter names, or type "type"; or nil
-    pub type_: Option<Expr<'a>>,       // field/method/parameter type, type list type; or nil
-    pub tag: Option<BasicLit<'a>>,     // field tag; or nil
-    pub comment: Option<CommentGroup>, // line comments; or nil
-}
-
-// https://pkg.go.dev/go/ast#File
-#[derive(Debug)]
-pub struct File<'a> {
-    // package name
-    pub decls: Vec<Decl<'a>>, // top-level declarations; or nil // list of all comments in the source file
-}
-
-// https://pkg.go.dev/go/ast#FuncDecl
-#[derive(Debug)]
-pub struct FuncDecl<'a> {
-    pub doc: Option<CommentGroup>,   // associated documentation; or nil
-    pub recv: Option<FieldList<'a>>, // receiver (methods); or nil (functions)
-    pub name: Ident<'a>,             // function/method name
-    pub type_: FuncType<'a>, // function signature: type and value parameters, results, and position of "func" keyword
-    pub body: Option<BlockStmt<'a>>, // function body; or nil for external (non-Go) function
-}
-
-// https://pkg.go.dev/go/ast#BlockStmt
-#[derive(Debug)]
-pub struct BlockStmt<'a> {
-    pub lbrace: Position<'a>, // position of "{"
-    pub list: Vec<Stmt>,
-    pub rbrace: Position<'a>, // position of "}", if any (may be absent due to syntax error)
-}
-
-// https://pkg.go.dev/go/ast#FuncType
-#[derive(Debug)]
-pub struct FuncType<'a> {
-    pub func: Option<Position<'a>>, // position of "func" keyword (token.NoPos if there is no "func")
-    pub params: FieldList<'a>,      // (incoming) parameters; non-nil
-    pub results: Option<FieldList<'a>>, // (outgoing) results; or nil
-}
-
-// https://pkg.go.dev/go/ast#Ident
-#[derive(Debug)]
-pub struct Ident<'a> {
-    pub name_pos: Position<'a>,       // identifier position
-    pub name: &'a str,                // identifier name
-    pub obj: Option<Box<Object<'a>>>, // denoted object; or nil
-}
-
-// https://pkg.go.dev/go/ast#ValueSpec
-#[derive(Debug)]
-pub struct ValueSpec<'a> {
-    pub doc: Option<CommentGroup>,     // associated documentation; or nil
-    pub names: Vec<Ident<'a>>,         // value names (len(Names) > 0)
-    pub type_: Option<Expr<'a>>,       // value type; or nil
-    pub values: Option<Vec<Expr<'a>>>, // initial values; or nil
-    pub comment: Option<CommentGroup>, // line comments; or nil
-}
-
-// https://pkg.go.dev/go/ast#BasicLit
-#[derive(Debug)]
-pub struct BasicLit<'a> {
-    pub value_pos: Position<'a>, // literal position
-    pub kind: Token,             // token.INT, token.FLOAT, token.IMAG, token.CHAR, or token.STRING
-    pub value: &'a str, // literal string; e.g. 42, 0x7f, 3.14, 1e-9, 2.4i, 'a', '\x7f', "foo" or `\m\n\o`
-}
-
-// https://pkg.go.dev/go/ast#Object
-#[derive(Debug)]
-pub struct Object<'a> {
-    pub kind: ObjKind,
-    pub name: &'a str,         // declared name
-    pub decl: Option<ObjDecl>, // corresponding Field, XxxSpec, FuncDecl, LabeledStmt, AssignStmt, Scope; or nil
-    pub data: Option<usize>,   // object-specific data; or nil
-    pub type_: Option<()>,     // placeholder for type information; may be nil
-}
-
-// https://pkg.go.dev/go/ast#Ellipsis
-#[derive(Debug)]
-pub struct Ellipsis<'a> {
-    pub ellipsis: Position<'a>,     // position of "..."
-    pub elt: Option<Box<Expr<'a>>>, // ellipsis element type (parameter lists only); or nil
-}
-
-// https://pkg.go.dev/go/ast#Ellipsis
-#[derive(Debug)]
-pub struct TypeAssertExpr<'a> {
-    pub x: Box<Expr<'a>>,     // expression
-    pub lparen: Position<'a>, // position of "("
-    pub type_: Box<Expr<'a>>, // asserted type; nil means type switch X.(type)
-    pub rparen: Position<'a>, // position of ")"
-}
-
-// https://pkg.go.dev/go/ast#SliceExpr
-#[derive(Debug)]
-pub struct SliceExpr<'a> {
-    pub x: Box<Expr<'a>>,            // expression
-    pub lbrack: Position<'a>,        // position of "["
-    pub low: Option<Box<Expr<'a>>>,  // begin of slice range; or nil
-    pub high: Option<Box<Expr<'a>>>, // end of slice range; or nil
-    pub max: Option<Box<Expr<'a>>>,  // maximum capacity of slice; or nil
-    pub slice3: bool,                // true if 3-index slice (2 colons present)
-    pub rbrack: Position<'a>,        // position of "]"
-}
-
-// https://pkg.go.dev/go/ast#ObjKind
-#[derive(Debug)]
-pub enum ObjKind {}
-
-#[derive(Debug)]
-pub enum ObjDecl {}
-
-// https://pkg.go.dev/go/ast#Decl
-#[derive(Debug)]
-pub enum Decl<'a> {
-    FuncDecl(FuncDecl<'a>),
-}
-
-// https://pkg.go.dev/go/ast#Scope
-#[derive(Debug)]
-pub struct Scope<'a> {
-    pub outer: Option<Box<Scope<'a>>>,
-    pub objects: BTreeMap<&'a str, Object<'a>>,
-}
-
-// https://pkg.go.dev/go/ast#GenDecl
-#[derive(Debug)]
-pub struct GenDecl<'a> {
-    pub doc: Option<CommentGroup>,    // associated documentation; or nil
-    pub tok_pos: Position<'a>,        // position of Tok
-    pub tok: Token,                   // IMPORT, CONST, TYPE, or VAR
-    pub lparen: Option<Position<'a>>, // position of '(', if any
-    pub specs: Vec<Spec>,
-    pub rparen: Option<Position<'a>>, // position of ')', if any
-}
-
-// https://pkg.go.dev/go/ast#AssignStmt
-#[derive(Debug)]
-pub struct AssignStmt<'a> {
-    pub lhs: Vec<Expr<'a>>,
-    pub tok_pos: Position<'a>, // position of Tok
-    pub tok: Token,            // assignment token, DEFINE
-    pub rhs: Vec<Expr<'a>>,
-}
-
-// https://pkg.go.dev/go/ast#BinaryExpr
-#[derive(Debug)]
-pub struct BinaryExpr<'a> {
-    pub x: Box<Expr<'a>>,     // left operand
-    pub op_pos: Position<'a>, // position of Op
-    pub op: Token,            // operator
-    pub y: Box<Expr<'a>>,     // right operand
-}
-
-// https://pkg.go.dev/go/ast#ReturnStmt
-#[derive(Debug)]
-pub struct ReturnStmt<'a> {
-    pub return_: Position<'a>,  // position of "return" keyword
-    pub results: Vec<Expr<'a>>, // result expressions; or nil
-}
-
-// https://pkg.go.dev/go/ast#TypeSpec
-#[derive(Debug)]
-pub struct TypeSpec<'a> {
-    pub doc: Option<CommentGroup>,     // associated documentation; or nil
-    pub name: Option<Ident<'a>>,       // type name
-    pub assign: Option<Position<'a>>,  // position of '=', if any
-    pub type_: Expr<'a>, // *Ident, *ParenExpr, *SelectorExpr, *StarExpr, or any of the *XxxTypes
-    pub comment: Option<CommentGroup>, // line comments; or nil
-}
-
-// https://pkg.go.dev/go/ast#StructType
-#[derive(Debug)]
-pub struct StructType<'a> {
-    pub struct_: Position<'a>,         // position of "struct" keyword
-    pub fields: Option<FieldList<'a>>, // list of field declarations
-    pub incomplete: bool,              // true if (source) fields are missing in the Fields list
-}
-
-// https://pkg.go.dev/go/ast#StarExpr
-#[derive(Debug)]
-pub struct StarExpr<'a> {
-    pub star: Position<'a>, // position of "*"
-    pub x: Box<Expr<'a>>,   // operand
-}
-
-// https://pkg.go.dev/go/ast#InterfaceType
-#[derive(Debug)]
-pub struct InterfaceType<'a> {
-    pub interface: Position<'a>,        // position of "interface" keyword
-    pub methods: Option<FieldList<'a>>, // list of embedded interfaces, methods, or types
-    pub incomplete: bool, // true if (source) methods or types are missing in the Methods list
-}
-
-// https://pkg.go.dev/go/ast#UnaryExpr
-#[derive(Debug)]
-pub struct UnaryExpr<'a> {
-    pub op_pos: Position<'a>, // position of Op
-    pub op: Token,            // operator
-    pub x: Box<Expr<'a>>,     // operand
-}
-
-// https://pkg.go.dev/go/ast#CallExpr
-#[derive(Debug)]
-pub struct CallExpr<'a> {
-    pub fun: Box<Expr<'a>>,             // function expression
-    pub lparen: Position<'a>,           // position of "("
-    pub args: Option<Vec<Expr<'a>>>,    // function arguments; or nil
-    pub ellipsis: Option<Position<'a>>, // position of "..." (token.NoPos if there is no "...")
-    pub rparen: Position<'a>,           // position of ")"
-}
-
-// https://pkg.go.dev/go/ast#SelectorExpr
-#[derive(Debug)]
-pub struct SelectorExpr<'a> {
-    pub x: Box<Expr<'a>>, // expression
-    pub sel: Ident<'a>,   // field selector
-}
-
-// https://pkg.go.dev/go/ast#ParenExpr
-#[derive(Debug)]
-pub struct ParenExpr<'a> {
-    pub lparen: Position<'a>, // position of "("
-    pub x: Box<Expr<'a>>,     // parenthesized expression
-    pub rparen: Position<'a>, // position of ")"
-}
-
-// https://pkg.go.dev/go/ast#FuncLit
-#[derive(Debug)]
-pub struct FuncLit<'a> {
-    pub type_: FuncType<'a>, // function type
-    pub body: BlockStmt<'a>, // function body
-}
-
-// https://pkg.go.dev/go/ast#ChanType
-#[derive(Debug)]
-pub struct ChanType<'a> {
-    pub begin: Position<'a>, // position of "chan" keyword or "<-" (whichever comes first)
-    pub arrow: Option<Position<'a>>, // position of "<-" (token.NoPos if there is no "<-")
-    pub dir: u8,             // channel direction
-    pub value: Box<Expr<'a>>, // value type
-}
-
-// htt/opt/visual-studio-code/resources/app/out/vs/code/electron-sandbox/workbench/workbench.htmlps://pkg.go.dev/go/ast#IndexExpr
-#[derive(Debug)]
-pub struct IndexExpr<'a> {
-    pub x: Box<Expr<'a>>,     // expression
-    pub lbrack: Position<'a>, // position of "["
-    pub index: Box<Expr<'a>>, // index expression
-    pub rbrack: Position<'a>, // position of "]"
-}
-
-// https://pkg.go.dev/go/ast#MapType
-#[derive(Debug)]
-pub struct MapType<'a> {
-    pub map: Position<'a>,
-    pub key: Box<Expr<'a>>,
-    pub value: Box<Expr<'a>>,
-}
-
-// https://pkg.go.dev/go/ast#CompositeLit
-#[derive(Debug)]
-pub struct CompositeLit<'a> {
-    pub type_: Box<Expr<'a>>,        // literal type; or nil
-    pub lbrace: Position<'a>,        // position of "{"
-    pub elts: Option<Vec<Expr<'a>>>, // list of composite elements; or nil
-    pub rbrace: Position<'a>,        // position of "}"
-    pub incomplete: bool,            // true if (source) expressions are missing in the Elts list
-}
-
-// https://pkg.go.dev/go/ast#KeyValueExpr
-#[derive(Debug)]
-pub struct KeyValueExpr<'a> {
-    pub key: Box<Expr<'a>>,
-    pub colon: Position<'a>, // position of ":"
-    pub value: Box<Expr<'a>>,
-}
-
-// https://pkg.go.dev/go/ast#ArrayType
-#[derive(Debug)]
-pub struct ArrayType<'a> {
-    pub lbrack: Position<'a>,       // position of "["
-    pub len: Option<Box<Expr<'a>>>, // Ellipsis node for [...]T array types, nil for slice types
-    pub elt: Box<Expr<'a>>,         // element type
-}
-
-// https://pkg.go.dev/go/ast#ChanDir
-#[derive(Debug)]
-pub enum ChanDir {
-    SEND = 1 << 0,
-    RECV = 1 << 1,
-}
-
-// https://pkg.go.dev/go/ast#Spec
-#[derive(Debug)]
-pub enum Spec {}
-
-// https://pkg.go.dev/go/ast#Expr
-#[derive(Debug)]
-pub enum Expr<'a> {
-    ArrayType(ArrayType<'a>),
-    BasicLit(BasicLit<'a>),
-    BinaryExpr(BinaryExpr<'a>),
-    CallExpr(CallExpr<'a>),
-    ChanType(ChanType<'a>),
-    CompositeLit(CompositeLit<'a>),
-    Ellipsis(Ellipsis<'a>),
-    FuncLit(FuncLit<'a>),
-    FuncType(FuncType<'a>),
-    Ident(Ident<'a>),
-    IndexExpr(IndexExpr<'a>),
-    InterfaceType(InterfaceType<'a>),
-    KeyValueExpr(KeyValueExpr<'a>),
-    MapType(MapType<'a>),
-    ParenExpr(ParenExpr<'a>),
-    SelectorExpr(SelectorExpr<'a>),
-    SliceExpr(SliceExpr<'a>),
-    StarExpr(StarExpr<'a>),
-    StructType(StructType<'a>),
-    TypeAssertExpr(TypeAssertExpr<'a>),
-    UnaryExpr(UnaryExpr<'a>),
-}
-
-// https://pkg.go.dev/go/ast#Stmt
-#[derive(Debug)]
-pub enum Stmt {}

backend/parsers/windmill-parser-go/src/parser_go_scanner.rs

@@ -1,948 +0,0 @@
-// https://golang.org/ref/spec#Lexical_elements
-
-use crate::parser_go_token::{Position, Token};
-use phf::{phf_map, Map};
-use std::fmt;
-use unicode_general_category::{get_general_category, GeneralCategory};
-
-pub type Step<'a> = (Position<'a>, Token, &'a str);
-
-#[derive(Debug)]
-pub enum ScannerError {
-    HexadecimalNotFound,
-    OctalNotFound,
-    UnterminatedComment,
-    UnterminatedEscapedChar,
-    UnterminatedRune,
-    UnterminatedString,
-    InvalidDirective,
-}
-
-impl std::error::Error for ScannerError {}
-
-impl fmt::Display for ScannerError {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "scanner error: {:?}", self)
-    }
-}
-
-pub type Result<T> = std::result::Result<T, ScannerError>;
-
-#[derive(Debug)]
-pub struct Scanner<'a> {
-    directory: &'a str,
-    file: &'a str,
-    buffer: &'a str,
-    //
-    chars: std::iter::Peekable<std::str::Chars<'a>>,
-    current_char: Option<char>,
-    current_char_len: usize,
-    //
-    offset: usize,
-    line: usize,
-    column: usize,
-    start_offset: usize,
-    start_line: usize,
-    start_column: usize,
-    //
-    hide_column: bool,
-    insert_semi: bool,
-    pending_line_info: Option<LineInfo<'a>>,
-}
-
-type LineInfo<'a> = (Option<&'a str>, usize, Option<usize>, bool);
-
-impl<'a> Scanner<'a> {
-    pub fn new(filename: &'a str, buffer: &'a str) -> Self {
-        let (directory, file) = filename.rsplit_once('/').unwrap_or(("", filename));
-        let mut s = Scanner {
-            directory,
-            file,
-            buffer,
-            //
-            chars: buffer.chars().peekable(),
-            current_char: None,
-            current_char_len: 0,
-            //
-            offset: 0,
-            line: 1,
-            column: 1,
-            start_offset: 0,
-            start_line: 1,
-            start_column: 1,
-            //
-            hide_column: false,
-            insert_semi: false,
-            pending_line_info: None,
-        };
-        s.next(); // read the first character
-        s
-    }
-
-    #[allow(clippy::cognitive_complexity)] // Allow complex scan function
-    pub fn scan(&mut self) -> Result<Step<'a>> {
-        let insert_semi = self.insert_semi;
-        self.insert_semi = false;
-
-        while let Some(c) = self.current_char {
-            self.reset_start();
-
-            match c {
-                ' ' | '\t' | '\r' => {
-                    self.next();
-                }
-
-                '\n' => {
-                    self.next();
-                    if insert_semi {
-                        return Ok((self.position(), Token::SEMICOLON, "\n"));
-                    }
-                }
-
-                _ => break,
-            }
-        }
-
-        if let Some(c) = self.current_char {
-            match c {
-                '+' => {
-                    self.next();
-                    match self.current_char {
-                        Some('=') => {
-                            self.next();
-                            return Ok((self.position(), Token::ADD_ASSIGN, ""));
-                        }
-                        Some('+') => {
-                            self.insert_semi = true;
-                            self.next();
-                            return Ok((self.position(), Token::INC, ""));
-                        }
-                        _ => return Ok((self.position(), Token::ADD, "")),
-                    }
-                }
-
-                '-' => {
-                    self.next();
-                    match self.current_char {
-                        Some('=') => {
-                            self.next();
-                            return Ok((self.position(), Token::SUB_ASSIGN, ""));
-                        }
-                        Some('-') => {
-                            self.insert_semi = true;
-                            self.next();
-                            return Ok((self.position(), Token::DEC, ""));
-                        }
-                        _ => return Ok((self.position(), Token::SUB, "")),
-                    }
-                }
-
-                '*' => {
-                    self.next();
-                    match self.current_char {
-                        Some('=') => {
-                            self.next();
-                            return Ok((self.position(), Token::MUL_ASSIGN, ""));
-                        }
-                        _ => return Ok((self.position(), Token::MUL, "")),
-                    }
-                }
-
-                '/' => match self.peek() {
-                    Some('=') => {
-                        self.next();
-                        self.next();
-                        return Ok((self.position(), Token::QUO_ASSIGN, ""));
-                    }
-                    Some('/') => {
-                        if insert_semi {
-                            return Ok((self.position(), Token::SEMICOLON, "\n"));
-                        }
-                        return self.scan_line_comment();
-                    }
-                    Some('*') => {
-                        if insert_semi && self.find_line_end() {
-                            return Ok((self.position(), Token::SEMICOLON, "\n"));
-                        }
-                        return self.scan_general_comment();
-                    }
-                    _ => {
-                        self.next();
-                        return Ok((self.position(), Token::QUO, ""));
-                    }
-                },
-
-                '%' => {
-                    self.next();
-                    match self.current_char {
-                        Some('=') => {
-                            self.next();
-                            return Ok((self.position(), Token::REM_ASSIGN, ""));
-                        }
-                        _ => return Ok((self.position(), Token::REM, "")),
-                    }
-                }
-
-                '&' => {
-                    self.next();
-                    match self.current_char {
-                        Some('=') => {
-                            self.next();
-                            return Ok((self.position(), Token::AND_ASSIGN, ""));
-                        }
-                        Some('&') => {
-                            self.next();
-                            return Ok((self.position(), Token::LAND, ""));
-                        }
-                        Some('^') => {
-                            self.next();
-                            match self.current_char {
-                                Some('=') => {
-                                    self.next();
-                                    return Ok((self.position(), Token::AND_NOT_ASSIGN, ""));
-                                }
-                                _ => return Ok((self.position(), Token::AND_NOT, "")),
-                            }
-                        }
-                        _ => return Ok((self.position(), Token::AND, "")),
-                    }
-                }
-
-                '|' => {
-                    self.next();
-                    match self.current_char {
-                        Some('=') => {
-                            self.next();
-                            return Ok((self.position(), Token::OR_ASSIGN, ""));
-                        }
-                        Some('|') => {
-                            self.next();
-                            return Ok((self.position(), Token::LOR, ""));
-                        }
-                        _ => return Ok((self.position(), Token::OR, "")),
-                    }
-                }
-
-                '^' => {
-                    self.next();
-                    match self.current_char {
-                        Some('=') => {
-                            self.next();
-                            return Ok((self.position(), Token::XOR_ASSIGN, ""));
-                        }
-                        _ => return Ok((self.position(), Token::XOR, "")),
-                    }
-                }
-
-                '<' => {
-                    self.next();
-                    match self.current_char {
-                        Some('<') => {
-                            self.next();
-                            match self.current_char {
-                                Some('=') => {
-                                    self.next();
-                                    return Ok((self.position(), Token::SHL_ASSIGN, ""));
-                                }
-                                _ => return Ok((self.position(), Token::SHL, "")),
-                            }
-                        }
-                        Some('=') => {
-                            self.next();
-                            return Ok((self.position(), Token::LEQ, ""));
-                        }
-                        Some('-') => {
-                            self.next();
-                            return Ok((self.position(), Token::ARROW, ""));
-                        }
-                        _ => return Ok((self.position(), Token::LSS, "")),
-                    }
-                }
-
-                '>' => {
-                    self.next();
-                    match self.current_char {
-                        Some('>') => {
-                            self.next();
-                            match self.current_char {
-                                Some('=') => {
-                                    self.next();
-                                    return Ok((self.position(), Token::SHR_ASSIGN, ""));
-                                }
-                                _ => {
-                                    return Ok((self.position(), Token::SHR, ""));
-                                }
-                            }
-                        }
-                        Some('=') => {
-                            self.next();
-                            return Ok((self.position(), Token::GEQ, ""));
-                        }
-                        _ => return Ok((self.position(), Token::GTR, "")),
-                    }
-                }
-
-                ':' => {
-                    self.next();
-                    match self.current_char {
-                        Some('=') => {
-                            self.next();
-                            return Ok((self.position(), Token::DEFINE, ""));
-                        }
-                        _ => return Ok((self.position(), Token::COLON, "")),
-                    }
-                }
-
-                '!' => {
-                    self.next();
-                    match self.current_char {
-                        Some('=') => {
-                            self.next();
-                            return Ok((self.position(), Token::NEQ, ""));
-                        }
-                        _ => return Ok((self.position(), Token::NOT, "")),
-                    }
-                }
-
-                ',' => {
-                    self.next();
-                    return Ok((self.position(), Token::COMMA, ""));
-                }
-
-                '(' => {
-                    self.next();
-                    return Ok((self.position(), Token::LPAREN, ""));
-                }
-
-                ')' => {
-                    self.insert_semi = true;
-                    self.next();
-                    return Ok((self.position(), Token::RPAREN, ""));
-                }
-
-                '[' => {
-                    self.next();
-                    return Ok((self.position(), Token::LBRACK, ""));
-                }
-
-                ']' => {
-                    self.insert_semi = true;
-                    self.next();
-                    return Ok((self.position(), Token::RBRACK, ""));
-                }
-
-                '{' => {
-                    self.next();
-                    return Ok((self.position(), Token::LBRACE, ""));
-                }
-
-                '}' => {
-                    self.insert_semi = true;
-                    self.next();
-                    return Ok((self.position(), Token::RBRACE, ""));
-                }
-
-                ';' => {
-                    self.next();
-                    return Ok((self.position(), Token::SEMICOLON, ";"));
-                }
-
-                '.' => {
-                    self.next();
-                    match self.current_char {
-                        Some('0'..='9') => return self.scan_int_or_float_or_imag(true),
-                        Some('.') => match self.peek() {
-                            Some('.') => {
-                                self.next();
-                                self.next();
-                                return Ok((self.position(), Token::ELLIPSIS, ""));
-                            }
-                            _ => return Ok((self.position(), Token::PERIOD, "")),
-                        },
-                        _ => return Ok((self.position(), Token::PERIOD, "")),
-                    }
-                }
-
-                '=' => {
-                    self.next();
-                    match self.current_char {
-                        Some('=') => {
-                            self.next();
-                            return Ok((self.position(), Token::EQL, ""));
-                        }
-                        _ => return Ok((self.position(), Token::ASSIGN, "")),
-                    }
-                }
-
-                '0'..='9' => return self.scan_int_or_float_or_imag(false),
-                '\'' => return self.scan_rune(),
-                '"' => return self.scan_interpreted_string(),
-                '`' => return self.scan_raw_string(),
-                _ => return self.scan_pkg_or_keyword_or_ident(),
-            };
-        }
-
-        self.reset_start();
-        if insert_semi {
-            Ok((self.position(), Token::SEMICOLON, "\n"))
-        } else {
-            Ok((self.position(), Token::EOF, ""))
-        }
-    }
-
-    // https://golang.org/ref/spec#Keywords
-    // https://golang.org/ref/spec#Identifiers
-    fn scan_pkg_or_keyword_or_ident(&mut self) -> Result<Step<'a>> {
-        self.next();
-
-        while let Some(c) = self.current_char {
-            if !(is_letter(c) || is_unicode_digit(c)) {
-                break;
-            }
-            self.next()
-        }
-
-        let pos = self.position();
-        let literal = self.literal();
-
-        if literal.len() > 1 {
-            if let Some(&token) = KEYWORDS.get(literal) {
-                self.insert_semi = matches!(
-                    token,
-                    Token::BREAK | Token::CONTINUE | Token::FALLTHROUGH | Token::RETURN
-                );
-                return Ok((pos, token, literal));
-            }
-        }
-
-        self.insert_semi = true;
-        Ok((pos, Token::IDENT, literal))
-    }
-
-    // https://golang.org/ref/spec#Integer_literals
-    // https://golang.org/ref/spec#Floating-point_literals
-    // https://golang.org/ref/spec#Imaginary_literals
-    fn scan_int_or_float_or_imag(&mut self, preceding_dot: bool) -> Result<Step<'a>> {
-        self.insert_semi = true;
-
-        let mut token = Token::INT;
-        let mut digits = "_0123456789";
-        let mut exp = "eE";
-
-        if !preceding_dot {
-            if matches!(self.current_char, Some('0')) {
-                self.next();
-                match self.current_char {
-                    Some('b' | 'B') => {
-                        digits = "_01";
-                        exp = "";
-                        self.next();
-                    }
-                    Some('o' | 'O') => {
-                        digits = "_01234567";
-                        exp = "";
-                        self.next();
-                    }
-                    Some('x' | 'X') => {
-                        digits = "_0123456789abcdefABCDEF";
-                        exp = "pP";
-                        self.next();
-                    }
-                    _ => {}
-                };
-            }
-
-            while let Some(c) = self.current_char {
-                if !digits.contains(c) {
-                    break;
-                }
-                self.next();
-            }
-        }
-
-        if preceding_dot || matches!(self.current_char, Some('.')) {
-            token = Token::FLOAT;
-            self.next();
-            while let Some(c) = self.current_char {
-                if !digits.contains(c) {
-                    break;
-                }
-                self.next();
-            }
-        }
-
-        if !exp.is_empty() {
-            if let Some(c) = self.current_char {
-                if exp.contains(c) {
-                    token = Token::FLOAT;
-                    self.next();
-                    if matches!(self.current_char, Some('-' | '+')) {
-                        self.next();
-                    }
-                    while let Some(c) = self.current_char {
-                        if !matches!(c, '_' | '0'..='9') {
-                            break;
-                        }
-                        self.next();
-                    }
-                }
-            }
-        }
-
-        if matches!(self.current_char, Some('i')) {
-            token = Token::IMAG;
-            self.next();
-        }
-
-        Ok((self.position(), token, self.literal()))
-    }
-
-    // https://golang.org/ref/spec#Rune_literals
-    fn scan_rune(&mut self) -> Result<Step<'a>> {
-        self.insert_semi = true;
-        self.next();
-
-        match self.current_char {
-            Some('\\') => self.require_escaped_char::<'\''>()?,
-            Some(_) => self.next(),
-            _ => return Err(ScannerError::UnterminatedRune),
-        }
-
-        if matches!(self.current_char, Some('\'')) {
-            self.next();
-            return Ok((self.position(), Token::CHAR, self.literal()));
-        }
-
-        Err(ScannerError::UnterminatedRune)
-    }
-
-    // https://golang.org/ref/spec#String_literals
-    fn scan_interpreted_string(&mut self) -> Result<Step<'a>> {
-        self.insert_semi = true;
-        self.next();
-
-        while let Some(c) = self.current_char {
-            match c {
-                '"' => {
-                    self.next();
-                    return Ok((self.position(), Token::STRING, self.literal()));
-                }
-                '\\' => self.require_escaped_char::<'"'>()?,
-                _ => self.next(),
-            }
-        }
-
-        Err(ScannerError::UnterminatedString)
-    }
-
-    // https://golang.org/ref/spec#String_literals
-    fn scan_raw_string(&mut self) -> Result<Step<'a>> {
-        self.insert_semi = true;
-        self.next();
-
-        while let Some(c) = self.current_char {
-            match c {
-                '`' => {
-                    self.next();
-                    return Ok((self.position(), Token::STRING, self.literal()));
-                }
-                _ => self.next(),
-            }
-        }
-
-        Err(ScannerError::UnterminatedString)
-    }
-
-    // https://golang.org/ref/spec#Comments
-    fn scan_general_comment(&mut self) -> Result<Step<'a>> {
-        self.next();
-        self.next();
-
-        while let Some(c) = self.current_char {
-            match c {
-                '*' => {
-                    self.next();
-                    if matches!(self.current_char, Some('/')) {
-                        self.next();
-
-                        let pos = self.position();
-                        let lit = self.literal();
-
-                        // look for compiler directives
-                        self.directive(&lit["/*".len()..lit.len() - "*/".len()], true)?;
-
-                        return Ok((pos, Token::COMMENT, lit));
-                    }
-                }
-                _ => self.next(),
-            }
-        }
-
-        Err(ScannerError::UnterminatedComment)
-    }
-
-    // https://golang.org/ref/spec#Comments
-    fn scan_line_comment(&mut self) -> Result<Step<'a>> {
-        self.next();
-        self.next();
-
-        while let Some(c) = self.current_char {
-            if is_newline(c) {
-                break;
-            }
-            self.next();
-        }
-
-        let pos = self.position();
-        let lit = self.literal();
-
-        // look for compiler directives (at the beginning of line)
-        if self.start_column == 1 {
-            self.directive(lit["//".len()..].trim_end(), false)?;
-        }
-
-        Ok((pos, Token::COMMENT, self.literal()))
-    }
-
-    // https://pkg.go.dev/cmd/compile#hdr-Compiler_Directives
-    fn directive(&mut self, input: &'a str, immediate: bool) -> Result<()> {
-        if let Some(line_directive) = input.strip_prefix("line ") {
-            self.pending_line_info = self.parse_line_directive(line_directive)?;
-            if immediate {
-                self.consume_pending_line_info();
-            }
-        }
-        Ok(())
-    }
-
-    fn parse_line_directive(&mut self, line_directive: &'a str) -> Result<Option<LineInfo<'a>>> {
-        if let Some((file, line)) = line_directive.rsplit_once(':') {
-            let line = line.parse().map_err(|_| ScannerError::InvalidDirective)?;
-
-            if let Some((file, l)) = file.rsplit_once(':') {
-                if let Ok(l) = l.parse() {
-                    //line :line:col
-                    //line filename:line:col
-                    /*line :line:col*/
-                    /*line filename:line:col*/
-                    let file = if !file.is_empty() { Some(file) } else { None };
-                    let col = Some(line);
-                    let line = l;
-                    let hide_column = false;
-                    return Ok(Some((file, line, col, hide_column)));
-                }
-            }
-
-            //line :line
-            //line filename:line
-            /*line :line*/
-            /*line filename:line*/
-            Ok(Some((Some(file), line, None, true)))
-        } else {
-            Ok(None)
-        }
-    }
-
-    const fn find_line_end(&self) -> bool {
-        let buffer = self.buffer.as_bytes();
-        let mut in_comment = true;
-
-        let mut i = self.offset;
-        let max = self.buffer.len();
-        while i < max {
-            let c = buffer[i] as char;
-
-            if i < max - 1 {
-                let n = buffer[i + 1] as char;
-
-                if !in_comment && c == '/' && n == '/' {
-                    return true;
-                }
-
-                if c == '/' && n == '*' {
-                    i += 2;
-                    in_comment = true;
-                    continue;
-                }
-
-                if c == '*' && n == '/' {
-                    i += 2;
-                    in_comment = false;
-                    continue;
-                }
-            }
-
-            if is_newline(c) {
-                return true;
-            }
-
-            if !in_comment && !matches!(c, ' ' | '\t' | '\r') {
-                return false;
-            }
-
-            i += 1;
-        }
-
-        !in_comment
-    }
-
-    fn consume_pending_line_info(&mut self) {
-        if let Some(line_info) = self.pending_line_info.take() {
-            if let Some(file) = line_info.0 {
-                self.file = file;
-            }
-
-            self.line = line_info.1;
-
-            if let Some(column) = line_info.2 {
-                self.column = column;
-            }
-
-            self.hide_column = line_info.3;
-        }
-    }
-
-    fn peek(&mut self) -> Option<char> {
-        self.chars.peek().copied()
-    }
-
-    fn next(&mut self) {
-        self.offset += self.current_char_len;
-        self.column += self.current_char_len;
-        let last_char = self.current_char;
-
-        self.current_char = self.chars.next();
-        if let Some(c) = self.current_char {
-            self.current_char_len = c.len_utf8();
-            if matches!(last_char, Some('\n')) {
-                self.line += 1;
-                self.column = 1;
-                self.consume_pending_line_info();
-            }
-        } else {
-            self.current_char_len = 0
-        }
-    }
-
-    const fn position(&self) -> Position<'a> {
-        Position {
-            directory: self.directory,
-            file: self.file,
-            offset: self.start_offset,
-            line: self.start_line,
-            column: if self.hide_column {
-                0
-            } else {
-                self.start_column
-            },
-        }
-    }
-
-    fn reset_start(&mut self) {
-        self.start_offset = self.offset;
-        self.start_line = self.line;
-        self.start_column = self.column;
-    }
-
-    fn literal(&self) -> &'a str {
-        &self.buffer[self.start_offset..self.offset]
-    }
-
-    fn require_escaped_char<const DELIM: char>(&mut self) -> Result<()> {
-        self.next();
-
-        let c = self
-            .current_char
-            .ok_or(ScannerError::UnterminatedEscapedChar)?;
-
-        // TODO: move this to the match when const generics can be referenced in patterns
-        if c == DELIM {
-            self.next();
-            return Ok(());
-        }
-
-        match c {
-            'a' | 'b' | 'f' | 'n' | 'r' | 't' | 'v' | '\\' => self.next(),
-            'x' => {
-                self.next();
-                self.require_hex_digits::<2>()?
-            }
-            'u' => {
-                self.next();
-                self.require_hex_digits::<4>()?;
-            }
-            'U' => {
-                self.next();
-                self.require_hex_digits::<8>()?;
-            }
-            '0'..='7' => self.require_octal_digits::<3>()?,
-            _ => return Err(ScannerError::UnterminatedEscapedChar),
-        }
-
-        Ok(())
-    }
-
-    fn require_octal_digits<const COUNT: usize>(&mut self) -> Result<()> {
-        for _ in 0..COUNT {
-            let c = self.current_char.ok_or(ScannerError::OctalNotFound)?;
-
-            if !is_octal_digit(c) {
-                return Err(ScannerError::OctalNotFound);
-            }
-
-            self.next();
-        }
-
-        Ok(())
-    }
-
-    fn require_hex_digits<const COUNT: usize>(&mut self) -> Result<()> {
-        for _ in 0..COUNT {
-            let c = self.current_char.ok_or(ScannerError::HexadecimalNotFound)?;
-
-            if !is_hex_digit(c) {
-                return Err(ScannerError::HexadecimalNotFound);
-            }
-
-            self.next();
-        }
-
-        Ok(())
-    }
-}
-
-impl<'a> IntoIterator for Scanner<'a> {
-    type Item = Result<Step<'a>>;
-    type IntoIter = IntoIter<'a>;
-
-    fn into_iter(self) -> Self::IntoIter {
-        Self::IntoIter::new(self)
-    }
-}
-
-pub struct IntoIter<'a> {
-    scanner: Scanner<'a>,
-    done: bool,
-}
-
-impl<'a> IntoIter<'a> {
-    const fn new(scanner: Scanner<'a>) -> Self {
-        Self { scanner, done: false }
-    }
-}
-
-impl<'a> Iterator for IntoIter<'a> {
-    type Item = Result<Step<'a>>;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        if self.done {
-            return None;
-        }
-
-        match self.scanner.scan() {
-            Ok((pos, tok, lit)) => {
-                if tok == Token::EOF {
-                    self.done = true;
-                }
-                Some(Ok((pos, tok, lit)))
-            }
-            Err(err) => {
-                self.done = true;
-                Some(Err(err))
-            }
-        }
-    }
-}
-
-// https://golang.org/ref/spec#Letters_and_digits
-
-fn is_letter(c: char) -> bool {
-    c == '_' || is_unicode_letter(c)
-}
-
-//const fn is_decimal_digit(c: char) -> bool {
-//matches!(c, '0'..='9')
-//}
-
-//const fn is_binary_digit(c: char) -> bool {
-//matches!(c, '0'..='1')
-//}
-
-const fn is_octal_digit(c: char) -> bool {
-    matches!(c, '0'..='7')
-}
-
-const fn is_hex_digit(c: char) -> bool {
-    matches!(c, '0'..='9' | 'A'..='F' | 'a'..='f')
-}
-
-// https://golang.org/ref/spec#Characters
-
-const fn is_newline(c: char) -> bool {
-    c == '\n'
-}
-
-//const fn is_unicode_char(c: char) -> bool {
-//c != '\n'
-//}
-
-fn is_unicode_letter(c: char) -> bool {
-    matches!(
-        get_general_category(c),
-        GeneralCategory::UppercaseLetter
-            | GeneralCategory::LowercaseLetter
-            | GeneralCategory::TitlecaseLetter
-            | GeneralCategory::ModifierLetter
-            | GeneralCategory::OtherLetter
-    )
-}
-
-fn is_unicode_digit(c: char) -> bool {
-    get_general_category(c) == GeneralCategory::DecimalNumber
-}
-
-// https://golang.org/ref/spec#Keywords
-
-static KEYWORDS: Map<&'static str, Token> = phf_map! {
-  "break" => Token::BREAK,
-  "case" => Token::CASE,
-  "chan" => Token::CHAN,
-  "const" => Token::CONST,
-  "continue" => Token::CONTINUE,
-
-  "default" => Token::DEFAULT,
-  "defer" => Token::DEFER,
-  "else" => Token::ELSE,
-  "fallthrough" => Token::FALLTHROUGH,
-  "for" => Token::FOR,
-
-  "func" => Token::FUNC,
-  "go" => Token::GO,
-  "goto" => Token::GOTO,
-  "if" => Token::IF,
-  "import" => Token::IMPORT,
-
-  "interface" => Token::INTERFACE,
-  "map" => Token::MAP,
-  "package" => Token::PACKAGE,
-  "range" => Token::RANGE,
-  "return" => Token::RETURN,
-
-  "select" => Token::SELECT,
-  "struct" => Token::STRUCT,
-  "switch" => Token::SWITCH,
-  "type" => Token::TYPE,
-  "var" => Token::VAR,
-};
-
-#[cfg(test)]
-mod tests {
-    use super::Scanner;
-
-    #[test] // fuzz
-    fn it_should_return_an_error_on_missing_line_number() {
-        let input = "/*line :*/";
-        let mut out: Vec<_> = Scanner::new(file!(), input).into_iter().collect();
-        assert!(out.pop().unwrap().is_err());
-    }
-}

backend/parsers/windmill-parser-go/src/parser_go_token.rs

@@ -1,273 +0,0 @@
-// https://cs.opensource.google/go/go/+/refs/tags/go1.17.2:src/go/token/token.go
-
-#![allow(non_camel_case_types)] // For consistency with the Go tokens
-
-use std::fmt;
-
-#[derive(Clone, Copy, Debug, Default)]
-pub struct Position<'a> {
-    pub directory: &'a str,
-    pub file: &'a str,
-    pub offset: usize,
-    pub line: usize,
-    pub column: usize,
-}
-
-impl<'a> fmt::Display for Position<'a> {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        if self.file.is_empty() {
-            write!(f, ":{}:{}", self.line, self.column)
-        } else if self.file.starts_with('/') {
-            write!(f, "{}:{}:{}", self.file, self.line, self.column)
-        } else {
-            write!(
-                f,
-                "{}/{}:{}:{}",
-                self.directory, self.file, self.line, self.column
-            )
-        }
-    }
-}
-
-#[derive(Debug, Clone, Copy, Eq, PartialEq)]
-pub enum Token {
-    EOF,
-    COMMENT,
-
-    IDENT,  // main
-    INT,    // 12345
-    FLOAT,  // 123.45
-    IMAG,   // 123.45i
-    CHAR,   // 'a'
-    STRING, // "abc"
-
-    ADD, // +
-    SUB, // -
-    MUL, // *
-    QUO, // /
-    REM, // %
-
-    AND,     // &
-    OR,      // |
-    XOR,     // ^
-    SHL,     // <<
-    SHR,     // >>
-    AND_NOT, // &^
-
-    ADD_ASSIGN, // +=
-    SUB_ASSIGN, // -=
-    MUL_ASSIGN, // *=
-    QUO_ASSIGN, // /=
-    REM_ASSIGN, // %=
-
-    AND_ASSIGN,     // &=
-    OR_ASSIGN,      // |=
-    XOR_ASSIGN,     // ^=
-    SHL_ASSIGN,     // <<=
-    SHR_ASSIGN,     // >>=
-    AND_NOT_ASSIGN, // &^=
-
-    LAND,  // &&
-    LOR,   // ||
-    ARROW, // <-
-    INC,   // ++
-    DEC,   // --
-
-    EQL,    // ==
-    LSS,    // <
-    GTR,    // >
-    ASSIGN, // =
-    NOT,    // !
-
-    NEQ,      // !=
-    LEQ,      // <=
-    GEQ,      // >=
-    DEFINE,   // :=
-    ELLIPSIS, // ...
-
-    LPAREN, // (
-    LBRACK, // [
-    LBRACE, // {
-    COMMA,  // ,
-    PERIOD, // .
-
-    RPAREN,    // )
-    RBRACK,    // ]
-    RBRACE,    // }
-    SEMICOLON, // ;
-    COLON,     // :
-
-    BREAK,
-    CASE,
-    CHAN,
-    CONST,
-    CONTINUE,
-
-    DEFAULT,
-    DEFER,
-    ELSE,
-    FALLTHROUGH,
-    FOR,
-
-    FUNC,
-    GO,
-    GOTO,
-    IF,
-    IMPORT,
-
-    INTERFACE,
-    MAP,
-    PACKAGE,
-    RANGE,
-    RETURN,
-
-    SELECT,
-    STRUCT,
-    SWITCH,
-    TYPE,
-    VAR,
-}
-
-impl Token {
-    pub const fn is_assign_op(&self) -> bool {
-        use Token::*;
-        matches!(
-            self,
-            ADD_ASSIGN
-                | SUB_ASSIGN
-                | MUL_ASSIGN
-                | QUO_ASSIGN
-                | REM_ASSIGN
-                | AND_ASSIGN
-                | OR_ASSIGN
-                | XOR_ASSIGN
-                | SHL_ASSIGN
-                | SHR_ASSIGN
-                | AND_NOT_ASSIGN
-        )
-    }
-
-    // https://go.dev/ref/spec#Operator_precedence
-    pub fn precedence(&self) -> u8 {
-        use Token::*;
-        match self {
-            MUL | QUO | REM | SHL | SHR | AND | AND_NOT => 5,
-            ADD | SUB | OR | XOR => 4,
-            EQL | NEQ | LSS | LEQ | GTR | GEQ => 3,
-            LAND => 2,
-            LOR => 1,
-            _ => unreachable!(
-                "precedence() is only supported for binary operators, called with: {:?}",
-                self
-            ),
-        }
-    }
-
-    pub const fn lowest_precedence() -> u8 {
-        0
-    }
-}
-
-impl From<&Token> for &'static str {
-    fn from(token: &Token) -> Self {
-        use Token::*;
-
-        match token {
-            EOF => "EOF",
-            COMMENT => "COMMENT",
-
-            IDENT => "IDENT",
-            INT => "INT",
-            FLOAT => "FLOAT",
-            IMAG => "IMAG",
-            CHAR => "CHAR",
-            STRING => "STRING",
-
-            ADD => "+",
-            SUB => "-",
-            MUL => "*",
-            QUO => "/",
-            REM => "%",
-
-            AND => "&",
-            OR => "|",
-            XOR => "^",
-            SHL => "<<",
-            SHR => ">>",
-            AND_NOT => "&^",
-
-            ADD_ASSIGN => "+=",
-            SUB_ASSIGN => "-=",
-            MUL_ASSIGN => "*=",
-            QUO_ASSIGN => "/=",
-            REM_ASSIGN => "%=",
-
-            AND_ASSIGN => "&=",
-            OR_ASSIGN => "|=",
-            XOR_ASSIGN => "^=",
-            SHL_ASSIGN => "<<=",
-            SHR_ASSIGN => ">>=",
-            AND_NOT_ASSIGN => "&^=",
-
-            LAND => "&&",
-            LOR => "||",
-            ARROW => "<-",
-            INC => "++",
-            DEC => "--",
-
-            EQL => "==",
-            LSS => "<",
-            GTR => ">",
-            ASSIGN => "=",
-            NOT => "!",
-
-            NEQ => "!=",
-            LEQ => "<=",
-            GEQ => ">=",
-            DEFINE => ":=",
-            ELLIPSIS => "...",
-
-            LPAREN => "(",
-            LBRACK => "[",
-            LBRACE => "{",
-            COMMA => ",",
-            PERIOD => ".",
-
-            RPAREN => ")",
-            RBRACK => "]",
-            RBRACE => "}",
-            SEMICOLON => ";",
-            COLON => ":",
-
-            BREAK => "break",
-            CASE => "case",
-            CHAN => "chan",
-            CONST => "const",
-            CONTINUE => "continue",
-
-            DEFAULT => "default",
-            DEFER => "defer",
-            ELSE => "else",
-            FALLTHROUGH => "fallthrough",
-            FOR => "for",
-
-            FUNC => "func",
-            GO => "go",
-            GOTO => "goto",
-            IF => "if",
-            IMPORT => "import",
-
-            INTERFACE => "interface",
-            MAP => "map",
-            PACKAGE => "package",
-            RANGE => "range",
-            RETURN => "return",
-
-            SELECT => "select",
-            STRUCT => "struct",
-            SWITCH => "switch",
-            TYPE => "type",
-            VAR => "var",
-        }
-    }
-}

backend/parsers/windmill-parser-py/src/lib.rs

@@ -218,7 +218,7 @@ pub fn parse_python_imports(code: &str) -> error::Result<Vec<String>> {
         let ast = parser::parse_program(code, "main.py").map_err(|e| {
             error::Error::ExecutionErr(format!("Error parsing code: {}", e.to_string()))
         })?;
-        let imports = ast
+        let mut imports: Vec<String> = ast
             .into_iter()
             .filter_map(|x| match x {
                 Located { node, .. } => match node {
@@ -250,7 +250,7 @@ pub fn parse_python_imports(code: &str) -> error::Result<Vec<String>> {
             .filter(|x| !STDIMPORTS.contains(&x.as_str()))
             .unique()
             .collect();
-
+        imports.sort();
         Ok(imports)
     }
 }
@@ -460,7 +460,7 @@ def main():
 ";
         let r = parse_python_imports(code)?;
         // println!("{}", serde_json::to_string(&r)?);
-        assert_eq!(r, vec!["wmill", "zanzibar", "matplotlib", "requests"]);
+        assert_eq!(r, vec!["matplotlib", "requests", "wmill", "zanzibar"]);
         Ok(())
     }
 

backend/parsers/windmill-parser-ts/src/lib.rs

@@ -6,21 +6,22 @@
  * LICENSE-AGPL for a copy of the license.
  */
 use deno_core::{serde_v8, v8, JsRuntime, RuntimeOptions};
+use serde_json::Value;
 use windmill_common::error;
 use windmill_parser::{json_to_typ, Arg, MainArgSignature, ObjectProperty, Typ};
 
-use swc_common::{sync::Lrc, FileName, SourceMap, SourceMapper, Spanned};
+use swc_common::{sync::Lrc, FileName, SourceMap, SourceMapper, Span, Spanned};
 use swc_ecma_ast::{
     ArrayLit, AssignPat, BigInt, BindingIdent, Bool, Decl, ExportDecl, Expr, FnDecl, Ident, Lit,
-    ModuleDecl, ModuleItem, Number, ObjectLit, Pat, Str, TsArrayType, TsEntityName, TsKeywordType,
-    TsKeywordTypeKind, TsLit, TsLitType, TsOptionalType, TsPropertySignature, TsType,
-    TsTypeElement, TsTypeLit, TsTypeRef, TsUnionOrIntersectionType, TsUnionType,
+    ModuleDecl, ModuleItem, Number, ObjectLit, Param, Pat, Str, TsArrayType, TsEntityName,
+    TsKeywordType, TsKeywordTypeKind, TsLit, TsLitType, TsOptionalType, TsPropertySignature,
+    TsType, TsTypeElement, TsTypeLit, TsTypeRef, TsUnionOrIntersectionType, TsUnionType,
 };
 use swc_ecma_parser::{lexer::Lexer, Parser, StringInput, Syntax, TsConfig};
 
-pub fn parse_deno_signature(code: &str) -> error::Result<MainArgSignature> {
+pub fn parse_deno_signature(code: &str, skip_dflt: bool) -> error::Result<MainArgSignature> {
     let cm: Lrc<SourceMap> = Default::default();
-    let fm = cm.new_source_file(FileName::Custom("test.ts".into()), code.into());
+    let fm = cm.new_source_file(FileName::Custom("main.ts".into()), code.into());
     let lexer = Lexer::new(
         // We want to parse ecmascript
         Syntax::Typescript(TsConfig::default()),
@@ -54,65 +55,17 @@ pub fn parse_deno_signature(code: &str) -> error::Result<MainArgSignature> {
         })) if &sym.to_string() == "main" => Some(function.params),
         _ => None,
     });
+
     if let Some(params) = params {
-        Ok(MainArgSignature {
+        let r = MainArgSignature {
             star_args: false,
             star_kwargs: false,
             args: params
                 .into_iter()
-                .map(|x| match x.pat {
-                    Pat::Ident(ident) => {
-                        let (name, typ, nullable) = binding_ident_to_arg(&ident);
-                        Ok(Arg {
-                            otyp: None,
-                            name,
-                            typ,
-                            default: None,
-                            has_default: ident.id.optional || nullable,
-                        })
-                    }
-                    Pat::Assign(AssignPat { left, right, .. }) => {
-                        let (name, mut typ, _nullable) =
-                            left.as_ident().map(binding_ident_to_arg).ok_or_else(|| {
-                                error::Error::ExecutionErr(format!(
-                                    "parameter syntax unsupported: `{}`",
-                                    cm.span_to_snippet(left.span())
-                                        .unwrap_or_else(|_| cm.span_to_string(left.span()))
-                                ))
-                            })?;
-
-                        let span = match *right {
-                            Expr::Lit(Lit::Str(Str { span, .. })) => Some(span),
-                            Expr::Lit(Lit::Num(Number { span, .. })) => Some(span),
-                            Expr::Lit(Lit::BigInt(BigInt { span, .. })) => Some(span),
-                            Expr::Lit(Lit::Bool(Bool { span, .. })) => Some(span),
-                            Expr::Object(ObjectLit { span, .. }) => Some(span),
-                            Expr::Array(ArrayLit { span, .. }) => Some(span),
-                            _ => None,
-                        };
-                        let expr = span
-                            .and_then(|x| cm.span_to_snippet(x).ok())
-                            .map(|x| serde_json::from_str(&x).map_err(|_| x));
-
-                        let default = match expr.clone() {
-                            Some(Ok(x)) => Some(x),
-                            Some(Err(x)) => eval_sync(&x).ok(),
-                            None => None,
-                        };
-
-                        if typ == Typ::Unknown && default.is_some() {
-                            typ = json_to_typ(default.as_ref().unwrap());
-                        }
-                        Ok(Arg { otyp: None, name, typ, default, has_default: true })
-                    }
-                    _ => Err(error::Error::ExecutionErr(format!(
-                        "parameter syntax unsupported: `{}`",
-                        cm.span_to_snippet(x.span())
-                            .unwrap_or_else(|_| cm.span_to_string(x.span()))
-                    ))),
-                })
+                .map(|x| parse_param(x, &cm, skip_dflt))
                 .collect::<Result<Vec<Arg>, error::Error>>()?,
-        })
+        };
+        Ok(r)
     } else {
         Err(error::Error::ExecutionErr(
             "main function was not findable (expected to find 'export function main(...)'"
@@ -121,6 +74,75 @@ pub fn parse_deno_signature(code: &str) -> error::Result<MainArgSignature> {
     }
 }
 
+fn parse_param(x: Param, cm: &Lrc<SourceMap>, skip_dflt: bool) -> error::Result<Arg> {
+    let r = match x.pat {
+        Pat::Ident(ident) => {
+            let (name, typ, nullable) = binding_ident_to_arg(&ident);
+            Ok(Arg {
+                otyp: None,
+                name,
+                typ,
+                default: None,
+                has_default: ident.id.optional || nullable,
+            })
+        }
+        Pat::Assign(AssignPat { left, right, .. }) => {
+            let (name, mut typ, _nullable) =
+                left.as_ident().map(binding_ident_to_arg).ok_or_else(|| {
+                    error::Error::ExecutionErr(format!(
+                        "parameter syntax unsupported: `{}`",
+                        cm.span_to_snippet(left.span())
+                            .unwrap_or_else(|_| cm.span_to_string(left.span()))
+                    ))
+                })?;
+
+            let dflt = if skip_dflt {
+                None
+            } else {
+                match *right {
+                    Expr::Lit(Lit::Str(Str { value, .. })) => {
+                        Some(Value::String(value.to_string()))
+                    }
+                    Expr::Lit(Lit::Num(Number { value, .. }))
+                        if (value == (value as u64) as f64) =>
+                    {
+                        Some(serde_json::json!(value as u64))
+                    }
+                    Expr::Lit(Lit::Num(Number { value, .. })) => Some(serde_json::json!(value)),
+                    Expr::Lit(Lit::BigInt(BigInt { value, .. })) => Some(serde_json::json!(value)),
+                    Expr::Lit(Lit::Bool(Bool { value, .. })) => Some(Value::Bool(value)),
+                    Expr::Object(ObjectLit { span, .. }) => eval_span(span, cm),
+                    Expr::Array(ArrayLit { span, .. }) => eval_span(span, cm),
+                    _ => None,
+                }
+            };
+
+            if typ == Typ::Unknown && dflt.is_some() {
+                typ = json_to_typ(dflt.as_ref().unwrap());
+            }
+            Ok(Arg { otyp: None, name, typ, default: dflt, has_default: true })
+        }
+        _ => Err(error::Error::ExecutionErr(format!(
+            "parameter syntax unsupported: `{}`",
+            cm.span_to_snippet(x.span())
+                .unwrap_or_else(|_| cm.span_to_string(x.span()))
+        ))),
+    };
+    r
+}
+
+fn eval_span(span: Span, cm: &Lrc<SourceMap>) -> Option<Value> {
+    let expr = cm
+        .span_to_snippet(span)
+        .ok()
+        .map(|x| serde_json::from_str(&x).map_err(|_| x));
+
+    match expr {
+        Some(Ok(x)) => Some(x),
+        Some(Err(x)) => eval_sync(&x).ok(),
+        None => None,
+    }
+}
 fn binding_ident_to_arg(BindingIdent { id, type_ann }: &BindingIdent) -> (String, Typ, bool) {
     let (typ, nullable) = type_ann
         .as_ref()
@@ -282,7 +304,7 @@ export function main(test1?: string, test2: string = \"burkina\",
 }
 ";
         assert_eq!(
-            parse_deno_signature(code)?,
+            parse_deno_signature(code, false)?,
             MainArgSignature {
                 star_args: false,
                 star_kwargs: false,
@@ -394,7 +416,7 @@ export function main(test2 = \"burkina\",
 }
 ";
         assert_eq!(
-            parse_deno_signature(code)?,
+            parse_deno_signature(code, false)?,
             MainArgSignature {
                 star_args: false,
                 star_kwargs: false,

backend/src/ee.rs

@@ -1,4 +1,6 @@
 #[cfg(feature = "enterprise")]
+use base64::Engine;
+#[cfg(feature = "enterprise")]
 use rsa::{pkcs8::DecodePublicKey, signature::Verifier};
 #[cfg(feature = "enterprise")]
 use sha2::Sha256;
@@ -11,9 +13,9 @@ pub fn verify_license_key(license_key: Option<String>) -> anyhow::Result<()> {
             .expect("license_key can be splitted with a .");
 
         let pub_key = rsa::RsaPublicKey::from_public_key_der(
-    &base64::decode("MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDgVShzcLSPiOi+8ET8fggob1kmi47/cE12JaidPkwfGnScZItghkqtiLsct0U4kJhlp5gO89DYTBmIKadvxwY7kMsLlZzmi2emVH7c27cByGASY8QmWDNdG4Ggy/NDflGGBdAtN6gHawZAg4zHv3qpbPQGHH1/6sXIohcXhOnouwIDAQAB")?)?;
-        let msg = base64::decode(splitted_lk.0)?;
-        let signature = base64::decode(splitted_lk.1)?;
+    &base64::engine::general_purpose::STANDARD.decode("MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDgVShzcLSPiOi+8ET8fggob1kmi47/cE12JaidPkwfGnScZItghkqtiLsct0U4kJhlp5gO89DYTBmIKadvxwY7kMsLlZzmi2emVH7c27cByGASY8QmWDNdG4Ggy/NDflGGBdAtN6gHawZAg4zHv3qpbPQGHH1/6sXIohcXhOnouwIDAQAB")?)?;
+        let msg = base64::engine::general_purpose::STANDARD.decode(splitted_lk.0)?;
+        let signature = base64::engine::general_purpose::STANDARD.decode(splitted_lk.1)?;
         rsa::pss::VerifyingKey::<Sha256>::new(pub_key)
             .verify(&msg, &rsa::pss::Signature::from(signature))
             .map_err(|_| anyhow::anyhow!("Invalid license key".to_string()))?;

backend/src/main.rs

@@ -10,7 +10,7 @@ use std::net::{IpAddr, Ipv4Addr, SocketAddr};
 
 use git_version::git_version;
 use sqlx::{Pool, Postgres};
-use windmill_common::utils::rd_string;
+use windmill_common::{utils::rd_string, METRICS_ADDR};
 
 const GIT_VERSION: &str = git_version!(args = ["--tag", "--always"], fallback = "unknown-version");
 const DEFAULT_NUM_WORKERS: usize = 3;
@@ -30,15 +30,7 @@ async fn main() -> anyhow::Result<()> {
         .and_then(|x| x.parse::<i32>().ok())
         .unwrap_or(DEFAULT_NUM_WORKERS as i32);
 
-    let metrics_addr: Option<SocketAddr> = std::env::var("METRICS_ADDR")
-        .ok()
-        .map(|s| {
-            s.parse::<bool>()
-                .map(|b| b.then(|| SocketAddr::from(([0, 0, 0, 0], 8001))))
-                .or_else(|_| s.parse::<SocketAddr>().map(Some))
-        })
-        .transpose()?
-        .flatten();
+    let metrics_addr: Option<SocketAddr> = *METRICS_ADDR;
 
     let server_bind_address: IpAddr = std::env::var("SERVER_BIND_ADDR")
         .ok()
@@ -57,8 +49,40 @@ async fn main() -> anyhow::Result<()> {
         .and_then(|x| x.parse::<bool>().ok())
         .unwrap_or(false);
 
+    let rsmq_config = std::env::var("REDIS_URL").ok().map(|x| {
+        let url = x.parse::<url::Url>().unwrap();
+        let mut config = rsmq_async::RsmqOptions { ..Default::default() };
+
+        config.host = url.host_str().expect("redis host required").to_owned();
+        config.password = url.password().map(|s| s.to_owned());
+        config.db = url
+            .path_segments()
+            .and_then(|mut segments| segments.next())
+            .and_then(|segment| segment.parse().ok())
+            .unwrap_or(0);
+        config.ns = url
+            .query_pairs()
+            .find(|s| s.0 == "rsmq_namespace")
+            .map(|s| s.1)
+            .unwrap_or(std::borrow::Cow::Borrowed("rsmq"))
+            .into_owned();
+        config.port = url.port().unwrap_or(6379).to_string();
+
+        config
+    });
+
     let db = windmill_common::connect_db(server_mode).await?;
 
+    let rsmq = if let Some(config) = rsmq_config {
+        let mut rsmq = rsmq_async::MultiplexedRsmq::new(config).await.unwrap();
+
+        let _ = rsmq_async::RsmqConnection::create_queue(&mut rsmq, "main_queue", None, None, None)
+            .await;
+        Some(rsmq)
+    } else {
+        None
+    };
+
     if server_mode {
         windmill_api::migrate_db(&db).await?;
     }
@@ -104,6 +128,8 @@ Windmill Community Edition {GIT_VERSION}
         "PYTHON_PATH",
         "DENO_PATH",
         "GO_PATH",
+        "GOPRIVATE",
+        "NETRC",
         "PIP_INDEX_URL",
         "PIP_EXTRA_INDEX_URL",
         "PIP_TRUSTED_HOST",
@@ -120,16 +146,17 @@ Windmill Community Edition {GIT_VERSION}
         "INCLUDE_HEADERS",
         "WHITELIST_WORKSPACES",
         "BLACKLIST_WORKSPACES",
-        "NEW_USER_WEBHOOK",
+        "INSTANCE_EVENTS_WEBHOOK",
         "CLOUD_HOSTED",
     ]);
 
     if server_mode || num_workers > 0 {
         let addr = SocketAddr::from((server_bind_address, port));
 
+        let rsmq2 = rsmq.clone();
         let server_f = async {
             if server_mode {
-                windmill_api::run_server(db.clone(), addr, rx.resubscribe()).await?;
+                windmill_api::run_server(db.clone(), rsmq2, addr, rx.resubscribe()).await?;
             }
             Ok(()) as anyhow::Result<()>
         };
@@ -141,24 +168,28 @@ Windmill Community Edition {GIT_VERSION}
                     rx.resubscribe(),
                     num_workers,
                     base_internal_url.clone(),
+                    rsmq.clone(),
                 )
                 .await?;
             }
             Ok(()) as anyhow::Result<()>
         };
 
+        let rsmq2 = rsmq.clone();
         let monitor_f = async {
             if server_mode {
-                monitor_db(&db, rx.resubscribe(), &base_internal_url);
+                monitor_db(&db, rx.resubscribe(), &base_internal_url, rsmq2);
             }
             Ok(()) as anyhow::Result<()>
         };
 
         let metrics_f = async {
             match metrics_addr {
-                Some(addr) => windmill_common::serve_metrics(addr, rx.resubscribe())
-                    .await
-                    .map_err(anyhow::Error::from),
+                Some(addr) => {
+                    windmill_common::serve_metrics(addr, rx.resubscribe(), num_workers > 0)
+                        .await
+                        .map_err(anyhow::Error::from)
+                }
                 None => Ok(()),
             }
         };
@@ -187,10 +218,11 @@ fn display_config(envs: Vec<&str>) {
     )
 }
 
-pub fn monitor_db(
+pub fn monitor_db<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 'static>(
     db: &Pool<Postgres>,
     rx: tokio::sync::broadcast::Receiver<()>,
     base_internal_url: &str,
+    rsmq: Option<R>,
 ) {
     let db1 = db.clone();
     let db2 = db.clone();
@@ -198,16 +230,17 @@ pub fn monitor_db(
     let rx2 = rx.resubscribe();
     let base_internal_url = base_internal_url.to_string();
     tokio::spawn(async move {
-        windmill_worker::handle_zombie_jobs_periodically(&db1, rx, &base_internal_url).await
+        windmill_worker::handle_zombie_jobs_periodically(&db1, rx, &base_internal_url, rsmq).await
     });
     tokio::spawn(async move { windmill_api::delete_expired_items_perdiodically(&db2, rx2).await });
 }
 
-pub async fn run_workers(
+pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 'static>(
     db: Pool<Postgres>,
     rx: tokio::sync::broadcast::Receiver<()>,
     num_workers: i32,
     base_internal_url: String,
+    rsmq: Option<R>,
 ) -> anyhow::Result<()> {
     let license_key = std::env::var("LICENSE_KEY").ok();
     #[cfg(feature = "enterprise")]
@@ -237,6 +270,7 @@ pub async fn run_workers(
         let ip = ip.clone();
         let rx = rx.resubscribe();
         let base_internal_url = base_internal_url.clone();
+        let rsmq2 = rsmq.clone();
         handles.push(tokio::spawn(monitor.instrument(async move {
             tracing::info!(worker = %worker_name, "starting worker");
             windmill_worker::run_worker(
@@ -247,6 +281,7 @@ pub async fn run_workers(
                 &ip,
                 rx,
                 &base_internal_url,
+                rsmq2,
             )
             .await
         })));

backend/tests/worker.rs

@@ -1,4 +1,5 @@
 use futures::{stream, Stream};
+use serde::Deserialize;
 use serde_json::json;
 use sqlx::{postgres::PgListener, types::Uuid, Pool, Postgres, Transaction};
 use windmill_api::jobs::{CompletedJob, Job};
@@ -87,7 +88,7 @@ impl ApiServer {
         let addr = sock.local_addr().unwrap();
         drop(sock);
 
-        let task = tokio::task::spawn(windmill_api::run_server(db.clone(), addr, rx));
+        let task = tokio::task::spawn(windmill_api::run_server(db.clone(), None, addr, rx));
 
         return Self { addr, tx, task };
     }
@@ -252,9 +253,7 @@ mod suspend_resume {
                 let second = completed.next().await.unwrap();
                 // print_job(second, &db).await;
 
-                let tx = db.begin().await.unwrap();
-                let (tx, token) = windmill_worker::create_token_for_owner(tx, "test-workspace", "u/test-user", "", 100, "").await.unwrap();
-                tx.commit().await.unwrap();
+                let token = windmill_worker::create_token_for_owner(&db, "test-workspace", "u/test-user", "", 100, "").await.unwrap();
                 let secret = reqwest::get(format!(
                     "http://localhost:{port}/api/w/test-workspace/jobs/job_signature/{second}/0?token={token}&approver=ruben"
                 ))
@@ -355,9 +354,7 @@ mod suspend_resume {
                 /* ... and send a request resume it. */
                 let second = completed.next().await.unwrap();
 
-                let tx = db.begin().await.unwrap();
-                let (tx, token) = windmill_worker::create_token_for_owner(tx, "test-workspace", "u/test-user", "", 100, "").await.unwrap();
-                tx.commit().await.unwrap();
+                let token = windmill_worker::create_token_for_owner(&db, "test-workspace", "u/test-user", "", 100, "").await.unwrap();
                 let secret = reqwest::get(format!(
                     "http://localhost:{port}/api/w/test-workspace/jobs/job_signature/{second}/0?token={token}"
                 ))
@@ -659,7 +656,7 @@ def main(error, port):
             },
         }))
         .unwrap();
-        let (attempts, responses) = [
+        let (_attempts, responses) = [
             /* fail the first step twice */
             (0x00, None),
             (0x00, None),
@@ -681,14 +678,20 @@ def main(error, port):
             _ => panic!("expected failure module"),
         }
 
-        assert_eq!(server.close().await, attempts);
+        println!("result: {:#?}", result);
         assert_eq!(
-            result,
-            json!({
-                "recv": 42,
-                "from failure module": {"error": {"name": "IndexError", "stack": "  File \"/tmp/tmp/main/step_0.py\", line 5, in main\n    return sock.recv(1)[0]\n", "message": "index out of range"}},
-            })
+            result
+                .get("from failure module")
+                .unwrap()
+                .get("error")
+                .unwrap()
+                .get("name")
+                .unwrap()
+                .clone(),
+            json!("IndexError")
         );
+
+        assert_eq!(result.get("recv").unwrap().clone(), json!(42));
     }
 }
 
@@ -824,9 +827,8 @@ impl RunJob {
 
     async fn push(self, db: &Pool<Postgres>) -> Uuid {
         let RunJob { payload, args } = self;
-        let tx = db.begin().await.unwrap();
-        let (uuid, tx) = windmill_queue::push(
-            tx,
+        let (uuid, tx) = windmill_queue::push::<rsmq_async::MultiplexedRsmq>(
+            (None, db.begin().await.unwrap()).into(),
             "test-workspace",
             payload,
             args,
@@ -844,8 +846,7 @@ impl RunJob {
         )
         .await
         .expect("push has to succeed");
-
-        tx.commit().await.expect("push has to commit");
+        tx.commit().await.unwrap();
 
         uuid
     }
@@ -915,7 +916,7 @@ fn spawn_test_worker(
     let ip: &str = Default::default();
     let future = async move {
         let base_internal_url = format!("http://localhost:{}", port);
-        windmill_worker::run_worker(
+        windmill_worker::run_worker::<rsmq_async::MultiplexedRsmq>(
             &db,
             worker_instance,
             worker_name,
@@ -923,6 +924,7 @@ fn spawn_test_worker(
             ip,
             rx,
             &base_internal_url,
+            None,
         )
         .await
     };
@@ -1518,6 +1520,8 @@ async fn test_go_job(db: Pool<Postgres>) {
     let port = server.addr.port();
 
     let content = r#"
+package inner
+
 import "fmt"
 
 func main(derp string) (string, error) {
@@ -2024,6 +2028,16 @@ async fn test_branchall_simple(db: Pool<Postgres>) {
     assert_eq!(result, serde_json::json!([[1, 2], [1, 3]]));
 }
 
+#[derive(Deserialize)]
+struct ErrorResult {
+    error: NamedError,
+}
+
+#[derive(Deserialize)]
+struct NamedError {
+    name: String,
+}
+
 #[sqlx::test(fixtures("base"))]
 async fn test_branchall_skip_failure(db: Pool<Postgres>) {
     initialize_tracing().await;
@@ -2059,8 +2073,11 @@ async fn test_branchall_skip_failure(db: Pool<Postgres>) {
         .unwrap();
 
     assert_eq!(
-        result,
-        serde_json::json!([{"error": {"name": "Error", "stack": "Error: failure\n    at main (file:///tmp/main.ts:1:31)\n    at run (file:///tmp/wrapper.ts:9:26)\n    at file:///tmp/wrapper.ts:14:1", "message": "failure"}}, [1,3]])
+        serde_json::from_value::<ErrorResult>(result.get(0).unwrap().clone())
+            .unwrap()
+            .error
+            .name,
+        "Error"
     );
 
     let flow: FlowValue = serde_json::from_value(json!({
@@ -2093,8 +2110,11 @@ async fn test_branchall_skip_failure(db: Pool<Postgres>) {
         .unwrap();
 
     assert_eq!(
-        result,
-        serde_json::json!([ {"error": {"name": "Error", "stack": "Error: failure\n    at main (file:///tmp/main.ts:1:31)\n    at run (file:///tmp/wrapper.ts:9:26)\n    at file:///tmp/wrapper.ts:14:1", "message": "failure"}}, [1, 2]])
+        serde_json::from_value::<ErrorResult>(result.get(0).unwrap().clone())
+            .unwrap()
+            .error
+            .name,
+        "Error"
     );
 }
 

backend/windmill-api/Cargo.toml

@@ -47,6 +47,7 @@ tracing.workspace = true
 sql-builder.workspace = true
 serde_json.workspace = true
 chrono.workspace = true
+chrono-tz.workspace = true
 hex.workspace = true
 base64.workspace = true
 serde_urlencoded.workspace = true
@@ -69,4 +70,5 @@ async-stripe = { workspace = true, optional = true }
 lazy_static.workspace = true
 prometheus.workspace = true
 async_zip.workspace = true
+rsmq_async.workspace = true
 regex.workspace = true

backend/windmill-api/openapi.yaml

@@ -1,7 +1,7 @@
 openapi: "3.0.3"
 
 info:
-  version: 1.84.1
+  version: 1.87.0
   title: Windmill API
 
   contact:
@@ -1048,6 +1048,27 @@ paths:
               schema:
                 type: string
 
+  /users/tokens/impersonate:
+    post:
+      summary: create token to impersonate a user (require superadmin)
+      operationId: createTokenImpersonate
+      tags:
+        - user
+      requestBody:
+        description: new token
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/NewTokenImpersonate"
+      responses:
+        "201":
+          description: token created
+          content:
+            text/plain:
+              schema:
+                type: string
+
   /users/tokens/delete/{token_prefix}:
     delete:
       summary: delete token
@@ -2021,8 +2042,7 @@ paths:
           schema:
             type: string
         - name: first_parent_hash
-          description:
-            mask to filter scripts whom first direct parent has exact hash
+          description: mask to filter scripts whom first direct parent has exact hash
           in: query
           schema:
             type: string
@@ -2279,8 +2299,7 @@ paths:
 
   /w/{workspace}/scripts/delete/h/{hash}:
     post:
-      summary:
-        delete script by hash (erase content but keep hash, require admin)
+      summary: delete script by hash (erase content but keep hash, require admin)
       operationId: deleteScriptByHash
       tags:
         - script
@@ -2346,6 +2365,24 @@ paths:
               schema:
                 type: string
 
+  /scripts_u/tokened_raw/{workspace}/{token}/{path}:
+    get:
+      summary: raw script by path with a token (mostly used by lsp to be used with import maps to resolve scripts)
+      operationId: rawScriptByPathTokened
+      tags:
+        - script
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/Token"
+        - $ref: "#/components/parameters/ScriptPath"
+      responses:
+        "200":
+          description: script content
+          content:
+            text/plain:
+              schema:
+                type: string
+
   /w/{workspace}/scripts/exists/p/{path}:
     get:
       summary: exists script by path
@@ -2435,15 +2472,13 @@ paths:
             type: string
             format: date-time
         - name: scheduled_in_secs
-          description:
-            schedule the script to execute in the number of seconds starting now
+          description: schedule the script to execute in the number of seconds starting now
           in: query
           schema:
             type: integer
         - $ref: "#/components/parameters/ParentJob"
         - name: invisible_to_owner
-          description:
-            make the run invisible to the the script owner (default false)
+          description: make the run invisible to the the script owner (default false)
           in: query
           schema:
             type: boolean
@@ -2473,18 +2508,6 @@ paths:
       parameters:
         - $ref: "#/components/parameters/WorkspaceId"
         - $ref: "#/components/parameters/ScriptPath"
-        - name: scheduled_for
-          description: when to schedule this job (leave empty for immediate run)
-          in: query
-          schema:
-            type: string
-            format: date-time
-        - name: scheduled_in_secs
-          description:
-            schedule the script to execute in the number of seconds starting now
-          in: query
-          schema:
-            type: integer
         - $ref: "#/components/parameters/ParentJob"
         - $ref: "#/components/parameters/IncludeHeader"
         - $ref: "#/components/parameters/QueueLimit"
@@ -2504,6 +2527,26 @@ paths:
             application/json:
               schema: {}
 
+    get:
+      summary: run script by path with get
+      operationId: runWaitResultScriptByPathGet
+      tags:
+        - job
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/ScriptPath"
+        - $ref: "#/components/parameters/ParentJob"
+        - $ref: "#/components/parameters/IncludeHeader"
+        - $ref: "#/components/parameters/QueueLimit"
+        - $ref: "#/components/parameters/Payload"
+
+      responses:
+        "200":
+          description: job result
+          content:
+            application/json:
+              schema: {}
+
   /w/{workspace}/jobs/run_wait_result/f/{path}:
     post:
       summary: run flow by path and wait until completion
@@ -2513,18 +2556,6 @@ paths:
       parameters:
         - $ref: "#/components/parameters/WorkspaceId"
         - $ref: "#/components/parameters/ScriptPath"
-        - name: scheduled_for
-          description: when to schedule this job (leave empty for immediate run)
-          in: query
-          schema:
-            type: string
-            format: date-time
-        - name: scheduled_in_secs
-          description:
-            schedule the script to execute in the number of seconds starting now
-          in: query
-          schema:
-            type: integer
         - $ref: "#/components/parameters/IncludeHeader"
         - $ref: "#/components/parameters/QueueLimit"
 
@@ -2751,6 +2782,27 @@ paths:
               schema:
                 type: string
 
+  /w/{workspace}/flows/input_history/p/{path}:
+    get:
+      summary: list inputs for previous completed flow jobs
+      operationId: getFlowInputHistoryByPath
+      tags:
+        - flow
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/ScriptPath"
+        - $ref: "#/components/parameters/Page"
+        - $ref: "#/components/parameters/PerPage"
+      responses:
+        "200":
+          description: input history for completed jobs with this flow path
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: "#/components/schemas/Input"
+
   /w/{workspace}/apps/list:
     get:
       summary: list all available apps
@@ -3023,16 +3075,14 @@ paths:
             type: string
             format: date-time
         - name: scheduled_in_secs
-          description:
-            schedule the script to execute in the number of seconds starting now
+          description: schedule the script to execute in the number of seconds starting now
           in: query
           schema:
             type: integer
         - $ref: "#/components/parameters/ParentJob"
         - $ref: "#/components/parameters/IncludeHeader"
         - name: invisible_to_owner
-          description:
-            make the run invisible to the the flow owner (default false)
+          description: make the run invisible to the the flow owner (default false)
           in: query
           schema:
             type: boolean
@@ -3070,16 +3120,14 @@ paths:
             type: string
             format: date-time
         - name: scheduled_in_secs
-          description:
-            schedule the script to execute in the number of seconds starting now
+          description: schedule the script to execute in the number of seconds starting now
           in: query
           schema:
             type: integer
         - $ref: "#/components/parameters/ParentJob"
         - $ref: "#/components/parameters/IncludeHeader"
         - name: invisible_to_owner
-          description:
-            make the run invisible to the the script owner (default false)
+          description: make the run invisible to the the script owner (default false)
           in: query
           schema:
             type: boolean
@@ -3110,8 +3158,7 @@ paths:
         - $ref: "#/components/parameters/WorkspaceId"
         - $ref: "#/components/parameters/IncludeHeader"
         - name: invisible_to_owner
-          description:
-            make the run invisible to the the script owner (default false)
+          description: make the run invisible to the the script owner (default false)
           in: query
           schema:
             type: boolean
@@ -3142,8 +3189,7 @@ paths:
         - $ref: "#/components/parameters/WorkspaceId"
         - $ref: "#/components/parameters/IncludeHeader"
         - name: invisible_to_owner
-          description:
-            make the run invisible to the the script owner (default false)
+          description: make the run invisible to the the script owner (default false)
           in: query
           schema:
             type: boolean
@@ -3297,23 +3343,6 @@ paths:
               schema:
                 $ref: "#/components/schemas/Job"
 
-  # /w/{workspace}/jobs/flow/current_state/{id}:
-  #   get:
-  #     summary: get flow current step state
-  #     operationId: getJob
-  #     tags:
-  #       - job
-  #     parameters:
-  #       - $ref: "#/components/parameters/WorkspaceId"
-  #       - $ref: "#/components/parameters/JobId"
-  #     responses:
-  #       "200":
-  #         description: state details
-  #         content:
-  #           application/json:
-  #             schema:
-  #               type: string
-
   /w/{workspace}/jobs_u/getupdate/{id}:
     get:
       summary: get job updates
@@ -3483,8 +3512,7 @@ paths:
 
   /w/{workspace}/jobs/resume_urls/{id}/{resume_id}:
     get:
-      summary:
-        get resume urls given a job_id, resume_id and a nonce to resume a flow
+      summary: get resume urls given a job_id, resume_id and a nonce to resume a flow
       operationId: getResumeUrls
       tags:
         - job
@@ -3528,6 +3556,7 @@ paths:
       parameters:
         - $ref: "#/components/parameters/WorkspaceId"
         - $ref: "#/components/parameters/JobId"
+        - $ref: "#/components/parameters/Payload"
         - name: resume_id
           in: path
           required: true
@@ -3741,14 +3770,14 @@ paths:
               properties:
                 schedule:
                   type: string
-                offset:
-                  type: integer
+                timezone:
+                  type: string
               required:
                 - schedule
+                - timezone
       responses:
         "200":
-          description:
-            the preview of the next 10 time this schedule would apply to
+          description: List of 5 estimated upcoming execution events (in UTC)
           content:
             application/json:
               schema:
@@ -3935,8 +3964,7 @@ paths:
         - $ref: "#/components/parameters/WorkspaceId"
         - name: only_member_of
           in: query
-          description:
-            only list the groups the user is member of (default false)
+          description: only list the groups the user is member of (default false)
           schema:
             type: boolean
       responses:
@@ -4124,8 +4152,7 @@ paths:
         - $ref: "#/components/parameters/WorkspaceId"
         - name: only_member_of
           in: query
-          description:
-            only list the folders the user is member of (default false)
+          description: only list the folders the user is member of (default false)
           schema:
             type: boolean
       responses:
@@ -4535,6 +4562,118 @@ paths:
         "200":
           description: unstar item
 
+  /w/{workspace}/inputs/history:
+    get:
+      summary: List Inputs used in previously completed jobs
+      operationId: getInputHistory
+      tags:
+        - input
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/RunnableId"
+        - $ref: "#/components/parameters/RunnableType"
+        - $ref: "#/components/parameters/Page"
+        - $ref: "#/components/parameters/PerPage"
+      responses:
+        "200":
+          description: Input history for completed jobs
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: "#/components/schemas/Input"
+
+  /w/{workspace}/inputs/list:
+    get:
+      summary: List saved Inputs for a Runnable
+      operationId: listInputs
+      tags:
+        - input
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/RunnableId"
+        - $ref: "#/components/parameters/RunnableType"
+        - $ref: "#/components/parameters/Page"
+        - $ref: "#/components/parameters/PerPage"
+      responses:
+        "200":
+          description: Saved Inputs for a Runnable
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: "#/components/schemas/Input"
+
+  /w/{workspace}/inputs/create:
+    post:
+      summary: Create an Input for future use in a script or flow
+      operationId: createInput
+      tags:
+        - input
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/RunnableId"
+        - $ref: "#/components/parameters/RunnableType"
+      requestBody:
+        description: Input
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/CreateInput"
+      responses:
+        "201":
+          description: Input created
+          content:
+            text/plain:
+              schema:
+                type: string
+                format: uuid
+
+  /w/{workspace}/inputs/update:
+    post:
+      summary: Update an Input
+      operationId: updateInput
+      tags:
+        - input
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+      requestBody:
+        description: UpdateInput
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/UpdateInput"
+      responses:
+        "201":
+          description: Input updated
+          content:
+            text/plain:
+              schema:
+                type: string
+                format: uuid
+
+  /w/{workspace}/inputs/delete/{input}:
+    post:
+      summary: Delete a Saved Input
+      operationId: deleteInput
+      tags:
+        - input
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/InputId"
+      responses:
+        "200":
+          description: Input deleted
+          content:
+            text/plain:
+              schema:
+                type: string
+                format: uuid
+
 components:
   securitySchemes:
     bearerAuth:
@@ -4544,6 +4683,7 @@ components:
       type: apiKey
       in: cookie
       name: token
+
   parameters:
     WorkspaceId:
       name: workspace
@@ -4551,6 +4691,12 @@ components:
       required: true
       schema:
         type: string
+    Token:
+      name: token
+      in: path
+      required: true
+      schema:
+        type: string
     AccountId:
       name: id
       in: path
@@ -4608,8 +4754,7 @@ components:
         type: integer
     PerPage:
       name: per_page
-      description:
-        number of items to return for a given page (default 30, max 100)
+      description: number of items to return for a given page (default 30, max 100)
       in: query
       schema:
         type: integer
@@ -4649,7 +4794,14 @@ components:
       in: query
       schema:
         type: string
-
+    Payload:
+      name: payload
+      description: |
+        The base64 encoded payload that has been encoded as a JSON. e.g how to encode such payload encodeURIComponent
+        `encodeURIComponent(btoa(JSON.stringify({a: 2})))`
+      in: query
+      schema:
+        type: string
     ScriptStartPath:
       name: script_path_start
       description: mask to filter matching starting path
@@ -4702,16 +4854,13 @@ components:
         type: boolean
     ArgsFilter:
       name: args
-      description:
-        filter on jobs containing those args as a json subset (@> in postgres)
+      description: filter on jobs containing those args as a json subset (@> in postgres)
       in: query
       schema:
         type: string
-
     ResultFilter:
       name: result
-      description:
-        filter on jobs containing those result as a json subset (@> in postgres)
+      description: filter on jobs containing those result as a json subset (@> in postgres)
       in: query
       schema:
         type: string
@@ -4768,9 +4917,26 @@ components:
       #     type: string
       #     enum: ["preview", "script", "dependencies"]
       # explode: false
+    RunnableId:
+      name: runnable_id
+      in: query
+      schema:
+        type: string
+    RunnableType:
+      name: runnable_type
+      in: query
+      schema:
+        $ref: "#/components/schemas/RunnableType"
+    InputId:
+      name: input
+      in: path
+      required: true
+      schema:
+        type: string
 
   schemas:
     $ref: "../../openflow.openapi.yaml#/components/schemas"
+
     Script:
       type: object
       properties:
@@ -4841,6 +5007,60 @@ components:
       type: object
       additionalProperties: {}
 
+    Input:
+      type: object
+      properties:
+        id:
+          type: string
+        name:
+          type: string
+        args:
+          type: object
+        created_by:
+          type: string
+        created_at:
+          type: string
+          format: date-time
+        is_public:
+          type: boolean
+      required:
+        - id
+        - name
+        - args
+        - created_by
+        - created_at
+        - is_public
+
+    CreateInput:
+      type: object
+      properties:
+        name:
+          type: string
+        args:
+          type: object
+      required:
+        - name
+        - args
+        - created_by
+
+    UpdateInput:
+      type: object
+      properties:
+        id:
+          type: string
+        name:
+          type: string
+        is_public:
+          type: boolean
+      required:
+        - id
+        - name
+        - is_public
+
+    RunnableType:
+      type: string
+      enum: ["ScriptHash", "ScriptPath", "FlowPath"]
+
     QueuedJob:
       type: object
       properties:
@@ -5143,6 +5363,19 @@ components:
           type: string
           format: date-time
 
+    NewTokenImpersonate:
+      type: object
+      properties:
+        label:
+          type: string
+        expiration:
+          type: string
+          format: date-time
+        impersonate_email:
+          type: string
+      required:
+        - impersonate_email
+
     ListableVariable:
       type: object
       properties:
@@ -5268,6 +5501,11 @@ components:
     MainArgSignature:
       type: object
       properties:
+        type:
+          type: string
+          enum: ["Valid", "Invalid"]
+        error:
+          type: string
         star_args:
           type: boolean
         star_kwargs:
@@ -5377,6 +5615,8 @@ components:
         - star_args
         - start_kwargs
         - args
+        - type
+        - error
 
     Preview:
       type: object
@@ -5511,8 +5751,8 @@ components:
           format: date-time
         schedule:
           type: string
-        offset_:
-          type: integer
+        timezone:
+          type: string
         enabled:
           type: boolean
         script_path:
@@ -5535,7 +5775,7 @@ components:
         - edited_at
         - schedule
         - script_path
-        - offset_
+        - timezone
         - extra_perms
         - is_flow
         - enabled
@@ -5548,8 +5788,8 @@ components:
           type: string
         schedule:
           type: string
-        offset:
-          type: integer
+        timezone:
+          type: string
         script_path:
           type: string
         is_flow:
@@ -5561,6 +5801,7 @@ components:
       required:
         - path
         - schedule
+        - timezone
         - script_path
         - is_flow
         - args
@@ -5570,10 +5811,13 @@ components:
       properties:
         schedule:
           type: string
+        timezone:
+          type: string
         args:
           $ref: "#/components/schemas/ScriptArgs"
       required:
         - schedule
+        - timezone
         - script_path
         - is_flow
         - args
@@ -5838,6 +6082,7 @@ components:
         - extra_perms
         - edited_at
         - execution_mode
+
     AppWithLastVersion:
       type: object
       properties:

backend/windmill-api/src/apps.rs

@@ -37,7 +37,7 @@ use windmill_common::{
         http_get_from_hub, list_elems_from_hub, not_found_if_none, paginate, Pagination, StripPath,
     },
 };
-use windmill_queue::{push, JobPayload, RawCode};
+use windmill_queue::{push, JobPayload, QueueTransaction, RawCode};
 
 pub fn workspaced_service() -> Router {
     Router::new()
@@ -565,6 +565,7 @@ fn digest(code: &str) -> String {
 async fn execute_component(
     OptAuthed(opt_authed): OptAuthed,
     Extension(db): Extension<DB>,
+    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
     Path((w_id, path)): Path<(String, StripPath)>,
     Json(payload): Json<ExecuteApp>,
 ) -> Result<String> {
@@ -583,7 +584,7 @@ async fn execute_component(
     };
 
     let path = path.to_path();
-    let mut tx = db.begin().await?;
+    let mut tx: QueueTransaction<'_, _> = (rsmq, db.begin().await?).into();
 
     let policy = if let Some(static_fields) = payload.clone().force_viewer_static_fields {
         let mut hm = HashMap::new();
@@ -652,8 +653,12 @@ async fn execute_component(
         }
         ExecuteApp { args, raw_code: None, path: Some(path), .. } => {
             let payload = if path.starts_with("script/") {
-                script_path_to_payload(path.strip_prefix("script/").unwrap(), &mut tx, &w_id)
-                    .await?
+                script_path_to_payload(
+                    path.strip_prefix("script/").unwrap(),
+                    tx.transaction_mut(),
+                    &w_id,
+                )
+                .await?
             } else if path.starts_with("flow/") {
                 JobPayload::Flow(path.strip_prefix("flow/").unwrap().to_string())
             } else {
@@ -686,8 +691,8 @@ async fn execute_component(
         true,
     )
     .await?;
-
     tx.commit().await?;
+
     Ok(uuid.to_string())
 }
 

backend/windmill-api/src/flows.rs

@@ -6,14 +6,20 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
-use hyper::StatusCode;
-use sql_builder::prelude::*;
-
+use crate::{
+    db::{UserDB, DB},
+    schedule::clear_schedule,
+    users::{maybe_refresh_folders, require_owner_of_path, Authed},
+    webhook_util::{WebhookMessage, WebhookShared},
+    HTTP_CLIENT,
+};
 use axum::{
     extract::{Extension, Path, Query},
     routing::{delete, get, post},
     Json, Router,
 };
+use hyper::StatusCode;
+use sql_builder::prelude::*;
 use sql_builder::SqlBuilder;
 use sqlx::{Postgres, Transaction};
 use windmill_audit::{audit_log, ActionKind};
@@ -25,15 +31,7 @@ use windmill_common::{
         http_get_from_hub, list_elems_from_hub, not_found_if_none, paginate, Pagination, StripPath,
     },
 };
-use windmill_queue::{push, schedule::push_scheduled_job, JobPayload};
-
-use crate::{
-    db::{UserDB, DB},
-    schedule::clear_schedule,
-    users::{require_owner_of_path, Authed},
-    webhook_util::{WebhookMessage, WebhookShared},
-    HTTP_CLIENT,
-};
+use windmill_queue::{push, schedule::push_scheduled_job, JobPayload, QueueTransaction};
 
 pub fn workspaced_service() -> Router {
     Router::new()
@@ -178,16 +176,20 @@ async fn check_path_conflict<'c>(
 
 async fn create_flow(
     authed: Authed,
+    Extension(db): Extension<DB>,
     Extension(user_db): Extension<UserDB>,
+    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
     Extension(webhook): Extension<WebhookShared>,
     Path(w_id): Path<String>,
     Json(nf): Json<NewFlow>,
 ) -> Result<(StatusCode, String)> {
     // cron::Schedule::from_str(&ns.schedule).map_err(|e| error::Error::BadRequest(e.to_string()))?;
-    let mut tx = user_db.clone().begin(&authed).await?;
+    let authed = maybe_refresh_folders(&nf.path, &w_id, authed, &db).await;
 
-    check_path_conflict(&mut tx, &w_id, &nf.path).await?;
-    check_schedule_conflict(&mut tx, &w_id, &nf.path).await?;
+    let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.clone().begin(&authed).await?).into();
+
+    check_path_conflict(tx.transaction_mut(), &w_id, &nf.path).await?;
+    check_schedule_conflict(tx.transaction_mut(), &w_id, &nf.path).await?;
 
     sqlx::query!(
         "INSERT INTO flow (workspace_id, path, summary, description, value, edited_by, edited_at, \
@@ -219,13 +221,11 @@ async fn create_flow(
     )
     .await?;
 
-    tx.commit().await?;
     webhook.send_message(
         w_id.clone(),
         WebhookMessage::CreateFlow { workspace: w_id.clone(), path: nf.path.clone() },
     );
 
-    let tx = user_db.begin(&authed).await?;
     let (dependency_job_uuid, mut tx) = push(
         tx,
         &w_id,
@@ -244,6 +244,7 @@ async fn create_flow(
         true,
     )
     .await?;
+
     sqlx::query!(
         "UPDATE flow SET dependency_job = $1 WHERE path = $2 AND workspace_id = $3",
         dependency_job_uuid,
@@ -283,15 +284,18 @@ async fn check_schedule_conflict<'c>(
 async fn update_flow(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
+    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
     Extension(db): Extension<DB>,
     Extension(webhook): Extension<WebhookShared>,
     Path((w_id, flow_path)): Path<(String, StripPath)>,
     Json(nf): Json<NewFlow>,
 ) -> Result<String> {
-    let mut tx = user_db.clone().begin(&authed).await?;
-
     let flow_path = flow_path.to_path();
-    check_schedule_conflict(&mut tx, &w_id, flow_path).await?;
+    let authed = maybe_refresh_folders(&flow_path, &w_id, authed, &db).await;
+
+    let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.clone().begin(&authed).await?).into();
+
+    check_schedule_conflict(tx.transaction_mut(), &w_id, flow_path).await?;
 
     let schema = nf.schema.map(|x| x.0);
     let old_dep_job = sqlx::query_scalar!(
@@ -318,13 +322,13 @@ async fn update_flow(
     .await?;
 
     if nf.path != flow_path {
-        check_schedule_conflict(&mut tx, &w_id, &nf.path).await?;
+        check_schedule_conflict(tx.transaction_mut(), &w_id, &nf.path).await?;
 
         if !authed.is_admin {
             require_owner_of_path(&w_id, &authed.username, &authed.groups, &flow_path, &db).await?;
         }
 
-        let mut schedulables = sqlx::query_as!(
+        let mut schedulables: Vec<Schedule> = sqlx::query_as!(
             Schedule,
                 "UPDATE schedule SET script_path = $1 WHERE script_path = $2 AND path != $2 AND workspace_id = $3 AND is_flow IS true RETURNING *",
                 nf.path,
@@ -347,32 +351,32 @@ async fn update_flow(
             schedulables.push(schedule);
         }
 
-        for schedule in schedulables {
-            clear_schedule(&mut tx, flow_path, true).await?;
+        for schedule in schedulables.into_iter() {
+            // TODO: Why is this in the loop in the first place? Seems like it's just doing nothing after the first iteration? Should this use schedule.path?
+            clear_schedule(tx.transaction_mut(), flow_path, true).await?;
 
             if schedule.enabled {
                 tx = push_scheduled_job(tx, schedule).await?;
             }
         }
+
+        audit_log(
+            &mut tx,
+            &authed.username,
+            "flows.update",
+            ActionKind::Create,
+            &w_id,
+            Some(&nf.path.to_string()),
+            Some(
+                [Some(("flow", nf.path.as_str()))]
+                    .into_iter()
+                    .flatten()
+                    .collect(),
+            ),
+        )
+        .await?;
     }
 
-    audit_log(
-        &mut tx,
-        &authed.username,
-        "flows.update",
-        ActionKind::Create,
-        &w_id,
-        Some(&nf.path.to_string()),
-        Some(
-            [Some(("flow", nf.path.as_str()))]
-                .into_iter()
-                .flatten()
-                .collect(),
-        ),
-    )
-    .await?;
-
-    tx.commit().await?;
     webhook.send_message(
         w_id.clone(),
         WebhookMessage::UpdateFlow {
@@ -382,7 +386,6 @@ async fn update_flow(
         },
     );
 
-    let tx = user_db.begin(&authed).await?;
     let (dependency_job_uuid, mut tx) = push(
         tx,
         &w_id,
@@ -625,7 +628,6 @@ mod tests {
           "modules": [
             {
               "id": "a",
-              "input_transforms": {},
               "value": {
                 "input_transforms": {
                     "test": {
@@ -639,7 +641,6 @@ mod tests {
             },
             {
               "id": "b",
-              "input_transforms": {},
               "value": {
                 "input_transforms": {},
                 "type": "rawscript",
@@ -653,7 +654,6 @@ mod tests {
             },
             {
               "id": "c",
-              "input_transforms": {},
               "value": {
                 "type": "forloopflow",
                 "iterator": {
@@ -676,7 +676,6 @@ mod tests {
           ],
           "failure_module": {
             "id": "d",
-            "input_transforms": {},
             "value": {
               "input_transforms": {},
               "type": "script",

backend/windmill-api/src/folders.rs

@@ -6,9 +6,11 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
+use std::sync::Arc;
+
 use crate::{
     db::{UserDB, DB},
-    users::Authed,
+    users::{AuthCache, Authed, Tokened},
     webhook_util::{WebhookMessage, WebhookShared},
 };
 use axum::{
@@ -144,8 +146,10 @@ lazy_static! {
 
 async fn create_folder(
     authed: Authed,
+    Tokened { token }: Tokened,
     Extension(user_db): Extension<UserDB>,
     Extension(webhook): Extension<WebhookShared>,
+    Extension(cache): Extension<Arc<AuthCache>>,
     Path(w_id): Path<String>,
     Json(ng): Json<NewFolder>,
 ) -> Result<String> {
@@ -156,8 +160,8 @@ async fn create_folder(
             "Folder name can only contain alphanumeric characters, underscores"
         )));
     }
-
     check_name_conflict(&mut tx, &w_id, &ng.name).await?;
+    cache.invalidate(&w_id, token).await;
     let owner = username_to_permissioned_as(&authed.username);
     let owners = &ng.owners.unwrap_or(vec![owner.clone()]);
 

backend/windmill-api/src/inputs.rs

@@ -0,0 +1,282 @@
+/*
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
+ * This file and its contents are licensed under the AGPLv3 License.
+ * Please see the included NOTICE for copyright information and
+ * LICENSE-AGPL for a copy of the license.
+ */
+
+use crate::{db::UserDB, jobs::CompletedJob, users::Authed};
+use axum::{
+    extract::{Path, Query},
+    routing::{get, post},
+    Extension, Json, Router,
+};
+use chrono::{DateTime, Utc};
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+use sqlx::types::Uuid;
+use std::{
+    fmt::{Display, Formatter},
+    vec,
+};
+use windmill_common::{
+    error::JsonResult,
+    scripts::to_i64,
+    utils::{paginate, Pagination},
+};
+use windmill_queue::JobKind;
+
+pub fn workspaced_service() -> Router {
+    Router::new()
+        .route("/history", get(get_input_history))
+        .route("/list", get(list_saved_inputs))
+        .route("/create", post(create_input))
+        .route("/update", post(update_input))
+        .route("/delete/:id", post(delete_input))
+}
+
+#[derive(Debug, sqlx::FromRow, Serialize, Deserialize)]
+pub struct InputRow {
+    pub id: Uuid,
+    pub workspace_id: String,
+    pub runnable_id: String,
+    pub runnable_type: RunnableType,
+    pub name: String,
+    pub args: Value,
+    pub created_at: DateTime<Utc>,
+    pub created_by: String,
+    pub is_public: bool,
+}
+
+#[derive(Debug, Serialize, Deserialize, sqlx::Type)]
+#[sqlx(type_name = "runnable_type")]
+pub enum RunnableType {
+    ScriptHash,
+    ScriptPath,
+    FlowPath,
+}
+
+impl Display for RunnableType {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        match self {
+            RunnableType::ScriptHash => write!(f, "ScriptHash"),
+            RunnableType::ScriptPath => write!(f, "ScriptPath"),
+            RunnableType::FlowPath => write!(f, "FlowPath"),
+        }
+    }
+}
+
+impl RunnableType {
+    fn job_kind(&self) -> JobKind {
+        match self {
+            RunnableType::ScriptHash => JobKind::Script,
+            RunnableType::ScriptPath => JobKind::Script,
+            RunnableType::FlowPath => JobKind::Flow,
+        }
+    }
+
+    fn column_name(&self) -> &'static str {
+        match self {
+            RunnableType::ScriptHash => "script_hash",
+            RunnableType::ScriptPath => "script_path",
+            RunnableType::FlowPath => "script_path",
+        }
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct RunnableParams {
+    pub runnable_id: String,
+    pub runnable_type: RunnableType,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct Input {
+    id: Uuid,
+    name: String,
+    created_at: chrono::DateTime<chrono::Utc>,
+    args: serde_json::Value,
+    created_by: String,
+    is_public: bool,
+}
+
+async fn get_input_history(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    Path(w_id): Path<String>,
+    Query(pagination): Query<Pagination>,
+    Query(r): Query<RunnableParams>,
+) -> JsonResult<Vec<Input>> {
+    let (per_page, offset) = paginate(pagination);
+
+    let mut tx = user_db.begin(&authed).await?;
+
+    let sql = &format!(
+        "select distinct on (args) * from completed_job \
+        where {} = $1 and job_kind = $2 and workspace_id = $3 \
+        order by args, started_at desc limit $4 offset $5",
+        r.runnable_type.column_name()
+    );
+
+    let query = sqlx::query_as::<_, CompletedJob>(sql);
+
+    let query = match r.runnable_type {
+        RunnableType::ScriptHash => query.bind(to_i64(&r.runnable_id)?),
+        _ => query.bind(&r.runnable_id),
+    };
+
+    let rows = query
+        .bind(r.runnable_type.job_kind())
+        .bind(&w_id)
+        .bind(per_page as i32)
+        .bind(offset as i32)
+        .fetch_all(&mut tx)
+        .await?;
+
+    tx.commit().await?;
+
+    let mut inputs = vec![];
+
+    for row in rows {
+        inputs.push(Input {
+            id: row.id,
+            name: format!(
+                "{} {}",
+                row.created_at.format("%H:%M %-d/%-m"),
+                row.created_by
+            ),
+            created_at: row.created_at,
+            args: row.args.unwrap_or(serde_json::json!({})),
+            created_by: row.created_by,
+            is_public: true,
+        });
+    }
+
+    Ok(Json(inputs))
+}
+
+async fn list_saved_inputs(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    Path(w_id): Path<String>,
+    Query(pagination): Query<Pagination>,
+    Query(r): Query<RunnableParams>,
+) -> JsonResult<Vec<Input>> {
+    let (per_page, offset) = paginate(pagination);
+
+    let mut tx = user_db.begin(&authed).await?;
+
+    let rows = sqlx::query_as::<_, InputRow>(
+        "select * from input \
+         where runnable_id = $1 and runnable_type = $2 and workspace_id = $3 \
+         and is_public IS true OR created_by = $4 \
+         order by created_at desc limit $5 offset $6",
+    )
+    .bind(&r.runnable_id)
+    .bind(&r.runnable_type)
+    .bind(&w_id)
+    .bind(&authed.username)
+    .bind(per_page as i32)
+    .bind(offset as i32)
+    .fetch_all(&mut tx)
+    .await?;
+
+    tx.commit().await?;
+
+    let mut inputs: Vec<Input> = Vec::new();
+
+    for row in rows {
+        inputs.push(Input {
+            id: row.id,
+            name: row.name,
+            args: row.args,
+            created_by: row.created_by,
+            created_at: row.created_at,
+            is_public: row.is_public,
+        })
+    }
+
+    Ok(Json(inputs))
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct CreateInput {
+    name: String,
+    args: serde_json::Value,
+}
+
+async fn create_input(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    Path(w_id): Path<String>,
+    Query(r): Query<RunnableParams>,
+    Json(input): Json<CreateInput>,
+) -> JsonResult<String> {
+    let mut tx = user_db.begin(&authed).await?;
+
+    let id = Uuid::new_v4();
+
+    sqlx::query(
+        "INSERT INTO input (id, workspace_id, runnable_id, runnable_type, name, args, created_by) VALUES ($1, $2, $3, $4, $5, $6, $7)",
+    )
+    .bind(&id)
+    .bind(&w_id)
+    .bind(&r.runnable_id)
+    .bind(&r.runnable_type)
+    .bind(&input.name)
+    .bind(&input.args)
+    .bind(&authed.username)
+    .execute(&mut tx)
+    .await?;
+
+    tx.commit().await?;
+
+    Ok(Json(id.to_string()))
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct UpdateInput {
+    id: Uuid,
+    name: String,
+    is_public: bool,
+}
+
+async fn update_input(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    Path(w_id): Path<String>,
+    Json(input): Json<UpdateInput>,
+) -> JsonResult<String> {
+    let mut tx = user_db.begin(&authed).await?;
+
+    sqlx::query("UPDATE input SET name = $1, is_public = $2 WHERE id = $3 and workspace_id = $4")
+        .bind(&input.name)
+        .bind(&input.is_public)
+        .bind(&input.id)
+        .bind(&w_id)
+        .execute(&mut tx)
+        .await?;
+
+    tx.commit().await?;
+
+    Ok(Json(input.id.to_string()))
+}
+
+async fn delete_input(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    Path((w_id, i_id)): Path<(String, Uuid)>,
+) -> JsonResult<String> {
+    let mut tx = user_db.begin(&authed).await?;
+
+    sqlx::query("DELETE FROM input WHERE id = $1 and workspace_id = $2")
+        .bind(&i_id)
+        .bind(&w_id)
+        .execute(&mut tx)
+        .await?;
+
+    tx.commit().await?;
+
+    Ok(Json(i_id.to_string()))
+}

backend/windmill-api/src/jobs.rs

@@ -6,6 +6,12 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
+use crate::{
+    db::{UserDB, DB},
+    users::{require_owner_of_path, Authed, OptAuthed},
+    variables::get_workspace_key,
+    BASE_URL,
+};
 use anyhow::Context;
 use axum::{
     extract::{FromRequest, Json, Path, Query},
@@ -30,13 +36,8 @@ use windmill_common::{
     users::username_to_permissioned_as,
     utils::{not_found_if_none, now_from_db, paginate, require_admin, Pagination, StripPath},
 };
-use windmill_queue::{get_queued_job, push, JobKind, JobPayload, QueuedJob, RawCode};
-
-use crate::{
-    db::{UserDB, DB},
-    users::{require_owner_of_path, Authed, OptAuthed},
-    variables::get_workspace_key,
-    BASE_URL,
+use windmill_queue::{
+    get_queued_job, push, JobKind, JobPayload, QueueTransaction, QueuedJob, RawCode,
 };
 
 pub fn workspaced_service() -> Router {
@@ -47,6 +48,10 @@ pub fn workspaced_service() -> Router {
             "/run_wait_result/p/*script_path",
             post(run_wait_result_job_by_path),
         )
+        .route(
+            "/run_wait_result/p/*script_path",
+            get(run_wait_result_job_by_path_get),
+        )
         .route(
             "/run_wait_result/h/:hash",
             post(run_wait_result_job_by_hash),
@@ -60,6 +65,7 @@ pub fn workspaced_service() -> Router {
         .route("/run/preview_flow", post(run_preview_flow_job))
         .route("/list", get(list_jobs))
         .route("/queue/list", get(list_queue_jobs))
+        .route("/queue/count", get(count_queue_jobs))
         .route("/completed/list", get(list_completed_jobs))
         .route("/completed/get/:id", get(get_completed_job))
         .route("/completed/get_result/:id", get(get_completed_job_result))
@@ -110,6 +116,7 @@ async fn get_result_by_id(
 }
 
 async fn cancel_job_api(
+    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
     OptAuthed(opt_authed): OptAuthed,
     Extension(db): Extension<DB>,
     Path((w_id, id)): Path<(String, Uuid)>,
@@ -123,7 +130,7 @@ async fn cancel_job_api(
     };
 
     let (mut tx, job_option) =
-        windmill_queue::cancel_job(&username, reason, id, &w_id, tx, false).await?;
+        windmill_queue::cancel_job(&username, reason, id, &w_id, tx, rsmq, false).await?;
 
     if let Some(id) = job_option {
         audit_log(
@@ -152,6 +159,7 @@ async fn cancel_job_api(
 }
 
 async fn force_cancel(
+    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
     OptAuthed(opt_authed): OptAuthed,
     Extension(db): Extension<DB>,
     Path((w_id, id)): Path<(String, Uuid)>,
@@ -165,7 +173,7 @@ async fn force_cancel(
     };
 
     let (mut tx, job_option) =
-        windmill_queue::cancel_job(&username, reason, id, &w_id, tx, true).await?;
+        windmill_queue::cancel_job(&username, reason, id, &w_id, tx, rsmq, true).await?;
 
     if let Some(id) = job_option {
         audit_log(
@@ -309,6 +317,7 @@ pub struct RunJobQuery {
     include_header: Option<String>,
     invisible_to_owner: Option<bool>,
     queue_limit: Option<i64>,
+    payload: Option<String>,
 }
 
 lazy_static::lazy_static! {
@@ -491,6 +500,26 @@ async fn list_queue_jobs(
     Ok(Json(jobs))
 }
 
+#[derive(Serialize, Debug, FromRow)]
+struct QueueStats {
+    database_length: i64,
+}
+
+async fn count_queue_jobs(
+    Extension(db): Extension<DB>,
+    Path(w_id): Path<String>,
+) -> error::JsonResult<QueueStats> {
+    Ok(Json(
+        sqlx::query_as!(
+            QueueStats,
+            "SELECT coalesce(COUNT(*), 0) as \"database_length!\" FROM queue WHERE workspace_id = $1",
+            w_id
+        )
+        .fetch_one(&db)
+        .await?,
+    ))
+}
+
 async fn list_jobs(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
@@ -788,6 +817,7 @@ async fn get_suspended_flow_info<'c>(
 pub async fn cancel_suspended_job(
     /* unauthed */
     Extension(db): Extension<DB>,
+    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
     Path((w_id, job, resume_id, secret)): Path<(String, Uuid, u32, String)>,
     Query(approver): Query<QueryApprover>,
 ) -> error::Result<String> {
@@ -810,6 +840,7 @@ pub async fn cancel_suspended_job(
         parent_flow,
         &w_id,
         tx,
+        rsmq,
         false,
     )
     .await?;
@@ -1194,7 +1225,7 @@ where
     }
 }
 
-fn decode_payload<D: DeserializeOwned, T: AsRef<[u8]>>(t: T) -> anyhow::Result<D> {
+fn decode_payload<D: DeserializeOwned>(t: String) -> anyhow::Result<D> {
     let vec = base64::engine::general_purpose::URL_SAFE
         .decode(t)
         .context("invalid base64")?;
@@ -1203,14 +1234,15 @@ fn decode_payload<D: DeserializeOwned, T: AsRef<[u8]>>(t: T) -> anyhow::Result<D
 pub async fn run_flow_by_path(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
+    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
     Path((w_id, flow_path)): Path<(String, StripPath)>,
     Query(run_query): Query<RunJobQuery>,
     headers: HeaderMap,
     Json(args): Json<Option<serde_json::Map<String, serde_json::Value>>>,
 ) -> error::Result<(StatusCode, String)> {
     let flow_path = flow_path.to_path();
-    let mut tx = user_db.begin(&authed).await?;
-    let scheduled_for = run_query.get_scheduled_for(&mut tx).await?;
+    let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
+    let scheduled_for = run_query.get_scheduled_for(tx.transaction_mut()).await?;
     let args = run_query.add_include_headers(headers, args.unwrap_or_default());
 
     let (uuid, tx) = push(
@@ -1238,15 +1270,16 @@ pub async fn run_flow_by_path(
 pub async fn run_job_by_path(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
+    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
     Path((w_id, script_path)): Path<(String, StripPath)>,
     Query(run_query): Query<RunJobQuery>,
     headers: HeaderMap,
     Json(args): Json<Option<serde_json::Map<String, serde_json::Value>>>,
 ) -> error::Result<(StatusCode, String)> {
     let script_path = script_path.to_path();
-    let mut tx = user_db.begin(&authed).await?;
-    let job_payload = script_path_to_payload(script_path, &mut tx, &w_id).await?;
-    let scheduled_for = run_query.get_scheduled_for(&mut tx).await?;
+    let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
+    let job_payload = script_path_to_payload(script_path, tx.transaction_mut(), &w_id).await?;
+    let scheduled_for = run_query.get_scheduled_for(tx.transaction_mut()).await?;
     let args = run_query.add_include_headers(headers, args.unwrap_or_default());
 
     let (uuid, tx) = push(
@@ -1385,9 +1418,65 @@ lazy_static::lazy_static! {
         .and_then(|x| x.parse().ok())
         .unwrap_or(20);
 }
+
+pub async fn run_wait_result_job_by_path_get(
+    authed: Authed,
+    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
+    Extension(user_db): Extension<UserDB>,
+    Extension(db): Extension<DB>,
+    Path((w_id, script_path)): Path<(String, StripPath)>,
+    Query(run_query): Query<RunJobQuery>,
+) -> error::JsonResult<serde_json::Value> {
+    let payload_r = run_query
+        .payload
+        .map(decode_payload)
+        .map(|x| x.map_err(|e| Error::InternalErr(e.to_string())));
+
+    let args = if let Some(payload) = payload_r {
+        payload?
+    } else {
+        serde_json::Map::new()
+    };
+
+    check_queue_too_long(db, QUEUE_LIMIT_WAIT_RESULT.or(run_query.queue_limit)).await?;
+    let script_path = script_path.to_path();
+    let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.clone().begin(&authed).await?).into();
+    let job_payload = script_path_to_payload(script_path, tx.transaction_mut(), &w_id).await?;
+
+    let (uuid, tx) = push(
+        tx,
+        &w_id,
+        job_payload,
+        args,
+        &authed.username,
+        &authed.email,
+        username_to_permissioned_as(&authed.username),
+        None,
+        None,
+        run_query.parent_job,
+        run_query.parent_job,
+        false,
+        false,
+        None,
+        !run_query.invisible_to_owner.unwrap_or(false),
+    )
+    .await?;
+    tx.commit().await?;
+
+    run_wait_result(
+        authed,
+        Extension(user_db),
+        *TIMEOUT_WAIT_RESULT,
+        uuid,
+        Path((w_id, script_path)),
+    )
+    .await
+}
+
 pub async fn run_wait_result_job_by_path(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
+    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
     Extension(db): Extension<DB>,
     Path((w_id, script_path)): Path<(String, StripPath)>,
     Query(run_query): Query<RunJobQuery>,
@@ -1396,9 +1485,8 @@ pub async fn run_wait_result_job_by_path(
 ) -> error::JsonResult<serde_json::Value> {
     check_queue_too_long(db, QUEUE_LIMIT_WAIT_RESULT.or(run_query.queue_limit)).await?;
     let script_path = script_path.to_path();
-    let mut tx = user_db.clone().begin(&authed).await?;
-    let job_payload = script_path_to_payload(script_path, &mut tx, &w_id).await?;
-    let scheduled_for = run_query.get_scheduled_for(&mut tx).await?;
+    let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.clone().begin(&authed).await?).into();
+    let job_payload = script_path_to_payload(script_path, tx.transaction_mut(), &w_id).await?;
 
     let args = run_query.add_include_headers(headers, args.unwrap_or_default());
 
@@ -1410,7 +1498,7 @@ pub async fn run_wait_result_job_by_path(
         &authed.username,
         &authed.email,
         username_to_permissioned_as(&authed.username),
-        scheduled_for,
+        None,
         None,
         run_query.parent_job,
         run_query.parent_job,
@@ -1435,6 +1523,7 @@ pub async fn run_wait_result_job_by_path(
 pub async fn run_wait_result_job_by_hash(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
+    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
     Extension(db): Extension<DB>,
     Path((w_id, script_hash)): Path<(String, ScriptHash)>,
     Query(run_query): Query<RunJobQuery>,
@@ -1444,9 +1533,9 @@ pub async fn run_wait_result_job_by_hash(
     check_queue_too_long(db, run_query.queue_limit).await?;
 
     let hash = script_hash.0;
-    let mut tx = user_db.clone().begin(&authed).await?;
-    let path = get_path_for_hash(&mut tx, &w_id, hash).await?;
-    let scheduled_for = run_query.get_scheduled_for(&mut tx).await?;
+    let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.clone().begin(&authed).await?).into();
+    let path = get_path_for_hash(tx.transaction_mut(), &w_id, hash).await?;
+
     let args = run_query.add_include_headers(headers, args.unwrap_or_default());
 
     let (uuid, tx) = push(
@@ -1457,7 +1546,7 @@ pub async fn run_wait_result_job_by_hash(
         &authed.username,
         &authed.email,
         username_to_permissioned_as(&authed.username),
-        scheduled_for,
+        None,
         None,
         run_query.parent_job,
         run_query.parent_job,
@@ -1482,6 +1571,7 @@ pub async fn run_wait_result_job_by_hash(
 pub async fn run_wait_result_flow_by_path(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
+    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
     Extension(db): Extension<DB>,
     Path((w_id, flow_path)): Path<(String, StripPath)>,
     Query(run_query): Query<RunJobQuery>,
@@ -1491,8 +1581,8 @@ pub async fn run_wait_result_flow_by_path(
     check_queue_too_long(db, run_query.queue_limit).await?;
 
     let flow_path = flow_path.to_path();
-    let mut tx = user_db.clone().begin(&authed).await?;
-    let scheduled_for = run_query.get_scheduled_for(&mut tx).await?;
+    let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.clone().begin(&authed).await?).into();
+    let scheduled_for = run_query.get_scheduled_for(tx.transaction_mut()).await?;
     let args = run_query.add_include_headers(headers, args.unwrap_or_default());
 
     let (uuid, tx) = push(
@@ -1513,7 +1603,6 @@ pub async fn run_wait_result_flow_by_path(
         !run_query.invisible_to_owner.unwrap_or(false),
     )
     .await?;
-
     tx.commit().await?;
 
     run_wait_result(
@@ -1544,13 +1633,14 @@ pub async fn script_path_to_payload<'c>(
 async fn run_preview_job(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
+    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
     Path(w_id): Path<String>,
     Query(run_query): Query<RunJobQuery>,
     headers: HeaderMap,
     Json(preview): Json<Preview>,
 ) -> error::Result<(StatusCode, String)> {
-    let mut tx = user_db.begin(&authed).await?;
-    let scheduled_for = run_query.get_scheduled_for(&mut tx).await?;
+    let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
+    let scheduled_for = run_query.get_scheduled_for(tx.transaction_mut()).await?;
     let args = run_query.add_include_headers(headers, preview.args.unwrap_or_default());
 
     let (uuid, tx) = push(
@@ -1577,19 +1667,21 @@ async fn run_preview_job(
     )
     .await?;
     tx.commit().await?;
+
     Ok((StatusCode::CREATED, uuid.to_string()))
 }
 
 async fn run_preview_flow_job(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
+    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
     Path(w_id): Path<String>,
     Query(run_query): Query<RunJobQuery>,
     headers: HeaderMap,
     Json(raw_flow): Json<PreviewFlow>,
 ) -> error::Result<(StatusCode, String)> {
-    let mut tx = user_db.begin(&authed).await?;
-    let scheduled_for = run_query.get_scheduled_for(&mut tx).await?;
+    let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
+    let scheduled_for = run_query.get_scheduled_for(tx.transaction_mut()).await?;
     let args = run_query.add_include_headers(headers, raw_flow.args.unwrap_or_default());
 
     let (uuid, tx) = push(
@@ -1611,21 +1703,23 @@ async fn run_preview_flow_job(
     )
     .await?;
     tx.commit().await?;
+
     Ok((StatusCode::CREATED, uuid.to_string()))
 }
 
 pub async fn run_job_by_hash(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
+    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
     Path((w_id, script_hash)): Path<(String, ScriptHash)>,
     Query(run_query): Query<RunJobQuery>,
     headers: HeaderMap,
     Json(args): Json<Option<serde_json::Map<String, serde_json::Value>>>,
 ) -> error::Result<(StatusCode, String)> {
     let hash = script_hash.0;
-    let mut tx = user_db.begin(&authed).await?;
-    let path = get_path_for_hash(&mut tx, &w_id, hash).await?;
-    let scheduled_for = run_query.get_scheduled_for(&mut tx).await?;
+    let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
+    let path = get_path_for_hash(tx.transaction_mut(), &w_id, hash).await?;
+    let scheduled_for = run_query.get_scheduled_for(tx.transaction_mut()).await?;
     let args = run_query.add_include_headers(headers, args.unwrap_or_default());
 
     let (uuid, tx) = push(
@@ -1647,6 +1741,7 @@ pub async fn run_job_by_hash(
     )
     .await?;
     tx.commit().await?;
+
     Ok((StatusCode::CREATED, uuid.to_string()))
 }
 

backend/windmill-api/src/lib.rs

@@ -7,17 +7,6 @@
  */
 
 use crate::oauth2::AllClients;
-use argon2::Argon2;
-use axum::{middleware::from_extractor, routing::get, Extension, Router};
-use db::DB;
-use git_version::git_version;
-use reqwest::Client;
-use std::{net::SocketAddr, sync::Arc};
-use tower::ServiceBuilder;
-use tower_cookies::CookieManagerLayer;
-use tower_http::trace::TraceLayer;
-use windmill_common::utils::rd_string;
-
 use crate::{
     db::UserDB,
     oauth2::{build_oauth_clients, SlackVerifier},
@@ -25,6 +14,20 @@ use crate::{
     users::{Authed, OptAuthed},
     webhook_util::WebhookShared,
 };
+use argon2::Argon2;
+use axum::{middleware::from_extractor, routing::get, Extension, Router};
+use db::DB;
+use git_version::git_version;
+use hyper::Method;
+use reqwest::Client;
+use std::{net::SocketAddr, sync::Arc};
+use tower::ServiceBuilder;
+use tower_cookies::CookieManagerLayer;
+use tower_http::{
+    cors::{Any, CorsLayer},
+    trace::TraceLayer,
+};
+use windmill_common::utils::rd_string;
 
 mod apps;
 mod audit;
@@ -35,6 +38,7 @@ mod flows;
 mod folders;
 mod granular_acls;
 mod groups;
+mod inputs;
 pub mod jobs;
 mod oauth2;
 mod resources;
@@ -77,6 +81,7 @@ lazy_static::lazy_static! {
 
 pub async fn run_server(
     db: DB,
+    rsmq: Option<rsmq_async::MultiplexedRsmq>,
     addr: SocketAddr,
     mut rx: tokio::sync::broadcast::Receiver<()>,
 ) -> anyhow::Result<()> {
@@ -96,10 +101,16 @@ pub async fn run_server(
                 .on_request(()),
         )
         .layer(Extension(db.clone()))
+        .layer(Extension(rsmq))
         .layer(Extension(user_db))
         .layer(Extension(auth_cache.clone()))
         .layer(CookieManagerLayer::new())
         .layer(Extension(WebhookShared::new(rx.resubscribe(), db.clone())));
+
+    let cors = CorsLayer::new()
+        .allow_methods([Method::GET, Method::POST])
+        .allow_origin(Any);
+
     // build our application with a route
     let app = Router::new()
         .nest(
@@ -108,25 +119,27 @@ pub async fn run_server(
                 .nest(
                     "/w/:workspace_id",
                     Router::new()
+                        // Reordered alphabetically
+                        .nest("/acls", granular_acls::workspaced_service())
+                        .nest("/apps", apps::workspaced_service())
+                        .nest("/audit", audit::workspaced_service())
+                        .nest("/capture", capture::workspaced_service())
+                        .nest("/favorites", favorite::workspaced_service())
+                        .nest("/flows", flows::workspaced_service())
+                        .nest("/folders", folders::workspaced_service())
+                        .nest("/groups", groups::workspaced_service())
+                        .nest("/inputs", inputs::workspaced_service())
+                        .nest("/jobs", jobs::workspaced_service().layer(cors.clone()))
+                        .nest("/oauth", oauth2::workspaced_service())
+                        .nest("/resources", resources::workspaced_service())
+                        .nest("/schedules", schedule::workspaced_service())
                         .nest("/scripts", scripts::workspaced_service())
-                        .nest("/jobs", jobs::workspaced_service())
                         .nest(
                             "/users",
                             users::workspaced_service().layer(Extension(argon2.clone())),
                         )
                         .nest("/variables", variables::workspaced_service())
-                        .nest("/oauth", oauth2::workspaced_service())
-                        .nest("/resources", resources::workspaced_service())
-                        .nest("/schedules", schedule::workspaced_service())
-                        .nest("/groups", groups::workspaced_service())
-                        .nest("/audit", audit::workspaced_service())
-                        .nest("/acls", granular_acls::workspaced_service())
-                        .nest("/workspaces", workspaces::workspaced_service())
-                        .nest("/apps", apps::workspaced_service())
-                        .nest("/flows", flows::workspaced_service())
-                        .nest("/capture", capture::workspaced_service())
-                        .nest("/favorites", favorite::workspaced_service())
-                        .nest("/folders", folders::workspaced_service()),
+                        .nest("/workspaces", workspaces::workspaced_service()),
                 )
                 .nest("/workspaces", workspaces::global_service())
                 .nest(
@@ -136,16 +149,25 @@ pub async fn run_server(
                 .nest("/workers", worker_ping::global_service())
                 .nest("/scripts", scripts::global_service())
                 .nest("/flows", flows::global_service())
-                .nest("/apps", apps::global_service())
+                .nest("/apps", apps::global_service().layer(cors.clone()))
                 .nest("/schedules", schedule::global_service())
                 .route_layer(from_extractor::<Authed>())
                 .route_layer(from_extractor::<users::Tokened>())
+                .nest("/scripts_u", scripts::global_unauthed_service())
                 .nest(
                     "/w/:workspace_id/apps_u",
-                    apps::unauthed_service().layer(from_extractor::<OptAuthed>()),
+                    apps::unauthed_service()
+                        .layer(from_extractor::<OptAuthed>())
+                        .layer(cors.clone()),
+                )
+                .nest(
+                    "/w/:workspace_id/jobs_u",
+                    jobs::global_service().layer(cors.clone()),
+                )
+                .nest(
+                    "/w/:workspace_id/capture_u",
+                    capture::global_service().layer(cors),
                 )
-                .nest("/w/:workspace_id/jobs_u", jobs::global_service())
-                .nest("/w/:workspace_id/capture_u", capture::global_service())
                 .nest(
                     "/auth",
                     users::make_unauthed_service().layer(Extension(argon2)),

backend/windmill-api/src/oauth2.rs

@@ -32,7 +32,8 @@ use windmill_audit::{audit_log, ActionKind};
 use windmill_common::users::username_to_permissioned_as;
 use windmill_common::utils::{not_found_if_none, now_from_db};
 
-use crate::users::{truncate_token, Authed, NEW_USER_WEBHOOK};
+use crate::users::{truncate_token, Authed};
+use crate::webhook_util::{InstanceEvent, WebhookShared};
 use crate::workspaces::invite_user_to_all_auto_invite_worspaces;
 use crate::{
     db::{UserDB, DB},
@@ -43,7 +44,7 @@ use crate::{BASE_URL, HTTP_CLIENT, IS_SECURE, OAUTH_CLIENTS, SLACK_SIGNING_SECRE
 use windmill_common::error::{self, to_anyhow, Error};
 use windmill_common::oauth2::*;
 
-use windmill_queue::JobPayload;
+use windmill_queue::{JobPayload, QueueTransaction};
 
 use std::{fs, str};
 
@@ -728,6 +729,7 @@ where
 async fn slack_command(
     SlackSig { sig, ts }: SlackSig,
     Extension(db): Extension<DB>,
+    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
     body: Bytes,
 ) -> error::Result<String> {
     let form: SlackCommand = serde_urlencoded::from_bytes(&body)
@@ -740,7 +742,7 @@ async fn slack_command(
         }
     }
 
-    let mut tx = db.begin().await?;
+    let mut tx: QueueTransaction<'_, _> = (rsmq, db.begin().await?).into();
     let settings = sqlx::query_as!(
         WorkspaceSettings,
         "SELECT * FROM workspace_settings WHERE slack_team_id = $1",
@@ -756,7 +758,7 @@ async fn slack_command(
             } else {
                 let path = path.strip_prefix("script/").unwrap_or_else(|| path);
                 let script_hash = windmill_common::get_latest_hash_for_path(
-                    &mut tx,
+                    tx.transaction_mut(),
                     &settings.workspace_id,
                     path,
                 )
@@ -788,14 +790,15 @@ async fn slack_command(
                 true,
             )
             .await?;
-            tx.commit().await?;
             let url = BASE_URL.to_owned();
+            tx.commit().await?;
             return Ok(format!(
                 "Job launched. See details at {url}/run/{uuid}?workspace={}",
                 &settings.workspace_id
             ));
         }
     }
+    tx.commit().await?;
 
     return Ok(format!(
         "workspace not properly configured (did you set the script to trigger in the settings?)"
@@ -815,6 +818,7 @@ async fn login_callback(
     Path(client_name): Path<String>,
     cookies: Cookies,
     Extension(db): Extension<DB>,
+    Extension(webhook): Extension<WebhookShared>,
     Json(callback): Json<OAuthCallback>,
 ) -> error::Result<String> {
     let client_w_config = &OAUTH_CLIENTS
@@ -849,7 +853,8 @@ async fn login_callback(
             _ => user.email.ok_or_else(|| {
                 error::Error::BadRequest("email address not fetchable from user info".to_string())
             })?,
-        };
+        }
+        .to_lowercase();
 
         if let Some(domains) = &client_w_config.allowed_domains {
             if !domains.iter().any(|d| email.ends_with(d)) {
@@ -939,14 +944,7 @@ async fn login_callback(
         }
         tx.commit().await?;
 
-        if let Some(new_user_webhook) = NEW_USER_WEBHOOK.clone() {
-            let _ = HTTP_CLIENT
-                .post(&new_user_webhook)
-                .json(&serde_json::json!({"email" : &email, "event": "oauth_signup"}))
-                .send()
-                .await
-                .map_err(|e| tracing::error!("Error sending new user webhook: {}", e.to_string()));
-        }
+        webhook.send_instance_event(InstanceEvent::UserSignupOAuth { email: email.clone() });
 
         Ok("Successfully logged in".to_string())
     } else {

backend/windmill-api/src/resources.rs

@@ -8,7 +8,7 @@
 
 use crate::{
     db::{UserDB, DB},
-    users::{require_owner_of_path, Authed},
+    users::{maybe_refresh_folders, require_owner_of_path, Authed},
     webhook_util::{WebhookMessage, WebhookShared},
 };
 use axum::{
@@ -264,9 +264,12 @@ async fn create_resource(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
     Extension(webhook): Extension<WebhookShared>,
+    Extension(db): Extension<DB>,
     Path(w_id): Path<String>,
     Json(resource): Json<CreateResource>,
 ) -> Result<(StatusCode, String)> {
+    let authed = maybe_refresh_folders(&resource.path, &w_id, authed, &db).await;
+
     let mut tx = user_db.begin(&authed).await?;
 
     check_path_conflict(&mut tx, &w_id, &resource.path).await?;
@@ -375,6 +378,7 @@ async fn update_resource(
     }
 
     sqlb.returning("path");
+    let authed = maybe_refresh_folders(path, &w_id, authed, &db).await;
 
     let mut tx = user_db.begin(&authed).await?;
 

backend/windmill-api/src/schedule.rs

@@ -6,27 +6,26 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
-use std::str::FromStr;
-
 use crate::{
     db::{UserDB, DB},
-    users::Authed,
+    users::{maybe_refresh_folders, Authed},
 };
 use axum::{
     extract::{Extension, Path, Query},
     routing::{delete, get, post},
     Json, Router,
 };
-use chrono::{DateTime, FixedOffset};
+use chrono::{DateTime, Utc};
 use serde::Deserialize;
 use sqlx::{Postgres, Transaction};
+use std::str::FromStr;
 use windmill_audit::{audit_log, ActionKind};
 use windmill_common::{
     error::{Error, JsonResult, Result},
     schedule::Schedule,
     utils::{not_found_if_none, paginate, Pagination, StripPath},
 };
-use windmill_queue::{self, schedule::push_scheduled_job, JobKind};
+use windmill_queue::{self, schedule::push_scheduled_job, JobKind, QueueTransaction};
 
 pub fn workspaced_service() -> Router {
     Router::new()
@@ -47,7 +46,7 @@ pub fn global_service() -> Router {
 pub struct NewSchedule {
     pub path: String,
     pub schedule: String,
-    pub offset: i32,
+    pub timezone: String,
     pub script_path: String,
     pub is_flow: bool,
     pub args: Option<serde_json::Value>,
@@ -78,23 +77,34 @@ async fn check_path_conflict<'c>(
 
 async fn create_schedule(
     authed: Authed,
+    Extension(db): Extension<DB>,
     Extension(user_db): Extension<UserDB>,
+    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
     Path(w_id): Path<String>,
     Json(ns): Json<NewSchedule>,
 ) -> Result<String> {
-    let mut tx = user_db.begin(&authed).await?;
+    let authed = maybe_refresh_folders(&ns.path, &w_id, authed, &db).await;
+    let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
+
     cron::Schedule::from_str(&ns.schedule).map_err(|e| Error::BadRequest(e.to_string()))?;
-    check_path_conflict(&mut tx, &w_id, &ns.path).await?;
-    check_flow_conflict(&mut tx, &w_id, &ns.path, ns.is_flow, &ns.script_path).await?;
+    check_path_conflict(tx.transaction_mut(), &w_id, &ns.path).await?;
+    check_flow_conflict(
+        tx.transaction_mut(),
+        &w_id,
+        &ns.path,
+        ns.is_flow,
+        &ns.script_path,
+    )
+    .await?;
 
     let schedule = sqlx::query_as!(
         Schedule,
-        "INSERT INTO schedule (workspace_id, path, schedule, offset_, edited_by, script_path, \
+        "INSERT INTO schedule (workspace_id, path, schedule, timezone, edited_by, script_path, \
          is_flow, args, enabled, email) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) RETURNING *",
         w_id,
         ns.path,
         ns.schedule,
-        ns.offset,
+        ns.timezone,
         &authed.username,
         ns.script_path,
         ns.is_flow,
@@ -135,13 +145,18 @@ async fn create_schedule(
 
 async fn edit_schedule(
     authed: Authed,
+    Extension(db): Extension<DB>,
     Extension(user_db): Extension<UserDB>,
+    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
     Path((w_id, path)): Path<(String, StripPath)>,
     Json(es): Json<EditSchedule>,
 ) -> Result<String> {
-    let mut tx = user_db.begin(&authed).await?;
     let path = path.to_path();
 
+    let authed = maybe_refresh_folders(&path, &w_id, authed, &db).await;
+      let mut tx: QueueTransaction<'_, rsmq_async::MultiplexedRsmq> =
+        (rsmq, user_db.begin(&authed).await?).into();
+
     cron::Schedule::from_str(&es.schedule).map_err(|e| Error::BadRequest(e.to_string()))?;
 
     let is_flow = sqlx::query_scalar!(
@@ -152,12 +167,13 @@ async fn edit_schedule(
     .fetch_one(&mut tx)
     .await?;
 
-    clear_schedule(&mut tx, path, is_flow).await?;
+    clear_schedule(tx.transaction_mut(), path, is_flow).await?;
     let schedule = sqlx::query_as!(
         Schedule,
-        "UPDATE schedule SET schedule = $1, args = $2 WHERE path \
-         = $3 AND workspace_id = $4 RETURNING *",
+        "UPDATE schedule SET schedule = $1, timezone = $2, args = $3 WHERE path \
+         = $4 AND workspace_id = $5 RETURNING *",
         es.schedule,
+        es.timezone,
         es.args,
         path,
         w_id,
@@ -166,10 +182,6 @@ async fn edit_schedule(
     .await
     .map_err(|e| Error::InternalErr(format!("updating schedule in {w_id}: {e}")))?;
 
-    if schedule.enabled {
-        tx = push_scheduled_job(tx, schedule).await?;
-    }
-
     audit_log(
         &mut tx,
         &authed.username,
@@ -185,6 +197,10 @@ async fn edit_schedule(
         ),
     )
     .await?;
+
+    if schedule.enabled {
+        tx = push_scheduled_job(tx, schedule).await?;
+    }
     tx.commit().await?;
 
     Ok(path.to_string())
@@ -235,15 +251,26 @@ async fn exists_schedule(
     Ok(Json(res))
 }
 
+#[derive(Deserialize)]
+pub struct PreviewPayload {
+    pub schedule: String,
+    pub timezone: String,
+}
+
 pub async fn preview_schedule(
     Json(payload): Json<PreviewPayload>,
-) -> JsonResult<Vec<DateTime<chrono::Utc>>> {
+) -> JsonResult<Vec<DateTime<Utc>>> {
     let schedule = cron::Schedule::from_str(&payload.schedule)
         .map_err(|e| Error::BadRequest(e.to_string()))?;
-    let upcoming: Vec<DateTime<chrono::Utc>> = schedule
-        .upcoming(get_offset(payload.offset))
-        .take(10)
-        .map(|x| x.into())
+
+    let tz =
+        chrono_tz::Tz::from_str(&payload.timezone).map_err(|e| Error::BadRequest(e.to_string()))?;
+
+    let upcoming: Vec<DateTime<Utc>> = schedule
+        .upcoming(tz)
+        .take(5)
+        // Convert back to UTC for a standardised API response. The client will convert to the local timezone.
+        .map(|x| x.with_timezone(&Utc))
         .collect();
 
     Ok(Json(upcoming))
@@ -252,10 +279,12 @@ pub async fn preview_schedule(
 pub async fn set_enabled(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
+    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
     Path((w_id, path)): Path<(String, StripPath)>,
     Json(payload): Json<SetEnabled>,
 ) -> Result<String> {
-    let mut tx = user_db.begin(&authed).await?;
+    let mut tx: QueueTransaction<'_, rsmq_async::MultiplexedRsmq> =
+        (rsmq, user_db.begin(&authed).await?).into();
     let path = path.to_path();
     let schedule_o = sqlx::query_as!(
         Schedule,
@@ -270,11 +299,8 @@ pub async fn set_enabled(
 
     let schedule = not_found_if_none(schedule_o, "Schedule", path)?;
 
-    clear_schedule(&mut tx, path, schedule.is_flow).await?;
+    clear_schedule(tx.transaction_mut(), path, schedule.is_flow).await?;
 
-    if payload.enabled {
-        tx = push_scheduled_job(tx, schedule).await?;
-    }
     audit_log(
         &mut tx,
         &authed.username,
@@ -285,7 +311,12 @@ pub async fn set_enabled(
         Some([("enabled", payload.enabled.to_string().as_ref())].into()),
     )
     .await?;
+
+    if payload.enabled {
+        tx = push_scheduled_job(tx, schedule).await?;
+    }
     tx.commit().await?;
+
     Ok(format!(
         "succesfully updated schedule at path {} to status {}",
         path, payload.enabled
@@ -353,6 +384,7 @@ async fn check_flow_conflict<'c>(
 #[derive(Deserialize)]
 pub struct EditSchedule {
     pub schedule: String,
+    pub timezone: String,
     pub args: Option<serde_json::Value>,
 }
 
@@ -376,16 +408,6 @@ pub async fn clear_schedule<'c>(
     Ok(())
 }
 
-#[derive(Deserialize)]
-pub struct PreviewPayload {
-    pub schedule: String,
-    pub offset: Option<i32>,
-}
-
-fn get_offset(offset: Option<i32>) -> FixedOffset {
-    FixedOffset::west_opt(offset.unwrap_or(0) * 60).expect("Invalid offset")
-}
-
 #[derive(Deserialize)]
 pub struct SetEnabled {
     pub enabled: bool,

backend/windmill-api/src/scripts.rs

@@ -6,13 +6,10 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
-use sql_builder::prelude::*;
-use windmill_audit::{audit_log, ActionKind};
-
 use crate::{
     db::{UserDB, DB},
     schedule::clear_schedule,
-    users::{require_owner_of_path, Authed},
+    users::{maybe_refresh_folders, require_owner_of_path, AuthCache, Authed},
     webhook_util::{WebhookMessage, WebhookShared},
     HTTP_CLIENT,
 };
@@ -24,12 +21,15 @@ use axum::{
 use hyper::StatusCode;
 use serde::Serialize;
 use serde_json::json;
+use sql_builder::prelude::*;
 use sql_builder::SqlBuilder;
 use sqlx::{FromRow, Postgres, Transaction};
 use std::{
     collections::hash_map::DefaultHasher,
     hash::{Hash, Hasher},
+    sync::Arc,
 };
+use windmill_audit::{audit_log, ActionKind};
 use windmill_common::{
     error::{Error, JsonResult, Result},
     schedule::Schedule,
@@ -42,7 +42,8 @@ use windmill_common::{
         list_elems_from_hub, not_found_if_none, paginate, require_admin, Pagination, StripPath,
     },
 };
-use windmill_queue::{self, schedule::push_scheduled_job};
+use windmill_parser::MainArgSignature;
+use windmill_queue::{self, schedule::push_scheduled_job, QueueTransaction};
 
 const MAX_HASH_HISTORY_LENGTH_STORED: usize = 20;
 
@@ -60,6 +61,13 @@ pub fn global_service() -> Router {
         .route("/hub/get_full/*path", get(get_full_hub_script_by_path))
 }
 
+pub fn global_unauthed_service() -> Router {
+    Router::new().route(
+        "/tokened_raw/:workspace/:token/*path",
+        get(get_tokened_raw_script_by_path),
+    )
+}
+
 pub fn workspaced_service() -> Router {
     Router::new()
         .route("/list", get(list_scripts))
@@ -76,6 +84,7 @@ pub fn workspaced_service() -> Router {
         .route("/deployment_status/h/:hash", get(get_deployment_status))
         .route("/list_paths", get(list_paths))
 }
+
 async fn list_scripts(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
@@ -184,20 +193,22 @@ fn hash_script(ns: &NewScript) -> i64 {
 async fn create_script(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
+    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
     Extension(webhook): Extension<WebhookShared>,
     Extension(db): Extension<DB>,
     Path(w_id): Path<String>,
     Json(ns): Json<NewScript>,
 ) -> Result<(StatusCode, String)> {
     let hash = ScriptHash(hash_script(&ns));
-    let mut tx = user_db.begin(&authed).await?;
+    let authed = maybe_refresh_folders(&ns.path, &w_id, authed, &db).await;
+    let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
 
     if sqlx::query_scalar!(
         "SELECT 1 FROM script WHERE hash = $1 AND workspace_id = $2",
         hash.0,
         &w_id
     )
-    .fetch_optional(&mut tx)
+    .fetch_optional(tx.transaction_mut())
     .await?
     .is_some()
     {
@@ -258,7 +269,7 @@ async fn create_script(
                 )));
             };
 
-            let ps = get_script_by_hash_internal(&mut tx, &w_id, p_hash).await?;
+            let ps = get_script_by_hash_internal(tx.transaction_mut(), &w_id, p_hash).await?;
 
             if ps.path != ns.path {
                 if !authed.is_admin {
@@ -353,7 +364,7 @@ async fn create_script(
         .await?;
 
         for schedule in schedulables {
-            clear_schedule(&mut tx, &schedule.path, false).await?;
+            clear_schedule(tx.transaction_mut(), &schedule.path, false).await?;
 
             if schedule.enabled {
                 tx = push_scheduled_job(tx, schedule).await?;
@@ -361,36 +372,6 @@ async fn create_script(
         }
     }
 
-    let mut tx = if needs_lock_gen {
-        let dependencies = match ns.language {
-            ScriptLang::Python3 => {
-                windmill_parser_py::parse_python_imports(&ns.content)?.join("\n")
-            }
-            _ => ns.content,
-        };
-        let (_, tx) = windmill_queue::push(
-            tx,
-            &w_id,
-            windmill_queue::JobPayload::Dependencies { hash, dependencies, language: ns.language },
-            serde_json::Map::new(),
-            &authed.username,
-            &authed.email,
-            username_to_permissioned_as(&authed.username),
-            None,
-            None,
-            None,
-            None,
-            false,
-            false,
-            None,
-            true,
-        )
-        .await?;
-        tx
-    } else {
-        tx
-    };
-
     if p_hashes.is_some() && !p_hashes.unwrap().is_empty() {
         audit_log(
             &mut tx,
@@ -405,7 +386,7 @@ async fn create_script(
         webhook.send_message(
             w_id.clone(),
             WebhookMessage::UpdateScript {
-                workspace: w_id,
+                workspace: w_id.clone(),
                 path: ns.path.clone(),
                 hash: hash.to_string(),
             },
@@ -430,13 +411,41 @@ async fn create_script(
         webhook.send_message(
             w_id.clone(),
             WebhookMessage::CreateScript {
-                workspace: w_id,
+                workspace: w_id.clone(),
                 path: ns.path.clone(),
                 hash: hash.to_string(),
             },
         );
     }
 
+    if needs_lock_gen {
+        let dependencies = match ns.language {
+            ScriptLang::Python3 => {
+                windmill_parser_py::parse_python_imports(&ns.content)?.join("\n")
+            }
+            _ => ns.content,
+        };
+        let (_, new_tx) = windmill_queue::push(
+            tx,
+            &w_id,
+            windmill_queue::JobPayload::Dependencies { hash, dependencies, language: ns.language },
+            serde_json::Map::new(),
+            &authed.username,
+            &authed.email,
+            username_to_permissioned_as(&authed.username),
+            None,
+            None,
+            None,
+            None,
+            false,
+            false,
+            None,
+            true,
+        )
+        .await?;
+        tx = new_tx;
+    }
+
     tx.commit().await?;
 
     Ok((StatusCode::CREATED, format!("{}", hash)))
@@ -496,6 +505,18 @@ async fn list_paths(
     Ok(Json(scripts))
 }
 
+async fn get_tokened_raw_script_by_path(
+    Extension(user_db): Extension<UserDB>,
+    Path((w_id, token, path)): Path<(String, String, StripPath)>,
+    Extension(cache): Extension<Arc<AuthCache>>,
+) -> Result<String> {
+    let authed = cache
+        .get_authed(Some(w_id.clone()), &token)
+        .await
+        .ok_or_else(|| Error::NotAuthorized("Invalid token".to_string()))?;
+    return raw_script_by_path(authed, Extension(user_db), Path((w_id, path))).await;
+}
+
 async fn raw_script_by_path(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
@@ -556,11 +577,10 @@ async fn get_script_by_hash_internal<'c>(
 }
 
 async fn get_script_by_hash(
-    authed: Authed,
-    Extension(user_db): Extension<UserDB>,
+    Extension(db): Extension<DB>,
     Path((w_id, hash)): Path<(String, ScriptHash)>,
 ) -> JsonResult<Script> {
-    let mut tx = user_db.begin(&authed).await?;
+    let mut tx = db.begin().await?;
     let r = get_script_by_hash_internal(&mut tx, &w_id, &hash).await?;
     tx.commit().await?;
 
@@ -568,11 +588,10 @@ async fn get_script_by_hash(
 }
 
 async fn raw_script_by_hash(
-    authed: Authed,
-    Extension(user_db): Extension<UserDB>,
+    Extension(db): Extension<DB>,
     Path((w_id, hash_str)): Path<(String, String)>,
 ) -> Result<String> {
-    let mut tx = user_db.begin(&authed).await?;
+    let mut tx = db.begin().await?;
     let hash = ScriptHash(to_i64(hash_str.strip_suffix(".ts").ok_or_else(|| {
         Error::BadRequest("Raw script path must end with .ts".to_string())
     })?)?);
@@ -588,11 +607,10 @@ struct DeploymentStatus {
     lock_error_logs: Option<String>,
 }
 async fn get_deployment_status(
-    authed: Authed,
-    Extension(user_db): Extension<UserDB>,
+    Extension(db): Extension<DB>,
     Path((w_id, hash)): Path<(String, ScriptHash)>,
 ) -> JsonResult<DeploymentStatus> {
-    let mut tx = user_db.begin(&authed).await?;
+    let mut tx = db.begin().await?;
     let status_o: Option<DeploymentStatus> = sqlx::query_as!(
         DeploymentStatus,
         "SELECT lock, lock_error_logs FROM script WHERE hash = $1 AND workspace_id = $2",
@@ -761,25 +779,31 @@ async fn delete_script_by_path(
     Ok(Json(script))
 }
 
-async fn parse_python_code_to_jsonschema(
-    Json(code): Json<String>,
-) -> JsonResult<windmill_parser::MainArgSignature> {
-    windmill_parser_py::parse_python_signature(&code).map(Json)
+#[derive(Debug, Serialize)]
+#[serde(tag = "type")]
+enum SigParsing {
+    Valid(MainArgSignature),
+    Invalid { error: String },
 }
 
-async fn parse_deno_code_to_jsonschema(
-    Json(code): Json<String>,
-) -> JsonResult<windmill_parser::MainArgSignature> {
-    windmill_parser_ts::parse_deno_signature(&code).map(Json)
-}
-async fn parse_go_code_to_jsonschema(
-    Json(code): Json<String>,
-) -> JsonResult<windmill_parser::MainArgSignature> {
-    windmill_parser_go::parse_go_sig(&code).map(Json)
+fn result_to_sig_parsing(result: Result<MainArgSignature>) -> Json<SigParsing> {
+    match result {
+        Ok(sig) => Json(SigParsing::Valid(sig)),
+        Err(e) => Json(SigParsing::Invalid { error: e.to_string() }),
+    }
 }
 
-async fn parse_bash_code_to_jsonschema(
-    Json(code): Json<String>,
-) -> JsonResult<windmill_parser::MainArgSignature> {
-    windmill_parser_bash::parse_bash_sig(&code).map(Json)
+async fn parse_python_code_to_jsonschema(Json(code): Json<String>) -> Json<SigParsing> {
+    result_to_sig_parsing(windmill_parser_py::parse_python_signature(&code))
+}
+
+async fn parse_deno_code_to_jsonschema(Json(code): Json<String>) -> Json<SigParsing> {
+    result_to_sig_parsing(windmill_parser_ts::parse_deno_signature(&code, false))
+}
+async fn parse_go_code_to_jsonschema(Json(code): Json<String>) -> Json<SigParsing> {
+    result_to_sig_parsing(windmill_parser_go::parse_go_sig(&code))
+}
+
+async fn parse_bash_code_to_jsonschema(Json(code): Json<String>) -> Json<SigParsing> {
+    result_to_sig_parsing(windmill_parser_bash::parse_bash_sig(&code))
 }

backend/windmill-api/src/users.rs

@@ -12,8 +12,9 @@ use crate::{
     db::{UserDB, DB},
     folders::get_folders_for_user,
     utils::require_super_admin,
+    webhook_util::{InstanceEvent, WebhookShared},
     workspaces::invite_user_to_all_auto_invite_worspaces,
-    COOKIE_DOMAIN, HTTP_CLIENT, IS_SECURE,
+    COOKIE_DOMAIN, IS_SECURE,
 };
 use argon2::{password_hash::SaltString, Argon2, PasswordHash, PasswordHasher, PasswordVerifier};
 use axum::{
@@ -76,6 +77,7 @@ pub fn global_service() -> Router {
         .route("/tokens/create", post(create_token))
         .route("/tokens/delete/:token_prefix", delete(delete_token))
         .route("/tokens/list", get(list_tokens))
+        .route("/tokens/impersonate", post(impersonate))
         .route("/usage", get(get_usage))
     // .route("/list_invite_codes", get(list_invite_codes))
     // .route("/create_invite_code", post(create_invite_code))
@@ -102,6 +104,10 @@ impl AuthCache {
         AuthCache { cache: Cache::new(), db, superadmin_secret }
     }
 
+    pub async fn invalidate(&self, w_id: &str, token: String) {
+        self.cache.remove(&(w_id.to_string(), token)).await;
+    }
+
     pub async fn get_authed(&self, w_id: Option<String>, token: &str) -> Option<Authed> {
         let key = (
             w_id.as_ref().unwrap_or(&"".to_string()).to_string(),
@@ -356,6 +362,31 @@ pub struct Authed {
     pub folders: Vec<(String, bool)>,
 }
 
+pub async fn maybe_refresh_folders(path: &str, w_id: &str, authed: Authed, db: &DB) -> Authed {
+    if authed.is_admin {
+        return authed;
+    }
+    let splitted = path.split('/').collect::<Vec<_>>();
+    if splitted.len() >= 2
+        && splitted[0] == "f"
+        && !authed.folders.iter().any(|(f, _)| f == splitted[1])
+    {
+        let name = &authed.username;
+        let groups = get_groups_for_user(w_id, name, db)
+            .await
+            .ok()
+            .unwrap_or_default();
+
+        let folders = get_folders_for_user(w_id, name, &groups, db)
+            .await
+            .ok()
+            .unwrap_or_default();
+        Authed { folders, ..authed }
+    } else {
+        authed
+    }
+}
+
 #[async_trait]
 impl<S> FromRequestParts<S> for Authed
 where
@@ -564,6 +595,7 @@ pub struct TruncatedToken {
 pub struct NewToken {
     pub label: Option<String>,
     pub expiration: Option<chrono::DateTime<chrono::Utc>>,
+    pub impersonate_email: Option<String>,
 }
 
 #[derive(Deserialize)]
@@ -1006,13 +1038,10 @@ lazy_static! {
 
 async fn accept_invite(
     Authed { email, .. }: Authed,
+    Extension(webhook): Extension<WebhookShared>,
     Extension(db): Extension<DB>,
     Json(nu): Json<AcceptInvite>,
 ) -> Result<(StatusCode, String)> {
-    if &nu.username == "bot" {
-        return Err(Error::BadRequest("bot is a reserved username".to_string()));
-    }
-
     if !VALID_USERNAME.is_match(&nu.username) {
         return Err(windmill_common::error::Error::BadRequest(format!(
             "Usermame can only contain alphanumeric characters and underscores"
@@ -1055,6 +1084,11 @@ async fn accept_invite(
     }
 
     if is_some {
+        webhook.send_instance_event(InstanceEvent::UserJoinedWorkspace {
+            email: email.clone(),
+            workspace: nu.workspace_id.clone(),
+            username: nu.username.clone(),
+        });
         Ok((
             StatusCode::CREATED,
             format!(
@@ -1295,12 +1329,14 @@ lazy_static::lazy_static! {
 async fn create_user(
     Authed { email, .. }: Authed,
     Extension(db): Extension<DB>,
+    Extension(webhook): Extension<WebhookShared>,
     Extension(argon2): Extension<Arc<Argon2<'_>>>,
-    Json(nu): Json<NewUser>,
+    Json(mut nu): Json<NewUser>,
 ) -> Result<(StatusCode, String)> {
     let mut tx = db.begin().await?;
 
     require_super_admin(&mut tx, &email).await?;
+    nu.email = nu.email.to_lowercase();
 
     if nu.email == SUPERADMIN_SECRET_EMAIL {
         return Err(Error::BadRequest(
@@ -1334,19 +1370,9 @@ async fn create_user(
     .await?;
     tx.commit().await?;
 
-    if let Some(new_user_webhook) = NEW_USER_WEBHOOK.clone() {
-        let _ = HTTP_CLIENT
-            .post(&new_user_webhook)
-            .json(
-                &serde_json::json!({"email" : &nu.email, "name": &nu.name, "event": "global_add"}),
-            )
-            .send()
-            .await
-            .map_err(|e| tracing::error!("Error sending new user webhook: {}", e.to_string()));
-    }
-
     invite_user_to_all_auto_invite_worspaces(&db, &nu.email).await?;
 
+    webhook.send_instance_event(InstanceEvent::UserAdded { email: nu.email.clone() });
     Ok((StatusCode::CREATED, format!("email {} created", nu.email)))
 }
 
@@ -1592,7 +1618,7 @@ async fn login(
     Json(Login { email, password }): Json<Login>,
 ) -> Result<String> {
     let mut tx = db.begin().await?;
-
+    let email = email.to_lowercase();
     let email_w_h: Option<(String, String, bool, bool)> = sqlx::query_as(
         "SELECT email, password_hash, super_admin, first_time_user FROM password WHERE email = $1 AND login_type = \
          'password'",
@@ -1714,6 +1740,58 @@ async fn create_token(
     Ok((StatusCode::CREATED, token))
 }
 
+async fn impersonate(
+    Extension(db): Extension<DB>,
+    Authed { email, username, .. }: Authed,
+    Json(new_token): Json<NewToken>,
+) -> Result<(StatusCode, String)> {
+    let token = rd_string(30);
+    let mut tx = db.begin().await?;
+    require_super_admin(&mut tx, &email).await?;
+
+    if new_token.impersonate_email.is_none() {
+        return Err(Error::BadRequest(
+            "impersonate_username is required".to_string(),
+        ));
+    }
+
+    let impersonated = new_token.impersonate_email.unwrap();
+
+    let is_super_admin = sqlx::query_scalar!(
+        "SELECT super_admin FROM password WHERE email = $1",
+        impersonated
+    )
+    .fetch_optional(&mut tx)
+    .await?
+    .unwrap_or(false);
+    sqlx::query!(
+        "INSERT INTO token
+            (token, email, label, expiration, super_admin)
+            VALUES ($1, $2, $3, $4, $5)",
+        token,
+        impersonated,
+        new_token.label,
+        new_token.expiration,
+        is_super_admin
+    )
+    .execute(&mut tx)
+    .await?;
+
+    audit_log(
+        &mut tx,
+        &username,
+        "users.impersonate",
+        ActionKind::Delete,
+        &"global",
+        Some(&token[0..10]),
+        Some([("impersonated", &format!("{impersonated}")[..])].into()),
+    )
+    .instrument(tracing::info_span!("token", email = &impersonated))
+    .await?;
+    tx.commit().await?;
+    Ok((StatusCode::CREATED, token))
+}
+
 async fn list_tokens(
     Extension(db): Extension<DB>,
     Authed { email, .. }: Authed,
@@ -1814,6 +1892,17 @@ pub async fn delete_expired_items_perdiodically(
             Err(e) => tracing::error!("Error deleting token: {}", e.to_string()),
         }
 
+        let pip_resolution_r = sqlx::query_scalar!(
+            "DELETE FROM pip_resolution_cache WHERE expiration <= now() RETURNING hash",
+        )
+        .fetch_all(db)
+        .await;
+
+        match pip_resolution_r {
+            Ok(res) => tracing::debug!("deleted {} pip_resolution: {:?}", res.len(), res),
+            Err(e) => tracing::error!("Error deleting pip_resolution: {}", e.to_string()),
+        }
+
         let magic_links_deleted_r: std::result::Result<Vec<String>, _> = sqlx::query_scalar(
             "DELETE FROM magic_link WHERE expiration <= now()
         RETURNING concat(substring(token for 10), '*****')",

backend/windmill-api/src/variables.rs

@@ -9,7 +9,7 @@
 use crate::{
     db::{UserDB, DB},
     oauth2::_refresh_token,
-    users::{require_owner_of_path, Authed},
+    users::{maybe_refresh_folders, require_owner_of_path, Authed},
     webhook_util::{WebhookMessage, WebhookShared},
 };
 /*
@@ -206,12 +206,15 @@ async fn check_path_conflict<'c>(
 
 async fn create_variable(
     authed: Authed,
+    Extension(db): Extension<DB>,
     Extension(user_db): Extension<UserDB>,
     Extension(webhook): Extension<WebhookShared>,
     Path(w_id): Path<String>,
     Query(AlreadyEncrypted { already_encrypted }): Query<AlreadyEncrypted>,
     Json(variable): Json<CreateVariable>,
 ) -> Result<(StatusCode, String)> {
+    let authed = maybe_refresh_folders(&variable.path, &w_id, authed, &db).await;
+
     let mut tx = user_db.begin(&authed).await?;
 
     check_path_conflict(&mut tx, &w_id, &variable.path).await?;
@@ -330,6 +333,7 @@ async fn update_variable(
     use sql_builder::prelude::*;
 
     let path = path.to_path();
+    let authed = maybe_refresh_folders(&path, &w_id, authed, &db).await;
 
     let mut tx = user_db.begin(&authed).await?;
 

backend/windmill-api/src/webhook_util.rs

@@ -2,6 +2,7 @@ use std::time::Duration;
 
 use serde::Serialize;
 use tokio::{select, sync::mpsc, time::interval};
+use windmill_common::METRICS_ENABLED;
 
 use crate::db::DB;
 
@@ -12,6 +13,26 @@ lazy_static::lazy_static! {
         "Histogram of webhook requests made"
     )
     .unwrap();
+
+    pub static ref INSTANCE_EVENTS_WEBHOOK: Option<String> = std::env::var("INSTANCE_EVENTS_WEBHOOK").ok();
+
+}
+
+pub enum WebhookPayload {
+    WorkspaceEvent(String, WebhookMessage),
+    InstanceEvent(InstanceEvent),
+}
+
+#[derive(Serialize)]
+#[serde(tag = "type")]
+pub enum InstanceEvent {
+    UserSignupOAuth { email: String },
+    UserAdded { email: String },
+    // UserDeleted { email: String },
+    // UserDeletedWorkspace { workspace: String, email: String },
+    UserAddedWorkspace { workspace: String, email: String },
+    UserInvitedWorkspace { workspace: String, email: String },
+    UserJoinedWorkspace { workspace: String, email: String, username: String },
 }
 
 #[derive(Serialize)]
@@ -45,12 +66,12 @@ pub enum WebhookMessage {
 
 #[derive(Clone)]
 pub struct WebhookShared {
-    pub channel: mpsc::UnboundedSender<(String, WebhookMessage)>,
+    pub channel: mpsc::UnboundedSender<WebhookPayload>,
 }
 
 impl WebhookShared {
     pub fn new(mut shutdown_rx: tokio::sync::broadcast::Receiver<()>, db: DB) -> Self {
-        let (tx, mut rx) = mpsc::unbounded_channel::<(String, WebhookMessage)>();
+        let (tx, mut rx) = mpsc::unbounded_channel::<WebhookPayload>();
         let _process = tokio::spawn(async move {
             let client = reqwest::Client::builder()
                 // TODO: investigate pool timeouts and such if TCP load is high
@@ -65,7 +86,7 @@ impl WebhookShared {
                     biased;
                     _ = shutdown_rx.recv() => break,
                     r = rx.recv() => match r {
-                        Some((workspace_id, message)) => {
+                        Some(WebhookPayload::WorkspaceEvent(workspace_id, message)) => {
                             let url_guard = match cache.get(&workspace_id).await {
                                 Some(guard) => {
                                     guard
@@ -89,12 +110,19 @@ impl WebhookShared {
                             };
                             let webook_opt = url_guard.value();
                             if let Some(url) = webook_opt {
-                                let timer = WEBHOOK_REQUEST_COUNT.start_timer();
+                                let timer = if *METRICS_ENABLED { Some(WEBHOOK_REQUEST_COUNT.start_timer()) } else { None };
                                 let _ = client.post(url).json(&message).send().await;
-                                timer.stop_and_record();
+                                timer.map(|x| x.stop_and_record());
                                 drop(url_guard);
                             }
                         },
+                        Some(WebhookPayload::InstanceEvent(event)) => {
+                            if *METRICS_ENABLED { Some(WEBHOOK_REQUEST_COUNT.start_timer()) } else { None };
+                            let r = client.post(INSTANCE_EVENTS_WEBHOOK.as_ref().unwrap()).json(&event).send().await;
+                            if let Err(e) = r {
+                                tracing::error!("Error sending instance event: {}", e);
+                            }
+                        },
                         None => break,
                     },
                     _ = futures::future::poll_fn(|cx| cache_purge_interval.poll_tick(cx)) => {
@@ -109,6 +137,16 @@ impl WebhookShared {
     }
 
     pub fn send_message(&self, workspace_id: String, message: WebhookMessage) {
-        let _ = self.channel.send((workspace_id.clone(), message));
+        let _ = self.channel.send(WebhookPayload::WorkspaceEvent(
+            workspace_id.clone(),
+            message,
+        ));
+    }
+
+    pub fn send_instance_event(&self, event: InstanceEvent) {
+        if INSTANCE_EVENTS_WEBHOOK.is_none() {
+            return;
+        }
+        let _ = self.channel.send(WebhookPayload::InstanceEvent(event));
     }
 }

backend/windmill-api/src/workspaces.rs

@@ -16,10 +16,10 @@ use crate::{
     db::{UserDB, DB},
     folders::Folder,
     resources::{Resource, ResourceType},
-    users::{Authed, WorkspaceInvite, NEW_USER_WEBHOOK, VALID_USERNAME},
+    users::{Authed, WorkspaceInvite, VALID_USERNAME},
     utils::require_super_admin,
     variables::build_crypt,
-    HTTP_CLIENT,
+    webhook_util::{InstanceEvent, WebhookShared},
 };
 #[cfg(feature = "enterprise")]
 use axum::response::Redirect;
@@ -943,11 +943,14 @@ pub async fn invite_user_to_all_auto_invite_worspaces(db: &DB, email: &str) -> R
 async fn invite_user(
     Authed { username, is_admin, .. }: Authed,
     Extension(db): Extension<DB>,
+    Extension(webhook): Extension<WebhookShared>,
     Path(w_id): Path<String>,
-    Json(nu): Json<NewWorkspaceInvite>,
+    Json(mut nu): Json<NewWorkspaceInvite>,
 ) -> Result<(StatusCode, String)> {
     require_admin(is_admin, &username)?;
 
+    nu.email = nu.email.to_lowercase();
+
     let mut tx = db.begin().await?;
 
     sqlx::query!(
@@ -964,14 +967,10 @@ async fn invite_user(
 
     tx.commit().await?;
 
-    if let Some(new_user_webhook) = NEW_USER_WEBHOOK.clone() {
-        let _ = &HTTP_CLIENT
-            .post(&new_user_webhook)
-            .json(&serde_json::json!({"email" : &nu.email, "event": "workspace_invite"}))
-            .send()
-            .await
-            .map_err(|e| tracing::error!("Error sending new user webhook: {}", e.to_string()));
-    }
+    webhook.send_instance_event(InstanceEvent::UserInvitedWorkspace {
+        email: nu.email.clone(),
+        workspace: w_id,
+    });
 
     Ok((
         StatusCode::CREATED,
@@ -982,10 +981,12 @@ async fn invite_user(
 async fn add_user(
     Authed { username, is_admin, .. }: Authed,
     Extension(db): Extension<DB>,
+    Extension(webhook): Extension<WebhookShared>,
     Path(w_id): Path<String>,
-    Json(nu): Json<NewWorkspaceUser>,
+    Json(mut nu): Json<NewWorkspaceUser>,
 ) -> Result<(StatusCode, String)> {
     require_admin(is_admin, &username)?;
+    nu.email = nu.email.to_lowercase();
 
     let mut tx = db.begin().await?;
     if !VALID_USERNAME.is_match(&nu.username) {
@@ -1019,14 +1020,10 @@ async fn add_user(
 
     tx.commit().await?;
 
-    if let Some(new_user_webhook) = NEW_USER_WEBHOOK.clone() {
-        let _ = HTTP_CLIENT
-            .post(&new_user_webhook)
-            .json(&serde_json::json!({"email" : &nu.email, "event": "workspace_add"}))
-            .send()
-            .await
-            .map_err(|e| tracing::error!("Error sending new user webhook: {}", e.to_string()));
-    }
+    webhook.send_instance_event(InstanceEvent::UserAddedWorkspace {
+        workspace: w_id.clone(),
+        email: nu.email.clone(),
+    });
 
     Ok((
         StatusCode::CREATED,

backend/windmill-audit/src/lib.rs

@@ -41,8 +41,8 @@ pub struct AuditLog {
 }
 
 #[tracing::instrument(level = "trace", skip_all)]
-pub async fn audit_log<'c>(
-    db: &mut Transaction<'c, Postgres>,
+pub async fn audit_log<'c, E: sqlx::Executor<'c, Database = Postgres>>(
+    db: E,
     username: &str,
     operation: &str,
     action_kind: ActionKind,

backend/windmill-common/src/lib.rs

@@ -6,7 +6,7 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
-use std::net::SocketAddr;
+use std::{net::SocketAddr, sync::Arc};
 
 use error::Error;
 
@@ -30,7 +30,19 @@ pub const DEFAULT_MAX_CONNECTIONS_SERVER: u32 = 50;
 pub const DEFAULT_MAX_CONNECTIONS_WORKER: u32 = 3;
 
 lazy_static::lazy_static! {
+    pub static ref METRICS_ADDR: Option<SocketAddr> = std::env::var("METRICS_ADDR")
+    .ok()
+    .map(|s| {
+        s.parse::<bool>()
+            .map(|b| b.then(|| SocketAddr::from(([0, 0, 0, 0], 8001))))
+            .or_else(|_| s.parse::<SocketAddr>().map(Some))
+    })
+    .transpose().ok()
+    .flatten()
+    .flatten();
+    pub static ref METRICS_ENABLED: bool = METRICS_ADDR.is_some();
     pub static ref BASE_URL: String = std::env::var("BASE_URL").unwrap_or_else(|_| "http://localhost".to_string());
+    pub static ref IS_READY: Arc<std::sync::atomic::AtomicBool> = Arc::new(std::sync::atomic::AtomicBool::new(false));
 }
 
 #[cfg(feature = "tokio")]
@@ -58,14 +70,31 @@ pub async fn shutdown_signal(tx: tokio::sync::broadcast::Sender<()>) -> anyhow::
 pub async fn serve_metrics(
     addr: SocketAddr,
     mut rx: tokio::sync::broadcast::Receiver<()>,
+    ready_worker_endpoint: bool,
 ) -> Result<(), hyper::Error> {
+    use std::sync::atomic::Ordering;
+
     use axum::{routing::get, Router};
-    axum::Server::bind(&addr)
-        .serve(
-            Router::new()
-                .route("/metrics", get(metrics))
-                .into_make_service(),
+    use hyper::StatusCode;
+    let router = Router::new().route("/metrics", get(metrics));
+
+    let router = if ready_worker_endpoint {
+        router.route(
+            "/ready",
+            get(|| async {
+                if IS_READY.load(Ordering::Relaxed) {
+                    (StatusCode::OK, "ready")
+                } else {
+                    (StatusCode::INTERNAL_SERVER_ERROR, "not ready")
+                }
+            }),
         )
+    } else {
+        router
+    };
+
+    axum::Server::bind(&addr)
+        .serve(router.into_make_service())
         .with_graceful_shutdown(async {
             rx.recv().await.ok();
             println!("Graceful shutdown of metrics");

backend/windmill-common/src/schedule.rs

@@ -17,7 +17,7 @@ pub struct Schedule {
     pub edited_by: String,
     pub edited_at: DateTime<chrono::Utc>,
     pub schedule: String,
-    pub offset_: i32,
+    pub timezone: String,
     pub enabled: bool,
     pub script_path: String,
     pub is_flow: bool,

backend/windmill-common/src/scripts.rs

@@ -17,7 +17,7 @@ use serde_json::to_string_pretty;
 
 use crate::utils::StripPath;
 
-#[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Hash)]
+#[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Hash, Eq)]
 #[cfg_attr(feature = "sqlx", derive(sqlx::Type))]
 #[cfg_attr(
     feature = "sqlx",

backend/windmill-common/src/utils.rs

@@ -6,10 +6,10 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
-use rand::{distributions::Alphanumeric, thread_rng, Rng};
-use serde::Deserialize;
-
 use crate::error::{Error, Result};
+use rand::{distributions::Alphanumeric, thread_rng, Rng};
+use serde::{Deserialize, Serialize};
+use sha2::{Digest, Sha256};
 
 pub const MAX_PER_PAGE: usize = 10000;
 pub const DEFAULT_PER_PAGE: usize = 1000;
@@ -19,7 +19,8 @@ pub struct Pagination {
     pub page: Option<usize>,
     pub per_page: Option<usize>,
 }
-#[derive(Deserialize)]
+
+#[derive(Debug, Serialize, Deserialize)]
 pub struct StripPath(pub String);
 
 impl StripPath {
@@ -51,8 +52,8 @@ pub fn paginate(pagination: Pagination) -> (usize, usize) {
 }
 
 #[cfg(feature = "sqlx")]
-pub async fn now_from_db<'c>(
-    db: &mut sqlx::Transaction<'c, sqlx::Postgres>,
+pub async fn now_from_db<'c, E: sqlx::PgExecutor<'c>>(
+    db: E,
 ) -> Result<chrono::DateTime<chrono::Utc>> {
     Ok(sqlx::query_scalar!("SELECT now()")
         .fetch_one(db)
@@ -118,3 +119,9 @@ pub fn rd_string(len: usize) -> String {
         .map(char::from)
         .collect()
 }
+
+pub fn calculate_hash(s: &str) -> String {
+    let mut hasher = Sha256::new();
+    hasher.update(s);
+    format!("{:x}", hasher.finalize())
+}

backend/windmill-queue/Cargo.toml

@@ -25,8 +25,12 @@ serde_json.workspace = true
 ulid.workspace = true
 uuid.workspace = true
 chrono.workspace = true
+chrono-tz.workspace = true
 hex.workspace = true
 reqwest.workspace = true
 lazy_static.workspace = true
 prometheus.workspace = true
 cron.workspace = true
+rsmq_async.workspace = true
+tokio.workspace = true
+futures-core.workspace = true

backend/windmill-queue/src/jobs.rs

@@ -22,8 +22,11 @@ use windmill_common::{
     flows::{FlowModule, FlowModuleValue, FlowValue},
     scripts::{get_full_hub_script_by_path, HubScript, ScriptHash, ScriptLang},
     utils::StripPath,
+    METRICS_ENABLED,
 };
 
+use crate::QueueTransaction;
+
 lazy_static::lazy_static! {
     pub static ref HTTP_CLIENT: Client = reqwest::ClientBuilder::new()
         .user_agent("windmill/beta")
@@ -51,13 +54,15 @@ lazy_static::lazy_static! {
 
 const MAX_FREE_EXECS: i32 = 1000;
 const MAX_FREE_CONCURRENT_RUNS: i32 = 15;
+const RSMQ_MAIN_QUEUE: &'static str = "main_queue";
 
-pub async fn cancel_job<'c>(
+pub async fn cancel_job<'c, R: rsmq_async::RsmqConnection + Clone>(
     username: &str,
     reason: Option<String>,
     id: Uuid,
     w_id: &str,
     mut tx: Transaction<'c, Postgres>,
+    rsmq: Option<R>,
     force_rerun: bool,
 ) -> error::Result<(Transaction<'c, Postgres>, Option<Uuid>)> {
     let job_option = sqlx::query_scalar!(
@@ -71,6 +76,12 @@ pub async fn cancel_job<'c>(
     )
     .fetch_optional(&mut tx)
     .await?;
+    if let Some(mut rsmq) = rsmq {
+        rsmq.change_message_visibility(RSMQ_MAIN_QUEUE, &id.to_string(), 0)
+            .await
+            .map_err(|e| anyhow::anyhow!(e))?;
+    }
+
     let mut jobs = job_option.map(|j| vec![j]).unwrap_or_default();
     while !jobs.is_empty() {
         let p_job = jobs.pop();
@@ -90,10 +101,11 @@ pub async fn cancel_job<'c>(
     Ok((tx, job_option))
 }
 
-pub async fn pull(
+pub async fn pull<R: rsmq_async::RsmqConnection + Clone>(
     db: &Pool<Postgres>,
     whitelist_workspaces: Option<Vec<String>>,
     blacklist_workspaces: Option<Vec<String>>,
+    rsmq: Option<R>,
 ) -> windmill_common::error::Result<Option<QueuedJob>> {
     let mut workspaces_filter = String::new();
     if let Some(whitelist) = whitelist_workspaces {
@@ -105,6 +117,9 @@ pub async fn pull(
                 .collect::<Vec<String>>()
                 .join(",")
         ));
+        if let Some(_rsmq) = rsmq {
+            todo!("REDIS: Implement workspace filters for redis");
+        }
     }
     if let Some(blacklist) = blacklist_workspaces {
         workspaces_filter.push_str(&format!(
@@ -115,16 +130,50 @@ pub async fn pull(
                 .collect::<Vec<String>>()
                 .join(",")
         ));
+        if let Some(_rsmq) = rsmq {
+            todo!("REDIS: Implement workspace filters for redis");
+        }
     }
-    /* Jobs can be started if they:
-     * - haven't been started before,
-     *   running = false
-     * - are flows with a step that needed resume,
-     *   suspend_until is non-null
-     *   and suspend = 0 when the resume messages are received
-     *   or suspend_until <= now() if it has timed out */
-    let job: Option<QueuedJob> = sqlx::query_as::<_, QueuedJob>(&format!(
-        "UPDATE queue
+
+    let job: Option<QueuedJob> = if let Some(mut rsmq) = rsmq {
+        // TODO: REDIS: Race conditions / replace last_ping
+        let msg = rsmq
+            .pop_message::<Vec<u8>>(RSMQ_MAIN_QUEUE)
+            .await
+            .map_err(|e| anyhow::anyhow!(e))?;
+
+        if let Some(msg) = msg {
+            let uuid = Uuid::from_bytes_le(
+                msg.message
+                    .try_into()
+                    .map_err(|_| anyhow::anyhow!("Failed to parsed Redis message"))?,
+            );
+
+            sqlx::query_as::<_, QueuedJob>(
+                "UPDATE queue
+            SET running = true
+            , started_at = coalesce(started_at, now())
+            , last_ping = now()
+            , suspend_until = null
+            WHERE id = $1
+            RETURNING *",
+            )
+            .bind(uuid)
+            .fetch_optional(db)
+            .await?
+        } else {
+            None
+        }
+    } else {
+        /* Jobs can be started if they:
+         * - haven't been started before,
+         *   running = false
+         * - are flows with a step that needed resume,
+         *   suspend_until is non-null
+         *   and suspend = 0 when the resume messages are received
+         *   or suspend_until <= now() if it has timed out */
+        sqlx::query_as::<_, QueuedJob>(&format!(
+            "UPDATE queue
             SET running = true
               , started_at = coalesce(started_at, now())
               , last_ping = now()
@@ -142,11 +191,12 @@ pub async fn pull(
                 LIMIT 1
             )
             RETURNING *"
-    ))
-    .fetch_optional(db)
-    .await?;
+        ))
+        .fetch_optional(db)
+        .await?
+    };
 
-    if job.is_some() {
+    if job.is_some() && *METRICS_ENABLED {
         QUEUE_PULL_COUNT.inc();
     }
 
@@ -206,24 +256,26 @@ pub async fn get_result_by_id(
 }
 
 #[instrument(level = "trace", skip_all)]
-pub async fn delete_job(
-    db: &Pool<Postgres>,
+pub async fn delete_job<'c, R: rsmq_async::RsmqConnection + Clone + Send>(
+    mut tx: QueueTransaction<'c, R>,
     w_id: &str,
     job_id: Uuid,
-) -> windmill_common::error::Result<()> {
-    QUEUE_DELETE_COUNT.inc();
+) -> windmill_common::error::Result<QueueTransaction<'c, R>> {
+    if *METRICS_ENABLED {
+        QUEUE_DELETE_COUNT.inc();
+    }
     let job_removed = sqlx::query_scalar!(
         "DELETE FROM queue WHERE workspace_id = $1 AND id = $2 RETURNING 1",
         w_id,
         job_id
     )
-    .fetch_one(db)
+    .fetch_one(&mut tx)
     .await
     .map_err(|e| Error::InternalErr(format!("Error during deletion of job {job_id}: {e}")))?
     .unwrap_or(0)
         == 1;
     tracing::debug!("Job {job_id} deleted: {job_removed}");
-    Ok(())
+    Ok(tx)
 }
 
 pub async fn get_queued_job<'c>(
@@ -242,9 +294,9 @@ pub async fn get_queued_job<'c>(
     Ok(r)
 }
 
-#[instrument(level = "trace", skip_all)]
-pub async fn push<'c>(
-    mut tx: Transaction<'c, Postgres>,
+// #[instrument(level = "trace", skip_all)]
+pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
+    mut tx: QueueTransaction<'c, R>,
     workspace_id: &str,
     job_payload: JobPayload,
     args: serde_json::Map<String, serde_json::Value>,
@@ -259,8 +311,7 @@ pub async fn push<'c>(
     mut same_worker: bool,
     pre_run_error: Option<&windmill_common::error::Error>,
     visible_to_owner: bool,
-) -> Result<(Uuid, Transaction<'c, Postgres>), Error> {
-    let scheduled_for = scheduled_for_o.unwrap_or_else(chrono::Utc::now);
+) -> Result<(Uuid, QueueTransaction<'c, R>), Error> {
     let args_json = serde_json::Value::Object(args);
     let job_id: Uuid = Ulid::new().into();
 
@@ -512,7 +563,7 @@ pub async fn push<'c>(
             (workspace_id, id, running, parent_job, created_by, permissioned_as, scheduled_for, 
                 script_hash, script_path, raw_code, raw_lock, args, job_kind, schedule_path, raw_flow, \
          flow_status, is_flow_step, language, started_at, same_worker, pre_run_error, email, visible_to_owner, root_job)
-            VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, CASE WHEN $3 THEN now() END, $19, $20, $21, $22, $23) \
+            VALUES ($1, $2, $3, $4, $5, $6, COALESCE($7, now()), $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, CASE WHEN $3 THEN now() END, $19, $20, $21, $22, $23) \
          RETURNING id",
         workspace_id,
         job_id,
@@ -520,7 +571,7 @@ pub async fn push<'c>(
         parent_job,
         user,
         permissioned_as,
-        scheduled_for,
+        scheduled_for_o,
         script_hash,
         script_path.clone(),
         raw_code,
@@ -542,7 +593,9 @@ pub async fn push<'c>(
     .await
     .map_err(|e| Error::InternalErr(format!("Could not insert into queue {job_id}: {e}")))?;
     // TODO: technically the job isn't queued yet, as the transaction can be rolled back. Should be solved when moving these metrics to the queue abstraction.
-    QUEUE_PUSH_COUNT.inc();
+    if *METRICS_ENABLED {
+        QUEUE_PUSH_COUNT.inc();
+    }
 
     {
         let uuid_string = job_id.to_string();
@@ -577,6 +630,10 @@ pub async fn push<'c>(
         .instrument(tracing::info_span!("job_run", email = &email))
         .await?;
     }
+    if let Some(ref mut rsmq) = tx.rsmq {
+        rsmq.send_message(job_id.to_bytes_le().to_vec(), scheduled_for_o);
+    }
+
     Ok((uuid, tx))
 }
 

backend/windmill-queue/src/lib.rs

@@ -7,6 +7,8 @@
  */
 
 mod jobs;
+mod queue_transaction;
 pub mod schedule;
 
 pub use jobs::*;
+pub use queue_transaction::*;

backend/windmill-queue/src/queue_transaction.rs

@@ -0,0 +1,161 @@
+use std::fmt::Debug;
+
+use futures_core::{future::BoxFuture, stream::BoxStream};
+use rsmq_async::{RedisBytes, RsmqConnection};
+use sqlx::{Postgres, Transaction};
+
+pub enum RedisOp {
+    SendMessage(RedisBytes, Option<chrono::DateTime<chrono::Utc>>),
+    DeleteMessage(String),
+}
+
+impl RedisOp {
+    pub async fn apply<R: RsmqConnection>(self, rsmq: &mut R) -> Result<(), rsmq_async::RsmqError> {
+        match self {
+            RedisOp::SendMessage(bytes, time) => {
+                rsmq.send_message(
+                    "main_queue",
+                    bytes,
+                    time.map(|t| (t - chrono::Utc::now()).num_seconds())
+                        .and_then(|e| e.try_into().ok()),
+                )
+                .await?;
+            }
+            RedisOp::DeleteMessage(id) => {
+                rsmq.delete_message("main_queue", &id).await?;
+            }
+        };
+
+        Ok(())
+    }
+}
+
+pub struct RedisTransaction<R: RsmqConnection> {
+    rsmq: R,
+    queued_ops: Vec<RedisOp>,
+}
+
+impl<R: RsmqConnection> From<R> for RedisTransaction<R> {
+    fn from(value: R) -> Self {
+        Self { rsmq: value, queued_ops: Vec::new() }
+    }
+}
+
+impl<R: RsmqConnection> RedisTransaction<R> {
+    pub async fn commit(self) -> Result<(), rsmq_async::RsmqError> {
+        let mut rsmq = self.rsmq;
+        for op in self.queued_ops {
+            op.apply(&mut rsmq).await?;
+        }
+        Ok(())
+    }
+
+    pub fn send_message<E: Into<RedisBytes>>(
+        &mut self,
+        bytes: E,
+        delay_until: Option<chrono::DateTime<chrono::Utc>>,
+    ) {
+        self.queued_ops
+            .push(RedisOp::SendMessage(bytes.into(), delay_until))
+    }
+
+    pub fn delete_message(&mut self, id: String) {
+        self.queued_ops.push(RedisOp::DeleteMessage(id))
+    }
+}
+
+pub struct QueueTransaction<'c, R: RsmqConnection> {
+    pub rsmq: Option<RedisTransaction<R>>,
+    transaction: Transaction<'c, Postgres>,
+}
+
+impl<'c, R: RsmqConnection> From<(Option<R>, Transaction<'c, Postgres>)>
+    for QueueTransaction<'c, R>
+{
+    fn from(value: (Option<R>, Transaction<'c, Postgres>)) -> Self {
+        Self { rsmq: value.0.map(|e| e.into()), transaction: value.1 }
+    }
+}
+
+impl<'c, R: RsmqConnection> Debug for QueueTransaction<'c, R> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("QueueTransaction")
+            .field("rsmq", &self.rsmq.as_ref().map(|_| ())) // do not require R: Debug
+            .field("transaction", &self.transaction)
+            .finish()
+    }
+}
+
+impl<'c, R: RsmqConnection> QueueTransaction<'c, R> {
+    pub async fn commit(self) -> Result<(), windmill_common::error::Error> {
+        self.transaction.commit().await?;
+        if let Some(rsmq) = self.rsmq {
+            rsmq.commit().await.map_err(|e| anyhow::anyhow!(e))?;
+        }
+
+        Ok(())
+    }
+
+    pub fn transaction_mut<'a>(&'a mut self) -> &'a mut Transaction<'c, Postgres> {
+        &mut self.transaction
+    }
+}
+
+impl<'c, 'b, R: RsmqConnection + Send> sqlx::Executor<'b> for &'b mut QueueTransaction<'c, R> {
+    type Database = Postgres;
+
+    fn fetch_many<'e, 'q: 'e, E: 'q>(
+        self,
+        query: E,
+    ) -> BoxStream<
+        'e,
+        Result<
+            sqlx::Either<
+                <Self::Database as sqlx::Database>::QueryResult,
+                <Self::Database as sqlx::Database>::Row,
+            >,
+            sqlx::Error,
+        >,
+    >
+    where
+        'b: 'e,
+        E: sqlx::Execute<'q, Self::Database>,
+    {
+        self.transaction.fetch_many(query)
+    }
+
+    fn fetch_optional<'e, 'q: 'e, E: 'q>(
+        self,
+        query: E,
+    ) -> BoxFuture<'e, Result<Option<<Self::Database as sqlx::Database>::Row>, sqlx::Error>>
+    where
+        'b: 'e,
+        E: sqlx::Execute<'q, Self::Database>,
+    {
+        self.transaction.fetch_optional(query)
+    }
+
+    fn prepare_with<'e, 'q: 'e>(
+        self,
+        sql: &'q str,
+        parameters: &'e [<Self::Database as sqlx::Database>::TypeInfo],
+    ) -> BoxFuture<
+        'e,
+        Result<<Self::Database as sqlx::database::HasStatement<'q>>::Statement, sqlx::Error>,
+    >
+    where
+        'b: 'e,
+    {
+        self.transaction.prepare_with(sql, parameters)
+    }
+
+    fn describe<'e, 'q: 'e>(
+        self,
+        sql: &'q str,
+    ) -> BoxFuture<'e, Result<sqlx::Describe<Self::Database>, sqlx::Error>>
+    where
+        'b: 'e,
+    {
+        self.transaction.describe(sql)
+    }
+}

backend/windmill-queue/src/schedule.rs

@@ -6,33 +6,39 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
-use std::str::FromStr;
-
-use chrono::Duration;
+use crate::{push, JobPayload};
 use sqlx::{query_scalar, Postgres, Transaction};
+use std::str::FromStr;
 use windmill_common::{
     error::{self, Result},
     schedule::Schedule,
     users::username_to_permissioned_as,
     utils::{now_from_db, StripPath},
 };
+use crate::{QueueTransaction};
 
-use crate::{push, JobPayload};
-
-pub async fn push_scheduled_job<'c>(
-    mut tx: Transaction<'c, Postgres>,
+pub async fn push_scheduled_job<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
+    mut tx: QueueTransaction<'c, R>,
     schedule: Schedule,
-) -> Result<Transaction<'c, Postgres>> {
+) -> Result<QueueTransaction<'c, R>> {
     let sched = cron::Schedule::from_str(&schedule.schedule)
         .map_err(|e| error::Error::BadRequest(e.to_string()))?;
 
-    let offset = Duration::minutes(schedule.offset_.into());
-    let now = now_from_db(&mut tx).await?;
+    let tz = chrono_tz::Tz::from_str(&schedule.timezone)
+        .map_err(|e| error::Error::BadRequest(e.to_string()))?;
+
+    let now = now_from_db(&mut tx).await?.with_timezone(&tz);
+
     let next = sched
-        .after(&(now - offset + Duration::seconds(1)))
+        .after(&now)
         .next()
-        .expect("a schedule should have a next event")
-        + offset;
+        .expect("a schedule should have a next event");
+
+    // println!("next event ({:?}): {}", tz, next);
+    // println!("next event(UTC): {}", next.with_timezone(&chrono::Utc));
+
+    // Scheduled events must be stored in the database in UTC
+    let next = next.with_timezone(&chrono::Utc);
 
     let already_exists: bool = query_scalar!(
         "SELECT EXISTS (SELECT 1 FROM queue WHERE workspace_id = $1 AND schedule_path = $2 AND scheduled_for = $3)",
@@ -65,7 +71,7 @@ pub async fn push_scheduled_job<'c>(
     } else {
         JobPayload::ScriptHash {
             hash: windmill_common::get_latest_hash_for_path(
-                &mut tx,
+                tx.transaction_mut(),
                 &schedule.workspace_id,
                 &schedule.script_path,
             )
@@ -74,7 +80,15 @@ pub async fn push_scheduled_job<'c>(
         }
     };
 
-    let (_, mut tx) = push(
+    sqlx::query!(
+        "UPDATE schedule SET error = NULL WHERE workspace_id = $1 AND path = $2",
+        &schedule.workspace_id,
+        &schedule.path
+    )
+    .execute(&mut tx)
+    .await?;
+
+    let (_, tx) = push(
         tx,
         &schedule.workspace_id,
         payload,
@@ -92,14 +106,7 @@ pub async fn push_scheduled_job<'c>(
         true,
     )
     .await?;
-    sqlx::query!(
-        "UPDATE schedule SET error = NULL WHERE workspace_id = $1 AND path = $2",
-        &schedule.workspace_id,
-        &schedule.path
-    )
-    .execute(&mut tx)
-    .await?;
-    Ok(tx)
+    Ok(tx) // TODO: Bubble up pushed UUID from here
 }
 
 pub async fn get_schedule_opt<'c>(

backend/windmill-worker/Cargo.toml

@@ -48,4 +48,5 @@ deno_core.workspace = true
 const_format.workspace = true
 git-version.workspace = true
 dyn-iter.workspace = true
-once_cell.workspace = true
+once_cell.workspace = true
+rsmq_async.workspace = true

backend/windmill-worker/nsjail/run.deno.config.proto

@@ -1,129 +0,0 @@
-name: "deno run script"
-
-mode: ONCE
-hostname: "deno"
-log_level: ERROR
-
-rlimit_as: 16000
-rlimit_cpu: 1000
-rlimit_fsize: 1000
-rlimit_nofile: 10000
-
-cwd: "/tmp"
-
-
-clone_newnet: false
-clone_newuser: {CLONE_NEWUSER}
-
-keep_caps: false
-keep_env: true
-mount_proc: true
-
-mount {
-    src: "/bin"
-    dst: "/bin"
-	is_bind: true
-}
-
-mount {
-    src: "/lib"
-    dst: "/lib"
-	is_bind: true
-}
-
-
-mount {
-    src: "/lib64"
-    dst: "/lib64"
-	is_bind: true
-}
-
-
-mount {
-    src: "/usr"
-    dst: "/usr"
-	is_bind: true
-}
-
-mount {
-	src: "/dev/null"
-	dst: "/dev/null"
-	is_bind: true
-	rw: true
-}
-
-mount {
-	dst: "/tmp"
-	fstype: "tmpfs"
-	rw: true
-    options: "size=500000000"
-}
-
-
-mount {
-    src: "{JOB_DIR}/wrapper.ts"
-    dst: "/tmp/wrapper.ts"
-    is_bind: true
-}
-
-mount {
-    src: "{JOB_DIR}/main.ts"
-    dst: "/tmp/main.ts"
-    is_bind: true
-}
-
-mount {
-    src: "{JOB_DIR}/import_map.json"
-    dst: "/tmp/import_map.json"
-    is_bind: true
-}
-
-mount {
-    src: "{JOB_DIR}/args.json"
-    dst: "/tmp/args.json"
-    is_bind: true
-}
-
-mount {
-    src: "{JOB_DIR}/result.json"
-    dst: "/tmp/result.json"
-    is_bind: true
-    rw: true
-}
-
-mount {
-    src: "/etc"
-    dst: "/etc"
-	is_bind: true
-}
-
-mount {
-    src: "/dev/random"
-    dst: "/dev/random"
-    is_bind: true
-}
-
-mount {
-    src: "/dev/urandom"
-    dst: "/dev/urandom"
-    is_bind: true
-}
-
-mount {
-    src: "{CACHE_DIR}"
-    dst: "/tmp/.cache/deno"
-    is_bind: true
-    rw: true
-    mandatory: false
-}
-
-{SHARED_MOUNT}
-
-iface_no_lo: true
-
-envar: "DENO_DIR=/tmp/.cache/deno"
-envar: "NO_COLOR=true"
-envar: "HOME=/tmp"
-
-
-

backend/windmill-worker/src/jobs.rs

@@ -6,23 +6,30 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
-use sqlx::{Pool, Postgres, Transaction};
+use sqlx::{Pool, Postgres};
 use tracing::instrument;
 use uuid::Uuid;
-use windmill_common::{error::Error, flow_status::FlowStatusModule, schedule::Schedule};
-use windmill_queue::{delete_job, schedule::get_schedule_opt, JobKind, QueuedJob, CLOUD_HOSTED};
+use windmill_common::{
+    error::Error, flow_status::FlowStatusModule, schedule::Schedule, METRICS_ENABLED,
+};
+use windmill_queue::{
+    delete_job, schedule::get_schedule_opt, JobKind, QueueTransaction, QueuedJob, CLOUD_HOSTED,
+};
 
 #[instrument(level = "trace", skip_all)]
-pub async fn add_completed_job_error(
+pub async fn add_completed_job_error<R: rsmq_async::RsmqConnection + Clone + Send>(
     db: &Pool<Postgres>,
     queued_job: &QueuedJob,
     logs: String,
     e: serde_json::Value,
     metrics: Option<crate::worker::Metrics>,
+    rsmq: Option<R>,
 ) -> Result<serde_json::Value, Error> {
-    metrics.map(|m| m.worker_execution_failed.inc());
+    if *METRICS_ENABLED {
+        metrics.map(|m| m.worker_execution_failed.inc());
+    }
     let result = serde_json::json!({ "error": e });
-    let _ = add_completed_job(db, &queued_job, false, false, result.clone(), logs).await?;
+    let _ = add_completed_job(db, &queued_job, false, false, result.clone(), logs, rsmq).await?;
     Ok(result)
 }
 
@@ -45,13 +52,14 @@ fn flatten_jobs(modules: Vec<FlowStatusModule>) -> Vec<Uuid> {
 }
 
 #[instrument(level = "trace", skip_all)]
-pub async fn add_completed_job(
+pub async fn add_completed_job<R: rsmq_async::RsmqConnection + Clone + Send>(
     db: &Pool<Postgres>,
     queued_job: &QueuedJob,
     success: bool,
     skipped: bool,
     result: serde_json::Value,
     logs: String,
+    rsmq: Option<R>,
 ) -> Result<Uuid, Error> {
     let duration =
         if queued_job.job_kind == JobKind::Flow || queued_job.job_kind == JobKind::FlowPreview {
@@ -83,7 +91,7 @@ pub async fn add_completed_job(
         .ok()
         .flatten()
         .flatten();
-    let mut tx = db.begin().await?;
+    let mut tx: QueueTransaction<'_, R> = (rsmq, db.begin().await?).into();
     let job_id = queued_job.id.clone();
     sqlx::query!(
         "INSERT INTO completed_job AS cj
@@ -153,7 +161,7 @@ pub async fn add_completed_job(
     .execute(&mut tx)
     .await
     .map_err(|e| Error::InternalErr(format!("Could not add completed job {job_id}: {e}")))?;
-    let _ = delete_job(db, &queued_job.workspace_id, job_id).await?;
+    tx = delete_job(tx, &queued_job.workspace_id, job_id).await?;
     if !queued_job.is_flow_step
         && queued_job.job_kind != JobKind::Flow
         && queued_job.job_kind != JobKind::FlowPreview
@@ -208,21 +216,24 @@ pub async fn add_completed_job(
 }
 
 #[instrument(level = "trace", skip_all)]
-pub async fn schedule_again_if_scheduled<'c>(
-    mut tx: Transaction<'c, Postgres>,
+pub async fn schedule_again_if_scheduled<'c, R: rsmq_async::RsmqConnection + Clone + Send + 'c>(
+    mut tx: QueueTransaction<'c, R>,
     db: &Pool<Postgres>,
     schedule_path: &str,
     script_path: &str,
     w_id: &str,
-) -> windmill_common::error::Result<Transaction<'c, Postgres>> {
-    let schedule = get_schedule_opt(&mut tx, w_id, schedule_path)
-        .await?
-        .ok_or_else(|| {
-            Error::InternalErr(format!(
-                "Could not find schedule {:?} for workspace {}",
-                schedule_path, w_id
-            ))
-        })?;
+) -> windmill_common::error::Result<QueueTransaction<'c, R>> {
+    let schedule = get_schedule_opt(tx.transaction_mut(), w_id, schedule_path).await?;
+
+    if schedule.is_none() {
+        tracing::error!(
+            "Schedule {schedule_path} in {w_id} not found. Impossible to schedule again"
+        );
+        return Ok(tx);
+    }
+
+    let schedule = schedule.unwrap();
+
     if schedule.enabled && script_path == schedule.script_path {
         let res = windmill_queue::schedule::push_scheduled_job(
             tx,
@@ -232,7 +243,7 @@ pub async fn schedule_again_if_scheduled<'c>(
                 edited_by: schedule.edited_by,
                 edited_at: schedule.edited_at,
                 schedule: schedule.schedule,
-                offset_: schedule.offset_,
+                timezone: schedule.timezone,
                 enabled: schedule.enabled,
                 script_path: schedule.script_path,
                 is_flow: schedule.is_flow,

backend/windmill-worker/src/worker_flow.rs

@@ -31,12 +31,13 @@ use windmill_common::{
 type DB = sqlx::Pool<sqlx::Postgres>;
 
 use windmill_queue::{
-    canceled_job_to_result, get_queued_job, push, JobPayload, QueuedJob, RawCode,
+    canceled_job_to_result, get_queued_job, push, JobPayload, QueueTransaction, QueuedJob, RawCode,
 };
 
-#[async_recursion]
 // #[instrument(level = "trace", skip_all)]
-pub async fn update_flow_status_after_job_completion(
+pub async fn update_flow_status_after_job_completion<
+    R: rsmq_async::RsmqConnection + Send + Sync + Clone,
+>(
     db: &DB,
     client: &AuthedClient,
     flow: uuid::Uuid,
@@ -50,58 +51,132 @@ pub async fn update_flow_status_after_job_completion(
     worker_dir: &str,
     stop_early_override: Option<bool>,
     base_internal_url: &str,
+    rsmq: Option<R>,
 ) -> error::Result<()> {
-    tracing::debug!("UPDATE FLOW STATUS: {flow:?} {success} {result:?} {w_id}");
-
-    let mut tx = db.begin().await?;
-
-    let old_status_json = sqlx::query_scalar!(
-        "SELECT flow_status FROM queue WHERE id = $1 AND workspace_id = $2",
+    // this is manual tailrecursion because async_recursion blows up the stack
+    let mut depth = 0;
+    let mut rec = update_flow_status_after_job_completion_internal(
+        db,
+        client,
         flow,
-        w_id
+        job_id_for_status,
+        w_id,
+        success,
+        result,
+        metrics.clone(),
+        unrecoverable,
+        same_worker_tx.clone(),
+        worker_dir,
+        stop_early_override,
+        base_internal_url,
+        depth,
+        rsmq.clone(),
     )
-    .fetch_one(&mut tx)
-    .await
-    .map_err(|e| {
-        Error::InternalErr(format!(
-            "fetching flow status {flow} while reporting {success} {result:?}: {e}"
-        ))
-    })?
-    .ok_or_else(|| Error::InternalErr(format!("requiring a previous status")))?;
+    .await?;
+    while let Some(nrec) = rec {
+        depth += 1;
+        rec = update_flow_status_after_job_completion_internal(
+            db,
+            client,
+            nrec.flow,
+            &nrec.job_id_for_status,
+            w_id,
+            nrec.success,
+            nrec.result,
+            metrics.clone(),
+            false,
+            same_worker_tx.clone(),
+            worker_dir,
+            nrec.stop_early_override,
+            base_internal_url,
+            depth,
+            rsmq.clone(),
+        )
+        .await?;
+    }
+    Ok(())
+}
+pub struct RecUpdateFlowStatusAfterJobCompletion {
+    flow: uuid::Uuid,
+    job_id_for_status: Uuid,
+    success: bool,
+    result: serde_json::Value,
+    stop_early_override: Option<bool>,
+}
+// #[instrument(level = "trace", skip_all)]
+pub async fn update_flow_status_after_job_completion_internal<
+    R: rsmq_async::RsmqConnection + Send + Sync + Clone,
+>(
+    db: &DB,
+    client: &AuthedClient,
+    flow: uuid::Uuid,
+    job_id_for_status: &Uuid,
+    w_id: &str,
+    success: bool,
+    result: serde_json::Value,
+    metrics: Option<worker::Metrics>,
+    unrecoverable: bool,
+    same_worker_tx: Sender<Uuid>,
+    worker_dir: &str,
+    stop_early_override: Option<bool>,
+    base_internal_url: &str,
+    depth: u8,
+    rsmq: Option<R>,
+) -> error::Result<Option<RecUpdateFlowStatusAfterJobCompletion>> {
+    let (should_continue_flow, flow_job, stop_early, skip_if_stop_early, nresult) = {
+        tracing::debug!("UPDATE FLOW STATUS: {flow:?} {success} {result:?} {w_id} {depth}");
 
-    let old_status = serde_json::from_value::<FlowStatus>(old_status_json)
-        .ok()
-        .ok_or_else(|| {
-            Error::InternalErr(format!("requiring status to be parsabled as FlowStatus"))
-        })?;
+        let mut tx: QueueTransaction<'_, _> = (rsmq.clone(), db.begin().await?).into();
 
-    let module_index = usize::try_from(old_status.step).ok();
+        let old_status_json = sqlx::query_scalar!(
+            "SELECT flow_status FROM queue WHERE id = $1 AND workspace_id = $2",
+            flow,
+            w_id
+        )
+        .fetch_one(&mut tx)
+        .await
+        .map_err(|e| {
+            Error::InternalErr(format!(
+                "fetching flow status {flow} while reporting {success} {result:?}: {e}"
+            ))
+        })?
+        .ok_or_else(|| Error::InternalErr(format!("requiring a previous status")))?;
 
-    let module_status = module_index
-        .and_then(|i| old_status.modules.get(i))
-        .unwrap_or(&old_status.failure_module.module_status);
+        let old_status = serde_json::from_value::<FlowStatus>(old_status_json)
+            .ok()
+            .ok_or_else(|| {
+                Error::InternalErr(format!("requiring status to be parsabled as FlowStatus"))
+            })?;
 
-    tracing::debug!("UPDATE FLOW STATUS 2: {module_index:#?} {module_status:#?} {old_status:#?} ");
+        let module_index = usize::try_from(old_status.step).ok();
 
-    let skip_loop_failures = if matches!(
-        module_status,
-        FlowStatusModule::InProgress { iterator: Some(_), .. }
-    ) {
-        compute_skip_loop_failures(flow, old_status.step, &mut tx)
-            .await?
-            .unwrap_or(false)
-    } else {
-        false
-    };
+        let module_status = module_index
+            .and_then(|i| old_status.modules.get(i))
+            .unwrap_or(&old_status.failure_module.module_status);
 
-    let is_failure_step = old_status.step >= old_status.modules.len() as i32;
+        tracing::debug!(
+            "UPDATE FLOW STATUS 2: {module_index:#?} {module_status:#?} {old_status:#?} "
+        );
 
-    let (mut stop_early, skip_if_stop_early) = if let Some(se) = stop_early_override {
-        (true, se)
-    } else if is_failure_step {
-        (false, false)
-    } else {
-        let r = sqlx::query!(
+        let skip_loop_failures = if matches!(
+            module_status,
+            FlowStatusModule::InProgress { iterator: Some(_), .. }
+        ) {
+            compute_skip_loop_failures(flow, old_status.step, tx.transaction_mut())
+                .await?
+                .unwrap_or(false)
+        } else {
+            false
+        };
+
+        let is_failure_step = old_status.step >= old_status.modules.len() as i32;
+
+        let (mut stop_early, skip_if_stop_early) = if let Some(se) = stop_early_override {
+            (true, se)
+        } else if is_failure_step {
+            (false, false)
+        } else {
+            let r = sqlx::query!(
             "
             SELECT raw_flow->'modules'->$1::int->'stop_after_if'->>'expr' as stop_early_expr,
             (raw_flow->'modules'->$1::int->'stop_after_if'->>'skip_if_stopped')::bool as skip_if_stopped,
@@ -116,37 +191,41 @@ pub async fn update_flow_status_after_job_completion(
         .await
         .map_err(|e| Error::InternalErr(format!("retrieval of stop_early_expr from state: {e}")))?;
 
-        let stop_early = success
-            && if let Some(expr) = r.stop_early_expr.clone() {
-                compute_bool_from_expr(expr, &r.args, result.clone(), None, Some(client)).await?
-            } else {
-                false
-            };
-        (stop_early, r.skip_if_stopped.unwrap_or(false))
-    };
+            let stop_early = success
+                && if let Some(expr) = r.stop_early_expr.clone() {
+                    compute_bool_from_expr(expr, &r.args, result.clone(), None, Some(client))
+                        .await?
+                } else {
+                    false
+                };
+            (stop_early, r.skip_if_stopped.unwrap_or(false))
+        };
 
-    let skip_branch_failure = match module_status {
-        FlowStatusModule::InProgress {
-            branchall: Some(BranchAllStatus { branch, .. }), ..
-        } => compute_skip_branchall_failure(flow, old_status.step, *branch, &mut tx)
-            .await?
-            .unwrap_or(false),
-        _ => false,
-    };
+        let skip_branch_failure = match module_status {
+            FlowStatusModule::InProgress {
+                branchall: Some(BranchAllStatus { branch, .. }),
+                ..
+            } => {
+                compute_skip_branchall_failure(flow, old_status.step, *branch, tx.transaction_mut())
+                    .await?
+                    .unwrap_or(false)
+            }
+            _ => false,
+        };
 
-    let skip_failure = skip_branch_failure || skip_loop_failures;
+        let skip_failure = skip_branch_failure || skip_loop_failures;
 
-    let (inc_step_counter, new_status) = match module_status {
-        FlowStatusModule::InProgress {
-            iterator,
-            branchall,
-            parallel,
-            flow_jobs: Some(jobs),
-            ..
-        } if *parallel => {
-            let (nindex, len) = match (iterator, branchall) {
-                (Some(Iterator { itered, .. }), _) => {
-                    let nindex = sqlx::query_scalar!(
+        let (inc_step_counter, new_status) = match module_status {
+            FlowStatusModule::InProgress {
+                iterator,
+                branchall,
+                parallel,
+                flow_jobs: Some(jobs),
+                ..
+            } if *parallel => {
+                let (nindex, len) = match (iterator, branchall) {
+                    (Some(Iterator { itered, .. }), _) => {
+                        let nindex = sqlx::query_scalar!(
                 "
             UPDATE queue
                SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'iterator', 'index'], ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb)
@@ -159,10 +238,10 @@ pub async fn update_flow_status_after_job_completion(
             .fetch_one(&mut tx)
             .await?
             .ok_or_else(|| Error::InternalErr(format!("requiring an index in InProgress")))?;
-                    (nindex, itered.len() as i32)
-                }
-                (_, Some(BranchAllStatus { len, .. })) => {
-                    let nindex = sqlx::query_scalar!(
+                        (nindex, itered.len() as i32)
+                    }
+                    (_, Some(BranchAllStatus { len, .. })) => {
+                        let nindex = sqlx::query_scalar!(
                 "
             UPDATE queue
                SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'branchall', 'branch'], ((flow_status->'modules'->$1::int->'branchall'->>'branch')::int + 1)::text::jsonb)
@@ -175,269 +254,294 @@ pub async fn update_flow_status_after_job_completion(
             .fetch_one(&mut tx)
             .await?
             .ok_or_else(|| Error::InternalErr(format!("requiring an index in InProgress")))?;
-                    (nindex, *len as i32)
-                }
-                _ => Err(Error::InternalErr(format!(
-                    "unexpected status for parallel module"
-                )))?,
-            };
-            if nindex == len {
-                let new_status = if skip_loop_failures
-                    || sqlx::query_scalar!(
-                        "
+                        (nindex, *len as i32)
+                    }
+                    _ => Err(Error::InternalErr(format!(
+                        "unexpected status for parallel module"
+                    )))?,
+                };
+                if nindex == len {
+                    let new_status = if skip_loop_failures
+                        || sqlx::query_scalar!(
+                            "
                       SELECT success
                         FROM completed_job
                        WHERE id = ANY($1)
                         ",
-                        jobs.as_slice(),
-                    )
-                    .fetch_all(&mut tx)
-                    .await?
-                    .into_iter()
-                    .all(|x| x)
-                {
-                    FlowStatusModule::Success {
-                        id: module_status.id(),
-                        job: job_id_for_status.clone(),
-                        flow_jobs: Some(jobs.clone()),
-                        branch_chosen: None,
-                        approvers: vec![],
-                    }
+                            jobs.as_slice(),
+                        )
+                        .fetch_all(&mut tx)
+                        .await?
+                        .into_iter()
+                        .all(|x| x)
+                    {
+                        FlowStatusModule::Success {
+                            id: module_status.id(),
+                            job: job_id_for_status.clone(),
+                            flow_jobs: Some(jobs.clone()),
+                            branch_chosen: None,
+                            approvers: vec![],
+                        }
+                    } else {
+                        FlowStatusModule::Failure {
+                            id: module_status.id(),
+                            job: job_id_for_status.clone(),
+                            flow_jobs: Some(jobs.clone()),
+                            branch_chosen: None,
+                        }
+                    };
+                    (true, Some(new_status))
                 } else {
-                    FlowStatusModule::Failure {
-                        id: module_status.id(),
-                        job: job_id_for_status.clone(),
-                        flow_jobs: Some(jobs.clone()),
-                        branch_chosen: None,
-                    }
-                };
-                (true, Some(new_status))
-            } else {
-                tx.commit().await?;
-                return Ok(());
-            }
-        }
-        FlowStatusModule::InProgress {
-            iterator: Some(windmill_common::flow_status::Iterator { index, itered, .. }),
-            ..
-        } if (*index + 1 < itered.len() && (success || skip_loop_failures)) && !stop_early => {
-            (false, None)
-        }
-        FlowStatusModule::InProgress {
-            branchall: Some(BranchAllStatus { branch, len, .. }),
-            ..
-        } if branch.to_owned() < len - 1 && (success || skip_branch_failure) => (false, None),
-        _ => {
-            if stop_early
-                && matches!(
-                    module_status,
-                    FlowStatusModule::InProgress { iterator: Some(_), .. }
-                )
-            {
-                // if we're stopping early inside a loop, we just want to break the loop instead
-                stop_early = false;
-            }
-            let (flow_jobs, branch_chosen) = match module_status {
-                FlowStatusModule::InProgress { flow_jobs, branch_chosen, .. } => {
-                    (flow_jobs.clone(), branch_chosen.clone())
+                    tx.commit().await?;
+                    return Ok(None);
                 }
-                _ => (None, None),
-            };
-            if success || (flow_jobs.is_some() && (skip_loop_failures || skip_branch_failure)) {
-                (
-                    true,
-                    Some(FlowStatusModule::Success {
-                        id: module_status.id(),
-                        job: job_id_for_status.clone(),
-                        flow_jobs,
-                        branch_chosen,
-                        approvers: vec![],
-                    }),
-                )
-            } else {
-                (
-                    false,
-                    Some(FlowStatusModule::Failure {
-                        id: module_status.id(),
-                        job: job_id_for_status.clone(),
-                        flow_jobs,
-                        branch_chosen,
-                    }),
-                )
             }
-        }
-    };
+            FlowStatusModule::InProgress {
+                iterator: Some(windmill_common::flow_status::Iterator { index, itered, .. }),
+                ..
+            } if (*index + 1 < itered.len() && (success || skip_loop_failures)) && !stop_early => {
+                (false, None)
+            }
+            FlowStatusModule::InProgress {
+                branchall: Some(BranchAllStatus { branch, len, .. }),
+                ..
+            } if branch.to_owned() < len - 1 && (success || skip_branch_failure) => (false, None),
+            _ => {
+                if stop_early
+                    && matches!(
+                        module_status,
+                        FlowStatusModule::InProgress { iterator: Some(_), .. }
+                    )
+                {
+                    // if we're stopping early inside a loop, we just want to break the loop instead
+                    stop_early = false;
+                }
+                let (flow_jobs, branch_chosen) = match module_status {
+                    FlowStatusModule::InProgress { flow_jobs, branch_chosen, .. } => {
+                        (flow_jobs.clone(), branch_chosen.clone())
+                    }
+                    _ => (None, None),
+                };
+                if success || (flow_jobs.is_some() && (skip_loop_failures || skip_branch_failure)) {
+                    (
+                        true,
+                        Some(FlowStatusModule::Success {
+                            id: module_status.id(),
+                            job: job_id_for_status.clone(),
+                            flow_jobs,
+                            branch_chosen,
+                            approvers: vec![],
+                        }),
+                    )
+                } else {
+                    (
+                        false,
+                        Some(FlowStatusModule::Failure {
+                            id: module_status.id(),
+                            job: job_id_for_status.clone(),
+                            flow_jobs,
+                            branch_chosen,
+                        }),
+                    )
+                }
+            }
+        };
 
-    let step_counter = if inc_step_counter {
-        sqlx::query!(
-            "
+        let step_counter = if inc_step_counter {
+            sqlx::query!(
+                "
             UPDATE queue
                SET flow_status = JSONB_SET(flow_status, ARRAY['step'], $1)
              WHERE id = $2
             ",
-            json!(old_status.step + 1),
-            flow
-        )
-        .execute(&mut tx)
-        .await?;
-        old_status.step + 1
-    } else {
-        old_status.step
-    };
+                json!(old_status.step + 1),
+                flow
+            )
+            .execute(&mut tx)
+            .await?;
+            old_status.step + 1
+        } else {
+            old_status.step
+        };
 
-    /* is_last_step is true when the step_counter (the next step index) is an invalid index */
-    let is_last_step = usize::try_from(step_counter)
-        .map(|i| !(..old_status.modules.len()).contains(&i))
-        .unwrap_or(true);
+        /* is_last_step is true when the step_counter (the next step index) is an invalid index */
+        let is_last_step = usize::try_from(step_counter)
+            .map(|i| !(..old_status.modules.len()).contains(&i))
+            .unwrap_or(true);
 
-    if let Some(new_status) = new_status.as_ref() {
-        if is_failure_step {
-            let parent_module = sqlx::query_scalar!(
+        if let Some(new_status) = new_status.as_ref() {
+            if is_failure_step {
+                let parent_module = sqlx::query_scalar!(
                 "SELECT flow_status->'failure_module'->>'parent_module' FROM queue WHERE id = $1",
                 flow
             )
-            .fetch_one(&mut tx)
-            .await?;
+                .fetch_one(&mut tx)
+                .await?;
 
-            sqlx::query!(
-                "
+                sqlx::query!(
+                    "
             UPDATE queue
                SET flow_status = JSONB_SET(flow_status, ARRAY['failure_module'], $1)
              WHERE id = $2
             ",
-                json!(FlowStatusModuleWParent { parent_module, module_status: new_status.clone() }),
-                flow
-            )
-            .execute(&mut tx)
-            .await?;
-        } else {
-            sqlx::query!(
-                "
-            UPDATE queue
-               SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT], $2)
-             WHERE id = $3
-            ",
-                old_status.step.to_string(),
-                json!(new_status),
-                flow
-            )
-            .execute(&mut tx)
-            .await?;
-
-            if let Some(job_result) = new_status.job_result() {
-                sqlx::query!(
-                    "
-            UPDATE queue
-               SET leaf_jobs = JSONB_SET(coalesce(leaf_jobs, '{}'::jsonb), ARRAY[$1::TEXT], $2)
-             WHERE COALESCE((SELECT root_job FROM queue WHERE id = $3), $3) = id
-            ",
-                    new_status.id(),
-                    json!(job_result),
+                    json!(FlowStatusModuleWParent {
+                        parent_module,
+                        module_status: new_status.clone()
+                    }),
                     flow
                 )
                 .execute(&mut tx)
                 .await?;
+            } else {
+                sqlx::query!(
+                    "
+            UPDATE queue
+               SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT], $2)
+             WHERE id = $3
+            ",
+                    old_status.step.to_string(),
+                    json!(new_status),
+                    flow
+                )
+                .execute(&mut tx)
+                .await?;
+
+                if let Some(job_result) = new_status.job_result() {
+                    sqlx::query!(
+                        "
+            UPDATE queue
+               SET leaf_jobs = JSONB_SET(coalesce(leaf_jobs, '{}'::jsonb), ARRAY[$1::TEXT], $2)
+             WHERE COALESCE((SELECT root_job FROM queue WHERE id = $3), $3) = id
+            ",
+                        new_status.id(),
+                        json!(job_result),
+                        flow
+                    )
+                    .execute(&mut tx)
+                    .await?;
+                }
             }
         }
-    }
 
-    let result = match &new_status {
-        Some(FlowStatusModule::Success { flow_jobs: Some(jobs), .. })
-        | Some(FlowStatusModule::Failure { flow_jobs: Some(jobs), .. }) => {
-            let results = sqlx::query!(
-                "
+        let nresult = match &new_status {
+            Some(FlowStatusModule::Success { flow_jobs: Some(jobs), .. })
+            | Some(FlowStatusModule::Failure { flow_jobs: Some(jobs), .. }) => {
+                let results = sqlx::query!(
+                    "
                   SELECT result, id
                     FROM completed_job
                    WHERE id = ANY($1)
                      AND workspace_id = $2
                     ",
-                jobs.as_slice(),
-                w_id
-            )
-            .fetch_all(&mut tx)
-            .await?
-            .into_iter()
-            .map(|r| (r.id, r.result))
-            .collect::<HashMap<_, _>>();
+                    jobs.as_slice(),
+                    w_id
+                )
+                .fetch_all(&mut tx)
+                .await?
+                .into_iter()
+                .map(|r| (r.id, r.result))
+                .collect::<HashMap<_, _>>();
 
-            let results = jobs
-                .iter()
-                .map(|j| {
-                    results
-                        .get(j)
-                        .ok_or_else(|| Error::InternalErr(format!("missing job result for {}", j)))
-                })
-                .collect::<Result<Vec<_>, _>>()?;
+                let results = jobs
+                    .iter()
+                    .map(|j| {
+                        results.get(j).ok_or_else(|| {
+                            Error::InternalErr(format!("missing job result for {}", j))
+                        })
+                    })
+                    .collect::<Result<Vec<_>, _>>()?;
 
-            json!(results)
-        }
-        _ => result,
-    };
+                json!(results)
+            }
+            _ => result,
+        };
 
-    if matches!(&new_status, Some(FlowStatusModule::Success { .. })) {
-        sqlx::query(
-            "
+        if matches!(&new_status, Some(FlowStatusModule::Success { .. })) {
+            sqlx::query(
+                "
             UPDATE queue
                SET flow_status = flow_status - 'retry'
              WHERE id = $1
              RETURNING flow_status
             ",
-        )
-        .bind(flow)
-        .execute(&mut tx)
-        .await
-        .context("remove flow status retry")?;
-    }
-
-    let flow_job = get_queued_job(flow, w_id, &mut tx)
-        .await?
-        .ok_or_else(|| Error::InternalErr(format!("requiring flow to be in the queue")))?;
-
-    let job_root = flow_job
-        .root_job
-        .map(|x| x.to_string())
-        .unwrap_or_else(|| "none".to_string());
-    tracing::info!(id = %flow_job.id, root_id = %job_root, "update flow status");
-
-    let raw_flow = flow_job.parse_raw_flow();
-    let module = raw_flow.as_ref().and_then(|module| {
-        module_index.and_then(|i| module.modules.get(i).or(module.failure_module.as_ref()))
-    });
-
-    let should_continue_flow = match success {
-        _ if stop_early => false,
-        _ if flow_job.canceled => false,
-        true => !is_last_step,
-        false if unrecoverable => false,
-        false if skip_failure => !is_last_step,
-        false
-            if next_retry(
-                &module.and_then(|m| m.retry.clone()).unwrap_or_default(),
-                &old_status.retry,
             )
-            .is_some() =>
-        {
-            true
+            .bind(flow)
+            .execute(&mut tx)
+            .await
+            .context("remove flow status retry")?;
         }
-        false if has_failure_module(flow, &mut tx).await? && !is_failure_step => true,
-        false => false,
-    };
 
-    if old_status.step == 0
-        && !flow_job.is_flow_step
-        && flow_job.schedule_path.is_some()
-        && flow_job.script_path.is_some()
-    {
-        tx = schedule_again_if_scheduled(
-            tx,
-            db,
-            flow_job.schedule_path.as_ref().unwrap(),
-            flow_job.script_path.as_ref().unwrap(),
-            &w_id,
+        let flow_job = get_queued_job(flow, w_id, tx.transaction_mut())
+            .await?
+            .ok_or_else(|| Error::InternalErr(format!("requiring flow to be in the queue")))?;
+
+        let job_root = flow_job
+            .root_job
+            .map(|x| x.to_string())
+            .unwrap_or_else(|| "none".to_string());
+        tracing::info!(id = %flow_job.id, root_id = %job_root, "update flow status");
+
+        let module = {
+            let raw_flow = flow_job.parse_raw_flow();
+            if let Some(raw_flow) = raw_flow {
+                if let Some(i) = module_index {
+                    if let Some(module) = raw_flow.modules.get(i) {
+                        Some(module.clone())
+                    } else {
+                        raw_flow.failure_module
+                    }
+                } else {
+                    None
+                }
+            } else {
+                None
+            }
+        };
+
+        let should_continue_flow = match success {
+            _ if stop_early => false,
+            _ if flow_job.canceled => false,
+            true => !is_last_step,
+            false if unrecoverable => false,
+            false if skip_failure => !is_last_step,
+            false
+                if next_retry(
+                    &module.and_then(|m| m.retry.clone()).unwrap_or_default(),
+                    &old_status.retry,
+                )
+                .is_some() =>
+            {
+                true
+            }
+            false if has_failure_module(flow, tx.transaction_mut()).await? && !is_failure_step => {
+                true
+            }
+            false => false,
+        };
+
+        if old_status.step == 0
+            && !flow_job.is_flow_step
+            && flow_job.schedule_path.is_some()
+            && flow_job.script_path.is_some()
+        {
+            tx = schedule_again_if_scheduled(
+                tx,
+                db,
+                flow_job.schedule_path.as_ref().unwrap(),
+                flow_job.script_path.as_ref().unwrap(),
+                &w_id,
+            )
+            .await?;
+        }
+        tx.commit().await?;
+        (
+            should_continue_flow,
+            flow_job,
+            stop_early,
+            skip_if_stop_early,
+            nresult,
         )
-        .await?;
-    }
-    tx.commit().await?;
+    };
 
     let done = if !should_continue_flow {
         let logs = if flow_job.canceled {
@@ -456,6 +560,7 @@ pub async fn update_flow_status_after_job_completion(
                 logs,
                 canceled_job_to_result(&flow_job),
                 metrics.clone(),
+                rsmq.clone(),
             )
             .await?;
         } else {
@@ -464,8 +569,9 @@ pub async fn update_flow_status_after_job_completion(
                 &flow_job,
                 success,
                 stop_early && skip_if_stop_early,
-                result.clone(),
+                nresult.clone(),
                 logs,
+                rsmq.clone(),
             )
             .await?;
         }
@@ -475,10 +581,11 @@ pub async fn update_flow_status_after_job_completion(
             &flow_job,
             db,
             client,
-            result.clone(),
+            nresult.clone(),
             same_worker_tx.clone(),
             worker_dir,
             base_internal_url,
+            rsmq.clone(),
         )
         .await
         {
@@ -489,6 +596,7 @@ pub async fn update_flow_status_after_job_completion(
                     "Unexpected error during flow chaining:\n".to_string(),
                     json!({"message": err.to_string(), "name": "InternalError"}),
                     metrics.clone(),
+                    rsmq.clone(),
                 )
                 .await;
                 true
@@ -503,30 +611,22 @@ pub async fn update_flow_status_after_job_completion(
         }
 
         if let Some(parent_job) = flow_job.parent_job {
-            return Ok(update_flow_status_after_job_completion(
-                db,
-                client,
-                parent_job,
-                &flow,
-                w_id,
+            return Ok(Some(RecUpdateFlowStatusAfterJobCompletion {
+                flow: parent_job,
+                job_id_for_status: flow,
                 success,
-                result,
-                metrics,
-                false,
-                same_worker_tx.clone(),
-                worker_dir,
-                if stop_early {
+                result: nresult,
+                stop_early_override: if stop_early {
                     Some(skip_if_stop_early)
                 } else {
                     None
                 },
-                base_internal_url,
-            )
-            .await?);
+            }));
         }
+        Ok(None)
+    } else {
+        Ok(None)
     }
-
-    Ok(())
 }
 
 async fn compute_skip_loop_failures<'c>(
@@ -747,7 +847,7 @@ async fn transform_input(
 }
 
 #[instrument(level = "trace", skip_all)]
-pub async fn handle_flow(
+pub async fn handle_flow<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
     flow_job: &QueuedJob,
     db: &sqlx::Pool<sqlx::Postgres>,
     client: &AuthedClient,
@@ -755,6 +855,7 @@ pub async fn handle_flow(
     same_worker_tx: Sender<Uuid>,
     worker_dir: &str,
     base_internal_url: &str,
+    rsmq: Option<R>,
 ) -> anyhow::Result<()> {
     let value = flow_job
         .raw_flow
@@ -778,6 +879,7 @@ pub async fn handle_flow(
         same_worker_tx,
         worker_dir,
         base_internal_url,
+        rsmq,
     )
     .await?;
     Ok(())
@@ -785,7 +887,7 @@ pub async fn handle_flow(
 
 #[async_recursion]
 // #[instrument(level = "trace", skip_all)]
-async fn push_next_flow_job(
+async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
     flow_job: &QueuedJob,
     mut status: FlowStatus,
     flow: FlowValue,
@@ -795,6 +897,7 @@ async fn push_next_flow_job(
     same_worker_tx: Sender<Uuid>,
     worker_dir: &str,
     base_internal_url: &str,
+    rsmq: Option<R>,
 ) -> error::Result<()> {
     let job_root = flow_job
         .root_job
@@ -831,6 +934,7 @@ async fn push_next_flow_job(
             worker_dir,
             None,
             base_internal_url,
+            rsmq,
         )
         .await;
     }
@@ -1001,7 +1105,7 @@ async fn push_next_flow_job(
                 let logs = "Timed out waiting to be resumed".to_string();
                 let result = json!({ "error": {"message": logs, "name": "SuspendedTimeout"}});
                 let _uuid =
-                    add_completed_job(db, &flow_job, success, skipped, result, logs).await?;
+                    add_completed_job(db, &flow_job, success, skipped, result, logs, rsmq).await?;
 
                 return Ok(());
             }
@@ -1202,7 +1306,7 @@ async fn push_next_flow_job(
         flow.same_worker && module.suspend.is_none() && module.sleep.is_none();
 
     /* Finally, push the job into the queue */
-    let mut tx = db.begin().await?;
+    let mut tx = (rsmq.clone(), db.begin().await?).into();
     let mut uuids = vec![];
 
     let len = match &job_payloads {
@@ -1261,7 +1365,11 @@ async fn push_next_flow_job(
             Ok(v) => (Some(v), None),
             Err(e) => (None, Some(e)),
         };
-        let root_job = flow_job.root_job.or_else(|| Some(flow_job.id));
+        let root_job = if matches!(module.value, FlowModuleValue::Flow { .. }) {
+            None
+        } else {
+            flow_job.root_job.or_else(|| Some(flow_job.id))
+        };
         let (uuid, inner_tx) = push(
             tx,
             &flow_job.workspace_id,
@@ -1364,7 +1472,7 @@ async fn push_next_flow_job(
             json!(i),
             flow_job.id
         )
-        .execute(&mut tx)
+        .execute(db)
         .await?;
     } else {
         sqlx::query!(
@@ -1380,7 +1488,7 @@ async fn push_next_flow_job(
             json!(i),
             flow_job.id
         )
-        .execute(&mut tx)
+        .execute(db)
         .await?;
     };
 

benchmarks/example_fake_user.json

@@ -1,53 +0,0 @@
-{
-  "type": "RANDOM",
-  "actions": [
-    {
-      "weight": 1,
-      "action": {
-        "type": "RANDOM",
-        "actions": [
-          {
-            "weight": 1,
-            "action": {
-              "type": "PREVIEW_SCRIPT",
-              "workspace": "demo",
-              "language": "deno",
-              "args": {},
-              "content": "export async function main() { return \"Hello World\"; }"
-            }
-          },
-          {
-            "weight": 1,
-            "action": {
-              "type": "PREVIEW_SCRIPT",
-              "workspace": "demo",
-              "language": "python3",
-              "args": {},
-              "content": "def main(): return \"Hello World\";"
-            }
-          },
-          {
-            "weight": 1,
-            "action": {
-              "type": "PREVIEW_SCRIPT",
-              "workspace": "demo",
-              "language": "go",
-              "args": {},
-              "content": "func main() { return \"Hello World\" }"
-            }
-          }
-        ]
-      }
-    },
-    {
-      "weight": 0.5,
-      "action": {
-        "type": "PREVIEW_SCRIPT",
-        "workspace": "demo",
-        "language": "deno",
-        "args": {},
-        "content": "import { delay } from \"https://deno.land/std@0.131.0/async/delay.ts\"; export async function main() { await delay(1000); return \"Hello World\"; }"
-      }
-    }
-  ]
-}

benchmarks/identity.json

@@ -0,0 +1,7 @@
+{
+  "type": "PREVIEW_SCRIPT",
+  "workspace": "demo",
+  "language": "deno",
+  "args": {},
+  "content": "export async function main() { return \"Hello World\"; }"
+}

benchmarks/main.ts

@@ -27,60 +27,60 @@ await new Command()
     "The number of workers to run at once.",
     {
       default: 1,
-    }
+    },
   )
   .option(
     "-s --seconds <seconds:number>",
     "How long to run the benchmark for (in seconds).",
     {
       default: 30,
-    }
+    },
   )
   .option("--max <max:number>", "Maximum number of operations performed.")
   .option("-e --email <email:string>", "The email to use to login.")
   .option("-p --password <password:string>", "The password to use to login.")
   .env(
     "WM_TOKEN=<token:string>",
-    "The token to use when talking to the API server. Preferred over manual login."
+    "The token to use when talking to the API server. Preferred over manual login.",
   )
   .option(
     "-t --token <token:string>",
-    "The token to use when talking to the API server. Preferred over manual login."
+    "The token to use when talking to the API server. Preferred over manual login.",
   )
   .env(
     "WM_WORKSPACE=<workspace:string>",
-    "The workspace to spawn scripts from."
+    "The workspace to spawn scripts from.",
   )
   .option(
     "-w --workspace <workspace:string>",
     "The workspace to spawn scripts from.",
-    { default: "starter" }
+    { default: "starter" },
   )
   .option("-m --metrics <metrics:string>", "The url to scrape metrics from.", {
     default: "http://localhost:8001/metrics",
   })
   .option(
     "--export-json <export_json:string>",
-    "If set, exports will be into a JSON file."
+    "If set, exports will be into a JSON file.",
   )
   .option(
     "--export-csv <export_csv:string>",
-    "If set, exports will be into a csv file."
+    "If set, exports will be into a csv file.",
   )
   .option(
     "--export-histograms [histograms...:string]",
-    "Mark metrics (without label) that are reported as histograms to export."
+    "Mark metrics (without label) that are reported as histograms to export.",
   )
   .option(
     "--export-simple [simple...:string]",
-    "Mark metrics (without label) that are reported as simple values."
+    "Mark metrics (without label) that are reported as simple values.",
   )
   .option(
     "--maximum-throughput <maximum_throughput:number>",
     "Maximum number of jobs/flows to start in one second.",
     {
       default: Infinity,
-    }
+    },
   )
   .option("--use-flows", "Run flows instead of jobs.")
   .option("--custom <custom_path:string>", "Use custom actions during bench")
@@ -89,11 +89,11 @@ await new Command()
     "The maximum time in ms to wait for jobs to complete.",
     {
       default: 90000,
-    }
+    },
   )
   .option(
     "--continous",
-    "Run the benchmark forever. This effectively disables metric collection & exports. No zombie jobs will be tracked."
+    "Run the benchmark forever. This effectively disables metric collection & exports. No zombie jobs will be tracked.",
   )
   .option(
     "--histogram-buckets [buckets...:string]",
@@ -114,7 +114,7 @@ await new Command()
         "0.01",
         "0.005",
       ],
-    }
+    },
   )
   .action(
     async ({
@@ -162,7 +162,7 @@ await new Command()
           new URL("./scraper.ts", import.meta.url).href,
           {
             type: "module",
-          }
+          },
         );
 
         metrics_worker.postMessage({
@@ -192,8 +192,8 @@ await new Command()
             zombieTimeout,
           },
           null,
-          4
-        )
+          4,
+        ),
       );
 
       const config = {
@@ -245,19 +245,22 @@ await new Command()
       const updateState = setInterval(async () => {
         const elapsed = start ? Math.ceil((Date.now() - start) / 1000) : 0;
         const sum = jobsSent.reduce((a, b) => a + b, 0);
-        const queue_length = (
-          await windmill.JobService.listQueue({
-            workspace: config.workspace_id,
-          })
-        ).length;
+        const queue_length = (await (await fetch(
+          host + "/api/w/" + config.workspace_id + "/jobs/queue/count",
+          { headers: { ["Authorization"]: "Bearer " + config.token } },
+        )).json()).database_length;
         await Deno.stdout.write(
           enc(
-            `elapsed: ${elapsed}/${seconds} | jobs sent: ${JSON.stringify(
-              jobsSent
-            )} (sum: ${sum} thr: ${(sum / elapsed).toFixed(
-              2
-            )}) | queue: ${queue_length}                          \r`
-          )
+            `elapsed: ${elapsed}/${seconds} | jobs sent: ${
+              JSON.stringify(
+                jobsSent,
+              )
+            } (sum: ${sum} thr: ${
+              (sum / elapsed).toFixed(
+                2,
+              )
+            }) | queue: ${queue_length}                          \r`,
+          ),
         );
       }, 100);
 
@@ -284,7 +287,7 @@ await new Command()
 
       const sum = jobsSent.reduce((a, b) => a + b, 0);
       await Deno.stdout.write(
-        enc(" ".padStart(30) + `\rduration: ${seconds} | jobs sent: ${sum}\n`)
+        enc(" ".padStart(30) + `\rduration: ${seconds} | jobs sent: ${sum}\n`),
       );
 
       const shutdown_start = Date.now();
@@ -302,7 +305,7 @@ await new Command()
         };
         worker.addEventListener("message", l);
         worker.postMessage(
-          Number.isSafeInteger(zombieTimeout) ? zombieTimeout : 90000
+          Number.isSafeInteger(zombieTimeout) ? zombieTimeout : 90000,
         );
       });
 
@@ -318,11 +321,10 @@ await new Command()
       console.log("incorrect results: ", incorrect_results);
       console.log(
         "queue length:",
-        (
-          await windmill.JobService.listQueue({
-            workspace: config.workspace_id,
-          })
-        ).length
+        (await (await fetch(
+          host + "/api/w/" + config.workspace_id + "/jobs/queue/count",
+          { headers: { ["Authorization"]: "Bearer " + config.token } },
+        )).json()).database_length,
       );
 
       metrics_worker!.postMessage("stop");
@@ -346,7 +348,7 @@ await new Command()
           const value = values[i]!;
           const mean = value.reduce((acc, e) => acc + e, 0) / values.length;
           const stdev = Math.sqrt(
-            value.reduce((acc, e) => acc + (e - mean) ** 2) / values.length
+            value.reduce((acc, e) => acc + (e - mean) ** 2) / values.length,
           );
           obj[name] = { mean, stdev };
         }
@@ -374,6 +376,6 @@ await new Command()
         f.close();
       }
       console.log("done");
-    }
+    },
   )
   .parse();

benchmarks/worker.ts

@@ -13,6 +13,8 @@ const promise = new Promise<{
   continous: boolean;
   max_per_worker: number;
   custom: Action | undefined;
+  server: string;
+  token: string;
 }>((resolve, _reject) => {
   self.onmessage = (evt) => {
     const sharedConfig = evt.data;
@@ -24,6 +26,8 @@ const promise = new Promise<{
       continous: sharedConfig.continous,
       max_per_worker: sharedConfig.max_per_worker,
       custom: sharedConfig.custom,
+      server: sharedConfig.server,
+      token: sharedConfig.token,
     };
     self.name = "Worker " + sharedConfig.i;
     resolve(config);
@@ -46,19 +50,20 @@ const updateStatusInterval = setInterval(() => {
 }, 100);
 
 while (cont) {
-  const queue_length = (
-    await windmill.JobService.listQueue({ workspace: config.workspace_id })
-  ).length;
-  if (queue_length > 500) {
+  const queue_length = (await (await fetch(
+    config.server + "/api/w/" + config.workspace_id + "/jobs/queue/count",
+    { headers: { ["Authorization"]: "Bearer " + config.token } },
+  )).json()).database_length;
+  if (queue_length > 2500) {
     console.log(
-      `queue length: ${queue_length} > 500. waiting...                                                            `
+      `queue length: ${queue_length} > 2500. waiting...                                                            `,
     );
     await sleep(0.5);
     continue;
   }
   if (
     (total_spawned * 1000) / (Date.now() - start_time) >
-    config.per_worker_throughput
+      config.per_worker_throughput
   ) {
     console.log("at maximum throughput. waiting...");
     await sleep(0.1);
@@ -138,13 +143,13 @@ while (outstanding.length > 0 && Date.now() < end_time) {
     await Deno.stdout.write(
       enc(
         `uuid: ${uuid}, queue length: ${
-          (
-            await windmill.JobService.listQueue({
-              workspace: config.workspace_id,
-            })
-          ).length
-        }                                                                                   \r`
-      )
+          (await (await fetch(
+            config.server + "/api/w/" + config.workspace_id +
+              "/jobs/queue/count",
+            { headers: { ["Authorization"]: "Bearer " + config.token } },
+          )).json()).database_length
+        }                                                                                   \r`,
+      ),
     );
   } else if (!config.useFlows) {
     r = r as api.CompletedJob;
@@ -156,7 +161,7 @@ while (outstanding.length > 0 && Date.now() < end_time) {
             " != " +
             uuid +
             "job: \n" +
-            JSON.stringify(r, null, 2)
+            JSON.stringify(r, null, 2),
         );
         incorrect_results++;
       }

cli/main.ts

@@ -19,7 +19,7 @@ import sync from "./sync.ts";
 import { tryResolveVersion } from "./context.ts";
 import { GlobalOptions } from "./types.ts";
 
-export const VERSION = "v1.84.1";
+export const VERSION = "v1.87.0";
 
 let command: any = new Command()
   .name("wmill")

cli/script.ts

@@ -142,6 +142,7 @@ export async function handleFile(
             typed.summary === remote.summary &&
             typed.is_template === remote.is_template &&
             typed.kind == remote.kind &&
+            !remote.archived &&
             remote?.lock == typed.lock?.join("\n") &&
             JSON.stringify(typed.schema) == JSON.stringify(remote.schema))
         ) {

deno-client/mod.ts

@@ -80,6 +80,31 @@ export async function getResource(
   }
 }
 
+/**
+ * Get the full resource value by path
+ * @param path path of the resource,  default to internal state path
+ * @param undefinedIfEmpty if the resource does not exist, return undefined instead of throwing an error
+ * @returns full resource
+ */
+export async function getFullResource(
+  path?: string,
+  undefinedIfEmpty?: boolean
+): Promise<any> {
+  const workspace = getWorkspace();
+  path = path ?? getStatePath();
+  try {
+    const resource = await ResourceService.getResource({ workspace, path });
+    const value = await _transformLeaf(resource.value);
+    return { ...resource, value };
+  } catch (e: any) {
+    if (undefinedIfEmpty && e.status === 404) {
+      return undefined;
+    } else {
+      throw Error(`Resource not found at ${path} or not visible to you`);
+    }
+  }
+}
+
 export function getStatePath(): string {
   const state_path = Deno.env.get("WM_STATE_PATH");
   if (state_path === undefined) {

docker-compose-dev.yml

@@ -0,0 +1,86 @@
+version: "3.7"
+
+services:
+  db:
+    image: postgres:14
+    restart: unless-stopped
+    volumes:
+      - db_data:/var/lib/postgresql/data
+    expose:
+      - 5432
+    ports:
+      - 5432:5432
+    environment:
+      POSTGRES_PASSWORD: ${DB_PASSWORD}
+      POSTGRES_DB: windmill
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  windmill_server:
+    image: ghcr.io/windmill-labs/windmill:main
+    deploy:
+      replicas: 1
+    restart: unless-stopped
+    expose:
+      - 8000
+    ports:
+      - 8000:8000
+    environment:
+      - DATABASE_URL=postgres://postgres:${DB_PASSWORD}@db/windmill?sslmode=disable
+      - BASE_URL=http://${WM_BASE_URL}
+      - RUST_LOG=info
+      ## You can set the number of workers to > 0 and not need any separate worker service
+      - NUM_WORKERS=0
+      - DISABLE_SERVER=false
+      - METRICS_ADDR=false
+    depends_on:
+      db:
+        condition: service_healthy
+    # volumes:
+    # - ./oauth.json/:/usr/src/app/oauth.json
+
+  windmill_worker:
+    image: ghcr.io/windmill-labs/windmill:main
+    deploy:
+      replicas: 3
+    restart: unless-stopped
+    environment:
+      - DATABASE_URL=postgres://postgres:${DB_PASSWORD}@db/windmill?sslmode=disable
+      - BASE_URL=http://${WM_BASE_URL}
+      - BASE_INTERNAL_URL=http://windmill_server:8000
+      - RUST_LOG=info
+      - NUM_WORKERS=1
+      - DISABLE_SERVER=true
+      - KEEP_JOB_DIR=false
+      - DENO_PATH=/usr/bin/deno
+      - PYTHON_PATH=/usr/local/bin/python3
+      - METRICS_ADDR=false
+    depends_on:
+      db:
+        condition: service_healthy
+    # to mount the worker folder to debug,, KEEP_JOB_DIR=true and mount /tmp/windmill
+    volumes:
+      - worker_dependency_cache:/tmp/windmill/cache
+
+  lsp:
+    image: ghcr.io/windmill-labs/windmill-lsp:latest
+    restart: unless-stopped
+    expose:
+      - 3001
+  caddy:
+    image: caddy:2.5.2-alpine
+    restart: unless-stopped
+    volumes:
+      - ./Caddyfile:/etc/caddy/Caddyfile
+    ports:
+      - 80:80
+      - 443:443
+    environment:
+      - BASE_URL=${WM_BASE_URL}
+
+volumes:
+  db_data: null
+  worker_dependency_cache: null

docker-compose.yml

@@ -1,21 +1,18 @@
-version: '3.7'
+version: "3.7"
 
 services:
-
   db:
     image: postgres:14
     restart: unless-stopped
     volumes:
       - db_data:/var/lib/postgresql/data
-    ports:
-      - 5432:5432
-      # NOTE: if port 5432 are already in use locally, we'll also bind another port
-      - 5433:5432
+    expose:
+      - 5432
     environment:
       POSTGRES_PASSWORD: ${DB_PASSWORD}
       POSTGRES_DB: windmill
     healthcheck:
-      test: [ "CMD-SHELL", "pg_isready -U postgres" ]
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
       interval: 10s
       timeout: 5s
       retries: 5
@@ -25,11 +22,11 @@ services:
     deploy:
       replicas: 1
     restart: unless-stopped
-    ports:
-      - 8000:8000
+    expose:
+      - 8000
     environment:
       - DATABASE_URL=postgres://postgres:${DB_PASSWORD}@db/windmill?sslmode=disable
-      - BASE_URL=http://${WM_BASE_URL}
+      - BASE_URL=${WM_BASE_URL}
       - RUST_LOG=info
       ## You can set the number of workers to > 0 and not need any separate worker service
       - NUM_WORKERS=0
@@ -39,8 +36,8 @@ services:
       db:
         condition: service_healthy
     # volumes:
-      # - ./oauth.json/:/usr/src/app/oauth.json
-      
+    # - ./oauth.json/:/usr/src/app/oauth.json
+
   windmill_worker:
     image: ghcr.io/windmill-labs/windmill:main
     deploy:
@@ -48,7 +45,7 @@ services:
     restart: unless-stopped
     environment:
       - DATABASE_URL=postgres://postgres:${DB_PASSWORD}@db/windmill?sslmode=disable
-      - BASE_URL=http://${WM_BASE_URL}
+      - BASE_URL=${WM_BASE_URL}
       - BASE_INTERNAL_URL=http://windmill_server:8000
       - RUST_LOG=info
       - NUM_WORKERS=1
@@ -67,8 +64,8 @@ services:
   lsp:
     image: ghcr.io/windmill-labs/windmill-lsp:latest
     restart: unless-stopped
-    ports:
-      - 3001:3001
+    expose:
+      - 3001
 
   caddy:
     image: caddy:2.5.2-alpine
@@ -79,7 +76,7 @@ services:
       - 80:80
       - 443:443
     environment:
-      - BASE_URL=${WM_BASE_URL}
+      - BASE_URL=${CADDY_REVERSE_PROXY}
 
 volumes:
   db_data: null

frontend/Caddyfile

@@ -2,9 +2,11 @@ http://localhost {
   bind {$ADDRESS}
   reverse_proxy /api/* http://localhost:8000
   reverse_proxy /* http://localhost:3000
+  reverse_proxy /ws/* http://localhost:3001
 }
 
 https://localhost {  
   bind {$ADDRESS}
+  reverse_proxy /api/* http://localhost:8000
   reverse_proxy /ws/* http://localhost:3001
 }

frontend/CaddyfileDev

@@ -4,9 +4,9 @@
 
 :8080 {  
   bind {$ADDRESS}
-  reverse_proxy /api/* http://windmill:8000
+  reverse_proxy /api/* http://localhost:8000
   reverse_proxy /* http://localhost:3000
   reverse_proxy /ws/* http://lsp:3001 {
     lb_policy header "Authorization"
   }
-}
+}

frontend/README_DEV.md

@@ -17,7 +17,7 @@ In the root folder:
 
 ```bash
 docker build . -t windmill
-docker compose up db windmill_server windmill_worker
+docker compose -f docker-compose-dev.yml up db windmill_server windmill_worker
 ```
 
 ### 2. Backend is run by cargo
@@ -28,20 +28,23 @@ docker compose up db windmill_server windmill_worker
 - Install llvm
 
   **On OSX:**
+
   ```bash
   brew install llvm caddy gsed
-  
+
   # make LLVM tools available on PATH
   echo 'export PATH="/opt/homebrew/opt/llvm/bin:$PATH"' >> ~/.zshrc
-  
+
   # now, restart your shell. You should now have the `lld` binary on your PATH.
   ```
+
 - To test that you have Rust and Cargo installed run `cargo --version`
 
 - In your terminal, go to the backend directory and run `cargo build`
 - Run `cargo run`
 
 **Known issue on M1 Mac while running `cargo build`**
+
 - You may encounter `linking with cc failed` build time error.
 - To solve this run:
   ```bash
@@ -49,7 +52,6 @@ docker compose up db windmill_server windmill_worker
   source ~/.zshrc
   ```
 
-
 **Do a Frontend Build**
 
 In order to run the backend, you need to have a frontend build inside `frontend/build/`.
@@ -70,11 +72,14 @@ npm run build
 ```
 
 **Known issue while running `npm run build`**
+
 - You may encounter `FATAL ERROR: Ineffective mark-compacts near heap limit Allocation failed - JavaScript heap out of memory` error.
 - To solve this run:
+
 ```bash
 export NODE_OPTIONS=--max_old_space_size=8096
 ```
+
 - run `npm run build` again
 
 In the root folder:
@@ -86,7 +91,7 @@ docker-compose up db
 In the backend folder:
 
 ```bash
-DATABASE_URL=postgres://postgres:changeme@127.0.0.1:5433/windmill?sslmode=disable cargo run
+DATABASE_URL=postgres://postgres:changeme@127.0.0.1:5432/windmill?sslmode=disable cargo run
 ```
 
 You can now access [http://127.0.0.1:8000](http://127.0.0.1:8000).
@@ -146,13 +151,13 @@ Recommended config for VS Code:
 
 - turn _format on save_ on
 
-
 ## Building
 
 The project is built with [SvelteKit](https://kit.svelte.dev/) and uses as output static files.
 There are others adapters for sveltekit, but we use the static adapter.
 
 To build the frontend as static assets, use:
+
 ```
 npm run build
 ```
@@ -160,10 +165,11 @@ npm run build
 The output is in the `build` folder.
 
 The default build assume you serve every non static files as the 200.html file which is catchall. If you prefer a normal layout, you can use:
+
 ```
 NOTCATCHALL=true npm run build
 ```
+
 which will generate an index.html and allow you to serve the frontend with any static server.
 
 Env variables used for build are set in .env file. See [https://vitejs.dev/guide/env-and-mode.html#env-files](https://vitejs.dev/guide/env-and-mode.html#env-files) for more details.
-

frontend/copy_workers.sh

@@ -0,0 +1 @@
+cp -r node_modules/monaco-editor-workers/dist/workers static

frontend/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "windmill",
-	"version": "1.84.1",
+	"version": "1.87.0",
 	"scripts": {
 		"dev": "vite dev",
 		"build": "vite build",
@@ -16,7 +16,8 @@
 	},
 	"devDependencies": {
 		"@playwright/test": "^1.31.1",
-		"@sveltejs/adapter-static": "^1.0.0",
+		"@rgossiaux/svelte-headlessui": "^1.0.2",
+		"@sveltejs/adapter-static": "^2.0.1",
 		"@sveltejs/kit": "^1.0.0-next.589",
 		"@sveltejs/package": "^1.0.2",
 		"@tailwindcss/forms": "^0.5.3",
@@ -25,20 +26,21 @@
 		"@types/d3-zoom": "^3.0.2",
 		"@types/lodash": "^4.14.191",
 		"@types/node": "^18.11.18",
-		"@types/vscode": "~1.74.0",
+		"@types/vscode": "~1.77.0",
 		"@typescript-eslint/eslint-plugin": "^5.49.0",
 		"@typescript-eslint/parser": "^5.48.0",
 		"@zerodevx/svelte-toast": "^0.8.1",
 		"autoprefixer": "^10.4.13",
-		"cssnano": "^5.1.14",
+		"cssnano": "^6.0.0",
 		"d3-dag": "^0.11.5",
-		"eslint": "^8.28.0",
+		"eslint": "^8.38.0",
 		"eslint-config-prettier": "^8.6.0",
 		"eslint-plugin-svelte3": "^4.0.0",
+		"monaco-editor-workers": "0.36.0",
 		"ol": "^7.2.2",
 		"openapi-typescript-codegen": "^0.23.0",
 		"path-browserify": "^1.0.1",
-		"pdfjs-dist": "^3.4.120",
+		"pdfjs-dist": "^3.5.141",
 		"postcss": "^8.4.18",
 		"postcss-load-config": "^4.0.1",
 		"prettier": "^2.8.3",
@@ -47,16 +49,17 @@
 		"style-to-object": "^0.4.1",
 		"stylelint-config-recommended": "^9.0.0",
 		"svelte": "^3.55.1",
-		"svelte-awesome": "^3.0.0",
+		"svelte-awesome": "^3.2.0",
 		"svelte-awesome-color-picker": "^2.4.1",
 		"svelte-check": "^3.0.2",
-		"svelte-highlight": "^6.2.1",
+		"svelte-highlight": "^7.2.1",
+		"svelte-multiselect": "^8.6.0",
 		"svelte-overlay": "^1.4.1",
 		"svelte-popperjs": "^1.3.2",
 		"svelte-preprocess": "^5.0.1",
 		"svelte-range-slider-pips": "^2.1.1",
 		"svelte-splitpanes": "^0.7.3",
-		"svelte2tsx": "^0.6.10",
+		"svelte2tsx": "^0.6.11",
 		"tailwindcss": "^3.2.2",
 		"tslib": "^2.4.1",
 		"typescript": "^4.9.3",
@@ -67,12 +70,11 @@
 	"dependencies": {
 		"@fortawesome/free-brands-svg-icons": "^6.2.1",
 		"@fortawesome/free-solid-svg-icons": "^6.2.1",
-		"@leeoniya/ufuzzy": "^0.9.1",
+		"@leeoniya/ufuzzy": "^1.0.6",
 		"@popperjs/core": "^2.11.6",
 		"@redocly/json-to-json-schema": "^0.0.1",
 		"@tanstack/svelte-table": "^8.7.6",
 		"ag-grid-svelte": "^0.1.4",
-		"async-mutex": "^0.4.0",
 		"chart.js": "^3.9.1",
 		"chartjs-adapter-date-fns": "^3.0.0",
 		"chartjs-plugin-zoom": "^2.0.0",
@@ -82,16 +84,16 @@
 		"highlight.js": "^11.7.0",
 		"lodash": "^4.17.21",
 		"lucide-svelte": "^0.115.0",
-		"monaco-editor": "0.33.0",
-		"monaco-editor-workers": "0.33.0",
-		"monaco-languageclient": "2.1.0",
-		"monaco-yaml": "^4.0.2",
+		"monaco-editor": "0.36.1",
+		"monaco-languageclient": "5.0.1",
 		"svelte-autosize": "^1.0.1",
 		"svelte-chartjs": "^3.1.0",
 		"svelte-portal": "^2.2.0",
-		"svelte-select": "^5.3.1",
+		"svelte-select": "^5.6.0",
+		"svelte-timezone-picker": "^2.0.3",
 		"tailwind-merge": "^1.9.1",
-		"vscode-ws-jsonrpc": "^2.0.1"
+		"vscode-languageclient": "^8.1.0",
+		"vscode-ws-jsonrpc": "3.0.0"
 	},
 	"peerDependencies": {
 		"svelte": "^3.55.1"

frontend/src/lib/components/AppConnect.svelte

@@ -370,7 +370,7 @@
 			<h2 class="mt-8 mb-4">Non OAuth APIs & Resources</h2>
 			<div class="grid sm:grid-cols-2 md:grid-cols-3 gap-x-2 gap-y-1 items-center mb-2">
 				{#if filteredConnectsManual}
-					{#each filteredConnectsManual as [key, instructions]}
+					{#each filteredConnectsManual as [key, _]}
 						<Button
 							size="sm"
 							variant="border"
@@ -414,7 +414,7 @@
 				</div>
 				{#if apiTokenApps[resource_type].img}
 					<div class="mt-4 w-full overflow-hidden">
-						<img class="m-auto  max-h-60" alt="connect" src={apiTokenApps[resource_type].img} />
+						<img class="m-auto max-h-60" alt="connect" src={apiTokenApps[resource_type].img} />
 					</div>
 				{/if}
 			{/if}

frontend/src/lib/components/ArgInfo.svelte

@@ -1,13 +1,11 @@
 <script lang="ts">
-	import { truncate } from '$lib/utils'
-	import Tooltip from './Tooltip.svelte'
-	import json from 'svelte-highlight/languages/json'
-	import { Highlight } from 'svelte-highlight'
 	import { ResourceService } from '$lib/gen'
 	import { workspaceStore } from '$lib/stores'
-	import Drawer from './common/drawer/Drawer.svelte'
+	import { truncate } from '$lib/utils'
 	import { DrawerContent } from './common'
+	import Drawer from './common/drawer/Drawer.svelte'
 	import ObjectViewer from './propertyPicker/ObjectViewer.svelte'
+	import Tooltip from './Tooltip.svelte'
 
 	export let value: any
 	let jsonViewer: Drawer

frontend/src/lib/components/ArgInput.svelte

@@ -3,32 +3,31 @@
 		faChevronDown,
 		faChevronUp,
 		faDollarSign,
-		faMinus,
 		faPlus
 	} from '@fortawesome/free-solid-svg-icons'
 
-	import { setInputCat as computeInputCat } from '$lib/utils'
-	import { Badge, Button } from './common'
-	import { createEventDispatcher } from 'svelte'
-	import Icon from 'svelte-awesome'
-	import FieldHeader from './FieldHeader.svelte'
-	import ObjectResourceInput from './ObjectResourceInput.svelte'
-	import ObjectTypeNarrowing from './ObjectTypeNarrowing.svelte'
-	import ResourcePicker from './ResourcePicker.svelte'
-	import StringTypeNarrowing from './StringTypeNarrowing.svelte'
-	import SchemaForm from './SchemaForm.svelte'
 	import type { SchemaProperty } from '$lib/common'
-	import SimpleEditor from './SimpleEditor.svelte'
+	import { setInputCat as computeInputCat } from '$lib/utils'
+	import { X } from 'lucide-svelte'
+	import { createEventDispatcher } from 'svelte'
 	import autosize from 'svelte-autosize'
-	import Toggle from './Toggle.svelte'
-	import Password from './Password.svelte'
-	import type VariableEditor from './VariableEditor.svelte'
+	import Icon from 'svelte-awesome'
+	import { fade } from 'svelte/transition'
+	import JsonEditor from './apps/editor/settingsPanel/inputEditor/JsonEditor.svelte'
+	import { Badge, Button } from './common'
+	import FieldHeader from './FieldHeader.svelte'
 	import type ItemPicker from './ItemPicker.svelte'
 	import NumberTypeNarrowing from './NumberTypeNarrowing.svelte'
+	import ObjectResourceInput from './ObjectResourceInput.svelte'
+	import ObjectTypeNarrowing from './ObjectTypeNarrowing.svelte'
+	import Password from './Password.svelte'
 	import Range from './Range.svelte'
-	import JsonEditor from './apps/editor/settingsPanel/inputEditor/JsonEditor.svelte'
-	import { fade } from 'svelte/transition'
-	import { X } from 'lucide-svelte'
+	import ResourcePicker from './ResourcePicker.svelte'
+	import SchemaForm from './SchemaForm.svelte'
+	import SimpleEditor from './SimpleEditor.svelte'
+	import StringTypeNarrowing from './StringTypeNarrowing.svelte'
+	import Toggle from './Toggle.svelte'
+	import type VariableEditor from './VariableEditor.svelte'
 
 	export let label: string = ''
 	export let value: any
@@ -42,7 +41,6 @@
 	export let required = false
 	export let pattern: undefined | string = undefined
 	export let valid = required ? false : true
-	export let maxRows = 10
 	export let enum_: string[] | undefined = undefined
 	export let disabled = false
 	export let editableSchema = false
@@ -63,8 +61,6 @@
 	let seeEditable: boolean = enum_ != undefined || pattern != undefined
 	const dispatch = createEventDispatcher()
 
-	$: maxHeight = maxRows ? `${1 + maxRows * 1.2}em` : `auto`
-
 	$: validateInput(pattern, value)
 
 	let error: string = ''
@@ -168,6 +164,7 @@
 	let itemsLimit = 50
 </script>
 
+<!-- svelte-ignore a11y-autofocus -->
 <div class="flex flex-col w-full min-w-[250px]">
 	<div>
 		{#if displayHeader}
@@ -175,6 +172,7 @@
 		{/if}
 		{#if editableSchema}
 			<div class="p-2 my-1 text-xs border-solid border border-gray-400">
+				<!-- svelte-ignore a11y-click-events-have-key-events -->
 				<span
 					class="underline"
 					on:click={() => {

frontend/src/lib/components/CenteredModal.svelte

@@ -13,9 +13,11 @@
 	})
 </script>
 
-<div class="center-center min-h-screen p-4 relative bg-[#3b4252]">
-	<div class="border rounded-md shadow-md bg-white w-full max-w-[640px] 
-	p-4 sm:py-8 sm:px-10 mb-6 md:mb-20 z-10">
+<div class="center-center min-h-screen p-4 relative bg-gray-50">
+	<div
+		class="border rounded-md shadow-md bg-white w-full max-w-[640px]
+	p-4 sm:py-8 sm:px-10 mb-6 md:mb-20 z-10"
+	>
 		<div class="mb-10">
 			<h1 class="text-center">
 				{title}
@@ -30,13 +32,13 @@
 	</div>
 	{#if !disableLogo}
 		<div class="hidden lg:block absolute top-10 right-50">
-			<div class="animate-[spin_100s_linear_infinite] ">
-				<WindmillIcon white height="100px" width="100px" />
+			<div>
+				<WindmillIcon height="100px" width="100px" spin="slow" />
 			</div>
-			<h2 class="text-center pt-2 text-gray-100">Windmill</h2>
+			<h2 class="text-center pt-2 text-gray-800">Windmill</h2>
 		</div>
 	{/if}
-	<div class="absolute top-0 right-0 text-2xs text-gray-300 italic px-3 py-1">
+	<div class="absolute top-0 right-0 text-2xs text-gray-800 italic px-3 py-1">
 		<span class="font-mono">{version}</span>
 	</div>
 </div>

frontend/src/lib/components/CronInput.svelte

@@ -1,92 +1,328 @@
-<script context="module">
-	export const OFFSET = new Date().getTimezoneOffset()
-</script>
-
 <script lang="ts">
 	import { ScheduleService } from '$lib/gen'
-	import { displayDate, emptyString, formatCron, sendUserToast } from '$lib/utils'
-	import CollapseLink from './CollapseLink.svelte'
+	import { emptyString, formatCron } from '$lib/utils'
+	import Badge from './Badge.svelte'
 
+	// @ts-ignore
+	import Multiselect from 'svelte-multiselect'
+	import TimezonePicker from 'svelte-timezone-picker'
+	import CollapseLink from './CollapseLink.svelte'
+	import { Button } from './common'
+
+	export let schedule: string
+	// export let offset: number = -60 * Math.floor(new Date().getTimezoneOffset() / 60)
+	export let timezone: string // = Intl.DateTimeFormat().resolvedOptions().timeZone
+	export let disabled = false
 	export let validCRON = true
 
 	let preview: string[] = []
-	let cronError = ''
-	export let schedule: string = '0 0 12 * *'
-	export let disabled = false
-	let limit = 3
+	// If the user has already entered a cron string, switching to the basic tab will override it.
+	let executeEvery: 'second' | 'minute' | 'hour' | 'day-month' | 'month' | 'day-week' = 'minute'
 
-	$: !emptyString(schedule) && handleScheduleInput(schedule)
+	let seconds = 30
+	let minutes = 30
+	let hours = 1
+	const daysOfMonthOptions: number[] = Array.from(Array(31).keys()).map((i) => i + 1)
+	let daysOfMonth: number[] = []
+	// let lastDayOfMonth = false
+	const monthsOfYearOptions: string[] = [
+		'January',
+		'February',
+		'March',
+		'April',
+		'May',
+		'June',
+		'July',
+		'August',
+		'September',
+		'October',
+		'November',
+		'December'
+	]
+	let monthsOfYear: string[] = []
+	const daysOfWeekOptions: string[] = [
+		'Sunday',
+		'Monday',
+		'Tuesday',
+		'Wednesday',
+		'Thursday',
+		'Friday',
+		'Saturday'
+	]
+	let daysOfWeek: string[] = []
+	let UTCTime: string = ''
 
-	async function handleScheduleInput(input: string): Promise<void> {
+	$: !emptyString(schedule) && handleScheduleInput(schedule, timezone)
+
+	async function handleScheduleInput(input: string, timezone: string): Promise<void> {
 		try {
 			preview = await ScheduleService.previewSchedule({
-				requestBody: { schedule: formatCron(input), offset: OFFSET }
+				requestBody: { schedule: formatCron(input), timezone }
 			})
-			cronError = ''
 			validCRON = true
 		} catch (err) {
 			if (err.status == 400 && err.body.includes('cron')) {
-				cronError = `Invalid cron expression`
 				validCRON = false
 			} else {
 				validCRON = false
 			}
 		}
 	}
+
+	let nschedule = ''
+
+	$: {
+		// CRON string format
+		// sec  min   hour      day of month   month     day of week   year
+		// 0    30    9,12,15   1,15           May-Aug   Mon,Wed,Fri   2018/2
+
+		let s_daysOfMonth = ''
+		// if (lastDayOfMonth) {
+		// 	s_daysOfMonth = 'L'
+		// } else
+		if (daysOfMonth.length > 0) {
+			s_daysOfMonth = daysOfMonth.join(',')
+		} else {
+			s_daysOfMonth = '*'
+		}
+
+		let s_months = ''
+		if (monthsOfYear.length > 0) {
+			s_months = monthsOfYear.map((m) => m.slice(0, 3).toLowerCase()).join(',')
+		} else {
+			s_months = '*'
+		}
+
+		let s_daysOfWeek = ''
+		if (daysOfWeek.length > 0) {
+			s_daysOfWeek = daysOfWeek.map((d) => d.slice(0, 3).toLowerCase()).join(',')
+		} else {
+			s_daysOfWeek = '*'
+		}
+
+		const s_AtUTCHours = parseInt(UTCTime.split(':')[0]) || '0'
+		const s_AtUTCMinutes = parseInt(UTCTime.split(':')[1]) || '0'
+
+		// If using the basic editor, set the cron string based on the selected options
+		if (executeEvery === 'second') {
+			if (seconds > 0) {
+				nschedule = `*/${seconds} * * * *`
+			} else {
+				nschedule = `* * * * *`
+			}
+		} else if (executeEvery === 'minute') {
+			if (minutes > 0) {
+				nschedule = `0 */${minutes} * * *`
+			} else {
+				nschedule = `* * * * *`
+			}
+		} else if (executeEvery === 'hour') {
+			if (hours > 0) {
+				nschedule = `0 0 */${hours} * *`
+			} else {
+				nschedule = `* * * * *`
+			}
+		} else if (executeEvery === 'day-month') {
+			nschedule = `0 ${s_AtUTCMinutes} ${s_AtUTCHours} ${s_daysOfMonth} *`
+		} else if (executeEvery === 'month') {
+			nschedule = `0 ${s_AtUTCMinutes} ${s_AtUTCHours} ${s_daysOfMonth} ${s_months}`
+		} else if (executeEvery === 'day-week') {
+			nschedule = `0 ${s_AtUTCMinutes} ${s_AtUTCHours} * * ${s_daysOfWeek}`
+		}
+	}
+
+	$: dateFormatter = new Intl.DateTimeFormat('en-GB', {
+		weekday: 'short',
+		day: '2-digit',
+		month: 'short',
+		year: 'numeric',
+		hour: 'numeric',
+		minute: 'numeric',
+		second: 'numeric',
+		timeZone: timezone,
+		timeZoneName: 'short'
+	}).format
 </script>
 
-<div class="max-w-xl">
-	<div class="w-full text-right text-red-600 text-2xs grow">{cronError}</div>
-	<div class="flex flex-row items-end max-w-5xl">
-		<label class="text-xs min-w-max mr-2 self-center" for="cron-schedule">CRON expression</label>
-		<input
-			class="inline-block"
-			type="text"
-			id="cron-schedule"
-			name="cron-schedule"
-			bind:value={schedule}
-			{disabled}
-		/>
-	</div>
-	{#if !disabled}
-		<div class="flex flex-row text-xs text-blue-500 gap-3 pl-28 mb-2">
-			<button
-				on:click={() => {
-					schedule = '0 */15 * * *'
-					cronError = ''
-				}}>every 15 min</button
-			>
-			<button
-				on:click={() => {
-					schedule = '0 0 * * * *'
-					cronError = ''
-				}}>every hour</button
-			>
-			<button
-				on:click={() => {
-					schedule = '0 0 8 * * *'
-					cronError = ''
-				}}>once a day at 8AM</button
-			>
+<div class="w-full flex space-x-16 p-4">
+	<div class="w-full flex flex-col space-y-2">
+		<div class="w-full flex flex-col gap-1">
+			<small class="font-bold">Cron</small>
+			<input
+				class="inline-block"
+				type="text"
+				id="cron-schedule"
+				name="cron-schedule"
+				placeholder="*/30 * * * *"
+				bind:value={schedule}
+				{disabled}
+			/>
+			{#if !validCRON}
+				<small class="text-red-600"> Invalid cron syntax </small>
+			{/if}
 		</div>
-	{/if}
 
-	<CollapseLink text="preview next runs" open={true}>
-		{#if preview && preview.length > 0}
-			<div class="text-sm text-gray-700 border p-2 rounded-md">
-				<div class="flex flex-row justify-between">The next runs will be scheduled at:</div>
-				<ul class="list-disc mx-12">
-					{#each preview.slice(0, limit) as p}
-						<li class="mx-2 text-gray-700 text-sm">{displayDate(p, true)}</li>
-					{/each}
-					<li class="text-sm mx-2">...</li>
-					{#if limit != 10}
-						<button class="underline text-gray-400" on:click={() => (limit = 10)}>Load more</button>
-					{:else}
-						<button class="underline text-gray-400" on:click={() => (limit = 3)}>Load less</button>
-					{/if}
-				</ul>
+		<div class="w-full flex flex-col gap-1">
+			<small class="font-bold">Timezone</small>
+
+			{#if disabled}
+				<div>
+					<Badge>{timezone}</Badge>
+				</div>
+			{:else}
+				<TimezonePicker {timezone} on:update={(e) => (timezone = e.detail.timezone)} />
+			{/if}
+		</div>
+
+		{#if !disabled}
+			<div class="w-full">
+				<CollapseLink text="Use simplified builder">
+					<div class="w-full flex flex-col gap-4 mt-4">
+						<div class="w-full flex flex-col gap-1">
+							<small class="font-bold">Execute schedule every</small>
+
+							<div class="w-full flex gap-4">
+								<div class="w-full flex flex-col gap-1">
+									<select
+										{disabled}
+										name="execute_every"
+										id="execute_every"
+										bind:value={executeEvery}
+									>
+										<option value="second">Second(s)</option>
+										<option value="minute">Minute(s)</option>
+										<option value="hour">Hour(s)</option>
+										<option value="day-month">Day of the month</option>
+										<option value="month">Month(s)</option>
+										<option value="day-week">Day of the week</option>
+									</select>
+								</div>
+
+								<div class="w-full flex flex-col gap-1 justify-center">
+									{#if executeEvery == 'second'}
+										<input {disabled} type="number" min="0" max="59" bind:value={seconds} />
+										<small>Valid range 0-59</small>
+									{:else if executeEvery == 'minute'}
+										<input {disabled} type="number" min="0" max="59" bind:value={minutes} />
+										<small>Valid range 0-59</small>
+									{:else if executeEvery == 'hour'}
+										<input {disabled} type="number" min="0" max="23" bind:value={hours} />
+										<small>Valid range 0-23</small>
+									{:else if executeEvery == 'day-month'}
+										<!-- <div class="w-full flex">
+									<label for="lastDayOfMonth" class="w-full flex items-center gap-2">
+										<div class="flex">
+											<input type="checkbox" id="lastDayOfMonth" bind:checked={lastDayOfMonth} />
+										</div>
+										<small> Last day of the month </small>
+									</label>
+								</div> -->
+									{/if}
+								</div>
+							</div>
+						</div>
+
+						<div class="w-full flex flex-col gap-4">
+							{#if executeEvery == 'month'}
+								<div class="w-full flex flex-col">
+									<Multiselect
+										{disabled}
+										bind:selected={monthsOfYear}
+										options={monthsOfYearOptions}
+										selectedOptionsDraggable={false}
+										placeholder="Every month"
+									/>
+								</div>
+							{/if}
+
+							{#if executeEvery == 'day-week'}
+								<div class="w-full flex flex-col">
+									<Multiselect
+										{disabled}
+										bind:selected={daysOfWeek}
+										options={daysOfWeekOptions}
+										selectedOptionsDraggable={false}
+										placeholder="Every day"
+									/>
+								</div>
+							{/if}
+
+							{#if executeEvery == 'day-month' || executeEvery == 'month'}
+								<div class="w-full flex flex-col gap-1">
+									{#if executeEvery == 'month'}
+										<small class="font-bold">On day of the month</small>
+									{/if}
+									<div class="w-full flex gap-4">
+										<div class="w-full flex">
+											<Multiselect
+												{disabled}
+												bind:selected={daysOfMonth}
+												options={daysOfMonthOptions}
+												selectedOptionsDraggable={false}
+												placeholder="Every day"
+											/>
+										</div>
+
+										<!-- {#if executeEvery == 'month'}
+									<div class="w-full flex">
+										<label for="lastDayOfMonth" class="w-full flex items-center gap-2">
+											<div class="flex">
+												<input type="checkbox" id="lastDayOfMonth" bind:checked={lastDayOfMonth} />
+											</div>
+											<small> Last day of the month </small>
+										</label>
+									</div>
+								{/if} -->
+									</div>
+									<small>Schedule will only execute on valid calendar days</small>
+								</div>
+							{/if}
+
+							{#if executeEvery == 'day-month' || executeEvery == 'month' || executeEvery == 'day-week'}
+								<div class="w-full flex flex-col gap-1">
+									<small class="font-bold">At UTC Time</small>
+									<input
+										{disabled}
+										type="time"
+										name="atUTCTime"
+										id="atUTCTime"
+										bind:value={UTCTime}
+									/>
+								</div>
+							{/if}
+						</div>
+
+						<div class="w-full flex flex-col gap-1">
+							<small class="font-bold">Preview New Cron</small>
+
+							<div class="flex p-2 px-4 rounded-md bg-gray-100">
+								<span>{nschedule}</span>
+							</div>
+						</div>
+					</div>
+
+					<div class="mt-4">
+						<Button variant="border" color="dark" size="xs" on:click={() => (schedule = nschedule)}
+							>Set Schedule</Button
+						>
+					</div>
+				</CollapseLink>
 			</div>
 		{/if}
-	</CollapseLink>
+	</div>
+
+	<div class="w-full flex flex-col space-y-2">
+		<h3>Execution summary</h3>
+		<hr />
+		<div class="flex flex-col space-y-2">
+			<small>Estimated upcoming events ({timezone})</small>
+			<div class="flex flex-col rounded-md p-4 border text-gray-600">
+				{#each preview as date}
+					<div class="flex items-center space-x-2">
+						<span>{dateFormatter(new Date(date))}</span>
+					</div>
+				{/each}
+			</div>
+		</div>
+	</div>
 </div>

frontend/src/lib/components/DisplayResult.svelte

@@ -99,19 +99,17 @@
 			<div class="mb-2 text-gray-500 text-sm bg-gray-50/20">
 				as JSON&nbsp;<input class="windmillapp" type="checkbox" bind:checked={forceJson} /></div
 			>{/if}{#if typeof result == 'object' && Object.keys(result).length > 0}<div
-				class="mb-2 min-w-[360px] text-sm text-gray-700 relative"
+				class="mb-2 w-full text-sm text-gray-700 relative"
 				>The result keys are: <b>{truncate(Object.keys(result).join(', '), 50)}</b>
-				<div class="text-gray-500 text-sm  absolute top-0 right-2">
-					<button on:click={jsonViewer.openDrawer}>Expand JSON</button>
+				<div class="text-gray-500 text-xs absolute top-5 right-0">
+					<button on:click={jsonViewer.openDrawer}>Expand</button>
 				</div></div
 			>{/if}{#if !forceJson && resultKind == 'table-col'}<div
-				class="grid grid-flow-col-dense border border-gray-200 rounded-md "
+				class="grid grid-flow-col-dense border border-gray-200 rounded-md"
 			>
 				{#each Object.keys(result) as col}
 					<div class="flex flex-col max-h-40 min-w-full">
-						<div
-							class="px-12 text-left uppercase border-b bg-gray-50 overflow-hidden rounded-t-md "
-						>
+						<div class="px-12 text-left uppercase border-b bg-gray-50 overflow-hidden rounded-t-md">
 							{col}
 						</div>
 						{#if Array.isArray(result[col])}
@@ -125,7 +123,7 @@
 				{/each}
 			</div>
 		{:else if !forceJson && resultKind == 'table-row'}<div
-				class="grid grid-flow-col-dense border border-gray-200 "
+				class="grid grid-flow-col-dense border border-gray-200"
 			>
 				<TableCustom>
 					<tbody slot="body">

frontend/src/lib/components/Dropdown.svelte

@@ -47,8 +47,8 @@
 							dispatch('click', { item: item?.eventName })
 						}
 					}}
-					class="block w-full  whitespace-nowrap hover:drop-shadow-sm hover:bg-gray-50 hover:bg-opacity-30
-					 px-4 py-2 text-sm text-gray-700 text-left 
+					class="block w-full whitespace-nowrap hover:drop-shadow-sm hover:bg-gray-50 hover:bg-opacity-30
+					 px-4 py-2 text-sm text-gray-700 text-left
 					 {item.disabled ? 'bg-gray-200' : ''}
 					 {item.separatorTop ? 'border-t' : ''} {item.separatorBottom ? 'border-b' : ''} {item.type ==
 					'delete'
@@ -72,7 +72,7 @@
 				<a
 					href={item.href}
 					on:click|stopPropagation|preventDefault={() => goto(item.href ?? '')}
-					class="block w-full px-4 font-semibold text-left  py-2 text-sm text-gray-700 hover:drop-shadow-sm hover:bg-gray-50 hover:bg-opacity-30
+					class="block w-full px-4 font-semibold text-left py-2 text-sm text-gray-700 hover:drop-shadow-sm hover:bg-gray-50 hover:bg-opacity-30
 					{item.disabled ? 'bg-gray-200' : ''}"
 					role="menuitem"
 					tabindex="-1"
@@ -89,9 +89,10 @@
 					{item.displayName}
 				</a>
 			{:else}
+				<!-- svelte-ignore a11y-click-events-have-key-events -->
 				<span
 					class:bg-gray-50={item.disabled}
-					class="block  text-left px-4 py-2 text-sm text-gray-700 cursor-auto"
+					class="block text-left px-4 py-2 text-sm text-gray-700 cursor-auto"
 					role="menuitem"
 					tabindex="-1"
 					id="user-menu-item-{name}-{i}}"

frontend/src/lib/components/Editor.svelte

@@ -1,10 +1,12 @@
 <script lang="ts" context="module">
-	import getMessageServiceOverride from 'vscode/service-override/messages'
+	import getDialogServiceOverride from 'vscode/service-override/dialogs'
+	import getNotificationServiceOverride from 'vscode/service-override/notifications'
 	import { StandaloneServices } from 'vscode/services'
 
 	try {
 		StandaloneServices?.initialize({
-			...getMessageServiceOverride(document.body)
+			...getNotificationServiceOverride(document.body),
+			...getDialogServiceOverride()
 		})
 	} catch (e) {
 		console.error(e)
@@ -12,35 +14,34 @@
 </script>
 
 <script lang="ts">
-	import { browser, dev } from '$app/environment'
+	import { browser } from '$app/environment'
 	import { page } from '$app/stores'
 	import { sendUserToast } from '$lib/utils'
 
-	import editorWorker from 'monaco-editor/esm/vs/editor/editor.worker?worker'
-	import tsWorker from 'monaco-editor/esm/vs/language/typescript/ts.worker?worker'
-
-	import { buildWorkerDefinition } from 'monaco-editor-workers'
-	import type {
-		Disposable,
-		DocumentUri,
-		MessageTransports,
-		MonacoLanguageClient
-	} from 'monaco-languageclient'
 	import { createEventDispatcher, onDestroy, onMount } from 'svelte'
 
-	import { languages, editor as meditor, KeyCode, KeyMod, Uri as mUri } from 'monaco-editor'
-
-	languages.typescript.typescriptDefaults.setCompilerOptions({
-		target: languages.typescript.ScriptTarget.Latest,
-		allowNonTsExtensions: true,
-		noLib: true
+	import 'monaco-editor/esm/vs/editor/edcore.main'
+	import {
+		editor as meditor,
+		KeyCode,
+		KeyMod,
+		Uri as mUri,
+		languages
+	} from 'monaco-editor/esm/vs/editor/editor.api'
+	import 'monaco-editor/esm/vs/basic-languages/python/python.contribution'
+	import 'monaco-editor/esm/vs/basic-languages/go/go.contribution'
+	import 'monaco-editor/esm/vs/basic-languages/shell/shell.contribution'
+	import 'monaco-editor/esm/vs/basic-languages/typescript/typescript.contribution'
+	import 'monaco-editor/esm/vs/language/typescript/monaco.contribution'
+	import { MonacoLanguageClient, MonacoServices } from 'monaco-languageclient'
+	import { toSocket, WebSocketMessageReader, WebSocketMessageWriter } from 'vscode-ws-jsonrpc'
+	import { CloseAction, ErrorAction, RequestType } from 'vscode-languageclient'
+	import * as vscode from 'vscode'
+	languages.typescript.typescriptDefaults.setModeConfiguration({
+		completionItems: false,
+		definitions: false,
+		hovers: false
 	})
-	languages.typescript.typescriptDefaults.setDiagnosticsOptions({
-		noSemanticValidation: true,
-		noSuggestionDiagnostics: true,
-		noSyntaxValidation: true
-	})
-	languages.typescript.typescriptDefaults.setExtraLibs([])
 
 	meditor.defineTheme('myTheme', {
 		base: 'vs',
@@ -53,32 +54,45 @@
 	})
 	meditor.setTheme('myTheme')
 
-	import {
-		BASH_INIT_CODE,
-		DENO_INIT_CODE_CLEAR,
-		GO_INIT_CODE,
-		PYTHON_INIT_CODE_CLEAR
-	} from '$lib/script_helpers'
 	import {
 		createHash as randomHash,
 		editorConfig,
 		langToExt,
 		updateOptions
 	} from '$lib/editorUtils'
+	import {
+		BASH_INIT_CODE,
+		DENO_INIT_CODE_CLEAR,
+		GO_INIT_CODE,
+		PYTHON_INIT_CODE_CLEAR
+	} from '$lib/script_helpers'
+	import type { Disposable } from 'vscode'
+	import type { DocumentUri, MessageTransports } from 'vscode-languageclient'
 	import { dirtyStore } from './common/confirmationModal/dirtyStore'
+	import { buildWorkerDefinition } from './build_workers'
+	import { workspaceStore } from '$lib/stores'
+	import { UserService } from '$lib/gen'
 
 	let divEl: HTMLDivElement | null = null
 	let editor: meditor.IStandaloneCodeEditor
 
 	export let lang: 'typescript' | 'python' | 'go' | 'shell'
 	export let code: string = ''
-	export let hash: string = randomHash()
 	export let cmdEnterAction: (() => void) | undefined = undefined
 	export let formatAction: (() => void) | undefined = undefined
 	export let automaticLayout = true
 	export let websocketAlive = { pyright: false, black: false, deno: false, go: false }
 	export let shouldBindKey: boolean = true
 	export let fixedOverflowWidgets = true
+	export let path: string = randomHash()
+
+	if (path == '' || path == undefined || path.startsWith('/')) {
+		path = randomHash()
+	}
+
+	let initialPath: string = path
+
+	$: path != initialPath && handlePathChange()
 
 	let websockets: [MonacoLanguageClient, WebSocket][] = []
 	let websocketInterval: NodeJS.Timer | undefined
@@ -87,28 +101,12 @@
 	let disposeMethod: () => void | undefined
 	const dispatch = createEventDispatcher()
 
-	const uri = `file:///tmp/monaco/${hash}.${langToExt(lang)}`
+	const uri =
+		lang == 'go' ? `file:///tmp/monaco/${randomHash()}.go` : `file:///${path}.${langToExt(lang)}`
 
-	if (browser) {
-		if (dev) {
-			buildWorkerDefinition(
-				'../../../node_modules/monaco-editor-workers/dist/workers',
-				import.meta.url,
-				false
-			)
-		} else {
-			// @ts-ignore
-			self.MonacoEnvironment = {
-				getWorker: function (_moduleId: any, label: string) {
-					if (label === 'typescript' || label === 'javascript') {
-						return new tsWorker()
-					} else {
-						return new editorWorker()
-					}
-				}
-			}
-		}
-	}
+	// if (lang != 'typescript') {
+	buildWorkerDefinition('../../../workers', import.meta.url, false)
+	// }
 
 	export function getCode(): string {
 		return editor?.getValue() ?? ''
@@ -136,27 +134,31 @@
 		}
 	}
 
-	export function setCode(ncode: string): void {
+	export function setCode(ncode: string, noHistory: boolean = false): void {
 		code = ncode
-		if (editor?.getModel()) {
-			// editor.setValue(ncode)
-			editor.pushUndoStop()
+		if (noHistory) {
+			editor?.setValue(ncode)
+		} else {
+			if (editor?.getModel()) {
+				// editor.setValue(ncode)
+				editor.pushUndoStop()
 
-			editor.executeEdits('set', [
-				{
-					range: editor.getModel()!.getFullModelRange(), // full range
-					text: ncode
-				}
-			])
+				editor.executeEdits('set', [
+					{
+						range: editor.getModel()!.getFullModelRange(), // full range
+						text: ncode
+					}
+				])
 
-			editor.pushUndoStop()
+				editor.pushUndoStop()
+			}
 		}
 	}
 
 	export function format() {
 		if (editor) {
 			code = getCode()
-			editor.getAction('editor.action.formatDocument').run()
+			editor?.getAction('editor.action.formatDocument')?.run()
 			if (formatAction) {
 				formatAction()
 			}
@@ -182,22 +184,14 @@
 
 	export async function reloadWebsocket() {
 		await closeWebsockets()
-		const { MonacoLanguageClient } = await import('monaco-languageclient')
-		const { CloseAction, ErrorAction } = await import('vscode-languageclient')
-		const { toSocket, WebSocketMessageReader, WebSocketMessageWriter } = await import(
-			'vscode-ws-jsonrpc'
-		)
-		const vscode = await import('vscode')
-		const { RequestType } = await import('vscode-jsonrpc')
-		// install Monaco language client services
-		const { MonacoServices } = await import('monaco-languageclient')
 
 		monacoServices = MonacoServices.install()
 
 		function createLanguageClient(
 			transports: MessageTransports,
 			name: string,
-			initializationOptions?: any
+			initializationOptions: any,
+			middlewareOptions: ((params, token, next) => any) | undefined
 		) {
 			const client = new MonacoLanguageClient({
 				name: name,
@@ -216,13 +210,11 @@
 					initializationOptions,
 					middleware: {
 						workspace: {
-							configuration: (params, token, configuration) => {
-								return [
-									{
-										enable: true
-									}
-								]
-							}
+							configuration:
+								middlewareOptions ??
+								((params, token, next) => {
+									return [{ enabled: true }]
+								})
 						}
 					}
 				},
@@ -235,7 +227,12 @@
 			return client
 		}
 
-		async function connectToLanguageServer(url: string, name: string, options?: any) {
+		async function connectToLanguageServer(
+			url: string,
+			name: string,
+			initOptions: any,
+			middlewareOptions: any
+		) {
 			try {
 				const webSocket = new WebSocket(url)
 
@@ -243,7 +240,15 @@
 					const socket = toSocket(webSocket)
 					const reader = new WebSocketMessageReader(socket)
 					const writer = new WebSocketMessageWriter(socket)
-					const languageClient = createLanguageClient({ reader, writer }, name, options)
+					const languageClient = createLanguageClient(
+						{ reader, writer },
+						name,
+						initOptions,
+						middlewareOptions
+					)
+					// if (middlewareOptions != undefined) {
+					// 	languageClient.registerNotUsedFeatures()
+					// }
 					websockets.push([languageClient, webSocket])
 
 					// HACK ALERT: for some reasons, the client need to be restarted to take into account the 'go get <dep>' command
@@ -273,12 +278,13 @@
 					})
 
 					try {
-						console.log('started client')
+						console.log('starting client')
 						await languageClient.start()
+						console.log('started client')
 					} catch (err) {
 						console.log('err at client')
 						console.error(err)
-						throw new Error(err)
+						return
 					}
 
 					lastWsAttempt = new Date()
@@ -309,64 +315,138 @@
 		}
 
 		const wsProtocol = $page.url.protocol == 'https:' ? 'wss' : 'ws'
+
+		let encodedImportMap = ''
 		if (lang == 'typescript') {
-			await connectToLanguageServer(`${wsProtocol}://${$page.url.host}/ws/deno`, 'deno', {
-				certificateStores: null,
-				enablePaths: [],
-				config: null,
-				importMap: null,
-				internalDebug: false,
-				lint: false,
-				path: null,
-				tlsCertificate: null,
-				unsafelyIgnoreCertificateErrors: null,
-				unstable: true,
-				enable: true,
-				cache: null,
-				codeLens: {
-					implementations: true,
-					references: true
-				},
-				suggest: {
-					autoImports: true,
-					completeFunctionCalls: false,
-					names: true,
-					paths: true,
+			if (path && path.split('/').length > 2) {
+				let expiration = new Date()
+				expiration.setHours(expiration.getHours() + 2)
+				const token = await UserService.createToken({
+					requestBody: { label: 'Ephemeral lsp token', expiration: expiration.toISOString() }
+				})
+				let root =
+					$page.url.protocol +
+					'//' +
+					$page.url.host +
+					'/api/scripts_u/tokened_raw/' +
+					$workspaceStore +
+					'/' +
+					token
+				const importMap = {
 					imports: {
-						autoDiscover: true,
-						hosts: {
-							'https://deno.land': true
+						'file:///': root + '/'
+					}
+				}
+				let path_splitted = path.split('/')
+				for (let c = 0; c < path_splitted.length; c++) {
+					let key = 'file://./'
+					for (let i = 0; i < c; i++) {
+						key += '../'
+					}
+					let url = path_splitted.slice(0, -c - 1).join('/')
+					let ending = c == path_splitted.length - 1 ? '' : '/'
+					importMap['imports'][key] = `${root}/${url}${ending}`
+				}
+				encodedImportMap = 'data:text/plain;base64,' + btoa(JSON.stringify(importMap))
+			}
+			await connectToLanguageServer(
+				`${wsProtocol}://${$page.url.host}/ws/deno`,
+				'deno',
+				{
+					certificateStores: null,
+					enablePaths: [],
+					config: null,
+					importMap: encodedImportMap,
+					internalDebug: false,
+					lint: false,
+					path: null,
+					tlsCertificate: null,
+					unsafelyIgnoreCertificateErrors: null,
+					unstable: true,
+					enable: true,
+					codeLens: {
+						implementations: true,
+						references: true,
+						referencesAllFunction: false
+					},
+					suggest: {
+						autoImports: true,
+						completeFunctionCalls: false,
+						names: true,
+						paths: true,
+						imports: {
+							autoDiscover: true,
+							hosts: {
+								'https://deno.land': true
+							}
 						}
 					}
+				},
+				() => {
+					return [
+						{
+							enable: true
+						}
+					]
 				}
-			})
+			)
 		} else if (lang === 'python') {
-			await connectToLanguageServer(`${wsProtocol}://${$page.url.host}/ws/pyright`, 'pyright', {
-				executionEnvironments: [
-					{
-						root: '/tmp/pyright',
-						pythonVersion: '3.7',
-						pythonPlatform: 'platform',
-						extraPaths: []
+			await connectToLanguageServer(
+				`${wsProtocol}://${$page.url.host}/ws/pyright`,
+				'pyright',
+				{},
+				(params, token, next) => {
+					if (params.items.find((x) => x.section === 'python')) {
+						return [
+							{
+								analysis: {
+									useLibraryCodeForTypes: true,
+									autoImportCompletions: true,
+									diagnosticSeverityOverrides: { reportMissingImports: 'none' },
+									typeCheckingMode: 'basic'
+								}
+							}
+						]
 					}
-				]
-			})
+					if (params.items.find((x) => x.section === 'python.analysis')) {
+						return [
+							{
+								useLibraryCodeForTypes: true,
+								autoImportCompletions: true,
+								diagnosticSeverityOverrides: { reportMissingImports: 'none' },
+								typeCheckingMode: 'basic'
+							}
+						]
+					}
+					return next(params, token)
+				}
+			)
 
-			connectToLanguageServer(`${wsProtocol}://${$page.url.host}/ws/black`, 'black', {
-				formatters: {
-					black: {
-						command: 'black',
-						args: ['--quiet', '-']
+			connectToLanguageServer(
+				`${wsProtocol}://${$page.url.host}/ws/black`,
+				'black',
+				{
+					formatters: {
+						black: {
+							command: 'black',
+							args: ['--quiet', '-']
+						}
+					},
+					formatFiletypes: {
+						python: 'black'
 					}
 				},
-				formatFiletypes: {
-					python: 'black'
-				}
-			})
+				undefined
+			)
 		} else if (lang === 'go') {
-			connectToLanguageServer(`${wsProtocol}://${$page.url.host}/ws/go`, 'go', {
-				'build.allowImplicitNetworkAccess': true
-			})
+			connectToLanguageServer(
+				`${wsProtocol}://${$page.url.host}/ws/go`,
+				'go',
+				{
+					'build.allowImplicitNetworkAccess': true
+				},
+				undefined
+			)
 		}
 
 		websocketInterval && clearInterval(websocketInterval)
@@ -397,6 +477,13 @@
 		}, 5000)
 	}
 
+	let pathTimeout: NodeJS.Timeout | undefined = undefined
+	function handlePathChange() {
+		initialPath = path
+		pathTimeout && clearTimeout(pathTimeout)
+		pathTimeout = setTimeout(reloadWebsocket, 3000)
+	}
+
 	async function closeWebsockets() {
 		command && command.dispose()
 		command = undefined
@@ -417,12 +504,14 @@
 		websocketInterval && clearInterval(websocketInterval)
 	}
 
+	let widgets: HTMLElement | undefined = document.getElementById('monaco-widgets-root') ?? undefined
 	async function loadMonaco() {
 		const model = meditor.createModel(code, lang, mUri.parse(uri))
 
 		model.updateOptions(updateOptions)
 		editor = meditor.create(divEl as HTMLDivElement, {
 			...editorConfig(model, code, lang, automaticLayout, fixedOverflowWidgets),
+			overflowWidgetsDomNode: widgets,
 			tabSize: lang == 'python' ? 4 : 2
 		})
 
@@ -506,7 +595,7 @@
 
 <div bind:this={divEl} class="{$$props.class} editor" />
 
-<style>
+<style lang="postcss">
 	.editor {
 		@apply p-0 border rounded-md border-gray-50;
 	}

frontend/src/lib/components/EditorBar.svelte

@@ -4,7 +4,7 @@
 
 <script lang="ts">
 	import { ResourceService, VariableService } from '$lib/gen'
-	import { getScriptByPath, sendUserToast } from '$lib/utils'
+	import { getModifierKey, getScriptByPath, sendUserToast } from '$lib/utils'
 
 	import {
 		faBroom,
@@ -33,7 +33,7 @@
 	import Kbd from './common/kbd/Kbd.svelte'
 
 	export let lang: 'python3' | 'deno' | 'go' | 'bash'
-	export let editor: Editor
+	export let editor: Editor | undefined
 	export let websocketAlive: { pyright: boolean; black: boolean; deno: boolean; go: boolean }
 	export let iconOnly: boolean = false
 	export let validCode: boolean = true
@@ -50,10 +50,10 @@
 		undefined
 
 	function addEditorActions() {
-		editor.addAction('insert-variable', 'Windmill: Insert variable', () => {
+		editor?.addAction('insert-variable', 'Windmill: Insert variable', () => {
 			variablePicker.openDrawer()
 		})
-		editor.addAction('insert-resource', 'Windmill: Insert resource', () => {
+		editor?.addAction('insert-resource', 'Windmill: Insert resource', () => {
 			resourcePicker.openDrawer()
 		})
 	}
@@ -79,6 +79,8 @@
 		codeViewer?.openDrawer?.()
 		codeObj = await getScriptByPath(e.detail.path ?? '')
 	}
+
+	let version = __pkg__.version
 </script>
 
 <Drawer bind:this={scriptPicker} size="900px">
@@ -108,6 +110,7 @@
 <ItemPicker
 	bind:this={contextualVariablePicker}
 	pickCallback={(path, name) => {
+		if (!editor) return
 		if (lang == 'deno') {
 			editor.insertAtCursor(`Deno.env.get('${name}')`)
 		} else if (lang == 'python3') {
@@ -133,10 +136,11 @@
 <ItemPicker
 	bind:this={variablePicker}
 	pickCallback={(path, name) => {
+		if (!editor) return
 		if (lang == 'deno') {
 			if (!editor.getCode().includes('import * as wmill from')) {
 				editor.insertAtBeginning(
-					`import * as wmill from 'https://deno.land/x/windmill@v${__pkg__.version}/mod.ts'\n`
+					`import * as wmill from 'https://deno.land/x/windmill@v${version}/mod.ts'\n`
 				)
 			}
 			editor.insertAtCursor(`(await wmill.getVariable('${path}'))`)
@@ -180,10 +184,11 @@
 <ItemPicker
 	bind:this={resourcePicker}
 	pickCallback={(path, _) => {
+		if (!editor) return
 		if (lang == 'deno') {
 			if (!editor.getCode().includes('import * as wmill from')) {
 				editor.insertAtBeginning(
-					`import * as wmill from 'https://deno.land/x/windmill@v${__pkg__.version}/mod.ts'\n`
+					`import * as wmill from 'https://deno.land/x/windmill@v${version}/mod.ts'\n`
 				)
 			}
 			editor.insertAtCursor(`(await wmill.getResource('${path}'))`)
@@ -233,7 +238,13 @@
 			{validCode ? 'Valid' : 'Invalid'}
 		</Badge>
 		<div class="flex items-center divide-x">
-			<Popover notClickable placement="bottom" disapperTimoout={0} class="pr-1" disablePopup={!iconOnly}>
+			<Popover
+				notClickable
+				placement="bottom"
+				disapperTimoout={0}
+				class="pr-1"
+				disablePopup={!iconOnly}
+			>
 				<Button
 					color="light"
 					btnClasses="!font-medium !h-full"
@@ -245,11 +256,15 @@
 				>
 					+Context Var
 				</Button>
-				<svelte:fragment slot="text">
-					Add context variable
-				</svelte:fragment>
+				<svelte:fragment slot="text">Add context variable</svelte:fragment>
 			</Popover>
-			<Popover notClickable placement="bottom" disapperTimoout={0} class="px-1" disablePopup={!iconOnly}>
+			<Popover
+				notClickable
+				placement="bottom"
+				disapperTimoout={0}
+				class="px-1"
+				disablePopup={!iconOnly}
+			>
 				<Button
 					color="light"
 					btnClasses="!font-medium !h-full"
@@ -261,11 +276,15 @@
 				>
 					+Variable
 				</Button>
-				<svelte:fragment slot="text">
-					Add variable
-				</svelte:fragment>
+				<svelte:fragment slot="text">Add variable</svelte:fragment>
 			</Popover>
-			<Popover notClickable placement="bottom" disapperTimoout={0} class="px-1" disablePopup={!iconOnly}>
+			<Popover
+				notClickable
+				placement="bottom"
+				disapperTimoout={0}
+				class="px-1"
+				disablePopup={!iconOnly}
+			>
 				<Button
 					btnClasses="!font-medium !h-full"
 					size="xs"
@@ -277,33 +296,41 @@
 				>
 					+Resource
 				</Button>
-				<svelte:fragment slot="text">
-					Add resource
-				</svelte:fragment>
+				<svelte:fragment slot="text">Add resource</svelte:fragment>
 			</Popover>
-			<Popover notClickable placement="bottom" disapperTimoout={0} class="px-1" disablePopup={!iconOnly}>
+			<Popover
+				notClickable
+				placement="bottom"
+				disapperTimoout={0}
+				class="px-1"
+				disablePopup={!iconOnly}
+			>
 				<Button
 					btnClasses="!font-medium !h-full"
 					size="xs"
 					spacingSize="md"
 					color="light"
-					on:click={editor.clearContent}
+					on:click={editor?.clearContent}
 					{iconOnly}
 					startIcon={{ icon: faRotateLeft }}
 				>
 					Reset
 				</Button>
-				<svelte:fragment slot="text">
-					Reset
-				</svelte:fragment>
+				<svelte:fragment slot="text">Reset</svelte:fragment>
 			</Popover>
-			<Popover notClickable placement="bottom" disapperTimoout={0} class="px-1" disablePopup={!iconOnly}>
+			<Popover
+				notClickable
+				placement="bottom"
+				disapperTimoout={0}
+				class="px-1"
+				disablePopup={!iconOnly}
+			>
 				<Button
 					btnClasses="!font-medium !h-full"
 					size="xs"
 					spacingSize="md"
 					color="light"
-					on:click={editor.reloadWebsocket}
+					on:click={editor?.reloadWebsocket}
 					startIcon={{ icon: faRotate }}
 				>
 					{#if !iconOnly}
@@ -320,29 +347,41 @@
 						{/if}
 					</span>
 				</Button>
-				<svelte:fragment slot="text">
-					Reload assistant
-				</svelte:fragment>
+				<svelte:fragment slot="text">Reload assistant</svelte:fragment>
 			</Popover>
-			<Popover notClickable placement="bottom" disapperTimoout={0} class="px-1" disablePopup={!iconOnly}>
+			<Popover
+				notClickable
+				placement="bottom"
+				disapperTimoout={0}
+				class="px-1"
+				disablePopup={!iconOnly}
+			>
 				<Button
 					btnClasses="!font-medium"
 					size="xs"
 					spacingSize="md"
 					color="light"
-					on:click={editor.format}
+					on:click={editor?.format}
 					{iconOnly}
 					startIcon={{ icon: faBroom }}
 				>
-					Format (Ctrl+S)
+					Format ({getModifierKey()}+S)
 				</Button>
 				<svelte:fragment slot="text">
-					Format <Kbd class="!text-gray-800">Ctrl</Kbd> + <Kbd class="!text-gray-800">S</Kbd>
+					Format <Kbd class="!text-gray-800">{getModifierKey()}</Kbd> + <Kbd class="!text-gray-800">
+						S
+					</Kbd>
 				</svelte:fragment>
 			</Popover>
 		</div>
 	</div>
-	<Popover notClickable placement="bottom" disapperTimoout={0} class="px-1" disablePopup={!iconOnly}>
+	<Popover
+		notClickable
+		placement="bottom"
+		disapperTimoout={0}
+		class="px-1"
+		disablePopup={!iconOnly}
+	>
 		<Button
 			btnClasses="!font-medium"
 			size="xs"
@@ -352,15 +391,13 @@
 			{iconOnly}
 			startIcon={{ icon: faEye }}
 		>
-			Script
+			Explore other scripts
 		</Button>
-		<svelte:fragment slot="text">
-			Script
-		</svelte:fragment>
+		<svelte:fragment slot="text">Script</svelte:fragment>
 	</Popover>
 </div>
 
-<style>
+<style lang="postcss">
 	span.green {
 		@apply text-green-600 animate-[pulse_5s_ease-in-out_infinite];
 	}

frontend/src/lib/components/FlowBuilder.svelte

@@ -4,15 +4,14 @@
 	import { initHistory, redo, undo } from '$lib/history'
 	import { userStore, workspaceStore } from '$lib/stores'
 	import { encodeState, formatCron, loadHubScripts, sendUserToast } from '$lib/utils'
-	import { faCalendarAlt, faPen, faSave } from '@fortawesome/free-solid-svg-icons'
+	import { faCalendarAlt, faSave } from '@fortawesome/free-solid-svg-icons'
 	import { setContext } from 'svelte'
 	import { writable, type Writable } from 'svelte/store'
 	import CenteredPage from './CenteredPage.svelte'
-	import { Button, ButtonPopup, ButtonPopupItem, UndoRedo } from './common'
+	import { Badge, Button, UndoRedo } from './common'
 	import { dirtyStore } from './common/confirmationModal/dirtyStore'
-	import { OFFSET } from './CronInput.svelte'
-	import ScriptEditorDrawer from './flows/content/ScriptEditorDrawer.svelte'
 	import FlowEditor from './flows/FlowEditor.svelte'
+	import ScriptEditorDrawer from './flows/content/ScriptEditorDrawer.svelte'
 	import type { FlowState } from './flows/flowState'
 	import { dfs } from './flows/flowStore'
 	import FlowImportExportMenu from './flows/header/FlowImportExportMenu.svelte'
@@ -20,6 +19,7 @@
 	import { loadFlowSchedule, type Schedule } from './flows/scheduleUtils'
 	import type { FlowEditorContext } from './flows/types'
 	import { cleanInputs } from './flows/utils'
+	import { Pen } from 'lucide-svelte'
 
 	export let initialPath: string = ''
 	export let selectedId: string | undefined
@@ -29,7 +29,7 @@
 	export let flowStateStore: Writable<FlowState>
 
 	async function createSchedule(path: string) {
-		const { cron, args, enabled } = $scheduleStore
+		const { cron, timezone, args, enabled } = $scheduleStore
 
 		try {
 			await ScheduleService.createSchedule({
@@ -37,7 +37,7 @@
 				requestBody: {
 					path: path,
 					schedule: formatCron(cron),
-					offset: OFFSET,
+					timezone,
 					script_path: path,
 					is_flow: true,
 					args,
@@ -53,73 +53,79 @@
 
 	async function saveFlow(leave: boolean): Promise<void> {
 		loadingSave = true
-		const flow = cleanInputs($flowStore)
-		const { cron, args, enabled } = $scheduleStore
-		$dirtyStore = false
-		if (initialPath === '') {
-			localStorage.removeItem('flow')
-			await FlowService.createFlow({
-				workspace: $workspaceStore!,
-				requestBody: {
-					path: flow.path,
-					summary: flow.summary,
-					description: flow.description ?? '',
-					value: flow.value,
-					schema: flow.schema
+		try {
+			const flow = cleanInputs($flowStore)
+			const { cron, timezone, args, enabled } = $scheduleStore
+			$dirtyStore = false
+			if (initialPath === '') {
+				localStorage.removeItem('flow')
+				await FlowService.createFlow({
+					workspace: $workspaceStore!,
+					requestBody: {
+						path: flow.path,
+						summary: flow.summary,
+						description: flow.description ?? '',
+						value: flow.value,
+						schema: flow.schema
+					}
+				})
+				if (enabled) {
+					await createSchedule(flow.path)
 				}
-			})
-			if (enabled) {
-				await createSchedule(flow.path)
-			}
-		} else {
-			localStorage.removeItem(`flow-${initialPath}`)
-			await FlowService.updateFlow({
-				workspace: $workspaceStore!,
-				path: initialPath,
-				requestBody: {
-					path: flow.path,
-					summary: flow.summary,
-					description: flow.description ?? '',
-					value: flow.value,
-					schema: flow.schema
-				}
-			})
-			const scheduleExists = await ScheduleService.existsSchedule({
-				workspace: $workspaceStore ?? '',
-				path: flow.path
-			})
-			if (scheduleExists) {
-				const schedule = await ScheduleService.getSchedule({
+			} else {
+				localStorage.removeItem(`flow-${initialPath}`)
+				await FlowService.updateFlow({
+					workspace: $workspaceStore!,
+					path: initialPath,
+					requestBody: {
+						path: flow.path,
+						summary: flow.summary,
+						description: flow.description ?? '',
+						value: flow.value,
+						schema: flow.schema
+					}
+				})
+				const scheduleExists = await ScheduleService.existsSchedule({
 					workspace: $workspaceStore ?? '',
 					path: flow.path
 				})
-				if (JSON.stringify(schedule.args) != JSON.stringify(args) || schedule.schedule != cron) {
-					await ScheduleService.updateSchedule({
+				if (scheduleExists) {
+					const schedule = await ScheduleService.getSchedule({
 						workspace: $workspaceStore ?? '',
-						path: flow.path,
-						requestBody: {
-							schedule: formatCron(cron),
-							args
-						}
+						path: flow.path
 					})
+					if (JSON.stringify(schedule.args) != JSON.stringify(args) || schedule.schedule != cron) {
+						await ScheduleService.updateSchedule({
+							workspace: $workspaceStore ?? '',
+							path: flow.path,
+							requestBody: {
+								schedule: formatCron(cron),
+								timezone,
+								args
+							}
+						})
+					}
+					if (enabled != schedule.enabled) {
+						await ScheduleService.setScheduleEnabled({
+							workspace: $workspaceStore ?? '',
+							path: flow.path,
+							requestBody: { enabled }
+						})
+					}
+				} else if (enabled) {
+					await createSchedule(flow.path)
 				}
-				if (enabled != schedule.enabled) {
-					await ScheduleService.setScheduleEnabled({
-						workspace: $workspaceStore ?? '',
-						path: flow.path,
-						requestBody: { enabled }
-					})
-				}
-			} else if (enabled) {
-				await createSchedule(flow.path)
 			}
-		}
-		loadingSave = false
-		if (leave) {
-			goto(`/flows/get/${$flowStore.path}?workspace_id=${$workspaceStore}`)
-		} else if (initialPath !== $flowStore.path) {
-			initialPath = $flowStore.path
-			goto(`/flows/edit/${$flowStore.path}?workspace_id=${$workspaceStore}`)
+			loadingSave = false
+			if (leave) {
+				goto(`/flows/get/${$flowStore.path}?workspace_id=${$workspaceStore}`)
+			} else if (initialPath !== $flowStore.path) {
+				initialPath = $flowStore.path
+				goto(`/flows/edit/${$flowStore.path}?workspace_id=${$workspaceStore}`)
+			}
+		} catch (err) {
+			sendUserToast(`The flow could not be saved: ${err.body}`, true)
+			loadingSave = false
 		}
 	}
 
@@ -150,7 +156,12 @@
 
 	const selectedIdStore = writable<string>(selectedId ?? 'settings-metadata')
 
-	const scheduleStore = writable<Schedule>({ args: {}, cron: '', enabled: false })
+	const scheduleStore = writable<Schedule>({
+		args: {},
+		cron: '',
+		timezone: Intl.DateTimeFormat().resolvedOptions().timeZone,
+		enabled: false
+	})
 	const previewArgsStore = writable<Record<string, any>>(initialArgs)
 	const scriptEditorDrawer = writable<ScriptEditorDrawer | undefined>(undefined)
 	const moving = writable<{ module: FlowModule; modules: FlowModule[] } | undefined>(undefined)
@@ -182,6 +193,7 @@
 			.catch(() => {
 				scheduleStore.set({
 					cron: '0 */5 * * *',
+					timezone: 'UTC',
 					args: {},
 					enabled: false
 				})
@@ -205,20 +217,20 @@
 
 		switch (event.key) {
 			case 'Z':
-				if (event.ctrlKey) {
+				if (event.ctrlKey || event.metaKey) {
 					$flowStore = redo(history)
 					event.preventDefault()
 				}
 				break
 			case 'z':
-				if (event.ctrlKey) {
+				if (event.ctrlKey || event.metaKey) {
 					$flowStore = undo(history, $flowStore)
 					$selectedIdStore = 'Input'
 					event.preventDefault()
 				}
 				break
 			case 's':
-				if (event.ctrlKey) {
+				if (event.ctrlKey || event.metaKey) {
 					saveFlow(false)
 					event.preventDefault()
 				}
@@ -251,6 +263,23 @@
 			...dfs($flowStore.value.modules, (module) => module.id)
 		]
 	}
+
+	const dropdownItems: Array<{
+		label: string
+		onClick: () => void
+	}> = [
+		{
+			label: 'Save and exit',
+			onClick: () => saveFlow(true)
+		}
+	]
+
+	if (initialPath != '') {
+		dropdownItems.push({
+			label: 'Fork',
+			onClick: () => window.open(`/flows/add?template=${initialPath}`)
+		})
+	}
 </script>
 
 <svelte:window on:keydown={onKeyDown} />
@@ -261,10 +290,17 @@
 	<div class="flex flex-col flex-1 h-screen">
 		<!-- Nav between steps-->
 		<div
-			class="justify-between flex flex-row w-full items-center pl-2.5 pr-6  space-x-4 overflow-x-auto scrollbar-hidden max-h-12 h-full"
+			class="justify-between flex flex-row items-center pl-2.5 pr-6 space-x-4 scrollbar-hidden max-h-12 h-full"
 		>
-			<div class="flex flex-row gap-4 items-center">
-				<FlowImportExportMenu />
+			<div class="flex w-full max-w-md gap-4 items-center">
+				<div class="min-w-64 w-full">
+					<input
+						type="text"
+						placeholder="Flow summary"
+						class="text-sm w-full font-semibold"
+						bind:value={$flowStore.summary}
+					/>
+				</div>
 				<UndoRedo
 					undoProps={{ disabled: $history.index === 0 }}
 					redoProps={{ disabled: $history.index === $history.history.length - 1 }}
@@ -278,7 +314,7 @@
 				/>
 			</div>
 
-			<div class="gap-1 flex-row hidden md:flex shrink overflow-hidden">
+			<div class="gap-4 flex-row hidden md:flex w-full max-w-md">
 				{#if $scheduleStore.enabled}
 					<Button
 						btnClasses="hidden lg:inline-flex"
@@ -293,53 +329,47 @@
 						{$scheduleStore.cron ?? ''}
 					</Button>
 				{/if}
-				<Button
-					btnClasses="hidden lg:inline-flex"
-					startIcon={{ icon: faPen }}
-					variant="contained"
-					color="light"
-					size="xs"
-					on:click={async () => {
-						select('settings-metadata')
-						document.getElementById('path')?.focus()
-					}}
-				>
-					{$flowStore.path && $flowStore.path != '' ? $flowStore.path : 'Choose a path'}
-				</Button>
-				<Button
-					startIcon={{ icon: faPen }}
-					variant="contained"
-					color="light"
-					size="xs"
-					on:click={async () => {
-						select('settings-metadata')
-						document.getElementById('flow-summary')?.focus()
-					}}
-				>
-					<div class="max-w-[10em] !truncate">
-						{$flowStore.summary == '' || !$flowStore.summary ? 'No summary' : $flowStore.summary}
+				<div class="flex justify-start w-full">
+					<div>
+						<button
+							on:click={async () => {
+								select('settings-metadata')
+								document.getElementById('path')?.focus()
+							}}
+						>
+							<Badge
+								color="gray"
+								class="center-center !bg-gray-300 !text-gray-600 !h-[28px]  !w-[70px] rounded-r-none"
+							>
+								<Pen size={12} class="mr-2" /> Path
+							</Badge>
+						</button>
 					</div>
-				</Button>
+					<input
+						type="text"
+						readonly
+						value={$flowStore.path && $flowStore.path != '' ? $flowStore.path : 'Choose a path'}
+						class="font-mono !text-xs !min-w-[96px] !max-w-[300px] !w-full !h-[28px] !my-0 !py-0 !border-l-0 !rounded-l-none"
+						on:focus={({ currentTarget }) => {
+							currentTarget.select()
+						}}
+					/>
+				</div>
 			</div>
 			<div class="flex flex-row space-x-2">
+				<FlowImportExportMenu />
+
 				<FlowPreviewButtons />
 				<div class="center-center">
-					<ButtonPopup
+					<Button
 						loading={loadingSave}
-						size="sm"
+						size="xs"
 						startIcon={{ icon: faSave }}
 						on:click={() => saveFlow(false)}
+						{dropdownItems}
 					>
-						<svelte:fragment slot="main">Save</svelte:fragment>
-						<ButtonPopupItem on:click={() => saveFlow(true)}>Save and exit</ButtonPopupItem>
-						{#if initialPath != ''}
-							<ButtonPopupItem
-								on:click={() => {
-									window.open(`/flows/add?template=${initialPath}`)
-								}}>Fork</ButtonPopupItem
-							>
-						{/if}
-					</ButtonPopup>
+						Save
+					</Button>
 				</div>
 			</div>
 		</div>

frontend/src/lib/components/FlowGraphViewer.svelte

@@ -20,10 +20,10 @@
 	}
 	export let overflowAuto = false
 	export let noSide = false
+	export let download = false
 
 	let stepDetail: FlowModule | string | undefined = undefined
 	let codeViewer: Drawer
-	let topHeight = 0
 </script>
 
 <Drawer bind:this={codeViewer} size="900px">
@@ -48,7 +48,7 @@
 					<div class="mt-6">
 						<h3>Code</h3>
 						<iframe
-							class="w-full h-full  text-sm"
+							class="w-full h-full text-sm"
 							title="embedded script from hub"
 							frameborder="0"
 							src="https://hub.windmill.dev/embed/script/{stepDetail.value?.path?.substring(4)}"
@@ -77,6 +77,7 @@
 		class:overflow-auto={overflowAuto}
 	>
 		<FlowGraph
+			{download}
 			minHeight={400}
 			modules={flow?.value?.modules}
 			failureModule={flow?.value?.failure_module}
@@ -141,7 +142,7 @@
 								>
 							</h3>
 							<iframe
-								class="w-full h-full  text-sm"
+								class="w-full h-full text-sm"
 								title="embedded script from hub"
 								frameborder="0"
 								src="https://hub.windmill.dev/embed/script/{stepDetail.value?.path?.substring(4)}"
@@ -154,11 +155,7 @@
 								json={stepDetail.value.iterator.value}
 							/>{:else}
 							<span class="text-xs">
-								<Highlight
-									offsetTop={0}
-									language={typescript}
-									code={cleanExpr(stepDetail.value.iterator.expr)}
-								/>
+								<Highlight language={typescript} code={cleanExpr(stepDetail.value.iterator.expr)} />
 							</span>
 						{/if}</p
 					>

frontend/src/lib/components/FlowPreviewContent.svelte

@@ -13,6 +13,7 @@
 	import FlowProgressBar from './flows/FlowProgressBar.svelte'
 	import CapturePayload from './flows/content/CapturePayload.svelte'
 	import { Loader2 } from 'lucide-svelte'
+	import { getModifierKey } from '$lib/utils'
 
 	let capturePayload: CapturePayload
 	export let previewMode: 'upTo' | 'whole'
@@ -75,7 +76,7 @@
 		if (open) {
 			switch (event.key) {
 				case 'Enter':
-					if (event.ctrlKey) {
+					if (event.ctrlKey || event.metaKey) {
 						event.preventDefault()
 						runPreview($previewArgs)
 					}
@@ -95,12 +96,14 @@
 
 <div class="flex divide-y flex-col space-y-2 h-screen bg-white px-6 py-2 w-full">
 	<div class="flex flex-row justify-between w-full items-center gap-x-2">
-		<button
-			on:click={() => dispatch('close')}
-			class="hover:bg-gray-200 bg-gray-100 rounded-full w-16 h-8 flex items-center justify-center transition-all"
-		>
-			<Icon data={faClose} class="text-gray-500" />
-		</button>
+		<div class="w-8">
+			<button
+				on:click={() => dispatch('close')}
+				class="hover:bg-gray-200 bg-gray-100 rounded-full w-8 h-8 flex items-center justify-center transition-all"
+			>
+				<Icon data={faClose} class="text-gray-500" />
+			</button>
+		</div>
 
 		{#if isRunning}
 			<Button
@@ -132,21 +135,21 @@
 				btnClasses="w-full max-w-lg"
 				on:click={() => runPreview($previewArgs)}
 			>
-				Test flow <Kbd class="ml-2">Ctrl+Enter</Kbd>
+				Test flow <Kbd class="ml-2">{getModifierKey()}+Enter</Kbd>
 			</Button>
 		{/if}
 		<Button
-			btnClasses="h-full"
+			btnClasses="h-full truncate"
 			size="sm"
 			variant="border"
 			on:click={() => {
 				capturePayload.openDrawer()
-			}}>Fill test args from a request</Button
+			}}>Fill args from a request</Button
 		>
 	</div>
 	<FlowProgressBar {job} bind:reset={jobProgressReset} />
 
-	<div class="overflow-y-auto grow  divide-y divide-gray-600 pr-4">
+	<div class="overflow-y-auto grow divide-y divide-gray-600 pr-4">
 		<div class="max-h-1/2 overflow-auto border-b border-gray-700">
 			<SchemaForm
 				noVariablePicker

frontend/src/lib/components/FlowStatusViewer.svelte

@@ -173,6 +173,34 @@
 			return arg == true
 		}
 	}
+
+	function onJobsLoaded(mod: FlowStatusModule, job: Job): void {
+		if (mod.id && (mod.flow_jobs ?? []).length == 0) {
+			if (flowState && flowState[mod.id]) {
+				flowState[mod.id].previewResult = job['result']
+				flowState[mod.id].previewArgs = job.args
+			}
+			if (job.type == 'QueuedJob') {
+				localFlowModuleStates[mod.id] = {
+					type: FlowStatusModule.type.IN_PROGRESS,
+					logs: job.logs,
+					args: job.args,
+					parent_module: mod['parent_module']
+				}
+			} else {
+				localFlowModuleStates[mod.id] = {
+					args: job.args,
+					type: job['success'] ? FlowStatusModule.type.SUCCESS : FlowStatusModule.type.FAILURE,
+					logs: job.logs,
+					result: job['result'],
+					job_id: job.id,
+					parent_module: mod['parent_module'],
+					iteration_total: mod.iterator?.itered?.length
+					// retries: flowState?.raw_flow
+				}
+			}
+		}
+	}
 </script>
 
 {#if job}
@@ -379,49 +407,31 @@
 						<div class="line w-8 h-10" />
 						<li class="w-full border border-gray-600 p-6 space-y-2 bg-blue-50/50">
 							{#if [FlowStatusModule.type.IN_PROGRESS, FlowStatusModule.type.SUCCESS, FlowStatusModule.type.FAILURE].includes(mod.type)}
-								<svelte:self
-									{workspaceId}
-									bind:suspend_status
-									bind:retry_status
-									bind:flowState
-									bind:flowModuleStates={localFlowModuleStates}
-									jobId={mod.job}
-									flowJobIds={mod.flow_jobs
-										? {
-												moduleId: mod.id,
-												flowJobs: mod.flow_jobs
-										  }
-										: undefined}
-									on:jobsLoaded={(e) => {
-										if (mod.id && (mod.flow_jobs ?? []).length == 0) {
-											if (flowState && flowState[mod.id]) {
-												flowState[mod.id].previewResult = e.detail.result
-												flowState[mod.id].previewArgs = e.detail.args
-											}
-											if (e.detail.type == 'QueuedJob') {
-												localFlowModuleStates[mod.id] = {
-													type: FlowStatusModule.type.IN_PROGRESS,
-													logs: e.detail.logs,
-													args: e.detail.args,
-													parent_module: mod['parent_module']
-												}
-											} else {
-												localFlowModuleStates[mod.id] = {
-													args: e.detail.args,
-													type: e.detail.success
-														? FlowStatusModule.type.SUCCESS
-														: FlowStatusModule.type.FAILURE,
-													logs: e.detail.logs,
-													result: e.detail.result,
-													job_id: e.detail.id,
-													parent_module: mod['parent_module'],
-													iteration_total: mod.iterator?.itered?.length
-													// retries: flowState?.raw_flow
-												}
-											}
-										}
-									}}
-								/>
+								{#if job.raw_flow?.modules[i]?.value.type == 'flow'}
+									<svelte:self
+										{workspaceId}
+										jobId={mod.job}
+										bind:suspend_status
+										bind:retry_status
+										on:jobsLoaded={(e) => onJobsLoaded(mod, e.detail)}
+									/>
+								{:else}
+									<svelte:self
+										{workspaceId}
+										bind:suspend_status
+										bind:retry_status
+										bind:flowState
+										bind:flowModuleStates={localFlowModuleStates}
+										jobId={mod.job}
+										flowJobIds={mod.flow_jobs
+											? {
+													moduleId: mod.id,
+													flowJobs: mod.flow_jobs
+											  }
+											: undefined}
+										on:jobsLoaded={(e) => onJobsLoaded(mod, e.detail)}
+									/>
+								{/if}
 							{:else}
 								<ModuleStatus
 									type={mod.type}
@@ -492,7 +502,7 @@
 								<p class="p-2">No arguments</p>
 							{/if}
 						{:else if node}
-							<div class="px-2 flex gap-2 min-w-0 ">
+							<div class="px-2 flex gap-2 min-w-0">
 								<ModuleStatus type={node.type} scheduled_for={node['scheduled_for']} />
 								{#if node.job_id}
 									<div class="truncate"

frontend/src/lib/components/FlowViewer.svelte

@@ -75,7 +75,7 @@
 				{:else}
 					<div class="text-gray-700 text-xs italic mb-4">No inputs</div>
 				{/if}
-				<FlowGraphViewer {noSide} {flow} overflowAuto />
+				<FlowGraphViewer download {noSide} {flow} overflowAuto />
 			</div>
 		</TabContent>
 		<TabContent value="json">

frontend/src/lib/components/FolderEditor.svelte

@@ -19,11 +19,9 @@
 	type Role = 'viewer' | 'writer' | 'admin'
 	let folder: Folder | undefined
 	let perms: { owner_name: string; role: Role }[] | undefined = undefined
-	let managing_folders: string[] = []
 	let usernames: string[] = []
 	let groups: string[] = []
 	let ownerItem: string = ''
-	let folders: string[] = []
 
 	async function loadUsernames(): Promise<void> {
 		usernames = await UserService.listUsernames({ workspace: $workspaceStore! })
@@ -42,14 +40,9 @@
 	async function load() {
 		loadUsernames()
 		loadGroups()
-		loadFolders()
 		await loadFolder()
 	}
 
-	async function loadFolders(): Promise<void> {
-		folders = (await FolderService.listFolders({ workspace: $workspaceStore! })).map((x) => x.name)
-	}
-
 	async function addToFolder() {
 		await GranularAclService.addGranularAcls({
 			workspace: $workspaceStore ?? '',
@@ -67,7 +60,8 @@
 		can_write =
 			$userStore != undefined &&
 			(folder?.owners.includes('u/' + $userStore.username) ||
-				($userStore.is_admin ?? false) || ($userStore.is_super_admin ?? false) ||
+				($userStore.is_admin ?? false) ||
+				($userStore.is_super_admin ?? false) ||
 				$userStore.pgroups.findIndex((x) => folder?.owners.includes(x)) != -1)
 
 		perms = Array.from(
@@ -82,9 +76,6 @@
 				role: getRole(x)
 			}
 		})
-		managing_folders = Object.entries(folder?.extra_perms ?? {})
-			.filter(([k, v]) => k.startsWith('g/') && v)
-			.map(([k, v]) => k)
 	}
 
 	function getRole(x: string): Role {

frontend/src/lib/components/FolderUsageInfo.svelte

@@ -1,5 +1,5 @@
 <script lang="ts">
-	import { FolderService, GroupService } from '$lib/gen'
+	import { FolderService } from '$lib/gen'
 	import { workspaceStore } from '$lib/stores'
 
 	export let name: string

frontend/src/lib/components/GroupEditor.svelte

@@ -1,14 +1,14 @@
 <script lang="ts">
+	import { GranularAclService, GroupService, UserService, type Group } from '$lib/gen'
 	import { userStore, workspaceStore } from '$lib/stores'
-	import { type Group, GroupService, UserService, GranularAclService } from '$lib/gen'
-	import AutoComplete from 'simple-svelte-autocomplete'
-	import TableCustom from './TableCustom.svelte'
 	import { canWrite, sendUserToast } from '$lib/utils'
+	import AutoComplete from 'simple-svelte-autocomplete'
+	import { createEventDispatcher } from 'svelte'
+	import autosize from 'svelte-autosize'
 	import { Button, ToggleButton, ToggleButtonGroup } from './common'
 	import Skeleton from './common/skeleton/Skeleton.svelte'
+	import TableCustom from './TableCustom.svelte'
 	import Tooltip from './Tooltip.svelte'
-	import autosize from 'svelte-autosize'
-	import { createEventDispatcher } from 'svelte'
 
 	export let name: string
 	let can_write = false
@@ -16,10 +16,8 @@
 	type Role = 'member' | 'manager' | 'admin'
 	let group: Group | undefined
 	let members: { member_name: string; role: Role }[] | undefined = undefined
-	let managing_groups: string[] = []
 	let usernames: string[] | undefined = []
 	let username: string = ''
-	let groups: string[] = []
 
 	const dispatch = createEventDispatcher()
 
@@ -34,15 +32,10 @@
 	}
 
 	async function load() {
-		loadGroups()
 		await loadGroup()
 		loadUsernames()
 	}
 
-	async function loadGroups(): Promise<void> {
-		groups = (await GroupService.listGroups({ workspace: $workspaceStore! })).map((x) => x.name)
-	}
-
 	async function addToGroup() {
 		await GroupService.addUserToGroup({
 			workspace: $workspaceStore ?? '',
@@ -68,9 +61,6 @@
 				role: getRole(x)
 			}
 		})
-		managing_groups = Object.entries(group?.extra_perms ?? {})
-			.filter(([k, v]) => k.startsWith('g/') && v)
-			.map(([k, v]) => k)
 	}
 
 	function getRole(x: string): Role {

frontend/src/lib/components/HighlightCode.svelte

@@ -27,7 +27,7 @@
 </script>
 
 {#if code?.length < 5000}
-	<Highlight className="nowrap {$$props.class}" language={lang} {code} />
+	<Highlight class="nowrap {$$props.class}" language={lang} {code} />
 {:else}
 	<pre class="overflow-auto max-h-screen {$$props.class}"
 		><code class="language-{language}">{code}</code></pre

frontend/src/lib/components/InlineCodeCopy.svelte

@@ -6,7 +6,8 @@
 	export let content: string
 </script>
 
-<pre class="bg-gray-700 text-gray-100 p-2  font-mono text-sm whitespace-pre-wrap"
+<!-- svelte-ignore a11y-click-events-have-key-events -->
+<pre class="bg-gray-700 text-gray-100 p-2 font-mono text-sm whitespace-pre-wrap"
 	>{content} <span on:click={() => copyToClipboard(content)} class="cursor-pointer ml-2"
 		><Icon data={faClipboard} /></span
 	></pre

frontend/src/lib/components/InputTransformForm.svelte

@@ -15,7 +15,6 @@
 	import type ItemPicker from './ItemPicker.svelte'
 	import type { InputTransform } from '$lib/gen'
 	import TemplateEditor from './TemplateEditor.svelte'
-	import Tooltip from './Tooltip.svelte'
 	import { setInputCat as computeInputCat } from '$lib/utils'
 
 	export let schema: Schema
@@ -40,17 +39,24 @@
 		schema.properties[argName].enum,
 		schema.properties[argName].contentEncoding
 	)
+
 	let propertyType = getPropertyType(arg)
 
 	function getPropertyType(arg: InputTransform | any): 'static' | 'javascript' {
 		let type: 'static' | 'javascript' = arg?.type ?? 'static'
-		if (type == 'javascript') {
+
+		if (
+			type == 'javascript' &&
+			isStaticTemplate(inputCat) &&
+			(arg?.expr?.length === 0 || arg?.expr?.[0] === '`')
+		) {
 			const newValue = codeToStaticTemplate(arg.expr)
 			if (newValue) {
 				type = 'static'
 				arg.value = newValue
 			}
 		}
+
 		return type
 	}
 
@@ -112,8 +118,6 @@
 	const { focusProp, propPickerConfig } = getContext<PropPickerWrapperContext>('PropPickerWrapper')
 
 	$: isStaticTemplate(inputCat) && propertyType == 'static' && setPropertyType(arg?.value)
-	const openBracket = '${'
-	const closeBracket = '}'
 
 	function setDefaultCode() {
 		if (!arg?.value) {
@@ -148,86 +152,89 @@
 		</div>
 		{#if !noDynamicToggle}
 			<div class="flex flex-row gap-x-4 gap-y-1 flex-wrap z-10">
-				<ToggleButtonGroup
-					bind:selected={propertyType}
-					on:selected={(e) => {
-						const staticTemplate = isStaticTemplate(inputCat)
-						if (e.detail === 'javascript') {
-							if (arg.expr == undefined) {
-								arg.expr = getDefaultExpr(
-									argName,
-									previousModuleId,
-									staticTemplate
-										? `\`${arg?.value?.toString().replaceAll('`', '\\`') ?? ''}\``
-										: arg.value
-										? JSON.stringify(arg?.value, null, 4)
-										: ''
-								)
-							}
-							if (arg) {
-								arg.value = undefined
-							}
-							propertyType = 'javascript'
-							arg.type = 'javascript'
-						} else {
-							if (staticTemplate) {
-								if (arg) {
-									arg.value = codeToStaticTemplate(arg.expr)
-									arg.expr = undefined
+				<div>
+					<ToggleButtonGroup
+						bind:selected={propertyType}
+						on:selected={(e) => {
+							const staticTemplate = isStaticTemplate(inputCat)
+							if (e.detail === 'javascript') {
+								if (arg.expr == undefined) {
+									arg.expr = getDefaultExpr(
+										argName,
+										previousModuleId,
+										staticTemplate
+											? `\`${arg?.value?.toString().replaceAll('`', '\\`') ?? ''}\``
+											: arg.value
+											? JSON.stringify(arg?.value, null, 4)
+											: ''
+									)
 								}
-								setPropertyType(arg?.value)
-							} else {
 								if (arg) {
-									arg.type = 'static'
 									arg.value = undefined
-									arg.expr = undefined
+									arg.type = 'javascript'
 								}
+								propertyType = 'javascript'
+							} else {
+								if (staticTemplate) {
+									if (arg) {
+										arg.value = codeToStaticTemplate(arg.expr)
+										arg.expr = undefined
+									}
+									setPropertyType(arg?.value)
+								} else {
+									if (arg) {
+										arg.type = 'static'
+										arg.value = undefined
+										arg.expr = undefined
+									}
+								}
+								propertyType = 'static'
 							}
-							propertyType = 'static'
-						}
-					}}
-				>
-					{#if isStaticTemplate(inputCat)}
-						<ToggleButton light position="left" value="static" size="xs">
-							{'${} '}&nbsp;
-							<Tooltip
-								>Write text or surround javascript with "{openBracket}" and "{closeBracket}". Use
-								`result` to connect to another node's output.
-							</Tooltip></ToggleButton
-						>
-					{:else}
-						<ToggleButton light position="left" value="static" size="xs">Static</ToggleButton>
-					{/if}
-
-					<ToggleButton
-						light
-						position="right"
-						value="javascript"
-						startIcon={{ icon: faCode }}
-						size="xs"
-						>&nbsp;<Tooltip
-							>Write javascript expressions directly, using 'flow_input' or 'result'. You can use
-							multiline javascript.
-						</Tooltip></ToggleButton
+						}}
 					>
-				</ToggleButtonGroup>
-				<Button
-					variant="contained"
-					color="blue"
-					size="xs"
-					on:click={() => {
-						focusProp(argName, 'connect', (path) => {
-							connectProperty(path)
-							return true
-						})
-					}}>Connect &rightarrow;</Button
-				>
+						{#if isStaticTemplate(inputCat)}
+							<ToggleButton
+								title={`Write text or surround javascript with \`\$\{\` and \`\}\`. Use \`result\` to connect to another node\'s output.`}
+								light
+								position="left"
+								value="static"
+								size="xs2"
+							>
+								{'${} '}&nbsp;
+							</ToggleButton>
+						{:else}
+							<ToggleButton light position="left" value="static" size="xs2">Static</ToggleButton>
+						{/if}
+
+						<ToggleButton
+							light
+							title="Write javascript expressions directly, using 'flow_input' or 'result'. You can use multiline javascript."
+							position="right"
+							value="javascript"
+							startIcon={{ icon: faCode }}
+							size="xs2"
+						/>
+					</ToggleButtonGroup>
+				</div>
+				<div>
+					<Button
+						title="Connect to another node's output"
+						variant="border"
+						color="dark"
+						size="xs2"
+						on:click={() => {
+							focusProp(argName, 'connect', (path) => {
+								connectProperty(path)
+								return true
+							})
+						}}>Link &rightarrow;</Button
+					>
+				</div>
 			</div>
 		{/if}
 	</div>
 
 	<div class="max-w-xs" />
-
 	<div
 		class="relative mb-6 {$propPickerConfig?.propName == argName
 			? 'outline outline-offset-0 outline-2 outline-blue-500 rounded-md'
@@ -241,7 +248,7 @@
 			</span>
 		{/if}
 		{#if isStaticTemplate(inputCat) && propertyType == 'static' && !noDynamicToggle}
-			<div class="py-1 mt-2 min-h-[28px] rounded border border-1 border-gray-500">
+			<div class="mt-2 min-h-[28px] rounded border border-1 border-gray-500">
 				{#if arg}
 					<TemplateEditor
 						bind:this={monacoTemplate}

frontend/src/lib/components/InputTransformSchemaForm.svelte

@@ -55,7 +55,7 @@
 
 <div class="w-full {clazz}">
 	{#if keys.length > 0}
-		{#each keys as argName, i (argName)}
+		{#each keys as argName (argName)}
 			{#if (!filter || filter.includes(argName)) && Object.keys(schema.properties ?? {}).includes(argName)}
 				<div class="z-10">
 					<InputTransformForm

frontend/src/lib/components/InputTransformsViewer.svelte

@@ -15,7 +15,7 @@
 					json={val.value}
 				/>{:else}
 				<span class="text-xs">
-					<Highlight offsetTop={0} language={typescript} code={cleanExpr(val.expr)} />
+					<Highlight language={typescript} code={cleanExpr(val.expr)} />
 				</span>
 			{/if}
 		</li>

frontend/src/lib/components/LightweightArgInput.svelte

@@ -308,11 +308,23 @@
 								: 'border border-red-700 border-opacity-30 focus:border-red-700 focus:border-opacity-30 bg-red-100'}"
 							placeholder={defaultValue ?? ''}
 							bind:value
+							on:pointerdown|stopPropagation={(e) => {
+								dispatch('inputClicked', e)
+							}}
 						/>
 					</div>
 				</div>
 			{/if}
 			<slot name="actions" />
 		</div>
+		{#if error && error != ''}
+			<div class="text-right text-xs text-red-600">
+				{#if error === ''}
+					&nbsp;
+				{:else}
+					{error}
+				{/if}
+			</div>
+		{/if}
 	</div>
 </div>

frontend/src/lib/components/LightweightSchemaForm.svelte

@@ -11,7 +11,7 @@
 </script>
 
 <div class="w-full">
-	{#each Object.keys(schema.properties ?? {}) as argName, i (argName)}
+	{#each Object.keys(schema.properties ?? {}) as argName (argName)}
 		<div>
 			{#if typeof args == 'object' && schema?.properties[argName] && args}
 				<LightweightArgInput
@@ -28,6 +28,7 @@
 					properties={schema.properties[argName].properties}
 					itemsType={schema.properties[argName].items}
 					extra={schema.properties[argName]}
+					on:inputClicked
 				/>
 			{/if}
 		</div>