parallelize more stuff

.env

@@ -3,5 +3,5 @@ DB_PASSWORD=changeme
 # this is the url that your instance is publicly exposed to
 WM_BASE_URL=http://localhost
 
-# this is the addres caddy will listen on
-CADDY_REVERSE_PROXY=":80"
+# this is the url that caddy will reverse proxy from. It might be different than WM_BASE_URL if you re using a second proxy/load balancer in front
+CADDY_REVERSE_PROXY=http://localhost

.github/workflows/backend-test.yml

@@ -40,4 +40,4 @@ jobs:
             backend -> target
       - name: cargo test
         timeout-minutes: 10
-        run: mkdir frontend/build && cd backend && touch windmill-api/openapi-deref.yaml && DATABASE_URL=postgres://postgres:changeme@postgres:5432/windmill cargo test --all -- --nocapture
+        run: mkdir frontend/build && cd backend && touch windmill-api/openapi-deref.yaml && DATABASE_URL=postgres://postgres:changeme@postgres:5432/windmill DISABLE_NSJAIL=false cargo test --all -- --nocapture

.github/workflows/frontend-check.yml

@@ -4,8 +4,6 @@ on:
     types: [opened,synchronize,reopened,closed]
     paths:
       - "frontend/**"
-  merge_group:
-
 jobs:
   npm_check:
     runs-on: ubuntu-latest

CHANGELOG.md

@@ -1,183 +1,6 @@
 # Changelog
 
 
-## [1.93.0](https://github.com/windmill-labs/windmill/compare/v1.92.2...v1.93.0) (2023-05-03)
-
-
-### Features
-
-* **frontend:** add recompute others to background scripts ([392d0f8](https://github.com/windmill-labs/windmill/commit/392d0f8b876c9b587fe85421098f3eceb8a74dec))
-
-
-### Bug Fixes
-
-* **frontend:** deploy path for apps ([7ac9677](https://github.com/windmill-labs/windmill/commit/7ac96771a5c3d44234c790e8cea3d621d8c1d00e))
-
-## [1.92.2](https://github.com/windmill-labs/windmill/compare/v1.92.1...v1.92.2) (2023-05-02)
-
-
-### Bug Fixes
-
-* **go-client:** fix go-client gen ([82c4d66](https://github.com/windmill-labs/windmill/commit/82c4d6629e134f00389c87a948c52878e5a3f4f5))
-
-## [1.92.1](https://github.com/windmill-labs/windmill/compare/v1.92.0...v1.92.1) (2023-05-02)
-
-
-### Bug Fixes
-
-* **go-client:** fix go-client gen ([df333d9](https://github.com/windmill-labs/windmill/commit/df333d9739f601714f7a0124f47422dfb113d320))
-
-## [1.92.0](https://github.com/windmill-labs/windmill/compare/v1.91.0...v1.92.0) (2023-05-02)
-
-
-### Features
-
-* **frontend:** add labels as table action ([64065c1](https://github.com/windmill-labs/windmill/commit/64065c17f305fb4c7c078c7fa4935d5423da8f66))
-* **frontend:** add labels as table action ([2ab1714](https://github.com/windmill-labs/windmill/commit/2ab1714dfa7cffc46b4b6aa40dabdd92c5a6270f))
-* **frontend:** allow running eval in every field ([62acbb5](https://github.com/windmill-labs/windmill/commit/62acbb5ab3a0727b306e25a80b74ac8216619501))
-* **frontend:** background script can run script and flows ([#1515](https://github.com/windmill-labs/windmill/issues/1515)) ([607c803](https://github.com/windmill-labs/windmill/commit/607c803be91921b53f329a2c2c3c129ce53d6c0c))
-
-
-### Bug Fixes
-
-* **frontend:** fix small ui issues ([#1513](https://github.com/windmill-labs/windmill/issues/1513)) ([f6ff8ca](https://github.com/windmill-labs/windmill/commit/f6ff8ca232f5725f86a36379956da2731def2580))
-
-## [1.91.0](https://github.com/windmill-labs/windmill/compare/v1.90.0...v1.91.0) (2023-05-01)
-
-
-### Features
-
-* add drafts for apps ([f7374c8](https://github.com/windmill-labs/windmill/commit/f7374c8204f85b4371e61f34dcd4b66857c0f8ab))
-* introduce backend persisted draft systems for scripts ([88e37fe](https://github.com/windmill-labs/windmill/commit/88e37fe0bed58f396690622e925d5e078c60140c))
-* introduce draft for flows ([a196642](https://github.com/windmill-labs/windmill/commit/a1966427e893dc8a58c8f2862ded752884843813))
-
-## [1.90.0](https://github.com/windmill-labs/windmill/compare/v1.89.0...v1.90.0) (2023-04-28)
-
-
-### Features
-
-* **backend:** add EXIT_AFTER_NO_JOB_FOR_SECS for ephemeral workers ([de9abd1](https://github.com/windmill-labs/windmill/commit/de9abd129db13dcdf0e69e2c1e2d3aa558fb783a))
-* **backend:** add JOB_RETENTION_SECS to delete completed jobs completed after expiration period ([0b7bad3](https://github.com/windmill-labs/windmill/commit/0b7bad3816e61841ef4765db7881227274c20b23))
-* **backend:** expose tag in the job  ([#1486](https://github.com/windmill-labs/windmill/issues/1486)) ([324d4f5](https://github.com/windmill-labs/windmill/commit/324d4f5e9e89e6de600e882f23bf545c0b1dc539))
-* **frontend:** adapt style ([#1488](https://github.com/windmill-labs/windmill/issues/1488)) ([41a24ec](https://github.com/windmill-labs/windmill/commit/41a24ecd36d9cc537cbd1dd0cd1de6f689be1b8c))
-* **frontend:** add an eval input component for flow ([#1494](https://github.com/windmill-labs/windmill/issues/1494)) ([2815f1e](https://github.com/windmill-labs/windmill/commit/2815f1ec71177bb6e89d0d62a8df89030d37b1fc))
-* **frontend:** Add new integration icons ([#1479](https://github.com/windmill-labs/windmill/issues/1479)) ([7adacd4](https://github.com/windmill-labs/windmill/commit/7adacd4c9f03d17b47abc515bc391e348a7e6ec1))
-* **frontend:** refactor inline script ([#1480](https://github.com/windmill-labs/windmill/issues/1480)) ([05c837e](https://github.com/windmill-labs/windmill/commit/05c837e64f61bfb22ae1f80263deb1c879030985))
-* **frontend:** Schedules run now ([#1475](https://github.com/windmill-labs/windmill/issues/1475)) ([47f0f35](https://github.com/windmill-labs/windmill/commit/47f0f35236e02958f6bc00b5652e06e25eabeaf5))
-* **frontend:** Small style fix ([#1473](https://github.com/windmill-labs/windmill/issues/1473)) ([7ad496a](https://github.com/windmill-labs/windmill/commit/7ad496ad3f746ffd782856a35c1456999792fa94))
-* **frontend:** Support TS union type with a select field ([#1457](https://github.com/windmill-labs/windmill/issues/1457)) ([8b76324](https://github.com/windmill-labs/windmill/commit/8b763249cb1360c122cc81d50a1a95d1ad3ddd5b))
-
-
-### Bug Fixes
-
-* **frontend:** Allow 0 as select default value ([#1474](https://github.com/windmill-labs/windmill/issues/1474)) ([d8529ff](https://github.com/windmill-labs/windmill/commit/d8529ff3ed6168de60bb24626a3e36ab4beae15c))
-* **frontend:** close the modal before deleting a form modal ([#1484](https://github.com/windmill-labs/windmill/issues/1484)) ([430c733](https://github.com/windmill-labs/windmill/commit/430c73399b7e1524ee81d0ce8d7d2eaf16117f9a))
-* **frontend:** fix apply connection ([#1487](https://github.com/windmill-labs/windmill/issues/1487)) ([cf59cc0](https://github.com/windmill-labs/windmill/commit/cf59cc04efb853fe06a23042115128689b5d26ee))
-* **frontend:** Fix frontend script ([#1476](https://github.com/windmill-labs/windmill/issues/1476)) ([b60a7f6](https://github.com/windmill-labs/windmill/commit/b60a7f63d04b1fc851479060e77613c77d20198a))
-* **frontend:** fix recomputa all ([#1491](https://github.com/windmill-labs/windmill/issues/1491)) ([fb05a09](https://github.com/windmill-labs/windmill/commit/fb05a09955f937000fbba0826c03c52e65aa146e))
-* **frontend:** Flow editor design updates ([#1477](https://github.com/windmill-labs/windmill/issues/1477)) ([50d814c](https://github.com/windmill-labs/windmill/commit/50d814c3dc55841b16cd1df8ae86d021de4e880c))
-* **frontend:** Minor app editor updates ([#1458](https://github.com/windmill-labs/windmill/issues/1458)) ([8fd10b1](https://github.com/windmill-labs/windmill/commit/8fd10b1f5813b3f4200a980de4a85e7d701660a7))
-* **frontend:** register applyConnection as a callback to remove unnecessary reactivit ([#1485](https://github.com/windmill-labs/windmill/issues/1485)) ([d915f6b](https://github.com/windmill-labs/windmill/commit/d915f6b004ea7cc27a3c55b3504df902f5db1aef))
-* **frontend:** reset ui job loading state when submitting preview job triggers error ([#1483](https://github.com/windmill-labs/windmill/issues/1483)) ([6f8616f](https://github.com/windmill-labs/windmill/commit/6f8616fb273b1f4a1489878a617d239f30ecb1c0))
-* **frontend:** Update CLI login request styling ([#1454](https://github.com/windmill-labs/windmill/issues/1454)) ([c77393c](https://github.com/windmill-labs/windmill/commit/c77393c15444868308b44a72bee89e49fc23d80f))
-* **frontend:** Update direct exports ([#1456](https://github.com/windmill-labs/windmill/issues/1456)) ([4a2af13](https://github.com/windmill-labs/windmill/commit/4a2af1359ee29001236591f685cacfa9df6715df))
-
-## [1.89.0](https://github.com/windmill-labs/windmill/compare/v1.88.1...v1.89.0) (2023-04-23)
-
-
-### Features
-
-* **backend:** global cache refactor for pip using tar for each dependency ([#1443](https://github.com/windmill-labs/windmill/issues/1443)) ([369dd0d](https://github.com/windmill-labs/windmill/commit/369dd0dac61e5430856ed9abf7129bbad3b75860))
-* **backend:** only run fully deployed scripts ([3d031c7](https://github.com/windmill-labs/windmill/commit/3d031c701705459f418b11d2ca83e71943e4079b))
-* **backend:** worker groups ([#1452](https://github.com/windmill-labs/windmill/issues/1452)) ([722783f](https://github.com/windmill-labs/windmill/commit/722783f7f630e3123ffd2605deb21a915188bd20))
-* **backend:** workers are instantly ready and sync with global cache in background ([670ba51](https://github.com/windmill-labs/windmill/commit/670ba51d9bd9a9a0b07a2ed064c316234bb5819d))
-* **ee:** sync cache in background ([c919827](https://github.com/windmill-labs/windmill/commit/c919827cf8eb9437b6d9bd57b3d3ad883a66de3b))
-* **ee:** sync cache in background ([0e77e37](https://github.com/windmill-labs/windmill/commit/0e77e37fbddbb517d5e1b1f07a27a40b63371439))
-* **frontend:** Add documentation links ([#1399](https://github.com/windmill-labs/windmill/issues/1399)) ([36acbf7](https://github.com/windmill-labs/windmill/commit/36acbf793b6714dffe4dbb0e2501b9438f034858))
-* **frontend:** Add seconds input ([#1445](https://github.com/windmill-labs/windmill/issues/1445)) ([30bf7ad](https://github.com/windmill-labs/windmill/commit/30bf7ad3e9785420b4dcd814ce3c7a444d23cc9f))
-* **frontend:** add toast actions ([#1411](https://github.com/windmill-labs/windmill/issues/1411)) ([d173232](https://github.com/windmill-labs/windmill/commit/d17323286a05aa0b1680ef94d7058d2c8902782f))
-* **frontend:** reorder array items in app editor ([#1426](https://github.com/windmill-labs/windmill/issues/1426)) ([3615fb2](https://github.com/windmill-labs/windmill/commit/3615fb26fb91d64626d82ace6e6275e424ece832))
-* **frontend:** support showing metadata on script add via query param ([#1438](https://github.com/windmill-labs/windmill/issues/1438)) ([3c98452](https://github.com/windmill-labs/windmill/commit/3c98452f50913ef639eba96996f3a6c80508bd63))
-
-
-### Bug Fixes
-
-* **backend:** avoid potential conflict between pull from tar and background sync ([d76e907](https://github.com/windmill-labs/windmill/commit/d76e90757e209263da7f79fa85052969e7efd63d))
-* **backend:** global cache synco only start if all piptars have been downloaded ([5f8a730](https://github.com/windmill-labs/windmill/commit/5f8a730fdfbb9e3d518555f7272a3bb297725f28))
-* **frontend:** App color picker overflow issue ([#1449](https://github.com/windmill-labs/windmill/issues/1449)) ([32903d2](https://github.com/windmill-labs/windmill/commit/32903d2839a082d53168bb9177fc88f6ab0ec482))
-* **frontend:** fix copy content button width ([#1428](https://github.com/windmill-labs/windmill/issues/1428)) ([d96d4a5](https://github.com/windmill-labs/windmill/commit/d96d4a524edebea65bc602194c8f11f5d69e920a))
-* **frontend:** Minor update of app default codes ([#1440](https://github.com/windmill-labs/windmill/issues/1440)) ([fe75aa1](https://github.com/windmill-labs/windmill/commit/fe75aa18f2f27745db35329aa60938694640a8c6))
-* **frontend:** Update app default codes ([#1432](https://github.com/windmill-labs/windmill/issues/1432)) ([c8acfbc](https://github.com/windmill-labs/windmill/commit/c8acfbc1ff0f6c23e5a2229ca83a3b09eec826c3))
-* **frontend:** Update app mobile preview width ([#1431](https://github.com/windmill-labs/windmill/issues/1431)) ([1764613](https://github.com/windmill-labs/windmill/commit/17646130bcf8cf646a4ccdfa39f9a8791876a137))
-* **frontend:** Update flow tooltip z-indexes ([#1433](https://github.com/windmill-labs/windmill/issues/1433)) ([17cb8fc](https://github.com/windmill-labs/windmill/commit/17cb8fc3fa0b39a9750e61ca2731a15bbda690ec))
-* **frontend:** Update flow viewer styling ([#1441](https://github.com/windmill-labs/windmill/issues/1441)) ([46a29b5](https://github.com/windmill-labs/windmill/commit/46a29b5d27b8d9f7ea38c1063fc081ed5933db5d))
-
-## [1.88.1](https://github.com/windmill-labs/windmill/compare/v1.88.0...v1.88.1) (2023-04-18)
-
-
-### Bug Fixes
-
-* **frontend:** fix hub list ([1144329](https://github.com/windmill-labs/windmill/commit/1144329972fb61e2df62873ca1e485c88fabc478))
-
-## [1.88.0](https://github.com/windmill-labs/windmill/compare/v1.87.0...v1.88.0) (2023-04-17)
-
-
-### Features
-
-* **backend:** install python scripts on save ([cb7e686](https://github.com/windmill-labs/windmill/commit/cb7e686dd95397d5b37edd5aac50b6d1429c4a71))
-* **frontend:** Add runs preview popup ([#1405](https://github.com/windmill-labs/windmill/issues/1405)) ([4ab023f](https://github.com/windmill-labs/windmill/commit/4ab023f95085958ab1ad01dc249d308c7ebf423e))
-* **frontend:** cancellable inline script editor run ([e828d26](https://github.com/windmill-labs/windmill/commit/e828d2673e62e95e5e1235eeca8107ac7cfb7e45))
-* **frontend:** Remove gap when button label is empty ([#1402](https://github.com/windmill-labs/windmill/issues/1402)) ([568f59e](https://github.com/windmill-labs/windmill/commit/568f59eefb104047b8ef063f273fe238075d6407))
-* **frontend:** Unify main lists ([#1406](https://github.com/windmill-labs/windmill/issues/1406)) ([48bbbd0](https://github.com/windmill-labs/windmill/commit/48bbbd0e872a12ed1c562a6d14967a2a0f7c4735))
-* **frontend:** Update airtable instructions ([#1403](https://github.com/windmill-labs/windmill/issues/1403)) ([7dc7ece](https://github.com/windmill-labs/windmill/commit/7dc7ecef55b465fc096f71fc9de5c8b543136ff7))
-* inputs library on run page ([92a2934](https://github.com/windmill-labs/windmill/commit/92a293488e8e58350229931ab69f7924d58474be))
-
-
-### Bug Fixes
-
-* **backend:** deno uses --no-check ([a5499c2](https://github.com/windmill-labs/windmill/commit/a5499c26f3ebd8b07541a7e0cbf33a7008a8f476))
-* **backend:** do not fail on schedule not existing anymore ([a5f6d73](https://github.com/windmill-labs/windmill/commit/a5f6d73f7d53d7af9d285a85460509763263c508))
-* **frontend:** Fix app file uploads ([#1408](https://github.com/windmill-labs/windmill/issues/1408)) ([ac489ac](https://github.com/windmill-labs/windmill/commit/ac489ac2da0fbf01f5e2877612c14cfaf1ef79c2))
-* **frontend:** fix buttons width ([#1407](https://github.com/windmill-labs/windmill/issues/1407)) ([75a0482](https://github.com/windmill-labs/windmill/commit/75a0482ef046dd7e30f6d6039dbc66880182dc5e))
-* **frontend:** fix enum sync ([#1410](https://github.com/windmill-labs/windmill/issues/1410)) ([98060ce](https://github.com/windmill-labs/windmill/commit/98060ce55d5efa59a8989cf9357935976d57650b))
-* **frontend:** Handle scheduled runs in preview ([#1413](https://github.com/windmill-labs/windmill/issues/1413)) ([accdc1a](https://github.com/windmill-labs/windmill/commit/accdc1ac59ce9611f66567222a73995d3c0a3f9d))
-* **frontend:** Keep selected tab during renaming ([#1409](https://github.com/windmill-labs/windmill/issues/1409)) ([82cd048](https://github.com/windmill-labs/windmill/commit/82cd048ef4d08f31660f6f31a96940676a28996c))
-* **frontend:** Queued-running jobs preview ([#1414](https://github.com/windmill-labs/windmill/issues/1414)) ([b2a40a0](https://github.com/windmill-labs/windmill/commit/b2a40a05805344c1c34f2ba917b4cdd52dfffc3f))
-* **frontend:** Remove output when deleting a component ([#1397](https://github.com/windmill-labs/windmill/issues/1397)) ([6aa1008](https://github.com/windmill-labs/windmill/commit/6aa100893352870d5a99fdd56d7f1425a221a273))
-
-## [1.87.0](https://github.com/windmill-labs/windmill/compare/v1.86.0...v2.0.0) (2023-04-11)
-
-
-### ⚠ BREAKING CHANGES
-
-* **frontend:** Add option to return file names ([#1380](https://github.com/windmill-labs/windmill/issues/1380))
-
-### Features
-
-* **backend:** add instance events webhook ([f2d3c82](https://github.com/windmill-labs/windmill/commit/f2d3c8208b6daa49f304f355752145de47138a3c))
-* **backend:** extend cached resolution for go ([dac61d1](https://github.com/windmill-labs/windmill/commit/dac61d1c982576d7589e16ab01c8cc8bad6e1686))
-* **backend:** Redis based queue ([#1324](https://github.com/windmill-labs/windmill/issues/1324)) ([d45e6c9](https://github.com/windmill-labs/windmill/commit/d45e6c94abed609357b18d4daa7de6b2ea0ba978))
-* **frontend:** Add option to return file names ([#1380](https://github.com/windmill-labs/windmill/issues/1380)) ([3dabac1](https://github.com/windmill-labs/windmill/commit/3dabac153f302f48210d15ebaec514e72717300f))
-* **python:** cache dependency resolution ([facb670](https://github.com/windmill-labs/windmill/commit/facb67093ce7d3b0874d0d559fb272ed822ce360))
-
-
-### Bug Fixes
-
-* **backend:** nested deno relative imports ([955a213](https://github.com/windmill-labs/windmill/commit/955a213a504c1f3b8811c930823e87fe7dba101a))
-* **cli:** overwrite archived scripts ([1f705ca](https://github.com/windmill-labs/windmill/commit/1f705cab2ce8c79829f22fc6af9e06ecba7450b1))
-* **frontend:** Add missing stopPropagation ([#1394](https://github.com/windmill-labs/windmill/issues/1394)) ([58d4b55](https://github.com/windmill-labs/windmill/commit/58d4b556ebbd76c6f07f1a16d601a9d824b99f7e))
-* **frontend:** fix app init issue ([d0e0e1f](https://github.com/windmill-labs/windmill/commit/d0e0e1fdf27d9a7fb86c66e43398786b64d8b6b7))
-* **frontend:** Fix frontend dependencies ([#1379](https://github.com/windmill-labs/windmill/issues/1379)) ([8e9c491](https://github.com/windmill-labs/windmill/commit/8e9c49165060a4a7f831b8be075593f89d867784))
-* **frontend:** Fix icon picker input ([#1389](https://github.com/windmill-labs/windmill/issues/1389)) ([8a44f8e](https://github.com/windmill-labs/windmill/commit/8a44f8e7796f13698e2a99af9f3772f5e676604b))
-* **frontend:** Fix mac shortcuts ([#1381](https://github.com/windmill-labs/windmill/issues/1381)) ([41831d5](https://github.com/windmill-labs/windmill/commit/41831d58ed593bb283600b76170f6e76783e0eae))
-* **frontend:** fix popover configuration to avoid content shift ([#1377](https://github.com/windmill-labs/windmill/issues/1377)) ([2031e1e](https://github.com/windmill-labs/windmill/commit/2031e1ebd0dc020da104ee84a0294c86babcefaf))
-* **frontend:** remove stopPropagation that was preventing components dnd ([#1378](https://github.com/windmill-labs/windmill/issues/1378)) ([de8dc1e](https://github.com/windmill-labs/windmill/commit/de8dc1e9cd7beea2ce62656e9e7676214f77a110))
-
-
-### Performance Improvements
-
-* parallelize more operations for deno jobs ([e911869](https://github.com/windmill-labs/windmill/commit/e911869d990956463834ac9ff35c52ba8236e362))
-
 ## [1.86.0](https://github.com/windmill-labs/windmill/compare/v1.85.0...v1.86.0) (2023-04-08)
 
 

Dockerfile

@@ -39,7 +39,7 @@ WORKDIR /windmill
 ENV SQLX_OFFLINE=true
 ENV CARGO_INCREMENTAL=1
 
-FROM node:20-alpine as frontend
+FROM node:19-alpine as frontend
 
 # install dependencies
 WORKDIR /frontend
@@ -85,7 +85,7 @@ COPY .git/ .git/
 RUN CARGO_NET_GIT_FETCH_WITH_CLI=true cargo build --release --features "$features"
 
 
-FROM python:3.11.3-slim-buster
+FROM python:3.11.2-slim-buster
 ARG TARGETPLATFORM
 
 ARG APP=/usr/src/app

README.md

@@ -5,7 +5,7 @@
     <em>.</em>
 </p>
 <p align=center>
-Open-source developer infrastructure for internal tools. Self-hostable alternative to Airplane, Pipedream, Superblocks and a simplified Temporal with autogenerated UIsm and custom UIs to trigger workflows and scripts as internal apps. Scripts are turned into UIs and no-code modules, no-code modules can be composed into very rich flows, and script and flows can be triggered from internal UIs made with a low-code builder. The script languages supported are: Python, Typescript, Go, Bash, SQL.
+Open-source developer infrastructure for internal tools. Self-hostable alternative to Airplane, Pipedream, Superblocks and a simplified Temporal with autogenerated UIs to trigger workflows and scripts as internal apps. Scripts are turned into UIs and no-code modules, no-code modules can be composed into very rich flows, and script and flows can be triggered from internal UIs made with a low-code builder. The script languages supported are: Python, Typescript, Go, Bash, SQL.
 </p>
 
 <p align="center">
@@ -70,10 +70,10 @@ https://user-images.githubusercontent.com/275584/218350457-bc2fdc3b-e667-4da5-a2
 
 3. Make it flow! You can chain your scripts or scripts made by the community
    shared on [WindmillHub](https://hub.windmill.dev).
-   ![Step 3](./imgs/windmill-flow.png)
+   ![Step 4](./imgs/windmill-flow.png)
 
 4. Build complex UI on top of your scripts and flows.
-   ![Step 4](./imgs/windmill-builder.png)
+   ![Step 5](./imgs/windmill-builder.png)
 
 Scripts and flows can also be triggered by a cron schedule '_/5 _ \* \* \*' or
 through webhooks.
@@ -123,7 +123,7 @@ instance from local commands. See
 ### Running scripts locally
 
 You can run your script locally easily, you simply need to pass the right
-environment variables for the `wmill` client library to fetch resources and
+environment variables for the `wmill` client library to fetch resource and
 variables from your instance if necessary. See more:
 <https://docs.windmill.dev/docs/advanced/local_development/>
 
@@ -187,9 +187,9 @@ compiling from source or using without a postgres super user, see
 ### Docker compose
 
 ```
-curl https://raw.githubusercontent.com/windmill-labs/windmill/main/docker-compose.yml -o docker-compose.yml
-curl https://raw.githubusercontent.com/windmill-labs/windmill/main/Caddyfile -o Caddyfile
-curl https://raw.githubusercontent.com/windmill-labs/windmill/main/.env -o .env
+curl https://github.com/windmill-labs/windmill/blob/main/docker-compose.yml -o docker-compose.yml
+curl https://github.com/windmill-labs/windmill/blob/main/CaddyFile -o Caddyfile
+curl https://github.com/windmill-labs/windmill/blob/main/.env -o .env
 
 docker compose up -d --pull always
 ```
@@ -288,61 +288,51 @@ it being synced automatically everyday.
 
 ## Environment Variables
 
-| Environment Variable name           | Default                                    | Description                                                                                                                                                                                        | Api Server/Worker/All |
-| ----------------------------------- | ------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------- |
-| DATABASE_URL                        |                                            | The Postgres database url.                                                                                                                                                                         | All                   |
-| DISABLE_NSJAIL                      | true                                       | Disable Nsjail Sandboxing                                                                                                                                                                          | Worker                |
-| SERVER_BIND_ADDR                    | 0.0.0.0                                    | IP Address on which to bind listening socket                                                                                                                                                       | Server                |
-| PORT                                | 8000                                       | Exposed port                                                                                                                                                                                       | Server                |
-| NUM_WORKERS                         | 3                                          | The number of worker per Worker instance (set to 1 on Eks to have 1 pod = 1 worker, set to 0 for an API only instance)                                                                             | Worker                |
-| DISABLE_SERVER                      | false                                      | Binary would operate as a worker only instance                                                                                                                                                     | Worker                |
-| METRICS_ADDR                        | None                                       | The socket addr at which to expose Prometheus metrics at the /metrics path. Set to "true" to expose it on port 8001                                                                                | All                   |
-| JSON_FMT                            | false                                      | Output the logs in json format instead of logfmt                                                                                                                                                   | All                   |
-| BASE_URL                            | http://localhost:8000                      | The base url that is exposed publicly to access your instance                                                                                                                                      | Server                |
-| BASE_INTERNAL_URL                   | http://localhost:8000                      | The base url that is reachable by your workers to talk to the Servers. This help avoiding going through the external load balancer for VPC-internal requests.                                      | Worker                |
-| TIMEOUT                             | 300                                        | The maximum time of execution of a script. When reached, the job is failed as having timedout.                                                                                                     | Worker                |
-| ZOMBIE_JOB_TIMEOUT                  | 30                                         | The timeout after which a job is considered to be zombie if the worker did not send pings about processing the job (every server check for zombie jobs every 30s)                                  | Server                |
-| RESTART_ZOMBIE_JOBS                 | true                                       | If true then a zombie job is restarted (in-place with the same uuid and some logs), if false the zombie job is failed                                                                              | Server                |
-| SLEEP_QUEUE                         | 50                                         | The number of ms to sleep in between the last check for new jobs in the DB. It is multiplied by NUM_WORKERS such that in average, for one worker instance, there is one pull every SLEEP_QUEUE ms. | Worker                |
-| MAX_LOG_SIZE                        | 500000                                     | The maximum number of characters a job can emit (log + result)                                                                                                                                     | Worker                |
-| DISABLE_NUSER                       | false                                      | If Nsjail is enabled, disable the nsjail's `clone_newuser` setting                                                                                                                                 | Worker                |
-| KEEP_JOB_DIR                        | false                                      | Keep the job directory after the job is done. Useful for debugging.                                                                                                                                | Worker                |
-| LICENSE_KEY (EE only)               | None                                       | License key checked at startup for the Enterprise Edition of Windmill                                                                                                                              | Worker                |
-| S3_CACHE_BUCKET (EE only)           | None                                       | The S3 bucket to sync the cache of the workers to                                                                                                                                                  | Worker                |
-| TAR_CACHE_RATE (EE only)            | 100                                        | The rate at which to tar the cache of the workers. 100 means every 100th job in average (uniformly randomly distributed).                                                                          | Worker                |
-| SLACK_SIGNING_SECRET                | None                                       | The signing secret of your Slack app. See [Slack documentation](https://api.slack.com/authentication/verifying-requests-from-slack)                                                                | Server                |
-| COOKIE_DOMAIN                       | None                                       | The domain of the cookie. If not set, the cookie will be set by the browser based on the full origin                                                                                               | Server                |
-| DENO_PATH                           | /usr/bin/deno                              | The path to the deno binary.                                                                                                                                                                       | Worker                |
-| PYTHON_PATH                         | /usr/local/bin/python3                     | The path to the python binary.                                                                                                                                                                     | Worker                |
-| GO_PATH                             | /usr/bin/go                                | The path to the go binary.                                                                                                                                                                         | Worker                |
-| GOPRIVATE                           |                                            | The GOPRIVATE env variable to use private go modules                                                                                                                                               | Worker                |
-| NETRC                               |                                            | The netrc content to use a private go registry                                                                                                                                                     | Worker                |
-| PIP_INDEX_URL                       | None                                       | The index url to pass for pip.                                                                                                                                                                     | Worker                |
-| PIP_EXTRA_INDEX_URL                 | None                                       | The extra index url to pass to pip.                                                                                                                                                                | Worker                |
-| PIP_TRUSTED_HOST                    | None                                       | The trusted host to pass to pip.                                                                                                                                                                   | Worker                |
-| PATH                                | None                                       | The path environment variable, usually inherited                                                                                                                                                   | Worker                |
-| HOME                                | None                                       | The home directory to use for Go and Bash , usually inherited                                                                                                                                      | Worker                |
-| DATABASE_CONNECTIONS                | 50 (Server)/3 (Worker)                     | The max number of connections in the database connection pool                                                                                                                                      | All                   |
-| SUPERADMIN_SECRET                   | None                                       | A token that would let the caller act as a virtual superadmin superadmin@windmill.dev                                                                                                              | Server                |
-| TIMEOUT_WAIT_RESULT                 | 20                                         | The number of seconds to wait before timeout on the 'run_wait_result' endpoint                                                                                                                     | Worker                |
-| QUEUE_LIMIT_WAIT_RESULT             | None                                       | The number of max jobs in the queue before rejecting immediately the request in 'run_wait_result' endpoint. Takes precedence on the query arg. If none is specified, there are no limit.           | Worker                |
-| DENO_AUTH_TOKENS                    | None                                       | Custom DENO_AUTH_TOKENS to pass to worker to allow the use of private modules                                                                                                                      | Worker                |
-| DENO_FLAGS                          | None                                       | Override the flags passed to deno (default --allow-all) to tighten permissions. Minimum permissions needed are "--allow-read=args.json --allow-write=result.json"                                  | Worker                |
-| NPM_CONFIG_REGISTRY                 | None                                       | Registry to use for NPM dependencies, set if you have a private repository you need to use instead of the default public NPM registry                                                              | Worker                |
-| PIP_LOCAL_DEPENDENCIES              | None                                       | Specify dependencies that are installed locally and do not need to be solved nor installed again                                                                                                   |                       |
-| ADDITIONAL_PYTHON_PATHS             | None                                       | Specify python paths (separated by a :) to be appended to the PYTHONPATH of the python jobs. To be used with PIP_LOCAL_DEPENDENCIES to use python codebases within Windmill                        | Worker                |
-| INCLUDE_HEADERS                     | None                                       | Whitelist of headers that are passed to jobs as args (separated by a comma)                                                                                                                        | Server                |
-| WHITELIST_WORKSPACES                | None                                       | Whitelist of workspaces this worker takes job from                                                                                                                                                 | Worker                |
-| BLACKLIST_WORKSPACES                | None                                       | Blacklist of workspaces this worker takes job from                                                                                                                                                 | Worker                |
-| INSTANCE_EVENTS_WEBHOOK             | None                                       | Webhook to notify of events such as new user added, signup/invite. Can hook back to windmill to send emails                                                                                        |
-| GLOBAL_CACHE_INTERVAL               | 10\*60                                     | (Enterprise Edition only) Interval in seconds in between bucket sync of the cache. This interval \* 2 is the time at which you're guaranteed all the worker's caches are synced together.          | Worker                |
-| WORKER_TAGS                         | 'deno,go,python3,bash,flow,hub,dependency' | The worker groups assigned to that workers                                                                                                                                                         | Worker                |
-| CUSTOM_TAGS                         | None                                       | The custom tags assignable to scripts.                                                                                                                                                             | Server                |
-| JOB_RETENTION_SECS                  | 60*60*24\*60 //60 days                     | The time in seconds after which jobs get deleted. Set to 0 or -1 to never delete                                                                                                                   |
-| WAIT_RESULT_FAST_POLL_INTERVAL_MS   | 50                                         | The time in between polling for the run_wait_result endpoints in fast poll mode                                                                                                                    | Server                |
-| WAIT_RESULT_SLOW_POLL_INTERVAL_MS   | 200                                        | The time in between polling for the run_wait_result endpoints in fast poll mode                                                                                                                    | Server                |
-| WAIT_RESULT_FAST_POLL_DURATION_SECS | 2                                          | The duration of fast poll mode before switching to slow poll                                                                                                                                       | Server                |
-| EXIT_AFTER_NO_JOB_FOR_SECS          | None                                       | Exit worker if no job is received after duration in secs if defined                                                                                                                                | Worker                |
+| Environment Variable name | Default                | Description                                                                                                                                                                                        | Api Server/Worker/All |
+| ------------------------- | ---------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------- |
+| DATABASE_URL              |                        | The Postgres database url.                                                                                                                                                                         | All                   |
+| DISABLE_NSJAIL            | true                   | Disable Nsjail Sandboxing                                                                                                                                                                          | Worker                |
+| SERVER_BIND_ADDR          | 0.0.0.0                | IP Address on which to bind listening socket                                                                                                                                                       | Server                |
+| PORT                      | 8000                   | Exposed port                                                                                                                                                                                       | Server                |
+| NUM_WORKERS               | 3                      | The number of worker per Worker instance (set to 1 on Eks to have 1 pod = 1 worker, set to 0 for an API only instance)                                                                             | Worker                |
+| DISABLE_SERVER            | false                  | Binary would operate as a worker only instance                                                                                                                                                     | Worker                |
+| METRICS_ADDR              | None                   | The socket addr at which to expose Prometheus metrics at the /metrics path. Set to "true" to expose it on port 8001                                                                                | All                   |
+| JSON_FMT                  | false                  | Output the logs in json format instead of logfmt                                                                                                                                                   | All                   |
+| BASE_URL                  | http://localhost:8000  | The base url that is exposed publicly to access your instance                                                                                                                                      | Server                |
+| BASE_INTERNAL_URL         | http://localhost:8000  | The base url that is reachable by your workers to talk to the Servers. This help avoiding going through the external load balancer for VPC-internal requests.                                      | Worker                |
+| TIMEOUT                   | 300                    | The maximum time of execution of a script. When reached, the job is failed as having timedout.                                                                                                     | Worker                |
+| ZOMBIE_JOB_TIMEOUT        | 30                     | The timeout after which a job is considered to be zombie if the worker did not send pings about processing the job (every server check for zombie jobs every 30s)                                  | Server                |
+| RESTART_ZOMBIE_JOBS       | true                   | If true then a zombie job is restarted (in-place with the same uuid and some logs), if false the zombie job is failed                                                                              | Server                |
+| SLEEP_QUEUE               | 50                     | The number of ms to sleep in between the last check for new jobs in the DB. It is multiplied by NUM_WORKERS such that in average, for one worker instance, there is one pull every SLEEP_QUEUE ms. | Worker                |
+| MAX_LOG_SIZE              | 500000                 | The maximum number of characters a job can emit (log + result)                                                                                                                                     | Worker                |
+| DISABLE_NUSER             | false                  | If Nsjail is enabled, disable the nsjail's `clone_newuser` setting                                                                                                                                 | Worker                |
+| KEEP_JOB_DIR              | false                  | Keep the job directory after the job is done. Useful for debugging.                                                                                                                                | Worker                |
+| LICENSE_KEY (EE only)     | None                   | License key checked at startup for the Enterprise Edition of Windmill                                                                                                                              | Worker                |
+| S3_CACHE_BUCKET (EE only) | None                   | The S3 bucket to sync the cache of the workers to                                                                                                                                                  | Worker                |
+| TAR_CACHE_RATE (EE only)  | 100                    | The rate at which to tar the cache of the workers. 100 means every 100th job in average (uniformly randomly distributed).                                                                          | Worker                |
+| SLACK_SIGNING_SECRET      | None                   | The signing secret of your Slack app. See [Slack documentation](https://api.slack.com/authentication/verifying-requests-from-slack)                                                                | Server                |
+| COOKIE_DOMAIN             | None                   | The domain of the cookie. If not set, the cookie will be set by the browser based on the full origin                                                                                               | Server                |
+| DENO_PATH                 | /usr/bin/deno          | The path to the deno binary.                                                                                                                                                                       | Worker                |
+| PYTHON_PATH               | /usr/local/bin/python3 | The path to the python binary.                                                                                                                                                                     | Worker                |
+| GO_PATH                   | /usr/bin/go            | The path to the go binary.                                                                                                                                                                         | Worker                |
+| PIP_INDEX_URL             | None                   | The index url to pass for pip.                                                                                                                                                                     | Worker                |
+| PIP_EXTRA_INDEX_URL       | None                   | The extra index url to pass to pip.                                                                                                                                                                | Worker                |
+| PIP_TRUSTED_HOST          | None                   | The trusted host to pass to pip.                                                                                                                                                                   | Worker                |
+| PATH                      | None                   | The path environment variable, usually inherited                                                                                                                                                   | Worker                |
+| HOME                      | None                   | The home directory to use for Go and Bash , usually inherited                                                                                                                                      | Worker                |
+| DATABASE_CONNECTIONS      | 50 (Server)/3 (Worker) | The max number of connections in the database connection pool                                                                                                                                      | All                   |
+| SUPERADMIN_SECRET         | None                   | A token that would let the caller act as a virtual superadmin superadmin@windmill.dev                                                                                                              | Server                |
+| TIMEOUT_WAIT_RESULT       | 20                     | The number of seconds to wait before timeout on the 'run_wait_result' endpoint                                                                                                                     | Worker                |
+| QUEUE_LIMIT_WAIT_RESULT   | None                   | The number of max jobs in the queue before rejecting immediately the request in 'run_wait_result' endpoint. Takes precedence on the query arg. If none is specified, there are no limit.           | Worker                |
+| DENO_AUTH_TOKENS          | None                   | Custom DENO_AUTH_TOKENS to pass to worker to allow the use of private modules                                                                                                                      | Worker                |
+| DENO_FLAGS                | None                   | Override the flags passed to deno (default --allow-all) to tighten permissions. Minimum permissions needed are "--allow-read=args.json --allow-write=result.json"                                  | Worker                |
+| NPM_CONFIG_REGISTRY       | None                   | Registry to use for NPM dependencies, set if you have a private repository you need to use instead of the default public NPM registry                                                              | Worker                |
+| PIP_LOCAL_DEPENDENCIES    | None                   | Specify dependencies that are installed locally and do not need to be solved nor installed again                                                                                                   |                       |
+| ADDITIONAL_PYTHON_PATHS   | None                   | Specify python paths (separated by a :) to be appended to the PYTHONPATH of the python jobs. To be used with PIP_LOCAL_DEPENDENCIES to use python codebases within Windmill                        | Worker                |
+| INCLUDE_HEADERS           | None                   | Whitelist of headers that are passed to jobs as args (separated by a comma)                                                                                                                        | Server                |
+| WHITELIST_WORKSPACES      | None                   | Whitelist of workspaces this worker takes job from                                                                                                                                                 | Worker                |
+| BLACKLIST_WORKSPACES      | None                   | Blacklist of workspaces this worker takes job from                                                                                                                                                 | Worker                |
+| NEW_USER_WEBHOOK          | None                   | Webhook to notify of a new user added, signup/invite. Can hook back to windmill to send emails                                                                                                     | Server                |
 
 ## Run a local dev setup
 

backend/.cargo/config.toml

@@ -18,4 +18,4 @@ rustflags = [
 rustflags = [
     "-C", "link-arg=-undefined",
     "-C", "link-arg=dynamic_lookup",
-]
+]

backend/.vscode/settings.json

@@ -1,4 +1,3 @@
 {
-  "python.analysis.typeCheckingMode": "basic",
-  "rust-analyzer.linkedProjects": ["./windmill-common/Cargo.toml"]
+  "python.analysis.typeCheckingMode": "basic"
 }

backend/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "windmill"
-version = "1.93.0"
+version = "1.86.0"
 authors.workspace = true
 edition.workspace = true
 
@@ -19,7 +19,7 @@ members = [
 ]
 
 [workspace.package]
-version = "1.93.0"
+version = "1.86.0"
 authors = ["Ruben Fiszel <ruben@windmill.dev>"]
 edition = "2021"
 
@@ -48,12 +48,6 @@ git-version.workspace = true
 rsa.workspace = true
 base64.workspace = true
 sha2.workspace = true
-rsmq_async.workspace = true
-url.workspace = true
-lazy_static.workspace = true
-once_cell.workspace = true
-prometheus.workspace = true
-uuid.workspace = true
 
 
 [dev-dependencies]
@@ -147,7 +141,6 @@ sqlx = { version = "^0", features = [
 dotenv = "^0"
 ulid = { version = "^1", features = ["uuid"] }
 futures = "^0"
-futures-core = "^0"
 tokio-metrics = "0.1.0"
 lazy_static = "1.4.0"
 serde_derive = "1.0.147"
@@ -160,7 +153,4 @@ async-stripe = { version = "0.14", features = [
 ] }
 async_zip = { version = "0.0.11", features = ["full"] }
 once_cell = "1.17.1"
-rsmq_async = { version = "5.1.5" }
 gosyn = "0.2.2"
-
-[patch.crates-io]

backend/README.md

@@ -1,7 +1,6 @@
 # Windmill Backend
 
-This folder holds all backend components, the [src/](./src/) folder only
-contains files used to build the "root" binary.
+This folder holds all backend components, the [src/](./src/) folder only contains files used to build the "root" binary.
 
 ## Components
 
@@ -14,9 +13,3 @@ contains files used to build the "root" binary.
 | [windmill-queue](./windmill-queue/)           | Contains job & flow queuing functionality, commonly written to by the API server and read from by workers |
 | [windmill-worker](./windmill-worker/)         | The worker. Used to process and execute flows & jobs.                                                     |
 | [parsers](./parsers/)                         | Contains code to parse signatures in different langauges.                                                 |
-
-### Compile sqlx for offline ci
-
-```
-cargo sqlx prepare --merged -- --bin windmill --features enterprise
-```

backend/migrations/20230410122342_saved_inputs.down.sql

@@ -1,2 +0,0 @@
-DROP TABLE IF EXISTS input;
-DROP TYPE RUNNABLE_TYPE;

backend/migrations/20230410122342_saved_inputs.up.sql

@@ -1,13 +0,0 @@
-CREATE TYPE RUNNABLE_TYPE AS ENUM ('ScriptHash', 'ScriptPath', 'FlowPath');
-
-CREATE TABLE IF NOT EXISTS input (
-  id UUID PRIMARY KEY,
-  workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id),
-  runnable_id VARCHAR(255) NOT NULL,
-  runnable_type RUNNABLE_TYPE NOT NULL,
-  name TEXT NOT NULL,
-  args JSONB NOT NULL,
-  created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
-  created_by VARCHAR(50) NOT NULL,
-  is_public BOOLEAN NOT NULL DEFAULT FALSE
-);

backend/migrations/20230414163230_input_grant.down.sql

@@ -1 +0,0 @@
--- Add down migration script here

backend/migrations/20230414163230_input_grant.up.sql

@@ -1,3 +0,0 @@
--- Add up migration script here
-GRANT ALL ON input TO windmill_user;
-GRANT ALL ON input TO windmill_admin;

backend/migrations/20230421111850_add_tagged_scripts.down.sql

@@ -1,4 +0,0 @@
--- Add down migration script here
-ALTER TABLE script DROP COLUMN tag;
-ALTER TABLE completed_job DROP COLUMN tag;
-ALTER TABLE queue DROP COLUMN tag;

backend/migrations/20230421111850_add_tagged_scripts.up.sql

@@ -1,4 +0,0 @@
--- Add up migration script here
-ALTER TABLE script ADD COLUMN tag VARCHAR(50);
-ALTER TABLE completed_job  ADD COLUMN tag VARCHAR(50) NOT NULL DEFAULT 'other';
-ALTER TABLE queue ADD COLUMN tag VARCHAR(50) NOT NULL DEFAULT 'other';

backend/migrations/20230429082953_add_drafts.down.sql

@@ -1,3 +0,0 @@
--- Add down migration script here
-DROP TABLE draft;
-DROP TYPE DRAFT_TYPE;

backend/migrations/20230429082953_add_drafts.up.sql

@@ -1,18 +0,0 @@
--- Add up migration script here
-CREATE TYPE DRAFT_TYPE AS ENUM ('script', 'flow', 'app');
-
-CREATE TABLE draft (
-    workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id),
-    path VARCHAR(255) NOT NULL,
-    typ DRAFT_TYPE NOT NULL,
-    value JSONB NOT NULL,
-    created_at TIMESTAMP NOT NULL DEFAULT now(),
-    PRIMARY KEY (workspace_id, path, typ)
-);
-
-GRANT ALL ON draft TO windmill_admin;
-GRANT ALL ON draft TO windmill_user;
-
-ALTER TABLE script ADD COLUMN draft_only BOOLEAN;
-ALTER TABLE flow ADD COLUMN draft_only BOOLEAN;
-ALTER TABLE app ADD COLUMN draft_only BOOLEAN;

backend/parsers/windmill-parser-go/src/lib.rs

@@ -96,7 +96,7 @@ fn parse_go_typ(typ: &Expression) -> (Option<String>, Typ) {
                         .as_ref()
                         .and_then(|x| x.value.strip_prefix("`json:\""))
                         .and_then(|x| x.strip_suffix("\"`"))
-                        .and_then(|x| x.split(',').next().map(|x| x.to_string()));
+                        .and_then(|x| x.split(',').last().map(|x| x.to_string()));
                     let (otyp, typ) = parse_go_typ(&field.typ);
                     let name = get_name(field);
                     let key = json_tag.unwrap_or_else(|| name.to_string());

backend/parsers/windmill-parser-ts/src/lib.rs

@@ -103,11 +103,6 @@ fn parse_param(x: Param, cm: &Lrc<SourceMap>, skip_dflt: bool) -> error::Result<
                     Expr::Lit(Lit::Str(Str { value, .. })) => {
                         Some(Value::String(value.to_string()))
                     }
-                    Expr::Lit(Lit::Num(Number { value, .. }))
-                        if (value == (value as u64) as f64) =>
-                    {
-                        Some(serde_json::json!(value as u64))
-                    }
                     Expr::Lit(Lit::Num(Number { value, .. })) => Some(serde_json::json!(value)),
                     Expr::Lit(Lit::BigInt(BigInt { value, .. })) => Some(serde_json::json!(value)),
                     Expr::Lit(Lit::Bool(Bool { value, .. })) => Some(Value::Bool(value)),
@@ -268,7 +263,7 @@ fn tstype_to_typ(ts_type: &TsType) -> (Typ, bool) {
 pub fn eval_sync(code: &str) -> Result<serde_json::Value, String> {
     let mut context = JsRuntime::new(RuntimeOptions::default());
     let code = format!("let x = {}; x", code);
-    let res = context.execute_script("<anon>", code.into());
+    let res = context.execute_script("<anon>", code);
     match res {
         Ok(global) => {
             let scope = &mut context.handle_scope();

backend/src/main.rs

@@ -6,24 +6,11 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
-use std::{
-    net::{IpAddr, Ipv4Addr, SocketAddr},
-    sync::Arc,
-};
+use std::net::{IpAddr, Ipv4Addr, SocketAddr};
 
 use git_version::git_version;
-use monitor::handle_zombie_jobs_periodically;
 use sqlx::{Pool, Postgres};
-use tokio::{
-    fs::{metadata, DirBuilder},
-    join,
-    sync::RwLock,
-};
 use windmill_common::{utils::rd_string, METRICS_ADDR};
-use windmill_worker::{
-    DENO_CACHE_DIR, DENO_TMP_CACHE_DIR, GO_CACHE_DIR, GO_TMP_CACHE_DIR, PIP_CACHE_DIR,
-    ROOT_TMP_CACHE_DIR, TAR_PIP_TMP_CACHE_DIR,
-};
 
 const GIT_VERSION: &str = git_version!(args = ["--tag", "--always"], fallback = "unknown-version");
 const DEFAULT_NUM_WORKERS: usize = 3;
@@ -31,7 +18,6 @@ const DEFAULT_PORT: u16 = 8000;
 const DEFAULT_SERVER_BIND_ADDR: Ipv4Addr = Ipv4Addr::new(0, 0, 0, 0);
 
 mod ee;
-mod monitor;
 
 #[tokio::main]
 async fn main() -> anyhow::Result<()> {
@@ -63,46 +49,14 @@ async fn main() -> anyhow::Result<()> {
         .and_then(|x| x.parse::<bool>().ok())
         .unwrap_or(false);
 
-    let rsmq_config = std::env::var("REDIS_URL").ok().map(|x| {
-        let url = x.parse::<url::Url>().unwrap();
-        let mut config = rsmq_async::RsmqOptions { ..Default::default() };
-
-        config.host = url.host_str().expect("redis host required").to_owned();
-        config.password = url.password().map(|s| s.to_owned());
-        config.db = url
-            .path_segments()
-            .and_then(|mut segments| segments.next())
-            .and_then(|segment| segment.parse().ok())
-            .unwrap_or(0);
-        config.ns = url
-            .query_pairs()
-            .find(|s| s.0 == "rsmq_namespace")
-            .map(|s| s.1)
-            .unwrap_or(std::borrow::Cow::Borrowed("rsmq"))
-            .into_owned();
-        config.port = url.port().unwrap_or(6379).to_string();
-
-        config
-    });
-
     let db = windmill_common::connect_db(server_mode).await?;
 
-    let rsmq = if let Some(config) = rsmq_config {
-        let mut rsmq = rsmq_async::MultiplexedRsmq::new(config).await.unwrap();
-
-        let _ = rsmq_async::RsmqConnection::create_queue(&mut rsmq, "main_queue", None, None, None)
-            .await;
-        Some(rsmq)
-    } else {
-        None
-    };
-
     if server_mode {
         windmill_api::migrate_db(&db).await?;
     }
 
     let (tx, rx) = tokio::sync::broadcast::channel::<()>(3);
-    let shutdown_signal = windmill_common::shutdown_signal(tx.clone(), rx.resubscribe());
+    let shutdown_signal = windmill_common::shutdown_signal(tx);
 
     #[cfg(feature = "enterprise")]
     tracing::info!(
@@ -142,8 +96,6 @@ Windmill Community Edition {GIT_VERSION}
         "PYTHON_PATH",
         "DENO_PATH",
         "GO_PATH",
-        "GOPRIVATE",
-        "NETRC",
         "PIP_INDEX_URL",
         "PIP_EXTRA_INDEX_URL",
         "PIP_TRUSTED_HOST",
@@ -160,25 +112,16 @@ Windmill Community Edition {GIT_VERSION}
         "INCLUDE_HEADERS",
         "WHITELIST_WORKSPACES",
         "BLACKLIST_WORKSPACES",
-        "INSTANCE_EVENTS_WEBHOOK",
+        "NEW_USER_WEBHOOK",
         "CLOUD_HOSTED",
-        "GLOBAL_CACHE_INTERVAL",
-        "WORKER_TAGS",
-        "CUSTOM_TAGS",
-        "JOB_RETENTION_SECS",
-        "WAIT_RESULT_FAST_POLL_DURATION_SECS",
-        "WAIT_RESULT_SLOW_POLL_INTERVAL_MS",
-        "WAIT_RESULT_FAST_POLL_INTERVAL_MS",
-        "EXIT_AFTER_NO_JOB_FOR_SECS",
     ]);
 
     if server_mode || num_workers > 0 {
         let addr = SocketAddr::from((server_bind_address, port));
 
-        let rsmq2 = rsmq.clone();
         let server_f = async {
             if server_mode {
-                windmill_api::run_server(db.clone(), rsmq2, addr, rx.resubscribe()).await?;
+                windmill_api::run_server(db.clone(), addr, rx.resubscribe()).await?;
             }
             Ok(()) as anyhow::Result<()>
         };
@@ -190,19 +133,15 @@ Windmill Community Edition {GIT_VERSION}
                     rx.resubscribe(),
                     num_workers,
                     base_internal_url.clone(),
-                    rsmq.clone(),
                 )
                 .await?;
-                tracing::info!("All workers exited.");
-                tx.send(())?; // signal server to shutdown
             }
             Ok(()) as anyhow::Result<()>
         };
 
-        let rsmq2 = rsmq.clone();
         let monitor_f = async {
             if server_mode {
-                monitor_db(&db, rx.resubscribe(), &base_internal_url, rsmq2);
+                monitor_db(&db, rx.resubscribe(), &base_internal_url);
             }
             Ok(()) as anyhow::Result<()>
         };
@@ -242,11 +181,10 @@ fn display_config(envs: Vec<&str>) {
     )
 }
 
-pub fn monitor_db<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 'static>(
+pub fn monitor_db(
     db: &Pool<Postgres>,
     rx: tokio::sync::broadcast::Receiver<()>,
     base_internal_url: &str,
-    rsmq: Option<R>,
 ) {
     let db1 = db.clone();
     let db2 = db.clone();
@@ -254,19 +192,16 @@ pub fn monitor_db<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 'static>
     let rx2 = rx.resubscribe();
     let base_internal_url = base_internal_url.to_string();
     tokio::spawn(async move {
-        join!(
-            handle_zombie_jobs_periodically(&db1, rx, &base_internal_url, rsmq),
-            windmill_api::delete_expired_items_perdiodically(&db2, rx2)
-        );
+        windmill_worker::handle_zombie_jobs_periodically(&db1, rx, &base_internal_url).await
     });
+    tokio::spawn(async move { windmill_api::delete_expired_items_perdiodically(&db2, rx2).await });
 }
 
-pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 'static>(
+pub async fn run_workers(
     db: Pool<Postgres>,
     rx: tokio::sync::broadcast::Receiver<()>,
     num_workers: i32,
     base_internal_url: String,
-    rsmq: Option<R>,
 ) -> anyhow::Result<()> {
     let license_key = std::env::var("LICENSE_KEY").ok();
     #[cfg(feature = "enterprise")]
@@ -289,28 +224,6 @@ pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + '
 
     let mut handles = Vec::with_capacity(num_workers as usize);
 
-    if metadata(&ROOT_TMP_CACHE_DIR).await.is_ok() {
-        if let Err(e) = tokio::fs::remove_dir_all(&ROOT_TMP_CACHE_DIR).await {
-            tracing::info!(error = %e, "Could not remove root tmp cache dir");
-        }
-    }
-
-    for x in [
-        PIP_CACHE_DIR,
-        DENO_CACHE_DIR,
-        GO_CACHE_DIR,
-        TAR_PIP_TMP_CACHE_DIR,
-        DENO_TMP_CACHE_DIR,
-        GO_TMP_CACHE_DIR,
-    ] {
-        DirBuilder::new()
-            .recursive(true)
-            .create(x)
-            .await
-            .expect("could not create initial worker dir");
-    }
-
-    let sync_barrier = Arc::new(RwLock::new(None));
     for i in 1..(num_workers + 1) {
         let db1 = db.clone();
         let instance_name = instance_name.clone();
@@ -318,8 +231,6 @@ pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + '
         let ip = ip.clone();
         let rx = rx.resubscribe();
         let base_internal_url = base_internal_url.clone();
-        let rsmq2 = rsmq.clone();
-        let sync_barrier = sync_barrier.clone();
         handles.push(tokio::spawn(monitor.instrument(async move {
             tracing::info!(worker = %worker_name, "starting worker");
             windmill_worker::run_worker(
@@ -327,12 +238,9 @@ pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + '
                 &instance_name,
                 worker_name,
                 i as u64,
-                num_workers as u32,
                 &ip,
                 rx,
                 &base_internal_url,
-                rsmq2,
-                sync_barrier,
             )
             .await
         })));

backend/src/monitor.rs

@@ -1,148 +0,0 @@
-use std::time::Duration;
-
-use once_cell::sync::OnceCell;
-use sqlx::{Pool, Postgres};
-use tokio::sync::mpsc;
-use uuid::Uuid;
-use windmill_common::{error, jobs::{JobKind, QueuedJob}, METRICS_ENABLED};
-use windmill_worker::{
-    create_token_for_owner, handle_job_error, AuthedClient, SESSION_TOKEN_EXPIRY,
-};
-
-lazy_static::lazy_static! {
-    static ref ZOMBIE_JOB_TIMEOUT: String = std::env::var("ZOMBIE_JOB_TIMEOUT")
-    .ok()
-    .and_then(|x| x.parse::<String>().ok())
-    .unwrap_or_else(|| "30".to_string());
-
-
-    pub static ref RESTART_ZOMBIE_JOBS: bool = std::env::var("RESTART_ZOMBIE_JOBS")
-    .ok()
-    .and_then(|x| x.parse::<bool>().ok())
-    .unwrap_or(true);
-
-    static ref QUEUE_ZOMBIE_RESTART_COUNT: prometheus::IntCounter = prometheus::register_int_counter!(
-        "queue_zombie_restart_count",
-        "Total number of jobs restarted due to ping timeout."
-    )
-    .unwrap();
-    static ref QUEUE_ZOMBIE_DELETE_COUNT: prometheus::IntCounter = prometheus::register_int_counter!(
-        "queue_zombie_delete_count",
-        "Total number of jobs deleted due to their ping timing out in an unrecoverable state."
-    )
-    .unwrap();
-}
-
-pub async fn handle_zombie_jobs_periodically<
-    R: rsmq_async::RsmqConnection + Send + Sync + Clone,
->(
-    db: &Pool<Postgres>,
-    mut rx: tokio::sync::broadcast::Receiver<()>,
-    base_internal_url: &str,
-    rsmq: Option<R>,
-) {
-    loop {
-        handle_zombie_jobs(db, base_internal_url, rsmq.clone()).await;
-
-        tokio::select! {
-            _ = tokio::time::sleep(Duration::from_secs(30))    => (),
-            _ = rx.recv() => {
-                    println!("received killpill for monitor job");
-                    break;
-            }
-        }
-    }
-}
-
-async fn handle_zombie_jobs<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
-    db: &Pool<Postgres>,
-    base_internal_url: &str,
-    rsmq: Option<R>,
-) {
-    if *RESTART_ZOMBIE_JOBS {
-        let restarted = sqlx::query!(
-                "UPDATE queue SET running = false, started_at = null, logs = logs || '\nRestarted job after not receiving job''s ping for too long the ' || now() || '\n\n' WHERE last_ping < now() - ($1 || ' seconds')::interval AND running = true AND job_kind != $2 AND job_kind != $3 AND same_worker = false RETURNING id, workspace_id, last_ping",
-                *ZOMBIE_JOB_TIMEOUT,
-                JobKind::Flow: JobKind,
-                JobKind::FlowPreview: JobKind,
-            )
-            .fetch_all(db)
-            .await
-            .ok()
-            .unwrap_or_else(|| vec![]);
-
-        if *METRICS_ENABLED {
-            QUEUE_ZOMBIE_RESTART_COUNT.inc_by(restarted.len() as _);
-        }
-        for r in restarted {
-            tracing::info!(
-                "restarted zombie job {} {} {}",
-                r.id,
-                r.workspace_id,
-                r.last_ping
-            );
-        }
-    }
-
-    let mut timeout_query = "SELECT * FROM queue WHERE last_ping < now() - ($1 || ' seconds')::interval AND running = true AND job_kind != $2".to_string();
-    if *RESTART_ZOMBIE_JOBS {
-        timeout_query.push_str(" AND same_worker = true");
-    };
-    let timeouts = sqlx::query_as::<_, QueuedJob>(&timeout_query)
-        .bind(ZOMBIE_JOB_TIMEOUT.as_str())
-        .bind(JobKind::Flow)
-        .fetch_all(db)
-        .await
-        .ok()
-        .unwrap_or_else(|| vec![]);
-
-    if *METRICS_ENABLED {
-        QUEUE_ZOMBIE_DELETE_COUNT.inc_by(timeouts.len() as _);
-    }
-
-    for job in timeouts {
-        tracing::info!("timedout zombie job {} {}", job.id, job.workspace_id,);
-
-        // since the job is unrecoverable, the same worker queue should never be sent anything
-        let (same_worker_tx_never_used, _same_worker_rx_never_used) = mpsc::channel::<Uuid>(1);
-
-        let token = create_token_for_owner(
-            &db,
-            &job.workspace_id,
-            &job.permissioned_as,
-            "ephemeral-zombie-jobs",
-            *SESSION_TOKEN_EXPIRY,
-            &job.email,
-        )
-        .await
-        .expect("could not create job token");
-
-        let client = AuthedClient {
-            base_internal_url: base_internal_url.to_string(),
-            token,
-            workspace: job.workspace_id.to_string(),
-            client: OnceCell::new(),
-        };
-
-        let last_ping = job.last_ping.clone();
-        let _ = handle_job_error(
-            db,
-            &client,
-            job,
-            error::Error::ExecutionErr(format!(
-                "Job timed out after no ping from job since {} (ZOMBIE_JOB_TIMEOUT: {})",
-                last_ping
-                    .map(|x| x.to_string())
-                    .unwrap_or_else(|| "no ping".to_string()),
-                *ZOMBIE_JOB_TIMEOUT
-            )),
-            None,
-            true,
-            same_worker_tx_never_used,
-            "",
-            base_internal_url,
-            rsmq.clone(),
-        )
-        .await;
-    }
-}

backend/tests/worker.rs

@@ -1,19 +1,14 @@
-use std::sync::Arc;
-
 use futures::{stream, Stream};
 use serde::Deserialize;
 use serde_json::json;
 use sqlx::{postgres::PgListener, types::Uuid, Pool, Postgres, Transaction};
-use tokio::sync::RwLock;
 use windmill_api::jobs::{CompletedJob, Job};
-use windmill_api_client::types::CreateFlowBody;
 use windmill_common::{
     flow_status::{FlowStatus, FlowStatusModule},
     flows::{FlowModule, FlowModuleValue, FlowValue, InputTransform},
-    jobs::{JobPayload, RawCode},
     scripts::ScriptLang,
 };
-use windmill_queue::get_queued_job;
+use windmill_queue::{get_queued_job, JobPayload, RawCode};
 
 async fn initialize_tracing() {
     use std::sync::Once;
@@ -93,7 +88,7 @@ impl ApiServer {
         let addr = sock.local_addr().unwrap();
         drop(sock);
 
-        let task = tokio::task::spawn(windmill_api::run_server(db.clone(), None, addr, rx));
+        let task = tokio::task::spawn(windmill_api::run_server(db.clone(), addr, rx));
 
         return Self { addr, tx, task };
     }
@@ -133,7 +128,8 @@ mod suspend_resume {
     use futures::{Stream, StreamExt};
     use serde_json::json;
     use sqlx::{query_scalar, types::Uuid};
-    use windmill_common::{flows::FlowValue, jobs::JobPayload};
+    use windmill_common::flows::FlowValue;
+    use windmill_queue::JobPayload;
 
     use super::*;
 
@@ -397,6 +393,7 @@ mod retry {
     use serde_json::json;
     use tokio::io::{AsyncReadExt, AsyncWriteExt};
     use windmill_common::flows::FlowValue;
+    use windmill_queue::JobPayload;
 
     use super::*;
 
@@ -659,7 +656,7 @@ def main(error, port):
             },
         }))
         .unwrap();
-        let (_attempts, responses) = [
+        let (attempts, responses) = [
             /* fail the first step twice */
             (0x00, None),
             (0x00, None),
@@ -681,20 +678,14 @@ def main(error, port):
             _ => panic!("expected failure module"),
         }
 
-        println!("result: {:#?}", result);
+        assert_eq!(server.close().await, attempts);
         assert_eq!(
-            result
-                .get("from failure module")
-                .unwrap()
-                .get("error")
-                .unwrap()
-                .get("name")
-                .unwrap()
-                .clone(),
-            json!("IndexError")
+            result,
+            json!({
+                "recv": 42,
+                "from failure module": {"error": {"name": "IndexError", "stack": "  File \"/tmp/tmp/main/step_0.py\", line 5, in main\n    return sock.recv(1)[0]\n", "message": "index out of range"}},
+            })
         );
-
-        assert_eq!(result.get("recv").unwrap().clone(), json!(42));
     }
 }
 
@@ -830,8 +821,9 @@ impl RunJob {
 
     async fn push(self, db: &Pool<Postgres>) -> Uuid {
         let RunJob { payload, args } = self;
-        let (uuid, tx) = windmill_queue::push::<rsmq_async::MultiplexedRsmq>(
-            (None, db.begin().await.unwrap()).into(),
+        let tx = db.begin().await.unwrap();
+        let (uuid, tx) = windmill_queue::push(
+            tx,
             "test-workspace",
             payload,
             args,
@@ -846,11 +838,11 @@ impl RunJob {
             /* running */ false,
             None,
             true,
-            None,
         )
         .await
         .expect("push has to succeed");
-        tx.commit().await.unwrap();
+
+        tx.commit().await.expect("push has to commit");
 
         uuid
     }
@@ -881,7 +873,7 @@ async fn in_test_worker<Fut: std::future::Future>(
     port: u16,
 ) -> <Fut as std::future::Future>::Output {
     let (quit, worker) = spawn_test_worker(db, port);
-    let worker = tokio::time::timeout(std::time::Duration::from_secs(45), worker);
+    let worker = tokio::time::timeout(std::time::Duration::from_secs(19), worker);
     tokio::pin!(worker);
 
     let res = tokio::select! {
@@ -916,20 +908,18 @@ fn spawn_test_worker(
     let db = db.to_owned();
     let worker_instance: &str = "test worker instance";
     let worker_name: String = next_worker_name();
+    let i_worker: u64 = Default::default();
     let ip: &str = Default::default();
     let future = async move {
         let base_internal_url = format!("http://localhost:{}", port);
-        windmill_worker::run_worker::<rsmq_async::MultiplexedRsmq>(
+        windmill_worker::run_worker(
             &db,
             worker_instance,
             worker_name,
-            1,
-            1,
+            i_worker,
             ip,
             rx,
             &base_internal_url,
-            None,
-            Arc::new(RwLock::new(None)),
         )
         .await
     };
@@ -1007,7 +997,6 @@ async fn test_deno_flow(db: Pool<Postgres>) {
                         content: numbers.to_string(),
                         path: None,
                         lock: None,
-                        tag: None,
                     },
                     stop_after_if: Default::default(),
                     summary: Default::default(),
@@ -1035,7 +1024,6 @@ async fn test_deno_flow(db: Pool<Postgres>) {
                                 content: doubles.to_string(),
                                 path: None,
                                 lock: None,
-                                tag: None,
                             },
                             stop_after_if: Default::default(),
                             summary: Default::default(),
@@ -1136,7 +1124,6 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
                         content: write_file.clone(),
                         path: None,
                         lock: None,
-                        tag: None,
                     },
                     stop_after_if: Default::default(),
                     summary: Default::default(),
@@ -1175,7 +1162,6 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
                                     content: write_file,
                                     path: None,
                                     lock: None,
-                                    tag: None,
                                 },
                                 stop_after_if: Default::default(),
                                 summary: Default::default(),
@@ -1201,7 +1187,6 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
                                     .to_string(),
                                     path: None,
                                     lock: None,
-                                    tag: None,
                                 },
                                 stop_after_if: Default::default(),
                                 summary: Default::default(),
@@ -1244,7 +1229,6 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
                         .to_string(),
                         path: None,
                         lock: None,
-                        tag: None,
                     },
                     stop_after_if: Default::default(),
                     summary: Default::default(),
@@ -2084,7 +2068,7 @@ async fn test_branchall_skip_failure(db: Pool<Postgres>) {
         .unwrap();
 
     assert_eq!(
-        serde_json::from_value::<ErrorResult>(result.get(0).unwrap().clone())
+        serde_json::from_value::<ErrorResult>(result)
             .unwrap()
             .error
             .name,
@@ -2121,7 +2105,7 @@ async fn test_branchall_skip_failure(db: Pool<Postgres>) {
         .unwrap();
 
     assert_eq!(
-        serde_json::from_value::<ErrorResult>(result.get(0).unwrap().clone())
+        serde_json::from_value::<ErrorResult>(result)
             .unwrap()
             .error
             .name,
@@ -2443,12 +2427,9 @@ async fn test_flow_lock_all(db: Pool<Postgres>) {
     client
         .create_flow(
             "test-workspace",
-            &CreateFlowBody {
-                open_flow_w_path: windmill_api_client::types::OpenFlowWPath {
-                    open_flow: flow,
-                    path: "g/all/flow_lock_all".to_owned(),
-                },
-                draft_only: None,
+            &windmill_api_client::types::OpenFlowWPath {
+                open_flow: flow,
+                path: "g/all/flow_lock_all".to_owned(),
             },
         )
         .await
@@ -2467,17 +2448,16 @@ async fn test_flow_lock_all(db: Pool<Postgres>) {
         .modules
         .into_iter()
         .for_each(|m| {
-            tracing::error!("Module: {:?}", m.value);
             assert!(matches!(
                 m.value,
-                windmill_api_client::types::FlowModuleValue::RawScript {
+                windmill_api_client::types::FlowModuleValue::Rawscript {
                     language: windmill_api_client::types::RawScriptLanguage::Deno | windmill_api_client::types::RawScriptLanguage::Bash,
                     lock: Some(ref lock),
                     ..
                 } if lock == "")
                 || matches!(
                 m.value,
-                windmill_api_client::types::FlowModuleValue::RawScript {
+                windmill_api_client::types::FlowModuleValue::Rawscript {
                     language: windmill_api_client::types::RawScriptLanguage::Go | windmill_api_client::types::RawScriptLanguage::Python3,
                     lock: Some(ref lock),
                     ..

backend/windmill-api-client/Cargo.toml

@@ -21,7 +21,5 @@ rand.workspace = true
 base64.workspace = true
 
 [build-dependencies]
-prettyplease = "0.1.25"
 progenitor = { git = "https://github.com/oxidecomputer/progenitor" }
 serde_json = "1.0"
-syn = "1.0"

backend/windmill-api-client/build.rs

@@ -13,9 +13,7 @@ fn main() {
     let spec = serde_json::from_reader(file).unwrap();
     let mut generator = progenitor::Generator::default();
 
-    let tokens = generator.generate_tokens(&spec).unwrap();
-    let ast = syn::parse2(tokens).unwrap();
-    let content = prettyplease::unparse(&ast);
+    let content = generator.generate_text(&spec).unwrap();
 
     let mut out_file = Path::new(&env::var("OUT_DIR").unwrap()).to_path_buf();
     out_file.push("codegen.rs");

backend/windmill-api/Cargo.toml

@@ -70,5 +70,4 @@ async-stripe = { workspace = true, optional = true }
 lazy_static.workspace = true
 prometheus.workspace = true
 async_zip.workspace = true
-rsmq_async.workspace = true
 regex.workspace = true

backend/windmill-api/openapi.yaml

@@ -1,7 +1,7 @@
 openapi: "3.0.3"
 
 info:
-  version: 1.93.0
+  version: 1.86.0
   title: Windmill API
 
   contact:
@@ -2125,41 +2125,6 @@ paths:
                 items:
                   type: string
 
-
-  /w/{workspace}/drafts/create:
-    post:
-      summary: create draft
-      operationId: createDraft
-      tags:
-        - draft
-      parameters:
-        - $ref: "#/components/parameters/WorkspaceId"
-      requestBody:
-        required: true
-        content:
-          application/json:
-            schema:
-              type: object
-              properties:
-                path:
-                  type: string
-                typ:
-                  type: string
-                  enum: ["flow", "script", "app"]
-                value: {}
-              required:
-                - path
-                - typ
-                - enum
-      responses:
-        "201":
-          description: draft created
-          content:
-            text/plain:
-              schema:
-                type: string
-
-
   /w/{workspace}/scripts/create:
     post:
       summary: create script
@@ -2174,8 +2139,38 @@ paths:
         content:
           application/json:
             schema:
-              $ref: "#/components/schemas/NewScript"
-
+              type: object
+              properties:
+                path:
+                  type: string
+                parent_hash:
+                  type: string
+                summary:
+                  type: string
+                description:
+                  type: string
+                content:
+                  type: string
+                schema:
+                  type: object
+                is_template:
+                  type: boolean
+                lock:
+                  type: array
+                  items:
+                    type: string
+                language:
+                  type: string
+                  enum: [python3, deno, go, bash]
+                kind:
+                  type: string
+                  enum: [script, failure, trigger, command, approval]
+              required:
+                - path
+                - summary
+                - description
+                - content
+                - language
       responses:
         "201":
           description: script created
@@ -2184,22 +2179,6 @@ paths:
               schema:
                 type: string
 
-  /workers/custom_tags:
-    get:
-      summary: get all instance custom tags (tags are used to dispatch jobs to different worker groups)
-      operationId: getCustomTags
-      tags:
-        - worker
-      responses:
-        "200":
-          description: list of custom tags
-          content:
-            application/json:
-              schema:
-                type: array
-                items:
-                  type: string
-
   /scripts/python/tojsonschema:
     post:
       summary: inspect python code to infer jsonschema of arguments
@@ -2369,25 +2348,6 @@ paths:
               schema:
                 $ref: "#/components/schemas/Script"
 
-  /w/{workspace}/scripts/get/draft/{path}:
-    get:
-      summary: get script by path with draft
-      operationId: getScriptByPathWithDraft
-      tags:
-        - script
-      parameters:
-        - $ref: "#/components/parameters/WorkspaceId"
-        - $ref: "#/components/parameters/ScriptPath"
-      responses:
-        "200":
-          description: script details
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/NewScriptWithDraft"
-
-
-
   /w/{workspace}/scripts/raw/p/{path}:
     get:
       summary: raw script by path
@@ -2703,14 +2663,7 @@ paths:
               schema:
                 type: array
                 items:
-                  allOf:
-                    - $ref: "#/components/schemas/Flow"
-                    - type: object
-                      properties:
-                        has_draft:
-                          type: boolean
-                        draft_only:
-                          type: boolean
+                  $ref: "#/components/schemas/Flow"
 
   /w/{workspace}/flows/get/{path}:
     get:
@@ -2729,29 +2682,6 @@ paths:
               schema:
                 $ref: "#/components/schemas/Flow"
 
-  /w/{workspace}/flows/get/draft/{path}:
-    get:
-      summary: get flow by path with draft
-      operationId: getFlowByPathWithDraft
-      tags:
-        - flow
-      parameters:
-        - $ref: "#/components/parameters/WorkspaceId"
-        - $ref: "#/components/parameters/ScriptPath"
-      responses:
-        "200":
-          description: flow details with draft
-          content:
-            application/json:
-              schema:
-                allOf:
-                  - $ref: "#/components/schemas/Flow"
-                  - type: object
-                    properties:
-                      draft:
-                        $ref: "#/components/schemas/Flow"
-
-
   /w/{workspace}/flows/exists/{path}:
     get:
       summary: exists flow by path
@@ -2783,12 +2713,7 @@ paths:
         content:
           application/json:
             schema:
-              allOf:
-                - $ref: "#/components/schemas/OpenFlowWPath"
-                - type: object
-                  properties:
-                    draft_only:
-                      type: boolean
+              $ref: "#/components/schemas/OpenFlowWPath"
 
       responses:
         "201":
@@ -2832,16 +2757,6 @@ paths:
       parameters:
         - $ref: "#/components/parameters/WorkspaceId"
         - $ref: "#/components/parameters/ScriptPath"
-      requestBody:
-        description: archiveFlow
-        required: true
-        content:
-          application/json:
-            schema:
-              type: object
-              properties:
-                archived:
-                  type: boolean
       responses:
         "200":
           description: flow archived
@@ -2867,27 +2782,6 @@ paths:
               schema:
                 type: string
 
-  /w/{workspace}/flows/input_history/p/{path}:
-    get:
-      summary: list inputs for previous completed flow jobs
-      operationId: getFlowInputHistoryByPath
-      tags:
-        - flow
-      parameters:
-        - $ref: "#/components/parameters/WorkspaceId"
-        - $ref: "#/components/parameters/ScriptPath"
-        - $ref: "#/components/parameters/Page"
-        - $ref: "#/components/parameters/PerPage"
-      responses:
-        "200":
-          description: input history for completed jobs with this flow path
-          content:
-            application/json:
-              schema:
-                type: array
-                items:
-                  $ref: "#/components/schemas/Input"
-
   /w/{workspace}/apps/list:
     get:
       summary: list all available apps
@@ -2961,24 +2855,6 @@ paths:
               schema:
                 $ref: "#/components/schemas/AppWithLastVersion"
 
-  /w/{workspace}/apps/get/draft/{path}:
-    get:
-      summary: get app by path with draft
-      operationId: getAppByPathWithDraft
-      tags:
-        - app
-      parameters:
-        - $ref: "#/components/parameters/WorkspaceId"
-        - $ref: "#/components/parameters/ScriptPath"
-      responses:
-        "200":
-          description: app details with draft
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/AppWithLastVersionWDraft"
-
-
   /w/{workspace}/apps_u/public_app/{path}:
     get:
       summary: get public app by secret
@@ -3053,8 +2929,6 @@ paths:
                   type: string
                 policy:
                   $ref: "#/components/schemas/Policy"
-                draft_only:
-                  type: boolean
               required:
                 - path
                 - value
@@ -3338,7 +3212,6 @@ paths:
         - $ref: "#/components/parameters/Running"
         - $ref: "#/components/parameters/ArgsFilter"
         - $ref: "#/components/parameters/ResultFilter"
-        - $ref: "#/components/parameters/Tag"
       responses:
         "200":
           description: All available queued jobs
@@ -3369,7 +3242,6 @@ paths:
         - $ref: "#/components/parameters/JobKinds"
         - $ref: "#/components/parameters/ArgsFilter"
         - $ref: "#/components/parameters/ResultFilter"
-        - $ref: "#/components/parameters/Tag"
         - name: is_skipped
           description: is the job skipped
           in: query
@@ -3407,7 +3279,6 @@ paths:
         - $ref: "#/components/parameters/StartedAfter"
         - $ref: "#/components/parameters/JobKinds"
         - $ref: "#/components/parameters/ArgsFilter"
-        - $ref: "#/components/parameters/Tag"
         - $ref: "#/components/parameters/ResultFilter"
         - name: is_skipped
           description: is the job skipped
@@ -3451,6 +3322,23 @@ paths:
               schema:
                 $ref: "#/components/schemas/Job"
 
+  # /w/{workspace}/jobs/flow/current_state/{id}:
+  #   get:
+  #     summary: get flow current step state
+  #     operationId: getJob
+  #     tags:
+  #       - job
+  #     parameters:
+  #       - $ref: "#/components/parameters/WorkspaceId"
+  #       - $ref: "#/components/parameters/JobId"
+  #     responses:
+  #       "200":
+  #         description: state details
+  #         content:
+  #           application/json:
+  #             schema:
+  #               type: string
+
   /w/{workspace}/jobs_u/getupdate/{id}:
     get:
       summary: get job updates
@@ -4670,118 +4558,6 @@ paths:
         "200":
           description: unstar item
 
-  /w/{workspace}/inputs/history:
-    get:
-      summary: List Inputs used in previously completed jobs
-      operationId: getInputHistory
-      tags:
-        - input
-      parameters:
-        - $ref: "#/components/parameters/WorkspaceId"
-        - $ref: "#/components/parameters/RunnableId"
-        - $ref: "#/components/parameters/RunnableType"
-        - $ref: "#/components/parameters/Page"
-        - $ref: "#/components/parameters/PerPage"
-      responses:
-        "200":
-          description: Input history for completed jobs
-          content:
-            application/json:
-              schema:
-                type: array
-                items:
-                  $ref: "#/components/schemas/Input"
-
-  /w/{workspace}/inputs/list:
-    get:
-      summary: List saved Inputs for a Runnable
-      operationId: listInputs
-      tags:
-        - input
-      parameters:
-        - $ref: "#/components/parameters/WorkspaceId"
-        - $ref: "#/components/parameters/RunnableId"
-        - $ref: "#/components/parameters/RunnableType"
-        - $ref: "#/components/parameters/Page"
-        - $ref: "#/components/parameters/PerPage"
-      responses:
-        "200":
-          description: Saved Inputs for a Runnable
-          content:
-            application/json:
-              schema:
-                type: array
-                items:
-                  $ref: "#/components/schemas/Input"
-
-  /w/{workspace}/inputs/create:
-    post:
-      summary: Create an Input for future use in a script or flow
-      operationId: createInput
-      tags:
-        - input
-      parameters:
-        - $ref: "#/components/parameters/WorkspaceId"
-        - $ref: "#/components/parameters/RunnableId"
-        - $ref: "#/components/parameters/RunnableType"
-      requestBody:
-        description: Input
-        required: true
-        content:
-          application/json:
-            schema:
-              $ref: "#/components/schemas/CreateInput"
-      responses:
-        "201":
-          description: Input created
-          content:
-            text/plain:
-              schema:
-                type: string
-                format: uuid
-
-  /w/{workspace}/inputs/update:
-    post:
-      summary: Update an Input
-      operationId: updateInput
-      tags:
-        - input
-      parameters:
-        - $ref: "#/components/parameters/WorkspaceId"
-      requestBody:
-        description: UpdateInput
-        required: true
-        content:
-          application/json:
-            schema:
-              $ref: "#/components/schemas/UpdateInput"
-      responses:
-        "201":
-          description: Input updated
-          content:
-            text/plain:
-              schema:
-                type: string
-                format: uuid
-
-  /w/{workspace}/inputs/delete/{input}:
-    post:
-      summary: Delete a Saved Input
-      operationId: deleteInput
-      tags:
-        - input
-      parameters:
-        - $ref: "#/components/parameters/WorkspaceId"
-        - $ref: "#/components/parameters/InputId"
-      responses:
-        "200":
-          description: Input deleted
-          content:
-            text/plain:
-              schema:
-                type: string
-                format: uuid
-
 components:
   securitySchemes:
     bearerAuth:
@@ -4791,7 +4567,6 @@ components:
       type: apiKey
       in: cookie
       name: token
-
   parameters:
     WorkspaceId:
       name: workspace
@@ -4910,6 +4685,7 @@ components:
       in: query
       schema:
         type: string
+
     ScriptStartPath:
       name: script_path_start
       description: mask to filter matching starting path
@@ -4966,12 +4742,7 @@ components:
       in: query
       schema:
         type: string
-    Tag:
-      name: tag
-      description: filter on jobs with a given tag/worker group
-      in: query
-      schema:
-        type: string
+
     ResultFilter:
       name: result
       description: filter on jobs containing those result as a json subset (@> in postgres)
@@ -5031,26 +4802,9 @@ components:
       #     type: string
       #     enum: ["preview", "script", "dependencies"]
       # explode: false
-    RunnableId:
-      name: runnable_id
-      in: query
-      schema:
-        type: string
-    RunnableType:
-      name: runnable_type
-      in: query
-      schema:
-        $ref: "#/components/schemas/RunnableType"
-    InputId:
-      name: input
-      in: path
-      required: true
-      schema:
-        type: string
 
   schemas:
     $ref: "../../openflow.openapi.yaml#/components/schemas"
-
     Script:
       type: object
       properties:
@@ -5101,12 +4855,6 @@ components:
           enum: [script, failure, trigger, command, approval]
         starred:
           type: boolean
-        tag:
-          type: string
-        has_draft:
-          type: boolean
-        draft_only:
-          type: boolean
       required:
         - hash
         - path
@@ -5123,115 +4871,10 @@ components:
         - kind
         - starred
 
-    NewScript:
-      type: object
-      properties:
-        path:
-          type: string
-        parent_hash:
-          type: string
-        summary:
-          type: string
-        description:
-          type: string
-        content:
-          type: string
-        schema:
-          type: object
-        is_template:
-          type: boolean
-        lock:
-          type: array
-          items:
-            type: string
-        language:
-          type: string
-          enum: [python3, deno, go, bash]
-        kind:
-          type: string
-          enum: [script, failure, trigger, command, approval]
-        tag:
-          type: string
-        draft_only:
-          type: boolean
-      required:
-        - path
-        - summary
-        - description
-        - content
-        - language
-
-    NewScriptWithDraft:
-      allOf:
-        - $ref: "#/components/schemas/NewScript"
-        - type: object
-          properties:
-            draft:
-              $ref: "#/components/schemas/NewScript"
-            hash:
-              type: string
-          required:
-            - hash
-
-
     ScriptArgs:
       type: object
       additionalProperties: {}
 
-    Input:
-      type: object
-      properties:
-        id:
-          type: string
-        name:
-          type: string
-        args:
-          type: object
-        created_by:
-          type: string
-        created_at:
-          type: string
-          format: date-time
-        is_public:
-          type: boolean
-      required:
-        - id
-        - name
-        - args
-        - created_by
-        - created_at
-        - is_public
-
-    CreateInput:
-      type: object
-      properties:
-        name:
-          type: string
-        args:
-          type: object
-      required:
-        - name
-        - args
-        - created_by
-
-    UpdateInput:
-      type: object
-      properties:
-        id:
-          type: string
-        name:
-          type: string
-        is_public:
-          type: boolean
-      required:
-        - id
-        - name
-        - is_public
-
-    RunnableType:
-      type: string
-      enum: ["ScriptHash", "ScriptPath", "FlowPath"]
-
     QueuedJob:
       type: object
       properties:
@@ -5309,8 +4952,6 @@ components:
           type: boolean
         mem_peak:
           type: integer
-        tag:
-          type: string
       required:
         - id
         - running
@@ -5320,7 +4961,6 @@ components:
         - is_flow_step
         - email
         - visible_to_owner
-        - tag
 
     CompletedJob:
       type: object
@@ -5400,8 +5040,6 @@ components:
           type: boolean
         mem_peak:
           type: integer
-        tag:
-          type: string
       required:
         - id
         - created_by
@@ -5416,7 +5054,6 @@ components:
         - is_skipped
         - email
         - visible_to_owner
-        - tag
 
     Job:
       allOf:
@@ -5457,10 +5094,6 @@ components:
           type: array
           items:
             type: string
-        folders_owners:
-          type: array
-          items:
-            type: string
         usage:
           $ref: "#/components/schemas/Usage"
       required:
@@ -5472,7 +5105,6 @@ components:
         - operator
         - disabled
         - folders
-        - folders_owners
 
     Usage:
       type: object
@@ -5812,8 +5444,6 @@ components:
         language:
           type: string
           enum: [python3, deno, go, bash]
-        tag:
-          type: string
 
       required:
         - content
@@ -6185,8 +5815,6 @@ components:
           type: boolean
         starred:
           type: boolean
-        draft_only:
-          type: boolean
       required:
         - path
         - edited_by
@@ -6268,7 +5896,6 @@ components:
         - extra_perms
         - edited_at
         - execution_mode
-
     AppWithLastVersion:
       type: object
       properties:
@@ -6312,15 +5939,6 @@ components:
         - execution_mode
         - extra_perms
 
-    AppWithLastVersionWDraft:
-      allOf:
-        - $ref: "#/components/schemas/AppWithLastVersion"
-        - type: object
-          properties:
-            draft_only:
-              type: boolean
-            draft: {}
-
     SlackToken:
       type: object
       properties:

backend/windmill-api/src/apps.rs

@@ -9,6 +9,7 @@ use std::collections::HashMap;
  */
 use crate::{
     db::{UserDB, DB},
+    jobs::script_path_to_payload,
     users::{require_owner_of_path, Authed, OptAuthed},
     variables::build_crypt,
     webhook_util::{WebhookMessage, WebhookShared},
@@ -31,19 +32,17 @@ use windmill_audit::{audit_log, ActionKind};
 use windmill_common::{
     apps::ListAppQuery,
     error::{to_anyhow, Error, JsonResult, Result},
-    jobs::{script_path_to_payload, JobPayload, RawCode},
     users::username_to_permissioned_as,
     utils::{
         http_get_from_hub, list_elems_from_hub, not_found_if_none, paginate, Pagination, StripPath,
     },
 };
-use windmill_queue::{push, QueueTransaction};
+use windmill_queue::{push, JobPayload, RawCode};
 
 pub fn workspaced_service() -> Router {
     Router::new()
         .route("/list", get(list_apps))
         .route("/get/p/*path", get(get_app))
-        .route("/get/draft/*path", get(get_app_w_draft))
         .route("/secret_of/*path", get(get_secret_id))
         .route("/get/v/*id", get(get_app_by_id))
         .route("/exists/*path", get(exists_app))
@@ -75,9 +74,6 @@ pub struct ListableApp {
     pub execution_mode: String,
     pub starred: bool,
     pub edited_at: chrono::DateTime<chrono::Utc>,
-    pub has_draft: bool,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub draft_only: Option<bool>,
 }
 
 #[derive(FromRow, Serialize, Deserialize)]
@@ -102,23 +98,6 @@ pub struct AppWithLastVersion {
     pub extra_perms: serde_json::Value,
 }
 
-#[derive(Serialize, Deserialize)]
-pub struct AppWithLastVersionAndDraft {
-    pub id: i64,
-    pub path: String,
-    pub summary: String,
-    pub policy: serde_json::Value,
-    pub versions: Vec<i64>,
-    pub value: serde_json::Value,
-    pub created_by: String,
-    pub created_at: chrono::DateTime<chrono::Utc>,
-    pub extra_perms: serde_json::Value,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub draft: Option<serde_json::Value>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub draft_only: Option<bool>,
-}
-
 pub type StaticFields = Map<String, Value>;
 
 #[derive(Serialize, Deserialize, Debug, PartialEq, Clone)]
@@ -147,7 +126,6 @@ pub struct CreateApp {
     pub summary: String,
     pub value: serde_json::Value,
     pub policy: Policy,
-    pub draft_only: Option<bool>,
 }
 
 #[derive(Deserialize)]
@@ -178,8 +156,6 @@ async fn list_apps(
             "app_version.created_at as edited_at",
             "app.extra_perms",
             "favorite.path IS NOT NULL as starred",
-            "draft.path IS NOT NULL as has_draft",
-            "draft_only"
         ])
         .left()
         .join("favorite")
@@ -192,11 +168,6 @@ async fn list_apps(
         .on(
             "app_version.id = versions[array_upper(versions, 1)]"
         )
-        .left()
-        .join("draft")
-        .on(
-            "draft.path = app.path AND draft.workspace_id = app.workspace_id AND draft.typ = 'app'"
-        )
         .order_desc("favorite.path IS NOT NULL")
         .order_by("app_version.created_at", true)
         .and_where("app.workspace_id = ?".bind(&w_id))
@@ -244,37 +215,6 @@ async fn get_app(
     Ok(Json(app))
 }
 
-async fn get_app_w_draft(
-    authed: Authed,
-    Extension(user_db): Extension<UserDB>,
-    Path((w_id, path)): Path<(String, StripPath)>,
-) -> JsonResult<AppWithLastVersionAndDraft> {
-    let path = path.to_path();
-    let mut tx = user_db.begin(&authed).await?;
-
-    let app_o = sqlx::query_as!(
-        AppWithLastVersionAndDraft,
-        r#"SELECT app.id, app.path, app.summary, app.versions, app.policy,
-        app.extra_perms, app_version.value, 
-        app_version.created_at, app_version.created_by,
-        app.draft_only, draft.value as "draft?"
-        from app
-        INNER JOIN app_version ON
-        app_version.id = app.versions[array_upper(app.versions, 1)]
-        LEFT JOIN draft ON 
-        app.path = draft.path AND app.workspace_id = draft.workspace_id AND draft.typ = 'app' 
-        WHERE app.path = $1 AND app.workspace_id = $2"#,
-        path.to_owned(),
-        &w_id
-    )
-    .fetch_optional(&mut tx)
-    .await?;
-    tx.commit().await?;
-
-    let app = not_found_if_none(app_o, "App", path)?;
-    Ok(Json(app))
-}
-
 async fn get_app_by_id(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
@@ -384,23 +324,14 @@ async fn create_app(
         return Err(Error::BadRequest("App path cannot be empty".to_string()));
     }
 
-    sqlx::query!(
-        "DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'app'",
-        &app.path,
-        &w_id
-    )
-    .execute(&mut tx)
-    .await?;
-
     let id = sqlx::query_scalar!(
         "INSERT INTO app
-            (workspace_id, path, summary, policy, versions, draft_only)
-            VALUES ($1, $2, $3, $4, '{}', $5) RETURNING id",
+            (workspace_id, path, summary, policy, versions)
+            VALUES ($1, $2, $3, $4, '{}') RETURNING id",
         w_id,
         app.path,
         app.summary,
         json!(app.policy),
-        app.draft_only,
     )
     .fetch_one(&mut tx)
     .await?;
@@ -480,14 +411,6 @@ async fn delete_app(
     let path = path.to_path();
     let mut tx = user_db.begin(&authed).await?;
 
-    sqlx::query!(
-        "DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'app'",
-        path,
-        &w_id
-    )
-    .execute(&mut tx)
-    .await?;
-
     sqlx::query!(
         "DELETE FROM app WHERE path = $1 AND workspace_id = $2",
         path,
@@ -518,6 +441,7 @@ async fn update_app(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
     Extension(webhook): Extension<WebhookShared>,
+    Extension(db): Extension<DB>,
     Path((w_id, path)): Path<(String, StripPath)>,
     Json(ns): Json<EditApp>,
 ) -> Result<String> {
@@ -532,10 +456,12 @@ async fn update_app(
         sqlb.and_where_eq("path", "?".bind(&path));
         sqlb.and_where_eq("workspace_id", "?".bind(&w_id));
 
-        sqlb.set("draft_only", "NULL");
         if let Some(npath) = &ns.path {
             if npath != path {
-                require_owner_of_path(&authed, path)?;
+                if !authed.is_admin {
+                    require_owner_of_path(&w_id, &authed.username, &authed.groups, &path, &db)
+                        .await?;
+                }
             }
             sqlb.set_str("path", npath);
         }
@@ -595,15 +521,6 @@ async fn update_app(
         .execute(&mut tx)
         .await?;
     }
-
-    sqlx::query!(
-        "DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'app'",
-        path,
-        &w_id
-    )
-    .execute(&mut tx)
-    .await?;
-
     audit_log(
         &mut tx,
         &authed.username,
@@ -648,7 +565,6 @@ fn digest(code: &str) -> String {
 async fn execute_component(
     OptAuthed(opt_authed): OptAuthed,
     Extension(db): Extension<DB>,
-    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
     Path((w_id, path)): Path<(String, StripPath)>,
     Json(payload): Json<ExecuteApp>,
 ) -> Result<String> {
@@ -667,7 +583,7 @@ async fn execute_component(
     };
 
     let path = path.to_path();
-    let mut tx: QueueTransaction<'_, _> = (rsmq, db.begin().await?).into();
+    let mut tx = db.begin().await?;
 
     let policy = if let Some(static_fields) = payload.clone().force_viewer_static_fields {
         let mut hm = HashMap::new();
@@ -726,27 +642,20 @@ async fn execute_component(
         }
     };
 
-    let (job_payload, args, tag) = match &payload {
+    let (job_payload, args) = match &payload {
         ExecuteApp { args, raw_code: Some(raw_code), path: None, .. } => {
             let content = &raw_code.content;
             let payload = JobPayload::Code(raw_code.clone());
             let path = digest(content);
             let args = build_args(policy, path, args)?;
-            (payload, args, None)
+            (payload, args)
         }
         ExecuteApp { args, raw_code: None, path: Some(path), .. } => {
-            let (payload, tag) = if path.starts_with("script/") {
-                script_path_to_payload(
-                    path.strip_prefix("script/").unwrap(),
-                    tx.transaction_mut(),
-                    &w_id,
-                )
-                .await?
+            let payload = if path.starts_with("script/") {
+                script_path_to_payload(path.strip_prefix("script/").unwrap(), &mut tx, &w_id)
+                    .await?
             } else if path.starts_with("flow/") {
-                (
-                    JobPayload::Flow(path.strip_prefix("flow/").unwrap().to_string()),
-                    None,
-                )
+                JobPayload::Flow(path.strip_prefix("flow/").unwrap().to_string())
             } else {
                 return Err(Error::BadRequest(format!(
                     "path must start with script/ or flow/ (got {})",
@@ -754,7 +663,7 @@ async fn execute_component(
                 )));
             };
             let args = build_args(policy, path.to_string(), args)?;
-            (payload, args, tag)
+            (payload, args)
         }
         _ => unreachable!(),
     };
@@ -775,11 +684,10 @@ async fn execute_component(
         false,
         None,
         true,
-        tag,
     )
     .await?;
-    tx.commit().await?;
 
+    tx.commit().await?;
     Ok(uuid.to_string())
 }
 

backend/windmill-api/src/drafts.rs

@@ -1,94 +0,0 @@
-/*
- * Author: Ruben Fiszel
- * Copyright: Windmill Labs, Inc 2042
- * This file and its contents are licensed under the AGPLv3 License.
- * Please see the included NOTICE for copyright information and
- * LICENSE-AGPL for a copy of the license.
- */
-
-use crate::{
-    db::{UserDB, DB},
-    users::{maybe_refresh_folders, require_owner_of_path, Authed},
-};
-
-use axum::{
-    extract::{Extension, Path},
-    routing::post,
-    Json, Router,
-};
-use hyper::StatusCode;
-use serde::{Deserialize, Serialize};
-use windmill_common::error::Result;
-
-pub fn workspaced_service() -> Router {
-    Router::new().route("/create", post(create_draft))
-}
-
-#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Clone)]
-#[sqlx(type_name = "DRAFT_TYPE", rename_all = "lowercase")]
-#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))]
-pub enum DraftType {
-    Script,
-    Flow,
-    App,
-}
-
-#[derive(Deserialize, Serialize, Debug)]
-pub struct Draft {
-    pub path: String,
-    pub value: serde_json::Value,
-    pub typ: DraftType,
-}
-
-async fn create_draft(
-    authed: Authed,
-    Extension(db): Extension<DB>,
-    Extension(user_db): Extension<UserDB>,
-    Path(w_id): Path<String>,
-    Json(draft): Json<Draft>,
-) -> Result<(StatusCode, String)> {
-    let authed = maybe_refresh_folders(&draft.path, &w_id, authed, &db).await;
-
-    let mut tx = user_db.begin(&authed).await?;
-
-    require_owner_of_path(&authed, &draft.path)?;
-
-    sqlx::query!(
-        "INSERT INTO draft
-            (workspace_id, path, value, typ)
-            VALUES ($1, $2, $3, $4)
-            ON CONFLICT (workspace_id, path, typ) DO UPDATE SET value = $3",
-        &w_id,
-        draft.path,
-        draft.value,
-        draft.typ: DraftType,
-    )
-    .execute(&mut tx)
-    .await?;
-
-    tx.commit().await?;
-
-    Ok((StatusCode::CREATED, format!("draft {} created", draft.path)))
-}
-
-// async fn get_draft(
-//     authed: Authed,
-//     Extension(user_db): Extension<UserDB>,
-//     Path((w_id, path)): Path<(String, StripPath)>,
-// ) -> JsonResult<Draft> {
-//     let path = path.to_path();
-//     let mut tx = user_db.begin(&authed).await?;
-
-//     let script_o = sqlx::query_as!(
-//         Draft,
-//         r#"SELECT path, value, typ as "typ: DraftType" FROM draft WHERE path = $1 AND workspace_id = $2"#,
-//         path,
-//         w_id
-//     )
-//     .fetch_optional(&mut tx)
-//     .await?;
-//     tx.commit().await?;
-
-//     let draft = not_found_if_none(script_o, "draft", path)?;
-//     Ok(Json(draft))
-// }

backend/windmill-api/src/flows.rs

@@ -6,6 +6,27 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
+use hyper::StatusCode;
+use sql_builder::prelude::*;
+
+use axum::{
+    extract::{Extension, Path, Query},
+    routing::{delete, get, post},
+    Json, Router,
+};
+use sql_builder::SqlBuilder;
+use sqlx::{Postgres, Transaction};
+use windmill_audit::{audit_log, ActionKind};
+use windmill_common::{
+    error::{self, to_anyhow, Error, JsonResult, Result},
+    flows::{Flow, ListFlowQuery, ListableFlow, NewFlow},
+    schedule::Schedule,
+    utils::{
+        http_get_from_hub, list_elems_from_hub, not_found_if_none, paginate, Pagination, StripPath,
+    },
+};
+use windmill_queue::{push, schedule::push_scheduled_job, JobPayload};
+
 use crate::{
     db::{UserDB, DB},
     schedule::clear_schedule,
@@ -13,28 +34,6 @@ use crate::{
     webhook_util::{WebhookMessage, WebhookShared},
     HTTP_CLIENT,
 };
-use axum::{
-    extract::{Extension, Path, Query},
-    routing::{delete, get, post},
-    Json, Router,
-};
-use hyper::StatusCode;
-use serde::{Deserialize, Serialize};
-use sql_builder::prelude::*;
-use sql_builder::SqlBuilder;
-use sqlx::{Postgres, Transaction};
-use windmill_audit::{audit_log, ActionKind};
-use windmill_common::{
-    error::{self, to_anyhow, Error, JsonResult, Result},
-    flows::{Flow, ListFlowQuery, ListableFlow, NewFlow},
-    jobs::JobPayload,
-    schedule::Schedule,
-    scripts::Schema,
-    utils::{
-        http_get_from_hub, list_elems_from_hub, not_found_if_none, paginate, Pagination, StripPath,
-    },
-};
-use windmill_queue::{push, schedule::push_scheduled_job, QueueTransaction};
 
 pub fn workspaced_service() -> Router {
     Router::new()
@@ -44,7 +43,6 @@ pub fn workspaced_service() -> Router {
         .route("/archive/*path", post(archive_flow_by_path))
         .route("/delete/*path", delete(delete_flow_by_path))
         .route("/get/*path", get(get_flow_by_path))
-        .route("/get/draft/*path", get(get_flow_by_path_w_draft))
         .route("/exists/*path", get(exists_flow_by_path))
         .route("/list_paths", get(list_paths))
 }
@@ -75,8 +73,6 @@ async fn list_flows(
             "archived",
             "extra_perms",
             "favorite.path IS NOT NULL as starred",
-            "draft.path IS NOT NULL as has_draft",
-            "draft_only"
         ])
         .left()
         .join("favorite")
@@ -84,11 +80,6 @@ async fn list_flows(
             "favorite.favorite_kind = 'flow' AND favorite.workspace_id = o.workspace_id AND favorite.path = o.path AND favorite.usr = ?"
                 .bind(&authed.username),
         )
-        .left()
-        .join("draft")
-        .on(
-            "draft.path = o.path AND draft.workspace_id = o.workspace_id AND draft.typ = 'flow'"
-        )
         .order_desc("favorite.path IS NOT NULL")
         .order_by("edited_at", lq.order_desc.unwrap_or(true))
         .and_where("o.workspace_id = ?".bind(&w_id))
@@ -96,8 +87,9 @@ async fn list_flows(
         .limit(per_page)
         .clone();
 
-    sqlb.and_where_eq("archived", lq.show_archived.unwrap_or(false));
-
+    if !lq.show_archived.unwrap_or(false) {
+        sqlb.and_where_eq("archived", false);
+    }
     if let Some(ps) = &lq.path_start {
         sqlb.and_where_like_left("path", "?".bind(ps));
     }
@@ -188,7 +180,6 @@ async fn create_flow(
     authed: Authed,
     Extension(db): Extension<DB>,
     Extension(user_db): Extension<UserDB>,
-    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
     Extension(webhook): Extension<WebhookShared>,
     Path(w_id): Path<String>,
     Json(nf): Json<NewFlow>,
@@ -196,14 +187,14 @@ async fn create_flow(
     // cron::Schedule::from_str(&ns.schedule).map_err(|e| error::Error::BadRequest(e.to_string()))?;
     let authed = maybe_refresh_folders(&nf.path, &w_id, authed, &db).await;
 
-    let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.clone().begin(&authed).await?).into();
+    let mut tx = user_db.clone().begin(&authed).await?;
 
-    check_path_conflict(tx.transaction_mut(), &w_id, &nf.path).await?;
-    check_schedule_conflict(tx.transaction_mut(), &w_id, &nf.path).await?;
+    check_path_conflict(&mut tx, &w_id, &nf.path).await?;
+    check_schedule_conflict(&mut tx, &w_id, &nf.path).await?;
 
     sqlx::query!(
         "INSERT INTO flow (workspace_id, path, summary, description, value, edited_by, edited_at, \
-         schema, dependency_job, draft_only) VALUES ($1, $2, $3, $4, $5, $6, now(), $7::text::json, NULL, $8)",
+         schema, dependency_job) VALUES ($1, $2, $3, $4, $5, $6, now(), $7::text::json, NULL)",
         w_id,
         nf.path,
         nf.summary,
@@ -211,15 +202,6 @@ async fn create_flow(
         nf.value,
         &authed.username,
         nf.schema.and_then(|x| serde_json::to_string(&x.0).ok()),
-        nf.draft_only
-    )
-    .execute(&mut tx)
-    .await?;
-
-    sqlx::query!(
-        "DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'flow'",
-        nf.path,
-        &w_id
     )
     .execute(&mut tx)
     .await?;
@@ -240,11 +222,13 @@ async fn create_flow(
     )
     .await?;
 
+    tx.commit().await?;
     webhook.send_message(
         w_id.clone(),
         WebhookMessage::CreateFlow { workspace: w_id.clone(), path: nf.path.clone() },
     );
 
+    let tx = user_db.begin(&authed).await?;
     let (dependency_job_uuid, mut tx) = push(
         tx,
         &w_id,
@@ -261,10 +245,8 @@ async fn create_flow(
         false,
         None,
         true,
-        None,
     )
     .await?;
-
     sqlx::query!(
         "UPDATE flow SET dependency_job = $1 WHERE path = $2 AND workspace_id = $3",
         dependency_job_uuid,
@@ -304,7 +286,6 @@ async fn check_schedule_conflict<'c>(
 async fn update_flow(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
-    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
     Extension(db): Extension<DB>,
     Extension(webhook): Extension<WebhookShared>,
     Path((w_id, flow_path)): Path<(String, StripPath)>,
@@ -313,9 +294,9 @@ async fn update_flow(
     let flow_path = flow_path.to_path();
     let authed = maybe_refresh_folders(&flow_path, &w_id, authed, &db).await;
 
-    let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.clone().begin(&authed).await?).into();
+    let mut tx = user_db.clone().begin(&authed).await?;
 
-    check_schedule_conflict(tx.transaction_mut(), &w_id, flow_path).await?;
+    check_schedule_conflict(&mut tx, &w_id, flow_path).await?;
 
     let schema = nf.schema.map(|x| x.0);
     let old_dep_job = sqlx::query_scalar!(
@@ -328,7 +309,7 @@ async fn update_flow(
     let old_dep_job = not_found_if_none(old_dep_job, "Flow", flow_path)?;
     sqlx::query!(
         "UPDATE flow SET path = $1, summary = $2, description = $3, value = $4, edited_by = $5, \
-         edited_at = now(), schema = $6::text::json, dependency_job = NULL, draft_only = NULL WHERE path = $7 AND workspace_id = $8",
+         edited_at = now(), schema = $6::text::json, dependency_job = NULL WHERE path = $7 AND workspace_id = $8",
         nf.path,
         nf.summary,
         nf.description,
@@ -342,52 +323,44 @@ async fn update_flow(
     .await?;
 
     if nf.path != flow_path {
-        check_schedule_conflict(tx.transaction_mut(), &w_id, &nf.path).await?;
+        check_schedule_conflict(&mut tx, &w_id, &nf.path).await?;
 
         if !authed.is_admin {
-            require_owner_of_path(&authed, flow_path)?;
+            require_owner_of_path(&w_id, &authed.username, &authed.groups, &flow_path, &db).await?;
         }
-    }
 
-    let mut schedulables: Vec<Schedule> = sqlx::query_as!(
-        Schedule,
-            "UPDATE schedule SET script_path = $1 WHERE script_path = $2 AND path != $2 AND workspace_id = $3 AND is_flow IS true RETURNING *",
+        let mut schedulables = sqlx::query_as!(
+            Schedule,
+                "UPDATE schedule SET script_path = $1 WHERE script_path = $2 AND path != $2 AND workspace_id = $3 AND is_flow IS true RETURNING *",
+                nf.path,
+                flow_path,
+                w_id,
+            )
+            .fetch_all(&mut tx)
+            .await?;
+
+        let schedule = sqlx::query_as!(Schedule,
+            "UPDATE schedule SET path = $1, script_path = $1 WHERE path = $2 AND workspace_id = $3 AND is_flow IS true RETURNING *",
             nf.path,
             flow_path,
             w_id,
         )
-        .fetch_all(&mut tx)
+        .fetch_optional(&mut tx)
         .await?;
 
-    let schedule = sqlx::query_as!(Schedule,
-        "UPDATE schedule SET path = $1, script_path = $1 WHERE path = $2 AND workspace_id = $3 AND is_flow IS true RETURNING *",
-        nf.path,
-        flow_path,
-        w_id,
-    )
-    .fetch_optional(&mut tx)
-    .await?;
+        if let Some(schedule) = schedule {
+            schedulables.push(schedule);
+        }
 
-    if let Some(schedule) = schedule {
-        schedulables.push(schedule);
-    }
+        for schedule in schedulables {
+            clear_schedule(&mut tx, flow_path, true).await?;
 
-    for schedule in schedulables.into_iter() {
-        clear_schedule(tx.transaction_mut(), &schedule.path, true).await?;
-
-        if schedule.enabled {
-            tx = push_scheduled_job(tx, schedule).await?;
+            if schedule.enabled {
+                tx = push_scheduled_job(tx, schedule).await?;
+            }
         }
     }
 
-    sqlx::query!(
-        "DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'flow'",
-        flow_path,
-        &w_id
-    )
-    .execute(&mut tx)
-    .await?;
-
     audit_log(
         &mut tx,
         &authed.username,
@@ -404,6 +377,7 @@ async fn update_flow(
     )
     .await?;
 
+    tx.commit().await?;
     webhook.send_message(
         w_id.clone(),
         WebhookMessage::UpdateFlow {
@@ -413,6 +387,7 @@ async fn update_flow(
         },
     );
 
+    let tx = user_db.begin(&authed).await?;
     let (dependency_job_uuid, mut tx) = push(
         tx,
         &w_id,
@@ -429,7 +404,6 @@ async fn update_flow(
         false,
         None,
         true,
-        None,
     )
     .await?;
     sqlx::query!(
@@ -472,44 +446,6 @@ async fn get_flow_by_path(
     Ok(Json(flow))
 }
 
-#[derive(Serialize, sqlx::FromRow)]
-pub struct FlowWDraft {
-    pub path: String,
-    pub summary: String,
-    pub description: String,
-    pub schema: Option<Schema>,
-    pub value: serde_json::Value,
-    pub extra_perms: serde_json::Value,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub draft: Option<serde_json::Value>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub draft_only: Option<bool>,
-}
-
-async fn get_flow_by_path_w_draft(
-    authed: Authed,
-    Extension(user_db): Extension<UserDB>,
-    Path((w_id, path)): Path<(String, StripPath)>,
-) -> JsonResult<FlowWDraft> {
-    let path = path.to_path();
-    let mut tx = user_db.begin(&authed).await?;
-
-    let flow_o = sqlx::query_as::<_, FlowWDraft>(
-        "SELECT flow.path, flow.summary, flow,description, flow.schema, flow.value, flow.extra_perms, flow.draft_only, draft.value as draft FROM flow
-        LEFT JOIN draft ON 
-        flow.path = draft.path AND flow.workspace_id = draft.workspace_id AND draft.typ = 'flow' 
-        WHERE flow.path = $1 AND flow.workspace_id = $2",
-    )
-    .bind(path)
-    .bind(w_id)
-    .fetch_optional(&mut tx)
-    .await?;
-    tx.commit().await?;
-
-    let flow = not_found_if_none(flow_o, "Flow", path)?;
-    Ok(Json(flow))
-}
-
 async fn exists_flow_by_path(
     Extension(db): Extension<DB>,
     Path((w_id, path)): Path<(String, StripPath)>,
@@ -528,24 +464,17 @@ async fn exists_flow_by_path(
     Ok(Json(exists))
 }
 
-#[derive(Deserialize)]
-struct Archived {
-    archived: Option<bool>,
-}
-
 async fn archive_flow_by_path(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
     Extension(webhook): Extension<WebhookShared>,
     Path((w_id, path)): Path<(String, StripPath)>,
-    Json(archived): Json<Archived>,
 ) -> Result<String> {
     let path = path.to_path();
     let mut tx = user_db.begin(&authed).await?;
 
     sqlx::query!(
-        "UPDATE flow SET archived = $1 WHERE path = $2 AND workspace_id = $3",
-        archived.archived.unwrap_or(true),
+        "UPDATE flow SET archived = true WHERE path = $1 AND workspace_id = $2",
         path,
         &w_id
     )
@@ -580,14 +509,6 @@ async fn delete_flow_by_path(
     let path = path.to_path();
     let mut tx = user_db.begin(&authed).await?;
 
-    sqlx::query!(
-        "DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'flow'",
-        path,
-        &w_id
-    )
-    .execute(&mut tx)
-    .await?;
-
     sqlx::query!(
         "DELETE FROM flow WHERE path = $1 AND workspace_id = $2",
         path,
@@ -659,7 +580,6 @@ mod tests {
                         language: scripts::ScriptLang::Deno,
                         path: None,
                         lock: None,
-                        tag: None,
                     },
                     stop_after_if: Some(StopAfterIf {
                         expr: "foo = 'bar'".to_string(),

backend/windmill-api/src/folders.rs

@@ -22,13 +22,13 @@ use lazy_static::lazy_static;
 use regex::Regex;
 use windmill_audit::{audit_log, ActionKind};
 use windmill_common::{
-    error::{self, to_anyhow, JsonResult, Result},
+    error::{self, to_anyhow, Error, JsonResult, Result},
     users::username_to_permissioned_as,
     utils::{not_found_if_none, paginate, Pagination},
 };
 
 use serde::{Deserialize, Serialize};
-use sqlx::{FromRow, Postgres, Transaction};
+use sqlx::{query_scalar, FromRow, Postgres, Transaction};
 
 pub fn workspaced_service() -> Router {
     Router::new()
@@ -41,7 +41,7 @@ pub fn workspaced_service() -> Router {
         .route("/delete/:name", delete(delete_folder))
         .route("/addowner/:name", post(add_owner))
         .route("/removeowner/:name", post(remove_owner))
-        .route("/is_owner/*path", get(is_owner_api))
+        .route("/is_owner/*path", get(is_owner))
 }
 
 #[derive(FromRow, Serialize, Deserialize, Clone)]
@@ -219,29 +219,47 @@ async fn create_folder(
     Ok(format!("Created folder {}", ng.name))
 }
 
-pub async fn is_owner_api(
-    authed: Authed,
-    Path((_w_id, name)): Path<(String, String)>,
+pub async fn is_owner(
+    Authed { username, is_admin, groups, .. }: Authed,
+    Extension(db): Extension<DB>,
+    Path((w_id, name)): Path<(String, String)>,
 ) -> JsonResult<bool> {
-    Ok(Json(is_owner(&authed, &name)))
-}
-
-pub fn is_owner(Authed { is_admin, folders, .. }: &Authed, name: &str) -> bool {
-    if *is_admin {
-        true
+    if is_admin {
+        Ok(Json(true))
     } else {
-        folders.into_iter().any(|x| x.0 == name && x.2)
+        Ok(Json(
+            require_is_owner(&name, &username, &groups, &w_id, &db)
+                .await
+                .is_ok(),
+        ))
     }
 }
 
-pub fn require_is_owner(authed: &Authed, name: &str) -> Result<()> {
-    if is_owner(authed, name) {
-        Ok(())
-    } else {
-        Err(windmill_common::error::Error::NotAuthorized(format!(
-            "You are not owner of the folder {}",
-            name
+pub async fn require_is_owner(
+    folder_name: &str,
+    username: &str,
+    groups: &Vec<String>,
+    w_id: &str,
+    db: &DB,
+) -> Result<()> {
+    let is_owner = query_scalar!(
+        "SELECT EXISTS(SELECT 1 FROM folder WHERE CONCAT('u/', $1::text) = ANY(owners) AND name = $2 AND workspace_id = $4) OR exists(
+            SELECT 1 FROM folder, unnest(folder.owners) as o
+            WHERE o = ANY($3::text[]) AND folder.name = $2 AND folder.workspace_id = $4)",
+        username,
+        folder_name,
+        groups,
+        w_id,
+    ).fetch_one(db)
+    .await?
+    .unwrap_or(false);
+    if !is_owner {
+        Err(Error::BadRequest(format!(
+            "{} is not an owner of {} and hence is not authorized to perform this operation",
+            username, folder_name
         )))
+    } else {
+        Ok(())
     }
 }
 
@@ -473,6 +491,7 @@ async fn delete_folder(
 
 async fn add_owner(
     authed: Authed,
+    Extension(db): Extension<DB>,
     Extension(user_db): Extension<UserDB>,
     Extension(webhook): Extension<WebhookShared>,
     Path((w_id, name)): Path<(String, String)>,
@@ -481,7 +500,9 @@ async fn add_owner(
     let mut tx = user_db.begin(&authed).await?;
 
     not_found_if_none(get_folderopt(&mut tx, &w_id, &name).await?, "Folder", &name)?;
-    require_is_owner(&authed, &name)?;
+    if !authed.is_admin {
+        require_is_owner(&name, &authed.username, &authed.groups, &w_id, &db).await?;
+    }
 
     sqlx::query!(
         "UPDATE folder SET owners = array_append(owners, $1) WHERE name = $2 AND workspace_id = $3 AND NOT $1 = ANY(owners) RETURNING name",
@@ -517,14 +538,14 @@ pub async fn get_folders_for_user(
     username: &str,
     groups: &[String],
     db: &DB,
-) -> Result<Vec<(String, bool, bool)>> {
+) -> Result<Vec<(String, bool)>> {
     let mut perms = groups
         .into_iter()
         .map(|x| format!("g/{}", x))
         .collect::<Vec<_>>();
     perms.insert(0, format!("u/{}", username));
     let folders = sqlx::query!(
-        "SELECT name, (EXISTS (SELECT 1 FROM (SELECT key, value FROM jsonb_each_text(extra_perms) WHERE key = ANY($1)) t  WHERE value::boolean IS true)) as write, $1 && owners::text[] as owner  FROM folder
+        "SELECT name, (EXISTS (SELECT 1 FROM (SELECT key, value FROM jsonb_each_text(extra_perms) WHERE key = ANY($1)) t  WHERE value::boolean IS true)) as write  FROM folder
         WHERE extra_perms ?| $1  AND workspace_id = $2",
         &perms[..],
         w_id,
@@ -532,7 +553,7 @@ pub async fn get_folders_for_user(
     .fetch_all(db)
     .await?
     .into_iter()
-    .map(|x| (x.name, x.write.unwrap_or(false), x.owner.unwrap_or(false)))
+    .map(|x| (x.name, x.write.unwrap_or(false)))
     .collect();
 
     Ok(folders)
@@ -540,6 +561,7 @@ pub async fn get_folders_for_user(
 
 async fn remove_owner(
     authed: Authed,
+    Extension(db): Extension<DB>,
     Extension(user_db): Extension<UserDB>,
     Extension(webhook): Extension<WebhookShared>,
     Path((w_id, name)): Path<(String, String)>,
@@ -548,7 +570,9 @@ async fn remove_owner(
     let mut tx = user_db.begin(&authed).await?;
 
     not_found_if_none(get_folderopt(&mut tx, &w_id, &name).await?, "Folder", &name)?;
-    require_is_owner(&authed, &name)?;
+    if !authed.is_admin {
+        require_is_owner(&name, &authed.username, &authed.groups, &w_id, &db).await?;
+    }
 
     sqlx::query!(
         "UPDATE folder SET owners = array_remove(owners, $1) WHERE name = $2 AND workspace_id = $3 RETURNING name",

backend/windmill-api/src/granular_acls.rs

@@ -43,29 +43,25 @@ async fn add_granular_acl(
     Json(GranularAcl { owner, write }): Json<GranularAcl>,
 ) -> Result<String> {
     let path = path.to_path();
-
     let (kind, path) = path
         .split_once('/')
         .ok_or_else(|| Error::BadRequest("Invalid path or kind".to_string()))?;
     let mut tx = user_db.begin(&authed).await?;
 
+    if !authed.is_admin {
+        if kind == "folder" {
+            crate::folders::require_is_owner(&path, &authed.username, &authed.groups, &w_id, &db)
+                .await?;
+        } else if kind == "group_" {
+        } else {
+            require_owner_of_path(&w_id, &authed.username, &authed.groups, &path, &db).await?;
+        }
+    }
     let identifier = if kind == "group_" || kind == "folder" {
         "name"
     } else {
         "path"
     };
-
-    if !authed.is_admin {
-        if kind == "folder" {
-            crate::folders::require_is_owner(&authed, path)?;
-        } else if kind == "group_" {
-            crate::groups::require_is_owner(path, &authed.username, &authed.groups, &w_id, &db)
-                .await?;
-        } else {
-            require_owner_of_path(&authed, path)?;
-        }
-    }
-
     let obj_o = sqlx::query_scalar::<_, serde_json::Value>(&format!(
         "UPDATE {kind} SET extra_perms = jsonb_set(extra_perms, '{{\"{owner}\"}}', to_jsonb($1), \
          true) WHERE {identifier} = $2 AND workspace_id = $3 RETURNING extra_perms"
@@ -90,22 +86,12 @@ async fn remove_granular_acl(
     Json(GranularAcl { owner, write: _ }): Json<GranularAcl>,
 ) -> Result<String> {
     let path = path.to_path();
-
+    if !authed.is_admin {
+        require_owner_of_path(&w_id, &authed.username, &authed.groups, &path, &db).await?;
+    }
     let (kind, path) = path
         .split_once('/')
         .ok_or_else(|| Error::BadRequest("Invalid path or kind".to_string()))?;
-
-    if !authed.is_admin {
-        if kind == "folder" {
-            crate::folders::require_is_owner(&authed, path)?;
-        } else if kind == "group_" {
-            crate::groups::require_is_owner(path, &authed.username, &authed.groups, &w_id, &db)
-                .await?;
-        } else {
-            require_owner_of_path(&authed, path)?;
-        }
-    }
-
     let mut tx = user_db.begin(&authed).await?;
 
     let identifier = if kind == "group_" || kind == "folder" {
@@ -113,11 +99,6 @@ async fn remove_granular_acl(
     } else {
         "path"
     };
-
-    if identifier == "path" {
-        require_owner_of_path(&authed, path)?;
-    }
-
     let obj_o = sqlx::query_scalar::<_, serde_json::Value>(&format!(
         "UPDATE {kind} SET extra_perms = extra_perms - $1 WHERE {identifier} = $2 AND \
          workspace_id = $3 RETURNING extra_perms"

backend/windmill-api/src/inputs.rs

@@ -1,281 +0,0 @@
-/*
- * Author: Ruben Fiszel
- * Copyright: Windmill Labs, Inc 2022
- * This file and its contents are licensed under the AGPLv3 License.
- * Please see the included NOTICE for copyright information and
- * LICENSE-AGPL for a copy of the license.
- */
-
-use crate::{db::UserDB, jobs::CompletedJob, users::Authed};
-use axum::{
-    extract::{Path, Query},
-    routing::{get, post},
-    Extension, Json, Router,
-};
-use chrono::{DateTime, Utc};
-use serde::{Deserialize, Serialize};
-use serde_json::Value;
-use sqlx::types::Uuid;
-use std::{
-    fmt::{Display, Formatter},
-    vec,
-};
-use windmill_common::{
-    error::JsonResult,
-    jobs::JobKind,
-    scripts::to_i64,
-    utils::{paginate, Pagination},
-};
-pub fn workspaced_service() -> Router {
-    Router::new()
-        .route("/history", get(get_input_history))
-        .route("/list", get(list_saved_inputs))
-        .route("/create", post(create_input))
-        .route("/update", post(update_input))
-        .route("/delete/:id", post(delete_input))
-}
-
-#[derive(Debug, sqlx::FromRow, Serialize, Deserialize)]
-pub struct InputRow {
-    pub id: Uuid,
-    pub workspace_id: String,
-    pub runnable_id: String,
-    pub runnable_type: RunnableType,
-    pub name: String,
-    pub args: Value,
-    pub created_at: DateTime<Utc>,
-    pub created_by: String,
-    pub is_public: bool,
-}
-
-#[derive(Debug, Serialize, Deserialize, sqlx::Type)]
-#[sqlx(type_name = "runnable_type")]
-pub enum RunnableType {
-    ScriptHash,
-    ScriptPath,
-    FlowPath,
-}
-
-impl Display for RunnableType {
-    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        match self {
-            RunnableType::ScriptHash => write!(f, "ScriptHash"),
-            RunnableType::ScriptPath => write!(f, "ScriptPath"),
-            RunnableType::FlowPath => write!(f, "FlowPath"),
-        }
-    }
-}
-
-impl RunnableType {
-    fn job_kind(&self) -> JobKind {
-        match self {
-            RunnableType::ScriptHash => JobKind::Script,
-            RunnableType::ScriptPath => JobKind::Script,
-            RunnableType::FlowPath => JobKind::Flow,
-        }
-    }
-
-    fn column_name(&self) -> &'static str {
-        match self {
-            RunnableType::ScriptHash => "script_hash",
-            RunnableType::ScriptPath => "script_path",
-            RunnableType::FlowPath => "script_path",
-        }
-    }
-}
-
-#[derive(Debug, Serialize, Deserialize)]
-pub struct RunnableParams {
-    pub runnable_id: String,
-    pub runnable_type: RunnableType,
-}
-
-#[derive(Debug, Serialize, Deserialize)]
-pub struct Input {
-    id: Uuid,
-    name: String,
-    created_at: chrono::DateTime<chrono::Utc>,
-    args: serde_json::Value,
-    created_by: String,
-    is_public: bool,
-}
-
-async fn get_input_history(
-    authed: Authed,
-    Extension(user_db): Extension<UserDB>,
-    Path(w_id): Path<String>,
-    Query(pagination): Query<Pagination>,
-    Query(r): Query<RunnableParams>,
-) -> JsonResult<Vec<Input>> {
-    let (per_page, offset) = paginate(pagination);
-
-    let mut tx = user_db.begin(&authed).await?;
-
-    let sql = &format!(
-        "select * from (select distinct on (args) * from completed_job \
-        where {} = $1 and job_kind = $2 and workspace_id = $3 \
-        order by args, started_at desc) t ORDER BY started_at desc limit $4 offset $5",
-        r.runnable_type.column_name()
-    );
-
-    let query = sqlx::query_as::<_, CompletedJob>(sql);
-
-    let query = match r.runnable_type {
-        RunnableType::ScriptHash => query.bind(to_i64(&r.runnable_id)?),
-        _ => query.bind(&r.runnable_id),
-    };
-
-    let rows = query
-        .bind(r.runnable_type.job_kind())
-        .bind(&w_id)
-        .bind(per_page as i32)
-        .bind(offset as i32)
-        .fetch_all(&mut tx)
-        .await?;
-
-    tx.commit().await?;
-
-    let mut inputs = vec![];
-
-    for row in rows {
-        inputs.push(Input {
-            id: row.id,
-            name: format!(
-                "{} {}",
-                row.created_at.format("%H:%M %-d/%-m"),
-                row.created_by
-            ),
-            created_at: row.created_at,
-            args: row.args.unwrap_or(serde_json::json!({})),
-            created_by: row.created_by,
-            is_public: true,
-        });
-    }
-
-    Ok(Json(inputs))
-}
-
-async fn list_saved_inputs(
-    authed: Authed,
-    Extension(user_db): Extension<UserDB>,
-    Path(w_id): Path<String>,
-    Query(pagination): Query<Pagination>,
-    Query(r): Query<RunnableParams>,
-) -> JsonResult<Vec<Input>> {
-    let (per_page, offset) = paginate(pagination);
-
-    let mut tx = user_db.begin(&authed).await?;
-
-    let rows = sqlx::query_as::<_, InputRow>(
-        "select * from input \
-         where runnable_id = $1 and runnable_type = $2 and workspace_id = $3 \
-         and is_public IS true OR created_by = $4 \
-         order by created_at desc limit $5 offset $6",
-    )
-    .bind(&r.runnable_id)
-    .bind(&r.runnable_type)
-    .bind(&w_id)
-    .bind(&authed.username)
-    .bind(per_page as i32)
-    .bind(offset as i32)
-    .fetch_all(&mut tx)
-    .await?;
-
-    tx.commit().await?;
-
-    let mut inputs: Vec<Input> = Vec::new();
-
-    for row in rows {
-        inputs.push(Input {
-            id: row.id,
-            name: row.name,
-            args: row.args,
-            created_by: row.created_by,
-            created_at: row.created_at,
-            is_public: row.is_public,
-        })
-    }
-
-    Ok(Json(inputs))
-}
-
-#[derive(Debug, Serialize, Deserialize)]
-pub struct CreateInput {
-    name: String,
-    args: serde_json::Value,
-}
-
-async fn create_input(
-    authed: Authed,
-    Extension(user_db): Extension<UserDB>,
-    Path(w_id): Path<String>,
-    Query(r): Query<RunnableParams>,
-    Json(input): Json<CreateInput>,
-) -> JsonResult<String> {
-    let mut tx = user_db.begin(&authed).await?;
-
-    let id = Uuid::new_v4();
-
-    sqlx::query(
-        "INSERT INTO input (id, workspace_id, runnable_id, runnable_type, name, args, created_by) VALUES ($1, $2, $3, $4, $5, $6, $7)",
-    )
-    .bind(&id)
-    .bind(&w_id)
-    .bind(&r.runnable_id)
-    .bind(&r.runnable_type)
-    .bind(&input.name)
-    .bind(&input.args)
-    .bind(&authed.username)
-    .execute(&mut tx)
-    .await?;
-
-    tx.commit().await?;
-
-    Ok(Json(id.to_string()))
-}
-
-#[derive(Debug, Serialize, Deserialize)]
-pub struct UpdateInput {
-    id: Uuid,
-    name: String,
-    is_public: bool,
-}
-
-async fn update_input(
-    authed: Authed,
-    Extension(user_db): Extension<UserDB>,
-    Path(w_id): Path<String>,
-    Json(input): Json<UpdateInput>,
-) -> JsonResult<String> {
-    let mut tx = user_db.begin(&authed).await?;
-
-    sqlx::query("UPDATE input SET name = $1, is_public = $2 WHERE id = $3 and workspace_id = $4")
-        .bind(&input.name)
-        .bind(&input.is_public)
-        .bind(&input.id)
-        .bind(&w_id)
-        .execute(&mut tx)
-        .await?;
-
-    tx.commit().await?;
-
-    Ok(Json(input.id.to_string()))
-}
-
-async fn delete_input(
-    authed: Authed,
-    Extension(user_db): Extension<UserDB>,
-    Path((w_id, i_id)): Path<(String, Uuid)>,
-) -> JsonResult<String> {
-    let mut tx = user_db.begin(&authed).await?;
-
-    sqlx::query("DELETE FROM input WHERE id = $1 and workspace_id = $2")
-        .bind(&i_id)
-        .bind(&w_id)
-        .execute(&mut tx)
-        .await?;
-
-    tx.commit().await?;
-
-    Ok(Json(i_id.to_string()))
-}

backend/windmill-api/src/jobs.rs

@@ -6,12 +6,6 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
-use crate::{
-    db::{UserDB, DB},
-    users::{require_owner_of_path, Authed, OptAuthed},
-    variables::get_workspace_key,
-    BASE_URL,
-};
 use anyhow::Context;
 use axum::{
     extract::{FromRequest, Json, Path, Query},
@@ -31,13 +25,19 @@ use windmill_common::{
     error::{self, to_anyhow, Error},
     flow_status::{Approval, FlowStatus, FlowStatusModule},
     flows::FlowValue,
-    jobs::{script_path_to_payload, JobKind, JobPayload, QueuedJob, RawCode},
     oauth2::HmacSha256,
     scripts::{ScriptHash, ScriptLang},
     users::username_to_permissioned_as,
     utils::{not_found_if_none, now_from_db, paginate, require_admin, Pagination, StripPath},
 };
-use windmill_queue::{get_queued_job, push, QueueTransaction};
+use windmill_queue::{get_queued_job, push, JobKind, JobPayload, QueuedJob, RawCode};
+
+use crate::{
+    db::{UserDB, DB},
+    users::{require_owner_of_path, Authed, OptAuthed},
+    variables::get_workspace_key,
+    BASE_URL,
+};
 
 pub fn workspaced_service() -> Router {
     Router::new()
@@ -64,7 +64,6 @@ pub fn workspaced_service() -> Router {
         .route("/run/preview_flow", post(run_preview_flow_job))
         .route("/list", get(list_jobs))
         .route("/queue/list", get(list_queue_jobs))
-        .route("/queue/count", get(count_queue_jobs))
         .route("/completed/list", get(list_completed_jobs))
         .route("/completed/get/:id", get(get_completed_job))
         .route("/completed/get_result/:id", get(get_completed_job_result))
@@ -115,7 +114,6 @@ async fn get_result_by_id(
 }
 
 async fn cancel_job_api(
-    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
     OptAuthed(opt_authed): OptAuthed,
     Extension(db): Extension<DB>,
     Path((w_id, id)): Path<(String, Uuid)>,
@@ -129,7 +127,7 @@ async fn cancel_job_api(
     };
 
     let (mut tx, job_option) =
-        windmill_queue::cancel_job(&username, reason, id, &w_id, tx, rsmq, false).await?;
+        windmill_queue::cancel_job(&username, reason, id, &w_id, tx, false).await?;
 
     if let Some(id) = job_option {
         audit_log(
@@ -158,7 +156,6 @@ async fn cancel_job_api(
 }
 
 async fn force_cancel(
-    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
     OptAuthed(opt_authed): OptAuthed,
     Extension(db): Extension<DB>,
     Path((w_id, id)): Path<(String, Uuid)>,
@@ -172,7 +169,7 @@ async fn force_cancel(
     };
 
     let (mut tx, job_option) =
-        windmill_queue::cancel_job(&username, reason, id, &w_id, tx, rsmq, true).await?;
+        windmill_queue::cancel_job(&username, reason, id, &w_id, tx, true).await?;
 
     if let Some(id) = job_option {
         audit_log(
@@ -220,26 +217,6 @@ pub async fn get_path_for_hash<'c>(
     Ok(path)
 }
 
-pub async fn get_path_and_tag_for_hash<'c>(
-    db: &mut Transaction<'c, Postgres>,
-    w_id: &str,
-    hash: i64,
-) -> error::Result<(String, Option<String>)> {
-    let script = sqlx::query!(
-        "select path, tag from script where hash = $1 AND workspace_id = $2",
-        hash,
-        w_id
-    )
-    .fetch_one(db)
-    .await
-    .map_err(|e| {
-        Error::InternalErr(format!(
-            "querying getting path for hash {hash} in {w_id}: {e}"
-        ))
-    })?;
-    Ok((script.path, script.tag))
-}
-
 async fn get_job(
     Extension(db): Extension<DB>,
     Path((w_id, id)): Path<(String, Uuid)>,
@@ -326,7 +303,6 @@ pub struct CompletedJob {
     pub visible_to_owner: bool,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub mem_peak: Option<i32>,
-    pub tag: String,
 }
 
 #[derive(Deserialize, Clone)]
@@ -411,7 +387,6 @@ pub struct ListQueueQuery {
     pub suspended: Option<bool>,
     // filter by matching a subset of the args using base64 encoded json subset
     pub args: Option<String>,
-    pub tag: Option<String>,
 }
 
 fn list_queue_jobs_query(w_id: &str, lq: &ListQueueQuery, fields: &[&str]) -> SqlBuilder {
@@ -434,9 +409,6 @@ fn list_queue_jobs_query(w_id: &str, lq: &ListQueueQuery, fields: &[&str]) -> Sq
     if let Some(cb) = &lq.created_by {
         sqlb.and_where_eq("created_by", "?".bind(cb));
     }
-    if let Some(t) = &lq.tag {
-        sqlb.and_where_eq("tag", "?".bind(t));
-    }
     if let Some(r) = &lq.running {
         sqlb.and_where_eq("running", &r);
     }
@@ -524,26 +496,6 @@ async fn list_queue_jobs(
     Ok(Json(jobs))
 }
 
-#[derive(Serialize, Debug, FromRow)]
-struct QueueStats {
-    database_length: i64,
-}
-
-async fn count_queue_jobs(
-    Extension(db): Extension<DB>,
-    Path(w_id): Path<String>,
-) -> error::JsonResult<QueueStats> {
-    Ok(Json(
-        sqlx::query_as!(
-            QueueStats,
-            "SELECT coalesce(COUNT(*), 0) as \"database_length!\" FROM queue WHERE workspace_id = $1",
-            w_id
-        )
-        .fetch_one(&db)
-        .await?,
-    ))
-}
-
 async fn list_jobs(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
@@ -569,7 +521,6 @@ async fn list_jobs(
             job_kinds: lq.job_kinds,
             suspended: lq.suspended,
             args: lq.args,
-            tag: lq.tag,
         },
         &[
             "'QueuedJob' as typ",
@@ -599,7 +550,6 @@ async fn list_jobs(
             "visible_to_owner",
             "suspend",
             "mem_peak",
-            "tag",
         ],
     );
     let sqlc = list_completed_jobs_query(
@@ -635,7 +585,6 @@ async fn list_jobs(
             "visible_to_owner",
             "null as suspend",
             "mem_peak",
-            "tag",
         ],
     );
     let sql = format!(
@@ -654,7 +603,7 @@ async fn list_jobs(
 pub async fn resume_suspended_flow_as_owner(
     authed: Authed,
     Extension(db): Extension<DB>,
-    Path((_w_id, flow_id)): Path<(String, Uuid)>,
+    Path((w_id, flow_id)): Path<(String, Uuid)>,
     QueryOrBody(value): QueryOrBody<serde_json::Value>,
 ) -> error::Result<StatusCode> {
     let value = value.unwrap_or(serde_json::Value::Null);
@@ -662,11 +611,16 @@ pub async fn resume_suspended_flow_as_owner(
 
     let (flow, job_id) = get_suspended_flow_info(flow_id, &mut tx).await?;
 
-    require_owner_of_path(
-        &authed,
-        &flow.script_path.clone().unwrap_or_else(|| String::new()),
-    )?;
-
+    if !authed.is_admin {
+        require_owner_of_path(
+            &w_id,
+            &authed.username,
+            &authed.groups,
+            &flow.script_path.clone().unwrap_or_else(|| String::new()),
+            &db,
+        )
+        .await?;
+    }
     insert_resume_job(0, job_id, &flow, value, Some(authed.username), &mut tx).await?;
 
     resume_immediately_if_relevant(flow, job_id, &mut tx).await?;
@@ -839,7 +793,6 @@ async fn get_suspended_flow_info<'c>(
 pub async fn cancel_suspended_job(
     /* unauthed */
     Extension(db): Extension<DB>,
-    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
     Path((w_id, job, resume_id, secret)): Path<(String, Uuid, u32, String)>,
     Query(approver): Query<QueryApprover>,
 ) -> error::Result<String> {
@@ -862,7 +815,6 @@ pub async fn cancel_suspended_job(
         parent_flow,
         &w_id,
         tx,
-        rsmq,
         false,
     )
     .await?;
@@ -1112,7 +1064,6 @@ struct UnifiedJob {
     visible_to_owner: bool,
     suspend: Option<i32>,
     mem_peak: Option<i32>,
-    tag: String,
 }
 
 impl From<UnifiedJob> for Job {
@@ -1148,7 +1099,6 @@ impl From<UnifiedJob> for Job {
                 email: uj.email,
                 visible_to_owner: uj.visible_to_owner,
                 mem_peak: uj.mem_peak,
-                tag: uj.tag,
             }),
             "QueuedJob" => Job::QueuedJob(QueuedJob {
                 workspace_id: uj.workspace_id,
@@ -1184,7 +1134,6 @@ impl From<UnifiedJob> for Job {
                 mem_peak: uj.mem_peak,
                 root_job: None,
                 leaf_jobs: None,
-                tag: uj.tag,
             }),
             t => panic!("job type {} not valid", t),
         }
@@ -1201,7 +1150,6 @@ struct Preview {
     path: Option<String>,
     args: Option<serde_json::Map<String, serde_json::Value>>,
     language: ScriptLang,
-    tag: Option<String>,
 }
 
 #[derive(Deserialize)]
@@ -1260,15 +1208,14 @@ fn decode_payload<D: DeserializeOwned>(t: String) -> anyhow::Result<D> {
 pub async fn run_flow_by_path(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
-    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
     Path((w_id, flow_path)): Path<(String, StripPath)>,
     Query(run_query): Query<RunJobQuery>,
     headers: HeaderMap,
     Json(args): Json<Option<serde_json::Map<String, serde_json::Value>>>,
 ) -> error::Result<(StatusCode, String)> {
     let flow_path = flow_path.to_path();
-    let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
-    let scheduled_for = run_query.get_scheduled_for(tx.transaction_mut()).await?;
+    let mut tx = user_db.begin(&authed).await?;
+    let scheduled_for = run_query.get_scheduled_for(&mut tx).await?;
     let args = run_query.add_include_headers(headers, args.unwrap_or_default());
 
     let (uuid, tx) = push(
@@ -1287,7 +1234,6 @@ pub async fn run_flow_by_path(
         false,
         None,
         !run_query.invisible_to_owner.unwrap_or(false),
-        None,
     )
     .await?;
     tx.commit().await?;
@@ -1297,17 +1243,15 @@ pub async fn run_flow_by_path(
 pub async fn run_job_by_path(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
-    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
     Path((w_id, script_path)): Path<(String, StripPath)>,
     Query(run_query): Query<RunJobQuery>,
     headers: HeaderMap,
     Json(args): Json<Option<serde_json::Map<String, serde_json::Value>>>,
 ) -> error::Result<(StatusCode, String)> {
     let script_path = script_path.to_path();
-    let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
-    let (job_payload, tag) =
-        script_path_to_payload(script_path, tx.transaction_mut(), &w_id).await?;
-    let scheduled_for = run_query.get_scheduled_for(tx.transaction_mut()).await?;
+    let mut tx = user_db.begin(&authed).await?;
+    let job_payload = script_path_to_payload(script_path, &mut tx, &w_id).await?;
+    let scheduled_for = run_query.get_scheduled_for(&mut tx).await?;
     let args = run_query.add_include_headers(headers, args.unwrap_or_default());
 
     let (uuid, tx) = push(
@@ -1326,7 +1270,6 @@ pub async fn run_job_by_path(
         false,
         None,
         !run_query.invisible_to_owner.unwrap_or(false),
-        tag,
     )
     .await?;
     tx.commit().await?;
@@ -1376,13 +1319,14 @@ async fn run_wait_result<T>(
     uuid: Uuid,
     Path((w_id, _)): Path<(String, T)>,
 ) -> error::JsonResult<serde_json::Value> {
-    let mut result;
-    let timeout_ms = if timeout <= 0 {
-        2000
+    let mut result = None;
+    let iters = if timeout <= 0 {
+        20
+    } else if timeout <= 1 {
+        timeout * 10
     } else {
-        (timeout * 1000) as u64
+        10 + ((timeout - 1) * 2)
     };
-
     let mut g = Guard {
         done: false,
         id: uuid,
@@ -1390,10 +1334,7 @@ async fn run_wait_result<T>(
         db: user_db.clone(),
         authed: authed.clone(),
     };
-
-    let fast_poll_duration = *WAIT_RESULT_FAST_POLL_DURATION_SECS as u64 * 1000;
-    let mut accumulated_delay = 0 as u64;
-    loop {
+    for i in 0..iters {
         let mut tx = user_db.clone().begin(&authed).await?;
         result = sqlx::query_scalar!(
             "SELECT result FROM completed_job WHERE id = $1 AND workspace_id = $2",
@@ -1408,16 +1349,7 @@ async fn run_wait_result<T>(
         if result.is_some() {
             break;
         }
-
-        let delay = if accumulated_delay <= fast_poll_duration {
-            *WAIT_RESULT_FAST_POLL_INTERVAL_MS
-        } else {
-            *WAIT_RESULT_SLOW_POLL_INTERVAL_MS
-        };
-        accumulated_delay += delay;
-        if accumulated_delay > timeout_ms {
-            break;
-        };
+        let delay = if i < 10 { 100 } else { 500 };
         tokio::time::sleep(core::time::Duration::from_millis(delay)).await;
     }
     if let Some(result) = result {
@@ -1457,23 +1389,10 @@ lazy_static::lazy_static! {
         .ok()
         .and_then(|x| x.parse().ok())
         .unwrap_or(20);
-    pub static ref WAIT_RESULT_FAST_POLL_INTERVAL_MS: u64 = std::env::var("WAIT_RESULT_FAST_POLL_INTERVAL_MS")
-        .ok()
-        .and_then(|x| x.parse().ok())
-        .unwrap_or(50);
-    pub static ref WAIT_RESULT_FAST_POLL_DURATION_SECS: u16 = std::env::var("WAIT_RESULT_FAST_POLL_DURATION_SECS")
-        .ok()
-        .and_then(|x| x.parse().ok())
-        .unwrap_or(2);
-    pub static ref WAIT_RESULT_SLOW_POLL_INTERVAL_MS: u64 = std::env::var("WAIT_RESULT_SLOW_POLL_INTERVAL_MS")
-        .ok()
-        .and_then(|x| x.parse().ok())
-        .unwrap_or(200);
 }
 
 pub async fn run_wait_result_job_by_path_get(
     authed: Authed,
-    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
     Extension(user_db): Extension<UserDB>,
     Extension(db): Extension<DB>,
     Path((w_id, script_path)): Path<(String, StripPath)>,
@@ -1492,9 +1411,8 @@ pub async fn run_wait_result_job_by_path_get(
 
     check_queue_too_long(db, QUEUE_LIMIT_WAIT_RESULT.or(run_query.queue_limit)).await?;
     let script_path = script_path.to_path();
-    let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.clone().begin(&authed).await?).into();
-    let (job_payload, tag) =
-        script_path_to_payload(script_path, tx.transaction_mut(), &w_id).await?;
+    let mut tx = user_db.clone().begin(&authed).await?;
+    let job_payload = script_path_to_payload(script_path, &mut tx, &w_id).await?;
 
     let (uuid, tx) = push(
         tx,
@@ -1512,7 +1430,6 @@ pub async fn run_wait_result_job_by_path_get(
         false,
         None,
         !run_query.invisible_to_owner.unwrap_or(false),
-        tag,
     )
     .await?;
     tx.commit().await?;
@@ -1530,7 +1447,6 @@ pub async fn run_wait_result_job_by_path_get(
 pub async fn run_wait_result_job_by_path(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
-    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
     Extension(db): Extension<DB>,
     Path((w_id, script_path)): Path<(String, StripPath)>,
     Query(run_query): Query<RunJobQuery>,
@@ -1539,9 +1455,8 @@ pub async fn run_wait_result_job_by_path(
 ) -> error::JsonResult<serde_json::Value> {
     check_queue_too_long(db, QUEUE_LIMIT_WAIT_RESULT.or(run_query.queue_limit)).await?;
     let script_path = script_path.to_path();
-    let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.clone().begin(&authed).await?).into();
-    let (job_payload, tag) =
-        script_path_to_payload(script_path, tx.transaction_mut(), &w_id).await?;
+    let mut tx = user_db.clone().begin(&authed).await?;
+    let job_payload = script_path_to_payload(script_path, &mut tx, &w_id).await?;
 
     let args = run_query.add_include_headers(headers, args.unwrap_or_default());
 
@@ -1561,7 +1476,6 @@ pub async fn run_wait_result_job_by_path(
         false,
         None,
         !run_query.invisible_to_owner.unwrap_or(false),
-        tag,
     )
     .await?;
     tx.commit().await?;
@@ -1579,7 +1493,6 @@ pub async fn run_wait_result_job_by_path(
 pub async fn run_wait_result_job_by_hash(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
-    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
     Extension(db): Extension<DB>,
     Path((w_id, script_hash)): Path<(String, ScriptHash)>,
     Query(run_query): Query<RunJobQuery>,
@@ -1589,9 +1502,8 @@ pub async fn run_wait_result_job_by_hash(
     check_queue_too_long(db, run_query.queue_limit).await?;
 
     let hash = script_hash.0;
-    let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.clone().begin(&authed).await?).into();
-    let (path, tag) = get_path_and_tag_for_hash(tx.transaction_mut(), &w_id, hash).await?;
-
+    let mut tx = user_db.clone().begin(&authed).await?;
+    let path = get_path_for_hash(&mut tx, &w_id, hash).await?;
     let args = run_query.add_include_headers(headers, args.unwrap_or_default());
 
     let (uuid, tx) = push(
@@ -1610,7 +1522,6 @@ pub async fn run_wait_result_job_by_hash(
         false,
         None,
         !run_query.invisible_to_owner.unwrap_or(false),
-        tag,
     )
     .await?;
     tx.commit().await?;
@@ -1628,7 +1539,6 @@ pub async fn run_wait_result_job_by_hash(
 pub async fn run_wait_result_flow_by_path(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
-    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
     Extension(db): Extension<DB>,
     Path((w_id, flow_path)): Path<(String, StripPath)>,
     Query(run_query): Query<RunJobQuery>,
@@ -1638,8 +1548,8 @@ pub async fn run_wait_result_flow_by_path(
     check_queue_too_long(db, run_query.queue_limit).await?;
 
     let flow_path = flow_path.to_path();
-    let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.clone().begin(&authed).await?).into();
-    let scheduled_for = run_query.get_scheduled_for(tx.transaction_mut()).await?;
+    let mut tx = user_db.clone().begin(&authed).await?;
+    let scheduled_for = run_query.get_scheduled_for(&mut tx).await?;
     let args = run_query.add_include_headers(headers, args.unwrap_or_default());
 
     let (uuid, tx) = push(
@@ -1658,9 +1568,9 @@ pub async fn run_wait_result_flow_by_path(
         false,
         None,
         !run_query.invisible_to_owner.unwrap_or(false),
-        None,
     )
     .await?;
+
     tx.commit().await?;
 
     run_wait_result(
@@ -1673,17 +1583,31 @@ pub async fn run_wait_result_flow_by_path(
     .await
 }
 
+// a similar function exists on the worker
+pub async fn script_path_to_payload<'c>(
+    script_path: &str,
+    db: &mut Transaction<'c, Postgres>,
+    w_id: &String,
+) -> std::result::Result<JobPayload, Error> {
+    let job_payload = if script_path.starts_with("hub/") {
+        JobPayload::ScriptHub { path: script_path.to_owned() }
+    } else {
+        let script_hash = windmill_common::get_latest_hash_for_path(db, w_id, script_path).await?;
+        JobPayload::ScriptHash { hash: script_hash, path: script_path.to_owned() }
+    };
+    Ok(job_payload)
+}
+
 async fn run_preview_job(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
-    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
     Path(w_id): Path<String>,
     Query(run_query): Query<RunJobQuery>,
     headers: HeaderMap,
     Json(preview): Json<Preview>,
 ) -> error::Result<(StatusCode, String)> {
-    let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
-    let scheduled_for = run_query.get_scheduled_for(tx.transaction_mut()).await?;
+    let mut tx = user_db.begin(&authed).await?;
+    let scheduled_for = run_query.get_scheduled_for(&mut tx).await?;
     let args = run_query.add_include_headers(headers, preview.args.unwrap_or_default());
 
     let (uuid, tx) = push(
@@ -1707,25 +1631,22 @@ async fn run_preview_job(
         false,
         None,
         true,
-        preview.tag,
     )
     .await?;
     tx.commit().await?;
-
     Ok((StatusCode::CREATED, uuid.to_string()))
 }
 
 async fn run_preview_flow_job(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
-    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
     Path(w_id): Path<String>,
     Query(run_query): Query<RunJobQuery>,
     headers: HeaderMap,
     Json(raw_flow): Json<PreviewFlow>,
 ) -> error::Result<(StatusCode, String)> {
-    let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
-    let scheduled_for = run_query.get_scheduled_for(tx.transaction_mut()).await?;
+    let mut tx = user_db.begin(&authed).await?;
+    let scheduled_for = run_query.get_scheduled_for(&mut tx).await?;
     let args = run_query.add_include_headers(headers, raw_flow.args.unwrap_or_default());
 
     let (uuid, tx) = push(
@@ -1744,27 +1665,24 @@ async fn run_preview_flow_job(
         false,
         None,
         true,
-        None,
     )
     .await?;
     tx.commit().await?;
-
     Ok((StatusCode::CREATED, uuid.to_string()))
 }
 
 pub async fn run_job_by_hash(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
-    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
     Path((w_id, script_hash)): Path<(String, ScriptHash)>,
     Query(run_query): Query<RunJobQuery>,
     headers: HeaderMap,
     Json(args): Json<Option<serde_json::Map<String, serde_json::Value>>>,
 ) -> error::Result<(StatusCode, String)> {
     let hash = script_hash.0;
-    let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
-    let (path, tag) = get_path_and_tag_for_hash(tx.transaction_mut(), &w_id, hash).await?;
-    let scheduled_for = run_query.get_scheduled_for(tx.transaction_mut()).await?;
+    let mut tx = user_db.begin(&authed).await?;
+    let path = get_path_for_hash(&mut tx, &w_id, hash).await?;
+    let scheduled_for = run_query.get_scheduled_for(&mut tx).await?;
     let args = run_query.add_include_headers(headers, args.unwrap_or_default());
 
     let (uuid, tx) = push(
@@ -1783,11 +1701,9 @@ pub async fn run_job_by_hash(
         false,
         None,
         !run_query.invisible_to_owner.unwrap_or(false),
-        tag,
     )
     .await?;
     tx.commit().await?;
-
     Ok((StatusCode::CREATED, uuid.to_string()))
 }
 
@@ -1874,9 +1790,6 @@ fn list_completed_jobs_query(
     if let Some(h) = &lq.script_hash {
         sqlb.and_where_eq("script_hash", "?".bind(h));
     }
-    if let Some(t) = &lq.tag {
-        sqlb.and_where_eq("tag", "?".bind(t));
-    }
     if let Some(cb) = &lq.created_by {
         sqlb.and_where_eq("created_by", "?".bind(cb));
     }
@@ -1934,7 +1847,6 @@ pub struct ListCompletedQuery {
     pub args: Option<String>,
     // filter by matching a subset of the result using base64 encoded json subset
     pub result: Option<String>,
-    pub tag: Option<String>,
 }
 
 async fn list_completed_jobs(
@@ -1980,7 +1892,6 @@ async fn list_completed_jobs(
             "email",
             "visible_to_owner",
             "mem_peak",
-            "tag",
         ],
     )
     .sql()?;

backend/windmill-api/src/lib.rs

@@ -7,13 +7,6 @@
  */
 
 use crate::oauth2::AllClients;
-use crate::{
-    db::UserDB,
-    oauth2::{build_oauth_clients, SlackVerifier},
-    tracing_init::{MyMakeSpan, MyOnResponse},
-    users::{Authed, OptAuthed},
-    webhook_util::WebhookShared,
-};
 use argon2::Argon2;
 use axum::{middleware::from_extractor, routing::get, Extension, Router};
 use db::DB;
@@ -29,17 +22,23 @@ use tower_http::{
 };
 use windmill_common::utils::rd_string;
 
+use crate::{
+    db::UserDB,
+    oauth2::{build_oauth_clients, SlackVerifier},
+    tracing_init::{MyMakeSpan, MyOnResponse},
+    users::{Authed, OptAuthed},
+    webhook_util::WebhookShared,
+};
+
 mod apps;
 mod audit;
 mod capture;
 mod db;
-mod drafts;
 mod favorite;
 mod flows;
 mod folders;
 mod granular_acls;
 mod groups;
-mod inputs;
 pub mod jobs;
 mod oauth2;
 mod resources;
@@ -51,7 +50,7 @@ mod users;
 mod utils;
 mod variables;
 mod webhook_util;
-mod workers;
+mod worker_ping;
 mod workspaces;
 
 pub const GIT_VERSION: &str =
@@ -82,7 +81,6 @@ lazy_static::lazy_static! {
 
 pub async fn run_server(
     db: DB,
-    rsmq: Option<rsmq_async::MultiplexedRsmq>,
     addr: SocketAddr,
     mut rx: tokio::sync::broadcast::Receiver<()>,
 ) -> anyhow::Result<()> {
@@ -102,7 +100,6 @@ pub async fn run_server(
                 .on_request(()),
         )
         .layer(Extension(db.clone()))
-        .layer(Extension(rsmq))
         .layer(Extension(user_db))
         .layer(Extension(auth_cache.clone()))
         .layer(CookieManagerLayer::new())
@@ -120,35 +117,32 @@ pub async fn run_server(
                 .nest(
                     "/w/:workspace_id",
                     Router::new()
-                        // Reordered alphabetically
-                        .nest("/acls", granular_acls::workspaced_service())
-                        .nest("/apps", apps::workspaced_service())
-                        .nest("/audit", audit::workspaced_service())
-                        .nest("/capture", capture::workspaced_service())
-                        .nest("/favorites", favorite::workspaced_service())
-                        .nest("/flows", flows::workspaced_service())
-                        .nest("/folders", folders::workspaced_service())
-                        .nest("/groups", groups::workspaced_service())
-                        .nest("/inputs", inputs::workspaced_service())
-                        .nest("/jobs", jobs::workspaced_service().layer(cors.clone()))
-                        .nest("/oauth", oauth2::workspaced_service())
-                        .nest("/resources", resources::workspaced_service())
-                        .nest("/schedules", schedule::workspaced_service())
                         .nest("/scripts", scripts::workspaced_service())
-                        .nest("/drafts", drafts::workspaced_service())
+                        .nest("/jobs", jobs::workspaced_service().layer(cors.clone()))
                         .nest(
                             "/users",
                             users::workspaced_service().layer(Extension(argon2.clone())),
                         )
                         .nest("/variables", variables::workspaced_service())
-                        .nest("/workspaces", workspaces::workspaced_service()),
+                        .nest("/oauth", oauth2::workspaced_service())
+                        .nest("/resources", resources::workspaced_service())
+                        .nest("/schedules", schedule::workspaced_service())
+                        .nest("/groups", groups::workspaced_service())
+                        .nest("/audit", audit::workspaced_service())
+                        .nest("/acls", granular_acls::workspaced_service())
+                        .nest("/workspaces", workspaces::workspaced_service())
+                        .nest("/apps", apps::workspaced_service())
+                        .nest("/flows", flows::workspaced_service())
+                        .nest("/capture", capture::workspaced_service())
+                        .nest("/favorites", favorite::workspaced_service())
+                        .nest("/folders", folders::workspaced_service()),
                 )
                 .nest("/workspaces", workspaces::global_service())
                 .nest(
                     "/users",
                     users::global_service().layer(Extension(argon2.clone())),
                 )
-                .nest("/workers", workers::global_service())
+                .nest("/workers", worker_ping::global_service())
                 .nest("/scripts", scripts::global_service())
                 .nest("/flows", flows::global_service())
                 .nest("/apps", apps::global_service().layer(cors.clone()))

backend/windmill-api/src/oauth2.rs

@@ -29,12 +29,10 @@ use serde::{de::DeserializeOwned, Deserialize, Serialize};
 use sqlx::{Postgres, Transaction};
 use tower_cookies::{Cookie, Cookies};
 use windmill_audit::{audit_log, ActionKind};
-use windmill_common::jobs::JobPayload;
 use windmill_common::users::username_to_permissioned_as;
 use windmill_common::utils::{not_found_if_none, now_from_db};
 
-use crate::users::{truncate_token, Authed};
-use crate::webhook_util::{InstanceEvent, WebhookShared};
+use crate::users::{truncate_token, Authed, NEW_USER_WEBHOOK};
 use crate::workspaces::invite_user_to_all_auto_invite_worspaces;
 use crate::{
     db::{UserDB, DB},
@@ -45,7 +43,7 @@ use crate::{BASE_URL, HTTP_CLIENT, IS_SECURE, OAUTH_CLIENTS, SLACK_SIGNING_SECRE
 use windmill_common::error::{self, to_anyhow, Error};
 use windmill_common::oauth2::*;
 
-use windmill_queue::QueueTransaction;
+use windmill_queue::JobPayload;
 
 use std::{fs, str};
 
@@ -730,7 +728,6 @@ where
 async fn slack_command(
     SlackSig { sig, ts }: SlackSig,
     Extension(db): Extension<DB>,
-    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
     body: Bytes,
 ) -> error::Result<String> {
     let form: SlackCommand = serde_urlencoded::from_bytes(&body)
@@ -743,7 +740,7 @@ async fn slack_command(
         }
     }
 
-    let mut tx: QueueTransaction<'_, _> = (rsmq, db.begin().await?).into();
+    let mut tx = db.begin().await?;
     let settings = sqlx::query_as!(
         WorkspaceSettings,
         "SELECT * FROM workspace_settings WHERE slack_team_id = $1",
@@ -754,20 +751,17 @@ async fn slack_command(
 
     if let Some(settings) = settings {
         if let Some(path) = &settings.slack_command_script {
-            let (payload, tag) = if let Some(path) = path.strip_prefix("flow/") {
-                (JobPayload::Flow(path.to_string()), None)
+            let payload = if let Some(path) = path.strip_prefix("flow/") {
+                JobPayload::Flow(path.to_string())
             } else {
                 let path = path.strip_prefix("script/").unwrap_or_else(|| path);
-                let (script_hash, tag) = windmill_common::get_latest_deployed_hash_for_path(
-                    tx.transaction_mut(),
+                let script_hash = windmill_common::get_latest_hash_for_path(
+                    &mut tx,
                     &settings.workspace_id,
                     path,
                 )
                 .await?;
-                (
-                    JobPayload::ScriptHash { hash: script_hash, path: path.to_owned() },
-                    tag,
-                )
+                JobPayload::ScriptHash { hash: script_hash, path: path.to_owned() }
             };
             let mut map = serde_json::Map::new();
             map.insert("text".to_string(), serde_json::Value::String(form.text));
@@ -792,18 +786,16 @@ async fn slack_command(
                 false,
                 None,
                 true,
-                tag,
             )
             .await?;
-            let url = BASE_URL.to_owned();
             tx.commit().await?;
+            let url = BASE_URL.to_owned();
             return Ok(format!(
                 "Job launched. See details at {url}/run/{uuid}?workspace={}",
                 &settings.workspace_id
             ));
         }
     }
-    tx.commit().await?;
 
     return Ok(format!(
         "workspace not properly configured (did you set the script to trigger in the settings?)"
@@ -823,7 +815,6 @@ async fn login_callback(
     Path(client_name): Path<String>,
     cookies: Cookies,
     Extension(db): Extension<DB>,
-    Extension(webhook): Extension<WebhookShared>,
     Json(callback): Json<OAuthCallback>,
 ) -> error::Result<String> {
     let client_w_config = &OAUTH_CLIENTS
@@ -949,7 +940,14 @@ async fn login_callback(
         }
         tx.commit().await?;
 
-        webhook.send_instance_event(InstanceEvent::UserSignupOAuth { email: email.clone() });
+        if let Some(new_user_webhook) = NEW_USER_WEBHOOK.clone() {
+            let _ = HTTP_CLIENT
+                .post(&new_user_webhook)
+                .json(&serde_json::json!({"email" : &email, "event": "oauth_signup"}))
+                .send()
+                .await
+                .map_err(|e| tracing::error!("Error sending new user webhook: {}", e.to_string()));
+        }
 
         Ok("Successfully logged in".to_string())
     } else {

backend/windmill-api/src/resources.rs

@@ -386,8 +386,9 @@ async fn update_resource(
         if npath != path {
             check_path_conflict(&mut tx, &w_id, &npath).await?;
 
-            require_owner_of_path(&authed, path)?;
-
+            if !authed.is_admin {
+                require_owner_of_path(&w_id, &authed.username, &authed.groups, &path, &db).await?;
+            }
             sqlx::query!(
                 "UPDATE variable SET path = $1 WHERE path = $2 AND workspace_id = $3",
                 npath,

backend/windmill-api/src/schedule.rs

@@ -23,9 +23,9 @@ use windmill_audit::{audit_log, ActionKind};
 use windmill_common::{
     error::{Error, JsonResult, Result},
     schedule::Schedule,
-    utils::{not_found_if_none, paginate, Pagination, StripPath}, jobs::JobKind,
+    utils::{not_found_if_none, paginate, Pagination, StripPath},
 };
-use windmill_queue::{self, schedule::push_scheduled_job, QueueTransaction};
+use windmill_queue::{self, schedule::push_scheduled_job, JobKind};
 
 pub fn workspaced_service() -> Router {
     Router::new()
@@ -79,23 +79,15 @@ async fn create_schedule(
     authed: Authed,
     Extension(db): Extension<DB>,
     Extension(user_db): Extension<UserDB>,
-    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
     Path(w_id): Path<String>,
     Json(ns): Json<NewSchedule>,
 ) -> Result<String> {
     let authed = maybe_refresh_folders(&ns.path, &w_id, authed, &db).await;
-    let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
 
+    let mut tx = user_db.begin(&authed).await?;
     cron::Schedule::from_str(&ns.schedule).map_err(|e| Error::BadRequest(e.to_string()))?;
-    check_path_conflict(tx.transaction_mut(), &w_id, &ns.path).await?;
-    check_flow_conflict(
-        tx.transaction_mut(),
-        &w_id,
-        &ns.path,
-        ns.is_flow,
-        &ns.script_path,
-    )
-    .await?;
+    check_path_conflict(&mut tx, &w_id, &ns.path).await?;
+    check_flow_conflict(&mut tx, &w_id, &ns.path, ns.is_flow, &ns.script_path).await?;
 
     let schedule = sqlx::query_as!(
         Schedule,
@@ -147,15 +139,14 @@ async fn edit_schedule(
     authed: Authed,
     Extension(db): Extension<DB>,
     Extension(user_db): Extension<UserDB>,
-    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
     Path((w_id, path)): Path<(String, StripPath)>,
     Json(es): Json<EditSchedule>,
 ) -> Result<String> {
     let path = path.to_path();
 
     let authed = maybe_refresh_folders(&path, &w_id, authed, &db).await;
-      let mut tx: QueueTransaction<'_, rsmq_async::MultiplexedRsmq> =
-        (rsmq, user_db.begin(&authed).await?).into();
+
+    let mut tx = user_db.begin(&authed).await?;
 
     cron::Schedule::from_str(&es.schedule).map_err(|e| Error::BadRequest(e.to_string()))?;
 
@@ -167,7 +158,7 @@ async fn edit_schedule(
     .fetch_one(&mut tx)
     .await?;
 
-    clear_schedule(tx.transaction_mut(), path, is_flow).await?;
+    clear_schedule(&mut tx, path, is_flow).await?;
     let schedule = sqlx::query_as!(
         Schedule,
         "UPDATE schedule SET schedule = $1, timezone = $2, args = $3 WHERE path \
@@ -182,6 +173,10 @@ async fn edit_schedule(
     .await
     .map_err(|e| Error::InternalErr(format!("updating schedule in {w_id}: {e}")))?;
 
+    if schedule.enabled {
+        tx = push_scheduled_job(tx, schedule).await?;
+    }
+
     audit_log(
         &mut tx,
         &authed.username,
@@ -197,10 +192,6 @@ async fn edit_schedule(
         ),
     )
     .await?;
-
-    if schedule.enabled {
-        tx = push_scheduled_job(tx, schedule).await?;
-    }
     tx.commit().await?;
 
     Ok(path.to_string())
@@ -279,12 +270,10 @@ pub async fn preview_schedule(
 pub async fn set_enabled(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
-    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
     Path((w_id, path)): Path<(String, StripPath)>,
     Json(payload): Json<SetEnabled>,
 ) -> Result<String> {
-    let mut tx: QueueTransaction<'_, rsmq_async::MultiplexedRsmq> =
-        (rsmq, user_db.begin(&authed).await?).into();
+    let mut tx = user_db.begin(&authed).await?;
     let path = path.to_path();
     let schedule_o = sqlx::query_as!(
         Schedule,
@@ -299,8 +288,11 @@ pub async fn set_enabled(
 
     let schedule = not_found_if_none(schedule_o, "Schedule", path)?;
 
-    clear_schedule(tx.transaction_mut(), path, schedule.is_flow).await?;
+    clear_schedule(&mut tx, path, schedule.is_flow).await?;
 
+    if payload.enabled {
+        tx = push_scheduled_job(tx, schedule).await?;
+    }
     audit_log(
         &mut tx,
         &authed.username,
@@ -311,12 +303,7 @@ pub async fn set_enabled(
         Some([("enabled", payload.enabled.to_string().as_ref())].into()),
     )
     .await?;
-
-    if payload.enabled {
-        tx = push_scheduled_job(tx, schedule).await?;
-    }
     tx.commit().await?;
-
     Ok(format!(
         "succesfully updated schedule at path {} to status {}",
         path, payload.enabled

backend/windmill-api/src/scripts.rs

@@ -6,6 +6,10 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
+use sql_builder::prelude::*;
+use windmill_audit::{audit_log, ActionKind};
+use windmill_parser::MainArgSignature;
+
 use crate::{
     db::{UserDB, DB},
     schedule::clear_schedule,
@@ -21,7 +25,6 @@ use axum::{
 use hyper::StatusCode;
 use serde::Serialize;
 use serde_json::json;
-use sql_builder::prelude::*;
 use sql_builder::SqlBuilder;
 use sqlx::{FromRow, Postgres, Transaction};
 use std::{
@@ -29,13 +32,11 @@ use std::{
     hash::{Hash, Hasher},
     sync::Arc,
 };
-use windmill_audit::{audit_log, ActionKind};
 use windmill_common::{
     error::{Error, JsonResult, Result},
-    jobs::JobPayload,
     schedule::Schedule,
     scripts::{
-        to_i64, HubScript, ListScriptQuery, ListableScript, NewScript, Schema, Script, ScriptHash,
+        to_i64, HubScript, ListScriptQuery, ListableScript, NewScript, Script, ScriptHash,
         ScriptKind, ScriptLang,
     },
     users::username_to_permissioned_as,
@@ -43,35 +44,10 @@ use windmill_common::{
         list_elems_from_hub, not_found_if_none, paginate, require_admin, Pagination, StripPath,
     },
 };
-use windmill_parser::MainArgSignature;
-use windmill_queue::{self, schedule::push_scheduled_job, QueueTransaction};
-
-lazy_static::lazy_static! {
-    pub static ref CUSTOM_TAGS: Vec<String> = std::env::var("CUSTOM_TAGS")
-        .ok()
-        .map(|x| x.split(',').map(|x| x.to_string()).filter(|x| !x.is_empty()).collect::<Vec<_>>()).unwrap_or_default();
-
-}
+use windmill_queue::{self, schedule::push_scheduled_job};
 
 const MAX_HASH_HISTORY_LENGTH_STORED: usize = 20;
 
-#[derive(Serialize, sqlx::FromRow)]
-pub struct ScriptWDraft {
-    pub hash: ScriptHash,
-    pub path: String,
-    pub summary: String,
-    pub description: String,
-    pub content: String,
-    pub language: ScriptLang,
-    pub kind: ScriptKind,
-    pub tag: Option<String>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub draft: Option<serde_json::Value>,
-    pub schema: Option<Schema>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub draft_only: Option<bool>,
-}
-
 pub fn global_service() -> Router {
     Router::new()
         .route(
@@ -98,7 +74,6 @@ pub fn workspaced_service() -> Router {
         .route("/list", get(list_scripts))
         .route("/create", post(create_script))
         .route("/archive/p/*path", post(archive_script_by_path))
-        .route("/get/draft/*path", get(get_script_by_path_w_draft))
         .route("/get/p/*path", get(get_script_by_path))
         .route("/raw/p/*path", get(raw_script_by_path))
         .route("/exists/p/*path", get(exists_script_by_path))
@@ -110,7 +85,6 @@ pub fn workspaced_service() -> Router {
         .route("/deployment_status/h/:hash", get(get_deployment_status))
         .route("/list_paths", get(list_paths))
 }
-
 async fn list_scripts(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
@@ -122,18 +96,22 @@ async fn list_scripts(
 
     let mut sqlb = SqlBuilder::select_from("script as o")
         .fields(&[
+            "o.workspace_id",
             "hash",
             "o.path",
+            "array_remove(array[parent_hashes[1]], NULL) as parent_hashes",
             "summary",
-            "COALESCE(draft.created_at, o.created_at) as created_at",
+            "description",
+            "created_by",
+            "created_at",
             "archived",
+            "deleted",
+            "is_template",
             "extra_perms",
-            "CASE WHEN lock_error_logs IS NOT NULL THEN true ELSE false END as has_deploy_errors",
+            "CASE WHEN lock_error_logs IS NOT NULL THEN 'error' ELSE null END as lock_error_logs",
             "language",
+            "kind",
             "favorite.path IS NOT NULL as starred",
-            "tag",
-            "draft.path IS NOT NULL as has_draft",
-            "draft_only"
         ])
         .left()
         .join("favorite")
@@ -141,11 +119,6 @@ async fn list_scripts(
             "favorite.favorite_kind = 'script' AND favorite.workspace_id = o.workspace_id AND favorite.path = o.path AND favorite.usr = ?"
                 .bind(&authed.username),
         )
-        .left()
-        .join("draft")
-        .on(
-            "draft.path = o.path AND draft.workspace_id = o.workspace_id AND draft.typ = 'script'"
-        )
         .order_desc("favorite.path IS NOT NULL")
         .order_by("created_at", lq.order_desc.unwrap_or(true))
         .and_where("o.workspace_id = ?".bind(&w_id))
@@ -155,7 +128,7 @@ async fn list_scripts(
 
     if lq.show_archived.unwrap_or(false) {
         sqlb.and_where_eq(
-            "o.created_at",
+            "created_at",
             "(select max(created_at) from script where o.path = path 
             AND workspace_id = ?)"
                 .bind(&w_id),
@@ -220,7 +193,6 @@ fn hash_script(ns: &NewScript) -> i64 {
 async fn create_script(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
-    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
     Extension(webhook): Extension<WebhookShared>,
     Extension(db): Extension<DB>,
     Path(w_id): Path<String>,
@@ -228,14 +200,14 @@ async fn create_script(
 ) -> Result<(StatusCode, String)> {
     let hash = ScriptHash(hash_script(&ns));
     let authed = maybe_refresh_folders(&ns.path, &w_id, authed, &db).await;
-    let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
+    let mut tx = user_db.begin(&authed).await?;
 
     if sqlx::query_scalar!(
         "SELECT 1 FROM script WHERE hash = $1 AND workspace_id = $2",
         hash.0,
         &w_id
     )
-    .fetch_optional(tx.transaction_mut())
+    .fetch_optional(&mut tx)
     .await?
     .is_some()
     {
@@ -261,20 +233,10 @@ async fn create_script(
     }
     let parent_hashes_and_perms: Option<ParentInfo> = match (&ns.parent_hash, clashing_script) {
         (None, None) => Ok(None),
-        (None, Some(s)) if !s.draft_only.unwrap_or(false) => Err(Error::BadRequest(format!(
+        (None, Some(s)) => Err(Error::BadRequest(format!(
             "Path conflict for {} with non-archived hash {}",
             &ns.path, &s.hash
         ))),
-        (None, Some(s)) => {
-            sqlx::query!(
-                "DELETE FROM script WHERE hash = $1 AND workspace_id = $2",
-                s.hash.0,
-                &w_id
-            )
-            .execute(&mut tx)
-            .await?;
-            Ok(None)
-        }
         (Some(p_hash), o) => {
             if sqlx::query_scalar!(
                 "SELECT 1 FROM script WHERE hash = $1 AND workspace_id = $2",
@@ -306,10 +268,13 @@ async fn create_script(
                 )));
             };
 
-            let ps = get_script_by_hash_internal(tx.transaction_mut(), &w_id, p_hash).await?;
+            let ps = get_script_by_hash_internal(&mut tx, &w_id, p_hash).await?;
 
             if ps.path != ns.path {
-                require_owner_of_path(&authed, &ps.path)?;
+                if !authed.is_admin {
+                    require_owner_of_path(&w_id, &authed.username, &authed.groups, &ps.path, &db)
+                        .await?;
+                }
             }
 
             let ph = {
@@ -366,8 +331,8 @@ async fn create_script(
     //::text::json is to ensure we use serde_json with preserve order
     sqlx::query!(
         "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, \
-         content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, draft_only) \
-         VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16)",
+         content, created_by, schema, is_template, extra_perms, lock, language, kind) \
+         VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14)",
         &w_id,
         &hash.0,
         ns.path,
@@ -382,21 +347,11 @@ async fn create_script(
         lock,
         ns.language: ScriptLang,
         ns.kind.unwrap_or(ScriptKind::Script): ScriptKind,
-        ns.tag,
-        ns.draft_only
     )
     .execute(&mut tx)
     .await?;
 
     if let Some(p_path) = parent_hashes_and_perms.as_ref().map(|x| x.p_path.clone()) {
-        sqlx::query!(
-            "DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'script'",
-            p_path,
-            &w_id
-        )
-        .execute(&mut tx)
-        .await?;
-
         let schedulables = sqlx::query_as!(
         Schedule,
             "UPDATE schedule SET script_path = $1 WHERE script_path = $2 AND workspace_id = $3 AND is_flow IS false RETURNING *",
@@ -408,22 +363,44 @@ async fn create_script(
         .await?;
 
         for schedule in schedulables {
-            clear_schedule(tx.transaction_mut(), &schedule.path, false).await?;
+            clear_schedule(&mut tx, &schedule.path, false).await?;
 
             if schedule.enabled {
                 tx = push_scheduled_job(tx, schedule).await?;
             }
         }
-    } else {
-        sqlx::query!(
-            "DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'script'",
-            ns.path,
-            &w_id
-        )
-        .execute(&mut tx)
-        .await?;
     }
 
+    let mut tx = if needs_lock_gen {
+        let dependencies = match ns.language {
+            ScriptLang::Python3 => {
+                windmill_parser_py::parse_python_imports(&ns.content)?.join("\n")
+            }
+            _ => ns.content,
+        };
+        let (_, tx) = windmill_queue::push(
+            tx,
+            &w_id,
+            windmill_queue::JobPayload::Dependencies { hash, dependencies, language: ns.language },
+            serde_json::Map::new(),
+            &authed.username,
+            &authed.email,
+            username_to_permissioned_as(&authed.username),
+            None,
+            None,
+            None,
+            None,
+            false,
+            false,
+            None,
+            true,
+        )
+        .await?;
+        tx
+    } else {
+        tx
+    };
+
     if p_hashes.is_some() && !p_hashes.unwrap().is_empty() {
         audit_log(
             &mut tx,
@@ -438,7 +415,7 @@ async fn create_script(
         webhook.send_message(
             w_id.clone(),
             WebhookMessage::UpdateScript {
-                workspace: w_id.clone(),
+                workspace: w_id,
                 path: ns.path.clone(),
                 hash: hash.to_string(),
             },
@@ -463,42 +440,13 @@ async fn create_script(
         webhook.send_message(
             w_id.clone(),
             WebhookMessage::CreateScript {
-                workspace: w_id.clone(),
+                workspace: w_id,
                 path: ns.path.clone(),
                 hash: hash.to_string(),
             },
         );
     }
 
-    if needs_lock_gen {
-        let dependencies = match ns.language {
-            ScriptLang::Python3 => {
-                windmill_parser_py::parse_python_imports(&ns.content)?.join("\n")
-            }
-            _ => ns.content,
-        };
-        let (_, new_tx) = windmill_queue::push(
-            tx,
-            &w_id,
-            JobPayload::Dependencies { hash, dependencies, language: ns.language },
-            serde_json::Map::new(),
-            &authed.username,
-            &authed.email,
-            username_to_permissioned_as(&authed.username),
-            None,
-            None,
-            None,
-            None,
-            false,
-            false,
-            None,
-            true,
-            None,
-        )
-        .await?;
-        tx = new_tx;
-    }
-
     tx.commit().await?;
 
     Ok((StatusCode::CREATED, format!("{}", hash)))
@@ -540,31 +488,6 @@ async fn get_script_by_path(
     Ok(Json(script))
 }
 
-async fn get_script_by_path_w_draft(
-    authed: Authed,
-    Extension(user_db): Extension<UserDB>,
-    Path((w_id, path)): Path<(String, StripPath)>,
-) -> JsonResult<ScriptWDraft> {
-    let path = path.to_path();
-    let mut tx = user_db.begin(&authed).await?;
-
-    let script_o = sqlx::query_as::<_, ScriptWDraft>(
-        "SELECT hash, script.path, summary, description, content, language, kind, tag, schema, draft_only, draft.value as draft FROM script LEFT JOIN draft ON 
-         script.path = draft.path AND script.workspace_id = draft.workspace_id AND draft.typ = 'script'
-         WHERE script.path = $1 AND script.workspace_id = $2 \
-         AND script.created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND \
-         workspace_id = $2)",
-    )
-    .bind(path)
-    .bind(w_id)
-    .fetch_optional(&mut tx)
-    .await?;
-    tx.commit().await?;
-
-    let script = not_found_if_none(script_o, "Script", path)?;
-    Ok(Json(script))
-}
-
 async fn list_paths(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
@@ -714,8 +637,6 @@ async fn archive_script_by_path(
     let path = path.to_path();
     let mut tx = user_db.begin(&authed).await?;
 
-    require_owner_of_path(&authed, path)?;
-
     let hash: i64 = sqlx::query_scalar!(
         "UPDATE script SET archived = true WHERE path = $1 AND workspace_id = $2 RETURNING hash",
         path,
@@ -829,19 +750,7 @@ async fn delete_script_by_path(
     let mut tx = user_db.begin(&authed).await?;
     let path = path.to_path();
 
-    let draft_only = sqlx::query_scalar!(
-        "SELECT draft_only FROM script WHERE path = $1 AND workspace_id = $2",
-        path,
-        w_id
-    )
-    .fetch_one(&db)
-    .await?
-    .unwrap_or(false);
-
-    if !draft_only {
-        require_admin(authed.is_admin, &authed.username)?;
-    }
-
+    require_admin(authed.is_admin, &authed.username)?;
     let script = sqlx::query_scalar!(
         "DELETE FROM script WHERE path = $1 AND workspace_id = $2 RETURNING path",
         path,

backend/windmill-api/src/users.rs

@@ -12,9 +12,8 @@ use crate::{
     db::{UserDB, DB},
     folders::get_folders_for_user,
     utils::require_super_admin,
-    webhook_util::{InstanceEvent, WebhookShared},
     workspaces::invite_user_to_all_auto_invite_worspaces,
-    COOKIE_DOMAIN, IS_SECURE,
+    COOKIE_DOMAIN, HTTP_CLIENT, IS_SECURE,
 };
 use argon2::{password_hash::SaltString, Argon2, PasswordHash, PasswordHasher, PasswordVerifier};
 use axum::{
@@ -359,8 +358,7 @@ pub struct Authed {
     pub username: String,
     pub is_admin: bool,
     pub groups: Vec<String>,
-    // (folder name, can write, is owner)
-    pub folders: Vec<(String, bool, bool)>,
+    pub folders: Vec<(String, bool)>,
 }
 
 pub async fn maybe_refresh_folders(path: &str, w_id: &str, authed: Authed, db: &DB) -> Authed {
@@ -370,7 +368,7 @@ pub async fn maybe_refresh_folders(path: &str, w_id: &str, authed: Authed, db: &
     let splitted = path.split('/').collect::<Vec<_>>();
     if splitted.len() >= 2
         && splitted[0] == "f"
-        && !authed.folders.iter().any(|(f, _, _)| f == splitted[1])
+        && !authed.folders.iter().any(|(f, _)| f == splitted[1])
     {
         let name = &authed.username;
         let groups = get_groups_for_user(w_id, name, db)
@@ -510,7 +508,6 @@ pub struct UserInfo {
     pub disabled: bool,
     pub role: Option<String>,
     pub folders: Vec<String>,
-    pub folders_owners: Vec<String>,
 }
 
 #[derive(FromRow, Serialize)]
@@ -810,14 +807,9 @@ async fn whoami(
             disabled: false,
             role: Some("superadmin".to_string()),
             folders: folders
-                .clone()
                 .into_iter()
                 .filter_map(|x| if x.1 { Some(x.0) } else { None })
                 .collect(),
-            folders_owners: folders
-                .into_iter()
-                .filter_map(|x| if x.2 { Some(x.0) } else { None })
-                .collect(),
         }))
     }
 }
@@ -903,14 +895,9 @@ async fn get_user(w_id: &str, username: &str, db: &DB) -> Result<Option<UserInfo
         disabled: usr.disabled,
         role: usr.role,
         folders: folders
-            .clone()
             .into_iter()
             .filter_map(|x| if x.1 { Some(x.0) } else { None })
             .collect(),
-        folders_owners: folders
-            .into_iter()
-            .filter_map(|x| if x.2 { Some(x.0) } else { None })
-            .collect(),
     }))
 }
 
@@ -925,45 +912,47 @@ pub async fn get_groups_for_user(w_id: &str, username: &str, db: &DB) -> Result<
     Ok(groups)
 }
 pub async fn is_owner_of_path(
-    authed: Authed,
-    Path((_w_id, path)): Path<(String, StripPath)>,
+    Authed { username, is_admin, groups, .. }: Authed,
+    Extension(db): Extension<DB>,
+    Path((w_id, path)): Path<(String, StripPath)>,
 ) -> JsonResult<bool> {
     let path = path.to_path();
-    if authed.is_admin {
+    if is_admin {
         Ok(Json(true))
     } else {
-        Ok(Json(require_owner_of_path(&authed, path).is_ok()))
+        Ok(Json(
+            require_owner_of_path(&w_id, &username, &groups, path, &db)
+                .await
+                .is_ok(),
+        ))
     }
 }
 
-pub fn require_owner_of_path(authed: &Authed, path: &str) -> Result<()> {
-    if authed.is_admin {
-        return Ok(());
-    }
+pub async fn require_owner_of_path(
+    w_id: &str,
+    username: &str,
+    groups: &Vec<String>,
+    path: &str,
+    db: &DB,
+) -> Result<()> {
     if !path.is_empty() {
         let splitted = path.split("/").collect::<Vec<&str>>();
         if splitted[0] == "u" {
-            if splitted[1] == authed.username {
-                Ok(())
+            if splitted[1] == username {
+                return Ok(());
             } else {
-                Err(Error::BadRequest(format!(
+                return Err(Error::BadRequest(format!(
                     "only the owner {} is authorized to perform this operation",
                     splitted[1]
-                )))
+                )));
             }
+        } else if splitted[0] == "g" {
+            return crate::groups::require_is_owner(splitted[1], username, groups, w_id, db).await;
         } else if splitted[0] == "f" {
-            crate::folders::require_is_owner(authed, splitted[1])
-        } else {
-            Err(Error::BadRequest(format!(
-                "Not recognized path kind: {}",
-                path
-            )))
+            return crate::folders::require_is_owner(splitted[1], username, groups, w_id, db).await;
         }
-    } else {
-        Err(Error::BadRequest(format!(
-            "Cannot be owner of an empty path"
-        )))
     }
+    Err(Error::BadRequest(format!("not recognized owner kind")))
 }
 
 async fn whois(
@@ -1044,16 +1033,10 @@ async fn decline_invite(
 
 lazy_static! {
     pub static ref VALID_USERNAME: Regex = Regex::new(r#"^[a-zA-Z_0-9]+$"#).unwrap();
-
-    pub static ref JOB_RETENTION_SECS: u32 = std::env::var("JOB_RETENTION_SECS")
-    .ok()
-    .and_then(|x| x.parse::<u32>().ok())
-    .unwrap_or(60 * 60 * 24 * 60); // 60 days
 }
 
 async fn accept_invite(
     Authed { email, .. }: Authed,
-    Extension(webhook): Extension<WebhookShared>,
     Extension(db): Extension<DB>,
     Json(nu): Json<AcceptInvite>,
 ) -> Result<(StatusCode, String)> {
@@ -1099,11 +1082,6 @@ async fn accept_invite(
     }
 
     if is_some {
-        webhook.send_instance_event(InstanceEvent::UserJoinedWorkspace {
-            email: email.clone(),
-            workspace: nu.workspace_id.clone(),
-            username: nu.username.clone(),
-        });
         Ok((
             StatusCode::CREATED,
             format!(
@@ -1344,7 +1322,6 @@ lazy_static::lazy_static! {
 async fn create_user(
     Authed { email, .. }: Authed,
     Extension(db): Extension<DB>,
-    Extension(webhook): Extension<WebhookShared>,
     Extension(argon2): Extension<Arc<Argon2<'_>>>,
     Json(mut nu): Json<NewUser>,
 ) -> Result<(StatusCode, String)> {
@@ -1385,9 +1362,19 @@ async fn create_user(
     .await?;
     tx.commit().await?;
 
+    if let Some(new_user_webhook) = NEW_USER_WEBHOOK.clone() {
+        let _ = HTTP_CLIENT
+            .post(&new_user_webhook)
+            .json(
+                &serde_json::json!({"email" : &nu.email, "name": &nu.name, "event": "global_add"}),
+            )
+            .send()
+            .await
+            .map_err(|e| tracing::error!("Error sending new user webhook: {}", e.to_string()));
+    }
+
     invite_user_to_all_auto_invite_worspaces(&db, &nu.email).await?;
 
-    webhook.send_instance_event(InstanceEvent::UserAdded { email: nu.email.clone() });
     Ok((StatusCode::CREATED, format!("email {} created", nu.email)))
 }
 
@@ -1918,24 +1905,16 @@ pub async fn delete_expired_items_perdiodically(
             Err(e) => tracing::error!("Error deleting pip_resolution: {}", e.to_string()),
         }
 
-        if *JOB_RETENTION_SECS > 0 {
-            let deleted_jobs = sqlx::query_scalar!(
-                "DELETE FROM completed_job WHERE started_at + ((duration_ms/1000 + $1) || ' s')::interval <= now() RETURNING id",
-                *JOB_RETENTION_SECS as i64
-            )
-            .fetch_all(db)
-            .await;
+        let magic_links_deleted_r: std::result::Result<Vec<String>, _> = sqlx::query_scalar(
+            "DELETE FROM magic_link WHERE expiration <= now()
+        RETURNING concat(substring(token for 10), '*****')",
+        )
+        .fetch_all(db)
+        .await;
 
-            match deleted_jobs {
-                Ok(deleted_jobs) => {
-                    tracing::info!(
-                        "deleted {} jobs completed JOB_RETENTION_SECS ago: {:?}",
-                        deleted_jobs.len(),
-                        deleted_jobs
-                    )
-                }
-                Err(e) => tracing::error!("Error deleting jobs: {}", e.to_string()),
-            }
+        match magic_links_deleted_r {
+            Ok(tokens) => tracing::debug!("deleted {} tokens: {:?}", tokens.len(), tokens),
+            Err(e) => tracing::error!("Error deleting token: {}", e.to_string()),
         }
 
         tokio::select! {

backend/windmill-api/src/variables.rs

@@ -12,6 +12,13 @@ use crate::{
     users::{maybe_refresh_folders, require_owner_of_path, Authed},
     webhook_util::{WebhookMessage, WebhookShared},
 };
+/*
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
+ * This file and its contents are licensed under the AGPLv3 License.
+ * Please see the included NOTICE for copyright information and
+ * LICENSE-AGPL for a copy of the license.
+ */
 
 use axum::{
     extract::{Extension, Path, Query},
@@ -373,8 +380,9 @@ async fn update_variable(
     if let Some(npath) = ns.path {
         if npath != path {
             check_path_conflict(&mut tx, &w_id, &npath).await?;
-            require_owner_of_path(&authed, path)?;
-
+            if !authed.is_admin {
+                require_owner_of_path(&w_id, &authed.username, &authed.groups, &path, &db).await?;
+            }
             let mut v = sqlx::query_scalar!(
                 "SELECT value FROM resource  WHERE path = $1 AND workspace_id = $2",
                 path,

backend/windmill-api/src/webhook_util.rs

@@ -13,26 +13,6 @@ lazy_static::lazy_static! {
         "Histogram of webhook requests made"
     )
     .unwrap();
-
-    pub static ref INSTANCE_EVENTS_WEBHOOK: Option<String> = std::env::var("INSTANCE_EVENTS_WEBHOOK").ok();
-
-}
-
-pub enum WebhookPayload {
-    WorkspaceEvent(String, WebhookMessage),
-    InstanceEvent(InstanceEvent),
-}
-
-#[derive(Serialize)]
-#[serde(tag = "type")]
-pub enum InstanceEvent {
-    UserSignupOAuth { email: String },
-    UserAdded { email: String },
-    // UserDeleted { email: String },
-    // UserDeletedWorkspace { workspace: String, email: String },
-    UserAddedWorkspace { workspace: String, email: String },
-    UserInvitedWorkspace { workspace: String, email: String },
-    UserJoinedWorkspace { workspace: String, email: String, username: String },
 }
 
 #[derive(Serialize)]
@@ -66,12 +46,12 @@ pub enum WebhookMessage {
 
 #[derive(Clone)]
 pub struct WebhookShared {
-    pub channel: mpsc::UnboundedSender<WebhookPayload>,
+    pub channel: mpsc::UnboundedSender<(String, WebhookMessage)>,
 }
 
 impl WebhookShared {
     pub fn new(mut shutdown_rx: tokio::sync::broadcast::Receiver<()>, db: DB) -> Self {
-        let (tx, mut rx) = mpsc::unbounded_channel::<WebhookPayload>();
+        let (tx, mut rx) = mpsc::unbounded_channel::<(String, WebhookMessage)>();
         let _process = tokio::spawn(async move {
             let client = reqwest::Client::builder()
                 // TODO: investigate pool timeouts and such if TCP load is high
@@ -86,7 +66,7 @@ impl WebhookShared {
                     biased;
                     _ = shutdown_rx.recv() => break,
                     r = rx.recv() => match r {
-                        Some(WebhookPayload::WorkspaceEvent(workspace_id, message)) => {
+                        Some((workspace_id, message)) => {
                             let url_guard = match cache.get(&workspace_id).await {
                                 Some(guard) => {
                                     guard
@@ -116,13 +96,6 @@ impl WebhookShared {
                                 drop(url_guard);
                             }
                         },
-                        Some(WebhookPayload::InstanceEvent(event)) => {
-                            if *METRICS_ENABLED { Some(WEBHOOK_REQUEST_COUNT.start_timer()) } else { None };
-                            let r = client.post(INSTANCE_EVENTS_WEBHOOK.as_ref().unwrap()).json(&event).send().await;
-                            if let Err(e) = r {
-                                tracing::error!("Error sending instance event: {}", e);
-                            }
-                        },
                         None => break,
                     },
                     _ = futures::future::poll_fn(|cx| cache_purge_interval.poll_tick(cx)) => {
@@ -137,16 +110,6 @@ impl WebhookShared {
     }
 
     pub fn send_message(&self, workspace_id: String, message: WebhookMessage) {
-        let _ = self.channel.send(WebhookPayload::WorkspaceEvent(
-            workspace_id.clone(),
-            message,
-        ));
-    }
-
-    pub fn send_instance_event(&self, event: InstanceEvent) {
-        if INSTANCE_EVENTS_WEBHOOK.is_none() {
-            return;
-        }
-        let _ = self.channel.send(WebhookPayload::InstanceEvent(event));
+        let _ = self.channel.send((workspace_id.clone(), message));
     }
 }

backend/windmill-api/src/worker_ping.rs

@@ -21,16 +21,7 @@ use windmill_common::{
 };
 
 pub fn global_service() -> Router {
-    Router::new()
-        .route("/list", get(list_worker_pings))
-        .route("/custom_tags", get(get_custom_tags))
-}
-
-lazy_static::lazy_static! {
-    pub static ref CUSTOM_TAGS: Vec<String> = std::env::var("CUSTOM_TAGS")
-        .ok()
-        .map(|x| x.split(',').map(|x| x.to_string()).collect::<Vec<_>>()).unwrap_or_default();
-
+    Router::new().route("/list", get(list_worker_pings))
 }
 
 #[derive(FromRow, Serialize, Deserialize)]
@@ -63,7 +54,3 @@ async fn list_worker_pings(
     tx.commit().await?;
     Ok(Json(rows))
 }
-
-async fn get_custom_tags() -> Json<Vec<String>> {
-    Json(CUSTOM_TAGS.clone())
-}

backend/windmill-api/src/workspaces.rs

@@ -16,10 +16,10 @@ use crate::{
     db::{UserDB, DB},
     folders::Folder,
     resources::{Resource, ResourceType},
-    users::{Authed, WorkspaceInvite, VALID_USERNAME},
+    users::{Authed, WorkspaceInvite, NEW_USER_WEBHOOK, VALID_USERNAME},
     utils::require_super_admin,
     variables::build_crypt,
-    webhook_util::{InstanceEvent, WebhookShared},
+    HTTP_CLIENT,
 };
 #[cfg(feature = "enterprise")]
 use axum::response::Redirect;
@@ -943,7 +943,6 @@ pub async fn invite_user_to_all_auto_invite_worspaces(db: &DB, email: &str) -> R
 async fn invite_user(
     Authed { username, is_admin, .. }: Authed,
     Extension(db): Extension<DB>,
-    Extension(webhook): Extension<WebhookShared>,
     Path(w_id): Path<String>,
     Json(mut nu): Json<NewWorkspaceInvite>,
 ) -> Result<(StatusCode, String)> {
@@ -967,10 +966,14 @@ async fn invite_user(
 
     tx.commit().await?;
 
-    webhook.send_instance_event(InstanceEvent::UserInvitedWorkspace {
-        email: nu.email.clone(),
-        workspace: w_id,
-    });
+    if let Some(new_user_webhook) = NEW_USER_WEBHOOK.clone() {
+        let _ = &HTTP_CLIENT
+            .post(&new_user_webhook)
+            .json(&serde_json::json!({"email" : &nu.email, "event": "workspace_invite"}))
+            .send()
+            .await
+            .map_err(|e| tracing::error!("Error sending new user webhook: {}", e.to_string()));
+    }
 
     Ok((
         StatusCode::CREATED,
@@ -981,7 +984,6 @@ async fn invite_user(
 async fn add_user(
     Authed { username, is_admin, .. }: Authed,
     Extension(db): Extension<DB>,
-    Extension(webhook): Extension<WebhookShared>,
     Path(w_id): Path<String>,
     Json(mut nu): Json<NewWorkspaceUser>,
 ) -> Result<(StatusCode, String)> {
@@ -1020,10 +1022,14 @@ async fn add_user(
 
     tx.commit().await?;
 
-    webhook.send_instance_event(InstanceEvent::UserAddedWorkspace {
-        workspace: w_id.clone(),
-        email: nu.email.clone(),
-    });
+    if let Some(new_user_webhook) = NEW_USER_WEBHOOK.clone() {
+        let _ = HTTP_CLIENT
+            .post(&new_user_webhook)
+            .json(&serde_json::json!({"email" : &nu.email, "event": "workspace_add"}))
+            .send()
+            .await
+            .map_err(|e| tracing::error!("Error sending new user webhook: {}", e.to_string()));
+    }
 
     Ok((
         StatusCode::CREATED,
@@ -1165,8 +1171,6 @@ where
                 "edited_at",
                 "edited_by",
                 "archived",
-                "has_draft",
-                "draft_only",
             ] {
                 if obj.contains_key(key) {
                     obj.remove(key);

backend/windmill-audit/src/lib.rs

@@ -41,8 +41,8 @@ pub struct AuditLog {
 }
 
 #[tracing::instrument(level = "trace", skip_all)]
-pub async fn audit_log<'c, E: sqlx::Executor<'c, Database = Postgres>>(
-    db: E,
+pub async fn audit_log<'c>(
+    db: &mut Transaction<'c, Postgres>,
     username: &str,
     operation: &str,
     action_kind: ActionKind,

backend/windmill-common/src/external_ip.rs

@@ -11,16 +11,10 @@
 //! For users writing scripts to access their infrastructure with firewalls requiring incoming
 //! connections to be from whitelisted IP addresses.
 
-use std::time::Duration;
-
 use reqwest::Result;
 
 pub async fn get_ip() -> Result<String> {
-    reqwest::ClientBuilder::new()
-        .timeout(Duration::from_secs(3))
-        .build()?
-        .get("https://hub.windmill.dev/getip")
-        .send()
+    reqwest::get("https://hub.windmill.dev/getip")
         .await?
         .error_for_status()?
         .text()

backend/windmill-common/src/flows.rs

@@ -33,8 +33,6 @@ pub struct Flow {
     pub archived: bool,
     pub schema: Option<Schema>,
     pub extra_perms: serde_json::Value,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub draft_only: Option<bool>,
 }
 
 #[derive(Serialize)]
@@ -49,9 +47,6 @@ pub struct ListableFlow {
     pub archived: bool,
     pub extra_perms: serde_json::Value,
     pub starred: bool,
-    pub has_draft: bool,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub draft_only: Option<bool>,
 }
 
 #[derive(Deserialize)]
@@ -62,7 +57,6 @@ pub struct NewFlow {
     pub description: String,
     pub value: serde_json::Value,
     pub schema: Option<Schema>,
-    pub draft_only: Option<bool>,
 }
 
 #[derive(Deserialize, Serialize, Debug, Clone, Default)]
@@ -258,8 +252,6 @@ pub enum FlowModuleValue {
         lock: Option<String>,
         #[serde(skip_serializing_if = "Option::is_none")]
         path: Option<String>,
-        #[serde(skip_serializing_if = "Option::is_none")]
-        tag: Option<String>,
         language: ScriptLang,
     },
     Identity,

backend/windmill-common/src/jobs.rs

@@ -1,201 +1 @@
-use serde::{Deserialize, Serialize};
-use sqlx::{Postgres, Transaction};
-use uuid::Uuid;
 
-use crate::{
-    error,
-    flow_status::FlowStatus,
-    flows::FlowValue,
-    get_latest_deployed_hash_for_path,
-    scripts::{ScriptHash, ScriptLang},
-};
-
-#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Clone)]
-#[sqlx(type_name = "JOB_KIND", rename_all = "lowercase")]
-#[serde(rename_all(serialize = "lowercase"))]
-pub enum JobKind {
-    Script,
-    #[allow(non_camel_case_types)]
-    Script_Hub,
-    Preview,
-    Dependencies,
-    Flow,
-    FlowPreview,
-    Identity,
-    FlowDependencies,
-}
-
-#[derive(Debug, sqlx::FromRow, Serialize, Clone)]
-pub struct QueuedJob {
-    pub workspace_id: String,
-    pub id: Uuid,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub parent_job: Option<Uuid>,
-    pub created_by: String,
-    pub created_at: chrono::DateTime<chrono::Utc>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub started_at: Option<chrono::DateTime<chrono::Utc>>,
-    pub scheduled_for: chrono::DateTime<chrono::Utc>,
-    pub running: bool,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub script_hash: Option<ScriptHash>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub script_path: Option<String>,
-    pub args: Option<serde_json::Value>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub logs: Option<String>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub raw_code: Option<String>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub raw_lock: Option<String>,
-    pub canceled: bool,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub canceled_by: Option<String>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub canceled_reason: Option<String>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub last_ping: Option<chrono::DateTime<chrono::Utc>>,
-    pub job_kind: JobKind,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub schedule_path: Option<String>,
-    pub permissioned_as: String,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub flow_status: Option<serde_json::Value>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub raw_flow: Option<serde_json::Value>,
-    pub is_flow_step: bool,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub language: Option<ScriptLang>,
-    pub same_worker: bool,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub pre_run_error: Option<String>,
-    pub email: String,
-    pub visible_to_owner: bool,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub suspend: Option<i32>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub mem_peak: Option<i32>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub root_job: Option<Uuid>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub leaf_jobs: Option<serde_json::Value>,
-    pub tag: String,
-}
-
-impl QueuedJob {
-    pub fn script_path(&self) -> &str {
-        self.script_path
-            .as_ref()
-            .map(String::as_str)
-            .unwrap_or("tmp/main")
-    }
-}
-
-impl QueuedJob {
-    pub fn parse_raw_flow(&self) -> Option<FlowValue> {
-        self.raw_flow
-            .as_ref()
-            .and_then(|v| serde_json::from_value::<FlowValue>(v.clone()).ok())
-    }
-
-    pub fn parse_flow_status(&self) -> Option<FlowStatus> {
-        self.flow_status
-            .as_ref()
-            .and_then(|v| serde_json::from_value::<FlowStatus>(v.clone()).ok())
-    }
-}
-
-impl Default for QueuedJob {
-    fn default() -> Self {
-        Self {
-            workspace_id: "".to_string(),
-            id: Uuid::default(),
-            parent_job: None,
-            created_by: "".to_string(),
-            created_at: chrono::Utc::now(),
-            started_at: None,
-            scheduled_for: chrono::Utc::now(),
-            running: false,
-            script_hash: None,
-            script_path: None,
-            args: None,
-            logs: None,
-            raw_code: None,
-            raw_lock: None,
-            canceled: false,
-            canceled_by: None,
-            canceled_reason: None,
-            last_ping: None,
-            job_kind: JobKind::Identity,
-            schedule_path: None,
-            permissioned_as: "".to_string(),
-            flow_status: None,
-            raw_flow: None,
-            is_flow_step: false,
-            language: None,
-            same_worker: false,
-            pre_run_error: None,
-            email: "".to_string(),
-            visible_to_owner: false,
-            suspend: None,
-            mem_peak: None,
-            root_job: None,
-            leaf_jobs: None,
-            tag: "deno".to_string(),
-        }
-    }
-}
-
-#[derive(Debug, Clone)]
-pub enum JobPayload {
-    ScriptHub { path: String },
-    ScriptHash { hash: ScriptHash, path: String },
-    Code(RawCode),
-    Dependencies { hash: ScriptHash, dependencies: String, language: ScriptLang },
-    FlowDependencies { path: String },
-    Flow(String),
-    RawFlow { value: FlowValue, path: Option<String> },
-    Identity,
-}
-
-#[derive(Clone, Serialize, Deserialize, Debug)]
-pub struct RawCode {
-    pub content: String,
-    pub path: Option<String>,
-    pub language: ScriptLang,
-    pub lock: Option<String>,
-}
-
-type Tag = String;
-
-pub async fn script_path_to_payload<'c>(
-    script_path: &str,
-    db: &mut Transaction<'c, Postgres>,
-    w_id: &String,
-) -> error::Result<(JobPayload, Option<Tag>)> {
-    let (job_payload, tag) = if script_path.starts_with("hub/") {
-        (JobPayload::ScriptHub { path: script_path.to_owned() }, None)
-    } else {
-        let (script_hash, tag) = get_latest_deployed_hash_for_path(db, w_id, script_path).await?;
-        (
-            JobPayload::ScriptHash { hash: script_hash, path: script_path.to_owned() },
-            tag,
-        )
-    };
-    Ok((job_payload, tag))
-}
-
-pub async fn script_hash_to_tag<'c>(
-    script_hash: &ScriptHash,
-    db: &mut Transaction<'c, Postgres>,
-    w_id: &String,
-) -> error::Result<Option<Tag>> {
-    Ok(sqlx::query_scalar!(
-        "select tag from script where hash = $1 AND workspace_id = $2",
-        script_hash.0,
-        w_id
-    )
-    .fetch_optional(db)
-    .await?
-    .flatten())
-}

backend/windmill-common/src/lib.rs

@@ -6,7 +6,7 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
-use std::net::SocketAddr;
+use std::{net::SocketAddr, sync::Arc};
 
 use error::Error;
 
@@ -15,7 +15,6 @@ pub mod error;
 pub mod external_ip;
 pub mod flow_status;
 pub mod flows;
-pub mod jobs;
 pub mod more_serde;
 pub mod oauth2;
 pub mod schedule;
@@ -43,14 +42,11 @@ lazy_static::lazy_static! {
     .flatten();
     pub static ref METRICS_ENABLED: bool = METRICS_ADDR.is_some();
     pub static ref BASE_URL: String = std::env::var("BASE_URL").unwrap_or_else(|_| "http://localhost".to_string());
-    pub static ref IS_READY: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
+    pub static ref IS_READY: Arc<std::sync::atomic::AtomicBool> = Arc::new(std::sync::atomic::AtomicBool::new(false));
 }
 
 #[cfg(feature = "tokio")]
-pub async fn shutdown_signal(
-    tx: tokio::sync::broadcast::Sender<()>,
-    mut rx: tokio::sync::broadcast::Receiver<()>,
-) -> anyhow::Result<()> {
+pub async fn shutdown_signal(tx: tokio::sync::broadcast::Sender<()>) -> anyhow::Result<()> {
     use std::io;
     use tokio::signal::unix::SignalKind;
 
@@ -64,9 +60,6 @@ pub async fn shutdown_signal(
     tokio::select! {
         _ = terminate() => {},
         _ = tokio::signal::ctrl_c() => {},
-        _ = rx.recv() => {
-            tracing::info!("shutdown monitor received killpill");
-        },
     }
     println!("signal received, starting graceful shutdown");
     let _ = tx.send(());
@@ -153,44 +146,23 @@ pub async fn connect(
         .map_err(|err| Error::ConnectingToDatabase(err.to_string()))
 }
 
-type Tag = String;
-
-pub async fn get_latest_deployed_hash_for_path<'c>(
-    db: &mut sqlx::Transaction<'c, sqlx::Postgres>,
-    w_id: &str,
-    script_path: &str,
-) -> error::Result<(scripts::ScriptHash, Option<Tag>)> {
-    let r_o = sqlx::query!(
-        "select hash, tag from script where path = $1 AND workspace_id = $2 AND
-    created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2 AND
-    deleted = false AND archived = false AND lock IS not NULL AND lock_error_logs IS NULL)",
-        script_path,
-        w_id
-    )
-    .fetch_optional(db)
-    .await?;
-
-    let script = utils::not_found_if_none(r_o, "script", script_path)?;
-
-    Ok((scripts::ScriptHash(script.hash), script.tag))
-}
-
+// TODO: Move this elsewhere
 pub async fn get_latest_hash_for_path<'c>(
     db: &mut sqlx::Transaction<'c, sqlx::Postgres>,
     w_id: &str,
     script_path: &str,
-) -> error::Result<(scripts::ScriptHash, Option<Tag>)> {
-    let r_o = sqlx::query!(
-        "select hash, tag from script where path = $1 AND workspace_id = $2 AND
-    created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2 AND
-    deleted = false AND archived = false)",
+) -> error::Result<scripts::ScriptHash> {
+    let script_hash_o = sqlx::query_scalar!(
+        "select hash from script where path = $1 AND workspace_id = $2 AND
+    created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2) AND
+    deleted = false",
         script_path,
         w_id
     )
     .fetch_optional(db)
     .await?;
 
-    let script = utils::not_found_if_none(r_o, "script", script_path)?;
+    let script_hash = utils::not_found_if_none(script_hash_o, "ScriptHash", script_path)?;
 
-    Ok((scripts::ScriptHash(script.hash), script.tag))
+    Ok(scripts::ScriptHash(script_hash))
 }

backend/windmill-common/src/scripts.rs

@@ -17,7 +17,7 @@ use serde_json::to_string_pretty;
 
 use crate::utils::StripPath;
 
-#[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Hash, Eq)]
+#[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Hash)]
 #[cfg_attr(feature = "sqlx", derive(sqlx::Type))]
 #[cfg_attr(
     feature = "sqlx",
@@ -136,28 +136,27 @@ pub struct Script {
     pub lock_error_logs: Option<String>,
     pub language: ScriptLang,
     pub kind: ScriptKind,
-    pub tag: Option<String>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub draft_only: Option<bool>,
 }
 
 #[derive(Serialize)]
 #[cfg_attr(feature = "sqlx", derive(sqlx::FromRow))]
 pub struct ListableScript {
+    pub workspace_id: String,
     pub hash: ScriptHash,
     pub path: String,
+    pub parent_hashes: Option<ScriptHashes>,
     pub summary: String,
+    pub description: String,
+    pub created_by: String,
     pub created_at: chrono::DateTime<chrono::Utc>,
     pub archived: bool,
+    pub deleted: bool,
+    pub is_template: bool,
     pub extra_perms: serde_json::Value,
+    pub lock_error_logs: Option<String>,
     pub language: ScriptLang,
+    pub kind: ScriptKind,
     pub starred: bool,
-    pub tag: Option<String>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub has_draft: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub draft_only: Option<bool>,
-    pub has_deploy_errors: bool,
 }
 
 #[derive(Serialize, Deserialize, Debug)]
@@ -187,8 +186,6 @@ pub struct NewScript {
     pub lock: Option<Vec<String>>,
     pub language: ScriptLang,
     pub kind: Option<ScriptKind>,
-    pub tag: Option<String>,
-    pub draft_only: Option<bool>,
 }
 
 #[derive(Deserialize)]

backend/windmill-common/src/utils.rs

@@ -6,11 +6,12 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
-use crate::error::{Error, Result};
 use rand::{distributions::Alphanumeric, thread_rng, Rng};
-use serde::{Deserialize, Serialize};
+use serde::Deserialize;
 use sha2::{Digest, Sha256};
 
+use crate::error::{Error, Result};
+
 pub const MAX_PER_PAGE: usize = 10000;
 pub const DEFAULT_PER_PAGE: usize = 1000;
 
@@ -19,8 +20,7 @@ pub struct Pagination {
     pub page: Option<usize>,
     pub per_page: Option<usize>,
 }
-
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Deserialize)]
 pub struct StripPath(pub String);
 
 impl StripPath {
@@ -52,8 +52,8 @@ pub fn paginate(pagination: Pagination) -> (usize, usize) {
 }
 
 #[cfg(feature = "sqlx")]
-pub async fn now_from_db<'c, E: sqlx::PgExecutor<'c>>(
-    db: E,
+pub async fn now_from_db<'c>(
+    db: &mut sqlx::Transaction<'c, sqlx::Postgres>,
 ) -> Result<chrono::DateTime<chrono::Utc>> {
     Ok(sqlx::query_scalar!("SELECT now()")
         .fetch_one(db)

backend/windmill-queue/Cargo.toml

@@ -31,7 +31,3 @@ reqwest.workspace = true
 lazy_static.workspace = true
 prometheus.workspace = true
 cron.workspace = true
-rsmq_async.workspace = true
-tokio.workspace = true
-futures-core.workspace = true
-itertools.workspace = true

backend/windmill-queue/src/jobs.rs

@@ -9,8 +9,8 @@
 use std::collections::HashMap;
 
 use anyhow::Context;
-use itertools::Itertools;
 use reqwest::Client;
+use serde::{Deserialize, Serialize};
 use sqlx::{Pool, Postgres, Transaction};
 use tracing::{instrument, Instrument};
 use ulid::Ulid;
@@ -20,14 +20,11 @@ use windmill_common::{
     error::{self, Error},
     flow_status::{FlowStatus, JobResult, MAX_RETRY_ATTEMPTS, MAX_RETRY_INTERVAL},
     flows::{FlowModule, FlowModuleValue, FlowValue},
-    jobs::{JobKind, JobPayload, QueuedJob, RawCode},
     scripts::{get_full_hub_script_by_path, HubScript, ScriptHash, ScriptLang},
     utils::StripPath,
     METRICS_ENABLED,
 };
 
-use crate::QueueTransaction;
-
 lazy_static::lazy_static! {
     pub static ref HTTP_CLIENT: Client = reqwest::ClientBuilder::new()
         .user_agent("windmill/beta")
@@ -51,34 +48,17 @@ lazy_static::lazy_static! {
     .unwrap();
     pub static ref CLOUD_HOSTED: bool = std::env::var("CLOUD_HOSTED").is_ok();
 
-    pub static ref ACCEPTED_TAGS: Vec<String> = std::env::var("WORKER_TAGS")
-        .ok()
-        .map(|x| x.split(',').map(|x| x.to_string()).collect())
-        .unwrap_or_else(|| vec![
-            "deno".to_string(),
-            "python3".to_string(),
-            "go".to_string(),
-            "bash".to_string(),
-            "dependency".to_string(),
-            "flow".to_string(),
-            "hub".to_string(),
-            "other".to_string()]);
-
-    pub static ref ACCEPTED_TAGS_FILTER: String = format!(" AND ({})",
-        ACCEPTED_TAGS.clone().into_iter().map(|x| format!("(tag = '{x}')")).join(" OR "));
 }
 
 const MAX_FREE_EXECS: i32 = 1000;
 const MAX_FREE_CONCURRENT_RUNS: i32 = 15;
-const RSMQ_MAIN_QUEUE: &'static str = "main_queue";
 
-pub async fn cancel_job<'c, R: rsmq_async::RsmqConnection + Clone>(
+pub async fn cancel_job<'c>(
     username: &str,
     reason: Option<String>,
     id: Uuid,
     w_id: &str,
     mut tx: Transaction<'c, Postgres>,
-    rsmq: Option<R>,
     force_rerun: bool,
 ) -> error::Result<(Transaction<'c, Postgres>, Option<Uuid>)> {
     let job_option = sqlx::query_scalar!(
@@ -92,12 +72,6 @@ pub async fn cancel_job<'c, R: rsmq_async::RsmqConnection + Clone>(
     )
     .fetch_optional(&mut tx)
     .await?;
-    if let Some(mut rsmq) = rsmq {
-        rsmq.change_message_visibility(RSMQ_MAIN_QUEUE, &id.to_string(), 0)
-            .await
-            .map_err(|e| anyhow::anyhow!(e))?;
-    }
-
     let mut jobs = job_option.map(|j| vec![j]).unwrap_or_default();
     while !jobs.is_empty() {
         let p_job = jobs.pop();
@@ -117,11 +91,10 @@ pub async fn cancel_job<'c, R: rsmq_async::RsmqConnection + Clone>(
     Ok((tx, job_option))
 }
 
-pub async fn pull<R: rsmq_async::RsmqConnection + Clone>(
+pub async fn pull(
     db: &Pool<Postgres>,
     whitelist_workspaces: Option<Vec<String>>,
     blacklist_workspaces: Option<Vec<String>>,
-    rsmq: Option<R>,
 ) -> windmill_common::error::Result<Option<QueuedJob>> {
     let mut workspaces_filter = String::new();
     if let Some(whitelist) = whitelist_workspaces {
@@ -133,9 +106,6 @@ pub async fn pull<R: rsmq_async::RsmqConnection + Clone>(
                 .collect::<Vec<String>>()
                 .join(",")
         ));
-        if let Some(_rsmq) = rsmq {
-            todo!("REDIS: Implement workspace filters for redis");
-        }
     }
     if let Some(blacklist) = blacklist_workspaces {
         workspaces_filter.push_str(&format!(
@@ -146,51 +116,16 @@ pub async fn pull<R: rsmq_async::RsmqConnection + Clone>(
                 .collect::<Vec<String>>()
                 .join(",")
         ));
-        if let Some(_rsmq) = rsmq {
-            todo!("REDIS: Implement workspace filters for redis");
-        }
     }
-
-    let job: Option<QueuedJob> = if let Some(mut rsmq) = rsmq {
-        // TODO: REDIS: Race conditions / replace last_ping
-        let msg = rsmq
-            .pop_message::<Vec<u8>>(RSMQ_MAIN_QUEUE)
-            .await
-            .map_err(|e| anyhow::anyhow!(e))?;
-
-        if let Some(msg) = msg {
-            let uuid = Uuid::from_bytes_le(
-                msg.message
-                    .try_into()
-                    .map_err(|_| anyhow::anyhow!("Failed to parsed Redis message"))?,
-            );
-
-            sqlx::query_as::<_, QueuedJob>(
-                "UPDATE queue
-            SET running = true
-            , started_at = coalesce(started_at, now())
-            , last_ping = now()
-            , suspend_until = null
-            WHERE id = $1
-            RETURNING *",
-            )
-            .bind(uuid)
-            .fetch_optional(db)
-            .await?
-        } else {
-            None
-        }
-    } else {
-        let accepted_tags_filter = &*ACCEPTED_TAGS_FILTER;
-        /* Jobs can be started if they:
-         * - haven't been started before,
-         *   running = false
-         * - are flows with a step that needed resume,
-         *   suspend_until is non-null
-         *   and suspend = 0 when the resume messages are received
-         *   or suspend_until <= now() if it has timed out */
-        sqlx::query_as::<_, QueuedJob>(&format!(
-            "UPDATE queue
+    /* Jobs can be started if they:
+     * - haven't been started before,
+     *   running = false
+     * - are flows with a step that needed resume,
+     *   suspend_until is non-null
+     *   and suspend = 0 when the resume messages are received
+     *   or suspend_until <= now() if it has timed out */
+    let job: Option<QueuedJob> = sqlx::query_as::<_, QueuedJob>(&format!(
+        "UPDATE queue
             SET running = true
               , started_at = coalesce(started_at, now())
               , last_ping = now()
@@ -202,18 +137,15 @@ pub async fn pull<R: rsmq_async::RsmqConnection + Clone>(
                        AND scheduled_for <= now())
                    OR (suspend_until IS NOT NULL
                        AND (   suspend <= 0
-                            OR suspend_until <= now()))) 
-                    {workspaces_filter}
-                    {accepted_tags_filter}
+                            OR suspend_until <= now()))) {workspaces_filter}
                 ORDER BY scheduled_for
                 FOR UPDATE SKIP LOCKED
                 LIMIT 1
             )
             RETURNING *"
-        ))
-        .fetch_optional(db)
-        .await?
-    };
+    ))
+    .fetch_optional(db)
+    .await?;
 
     if job.is_some() && *METRICS_ENABLED {
         QUEUE_PULL_COUNT.inc();
@@ -275,11 +207,11 @@ pub async fn get_result_by_id(
 }
 
 #[instrument(level = "trace", skip_all)]
-pub async fn delete_job<'c, R: rsmq_async::RsmqConnection + Clone + Send>(
-    mut tx: QueueTransaction<'c, R>,
+pub async fn delete_job(
+    db: &Pool<Postgres>,
     w_id: &str,
     job_id: Uuid,
-) -> windmill_common::error::Result<QueueTransaction<'c, R>> {
+) -> windmill_common::error::Result<()> {
     if *METRICS_ENABLED {
         QUEUE_DELETE_COUNT.inc();
     }
@@ -288,13 +220,13 @@ pub async fn delete_job<'c, R: rsmq_async::RsmqConnection + Clone + Send>(
         w_id,
         job_id
     )
-    .fetch_one(&mut tx)
+    .fetch_one(db)
     .await
     .map_err(|e| Error::InternalErr(format!("Error during deletion of job {job_id}: {e}")))?
     .unwrap_or(0)
         == 1;
     tracing::debug!("Job {job_id} deleted: {job_removed}");
-    Ok(tx)
+    Ok(())
 }
 
 pub async fn get_queued_job<'c>(
@@ -313,9 +245,9 @@ pub async fn get_queued_job<'c>(
     Ok(r)
 }
 
-// #[instrument(level = "trace", skip_all)]
-pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
-    mut tx: QueueTransaction<'c, R>,
+#[instrument(level = "trace", skip_all)]
+pub async fn push<'c>(
+    mut tx: Transaction<'c, Postgres>,
     workspace_id: &str,
     job_payload: JobPayload,
     args: serde_json::Map<String, serde_json::Value>,
@@ -330,8 +262,8 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
     mut same_worker: bool,
     pre_run_error: Option<&windmill_common::error::Error>,
     visible_to_owner: bool,
-    mut tag: Option<String>,
-) -> Result<(Uuid, QueueTransaction<'c, R>), Error> {
+) -> Result<(Uuid, Transaction<'c, Postgres>), Error> {
+    let scheduled_for = scheduled_for_o.unwrap_or_else(chrono::Utc::now);
     let args_json = serde_json::Value::Object(args);
     let job_id: Uuid = Ulid::new().into();
 
@@ -578,34 +510,12 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
 
     let flow_status = raw_flow.as_ref().map(FlowStatus::new);
 
-    let tag = if job_kind == JobKind::Dependencies || job_kind == JobKind::FlowDependencies {
-        "dependency".to_string()
-    } else if job_kind == JobKind::Flow || job_kind == JobKind::FlowPreview {
-        "flow".to_string()
-    } else if job_kind == JobKind::Identity {
-        // identity is a light script, deno is too
-        "deno".to_string()
-    } else if job_kind == JobKind::Script_Hub {
-        "hub".to_string()
-    } else {
-        if tag == Some("".to_string()) {
-            tag = None;
-        }
-        tag.unwrap_or_else(|| {
-            language
-                .as_ref()
-                .map(|x| x.as_str())
-                .unwrap_or_else(|| "deno")
-                .to_string()
-        })
-    };
-
     let uuid = sqlx::query_scalar!(
         "INSERT INTO queue
             (workspace_id, id, running, parent_job, created_by, permissioned_as, scheduled_for, 
                 script_hash, script_path, raw_code, raw_lock, args, job_kind, schedule_path, raw_flow, \
-         flow_status, is_flow_step, language, started_at, same_worker, pre_run_error, email, visible_to_owner, root_job, tag)
-            VALUES ($1, $2, $3, $4, $5, $6, COALESCE($7, now()), $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, CASE WHEN $3 THEN now() END, $19, $20, $21, $22, $23, $24) \
+         flow_status, is_flow_step, language, started_at, same_worker, pre_run_error, email, visible_to_owner, root_job)
+            VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, CASE WHEN $3 THEN now() END, $19, $20, $21, $22, $23) \
          RETURNING id",
         workspace_id,
         job_id,
@@ -613,7 +523,7 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
         parent_job,
         user,
         permissioned_as,
-        scheduled_for_o,
+        scheduled_for,
         script_hash,
         script_path.clone(),
         raw_code,
@@ -629,8 +539,7 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
         pre_run_error.map(|e| e.to_string()),
         email,
         visible_to_owner,
-        root_job,
-        tag
+        root_job
     )
     .fetch_one(&mut tx)
     .await
@@ -673,10 +582,6 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
         .instrument(tracing::info_span!("job_run", email = &email))
         .await?;
     }
-    if let Some(ref mut rsmq) = tx.rsmq {
-        rsmq.send_message(job_id.to_bytes_le().to_vec(), scheduled_for_o);
-    }
-
     Ok((uuid, tx))
 }
 
@@ -698,3 +603,117 @@ pub async fn get_hub_script(
         .await
         .map(|e| e)
 }
+
+#[derive(Debug, sqlx::FromRow, Serialize, Clone)]
+pub struct QueuedJob {
+    pub workspace_id: String,
+    pub id: Uuid,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub parent_job: Option<Uuid>,
+    pub created_by: String,
+    pub created_at: chrono::DateTime<chrono::Utc>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub started_at: Option<chrono::DateTime<chrono::Utc>>,
+    pub scheduled_for: chrono::DateTime<chrono::Utc>,
+    pub running: bool,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub script_hash: Option<ScriptHash>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub script_path: Option<String>,
+    pub args: Option<serde_json::Value>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub logs: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub raw_code: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub raw_lock: Option<String>,
+    pub canceled: bool,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub canceled_by: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub canceled_reason: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub last_ping: Option<chrono::DateTime<chrono::Utc>>,
+    pub job_kind: JobKind,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub schedule_path: Option<String>,
+    pub permissioned_as: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub flow_status: Option<serde_json::Value>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub raw_flow: Option<serde_json::Value>,
+    pub is_flow_step: bool,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub language: Option<ScriptLang>,
+    pub same_worker: bool,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub pre_run_error: Option<String>,
+    pub email: String,
+    pub visible_to_owner: bool,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub suspend: Option<i32>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub mem_peak: Option<i32>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub root_job: Option<Uuid>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub leaf_jobs: Option<serde_json::Value>,
+}
+
+impl QueuedJob {
+    pub fn script_path(&self) -> &str {
+        self.script_path
+            .as_ref()
+            .map(String::as_str)
+            .unwrap_or("tmp/main")
+    }
+}
+
+impl QueuedJob {
+    pub fn parse_raw_flow(&self) -> Option<FlowValue> {
+        self.raw_flow
+            .as_ref()
+            .and_then(|v| serde_json::from_value::<FlowValue>(v.clone()).ok())
+    }
+
+    pub fn parse_flow_status(&self) -> Option<FlowStatus> {
+        self.flow_status
+            .as_ref()
+            .and_then(|v| serde_json::from_value::<FlowStatus>(v.clone()).ok())
+    }
+}
+
+#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Clone)]
+#[sqlx(type_name = "JOB_KIND", rename_all = "lowercase")]
+#[serde(rename_all(serialize = "lowercase"))]
+pub enum JobKind {
+    Script,
+    #[allow(non_camel_case_types)]
+    Script_Hub,
+    Preview,
+    Dependencies,
+    Flow,
+    FlowPreview,
+    Identity,
+    FlowDependencies,
+}
+
+#[derive(Debug, Clone)]
+pub enum JobPayload {
+    ScriptHub { path: String },
+    ScriptHash { hash: ScriptHash, path: String },
+    Code(RawCode),
+    Dependencies { hash: ScriptHash, dependencies: String, language: ScriptLang },
+    FlowDependencies { path: String },
+    Flow(String),
+    RawFlow { value: FlowValue, path: Option<String> },
+    Identity,
+}
+
+#[derive(Clone, Serialize, Deserialize, Debug)]
+pub struct RawCode {
+    pub content: String,
+    pub path: Option<String>,
+    pub language: ScriptLang,
+    pub lock: Option<String>,
+}

backend/windmill-queue/src/lib.rs

@@ -7,8 +7,6 @@
  */
 
 mod jobs;
-mod queue_transaction;
 pub mod schedule;
 
 pub use jobs::*;
-pub use queue_transaction::*;

backend/windmill-queue/src/queue_transaction.rs

@@ -1,161 +0,0 @@
-use std::fmt::Debug;
-
-use futures_core::{future::BoxFuture, stream::BoxStream};
-use rsmq_async::{RedisBytes, RsmqConnection};
-use sqlx::{Postgres, Transaction};
-
-pub enum RedisOp {
-    SendMessage(RedisBytes, Option<chrono::DateTime<chrono::Utc>>),
-    DeleteMessage(String),
-}
-
-impl RedisOp {
-    pub async fn apply<R: RsmqConnection>(self, rsmq: &mut R) -> Result<(), rsmq_async::RsmqError> {
-        match self {
-            RedisOp::SendMessage(bytes, time) => {
-                rsmq.send_message(
-                    "main_queue",
-                    bytes,
-                    time.map(|t| (t - chrono::Utc::now()).num_seconds())
-                        .and_then(|e| e.try_into().ok()),
-                )
-                .await?;
-            }
-            RedisOp::DeleteMessage(id) => {
-                rsmq.delete_message("main_queue", &id).await?;
-            }
-        };
-
-        Ok(())
-    }
-}
-
-pub struct RedisTransaction<R: RsmqConnection> {
-    rsmq: R,
-    queued_ops: Vec<RedisOp>,
-}
-
-impl<R: RsmqConnection> From<R> for RedisTransaction<R> {
-    fn from(value: R) -> Self {
-        Self { rsmq: value, queued_ops: Vec::new() }
-    }
-}
-
-impl<R: RsmqConnection> RedisTransaction<R> {
-    pub async fn commit(self) -> Result<(), rsmq_async::RsmqError> {
-        let mut rsmq = self.rsmq;
-        for op in self.queued_ops {
-            op.apply(&mut rsmq).await?;
-        }
-        Ok(())
-    }
-
-    pub fn send_message<E: Into<RedisBytes>>(
-        &mut self,
-        bytes: E,
-        delay_until: Option<chrono::DateTime<chrono::Utc>>,
-    ) {
-        self.queued_ops
-            .push(RedisOp::SendMessage(bytes.into(), delay_until))
-    }
-
-    pub fn delete_message(&mut self, id: String) {
-        self.queued_ops.push(RedisOp::DeleteMessage(id))
-    }
-}
-
-pub struct QueueTransaction<'c, R: RsmqConnection> {
-    pub rsmq: Option<RedisTransaction<R>>,
-    transaction: Transaction<'c, Postgres>,
-}
-
-impl<'c, R: RsmqConnection> From<(Option<R>, Transaction<'c, Postgres>)>
-    for QueueTransaction<'c, R>
-{
-    fn from(value: (Option<R>, Transaction<'c, Postgres>)) -> Self {
-        Self { rsmq: value.0.map(|e| e.into()), transaction: value.1 }
-    }
-}
-
-impl<'c, R: RsmqConnection> Debug for QueueTransaction<'c, R> {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("QueueTransaction")
-            .field("rsmq", &self.rsmq.as_ref().map(|_| ())) // do not require R: Debug
-            .field("transaction", &self.transaction)
-            .finish()
-    }
-}
-
-impl<'c, R: RsmqConnection> QueueTransaction<'c, R> {
-    pub async fn commit(self) -> Result<(), windmill_common::error::Error> {
-        self.transaction.commit().await?;
-        if let Some(rsmq) = self.rsmq {
-            rsmq.commit().await.map_err(|e| anyhow::anyhow!(e))?;
-        }
-
-        Ok(())
-    }
-
-    pub fn transaction_mut<'a>(&'a mut self) -> &'a mut Transaction<'c, Postgres> {
-        &mut self.transaction
-    }
-}
-
-impl<'c, 'b, R: RsmqConnection + Send> sqlx::Executor<'b> for &'b mut QueueTransaction<'c, R> {
-    type Database = Postgres;
-
-    fn fetch_many<'e, 'q: 'e, E: 'q>(
-        self,
-        query: E,
-    ) -> BoxStream<
-        'e,
-        Result<
-            sqlx::Either<
-                <Self::Database as sqlx::Database>::QueryResult,
-                <Self::Database as sqlx::Database>::Row,
-            >,
-            sqlx::Error,
-        >,
-    >
-    where
-        'b: 'e,
-        E: sqlx::Execute<'q, Self::Database>,
-    {
-        self.transaction.fetch_many(query)
-    }
-
-    fn fetch_optional<'e, 'q: 'e, E: 'q>(
-        self,
-        query: E,
-    ) -> BoxFuture<'e, Result<Option<<Self::Database as sqlx::Database>::Row>, sqlx::Error>>
-    where
-        'b: 'e,
-        E: sqlx::Execute<'q, Self::Database>,
-    {
-        self.transaction.fetch_optional(query)
-    }
-
-    fn prepare_with<'e, 'q: 'e>(
-        self,
-        sql: &'q str,
-        parameters: &'e [<Self::Database as sqlx::Database>::TypeInfo],
-    ) -> BoxFuture<
-        'e,
-        Result<<Self::Database as sqlx::database::HasStatement<'q>>::Statement, sqlx::Error>,
-    >
-    where
-        'b: 'e,
-    {
-        self.transaction.prepare_with(sql, parameters)
-    }
-
-    fn describe<'e, 'q: 'e>(
-        self,
-        sql: &'q str,
-    ) -> BoxFuture<'e, Result<sqlx::Describe<Self::Database>, sqlx::Error>>
-    where
-        'b: 'e,
-    {
-        self.transaction.describe(sql)
-    }
-}

backend/windmill-queue/src/schedule.rs

@@ -6,11 +6,9 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
-use crate::push;
-use crate::QueueTransaction;
+use crate::{push, JobPayload};
 use sqlx::{query_scalar, Postgres, Transaction};
 use std::str::FromStr;
-use windmill_common::jobs::JobPayload;
 use windmill_common::{
     error::{self, Result},
     schedule::Schedule,
@@ -18,10 +16,10 @@ use windmill_common::{
     utils::{now_from_db, StripPath},
 };
 
-pub async fn push_scheduled_job<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
-    mut tx: QueueTransaction<'c, R>,
+pub async fn push_scheduled_job<'c>(
+    mut tx: Transaction<'c, Postgres>,
     schedule: Schedule,
-) -> Result<QueueTransaction<'c, R>> {
+) -> Result<Transaction<'c, Postgres>> {
     let sched = cron::Schedule::from_str(&schedule.schedule)
         .map_err(|e| error::Error::BadRequest(e.to_string()))?;
 
@@ -67,30 +65,21 @@ pub async fn push_scheduled_job<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
         }
     }
 
-    let (payload, tag) = if schedule.is_flow {
-        (JobPayload::Flow(schedule.script_path), None)
+    let payload = if schedule.is_flow {
+        JobPayload::Flow(schedule.script_path)
     } else {
-        let (hash, tag) = windmill_common::get_latest_hash_for_path(
-            tx.transaction_mut(),
-            &schedule.workspace_id,
-            &schedule.script_path,
-        )
-        .await?;
-        (
-            JobPayload::ScriptHash { hash, path: schedule.script_path },
-            tag,
-        )
+        JobPayload::ScriptHash {
+            hash: windmill_common::get_latest_hash_for_path(
+                &mut tx,
+                &schedule.workspace_id,
+                &schedule.script_path,
+            )
+            .await?,
+            path: schedule.script_path,
+        }
     };
 
-    sqlx::query!(
-        "UPDATE schedule SET error = NULL WHERE workspace_id = $1 AND path = $2",
-        &schedule.workspace_id,
-        &schedule.path
-    )
-    .execute(&mut tx)
-    .await?;
-
-    let (_, tx) = push(
+    let (_, mut tx) = push(
         tx,
         &schedule.workspace_id,
         payload,
@@ -106,10 +95,16 @@ pub async fn push_scheduled_job<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
         false,
         None,
         true,
-        tag,
     )
     .await?;
-    Ok(tx) // TODO: Bubble up pushed UUID from here
+    sqlx::query!(
+        "UPDATE schedule SET error = NULL WHERE workspace_id = $1 AND path = $2",
+        &schedule.workspace_id,
+        &schedule.path
+    )
+    .execute(&mut tx)
+    .await?;
+    Ok(tx)
 }
 
 pub async fn get_schedule_opt<'c>(

backend/windmill-worker/Cargo.toml

@@ -48,5 +48,4 @@ deno_core.workspace = true
 const_format.workspace = true
 git-version.workspace = true
 dyn-iter.workspace = true
-once_cell.workspace = true
-rsmq_async.workspace = true
+once_cell.workspace = true

backend/windmill-worker/src/common.rs

@@ -1,41 +0,0 @@
-use sqlx::{Pool, Postgres};
-use tokio::{fs::File, io::AsyncReadExt};
-use windmill_common::error::{self, Error};
-use windmill_queue::CLOUD_HOSTED;
-
-use crate::MAX_RESULT_SIZE;
-
-pub async fn read_result(job_dir: &str) -> error::Result<serde_json::Value> {
-    let mut file = File::open(format!("{job_dir}/result.json")).await?;
-    let mut content = "".to_string();
-    file.read_to_string(&mut content).await?;
-    if *CLOUD_HOSTED && content.len() > MAX_RESULT_SIZE {
-        return Err(Error::ExecutionErr("Result is too large for the cloud app (limit 2MB). 
-        If using this script as part of the flow, use the shared folder to pass heavy data between steps.".to_owned()));
-    }
-    serde_json::from_str(&content)
-        .map_err(|e| Error::ExecutionErr(format!("Error parsing result: {e}")))
-}
-
-#[tracing::instrument(level = "trace", skip_all)]
-pub async fn set_logs(logs: &str, id: &uuid::Uuid, db: &Pool<Postgres>) {
-    if sqlx::query!(
-        "UPDATE queue SET logs = $1 WHERE id = $2",
-        logs.to_owned(),
-        id
-    )
-    .execute(db)
-    .await
-    .is_err()
-    {
-        tracing::error!(%id, "error updating logs for id {id}")
-    };
-}
-
-pub fn capitalize(s: &str) -> String {
-    let mut c = s.chars();
-    match c.next() {
-        None => String::new(),
-        Some(f) => f.to_uppercase().collect::<String>() + c.as_str(),
-    }
-}

backend/windmill-worker/src/global_cache.rs

@@ -1,521 +0,0 @@
-#[cfg(feature = "enterprise")]
-use crate::{ROOT_CACHE_DIR, ROOT_TMP_CACHE_DIR, TAR_CACHE_RATE, TAR_PIP_TMP_CACHE_DIR, TMP_DIR};
-#[cfg(feature = "enterprise")]
-use itertools::Itertools;
-#[cfg(feature = "enterprise")]
-use rand::Rng;
-#[cfg(feature = "enterprise")]
-use std::process::Stdio;
-
-#[cfg(feature = "enterprise")]
-use tokio::{process::Command, sync::mpsc::Sender, time::Instant};
-
-#[cfg(feature = "enterprise")]
-use windmill_common::error;
-
-#[cfg(feature = "enterprise")]
-const TAR_CACHE_FILENAME: &str = "denogocache.tar";
-
-#[cfg(feature = "enterprise")]
-pub async fn build_tar_and_push(bucket: &str, folder: String) -> error::Result<()> {
-    tracing::info!("Started building and pushing piptar {folder}");
-    let start = Instant::now();
-    let folder_name = folder.split("/").last().unwrap();
-    let tar_path = format!("{TAR_PIP_TMP_CACHE_DIR}/{folder_name}.tar",);
-
-    if let Err(e) = execute_command(
-        ROOT_TMP_CACHE_DIR,
-        "tar",
-        vec!["-c", "-f", &tar_path, "-C", &folder, "."],
-    )
-    .await
-    {
-        tracing::info!("Failed to tar cache. Error: {:?}", e);
-        return Err(e);
-    }
-
-    let tar_metadata = tokio::fs::metadata(&tar_path).await;
-    if tar_metadata.is_err() || tar_metadata.as_ref().unwrap().len() == 0 {
-        tracing::info!("Failed to tar cache: {folder}");
-        return Err(error::Error::ExecutionErr(format!(
-            "Failed to tar cache: {folder}"
-        )));
-    }
-
-    if let Err(e) = execute_command(
-        ROOT_TMP_CACHE_DIR,
-        "rclone",
-        vec![
-            "copyto",
-            &tar_path,
-            &format!(":s3,env_auth=true:{bucket}/tar/pip/{folder_name}.tar"),
-            "-v",
-            "--size-only",
-            "--fast-list",
-            "--s3-no-check-bucket",
-        ],
-    )
-    .await
-    {
-        tracing::info!("Failed to copy piptar {folder} to bucket. Error: {:?}", e);
-        return Err(e);
-    }
-
-    tracing::info!(
-        "Finished copying piptar {folder} to bucket {bucket} as tar, took: {:?}s. Size of tar: {}",
-        start.elapsed().as_secs(),
-        tar_metadata.unwrap().len()
-    );
-    Ok(())
-}
-
-#[cfg(feature = "enterprise")]
-pub async fn pull_from_tar(bucket: &str, folder: String) -> error::Result<()> {
-    use tokio::fs::metadata;
-    let folder_name = folder.split("/").last().unwrap();
-
-    tracing::info!("Attempting to pull piptar {folder_name} from bucket");
-
-    let start = Instant::now();
-    let tar_path = format!("tar/pip/{folder_name}.tar");
-    let target = format!("{ROOT_TMP_CACHE_DIR}/{tar_path}.single");
-    if let Err(e) = execute_command(
-        ROOT_TMP_CACHE_DIR,
-        "rclone",
-        vec![
-            "copyto",
-            &format!(":s3,env_auth=true:{bucket}/{tar_path}"),
-            &target,
-            "-v",
-            "--size-only",
-            "--fast-list",
-        ],
-    )
-    .await
-    {
-        tracing::info!(
-            "Failed to copy tar {folder_name} from bucket. Error: {:?}",
-            e
-        );
-        return Err(e);
-    }
-
-    if metadata(&target).await.is_err() {
-        tracing::info!(
-            "piptar {folder_name} not found in bucket. Took {:?}ms",
-            start.elapsed().as_millis()
-        );
-        return Err(error::Error::ExecutionErr(format!(
-            "tar {folder_name} does not exist in bucket"
-        )));
-    }
-
-    extract_pip_tar(&target, &folder).await?;
-    tokio::fs::remove_file(&target).await?;
-    tracing::info!(
-        "Finished pulling and extracting {folder_name}. Took {:?}ms",
-        start.elapsed().as_millis()
-    );
-
-    Ok(())
-}
-
-#[cfg(feature = "enterprise")]
-pub async fn cache_global(bucket: &str, tx: Sender<()>) -> error::Result<()> {
-    copy_cache_from_bucket(bucket, tx).await?;
-    copy_cache_to_bucket(bucket).await?;
-
-    // this is to prevent excessive tar upload. 1/100*15min = each worker sync its tar once per day on average
-    if rand::thread_rng().gen_range(0..*TAR_CACHE_RATE) == 0 {
-        copy_cache_to_bucket_as_tar(bucket).await;
-    }
-    Ok(())
-}
-
-#[cfg(feature = "enterprise")]
-pub async fn copy_cache_from_bucket(bucket: &str, tx: Sender<()>) -> error::Result<()> {
-    tracing::info!("Copying cache from bucket in the background {bucket}");
-    let bucket = bucket.to_string();
-
-    let start = Instant::now();
-
-    if let Err(e) = execute_command(
-        ROOT_TMP_CACHE_DIR,
-        "rclone",
-        vec![
-            "copy",
-            &format!(":s3,env_auth=true:{bucket}"),
-            &ROOT_TMP_CACHE_DIR,
-            "--size-only",
-            "--fast-list",
-            "--filter",
-            "- deno/gen/file/**",
-            "--filter",
-            "+ deno/**",
-            "--filter",
-            "+ go/**",
-            "--filter",
-            "+ tar/**",
-            "--filter",
-            "- *",
-        ],
-    )
-    .await
-    {
-        tracing::info!("Failed to copy cache from bucket. Error: {:?}", e);
-        return Err(e);
-    }
-
-    tracing::info!(
-        "Finished copying cache from bucket {bucket}, took {:?}s",
-        start.elapsed().as_secs()
-    );
-
-    tx.send(()).await.expect("can send copy cache signal");
-
-    Ok(())
-}
-
-#[cfg(feature = "enterprise")]
-pub async fn copy_cache_to_bucket(bucket: &str) -> error::Result<()> {
-    tracing::info!("Copying cache to bucket {bucket}");
-    let start = Instant::now();
-
-    if let Err(e) = execute_command(
-        ROOT_TMP_CACHE_DIR,
-        "rclone",
-        vec![
-            "copy",
-            &ROOT_TMP_CACHE_DIR,
-            &format!(":s3,env_auth=true:{bucket}"),
-            "--size-only",
-            "--fast-list",
-            "--filter",
-            "- deno/gen/file/**",
-            "--filter",
-            "+ deno/**",
-            "--filter",
-            "+ go/**",
-            "--filter",
-            "- *",
-        ],
-    )
-    .await
-    {
-        tracing::info!("Failed to copy cache to bucket. Error: {:?}", e);
-        return Err(e);
-    }
-    tracing::info!(
-        "Finished copying cache to bucket {bucket}, took: {:?}s",
-        start.elapsed().as_secs()
-    );
-    Ok(())
-}
-
-#[cfg(feature = "enterprise")]
-pub async fn copy_cache_to_bucket_as_tar(bucket: &str) {
-    tracing::info!("Copying cache to bucket {bucket} as tar");
-    let start = Instant::now();
-
-    if let Err(e) = execute_command(
-        ROOT_TMP_CACHE_DIR,
-        "tar",
-        vec![
-            "-c",
-            "-f",
-            &format!("{ROOT_TMP_CACHE_DIR}{TAR_CACHE_FILENAME}"),
-            "go",
-            "deno",
-        ],
-    )
-    .await
-    {
-        tracing::info!("Failed to tar cache. Error: {:?}", e);
-        return;
-    }
-
-    let tar_metadata =
-        tokio::fs::metadata(format!("{ROOT_TMP_CACHE_DIR}{TAR_CACHE_FILENAME}")).await;
-    if tar_metadata.is_err() || tar_metadata.as_ref().unwrap().len() == 0 {
-        tracing::info!("Failed to tar cache");
-        return;
-    }
-
-    if let Err(e) = execute_command(
-        ROOT_TMP_CACHE_DIR,
-        "rclone",
-        vec![
-            "copyto",
-            &format!("{ROOT_TMP_CACHE_DIR}{TAR_CACHE_FILENAME}"),
-            &format!(":s3,env_auth=true:{bucket}/{TAR_CACHE_FILENAME}"),
-            "-v",
-            "--size-only",
-            "--fast-list",
-            "--s3-no-check-bucket",
-        ],
-    )
-    .await
-    {
-        tracing::info!("Failed to copy tar to bucket. Error: {:?}", e);
-        return;
-    }
-
-    if let Err(e) =
-        tokio::fs::remove_file(format!("{ROOT_TMP_CACHE_DIR}{TAR_CACHE_FILENAME}")).await
-    {
-        tracing::info!("Failed to remove tar cache. Error: {:?}", e);
-    };
-
-    tracing::info!(
-        "Finished copying cache to bucket {bucket} as tar, took: {:?}s. Size of new tar: {}",
-        start.elapsed().as_secs(),
-        tar_metadata.unwrap().len()
-    );
-}
-
-#[cfg(feature = "enterprise")]
-pub async fn copy_denogo_cache_from_bucket_as_tar(bucket: &str) {
-    use tokio::fs::metadata;
-
-    tracing::info!("Copying denogo cache from bucket {bucket} as tar");
-
-    let start: Instant = Instant::now();
-
-    if let Err(e) = execute_command(
-        ROOT_TMP_CACHE_DIR,
-        "rclone",
-        vec![
-            "copyto",
-            &format!(":s3,env_auth=true:{bucket}/{TAR_CACHE_FILENAME}"),
-            &format!("{ROOT_TMP_CACHE_DIR}{TAR_CACHE_FILENAME}"),
-            "-v",
-            "--size-only",
-            "--fast-list",
-        ],
-    )
-    .await
-    {
-        tracing::info!("Failed copying denogo tar from cache. Error: {:?}", e);
-        return;
-    }
-
-    if let Err(e) = execute_command(
-        ROOT_CACHE_DIR,
-        "tar",
-        vec![
-            "-xpvf",
-            &format!("{ROOT_TMP_CACHE_DIR}{TAR_CACHE_FILENAME}"),
-        ],
-    )
-    .await
-    {
-        tracing::info!("Failed to untar denogo. Error: {:?}", e);
-        return;
-    }
-
-    let denogen = format!("{ROOT_TMP_CACHE_DIR}deno/gen/file");
-    if metadata(&denogen).await.is_ok() {
-        let _ = tokio::fs::remove_dir_all(denogen).await;
-    }
-
-    if let Err(e) =
-        tokio::fs::remove_file(format!("{ROOT_TMP_CACHE_DIR}{TAR_CACHE_FILENAME}")).await
-    {
-        tracing::info!("Failed to remove denotar cache. Error: {:?}", e);
-        return;
-    };
-
-    tracing::info!(
-        "Finished copying denogotar from bucket {bucket} as tar, took: {:?}s",
-        start.elapsed().as_secs()
-    );
-}
-
-#[cfg(feature = "enterprise")]
-pub async fn copy_all_piptars_from_bucket(bucket: &str) {
-    tracing::info!("Copying all piptars cache from bucket {bucket}");
-
-    let start = Instant::now();
-
-    if let Err(e) = execute_command(
-        ROOT_CACHE_DIR,
-        "rclone",
-        vec![
-            "copy",
-            &format!(":s3,env_auth=true:{bucket}/tar/pip/"),
-            &TAR_PIP_TMP_CACHE_DIR,
-            "-v",
-            "--size-only",
-            "--fast-list",
-        ],
-    )
-    .await
-    {
-        tracing::info!("Failed transferring all piptars from cache. Error: {:?}", e);
-        return;
-    }
-
-    tracing::info!(
-        "Finished transferring piptars from bucket {bucket} as tar, took: {:?}s",
-        start.elapsed().as_secs()
-    );
-}
-
-// async fn check_if_bucket_syncable(bucket: &str) -> bool {
-//     match Command::new("rclone")
-//         .arg("lsf")
-//         .arg(format!(":s3,env_auth=true:{bucket}/NOSYNC"))
-
-//         .arg("-vv")
-//         .arg("--fast-list")
-//         .stdin(Stdio::null())
-//         .stdout(Stdio::null())
-//         .output()
-//         .await;
-//     return true;
-// }
-
-#[cfg(feature = "enterprise")]
-pub async fn copy_tmp_cache_to_cache() -> error::Result<()> {
-    let start: Instant = Instant::now();
-    execute_command(
-        TMP_DIR,
-        "rclone",
-        vec![
-            "sync",
-            ROOT_TMP_CACHE_DIR,
-            ROOT_CACHE_DIR,
-            "--filter",
-            "- deno/gen/file/**",
-            "--filter",
-            "+ deno/**",
-            "--filter",
-            "+ go/**",
-            "--filter",
-            "- *",
-        ],
-    )
-    .await?;
-
-    tracing::info!(
-        "Finished copying local tmp cache to local cache. Took {}ms",
-        start.elapsed().as_millis(),
-    );
-
-    if let Err(e) = untar_all_piptars().await {
-        tracing::info!("Failed to untar piptars. Error: {:?}", e);
-    }
-
-    Ok(())
-}
-
-#[cfg(feature = "enterprise")]
-pub async fn untar_all_piptars() -> error::Result<()> {
-    use tokio::fs::{self, metadata};
-
-    use crate::PIP_CACHE_DIR;
-
-    let start: Instant = Instant::now();
-
-    let mut entries = fs::read_dir(TAR_PIP_TMP_CACHE_DIR).await?;
-    while let Some(entry) = entries.next_entry().await? {
-        if let Err(e) = {
-            let entry_path = entry.path();
-            let path = entry_path.to_str().expect("Could not convert path to str");
-            let folder = format!(
-                "{PIP_CACHE_DIR}/{}",
-                path.split('/')
-                    .last()
-                    .unwrap()
-                    .strip_suffix(".tar")
-                    .unwrap()
-            );
-            if metadata(&folder).await.is_ok() {
-                continue;
-            }
-            extract_pip_tar(&path, &folder).await?;
-            Ok(()) as error::Result<()>
-        } {
-            tracing::info!("Failed to extract pip tar. Error: {:?}", e);
-        }
-    }
-
-    tracing::info!(
-        "Finished untarring all piptars. Took {}ms",
-        start.elapsed().as_millis(),
-    );
-    Ok(())
-}
-
-#[cfg(feature = "enterprise")]
-pub async fn extract_pip_tar(tar: &str, folder: &str) -> error::Result<()> {
-    use tokio::fs;
-
-    let start: Instant = Instant::now();
-    fs::create_dir_all(&folder).await?;
-    if let Err(e) = execute_command(&folder, "tar", vec!["-xpvf", tar]).await {
-        tracing::info!("Failed to untar piptar. Error: {:?}", e);
-        fs::remove_dir_all(&folder).await?;
-        return Err(e);
-    }
-    tracing::info!(
-        "Finished extracting pip tar {folder}. Took {}ms",
-        start.elapsed().as_millis(),
-    );
-    Ok(())
-}
-
-#[cfg(feature = "enterprise")]
-pub async fn copy_cache_to_tmp_cache() -> error::Result<()> {
-    let start: Instant = Instant::now();
-    execute_command(
-        TMP_DIR,
-        "rclone",
-        vec![
-            "sync",
-            ROOT_CACHE_DIR,
-            ROOT_TMP_CACHE_DIR,
-            "--filter",
-            "- deno/gen/file/**",
-            "--filter",
-            "+ deno/**",
-            "--filter",
-            "+ go/**",
-            "--filter",
-            "- *",
-        ],
-    )
-    .await?;
-    tracing::info!(
-        "Finished copying local cache to local tmp cache. Took {}ms",
-        start.elapsed().as_millis()
-    );
-    Ok(())
-}
-
-#[cfg(feature = "enterprise")]
-pub async fn execute_command(dir: &str, command: &str, args: Vec<&str>) -> error::Result<()> {
-    match Command::new(command)
-        .current_dir(dir)
-        .args(args.clone())
-        .stdin(Stdio::null())
-        .stdout(Stdio::null())
-        .spawn()
-    {
-        Ok(mut h) => {
-            if !h.wait().await.unwrap().success() {
-                return Err(error::Error::ExecutionErr(format!(
-                    "Failed to apply {command} with args: {}",
-                    args.iter().join(" ")
-                )));
-            }
-        }
-        Err(e) => {
-            return Err(error::Error::ExecutionErr(format!(
-                "Failed to apply {command} with args: {}. Error: {e:?}",
-                args.iter().join(" ")
-            )));
-        }
-    }
-    Ok(())
-}

backend/windmill-worker/src/go_executor.rs

@@ -1,373 +0,0 @@
-use std::process::Stdio;
-
-use itertools::Itertools;
-use tokio::{
-    fs::{DirBuilder, File},
-    io::AsyncReadExt,
-    process::Command,
-};
-use uuid::Uuid;
-use windmill_common::{
-    error::{self, Error},
-    jobs::QueuedJob,
-    utils::calculate_hash,
-};
-use windmill_parser_go::parse_go_imports;
-
-use crate::{
-    common::{capitalize, read_result, set_logs},
-    create_args_and_out_file, get_reserved_variables, handle_child, write_file,
-    AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, GOPRIVATE, GO_CACHE_DIR, HOME_ENV,
-    NETRC, NSJAIL_PATH, PATH_ENV,
-};
-
-const GO_REQ_SPLITTER: &str = "//go.sum\n";
-const NSJAIL_CONFIG_RUN_GO_CONTENT: &str = include_str!("../nsjail/run.go.config.proto");
-
-lazy_static::lazy_static! {
-    static ref GO_PATH: String = std::env::var("GO_PATH").unwrap_or_else(|_| "/usr/bin/go".to_string());
-}
-
-#[tracing::instrument(level = "trace", skip_all)]
-pub async fn handle_go_job(
-    logs: &mut String,
-    job: &QueuedJob,
-    db: &sqlx::Pool<sqlx::Postgres>,
-    client: &AuthedClientBackgroundTask,
-    inner_content: &str,
-    job_dir: &str,
-    requirements_o: Option<String>,
-    shared_mount: &str,
-    base_internal_url: &str,
-    worker_name: &str,
-) -> Result<serde_json::Value, Error> {
-    //go does not like executing modules at temp root
-    let job_dir = &format!("{job_dir}/go");
-    let (skip_go_mod, skip_tidy) = if let Some(requirements) = requirements_o {
-        gen_go_mod(inner_content, job_dir, &requirements).await?
-    } else {
-        (false, false)
-    };
-
-    logs.push_str("\n\n--- GO DEPENDENCIES SETUP ---\n");
-    set_logs(logs, &job.id, db).await;
-
-    install_go_dependencies(
-        &job.id,
-        inner_content,
-        logs,
-        job_dir,
-        db,
-        true,
-        skip_go_mod,
-        skip_tidy,
-        worker_name,
-        &job.workspace_id,
-    )
-    .await?;
-
-    logs.push_str("\n\n--- GO CODE EXECUTION ---\n");
-    set_logs(logs, &job.id, db).await;
-    let client = &client.get_authed().await;
-    create_args_and_out_file(client, job, job_dir).await?;
-    {
-        let sig = windmill_parser_go::parse_go_sig(&inner_content)?;
-        drop(inner_content);
-
-        const WRAPPER_CONTENT: &str = r#"package main
-
-import (
-    "encoding/json"
-    "os"
-    "fmt"
-    "mymod/inner"
-)
-
-func main() {{
-
-    dat, err := os.ReadFile("args.json")
-    if err != nil {{
-        fmt.Println(err)
-        os.Exit(1)
-    }}
-
-    var req inner.Req
-
-    if err := json.Unmarshal(dat, &req); err != nil {{
-        fmt.Println(err)
-        os.Exit(1)
-    }}
-
-    res, err := inner.Run(req)
-    if err != nil {{
-        fmt.Println(err)
-        os.Exit(1)
-    }}
-    res_json, err := json.Marshal(res)
-    if err != nil {{
-        fmt.Println(err)
-        os.Exit(1)
-    }}
-    f, err := os.OpenFile("result.json", os.O_APPEND|os.O_WRONLY, os.ModeAppend)
-    if err != nil {{
-        fmt.Println(err)
-        os.Exit(1)
-    }}
-    _, err = f.WriteString(string(res_json))
-    if err != nil {{
-        fmt.Println(err)
-        os.Exit(1)
-    }}
-}}"#;
-
-        write_file(job_dir, "main.go", WRAPPER_CONTENT).await?;
-
-        {
-            let spread = &sig
-                .args
-                .clone()
-                .into_iter()
-                .map(|x| format!("req.{}", capitalize(&x.name)))
-                .join(", ");
-            let req_body = &sig
-                .args
-                .into_iter()
-                .map(|x| {
-                    format!(
-                        "{} {} `json:\"{}\"`",
-                        capitalize(&x.name),
-                        windmill_parser_go::otyp_to_string(x.otyp),
-                        x.name
-                    )
-                })
-                .join("\n");
-            let runner_content: String = format!(
-                r#"package inner
-type Req struct {{
-    {req_body}
-}}
-
-func Run(req Req) (interface{{}}, error){{
-    return main({spread})
-}}
-
-"#,
-            );
-            write_file(&format!("{job_dir}/inner"), "runner.go", &runner_content).await?;
-        }
-    }
-    let mut reserved_variables = get_reserved_variables(job, &client.token, db).await?;
-    reserved_variables.insert("RUST_LOG".to_string(), "info".to_string());
-
-    let child = if !*DISABLE_NSJAIL {
-        let _ = write_file(
-            job_dir,
-            "run.config.proto",
-            &NSJAIL_CONFIG_RUN_GO_CONTENT
-                .replace("{JOB_DIR}", job_dir)
-                .replace("{CACHE_DIR}", GO_CACHE_DIR)
-                .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string())
-                .replace("{SHARED_MOUNT}", shared_mount),
-        )
-        .await?;
-        let build_go = Command::new(GO_PATH.as_str())
-            .current_dir(job_dir)
-            .env_clear()
-            .env("PATH", PATH_ENV.as_str())
-            .env("BASE_INTERNAL_URL", base_internal_url)
-            .env("GOPATH", GO_CACHE_DIR)
-            .env("HOME", HOME_ENV.as_str())
-            .args(vec!["build", "main.go"])
-            .stdout(Stdio::piped())
-            .stderr(Stdio::piped())
-            .spawn()?;
-        handle_child(
-            &job.id,
-            db,
-            logs,
-            build_go,
-            false,
-            worker_name,
-            &job.workspace_id,
-            "go build",
-        )
-        .await?;
-
-        Command::new(NSJAIL_PATH.as_str())
-            .current_dir(job_dir)
-            .env_clear()
-            .envs(reserved_variables)
-            .env("PATH", PATH_ENV.as_str())
-            .env("BASE_INTERNAL_URL", base_internal_url)
-            .args(vec!["--config", "run.config.proto", "--", "/tmp/go/main"])
-            .stdout(Stdio::piped())
-            .stderr(Stdio::piped())
-            .spawn()?
-    } else {
-        if let Some(ref netrc) = *NETRC {
-            write_file(&HOME_ENV, ".netrc", netrc).await?;
-        }
-        Command::new(GO_PATH.as_str())
-            .current_dir(job_dir)
-            .env_clear()
-            .envs(reserved_variables)
-            .env("PATH", PATH_ENV.as_str())
-            .env("BASE_INTERNAL_URL", base_internal_url)
-            .env("GOPATH", GO_CACHE_DIR)
-            .env("GOPRIVATE", GOPRIVATE.as_ref().unwrap_or(&String::new()))
-            .env("HOME", HOME_ENV.as_str())
-            .args(vec!["run", "main.go"])
-            .stdout(Stdio::piped())
-            .stderr(Stdio::piped())
-            .spawn()?
-    };
-    handle_child(
-        &job.id,
-        db,
-        logs,
-        child,
-        !*DISABLE_NSJAIL,
-        worker_name,
-        &job.workspace_id,
-        "go run",
-    )
-    .await?;
-    read_result(job_dir).await
-}
-
-async fn gen_go_mod(
-    inner_content: &str,
-    job_dir: &str,
-    requirements: &str,
-) -> error::Result<(bool, bool)> {
-    gen_go_mymod(inner_content, job_dir).await?;
-
-    let md = requirements.split_once(GO_REQ_SPLITTER);
-    if let Some((req, sum)) = md {
-        write_file(job_dir, "go.mod", &req).await?;
-        write_file(job_dir, "go.sum", &sum).await?;
-        Ok((true, true))
-    } else {
-        write_file(job_dir, "go.mod", &requirements).await?;
-        Ok((true, false))
-    }
-}
-
-pub async fn install_go_dependencies(
-    job_id: &Uuid,
-    code: &str,
-    logs: &mut String,
-    job_dir: &str,
-    db: &sqlx::Pool<sqlx::Postgres>,
-    non_dep_job: bool,
-    skip_go_mod: bool,
-    has_sum: bool,
-    worker_name: &str,
-    w_id: &str,
-) -> error::Result<String> {
-    if !skip_go_mod {
-        gen_go_mymod(code, job_dir).await?;
-        let child = Command::new("go")
-            .current_dir(job_dir)
-            .args(vec!["mod", "init", "mymod"])
-            .stdout(Stdio::piped())
-            .stderr(Stdio::piped())
-            .spawn()?;
-
-        handle_child(job_id, db, logs, child, false, worker_name, w_id, "go init").await?;
-    }
-
-    let mut new_lockfile = false;
-
-    let hash = if !has_sum {
-        calculate_hash(parse_go_imports(&code)?.iter().join("\n").as_str())
-    } else {
-        "".to_string()
-    };
-
-    let mut skip_tidy = has_sum;
-
-    if !has_sum {
-        if let Some(cached) = sqlx::query_scalar!(
-            "SELECT lockfile FROM pip_resolution_cache WHERE hash = $1",
-            hash
-        )
-        .fetch_optional(db)
-        .await?
-        {
-            logs.push_str(&format!("\nfound cached resolution"));
-            gen_go_mod(code, job_dir, &cached).await?;
-            skip_tidy = true;
-            new_lockfile = false;
-        } else {
-            new_lockfile = true;
-        }
-    }
-
-    let mod_command = if skip_tidy { "download" } else { "tidy" };
-    let child = Command::new(GO_PATH.as_str())
-        .current_dir(job_dir)
-        .env("GOPATH", GO_CACHE_DIR)
-        .args(vec!["mod", mod_command])
-        .stdout(Stdio::piped())
-        .stderr(Stdio::piped())
-        .spawn()?;
-    handle_child(
-        job_id,
-        db,
-        logs,
-        child,
-        false,
-        worker_name,
-        &w_id,
-        &format!("go {mod_command}"),
-    )
-    .await
-    .map_err(|e| Error::ExecutionErr(format!("Lock file generation failed: {e:?}")))?;
-
-    if (!new_lockfile || has_sum) && non_dep_job {
-        return Ok("".to_string());
-    }
-
-    let mut req_content = "".to_string();
-
-    let mut file = File::open(format!("{job_dir}/go.mod")).await?;
-    file.read_to_string(&mut req_content).await?;
-    req_content.push_str(GO_REQ_SPLITTER);
-    let sum_path = format!("{job_dir}/go.sum");
-    if tokio::fs::metadata(&sum_path).await.is_ok() {
-        let mut file = File::open(sum_path).await?;
-        file.read_to_string(&mut req_content).await?;
-    }
-
-    if non_dep_job {
-        sqlx::query!(
-            "INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() + ('3 days')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = $2",
-            hash,
-            req_content
-        ).fetch_optional(db).await?;
-
-        return Ok(String::new());
-    } else {
-        Ok(req_content)
-    }
-}
-
-async fn gen_go_mymod(code: &str, job_dir: &str) -> error::Result<()> {
-    let code = if code.trim_start().starts_with("package") {
-        code.to_string()
-    } else {
-        format!("package inner; {code}")
-    };
-
-    let mymod_dir = format!("{job_dir}/inner");
-    DirBuilder::new()
-        .recursive(true)
-        .create(&mymod_dir)
-        .await
-        .expect("could not create go's mymod dir");
-
-    write_file(&mymod_dir, "inner_main.go", &code).await?;
-
-    Ok(())
-}

backend/windmill-worker/src/jobs.rs

@@ -6,32 +6,29 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
-use sqlx::{Pool, Postgres};
+use std::time::SystemTime;
+
+use sqlx::{Pool, Postgres, Transaction};
 use tracing::instrument;
 use uuid::Uuid;
 use windmill_common::{
-    error::Error,
-    flow_status::FlowStatusModule,
-    jobs::{JobKind, QueuedJob},
-    schedule::Schedule,
-    METRICS_ENABLED,
+    error::Error, flow_status::FlowStatusModule, schedule::Schedule, METRICS_ENABLED,
 };
-use windmill_queue::{delete_job, schedule::get_schedule_opt, QueueTransaction, CLOUD_HOSTED};
+use windmill_queue::{delete_job, schedule::get_schedule_opt, JobKind, QueuedJob, CLOUD_HOSTED};
 
 #[instrument(level = "trace", skip_all)]
-pub async fn add_completed_job_error<R: rsmq_async::RsmqConnection + Clone + Send>(
+pub async fn add_completed_job_error(
     db: &Pool<Postgres>,
     queued_job: &QueuedJob,
     logs: String,
     e: serde_json::Value,
     metrics: Option<crate::worker::Metrics>,
-    rsmq: Option<R>,
 ) -> Result<serde_json::Value, Error> {
     if *METRICS_ENABLED {
         metrics.map(|m| m.worker_execution_failed.inc());
     }
     let result = serde_json::json!({ "error": e });
-    let _ = add_completed_job(db, &queued_job, false, false, result.clone(), logs, rsmq).await?;
+    let _ = add_completed_job(db, &queued_job, false, false, result.clone(), logs).await?;
     Ok(result)
 }
 
@@ -54,14 +51,13 @@ fn flatten_jobs(modules: Vec<FlowStatusModule>) -> Vec<Uuid> {
 }
 
 #[instrument(level = "trace", skip_all)]
-pub async fn add_completed_job<R: rsmq_async::RsmqConnection + Clone + Send>(
+pub async fn add_completed_job(
     db: &Pool<Postgres>,
     queued_job: &QueuedJob,
     success: bool,
     skipped: bool,
     result: serde_json::Value,
     logs: String,
-    rsmq: Option<R>,
 ) -> Result<Uuid, Error> {
     let duration =
         if queued_job.job_kind == JobKind::Flow || queued_job.job_kind == JobKind::FlowPreview {
@@ -93,7 +89,7 @@ pub async fn add_completed_job<R: rsmq_async::RsmqConnection + Clone + Send>(
         .ok()
         .flatten()
         .flatten();
-    let mut tx: QueueTransaction<'_, R> = (rsmq, db.begin().await?).into();
+    let mut tx = db.begin().await?;
     let job_id = queued_job.id.clone();
     sqlx::query!(
         "INSERT INTO completed_job AS cj
@@ -126,10 +122,9 @@ pub async fn add_completed_job<R: rsmq_async::RsmqConnection + Clone + Send>(
                    , email
                    , visible_to_owner
                    , mem_peak
-                   , tag
                 )
             VALUES ($1, $2, $3, $4, $5, $6, COALESCE($26, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE($6, now()))))*1000), $7, $8, $9,\
-                    $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $27, $28, $29, $30)
+                    $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $27, $28, $29)
          ON CONFLICT (id) DO UPDATE SET success = $7, result = $11, logs = concat(cj.logs, $12)",
         queued_job.workspace_id,
         queued_job.id,
@@ -159,13 +154,14 @@ pub async fn add_completed_job<R: rsmq_async::RsmqConnection + Clone + Send>(
         duration: Option<i64>,
         queued_job.email,
         queued_job.visible_to_owner,
-        mem_peak,
-        queued_job.tag,
+        mem_peak
     )
     .execute(&mut tx)
     .await
     .map_err(|e| Error::InternalErr(format!("Could not add completed job {job_id}: {e}")))?;
-    tx = delete_job(tx, &queued_job.workspace_id, job_id).await?;
+    println!("{:?}", SystemTime::now());
+
+    let _ = delete_job(db, &queued_job.workspace_id, job_id).await?;
     if !queued_job.is_flow_step
         && queued_job.job_kind != JobKind::Flow
         && queued_job.job_kind != JobKind::FlowPreview
@@ -220,24 +216,21 @@ pub async fn add_completed_job<R: rsmq_async::RsmqConnection + Clone + Send>(
 }
 
 #[instrument(level = "trace", skip_all)]
-pub async fn schedule_again_if_scheduled<'c, R: rsmq_async::RsmqConnection + Clone + Send + 'c>(
-    mut tx: QueueTransaction<'c, R>,
+pub async fn schedule_again_if_scheduled<'c>(
+    mut tx: Transaction<'c, Postgres>,
     db: &Pool<Postgres>,
     schedule_path: &str,
     script_path: &str,
     w_id: &str,
-) -> windmill_common::error::Result<QueueTransaction<'c, R>> {
-    let schedule = get_schedule_opt(tx.transaction_mut(), w_id, schedule_path).await?;
-
-    if schedule.is_none() {
-        tracing::error!(
-            "Schedule {schedule_path} in {w_id} not found. Impossible to schedule again"
-        );
-        return Ok(tx);
-    }
-
-    let schedule = schedule.unwrap();
-
+) -> windmill_common::error::Result<Transaction<'c, Postgres>> {
+    let schedule = get_schedule_opt(&mut tx, w_id, schedule_path)
+        .await?
+        .ok_or_else(|| {
+            Error::InternalErr(format!(
+                "Could not find schedule {:?} for workspace {}",
+                schedule_path, w_id
+            ))
+        })?;
     if schedule.enabled && script_path == schedule.script_path {
         let res = windmill_queue::schedule::push_scheduled_job(
             tx,

backend/windmill-worker/src/js_eval.rs

@@ -252,7 +252,7 @@ async function resource(path) {{
             .join(""),
     );
     tracing::debug!("{}", code);
-    let global = context.execute_script("<anon>", code.into())?;
+    let global = context.execute_script("<anon>", code)?;
     let global = context.resolve_value(global).await?;
 
     let scope = &mut context.handle_scope();

backend/windmill-worker/src/lib.rs

@@ -1,9 +1,5 @@
-mod common;
-mod global_cache;
-mod go_executor;
 mod jobs;
 mod js_eval;
-mod python_executor;
 mod worker;
 mod worker_flow;
 

backend/windmill-worker/src/main2.rs

@@ -0,0 +1,92 @@
+/*
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
+ * This file and its contents are licensed under the AGPLv3 License.
+ * Please see the included NOTICE for copyright information and
+ * LICENSE-AGPL for a copy of the license.
+ */
+
+// use std::{net::SocketAddr, time::Duration};
+
+// use anyhow::Context;
+// use sqlx::{postgres::PgPoolOptions, Pool, Postgres};
+// use windmill_common::{
+//     error::{self, Error},
+//     utils::rd_string,
+// };
+
+// #[tokio::main]
+// async fn main() -> anyhow::Result<()> {
+//     // dotenv().ok();
+
+//     windmill_common::tracing_init::initialize_tracing();
+
+//     let db = async {
+//         let database_url = std::env::var("DATABASE_URL")
+//             .map_err(|_| Error::BadConfig("DATABASE_URL env var is missing".to_string()))?;
+
+//         let max_connections = match std::env::var("DATABASE_CONNECTIONS") {
+//             Ok(n) => n.parse::<u32>().context("invalid DATABASE_CONNECTIONS")?,
+//             Err(_) => 10,
+//         };
+
+//         Ok::<Pool<Postgres>, error::Error>(
+//             PgPoolOptions::new()
+//                 .max_connections(max_connections)
+//                 .max_lifetime(Duration::from_secs(30 * 60)) // 30 mins
+//                 .connect(&database_url)
+//                 .await
+//                 .map_err(|err| Error::ConnectingToDatabase(err.to_string()))?,
+//         )
+//     }
+//     .await?;
+
+//     let metrics_addr: Option<SocketAddr> = std::env::var("METRICS_ADDR")
+//         .ok()
+//         .map(|s| {
+//             s.parse::<bool>()
+//                 .map(|b| b.then(|| SocketAddr::from(([0, 0, 0, 0], 8001))))
+//                 .or_else(|_| s.parse::<SocketAddr>().map(Some))
+//         })
+//         .transpose()?
+//         .flatten();
+
+//     let (tx, rx) = tokio::sync::broadcast::channel::<()>(3);
+//     let shutdown_signal = windmill_common::shutdown_signal(tx);
+
+//     let workers_f = async {
+//         let instance_name = rd_string(5);
+
+//         let ip = windmill_common::external_ip::get_ip()
+//             .await
+//             .unwrap_or_else(|e| {
+//                 tracing::warn!(error = e.to_string(), "failed to get external IP");
+//                 "unretrievable IP".to_string()
+//             });
+//         let worker_name = format!("dt-worker-{}-{}", &instance_name, rd_string(5));
+//         windmill_worker::run_worker(
+//             &db.clone(),
+//             &instance_name,
+//             worker_name,
+//             1,
+//             1,
+//             &ip,
+//             rx.resubscribe(),
+//         )
+//         .await;
+//         Ok(()) as anyhow::Result<()>
+//     };
+
+//     let metrics_f = async {
+//         match metrics_addr {
+//             Some(addr) => windmill_common::serve_metrics(addr, rx.resubscribe())
+//                 .await
+//                 .map_err(anyhow::Error::from),
+//             None => Ok(()),
+//         }
+//     };
+
+//     futures::try_join!(shutdown_signal, workers_f, metrics_f)?;
+
+//     Ok(())
+// }

backend/windmill-worker/src/python_executor.rs

@@ -1,573 +0,0 @@
-use std::process::Stdio;
-
-use itertools::Itertools;
-use regex::Regex;
-use sqlx::{Pool, Postgres};
-use tokio::{
-    fs::{metadata, DirBuilder, File},
-    io::AsyncReadExt,
-    process::Command,
-};
-use uuid::Uuid;
-use windmill_common::{
-    error::{self, Error},
-    jobs::QueuedJob,
-    utils::calculate_hash,
-};
-
-lazy_static::lazy_static! {
-    static ref PYTHON_PATH: String =
-    std::env::var("PYTHON_PATH").unwrap_or_else(|_| "/usr/local/bin/python3".to_string());
-
-    static ref PIP_INDEX_URL: Option<String> = std::env::var("PIP_INDEX_URL").ok();
-    static ref PIP_EXTRA_INDEX_URL: Option<String> = std::env::var("PIP_EXTRA_INDEX_URL").ok();
-    static ref PIP_TRUSTED_HOST: Option<String> = std::env::var("PIP_TRUSTED_HOST").ok();
-    static ref PIP_LOCAL_DEPENDENCIES: Option<Vec<String>> = {
-        let pip_local_dependencies = std::env::var("PIP_LOCAL_DEPENDENCIES")
-            .ok()
-            .map(|x| x.split(',').map(|x| x.to_string()).collect());
-        if pip_local_dependencies == Some(vec!["".to_string()]) {
-            None
-        } else {
-            pip_local_dependencies
-        }
-    };
-
-    static ref ADDITIONAL_PYTHON_PATHS: Option<Vec<String>> = std::env::var("ADDITIONAL_PYTHON_PATHS")
-        .ok()
-        .map(|x| x.split(':').map(|x| x.to_string()).collect());
-
-    static ref RELATIVE_IMPORT_REGEX: Regex = Regex::new(r#"(import|from)\s(((u|f)\.)|\.)"#).unwrap();
-
-}
-
-const NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT: &str = include_str!("../nsjail/download.py.config.proto");
-const NSJAIL_CONFIG_RUN_PYTHON3_CONTENT: &str = include_str!("../nsjail/run.python3.config.proto");
-const RELATIVE_PYTHON_LOADER: &str = include_str!("../loader.py");
-
-#[cfg(feature = "enterprise")]
-use crate::global_cache::{build_tar_and_push, pull_from_tar};
-
-use crate::{
-    common::{read_result, set_logs},
-    create_args_and_out_file, get_reserved_variables, handle_child, write_file,
-    AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, NSJAIL_PATH, PATH_ENV,
-    PIP_CACHE_DIR, S3_CACHE_BUCKET,
-};
-
-pub async fn create_dependencies_dir(job_dir: &str) {
-    DirBuilder::new()
-        .recursive(true)
-        .create(&format!("{job_dir}/dependencies"))
-        .await
-        .expect("could not create dependencies dir");
-}
-
-pub async fn pip_compile(
-    job_id: &Uuid,
-    requirements: &str,
-    logs: &mut String,
-    job_dir: &str,
-    db: &Pool<Postgres>,
-    worker_name: &str,
-    w_id: &str,
-) -> error::Result<String> {
-    logs.push_str(&format!("\nresolving dependencies..."));
-    set_logs(logs, job_id, db).await;
-    logs.push_str(&format!("\ncontent of requirements:\n{}", requirements));
-    let req_hash = calculate_hash(&requirements);
-    if let Some(cached) = sqlx::query_scalar!(
-        "SELECT lockfile FROM pip_resolution_cache WHERE hash = $1",
-        req_hash
-    )
-    .fetch_optional(db)
-    .await?
-    {
-        logs.push_str(&format!("\nfound cached resolution"));
-        return Ok(cached);
-    }
-    let file = "requirements.in";
-    let requirements = if let Some(pip_local_dependencies) = PIP_LOCAL_DEPENDENCIES.as_ref() {
-        let deps = pip_local_dependencies.clone();
-        requirements
-            .lines()
-            .filter(|s| !deps.contains(&s.to_string()))
-            .join("\n")
-    } else {
-        requirements.to_string()
-    };
-    write_file(job_dir, file, &requirements).await?;
-
-    let mut args = vec!["-q", "--no-header", file, "--resolver=backtracking"];
-    if let Some(url) = PIP_EXTRA_INDEX_URL.as_ref() {
-        args.extend(["--extra-index-url", url]);
-    }
-    if let Some(url) = PIP_INDEX_URL.as_ref() {
-        args.extend(["--index-url", url]);
-    }
-    if let Some(host) = PIP_TRUSTED_HOST.as_ref() {
-        args.extend(["--trusted-host", host]);
-    }
-    let child = Command::new("pip-compile")
-        .current_dir(job_dir)
-        .args(args)
-        .stdout(Stdio::piped())
-        .stderr(Stdio::piped())
-        .spawn()?;
-    handle_child(
-        job_id,
-        db,
-        logs,
-        child,
-        false,
-        worker_name,
-        &w_id,
-        "pip-compile",
-    )
-    .await
-    .map_err(|e| Error::ExecutionErr(format!("Lock file generation failed: {e:?}")))?;
-    let path_lock = format!("{job_dir}/requirements.txt");
-    let mut file = File::open(path_lock).await?;
-    let mut req_content = "".to_string();
-    file.read_to_string(&mut req_content).await?;
-    let lockfile = req_content
-        .lines()
-        .filter(|x| !x.trim_start().starts_with('#'))
-        .map(|x| x.to_string())
-        .collect::<Vec<String>>()
-        .join("\n");
-    sqlx::query!(
-        "INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() + ('3 days')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = $2",
-        req_hash,
-        lockfile
-    ).fetch_optional(db).await?;
-    Ok(lockfile)
-}
-
-#[tracing::instrument(level = "trace", skip_all)]
-pub async fn handle_python_job(
-    requirements_o: Option<String>,
-    job_dir: &str,
-    worker_dir: &str,
-    worker_name: &str,
-    job: &QueuedJob,
-    logs: &mut String,
-    db: &sqlx::Pool<sqlx::Postgres>,
-    client: &AuthedClientBackgroundTask,
-    inner_content: &String,
-    shared_mount: &str,
-    base_internal_url: &str,
-) -> windmill_common::error::Result<serde_json::Value> {
-    create_dependencies_dir(job_dir).await;
-
-    let mut additional_python_paths: Vec<String> =
-        ADDITIONAL_PYTHON_PATHS.to_owned().unwrap_or_else(|| vec![]);
-
-    let requirements = match requirements_o {
-        Some(r) => r,
-        None => {
-            let requirements = windmill_parser_py::parse_python_imports(&inner_content)?.join("\n");
-            if requirements.is_empty() {
-                "".to_string()
-            } else {
-                pip_compile(
-                    &job.id,
-                    &requirements,
-                    logs,
-                    job_dir,
-                    db,
-                    worker_name,
-                    &job.workspace_id,
-                )
-                .await
-                .map_err(|e| {
-                    Error::ExecutionErr(format!("pip compile failed: {}", e.to_string()))
-                })?
-            }
-        }
-    };
-
-    if requirements.len() > 0 {
-        additional_python_paths = handle_python_reqs(
-            requirements
-                .split("\n")
-                .filter(|x| !x.starts_with("--"))
-                .collect(),
-            &job.id,
-            &job.workspace_id,
-            logs,
-            db,
-            worker_name,
-            job_dir,
-            worker_dir,
-        )
-        .await?;
-    }
-    logs.push_str("\n\n--- PYTHON CODE EXECUTION ---\n");
-
-    set_logs(logs, &job.id, db).await;
-
-    let relative_imports = RELATIVE_IMPORT_REGEX.is_match(&inner_content);
-
-    let script_path_splitted = &job.script_path().split("/");
-    let dirs_full = script_path_splitted
-        .clone()
-        .take(script_path_splitted.clone().count() - 1)
-        .join("/")
-        .replace("-", "_");
-    let dirs = if dirs_full.len() > 0 {
-        dirs_full
-    } else {
-        "tmp".to_string()
-    };
-    let last = script_path_splitted
-        .clone()
-        .last()
-        .unwrap()
-        .replace("-", "_")
-        .replace(" ", "_")
-        .to_lowercase();
-    let module_dir = format!("{}/{}", job_dir, dirs);
-    tokio::fs::create_dir_all(format!("{module_dir}/")).await?;
-    let _ = write_file(&module_dir, &format!("{last}.py"), inner_content).await?;
-    if relative_imports {
-        let _ = write_file(&job_dir, "loader.py", RELATIVE_PYTHON_LOADER).await?;
-    }
-
-    let sig = windmill_parser_py::parse_python_signature(inner_content)?;
-    let transforms = sig
-        .args
-        .iter()
-        .map(|x| match x.typ {
-            windmill_parser::Typ::Bytes => {
-                format!(
-                    "if \"{}\" in kwargs and kwargs[\"{}\"] is not None:\n    \
-                                     kwargs[\"{}\"] = base64.b64decode(kwargs[\"{}\"])\n",
-                    x.name, x.name, x.name, x.name
-                )
-            }
-            windmill_parser::Typ::Datetime => {
-                format!(
-                    "if \"{}\" in kwargs and kwargs[\"{}\"] is not None:\n    \
-                                     kwargs[\"{}\"] = datetime.strptime(kwargs[\"{}\"], \
-                                     '%Y-%m-%dT%H:%M')\n",
-                    x.name, x.name, x.name, x.name
-                )
-            }
-            _ => "".to_string(),
-        })
-        .collect::<Vec<String>>()
-        .join("");
-    let client = client.get_authed().await;
-    create_args_and_out_file(&client, job, job_dir).await?;
-
-    let import_loader = if relative_imports {
-        "import loader"
-    } else {
-        ""
-    };
-    let import_base64 = if sig
-        .args
-        .iter()
-        .any(|x| x.typ == windmill_parser::Typ::Bytes)
-    {
-        "import base64"
-    } else {
-        ""
-    };
-    let import_datetime = if sig
-        .args
-        .iter()
-        .any(|x| x.typ == windmill_parser::Typ::Datetime)
-    {
-        "from datetime import datetime"
-    } else {
-        ""
-    };
-    let spread = if sig.star_kwargs {
-        "args = kwargs".to_string()
-    } else {
-        sig.args
-            .into_iter()
-            .map(|x| format!("args[\"{}\"] = kwargs.get(\"{}\")", x.name, x.name))
-            .join("\n")
-    };
-
-    let module_dir_dot = dirs.replace("/", ".").replace("-", "_");
-    let wrapper_content: String = format!(
-        r#"
-import json
-{import_loader}
-{import_base64}
-{import_datetime}
-import traceback
-import sys
-from {module_dir_dot} import {last} as inner_script
-
-
-with open("args.json") as f:
-    kwargs = json.load(f, strict=False)
-args = {{}}
-{spread}
-{transforms}
-for k, v in list(args.items()):
-    if v == '<function call>':
-        del args[k]
-
-try:
-    res = inner_script.main(**args)
-    typ = type(res)
-    if typ.__name__ == 'DataFrame':
-        if typ.__module__ == 'pandas.core.frame':
-            res = res.values.tolist()
-        elif typ.__module__ == 'polars.dataframe.frame':
-            res = res.rows()
-    res_json = json.dumps(res, separators=(',', ':'), default=str).replace('\n', '')
-    with open("result.json", 'w') as f:
-        f.write(res_json)
-except Exception as e:
-    exc_type, exc_value, exc_traceback = sys.exc_info()
-    tb = traceback.format_tb(exc_traceback)
-    with open("result.json", 'w') as f:
-        err_json = json.dumps({{ "message": str(e), "name": e.__class__.__name__, "stack": '\n'.join(tb[1:])  }}, separators=(',', ':'), default=str).replace('\n', '')
-        f.write(err_json)
-        sys.exit(1)
-"#,
-    );
-    write_file(job_dir, "wrapper.py", &wrapper_content).await?;
-
-    let mut reserved_variables = get_reserved_variables(job, &client.token, db).await?;
-    let additional_python_paths_folders = additional_python_paths.iter().join(":");
-    if !*DISABLE_NSJAIL {
-        let shared_deps = additional_python_paths
-            .into_iter()
-            .map(|pp| {
-                format!(
-                    r#"
-mount {{
-    src: "{pp}"
-    dst: "{pp}"
-    is_bind: true
-    rw: false
-}}
-        "#
-                )
-            })
-            .join("\n");
-        let _ = write_file(
-            job_dir,
-            "run.config.proto",
-            &NSJAIL_CONFIG_RUN_PYTHON3_CONTENT
-                .replace("{JOB_DIR}", job_dir)
-                .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string())
-                .replace("{SHARED_MOUNT}", shared_mount)
-                .replace("{SHARED_DEPENDENCIES}", shared_deps.as_str())
-                .replace("{MAIN}", format!("{dirs}/{last}").as_str())
-                .replace(
-                    "{ADDITIONAL_PYTHON_PATHS}",
-                    additional_python_paths_folders.as_str(),
-                ),
-        )
-        .await?;
-    } else {
-        reserved_variables.insert("PYTHONPATH".to_string(), additional_python_paths_folders);
-    }
-
-    tracing::info!(
-        worker_name = %worker_name,
-        job_id = %job.id,
-        workspace_id = %job.workspace_id,
-        "started python code execution {}",
-        job.id
-    );
-    let child = if !*DISABLE_NSJAIL {
-        Command::new(NSJAIL_PATH.as_str())
-            .current_dir(job_dir)
-            .env_clear()
-            // inject PYTHONPATH here - for some reason I had to do it in nsjail conf
-            .envs(reserved_variables)
-            .env("PATH", PATH_ENV.as_str())
-            .env("BASE_INTERNAL_URL", base_internal_url)
-            .args(vec![
-                "--config",
-                "run.config.proto",
-                "--",
-                PYTHON_PATH.as_str(),
-                "-u",
-                "-m",
-                "wrapper",
-            ])
-            .stdout(Stdio::piped())
-            .stderr(Stdio::piped())
-            .spawn()?
-    } else {
-        Command::new(PYTHON_PATH.as_str())
-            .current_dir(job_dir)
-            .env_clear()
-            .envs(reserved_variables)
-            .env("PATH", PATH_ENV.as_str())
-            .env("BASE_INTERNAL_URL", base_internal_url)
-            .args(vec!["-u", "-m", "wrapper"])
-            .stdout(Stdio::piped())
-            .stderr(Stdio::piped())
-            .spawn()?
-    };
-
-    handle_child(
-        &job.id,
-        db,
-        logs,
-        child,
-        !*DISABLE_NSJAIL,
-        worker_name,
-        &job.workspace_id,
-        "python run",
-    )
-    .await?;
-    read_result(job_dir).await
-}
-
-pub async fn handle_python_reqs(
-    requirements: Vec<&str>,
-    job_id: &Uuid,
-    w_id: &str,
-    logs: &mut String,
-    db: &sqlx::Pool<sqlx::Postgres>,
-    worker_name: &str,
-    job_dir: &str,
-    worker_dir: &str,
-) -> error::Result<Vec<String>> {
-    let mut req_paths: Vec<String> = vec![];
-    let mut vars = vec![("PATH", PATH_ENV.as_str())];
-    if !*DISABLE_NSJAIL {
-        if let Some(url) = PIP_EXTRA_INDEX_URL.as_ref() {
-            vars.push(("EXTRA_INDEX_URL", url));
-        }
-        if let Some(url) = PIP_INDEX_URL.as_ref() {
-            vars.push(("INDEX_URL", url));
-        }
-        if let Some(host) = PIP_TRUSTED_HOST.as_ref() {
-            vars.push(("TRUSTED_HOST", host));
-        }
-        let _ = write_file(
-            job_dir,
-            "download.config.proto",
-            &NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT
-                .replace("{WORKER_DIR}", &worker_dir)
-                .replace("{CACHE_DIR}", PIP_CACHE_DIR)
-                .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()),
-        )
-        .await?;
-    };
-
-    for req in requirements {
-        // todo: handle many reqs
-        let venv_p = format!("{PIP_CACHE_DIR}/{req}");
-        if metadata(&venv_p).await.is_ok() {
-            req_paths.push(venv_p);
-            continue;
-        }
-
-        #[cfg(feature = "enterprise")]
-        if let Some(ref bucket) = *S3_CACHE_BUCKET {
-            sqlx::query_scalar!("UPDATE queue SET last_ping = now() WHERE id = $1", job_id)
-                .execute(db)
-                .await?;
-            if pull_from_tar(bucket, venv_p.clone()).await.is_ok() {
-                req_paths.push(venv_p.clone());
-                continue;
-            }
-        }
-
-        logs.push_str("\n--- PIP INSTALL ---\n");
-        logs.push_str(&format!("\n{req} is being installed for the first time.\n It will be cached for all ulterior uses."));
-
-        tracing::info!(
-            worker_name = %worker_name,
-            job_id = %job_id,
-            workspace_id = %w_id,
-            "started setup python dependencies"
-        );
-
-        let child = if !*DISABLE_NSJAIL {
-            tracing::info!(
-                worker_name = %worker_name,
-                job_id = %job_id,
-                workspace_id = %w_id,
-                "starting nsjail"
-            );
-            let mut vars = vars.clone();
-            let req = req.to_string();
-            vars.push(("REQ", &req));
-            vars.push(("TARGET", &venv_p));
-            Command::new(NSJAIL_PATH.as_str())
-                .current_dir(job_dir)
-                .env_clear()
-                .envs(vars)
-                .args(vec!["--config", "download.config.proto"])
-                .stdout(Stdio::piped())
-                .stderr(Stdio::piped())
-                .spawn()?
-        } else {
-            let mut args = vec![
-                "-m",
-                "pip",
-                "install",
-                &req,
-                "-I",
-                "--no-deps",
-                "--no-color",
-                "--isolated",
-                "--no-warn-conflicts",
-                "--disable-pip-version-check",
-                "-t",
-                venv_p.as_str(),
-            ];
-            if let Some(url) = PIP_EXTRA_INDEX_URL.as_ref() {
-                args.extend(["--extra-index-url", url]);
-            }
-            if let Some(url) = PIP_INDEX_URL.as_ref() {
-                args.extend(["--index-url", url]);
-            }
-            if let Some(host) = PIP_TRUSTED_HOST.as_ref() {
-                args.extend(["--trusted-host", &host]);
-            }
-            Command::new(PYTHON_PATH.as_str())
-                .env_clear()
-                .env("PATH", PATH_ENV.as_str())
-                .args(args)
-                .stdout(Stdio::piped())
-                .stderr(Stdio::piped())
-                .spawn()?
-        };
-
-        let child = handle_child(
-            &job_id,
-            db,
-            logs,
-            child,
-            false,
-            worker_name,
-            &w_id,
-            &format!("pip install {req}"),
-        )
-        .await;
-        tracing::info!(
-            worker_name = %worker_name,
-            job_id = %job_id,
-            workspace_id = %w_id,
-            is_ok = child.is_ok(),
-            "finished setting up python dependencies {}",
-            job_id
-        );
-        child?;
-
-        #[cfg(feature = "enterprise")]
-        if let Some(ref bucket) = *S3_CACHE_BUCKET {
-            let venv_p = venv_p.clone();
-            tokio::spawn(build_tar_and_push(bucket, venv_p));
-        }
-        req_paths.push(venv_p);
-    }
-    Ok(req_paths)
-}

backend/windmill-worker/src/worker_flow.rs

@@ -19,9 +19,6 @@ use tokio::sync::mpsc::Sender;
 use tracing::instrument;
 use uuid::Uuid;
 use windmill_common::flow_status::{FlowStatusModuleWParent, Iterator, JobResult};
-use windmill_common::jobs::{
-    script_hash_to_tag, script_path_to_payload, JobPayload, QueuedJob, RawCode,
-};
 use windmill_common::{
     error::{self, to_anyhow, Error},
     flow_status::{
@@ -33,12 +30,12 @@ use windmill_common::{
 
 type DB = sqlx::Pool<sqlx::Postgres>;
 
-use windmill_queue::{canceled_job_to_result, get_queued_job, push, QueueTransaction};
+use windmill_queue::{
+    canceled_job_to_result, get_queued_job, push, JobPayload, QueuedJob, RawCode,
+};
 
 // #[instrument(level = "trace", skip_all)]
-pub async fn update_flow_status_after_job_completion<
-    R: rsmq_async::RsmqConnection + Send + Sync + Clone,
->(
+pub async fn update_flow_status_after_job_completion(
     db: &DB,
     client: &AuthedClient,
     flow: uuid::Uuid,
@@ -52,7 +49,6 @@ pub async fn update_flow_status_after_job_completion<
     worker_dir: &str,
     stop_early_override: Option<bool>,
     base_internal_url: &str,
-    rsmq: Option<R>,
 ) -> error::Result<()> {
     // this is manual tailrecursion because async_recursion blows up the stack
     let mut depth = 0;
@@ -71,7 +67,6 @@ pub async fn update_flow_status_after_job_completion<
         stop_early_override,
         base_internal_url,
         depth,
-        rsmq.clone(),
     )
     .await?;
     while let Some(nrec) = rec {
@@ -91,7 +86,6 @@ pub async fn update_flow_status_after_job_completion<
             nrec.stop_early_override,
             base_internal_url,
             depth,
-            rsmq.clone(),
         )
         .await?;
     }
@@ -105,9 +99,7 @@ pub struct RecUpdateFlowStatusAfterJobCompletion {
     stop_early_override: Option<bool>,
 }
 // #[instrument(level = "trace", skip_all)]
-pub async fn update_flow_status_after_job_completion_internal<
-    R: rsmq_async::RsmqConnection + Send + Sync + Clone,
->(
+pub async fn update_flow_status_after_job_completion_internal(
     db: &DB,
     client: &AuthedClient,
     flow: uuid::Uuid,
@@ -122,12 +114,11 @@ pub async fn update_flow_status_after_job_completion_internal<
     stop_early_override: Option<bool>,
     base_internal_url: &str,
     depth: u8,
-    rsmq: Option<R>,
 ) -> error::Result<Option<RecUpdateFlowStatusAfterJobCompletion>> {
     let (should_continue_flow, flow_job, stop_early, skip_if_stop_early, nresult) = {
         tracing::debug!("UPDATE FLOW STATUS: {flow:?} {success} {result:?} {w_id} {depth}");
 
-        let mut tx: QueueTransaction<'_, _> = (rsmq.clone(), db.begin().await?).into();
+        let mut tx = db.begin().await?;
 
         let old_status_json = sqlx::query_scalar!(
             "SELECT flow_status FROM queue WHERE id = $1 AND workspace_id = $2",
@@ -163,7 +154,7 @@ pub async fn update_flow_status_after_job_completion_internal<
             module_status,
             FlowStatusModule::InProgress { iterator: Some(_), .. }
         ) {
-            compute_skip_loop_failures(flow, old_status.step, tx.transaction_mut())
+            compute_skip_loop_failures(flow, old_status.step, &mut tx)
                 .await?
                 .unwrap_or(false)
         } else {
@@ -206,11 +197,9 @@ pub async fn update_flow_status_after_job_completion_internal<
             FlowStatusModule::InProgress {
                 branchall: Some(BranchAllStatus { branch, .. }),
                 ..
-            } => {
-                compute_skip_branchall_failure(flow, old_status.step, *branch, tx.transaction_mut())
-                    .await?
-                    .unwrap_or(false)
-            }
+            } => compute_skip_branchall_failure(flow, old_status.step, *branch, &mut tx)
+                .await?
+                .unwrap_or(false),
             _ => false,
         };
 
@@ -472,7 +461,7 @@ pub async fn update_flow_status_after_job_completion_internal<
             .context("remove flow status retry")?;
         }
 
-        let flow_job = get_queued_job(flow, w_id, tx.transaction_mut())
+        let flow_job = get_queued_job(flow, w_id, &mut tx)
             .await?
             .ok_or_else(|| Error::InternalErr(format!("requiring flow to be in the queue")))?;
 
@@ -514,9 +503,7 @@ pub async fn update_flow_status_after_job_completion_internal<
             {
                 true
             }
-            false if has_failure_module(flow, tx.transaction_mut()).await? && !is_failure_step => {
-                true
-            }
+            false if has_failure_module(flow, &mut tx).await? && !is_failure_step => true,
             false => false,
         };
 
@@ -561,7 +548,6 @@ pub async fn update_flow_status_after_job_completion_internal<
                 logs,
                 canceled_job_to_result(&flow_job),
                 metrics.clone(),
-                rsmq.clone(),
             )
             .await?;
         } else {
@@ -572,7 +558,6 @@ pub async fn update_flow_status_after_job_completion_internal<
                 stop_early && skip_if_stop_early,
                 nresult.clone(),
                 logs,
-                rsmq.clone(),
             )
             .await?;
         }
@@ -586,7 +571,6 @@ pub async fn update_flow_status_after_job_completion_internal<
             same_worker_tx.clone(),
             worker_dir,
             base_internal_url,
-            rsmq.clone(),
         )
         .await
         {
@@ -597,7 +581,6 @@ pub async fn update_flow_status_after_job_completion_internal<
                     "Unexpected error during flow chaining:\n".to_string(),
                     json!({"message": err.to_string(), "name": "InternalError"}),
                     metrics.clone(),
-                    rsmq.clone(),
                 )
                 .await;
                 true
@@ -848,7 +831,7 @@ async fn transform_input(
 }
 
 #[instrument(level = "trace", skip_all)]
-pub async fn handle_flow<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
+pub async fn handle_flow(
     flow_job: &QueuedJob,
     db: &sqlx::Pool<sqlx::Postgres>,
     client: &AuthedClient,
@@ -856,7 +839,6 @@ pub async fn handle_flow<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
     same_worker_tx: Sender<Uuid>,
     worker_dir: &str,
     base_internal_url: &str,
-    rsmq: Option<R>,
 ) -> anyhow::Result<()> {
     let value = flow_job
         .raw_flow
@@ -880,7 +862,6 @@ pub async fn handle_flow<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
         same_worker_tx,
         worker_dir,
         base_internal_url,
-        rsmq,
     )
     .await?;
     Ok(())
@@ -888,7 +869,7 @@ pub async fn handle_flow<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
 
 #[async_recursion]
 // #[instrument(level = "trace", skip_all)]
-async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
+async fn push_next_flow_job(
     flow_job: &QueuedJob,
     mut status: FlowStatus,
     flow: FlowValue,
@@ -898,7 +879,6 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
     same_worker_tx: Sender<Uuid>,
     worker_dir: &str,
     base_internal_url: &str,
-    rsmq: Option<R>,
 ) -> error::Result<()> {
     let job_root = flow_job
         .root_job
@@ -935,7 +915,6 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
             worker_dir,
             None,
             base_internal_url,
-            rsmq,
         )
         .await;
     }
@@ -1106,7 +1085,7 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
                 let logs = "Timed out waiting to be resumed".to_string();
                 let result = json!({ "error": {"message": logs, "name": "SuspendedTimeout"}});
                 let _uuid =
-                    add_completed_job(db, &flow_job, success, skipped, result, logs, rsmq).await?;
+                    add_completed_job(db, &flow_job, success, skipped, result, logs).await?;
 
                 return Ok(());
             }
@@ -1307,7 +1286,7 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
         flow.same_worker && module.suspend.is_none() && module.sleep.is_none();
 
     /* Finally, push the job into the queue */
-    let mut tx = (rsmq.clone(), db.begin().await?).into();
+    let mut tx = db.begin().await?;
     let mut uuids = vec![];
 
     let len = match &job_payloads {
@@ -1316,7 +1295,7 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
         ContinuePayload::ForloopJobs { n, .. } => *n,
     };
     for i in (0..len).into_iter() {
-        let payload_tag = match &job_payloads {
+        let payload = match &job_payloads {
             ContinuePayload::SingleJob(payload) => payload.clone(),
             ContinuePayload::BranchAllJobs(payloads) => payloads[i].clone(),
             ContinuePayload::ForloopJobs { modules, .. } => {
@@ -1325,16 +1304,13 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
                     failure_module.id_append(&format!("{}/{}", status.step, i));
                     fm = Some(failure_module);
                 }
-                JobPayloadWithTag {
-                    payload: JobPayload::RawFlow {
-                        value: FlowValue {
-                            modules: (*modules).clone(),
-                            failure_module: fm.clone(),
-                            same_worker: flow.same_worker,
-                        },
-                        path: Some(format!("{}/loop-{}", flow_job.script_path(), i)),
+                JobPayload::RawFlow {
+                    value: FlowValue {
+                        modules: (*modules).clone(),
+                        failure_module: fm.clone(),
+                        same_worker: flow.same_worker,
                     },
-                    tag: None,
+                    path: Some(format!("{}/loop-{}", flow_job.script_path(), i)),
                 }
             }
         };
@@ -1374,11 +1350,10 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
         } else {
             flow_job.root_job.or_else(|| Some(flow_job.id))
         };
-
         let (uuid, inner_tx) = push(
             tx,
             &flow_job.workspace_id,
-            payload_tag.payload,
+            payload,
             ok.unwrap_or_else(|| Map::new()),
             &flow_job.created_by,
             &flow_job.email,
@@ -1391,7 +1366,6 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
             continue_on_same_worker,
             err,
             flow_job.visible_to_owner,
-            payload_tag.tag,
         )
         .await?;
         tx = inner_tx;
@@ -1478,7 +1452,7 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
             json!(i),
             flow_job.id
         )
-        .execute(db)
+        .execute(&mut tx)
         .await?;
     } else {
         sqlx::query!(
@@ -1494,7 +1468,7 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
             json!(i),
             flow_job.id
         )
-        .execute(db)
+        .execute(&mut tx)
         .await?;
     };
 
@@ -1610,15 +1584,10 @@ enum NextStatus {
     },
 }
 
-#[derive(Clone)]
-struct JobPayloadWithTag {
-    payload: JobPayload,
-    tag: Option<String>,
-}
 enum ContinuePayload {
-    SingleJob(JobPayloadWithTag),
+    SingleJob(JobPayload),
     ForloopJobs { n: usize, modules: Vec<FlowModule> },
-    BranchAllJobs(Vec<JobPayloadWithTag>),
+    BranchAllJobs(Vec<JobPayload>),
 }
 
 enum NextFlowTransform {
@@ -1626,6 +1595,22 @@ enum NextFlowTransform {
     Continue(ContinuePayload, NextStatus),
 }
 
+// a similar function exists on the backend
+// TODO: rewrite this to use an endpoint in the backend directly, instead of checking for hub itself, and then using the API
+async fn script_path_to_payload<'c>(
+    script_path: &str,
+    db: &mut sqlx::Transaction<'c, sqlx::Postgres>,
+    w_id: &String,
+) -> Result<JobPayload, Error> {
+    let job_payload = if script_path.starts_with("hub/") {
+        JobPayload::ScriptHub { path: script_path.to_owned() }
+    } else {
+        let script_hash = windmill_common::get_latest_hash_for_path(db, w_id, script_path).await?;
+        JobPayload::ScriptHash { hash: script_hash, path: script_path.to_owned() }
+    };
+    Ok(job_payload)
+}
+
 async fn compute_next_flow_transform<'c>(
     flow_job: &QueuedJob,
     flow: &FlowValue,
@@ -1642,10 +1627,7 @@ async fn compute_next_flow_transform<'c>(
         FlowModuleValue::Identity => Ok((
             tx,
             NextFlowTransform::Continue(
-                ContinuePayload::SingleJob(JobPayloadWithTag {
-                    payload: JobPayload::Identity,
-                    tag: None,
-                }),
+                ContinuePayload::SingleJob(JobPayload::Identity),
                 NextStatus::NextStep,
             ),
         )),
@@ -1654,46 +1636,41 @@ async fn compute_next_flow_transform<'c>(
             Ok((
                 tx,
                 NextFlowTransform::Continue(
-                    ContinuePayload::SingleJob(JobPayloadWithTag { payload, tag: None }),
+                    ContinuePayload::SingleJob(payload),
                     NextStatus::NextStep,
                 ),
             ))
         }
         FlowModuleValue::Script { path: script_path, hash: script_hash, .. } => {
-            let (payload, tag) = if script_hash.is_none() {
+            let payload = if script_hash.is_none() {
                 script_path_to_payload(script_path, &mut tx, &flow_job.workspace_id).await?
             } else {
-                let hash = script_hash.clone().unwrap();
-                let tag = script_hash_to_tag(&hash, &mut tx, &flow_job.workspace_id).await?;
-                (
-                    JobPayload::ScriptHash { hash, path: script_path.to_owned() },
-                    tag,
-                )
+                JobPayload::ScriptHash {
+                    hash: script_hash.clone().unwrap(),
+                    path: script_path.to_owned(),
+                }
             };
             Ok((
                 tx,
                 NextFlowTransform::Continue(
-                    ContinuePayload::SingleJob(JobPayloadWithTag { payload, tag }),
+                    ContinuePayload::SingleJob(payload),
                     NextStatus::NextStep,
                 ),
             ))
         }
-        FlowModuleValue::RawScript { path, content, language, lock, tag, .. } => {
+        FlowModuleValue::RawScript { path, content, language, lock, .. } => {
             let path = path
                 .clone()
                 .or_else(|| Some(format!("{}/step-{}", flow_job.script_path(), status.step)));
             Ok((
                 tx,
                 NextFlowTransform::Continue(
-                    ContinuePayload::SingleJob(JobPayloadWithTag {
-                        payload: JobPayload::Code(RawCode {
-                            path,
-                            content: content.clone(),
-                            language: language.clone(),
-                            lock: lock.clone(),
-                        }),
-                        tag: tag.clone(),
-                    }),
+                    ContinuePayload::SingleJob(JobPayload::Code(RawCode {
+                        path,
+                        content: content.clone(),
+                        language: language.clone(),
+                        lock: lock.clone(),
+                    })),
                     NextStatus::NextStep,
                 ),
             ))
@@ -1791,20 +1768,13 @@ async fn compute_next_flow_transform<'c>(
                     Ok((
                         tx,
                         NextFlowTransform::Continue(
-                            ContinuePayload::SingleJob(JobPayloadWithTag {
-                                payload: JobPayload::RawFlow {
-                                    value: FlowValue {
-                                        modules: (*modules).clone(),
-                                        failure_module: fm,
-                                        same_worker: flow.same_worker,
-                                    },
-                                    path: Some(format!(
-                                        "{}/loop-{}",
-                                        flow_job.script_path(),
-                                        ns.index
-                                    )),
+                            ContinuePayload::SingleJob(JobPayload::RawFlow {
+                                value: FlowValue {
+                                    modules: (*modules).clone(),
+                                    failure_module: fm,
+                                    same_worker: flow.same_worker,
                                 },
-                                tag: None,
+                                path: Some(format!("{}/loop-{}", flow_job.script_path(), ns.index)),
                             }),
                             NextStatus::NextLoopIteration(ns),
                         ),
@@ -1877,20 +1847,17 @@ async fn compute_next_flow_transform<'c>(
             Ok((
                 tx,
                 NextFlowTransform::Continue(
-                    ContinuePayload::SingleJob(JobPayloadWithTag {
-                        payload: JobPayload::RawFlow {
-                            value: FlowValue {
-                                modules,
-                                failure_module: fm,
-                                same_worker: flow.same_worker,
-                            },
-                            path: Some(format!(
-                                "{}/branchone-{}",
-                                flow_job.script_path(),
-                                status.step
-                            )),
+                    ContinuePayload::SingleJob(JobPayload::RawFlow {
+                        value: FlowValue {
+                            modules,
+                            failure_module: fm,
+                            same_worker: flow.same_worker,
                         },
-                        tag: None,
+                        path: Some(format!(
+                            "{}/branchone-{}",
+                            flow_job.script_path(),
+                            status.step
+                        )),
                     }),
                     NextStatus::BranchChosen(branch),
                 ),
@@ -1920,20 +1887,17 @@ async fn compute_next_flow_transform<'c>(
                                                     .id_append(&format!("{}/{i}", status.step));
                                                 fm = Some(failure_module);
                                             }
-                                            JobPayloadWithTag {
-                                                payload: JobPayload::RawFlow {
-                                                    value: FlowValue {
-                                                        modules: b.modules.clone(),
-                                                        failure_module: fm.clone(),
-                                                        same_worker: flow.same_worker,
-                                                    },
-                                                    path: Some(format!(
-                                                        "{}/branchall-{}",
-                                                        flow_job.script_path(),
-                                                        i
-                                                    )),
+                                            JobPayload::RawFlow {
+                                                value: FlowValue {
+                                                    modules: b.modules.clone(),
+                                                    failure_module: fm.clone(),
+                                                    same_worker: flow.same_worker,
                                                 },
-                                                tag: None,
+                                                path: Some(format!(
+                                                    "{}/branchall-{}",
+                                                    flow_job.script_path(),
+                                                    i
+                                                )),
                                             }
                                         })
                                         .collect(),
@@ -1994,20 +1958,17 @@ async fn compute_next_flow_transform<'c>(
             Ok((
                 tx,
                 NextFlowTransform::Continue(
-                    ContinuePayload::SingleJob(JobPayloadWithTag {
-                        payload: JobPayload::RawFlow {
-                            value: FlowValue {
-                                modules,
-                                failure_module: fm.clone(),
-                                same_worker: flow.same_worker,
-                            },
-                            path: Some(format!(
-                                "{}/branchall-{}",
-                                flow_job.script_path(),
-                                branch_status.branch
-                            )),
+                    ContinuePayload::SingleJob(JobPayload::RawFlow {
+                        value: FlowValue {
+                            modules,
+                            failure_module: fm.clone(),
+                            same_worker: flow.same_worker,
                         },
-                        tag: None,
+                        path: Some(format!(
+                            "{}/branchall-{}",
+                            flow_job.script_path(),
+                            branch_status.branch
+                        )),
                     }),
                     NextStatus::NextBranchStep(NextBranch { status: branch_status, flow_jobs }),
                 ),

benchmarks/example_fake_user.json

@@ -0,0 +1,53 @@
+{
+  "type": "RANDOM",
+  "actions": [
+    {
+      "weight": 1,
+      "action": {
+        "type": "RANDOM",
+        "actions": [
+          {
+            "weight": 1,
+            "action": {
+              "type": "PREVIEW_SCRIPT",
+              "workspace": "demo",
+              "language": "deno",
+              "args": {},
+              "content": "export async function main() { return \"Hello World\"; }"
+            }
+          },
+          {
+            "weight": 1,
+            "action": {
+              "type": "PREVIEW_SCRIPT",
+              "workspace": "demo",
+              "language": "python3",
+              "args": {},
+              "content": "def main(): return \"Hello World\";"
+            }
+          },
+          {
+            "weight": 1,
+            "action": {
+              "type": "PREVIEW_SCRIPT",
+              "workspace": "demo",
+              "language": "go",
+              "args": {},
+              "content": "func main() { return \"Hello World\" }"
+            }
+          }
+        ]
+      }
+    },
+    {
+      "weight": 0.5,
+      "action": {
+        "type": "PREVIEW_SCRIPT",
+        "workspace": "demo",
+        "language": "deno",
+        "args": {},
+        "content": "import { delay } from \"https://deno.land/std@0.131.0/async/delay.ts\"; export async function main() { await delay(1000); return \"Hello World\"; }"
+      }
+    }
+  ]
+}

benchmarks/identity.json

@@ -1,7 +0,0 @@
-{
-  "type": "PREVIEW_SCRIPT",
-  "workspace": "demo",
-  "language": "deno",
-  "args": {},
-  "content": "export async function main() { return \"Hello World\"; }"
-}

benchmarks/main.ts

@@ -27,60 +27,60 @@ await new Command()
     "The number of workers to run at once.",
     {
       default: 1,
-    },
+    }
   )
   .option(
     "-s --seconds <seconds:number>",
     "How long to run the benchmark for (in seconds).",
     {
       default: 30,
-    },
+    }
   )
   .option("--max <max:number>", "Maximum number of operations performed.")
   .option("-e --email <email:string>", "The email to use to login.")
   .option("-p --password <password:string>", "The password to use to login.")
   .env(
     "WM_TOKEN=<token:string>",
-    "The token to use when talking to the API server. Preferred over manual login.",
+    "The token to use when talking to the API server. Preferred over manual login."
   )
   .option(
     "-t --token <token:string>",
-    "The token to use when talking to the API server. Preferred over manual login.",
+    "The token to use when talking to the API server. Preferred over manual login."
   )
   .env(
     "WM_WORKSPACE=<workspace:string>",
-    "The workspace to spawn scripts from.",
+    "The workspace to spawn scripts from."
   )
   .option(
     "-w --workspace <workspace:string>",
     "The workspace to spawn scripts from.",
-    { default: "starter" },
+    { default: "starter" }
   )
   .option("-m --metrics <metrics:string>", "The url to scrape metrics from.", {
     default: "http://localhost:8001/metrics",
   })
   .option(
     "--export-json <export_json:string>",
-    "If set, exports will be into a JSON file.",
+    "If set, exports will be into a JSON file."
   )
   .option(
     "--export-csv <export_csv:string>",
-    "If set, exports will be into a csv file.",
+    "If set, exports will be into a csv file."
   )
   .option(
     "--export-histograms [histograms...:string]",
-    "Mark metrics (without label) that are reported as histograms to export.",
+    "Mark metrics (without label) that are reported as histograms to export."
   )
   .option(
     "--export-simple [simple...:string]",
-    "Mark metrics (without label) that are reported as simple values.",
+    "Mark metrics (without label) that are reported as simple values."
   )
   .option(
     "--maximum-throughput <maximum_throughput:number>",
     "Maximum number of jobs/flows to start in one second.",
     {
       default: Infinity,
-    },
+    }
   )
   .option("--use-flows", "Run flows instead of jobs.")
   .option("--custom <custom_path:string>", "Use custom actions during bench")
@@ -89,11 +89,11 @@ await new Command()
     "The maximum time in ms to wait for jobs to complete.",
     {
       default: 90000,
-    },
+    }
   )
   .option(
     "--continous",
-    "Run the benchmark forever. This effectively disables metric collection & exports. No zombie jobs will be tracked.",
+    "Run the benchmark forever. This effectively disables metric collection & exports. No zombie jobs will be tracked."
   )
   .option(
     "--histogram-buckets [buckets...:string]",
@@ -114,7 +114,7 @@ await new Command()
         "0.01",
         "0.005",
       ],
-    },
+    }
   )
   .action(
     async ({
@@ -162,7 +162,7 @@ await new Command()
           new URL("./scraper.ts", import.meta.url).href,
           {
             type: "module",
-          },
+          }
         );
 
         metrics_worker.postMessage({
@@ -192,8 +192,8 @@ await new Command()
             zombieTimeout,
           },
           null,
-          4,
-        ),
+          4
+        )
       );
 
       const config = {
@@ -245,22 +245,19 @@ await new Command()
       const updateState = setInterval(async () => {
         const elapsed = start ? Math.ceil((Date.now() - start) / 1000) : 0;
         const sum = jobsSent.reduce((a, b) => a + b, 0);
-        const queue_length = (await (await fetch(
-          host + "/api/w/" + config.workspace_id + "/jobs/queue/count",
-          { headers: { ["Authorization"]: "Bearer " + config.token } },
-        )).json()).database_length;
+        const queue_length = (
+          await windmill.JobService.listQueue({
+            workspace: config.workspace_id,
+          })
+        ).length;
         await Deno.stdout.write(
           enc(
-            `elapsed: ${elapsed}/${seconds} | jobs sent: ${
-              JSON.stringify(
-                jobsSent,
-              )
-            } (sum: ${sum} thr: ${
-              (sum / elapsed).toFixed(
-                2,
-              )
-            }) | queue: ${queue_length}                          \r`,
-          ),
+            `elapsed: ${elapsed}/${seconds} | jobs sent: ${JSON.stringify(
+              jobsSent
+            )} (sum: ${sum} thr: ${(sum / elapsed).toFixed(
+              2
+            )}) | queue: ${queue_length}                          \r`
+          )
         );
       }, 100);
 
@@ -287,7 +284,7 @@ await new Command()
 
       const sum = jobsSent.reduce((a, b) => a + b, 0);
       await Deno.stdout.write(
-        enc(" ".padStart(30) + `\rduration: ${seconds} | jobs sent: ${sum}\n`),
+        enc(" ".padStart(30) + `\rduration: ${seconds} | jobs sent: ${sum}\n`)
       );
 
       const shutdown_start = Date.now();
@@ -305,7 +302,7 @@ await new Command()
         };
         worker.addEventListener("message", l);
         worker.postMessage(
-          Number.isSafeInteger(zombieTimeout) ? zombieTimeout : 90000,
+          Number.isSafeInteger(zombieTimeout) ? zombieTimeout : 90000
         );
       });
 
@@ -321,10 +318,11 @@ await new Command()
       console.log("incorrect results: ", incorrect_results);
       console.log(
         "queue length:",
-        (await (await fetch(
-          host + "/api/w/" + config.workspace_id + "/jobs/queue/count",
-          { headers: { ["Authorization"]: "Bearer " + config.token } },
-        )).json()).database_length,
+        (
+          await windmill.JobService.listQueue({
+            workspace: config.workspace_id,
+          })
+        ).length
       );
 
       metrics_worker!.postMessage("stop");
@@ -348,7 +346,7 @@ await new Command()
           const value = values[i]!;
           const mean = value.reduce((acc, e) => acc + e, 0) / values.length;
           const stdev = Math.sqrt(
-            value.reduce((acc, e) => acc + (e - mean) ** 2) / values.length,
+            value.reduce((acc, e) => acc + (e - mean) ** 2) / values.length
           );
           obj[name] = { mean, stdev };
         }
@@ -376,6 +374,6 @@ await new Command()
         f.close();
       }
       console.log("done");
-    },
+    }
   )
   .parse();

benchmarks/worker.ts

@@ -13,8 +13,6 @@ const promise = new Promise<{
   continous: boolean;
   max_per_worker: number;
   custom: Action | undefined;
-  server: string;
-  token: string;
 }>((resolve, _reject) => {
   self.onmessage = (evt) => {
     const sharedConfig = evt.data;
@@ -26,8 +24,6 @@ const promise = new Promise<{
       continous: sharedConfig.continous,
       max_per_worker: sharedConfig.max_per_worker,
       custom: sharedConfig.custom,
-      server: sharedConfig.server,
-      token: sharedConfig.token,
     };
     self.name = "Worker " + sharedConfig.i;
     resolve(config);
@@ -50,20 +46,19 @@ const updateStatusInterval = setInterval(() => {
 }, 100);
 
 while (cont) {
-  const queue_length = (await (await fetch(
-    config.server + "/api/w/" + config.workspace_id + "/jobs/queue/count",
-    { headers: { ["Authorization"]: "Bearer " + config.token } },
-  )).json()).database_length;
-  if (queue_length > 2500) {
+  const queue_length = (
+    await windmill.JobService.listQueue({ workspace: config.workspace_id })
+  ).length;
+  if (queue_length > 500) {
     console.log(
-      `queue length: ${queue_length} > 2500. waiting...                                                            `,
+      `queue length: ${queue_length} > 500. waiting...                                                            `
     );
     await sleep(0.5);
     continue;
   }
   if (
     (total_spawned * 1000) / (Date.now() - start_time) >
-      config.per_worker_throughput
+    config.per_worker_throughput
   ) {
     console.log("at maximum throughput. waiting...");
     await sleep(0.1);
@@ -143,13 +138,13 @@ while (outstanding.length > 0 && Date.now() < end_time) {
     await Deno.stdout.write(
       enc(
         `uuid: ${uuid}, queue length: ${
-          (await (await fetch(
-            config.server + "/api/w/" + config.workspace_id +
-              "/jobs/queue/count",
-            { headers: { ["Authorization"]: "Bearer " + config.token } },
-          )).json()).database_length
-        }                                                                                   \r`,
-      ),
+          (
+            await windmill.JobService.listQueue({
+              workspace: config.workspace_id,
+            })
+          ).length
+        }                                                                                   \r`
+      )
     );
   } else if (!config.useFlows) {
     r = r as api.CompletedJob;
@@ -161,7 +156,7 @@ while (outstanding.length > 0 && Date.now() < end_time) {
             " != " +
             uuid +
             "job: \n" +
-            JSON.stringify(r, null, 2),
+            JSON.stringify(r, null, 2)
         );
         incorrect_results++;
       }

cli/main.ts

@@ -19,7 +19,7 @@ import sync from "./sync.ts";
 import { tryResolveVersion } from "./context.ts";
 import { GlobalOptions } from "./types.ts";
 
-export const VERSION = "v1.93.0";
+export const VERSION = "v1.86.0";
 
 let command: any = new Command()
   .name("wmill")

cli/script.ts

@@ -142,7 +142,6 @@ export async function handleFile(
             typed.summary === remote.summary &&
             typed.is_template === remote.is_template &&
             typed.kind == remote.kind &&
-            !remote.archived &&
             remote?.lock == typed.lock?.join("\n") &&
             JSON.stringify(typed.schema) == JSON.stringify(remote.schema))
         ) {

frontend/.env

@@ -1 +1 @@
-VITE_DEFAULT_WEBHOOK_TYPE=async
+VITE_DEFAULT_WEBHOOK_TYPE=async 

frontend/.gitignore

@@ -7,5 +7,4 @@ node_modules
 CaddyfileRemoteRuben
 tests-out/
 storageState.json
-.env.production
-dist/
+.env.production

frontend/Caddyfile

@@ -1,11 +1,11 @@
-:80 {  
+http://localhost {  
   bind {$ADDRESS}
   reverse_proxy /api/* http://localhost:8000
   reverse_proxy /* http://localhost:3000
   reverse_proxy /ws/* http://localhost:3001
 }
 
-:443 {  
+https://localhost {  
   bind {$ADDRESS}
   reverse_proxy /api/* http://localhost:8000
   reverse_proxy /ws/* http://localhost:3001

frontend/CaddyfileDev

@@ -4,9 +4,9 @@
 
 :8080 {  
   bind {$ADDRESS}
-  reverse_proxy /api/* http://localhost:8000
+  reverse_proxy /api/* http://windmill:8000
   reverse_proxy /* http://localhost:3000
   reverse_proxy /ws/* http://lsp:3001 {
     lb_policy header "Authorization"
   }
-}
+}

frontend/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "windmill",
-	"version": "1.93.0",
+	"version": "1.86.0",
 	"scripts": {
 		"dev": "vite dev",
 		"build": "vite build",
@@ -9,49 +9,49 @@
 		"check:watch": "svelte-check --tsconfig ./tsconfig.json --watch",
 		"lint": "prettier --ignore-path .gitignore --check --plugin-search-dir=. . && eslint --ignore-path .gitignore .",
 		"format": "prettier --ignore-path .gitignore --write --plugin-search-dir=. .",
-		"package": "svelte-package -o package",
+		"package": "svelte-package && cd package && rm README.md && rm README_DEV.md && sed -i -e 's/windmill/windmill-components/g' package.json",
 		"generate-backend-client": "openapi --input ../backend/windmill-api/openapi.yaml --output ./src/lib/gen --useOptions && sed -i '213 i \\    request.referrerPolicy = \"no-referrer\"\n' src/lib/gen/core/request.ts",
 		"pretest": "tsc --incremental -p tests/tsconfig.json",
 		"test": "playwright test --config=tests-out/playwright.config.js"
 	},
 	"devDependencies": {
-		"@playwright/test": "^1.32.3",
+		"@playwright/test": "^1.31.1",
 		"@rgossiaux/svelte-headlessui": "^1.0.2",
-		"@sveltejs/adapter-static": "^2.0.2",
-		"@sveltejs/kit": "^1.15.9",
-		"@sveltejs/package": "^2.0.0",
+		"@sveltejs/adapter-static": "^2.0.1",
+		"@sveltejs/kit": "^1.0.0-next.589",
+		"@sveltejs/package": "^1.0.2",
 		"@tailwindcss/forms": "^0.5.3",
 		"@tailwindcss/typography": "^0.5.8",
 		"@types/d3": "^7.4.0",
 		"@types/d3-zoom": "^3.0.2",
-		"@types/lodash": "^4.14.194",
-		"@types/node": "^18.16.3",
-		"@types/vscode": "~1.77.0",
-		"@typescript-eslint/eslint-plugin": "^5.59.2",
-		"@typescript-eslint/parser": "^5.59.1",
-		"@zerodevx/svelte-toast": "^0.9.3",
+		"@types/lodash": "^4.14.191",
+		"@types/node": "^18.11.18",
+		"@types/vscode": "~1.76.0",
+		"@typescript-eslint/eslint-plugin": "^5.49.0",
+		"@typescript-eslint/parser": "^5.48.0",
+		"@zerodevx/svelte-toast": "^0.8.1",
 		"autoprefixer": "^10.4.13",
-		"cssnano": "^6.0.1",
+		"cssnano": "^6.0.0",
 		"d3-dag": "^0.11.5",
-		"eslint": "^8.38.0",
+		"eslint": "^8.28.0",
 		"eslint-config-prettier": "^8.6.0",
 		"eslint-plugin-svelte3": "^4.0.0",
-		"monaco-editor-workers": "0.37.0",
+		"monaco-editor-workers": "0.36.0",
 		"ol": "^7.2.2",
 		"openapi-typescript-codegen": "^0.23.0",
 		"path-browserify": "^1.0.1",
-		"pdfjs-dist": "^3.5.141",
-		"postcss": "^8.4.23",
+		"pdfjs-dist": "^3.4.120",
+		"postcss": "^8.4.18",
 		"postcss-load-config": "^4.0.1",
-		"prettier": "^2.8.8",
+		"prettier": "^2.8.3",
 		"prettier-plugin-svelte": "^2.9.0",
 		"simple-svelte-autocomplete": "^2.5.1",
 		"style-to-object": "^0.4.1",
-		"stylelint-config-recommended": "^12.0.0",
+		"stylelint-config-recommended": "^9.0.0",
 		"svelte": "^3.55.1",
 		"svelte-awesome": "^3.2.0",
 		"svelte-awesome-color-picker": "^2.4.1",
-		"svelte-check": "^3.2.0",
+		"svelte-check": "^3.0.2",
 		"svelte-highlight": "^7.2.1",
 		"svelte-multiselect": "^8.6.0",
 		"svelte-overlay": "^1.4.1",
@@ -59,11 +59,11 @@
 		"svelte-preprocess": "^5.0.1",
 		"svelte-range-slider-pips": "^2.1.1",
 		"svelte-splitpanes": "^0.7.3",
-		"svelte2tsx": "^0.6.11",
-		"tailwindcss": "^3.3.2",
+		"svelte2tsx": "^0.6.10",
+		"tailwindcss": "^3.2.2",
 		"tslib": "^2.4.1",
-		"typescript": "^5.0.4",
-		"vite": "^4.3.3",
+		"typescript": "^4.9.3",
+		"vite": "^4.0.4",
 		"yootils": "^0.3.1"
 	},
 	"type": "module",
@@ -73,7 +73,7 @@
 		"@leeoniya/ufuzzy": "^1.0.6",
 		"@popperjs/core": "^2.11.6",
 		"@redocly/json-to-json-schema": "^0.0.1",
-		"@tanstack/svelte-table": "^8.8.5",
+		"@tanstack/svelte-table": "^8.7.6",
 		"ag-grid-svelte": "^0.1.4",
 		"chart.js": "^3.9.1",
 		"chartjs-adapter-date-fns": "^3.0.0",
@@ -81,173 +81,21 @@
 		"d3-zoom": "^3.0.0",
 		"date-fns": "^2.29.3",
 		"fast-equals": "^4.0.3",
-		"highlight.js": "^11.8.0",
+		"highlight.js": "^11.7.0",
 		"lodash": "^4.17.21",
-		"lucide-svelte": "^0.192.0",
+		"lucide-svelte": "^0.115.0",
 		"monaco-editor": "0.36.1",
 		"monaco-languageclient": "5.0.1",
 		"svelte-autosize": "^1.0.1",
 		"svelte-chartjs": "^3.1.0",
-		"svelte-dnd-action": "^0.9.22",
 		"svelte-portal": "^2.2.0",
-		"svelte-select": "^5.6.0",
+		"svelte-select": "^5.3.1",
 		"svelte-timezone-picker": "^2.0.3",
-		"tailwind-merge": "^1.12.0",
+		"tailwind-merge": "^1.9.1",
 		"vscode-languageclient": "^8.1.0",
 		"vscode-ws-jsonrpc": "3.0.0"
 	},
 	"peerDependencies": {
-		"@sveltejs/kit": "^1.15.9",
 		"svelte": "^3.55.1"
-	},
-	"exports": {
-		"./package.json": "./package.json",
-		".": {
-			"types": "./dist/index.d.ts",
-			"svelte": "./dist/index.js"
-		},
-		"./assets/app.css": "./package/assets/app.css",
-		"./components/IconedResourceType.svelte": {
-			"types": "./package/components/IconedResourceType.svelte.d.ts",
-			"svelte": "./package/components/IconedResourceType.svelte",
-			"default": "./package/components/IconedResourceType.svelte"
-		},
-		"./components/common/drawer/Drawer.svelte": {
-			"types": "./package/components/common/drawer/Drawer.svelte.d.ts",
-			"svelte": "./package/components/common/drawer/Drawer.svelte",
-			"default": "./package/components/common/drawer/Drawer.svelte"
-		},
-		"./components/common/drawer/DrawerContent.svelte": {
-			"types": "./package/components/common/drawer/DrawerContent.svelte.d.ts",
-			"svelte": "./package/components/common/drawer/DrawerContent.svelte",
-			"default": "./package/components/common/drawer/DrawerContent.svelte"
-		},
-		"./components/common/button/Button.svelte": {
-			"types": "./package/components/common/button/Button.svelte.d.ts",
-			"svelte": "./package/components/common/button/Button.svelte",
-			"default": "./package/components/common/button/Button.svelte"
-		},
-		"./components/RadioButton.svelte": {
-			"types": "./package/components/RadioButton.svelte.d.ts",
-			"svelte": "./package/components/RadioButton.svelte",
-			"default": "./package/components/RadioButton.svelte"
-		},
-		"./components/common/tabs/Tabs.svelte": {
-			"types": "./package/components/common/tabs/Tabs.svelte.d.ts",
-			"svelte": "./package/components/common/tabs/Tabs.svelte",
-			"default": "./package/components/common/tabs/Tabs.svelte"
-		},
-		"./components/common/tabs/Tab.svelte": {
-			"types": "./package/components/common/tabs/Tab.svelte.d.ts",
-			"svelte": "./package/components/common/tabs/Tab.svelte",
-			"default": "./package/components/common/tabs/Tab.svelte"
-		},
-		"./components/common/alert/Alert.svelte": {
-			"types": "./package/components/common/alert/Alert.svelte.d.ts",
-			"svelte": "./package/components/common/alert/Alert.svelte",
-			"default": "./package/components/common/alert/Alert.svelte"
-		},
-		"./components/apps/editor/AppPreview.svelte": {
-			"types": "./package/components/apps/editor/AppPreview.svelte.d.ts",
-			"svelte": "./package/components/apps/editor/AppPreview.svelte",
-			"default": "./package/components/apps/editor/AppPreview.svelte"
-		},
-		"./components/FlowViewer.svelte": {
-			"types": "./package/components/FlowViewer.svelte.d.ts",
-			"svelte": "./package/components/FlowViewer.svelte",
-			"default": "./package/components/FlowViewer.svelte"
-		},
-		"./components/SchemaViewer.svelte": {
-			"types": "./package/components/SchemaViewer.svelte.d.ts",
-			"svelte": "./package/components/SchemaViewer.svelte",
-			"default": "./package/components/SchemaViewer.svelte"
-		},
-		"./components/SchemaEditor.svelte": {
-			"types": "./package/components/SchemaEditor.svelte.d.ts",
-			"svelte": "./package/components/SchemaEditor.svelte",
-			"default": "./package/components/SchemaEditor.svelte"
-		},
-		"./common": {
-			"types": "./package/common.d.ts",
-			"default": "./package/common.js"
-		},
-		"./components/icons": {
-			"types": "./package/components/icons/index.d.ts",
-			"svelte": "./package/components/icons/index.js",
-			"default": "./package/components/icons/index.js"
-		},
-		"./components/apps/inputType": {
-			"types": "./package/components/apps/inputType.d.ts",
-			"default": "./package/components/apps/inputType.js"
-		},
-		"./components/apps/types": {
-			"types": "./package/components/apps/types.d.ts",
-			"default": "./package/components/apps/types.js"
-		},
-		"./components/apps/editor/inlineScriptsPanel/utils": {
-			"types": "./package/components/apps/editor/inlineScriptsPanel/utils.d.ts",
-			"default": "./package/components/apps/editor/inlineScriptsPanel/utils.js"
-		}
-	},
-	"files": [
-		"dist",
-		"package"
-	],
-	"license": "Apache-2.0",
-	"svelte": "./dist/index.js",
-	"typesVersions": {
-		">4.0": {
-			"components/IconedResourceType.svelte": [
-				"./package/components/IconedResourceType.svelte.d.ts"
-			],
-			"components/common/drawer/Drawer.svelte": [
-				"./package/components/common/drawer/Drawer.svelte.d.ts"
-			],
-			"components/common/drawer/DrawerContent.svelte": [
-				"./package/components/common/drawer/DrawerContent.svelte.d.ts"
-			],
-			"components/common/button/Button.svelte": [
-				"./package/components/common/button/Button.svelte.d.ts"
-			],
-			"components/RadioButton.svelte": [
-				"./package/components/RadioButton.svelte.d.ts"
-			],
-			"components/common/tabs/Tabs.svelte": [
-				"./package/components/common/tabs/Tabs.svelte.d.ts"
-			],
-			"components/common/tabs/Tab.svelte": [
-				"./package/components/common/tabs/Tab.svelte.d.ts"
-			],
-			"components/common/alert/Alert.svelte": [
-				"./package/components/common/alert/Alert.svelte.d.ts"
-			],
-			"components/apps/editor/AppPreview.svelte": [
-				"./package/components/apps/editor/AppPreview.svelte.d.ts"
-			],
-			"components/FlowViewer.svelte": [
-				"./package/components/FlowViewer.svelte.d.ts"
-			],
-			"components/SchemaViewer.svelte": [
-				"./package/components/SchemaViewer.svelte.d.ts"
-			],
-			"components/SchemaEditor.svelte": [
-				"./package/components/SchemaEditor.svelte.d.ts"
-			],
-			"common": [
-				"./package/common.d.ts"
-			],
-			"components/icons": [
-				"./package/components/icons/index.d.ts"
-			],
-			"components/apps/inputType": [
-				"./package/components/apps/inputType.d.ts"
-			],
-			"components/apps/types": [
-				"./package/components/apps/types.d.ts"
-			],
-			"components/apps/editor/inlineScriptsPanel/utils": [
-				"./package/components/apps/editor/inlineScriptsPanel/utils.d.ts"
-			]
-		}
 	}
 }

frontend/publish.sh

@@ -1,4 +0,0 @@
-npm run package
-sed -i -e 's/windmill/windmill-components/g' package.json
-npm publish
-sed -i -e 's/windmill-components/windmill/g' package.json

frontend/src/global.d.ts

@@ -1,10 +1,2 @@
 /// <reference types="@sveltejs/kit" />
 
-declare type Item = import('svelte-dnd-action').Item
-declare type DndEvent<ItemType = Item> = import('svelte-dnd-action').DndEvent<ItemType>
-declare namespace svelte.JSX {
-	interface HTMLAttributes<T> {
-		onconsider?: (event: CustomEvent<DndEvent<ItemType>> & { target: EventTarget & T }) => void
-		onfinalize?: (event: CustomEvent<DndEvent<ItemType>> & { target: EventTarget & T }) => void
-	}
-}

frontend/src/lib/components/AppConnect.svelte

@@ -2,12 +2,7 @@
 	const apiTokenApps: Record<string, { img?: string; instructions: string[]; key?: string }> = {
 		airtable: {
 			img: '/airtable_connect.png',
-			instructions: [
-				'Click to <a href="https://airtable.com/create/tokens" target="_blank" rel=”noopener noreferrer”>https://airtable.com/create/tokens</a>',
-				'Click on "Create new token"',
-				'Set a name, specify the scopes or the access level and click on "Create token"',
-				'Copy the token'
-			]
+			instructions: ['Click on the top-right avatar', 'Click on Account', 'Find "Api"']
 		},
 		discord_webhook: {
 			img: '/discord_webhook.png',
@@ -461,7 +456,7 @@
 				>
 			{/if}
 		{/if}
-		<div slot="actions" class="flex gap-1">
+		<div slot="actions">
 			{#if step > 1 && !no_back}
 				<Button variant="border" on:click={back}>Back</Button>
 			{/if}

frontend/src/lib/components/ArgInfo.svelte

@@ -31,9 +31,7 @@
 	</DrawerContent>
 </Drawer>
 
-{#if value == undefined || value == null}
-	<span class="text-gray-600">null</span>
-{:else if value == '<function call>'}
+{#if value == '<function call>'}
 	{'<function call>'}<Tooltip
 		>The arg was none and the default argument of the script is a function call, hence the actual
 		value used for this arg was the output of the script's function call for this arg</Tooltip

frontend/src/lib/components/ArgInput.svelte

@@ -14,7 +14,7 @@
 	import Icon from 'svelte-awesome'
 	import { fade } from 'svelte/transition'
 	import JsonEditor from './apps/editor/settingsPanel/inputEditor/JsonEditor.svelte'
-	import { Badge, Button, SecondsInput } from './common'
+	import { Badge, Button } from './common'
 	import FieldHeader from './FieldHeader.svelte'
 	import type ItemPicker from './ItemPicker.svelte'
 	import NumberTypeNarrowing from './NumberTypeNarrowing.svelte'
@@ -57,7 +57,6 @@
 	export let itemPicker: ItemPicker | undefined = undefined
 	export let noMargin = false
 	export let extra: Record<string, any> = {}
-	export let minW = true
 
 	let seeEditable: boolean = enum_ != undefined || pattern != undefined
 	const dispatch = createEventDispatcher()
@@ -142,12 +141,6 @@
 		}
 	}
 
-	function onKeyDown(e: KeyboardEvent) {
-		if ((e.ctrlKey || e.metaKey) && e.key == 'Enter') {
-			return
-		}
-		e.stopPropagation()
-	}
 	$: {
 		if (value == undefined || value == null) {
 			value = defaultValue
@@ -172,7 +165,7 @@
 </script>
 
 <!-- svelte-ignore a11y-autofocus -->
-<div class="flex flex-col w-full {minW ? 'min-w-[250px]' : ''}">
+<div class="flex flex-col w-full min-w-[250px]">
 	<div>
 		{#if displayHeader}
 			<FieldHeader {label} {required} {type} {contentEncoding} {format} />
@@ -199,7 +192,7 @@
 								use:autosize
 								rows="1"
 								bind:value={description}
-								on:keydown={onKeyDown}
+								on:keydown|stopPropagation
 								placeholder="Field description"
 							/>
 							{#if type == 'string' && format != 'date-time'}
@@ -243,12 +236,12 @@
 						<span>{extra['max']}</span>
 						<span class="mx-2"><Badge large color="blue">{value}</Badge></span>
 					</div>
-				{:else if extra['seconds'] !== undefined}
-					<SecondsInput bind:seconds={value} on:focus />
 				{:else}
 					<input
 						{autofocus}
-						on:focus
+						on:focus={(e) => {
+							dispatch('focus')
+						}}
 						{disabled}
 						type="number"
 						class={valid
@@ -318,23 +311,21 @@
 							{/if}
 						{/key}
 					</div>
-					<div class="flex mt-2">
-						<Button
-							variant="border"
-							color="dark"
-							size="xs"
-							btnClasses="mt-1"
-							on:click={() => {
-								if (value == undefined || !Array.isArray(value)) {
-									value = []
-								}
-								value = value.concat('')
-							}}
-						>
-							<Icon data={faPlus} class="mr-2" />
-							Add item
-						</Button>
-					</div>
+					<Button
+						variant="border"
+						color="dark"
+						size="xs"
+						btnClasses="mt-1"
+						on:click={() => {
+							if (value == undefined || !Array.isArray(value)) {
+								value = []
+							}
+							value = value.concat('')
+						}}
+					>
+						<Icon data={faPlus} class="mr-2" />
+						Add item
+					</Button>
 				</div>
 			{:else if inputCat == 'resource-object'}
 				<ObjectResourceInput {format} bind:value />
@@ -415,7 +406,7 @@
 									dispatch('focus')
 								}}
 								use:autosize
-								on:keydown={onKeyDown}
+								on:keydown|stopPropagation
 								type="text"
 								{disabled}
 								class="col-span-10 {valid
@@ -432,10 +423,8 @@
 										on:click={() => {
 											pickForField = label
 											itemPicker?.openDrawer?.()
-										}}
+										}}><Icon data={faDollarSign} /></div
 									>
-										<Icon data={faDollarSign} />
-									</div>
 								</div>
 							{/if}
 						{/if}

frontend/src/lib/components/CronInput.svelte

@@ -144,7 +144,7 @@
 	}).format
 </script>
 
-<div class="w-full flex space-x-16">
+<div class="w-full flex space-x-16 p-4">
 	<div class="w-full flex flex-col space-y-2">
 		<div class="w-full flex flex-col gap-1">
 			<small class="font-bold">Cron</small>
@@ -318,7 +318,7 @@
 			<small>Estimated upcoming events ({timezone})</small>
 			<div class="flex flex-col rounded-md p-4 border text-gray-600">
 				{#each preview as date}
-					<div class="flex items-center space-x-2 text-sm">
+					<div class="flex items-center space-x-2">
 						<span>{dateFormatter(new Date(date))}</span>
 					</div>
 				{/each}

frontend/src/lib/components/DisplayResult.svelte

@@ -11,7 +11,6 @@
 	export let result: any
 	export let requireHtmlApproval = false
 	export let filename: string | undefined = undefined
-	export let disableExpand = false
 
 	let resultKind:
 		| 'json'
@@ -102,12 +101,10 @@
 			>{/if}{#if typeof result == 'object' && Object.keys(result).length > 0}<div
 				class="mb-2 w-full text-sm text-gray-700 relative"
 				>The result keys are: <b>{truncate(Object.keys(result).join(', '), 50)}</b>
-				{#if !disableExpand}
-					<div class="text-gray-500 text-xs absolute top-5 right-0">
-						<button on:click={jsonViewer.openDrawer}>Expand</button>
-					</div>
-				{/if}
-			</div>{/if}{#if !forceJson && resultKind == 'table-col'}<div
+				<div class="text-gray-500 text-sm absolute top-6 right-0">
+					<button on:click={jsonViewer.openDrawer}>Expand JSON</button>
+				</div></div
+			>{/if}{#if !forceJson && resultKind == 'table-col'}<div
 				class="grid grid-flow-col-dense border border-gray-200 rounded-md"
 			>
 				{#each Object.keys(result) as col}
@@ -203,8 +200,7 @@
 			</div>
 		{:else if !forceJson && resultKind == 'error'}<div>
 				<span class="text-red-500 font-semibold text-sm whitespace-pre-wrap"
-					>{#if result.error.name || result.error.message}{result.error.name}: {result.error
-							.message}{:else}{JSON.stringify(result.error, null, 4)}{/if}</span
+					>{result.error.name}: {result.error.message}</span
 				>
 				<pre class="text-sm whitespace-pre-wrap text-gray-900">{result.error.stack ?? ''}</pre>
 			</div>
@@ -249,29 +245,19 @@
 	{/if}
 </div>
 
-{#if !disableExpand}
-	<Portal>
-		<Drawer bind:this={jsonViewer} size="900px">
-			<DrawerContent title="Expanded Result" on:close={jsonViewer.closeDrawer}>
-				<svelte:fragment slot="actions">
-					<Button
-						on:click={() => copyToClipboard(JSON.stringify(result, null, 4))}
-						color="light"
-						size="xs"
-					>
-						<div class="flex gap-2 items-center">Copy to clipboard <ClipboardCopy /> </div>
-					</Button>
-				</svelte:fragment>
-				{@const str = JSON.stringify(result, null, 4).replace(/\\n/g, '\n')}
-				{#if str.length > 100000}
-					JSON too large. <a
-						download="{filename ?? 'result'}.json"
-						href="data:text/json;charset=utf-8,{encodeURIComponent(str)}">Download</a
-					>
-				{:else}
-					<Highlight language={json} code={JSON.stringify(result, null, 4).replace(/\\n/g, '\n')} />
-				{/if}
-			</DrawerContent>
-		</Drawer>
-	</Portal>
-{/if}
+<Portal>
+	<Drawer bind:this={jsonViewer} size="900px">
+		<DrawerContent title="Expanded Result" on:close={jsonViewer.closeDrawer}>
+			<svelte:fragment slot="actions">
+				<Button
+					on:click={() => copyToClipboard(JSON.stringify(result, null, 4))}
+					color="light"
+					size="xs"
+				>
+					<div class="flex gap-2 items-center">Copy to clipboard <ClipboardCopy /> </div>
+				</Button>
+			</svelte:fragment>
+			<Highlight language={json} code={JSON.stringify(result, null, 4).replace(/\\n/g, '\n')} />
+		</DrawerContent>
+	</Drawer>
+</Portal>

frontend/src/lib/components/DraftBadge.svelte

@@ -1,21 +0,0 @@
-<script lang="ts">
-	import Popover from './Popover.svelte'
-	import { Badge } from './common'
-
-	export let has_draft: boolean = false
-	export let draft_only: boolean = false
-</script>
-
-{#if has_draft}
-	{#if draft_only}
-		<Popover notClickable>
-			<svelte:fragment slot="text">Never deployed and is only a draft</svelte:fragment>
-			<Badge small color="indigo" baseClass="border border-indigo-200">Draft only</Badge>
-		</Popover>
-	{:else}
-		<Popover notClickable>
-			<svelte:fragment slot="text">Is deployed and has a draft</svelte:fragment>
-			<Badge color="indigo" baseClass="border border-indigo-200">+Draft</Badge>
-		</Popover>
-	{/if}
-{/if}

frontend/src/lib/components/Dropdown.svelte

@@ -11,27 +11,19 @@
 	type Side = 'top' | 'bottom'
 	type Placement = `${Side}-${Alignment}`
 
-	export let dropdownItems: DropdownItem[] | (() => DropdownItem[]) = []
+	export let dropdownItems: DropdownItem[]
 	export let name: string | undefined = undefined
 	export let placement: Placement = 'bottom-start'
 	export let btnClasses = ''
 	$: buttonClass = twMerge('!border-0 bg-transparent !p-[6px]', btnClasses)
 
 	const dispatch = createEventDispatcher()
-
-	function computeDropdowns(): DropdownItem[] {
-		if (typeof dropdownItems === 'function') {
-			return dropdownItems()
-		} else {
-			return dropdownItems
-		}
-	}
 </script>
 
 <Menu {placement} let:close>
 	<Button
 		nonCaptureEvent
-		color="light"
+		color="dark"
 		variant="border"
 		size="xs"
 		btnClasses={buttonClass}
@@ -39,13 +31,13 @@
 		slot="trigger"
 	>
 		{#if !$$slots.default}
-			<MoreVertical size={16} />
+			<MoreVertical size={20} />
 		{:else}
 			<slot />
 		{/if}
 	</Button>
 	{#if dropdownItems}
-		{#each computeDropdowns() as item, i}
+		{#each dropdownItems as item, i}
 			{#if item.action}
 				<button
 					on:click|preventDefault|stopPropagation={(event) => {
@@ -99,8 +91,8 @@
 			{:else}
 				<!-- svelte-ignore a11y-click-events-have-key-events -->
 				<span
-					class:bg-gray-200={item.disabled}
-					class="block font-semibold text-left px-4 py-2 text-sm text-gray-700 cursor-auto"
+					class:bg-gray-50={item.disabled}
+					class="block text-left px-4 py-2 text-sm text-gray-700 cursor-auto"
 					role="menuitem"
 					tabindex="-1"
 					id="user-menu-item-{name}-{i}}"

frontend/src/lib/components/Editor.svelte

@@ -508,7 +508,7 @@
 	async function loadMonaco() {
 		const model = meditor.createModel(code, lang, mUri.parse(uri))
 
-		model.updateOptions(lang == 'python' ? { tabSize: 4, insertSpaces: true } : updateOptions)
+		model.updateOptions(updateOptions)
 		editor = meditor.create(divEl as HTMLDivElement, {
 			...editorConfig(model, code, lang, automaticLayout, fixedOverflowWidgets),
 			overflowWidgetsDomNode: widgets,
@@ -569,7 +569,7 @@
 		callback: (editor: meditor.IStandaloneCodeEditor) => void,
 		keybindings: number[] = []
 	) {
-		editor?.addAction({
+		editor.addAction({
 			id,
 			label,
 			keybindings,

frontend/src/lib/components/EditorBar.svelte

@@ -7,6 +7,7 @@
 	import { getScriptByPath, sendUserToast } from '$lib/utils'
 
 	import {
+		faBroom,
 		faCube,
 		faDollarSign,
 		faEye,
@@ -29,7 +30,7 @@
 	import ToggleHubWorkspace from './ToggleHubWorkspace.svelte'
 	import Skeleton from './common/skeleton/Skeleton.svelte'
 	import Popover from './Popover.svelte'
-	import { SCRIPT_EDITOR_SHOW_EXPLORE_OTHER_SCRIPTS } from '$lib/consts'
+	import Kbd from './common/kbd/Kbd.svelte'
 
 	export let lang: 'python3' | 'deno' | 'go' | 'bash'
 	export let editor: Editor | undefined
@@ -127,9 +128,6 @@
 		}
 		sendUserToast(`${name} inserted at cursor`)
 	}}
-	tooltip="Contextual Variables are variables whose values are contextual to the Script
-	execution. They are are automatically set by Windmill."
-	documentationLink="https://docs.windmill.dev/docs/core_concepts/variables_and_secrets#contextual-variables"
 	itemName="Contextual Variable"
 	extraField="name"
 	loadItems={loadContextualVariables}
@@ -163,8 +161,6 @@
 		}
 		sendUserToast(`${name} inserted at cursor`)
 	}}
-	tooltip="Variables are dynamic values that have a key associated to them and can be retrieved during the execution of a Script or Flow."
-	documentationLink="https://docs.windmill.dev/docs/core_concepts/variables_and_secrets"
 	itemName="Variable"
 	extraField="path"
 	loadItems={loadVariables}
@@ -213,8 +209,6 @@
 		}
 		sendUserToast(`${path} inserted at cursor`)
 	}}
-	tooltip="Resources represent connections to third party systems. Resources are a good way to define a connection to a frequently used third party system such as a database."
-	documentationLink="https://docs.windmill.dev/docs/core_concepts/resources_and_types"
 	itemName="Resource"
 	buttons={{ 'Edit/View': (x) => resourceEditor.initEdit(x) }}
 	extraField="description"
@@ -247,7 +241,7 @@
 			<Popover
 				notClickable
 				placement="bottom"
-				disappearTimeout={0}
+				disapperTimoout={0}
 				class="pr-1"
 				disablePopup={!iconOnly}
 			>
@@ -267,7 +261,7 @@
 			<Popover
 				notClickable
 				placement="bottom"
-				disappearTimeout={0}
+				disapperTimoout={0}
 				class="px-1"
 				disablePopup={!iconOnly}
 			>
@@ -287,7 +281,7 @@
 			<Popover
 				notClickable
 				placement="bottom"
-				disappearTimeout={0}
+				disapperTimoout={0}
 				class="px-1"
 				disablePopup={!iconOnly}
 			>
@@ -307,7 +301,7 @@
 			<Popover
 				notClickable
 				placement="bottom"
-				disappearTimeout={0}
+				disapperTimoout={0}
 				class="px-1"
 				disablePopup={!iconOnly}
 			>
@@ -327,7 +321,7 @@
 			<Popover
 				notClickable
 				placement="bottom"
-				disappearTimeout={0}
+				disapperTimoout={0}
 				class="px-1"
 				disablePopup={!iconOnly}
 			>
@@ -355,10 +349,10 @@
 				</Button>
 				<svelte:fragment slot="text">Reload assistant</svelte:fragment>
 			</Popover>
-			<!-- <Popover
+			<Popover
 				notClickable
 				placement="bottom"
-				disappearTimeout={0}
+				disapperTimoout={0}
 				class="px-1"
 				disablePopup={!iconOnly}
 			>
@@ -371,36 +365,32 @@
 					{iconOnly}
 					startIcon={{ icon: faBroom }}
 				>
-					Format ({getModifierKey()}+S)
+					Format (Ctrl+S)
 				</Button>
 				<svelte:fragment slot="text">
-					Format <Kbd class="!text-gray-800">{getModifierKey()}</Kbd> + <Kbd class="!text-gray-800">
-						S
-					</Kbd>
+					Format <Kbd class="!text-gray-800">Ctrl</Kbd> + <Kbd class="!text-gray-800">S</Kbd>
 				</svelte:fragment>
-			</Popover> -->
+			</Popover>
 		</div>
 	</div>
 	<Popover
 		notClickable
 		placement="bottom"
-		disappearTimeout={0}
+		disapperTimoout={0}
 		class="px-1"
 		disablePopup={!iconOnly}
 	>
-		{#if SCRIPT_EDITOR_SHOW_EXPLORE_OTHER_SCRIPTS}
-			<Button
-				btnClasses="!font-medium"
-				size="xs"
-				spacingSize="md"
-				color="light"
-				on:click={scriptPicker.openDrawer}
-				{iconOnly}
-				startIcon={{ icon: faEye }}
-			>
-				Explore other scripts
-			</Button>
-		{/if}
+		<Button
+			btnClasses="!font-medium"
+			size="xs"
+			spacingSize="md"
+			color="light"
+			on:click={scriptPicker.openDrawer}
+			{iconOnly}
+			startIcon={{ icon: faEye }}
+		>
+			Explore other scripts
+		</Button>
 		<svelte:fragment slot="text">Script</svelte:fragment>
 	</Popover>
 </div>

frontend/src/lib/components/FlowBuilder.svelte

@@ -1,6 +1,6 @@
 <script lang="ts">
 	import { goto } from '$app/navigation'
-	import { FlowService, ScheduleService, type Flow, type FlowModule, DraftService } from '$lib/gen'
+	import { FlowService, ScheduleService, type Flow, type FlowModule } from '$lib/gen'
 	import { initHistory, redo, undo } from '$lib/history'
 	import { userStore, workspaceStore } from '$lib/stores'
 	import { encodeState, formatCron, loadHubScripts, sendUserToast } from '$lib/utils'
@@ -8,7 +8,7 @@
 	import { setContext } from 'svelte'
 	import { writable, type Writable } from 'svelte/store'
 	import CenteredPage from './CenteredPage.svelte'
-	import { Badge, Button, Kbd, UndoRedo } from './common'
+	import { Badge, Button, UndoRedo } from './common'
 	import { dirtyStore } from './common/confirmationModal/dirtyStore'
 	import FlowEditor from './flows/FlowEditor.svelte'
 	import ScriptEditorDrawer from './flows/content/ScriptEditorDrawer.svelte'
@@ -20,7 +20,6 @@
 	import type { FlowEditorContext } from './flows/types'
 	import { cleanInputs } from './flows/utils'
 	import { Pen } from 'lucide-svelte'
-	import UnsavedConfirmationModal from './common/confirmationModal/UnsavedConfirmationModal.svelte'
 
 	export let initialPath: string = ''
 	export let selectedId: string | undefined
@@ -51,44 +50,8 @@
 	}
 
 	let loadingSave = false
-	let loadingDraft = false
 
-	async function saveDraft(): Promise<void> {
-		loadingDraft = true
-		try {
-			const flow = cleanInputs($flowStore)
-			$dirtyStore = false
-			localStorage.removeItem('flow')
-
-			if (initialPath == '') {
-				await FlowService.createFlow({
-					workspace: $workspaceStore!,
-					requestBody: {
-						path: flow.path,
-						summary: flow.summary,
-						description: flow.description ?? '',
-						value: flow.value,
-						schema: flow.schema,
-						draft_only: true
-					}
-				})
-			}
-			await DraftService.createDraft({
-				workspace: $workspaceStore!,
-				requestBody: { path: initialPath == '' ? flow.path : initialPath, typ: 'flow', value: flow }
-			})
-			if (initialPath == '') {
-				$dirtyStore = false
-				goto(`/flows/edit/${flow.path}`)
-			}
-			sendUserToast('Saved as draft')
-		} catch (error) {
-			sendUserToast(`Error while saving the flow as a draft: ${error.body || error.message}`, true)
-		}
-		loadingDraft = false
-	}
-
-	async function saveFlow(): Promise<void> {
+	async function saveFlow(leave: boolean): Promise<void> {
 		loadingSave = true
 		try {
 			const flow = cleanInputs($flowStore)
@@ -154,8 +117,12 @@
 				}
 			}
 			loadingSave = false
-			$dirtyStore = false
-			goto(`/flows/get/${$flowStore.path}?workspace=${$workspaceStore}`)
+			if (leave) {
+				goto(`/flows/get/${$flowStore.path}?workspace_id=${$workspaceStore}`)
+			} else if (initialPath !== $flowStore.path) {
+				initialPath = $flowStore.path
+				goto(`/flows/edit/${$flowStore.path}?workspace_id=${$workspaceStore}`)
+			}
 		} catch (err) {
 			sendUserToast(`The flow could not be saved: ${err.body}`, true)
 			loadingSave = false
@@ -166,11 +133,11 @@
 
 	$: {
 		if ($flowStore || $selectedIdStore) {
-			saveSessionDraft()
+			saveDraft()
 		}
 	}
 
-	function saveSessionDraft() {
+	function saveDraft() {
 		timeout && clearTimeout(timeout)
 		timeout = setTimeout(() => {
 			try {
@@ -250,21 +217,21 @@
 
 		switch (event.key) {
 			case 'Z':
-				if (event.ctrlKey || event.metaKey) {
+				if (event.ctrlKey) {
 					$flowStore = redo(history)
 					event.preventDefault()
 				}
 				break
 			case 'z':
-				if (event.ctrlKey || event.metaKey) {
+				if (event.ctrlKey) {
 					$flowStore = undo(history, $flowStore)
 					$selectedIdStore = 'Input'
 					event.preventDefault()
 				}
 				break
 			case 's':
-				if (event.ctrlKey || event.metaKey) {
-					saveDraft()
+				if (event.ctrlKey) {
+					saveFlow(false)
 					event.preventDefault()
 				}
 				break
@@ -302,8 +269,8 @@
 		onClick: () => void
 	}> = [
 		{
-			label: 'Exit & see details',
-			onClick: () => goto(`/flows/get/${$flowStore.path}?workspace=${$workspaceStore}`)
+			label: 'Save and exit',
+			onClick: () => saveFlow(true)
 		}
 	]
 
@@ -317,8 +284,6 @@
 
 <svelte:window on:keydown={onKeyDown} />
 
-<UnsavedConfirmationModal />
-
 {#if !$userStore?.operator}
 	<ScriptEditorDrawer bind:this={$scriptEditorDrawer} />
 
@@ -395,23 +360,17 @@
 				<FlowImportExportMenu />
 
 				<FlowPreviewButtons />
-				<Button
-					loading={loadingDraft}
-					size="xs"
-					startIcon={{ icon: faSave }}
-					on:click={() => saveDraft()}
-				>
-					Save draft&nbsp;<Kbd small>Ctrl</Kbd><Kbd small>S</Kbd>
-				</Button>
-				<Button
-					loading={loadingSave}
-					size="xs"
-					startIcon={{ icon: faSave }}
-					on:click={() => saveFlow()}
-					dropdownItems={initialPath != '' ? dropdownItems : undefined}
-				>
-					Deploy
-				</Button>
+				<div class="center-center">
+					<Button
+						loading={loadingSave}
+						size="xs"
+						startIcon={{ icon: faSave }}
+						on:click={() => saveFlow(false)}
+						{dropdownItems}
+					>
+						Save
+					</Button>
+				</div>
 			</div>
 		</div>
 

frontend/src/lib/components/FlowGraphViewer.svelte

@@ -5,7 +5,7 @@
 	import IconedPath from './IconedPath.svelte'
 	import { scriptPathToHref } from '../utils'
 	import type { FlowModule, FlowValue } from '$lib/gen'
-	import { Badge, Button, Drawer, DrawerContent } from './common'
+	import { Drawer, DrawerContent } from './common'
 	import { Highlight } from 'svelte-highlight'
 	import ObjectViewer from './propertyPicker/ObjectViewer.svelte'
 	import typescript from 'svelte-highlight/languages/typescript'
@@ -46,7 +46,7 @@
 				</div>
 				{#if stepDetail.value.path.startsWith('hub/')}
 					<div class="mt-6">
-						<h3 class="mb-2">Code</h3>
+						<h3>Code</h3>
 						<iframe
 							class="w-full h-full text-sm"
 							title="embedded script from hub"
@@ -69,11 +69,11 @@
 		{/if}
 	</DrawerContent>
 </Drawer>
-<div class="grid grid-cols-3 w-full h-full">
+<div class="grid grid-cols-3 w-full">
 	<div
 		class="{noSide
 			? 'col-span-3'
-			: 'sm:col-span-2 col-span-3'} w-full border border-gray-400 max-h-full"
+			: 'sm:col-span-2 col-span-3'} w-full border border-gray-400 max-h-screen"
 		class:overflow-auto={overflowAuto}
 	>
 		<FlowGraph
@@ -86,101 +86,63 @@
 	</div>
 	{#if !noSide}
 		<div
-			class="relative w-full h-full min-h-[150px] max-h-[90vh] border-r border-b border-t border-gray-400
-			p-2 pt-0 overflow-auto hidden sm:flex flex-col gap-4"
+			class="w-full border-r border-b border-t border-gray-400 min-h-[150px] p-2 overflow-auto hidden sm:block"
 		>
 			{#if stepDetail == undefined}
-				<div>
-					<p class="font-medium text-gray-600 text-center pt-4 pb-8">
-						Click on a step to see its details
-					</p>
-					<h3 class="mb-2 font-semibold">Flow Inputs</h3>
-					<SchemaViewer schema={flow?.schema} />
-				</div>
+				<SchemaViewer schema={flow?.schema} />
+
+				<span class="font-black text-lg w-full my-4 mt-14">
+					<span>Click on a step to see its details</span>
+				</span>
 			{:else if stepDetail == 'Input'}
 				<SchemaViewer schema={flow?.schema} />
 			{:else if stepDetail == 'Result'}
-				<p class="font-medium text-gray-600 text-center pt-4 pb-8"> End of the flow </p>
+				End of the flow
 			{:else if typeof stepDetail != 'string' && stepDetail.value}
-				<div class="">
-					<div class="sticky top-0 bg-white w-full flex items-center py-2">
-						{#if stepDetail.id}
-							<Badge color="indigo">
-								{stepDetail.id}
-							</Badge>
-						{/if}
-						<span class="ml-2 font-medium text-lg">
-							{#if stepDetail.summary}
-								{stepDetail.summary}
-							{:else if stepDetail.value.type == 'identity'}
-								Identity
-							{:else if stepDetail.value.type == 'forloopflow'}
-								For loop
-							{:else if stepDetail.value.type == 'branchall'}
-								Run all branches
-							{:else if stepDetail.value.type == 'branchone'}
-								Run one branch
-							{:else if stepDetail.value.type == 'flow'}
-								Inner flow
-							{:else}
-								Anonymous step
-							{/if}
-						</span>
-					</div>
-					{#if stepDetail.value.type == 'script'}
-						<div class="pb-2">
-							<a
-								rel="noreferrer"
-								target="_blank"
-								href={scriptPathToHref(stepDetail?.value?.path ?? '')}
-								class=""
-							>
-								<IconedPath path={stepDetail?.value?.path ?? ''} />
-							</a>
-						</div>
-					{/if}
-				</div>
+				<div class="font-black text-lg w-full mb-6"
+					>Step {stepDetail.id ?? ''}<span class="ml-2 font-normal">{stepDetail.summary || ''}</span
+					></div
+				>
 				{#if stepDetail.value.type == 'identity'}
-					<p class="font-medium text-gray-600 text-center pt-4 pb-8">
-						An identity step returns its inputs as outputs
-					</p>
+					<div> An identity step return as output its input </div>
 				{:else if stepDetail.value.type == 'rawscript'}
-					<div class="text-xs">
-						<h3 class="mb-2 font-semibold">Step Inputs</h3>
+					<div class="text-2xs mb-4">
+						<h3 class="mb-2">Step Inputs</h3>
 						<InputTransformsViewer inputTransforms={stepDetail?.value?.input_transforms ?? {}} />
 					</div>
 
-					<div>
-						<div class="mb-2 flex justify-between items-center">
-							<h3 class="font-semibold">Code</h3>
-							<Button size="xs2" color="light" variant="contained" on:click={codeViewer.openDrawer}>
-								Expand
-							</Button>
-						</div>
-						<div class="w-full overflow-auto">
-							<HighlightCode language={stepDetail.value.language} code={stepDetail.value.content} />
-						</div>
-					</div>
+					<h3 class="mb-2"
+						>Code &nbsp; <button class="text-gray-600 text-xs" on:click={codeViewer.openDrawer}
+							>Expand</button
+						>
+					</h3>
+					<span class="!text-xs">
+						<HighlightCode language={stepDetail.value.language} code={stepDetail.value.content} />
+					</span>
 				{:else if stepDetail.value.type == 'script'}
-					<div class="text-2xs">
-						<h3 class="mb-2 font-semibold">Step Inputs</h3>
+					<div class="mb-4">
+						<a
+							rel="noreferrer"
+							target="_blank"
+							href={scriptPathToHref(stepDetail?.value?.path ?? '')}
+							class=""
+						>
+							<IconedPath path={stepDetail?.value?.path ?? ''} />
+						</a>
+					</div>
+					<div class="text-2xs mb-4">
+						<h3 class="mb-2">Step Inputs</h3>
 						<InputTransformsViewer inputTransforms={stepDetail?.value?.input_transforms ?? {}} />
 					</div>
 					{#if stepDetail.value.path.startsWith('hub/')}
-						<div class="flex flex-col grow">
-							<div class="mb-2 flex justify-between items-center">
-								<h3 class="font-semibold">Code</h3>
-								<Button
-									size="xs2"
-									color="light"
-									variant="contained"
-									on:click={codeViewer.openDrawer}
+						<div class="mt-6">
+							<h3
+								>Code&nbsp; <button class="text-gray-600 text-xs" on:click={codeViewer.openDrawer}
+									>Expand</button
 								>
-									Expand
-								</Button>
-							</div>
+							</h3>
 							<iframe
-								class="w-full grow text-sm"
+								class="w-full h-full text-sm"
 								title="embedded script from hub"
 								frameborder="0"
 								src="https://hub.windmill.dev/embed/script/{stepDetail.value?.path?.substring(4)}"
@@ -188,24 +150,19 @@
 						</div>
 					{/if}
 				{:else if stepDetail.value.type == 'forloopflow'}
-					<div>
-						<p class="font-medium text-gray-600 pb-2"> Iterator expression: </p>
-						{#if stepDetail.value.iterator.type == 'static'}
-							<ObjectViewer json={stepDetail.value.iterator.value} />
-						{:else}
+					<p
+						>For loop with iterator: {#if stepDetail.value.iterator.type == 'static'}<ObjectViewer
+								json={stepDetail.value.iterator.value}
+							/>{:else}
 							<span class="text-xs">
 								<Highlight language={typescript} code={cleanExpr(stepDetail.value.iterator.expr)} />
 							</span>
-						{/if}
-					</div>
+						{/if}</p
+					>
 				{:else if stepDetail.value.type == 'branchall'}
-					<p class="font-medium text-gray-600 text-center pt-4 pb-8">
-						All branches will run, regardless of the inputs
-					</p>
+					<p>Run all branches</p>
 				{:else if stepDetail.value.type == 'branchone'}
-					<p class="font-medium text-gray-600 text-center pt-4 pb-8">
-						Only one branch will run based on a predicate
-					</p>
+					<p>Run one branch based on a predicate</p>
 				{:else if stepDetail.value.type == 'flow'}
 					<FlowPathViewer noSide path={stepDetail.value.path} />
 				{/if}

frontend/src/lib/components/FlowPreviewContent.svelte

@@ -13,7 +13,6 @@
 	import FlowProgressBar from './flows/FlowProgressBar.svelte'
 	import CapturePayload from './flows/content/CapturePayload.svelte'
 	import { Loader2 } from 'lucide-svelte'
-	import { getModifierKey } from '$lib/utils'
 
 	let capturePayload: CapturePayload
 	export let previewMode: 'upTo' | 'whole'
@@ -76,7 +75,7 @@
 		if (open) {
 			switch (event.key) {
 				case 'Enter':
-					if (event.ctrlKey || event.metaKey) {
+					if (event.ctrlKey) {
 						event.preventDefault()
 						runPreview($previewArgs)
 					}
@@ -130,13 +129,12 @@
 			<Button
 				variant="contained"
 				startIcon={{ icon: isRunning ? faRefresh : faPlay }}
-				color="dark"
+				color="blue"
 				size="sm"
 				btnClasses="w-full max-w-lg"
 				on:click={() => runPreview($previewArgs)}
 			>
-				Test flow &nbsp;<Kbd small>{getModifierKey()}</Kbd>
-				<Kbd small><span class="text-lg font-bold">⏎</span></Kbd>
+				Test flow <Kbd class="ml-2">Ctrl+Enter</Kbd>
 			</Button>
 		{/if}
 		<Button

frontend/src/lib/components/FlowStatusViewer.svelte

@@ -158,8 +158,8 @@
 		timeout && clearTimeout(timeout)
 	})
 
-	function loadOwner(path: string) {
-		is_owner = isOwner(path, $userStore!, workspaceId ?? $workspaceStore!)
+	async function loadOwner(path: string) {
+		is_owner = await isOwner(path, $userStore!, workspaceId ?? $workspaceStore!)
 	}
 
 	let selected: 'graph' | 'sequence' = 'graph'

frontend/src/lib/components/FlowViewer.svelte

@@ -8,15 +8,6 @@
 	import FieldHeader from './FieldHeader.svelte'
 	import { copyToClipboard } from '../utils'
 	import FlowGraphViewer from './FlowGraphViewer.svelte'
-	import { setContext } from 'svelte'
-	import {
-		SVELVET_CONTEXT_KEY,
-		type SvelvetSettingsContext
-	} from './graph/svelvet/container/models'
-
-	setContext<SvelvetSettingsContext>(SVELVET_CONTEXT_KEY, {
-		fullHeight: true
-	})
 
 	export let flow: {
 		summary: string
@@ -49,7 +40,7 @@
 
 <Tabs bind:selected={tab}>
 	<Tab value="ui">Graph</Tab>
-	<Tab value="json">JSON</Tab>
+	<Tab value="json">Json</Tab>
 	<Tab value="schema">Input Schema</Tab>
 
 	<svelte:fragment slot="content">
@@ -88,22 +79,18 @@
 			</div>
 		</TabContent>
 		<TabContent value="json">
-			<div class="relative pt-2">
+			<div class="relative">
 				<Button
 					on:click={() => copyToClipboard(JSON.stringify(flowFiltered, null, 4))}
 					color="dark"
 					variant="border"
 					size="sm"
 					startIcon={{ icon: faClipboard }}
-					btnClasses="absolute top-2 right-2 w-min"
+					btnClasses="absolute top-2 right-2"
 				>
 					Copy content
 				</Button>
-				<Highlight
-					language={json}
-					code={JSON.stringify(flowFiltered, null, 4)}
-					class="overflow-auto"
-				/>
+				<Highlight language={json} code={JSON.stringify(flowFiltered, null, 4)} />
 			</div>
 		</TabContent>
 		<TabContent value="schema">

frontend/src/lib/components/InputTransformForm.svelte

@@ -16,7 +16,6 @@
 	import type { InputTransform } from '$lib/gen'
 	import TemplateEditor from './TemplateEditor.svelte'
 	import { setInputCat as computeInputCat } from '$lib/utils'
-	import { Plug } from 'lucide-svelte'
 
 	export let schema: Schema
 	export let arg: InputTransform | any
@@ -28,7 +27,6 @@
 	export let variableEditor: VariableEditor | undefined = undefined
 	export let itemPicker: ItemPicker | undefined = undefined
 	export let noDynamicToggle = false
-	export let argExtra: Record<string, any> = {}
 
 	let monaco: SimpleEditor | undefined = undefined
 	let monacoTemplate: TemplateEditor | undefined = undefined
@@ -222,14 +220,14 @@
 					<Button
 						title="Connect to another node's output"
 						variant="border"
-						color="light"
+						color="dark"
 						size="xs2"
 						on:click={() => {
 							focusProp(argName, 'connect', (path) => {
 								connectProperty(path)
 								return true
 							})
-						}}><Plug size={16} /> &rightarrow;</Button
+						}}>Link &rightarrow;</Button
 					>
 				</div>
 			</div>
@@ -238,7 +236,7 @@
 
 	<div class="max-w-xs" />
 	<div
-		class="relative {$propPickerConfig?.propName == argName
+		class="relative mb-6 {$propPickerConfig?.propName == argName
 			? 'outline outline-offset-0 outline-2 outline-blue-500 rounded-md'
 			: ''}"
 	>
@@ -282,7 +280,6 @@
 				bind:itemsType={schema.properties[argName].items}
 				properties={schema.properties[argName].properties}
 				displayHeader={false}
-				extra={argExtra}
 				{variableEditor}
 				{itemPicker}
 				bind:pickForField

frontend/src/lib/components/InputTransformsViewer.svelte

@@ -1,27 +1,23 @@
 <script lang="ts">
 	import type { InputTransform } from '$lib/gen'
+	import { Highlight } from 'svelte-highlight'
 	import ObjectViewer from './propertyPicker/ObjectViewer.svelte'
+	import typescript from 'svelte-highlight/languages/typescript'
 	import { cleanExpr } from './flows/utils'
 
 	export let inputTransforms: Record<string, InputTransform>
-	$: entries = Object.entries(inputTransforms)
 </script>
 
-{#if entries.length}
-	<ul class="mb-1">
-		{#each entries as [key, val]}
-			<li class="flex pb-2 last:pb-0">
-				<span class="font-black text-gray-700 text-xs">{key}:</span>
-				{#if val.type == 'static'}
-					<ObjectViewer json={val.value} />
-				{:else}
-					<span class="text-xs text-black whitespace-pre-wrap ml-2">
-						{cleanExpr(val.expr)}
-					</span>
-				{/if}
-			</li>
-		{/each}
-	</ul>
-{:else}
-	<div class="text-gray-600 text-sm"> No inputs </div>
-{/if}
+<ul class="mb-1">
+	{#each Object.entries(inputTransforms) as [key, val]}
+		<li>
+			<span class="font-black text-gray-700">{key}</span>: {#if val.type == 'static'}<ObjectViewer
+					json={val.value}
+				/>{:else}
+				<span class="text-xs">
+					<Highlight language={typescript} code={cleanExpr(val.expr)} />
+				</span>
+			{/if}
+		</li>
+	{/each}
+</ul>

frontend/src/lib/components/ItemPicker.svelte

@@ -16,8 +16,6 @@
 	export let noItemMessage = 'There are no items in the list'
 	/** Displayed if the search returns no items. */
 	export let buttons: Record<string, (x: string) => void> = {}
-	export let tooltip: string = ''
-	export let documentationLink: string | undefined = undefined
 
 	let loading = false
 	let items: Item[] | undefined = []
@@ -50,7 +48,7 @@
 />
 
 <Drawer bind:this={drawer} size="600px">
-	<DrawerContent {tooltip} {documentationLink} overflow_y={false} title="Search {itemName}s" on:close={drawer.closeDrawer}>
+	<DrawerContent overflow_y={false} title="Search {itemName}s" on:close={drawer.closeDrawer}>
 		<div class="w-full h-full flex flex-col">
 			<div class="w-12/12 pb-4">
 				<input