chore(main): release 1.83.1 (#1335)

* chore(main): release 1.83.1

* Apply automatic changes

---------

Co-authored-by: rubenfiszel <rubenfiszel@users.noreply.github.com>

.github/change-versions.sh

@@ -4,7 +4,7 @@ VERSION=$1
 echo "Updating versions to: $VERSION"
 
 sed -i -e "/^version =/s/= .*/= \"$VERSION\"/" backend/Cargo.toml
-sed -i -e "/^const VERSION =/s/= .*/= \"v$VERSION\";/" cli/main.ts
+sed -i -e "/^export const VERSION =/s/= .*/= \"v$VERSION\";/" cli/main.ts
 sed -i -e "/version: /s/: .*/: $VERSION/" backend/windmill-api/openapi.yaml
 sed -i -e "/version: /s/: .*/: $VERSION/" openflow.openapi.yaml
 sed -i -e "/\"version\": /s/: .*,/: \"$VERSION\",/" frontend/package.json

.github/workflows/deploy_to_windmill.yml

@@ -1,20 +0,0 @@
-name: Deploy to windmill.dev
-
-on:
-  push:
-    branches: [main]
-    paths:
-      - "community/**"
-
-jobs:
-  deploy:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: Deploy to windmill.dev
-        uses: windmill-labs/windmill-gh-action-deploy@v2.0.0
-        with:
-          dry_run: false
-          input_dir: community
-          windmill_workspace: starter
-          windmill_token: ${{ secrets.WINDMILL_API_TOKEN }}

.github/workflows/docker-image.yml

@@ -109,38 +109,38 @@ jobs:
             ${{ steps.meta-ee-public.outputs.labels }}
             org.opencontainers.image.licenses=Windmill-Enterprise-License
 
-
-  playwright:
-    runs-on: [self-hosted, new]
-    needs: [build]
-    services:
-      postgres:
-        image: postgres
-        env:
-          POSTGRES_DB: windmill
-          POSTGRES_USER: admin
-          POSTGRES_PASSWORD: changeme
-        ports:
-          - 5432:5432
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-    steps:
-      - uses: actions/checkout@v3
-      - name: "Docker"
-        run: echo "::set-output name=id::$(docker run --network=host --rm -d -p 8000:8000 --privileged -it -e DATABASE_URL=postgres://admin:changeme@localhost:5432/windmill -e BASE_INTERNAL_URL=http://localhost:8000 ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest)"
-        id: docker-container
-      - uses: actions/setup-node@v3
-        with:
-          node-version: 16
-      - name: "Playwright run"
-        timeout-minutes: 2
-        run: cd frontend && npm ci @playwright/test && npx playwright install && export BASE_URL=http://localhost:8000 && npm run test
-      - name: "Clean up"
-        run: docker kill ${{ steps.docker-container.outputs.id }}
-        if: always()
+  # disabled until we make it 100% reliable and add more meaningful tests
+  # playwright:
+  #   runs-on: [self-hosted, new]
+  #   needs: [build]
+  #   services:
+  #     postgres:
+  #       image: postgres
+  #       env:
+  #         POSTGRES_DB: windmill
+  #         POSTGRES_USER: admin
+  #         POSTGRES_PASSWORD: changeme
+  #       ports:
+  #         - 5432:5432
+  #       options: >-
+  #         --health-cmd pg_isready
+  #         --health-interval 10s
+  #         --health-timeout 5s
+  #         --health-retries 5
+  #   steps:
+  #     - uses: actions/checkout@v3
+  #     - name: "Docker"
+  #       run: echo "::set-output name=id::$(docker run --network=host --rm -d -p 8000:8000 --privileged -it -e DATABASE_URL=postgres://admin:changeme@localhost:5432/windmill -e BASE_INTERNAL_URL=http://localhost:8000 ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest)"
+  #       id: docker-container
+  #     - uses: actions/setup-node@v3
+  #       with:
+  #         node-version: 16
+  #     - name: "Playwright run"
+  #       timeout-minutes: 2
+  #       run: cd frontend && npm ci @playwright/test && npx playwright install && export BASE_URL=http://localhost:8000 && npm run test
+  #     - name: "Clean up"
+  #       run: docker kill ${{ steps.docker-container.outputs.id }}
+  #       if: always()
 
 
   publish_privately_heavy:

CHANGELOG.md

@@ -1,6 +1,279 @@
 # Changelog
 
 
+## [1.83.1](https://github.com/windmill-labs/windmill/compare/v1.83.0...v1.83.1) (2023-03-28)
+
+
+### Bug Fixes
+
+* **cli:** plain secrets might be undefined ([569a55e](https://github.com/windmill-labs/windmill/commit/569a55e45b34641b0fb4569387166f3aa89ce35f))
+
+## [1.83.0](https://github.com/windmill-labs/windmill/compare/v1.82.0...v1.83.0) (2023-03-28)
+
+
+### Features
+
+* **backend:** allow relative imports for python ([a5500ea](https://github.com/windmill-labs/windmill/commit/a5500ea40a77b2e0408e2a644190a8f65b18cd1d))
+* **backend:** execute /bin/bash instead of /bin/sh for bash scripts ([021fa23](https://github.com/windmill-labs/windmill/commit/021fa23f9ffcd11548977a4589eb9bc2815243cf))
+* **backend:** improve relative importsfor deno ([eaac598](https://github.com/windmill-labs/windmill/commit/eaac598af308cedea8f0f8fc7c189a4640b4366b))
+* **backend:** increase timeout for premium workspace ([00b70d9](https://github.com/windmill-labs/windmill/commit/00b70d9aaac8ae979782492d7754060a3c2c9567))
+* **frontend:** add pagination ([33c07d3](https://github.com/windmill-labs/windmill/commit/33c07d3e63f96673719ecb15e45f4cd9e18be80e))
+* **frontend:** Add quick style settings to app editor ([#1308](https://github.com/windmill-labs/windmill/issues/1308)) ([ac24862](https://github.com/windmill-labs/windmill/commit/ac2486219cd91df3a7fe11d37894797a881cac6c))
+* **frontend:** add recompute as a primitive ([449d3ae](https://github.com/windmill-labs/windmill/commit/449d3ae5ddeceef3fbcb7a815a4dba16c9639fb3))
+* **frontend:** add textareacomponent + fix multiselect style + select multi components ([2b31653](https://github.com/windmill-labs/windmill/commit/2b31653a8aa06807678e8609cfa62cf0f2f55dce))
+* **frontend:** multiselect components for apps ([577dec5](https://github.com/windmill-labs/windmill/commit/577dec5c5733cdf88e8586ce6c27159920c69c8a))
+* **frontend:** use rich json editor for arrays of objects and for object in ArgInput ([b95afaa](https://github.com/windmill-labs/windmill/commit/b95afaa9bb41b102181657453a564f44f4511983))
+
+
+### Bug Fixes
+
+* **apps:** improve app table actionButtons behavior under many clicks ([8e3d8ac](https://github.com/windmill-labs/windmill/commit/8e3d8acc80de971ee115d6903d24864d8263f08b))
+* **cli:** add --plain-secrets ([98d51e2](https://github.com/windmill-labs/windmill/commit/98d51e219df1680507114f9b57ec0b0a4a234b5c))
+* **frontend:** add a modal that is always mounted to make sure compon… ([#1328](https://github.com/windmill-labs/windmill/issues/1328)) ([a527cb8](https://github.com/windmill-labs/windmill/commit/a527cb8222a2ff80dae38ebae7dc5ea0979d74c5))
+* **frontend:** Disable app keyboard navigation on focused inputs ([#1326](https://github.com/windmill-labs/windmill/issues/1326)) ([da24e9a](https://github.com/windmill-labs/windmill/commit/da24e9ab0625a7503c498c179022ea4011a03170))
+* **frontend:** persist description for schemas ([1a48673](https://github.com/windmill-labs/windmill/commit/1a4867302f72aaae8f422ac8f53812c116cc383d))
+* **frontend:** Revert app upload input ([#1330](https://github.com/windmill-labs/windmill/issues/1330)) ([fa457bb](https://github.com/windmill-labs/windmill/commit/fa457bb7099bd31c2315eaf7f7f2c40900b2ae39))
+* **frontend:** Small app fixes ([#1331](https://github.com/windmill-labs/windmill/issues/1331)) ([75306c8](https://github.com/windmill-labs/windmill/commit/75306c831616d9a01cc3a4681732aab93153f1a9))
+
+## [1.82.0](https://github.com/windmill-labs/windmill/compare/v1.81.0...v1.82.0) (2023-03-24)
+
+
+### Features
+
+* **backend:** introduce RESTART_ZOMBIE_JOBS and ZOMBIE_JOB_TIMEOUT ([47a7f71](https://github.com/windmill-labs/windmill/commit/47a7f7163aae3fe807e766c824085b4d1b75c8c8))
+
+
+### Bug Fixes
+
+* **backend:** do not consider FlowPreview as potential zombie job ([f7c30b5](https://github.com/windmill-labs/windmill/commit/f7c30b5d2f16e15f36208e07126557fd7ed84801))
+* **backend:** increase dynamic js timeout + improve client passing ([34e25f0](https://github.com/windmill-labs/windmill/commit/34e25f0f96fe637cc42f4017a064c40def5d67ef))
+* **cli:** improve diff speed + fix replacing cli ([b999c98](https://github.com/windmill-labs/windmill/commit/b999c9894b4011b735f37df485fe403c22c00512))
+* **frontend:** Fix AppTable error display + clear errors when removing a component + properly detect that latest component run had an error ([#1322](https://github.com/windmill-labs/windmill/issues/1322)) ([c15bc8a](https://github.com/windmill-labs/windmill/commit/c15bc8a7bfb3bef2634e6093088967137cd06239))
+* **frontend:** fix refresh with manual dependencies ([#1319](https://github.com/windmill-labs/windmill/issues/1319)) ([a47031a](https://github.com/windmill-labs/windmill/commit/a47031a41e6a3392101e280dcd1aea098f898447))
+* **frontend:** fix settings panel ([#1323](https://github.com/windmill-labs/windmill/issues/1323)) ([30b8e47](https://github.com/windmill-labs/windmill/commit/30b8e474df5b71b7e7b36d3fe5974a289cf0dfae))
+* **frontend:** Fix transformer ([#1321](https://github.com/windmill-labs/windmill/issues/1321)) ([addabcc](https://github.com/windmill-labs/windmill/commit/addabcceb0c90782ba4a934bb3822f8cc9865069))
+* **frontend:** remove unnecessary div ([#1318](https://github.com/windmill-labs/windmill/issues/1318)) ([e193a0b](https://github.com/windmill-labs/windmill/commit/e193a0bcdf6690b007594d2f1325a7ec26603129))
+
+## [1.81.0](https://github.com/windmill-labs/windmill/compare/v1.80.1...v1.81.0) (2023-03-21)
+
+
+### Features
+
+* **apps:** add action on form/button/formbutton ([2593218](https://github.com/windmill-labs/windmill/commit/2593218cbf07c05521a270797055ddb22dc22b8d))
+
+
+### Bug Fixes
+
+* **frontend:** Remove action outline on preview mode ([#1313](https://github.com/windmill-labs/windmill/issues/1313)) ([a7c4f1a](https://github.com/windmill-labs/windmill/commit/a7c4f1a12e02e8627a5955b75d572e9cf11d8122))
+
+## [1.80.1](https://github.com/windmill-labs/windmill/compare/v1.80.0...v1.80.1) (2023-03-21)
+
+
+### Bug Fixes
+
+* **cli:** add support for non metadataed scripts ([42f6d2e](https://github.com/windmill-labs/windmill/commit/42f6d2e0ee6294f8a1d97f5f62f2adb6edfd2fed))
+
+## [1.80.0](https://github.com/windmill-labs/windmill/compare/v1.79.0...v1.80.0) (2023-03-20)
+
+
+### Features
+
+* **apps:** add transformers for data sources ([0abacac](https://github.com/windmill-labs/windmill/commit/0abacac06c7dd586b48c66ff47b7589fe692205b))
+* **frontend:** App set tab ([#1307](https://github.com/windmill-labs/windmill/issues/1307)) ([48413a7](https://github.com/windmill-labs/windmill/commit/48413a78c5e7e0ee8208711f15135d81136b7386))
+
+
+### Bug Fixes
+
+* **frontend:** add missing optional chaining ([#1306](https://github.com/windmill-labs/windmill/issues/1306)) ([29b1cc6](https://github.com/windmill-labs/windmill/commit/29b1cc6ff0ebc5edcad24a1780113889c507075d))
+* **frontend:** App button triggered by ([#1304](https://github.com/windmill-labs/windmill/issues/1304)) ([cf2d031](https://github.com/windmill-labs/windmill/commit/cf2d031e8e89faa2cd7fa58436cbe7cf4d9045f9))
+
+## [1.79.0](https://github.com/windmill-labs/windmill/compare/v1.78.0...v1.79.0) (2023-03-17)
+
+
+### Features
+
+* **frontend:** add listeners for frontend scripts ([597e38e](https://github.com/windmill-labs/windmill/commit/597e38ef367d38fa97fc443ccb2c721e5964fece))
+* **frontend:** add table actions navigation ([#1298](https://github.com/windmill-labs/windmill/issues/1298)) ([c3ba1a6](https://github.com/windmill-labs/windmill/commit/c3ba1a6ab97484a08a5a20187bb858a5af7025cb))
+* **frontend:** App component triggers ([#1303](https://github.com/windmill-labs/windmill/issues/1303)) ([078cb1b](https://github.com/windmill-labs/windmill/commit/078cb1bf3e4de08cb018578f04d24392a6462f69))
+* **frontend:** Component control ([#1293](https://github.com/windmill-labs/windmill/issues/1293)) ([bd927a2](https://github.com/windmill-labs/windmill/commit/bd927a27ed9581dbf67ea3694f9d989f8d71d2ed))
+
+
+### Bug Fixes
+
+* **frontend:** App panel styling ([#1284](https://github.com/windmill-labs/windmill/issues/1284)) ([c1dd35c](https://github.com/windmill-labs/windmill/commit/c1dd35c3f0fcbc1be43273f82a873c3c07863417))
+* **frontend:** Display app context search on top ([#1300](https://github.com/windmill-labs/windmill/issues/1300)) ([bd3ee81](https://github.com/windmill-labs/windmill/commit/bd3ee81b14846f16ccd16461de99b46fe68be6ba))
+* **frontend:** fix horizontal splitpanes ([#1301](https://github.com/windmill-labs/windmill/issues/1301)) ([ea3dab4](https://github.com/windmill-labs/windmill/commit/ea3dab411b3d5dd772e04c8831e789e2470aaf28))
+* **frontend:** fix map render ([#1297](https://github.com/windmill-labs/windmill/issues/1297)) ([0092721](https://github.com/windmill-labs/windmill/commit/00927210fd68c31cb793ef4f0efea05711ebcf00))
+* **frontend:** Hide archive toggle with empty list ([#1296](https://github.com/windmill-labs/windmill/issues/1296)) ([bac831b](https://github.com/windmill-labs/windmill/commit/bac831b23ce85a683ddbd4537900670a0b7d12a8))
+
+## [1.78.0](https://github.com/windmill-labs/windmill/compare/v1.77.0...v1.78.0) (2023-03-16)
+
+
+### Features
+
+* **frontend:** app textcomponent editable + tooltip ([11567d6](https://github.com/windmill-labs/windmill/commit/11567d6280ea60f1a8c3c6607c724179775cbbe3))
+
+
+### Bug Fixes
+
+* **backend:** whitelist for include_header was ignored in some cases ([183a459](https://github.com/windmill-labs/windmill/commit/183a4591df700ab4720de6e92a83631256940089))
+* **frontend:** improve rendering performance after component moving ([6f890f2](https://github.com/windmill-labs/windmill/commit/6f890f2120885f90d986fbd655096b45bf9de539))
+* **frontend:** remove staticOutputs from apps ([dbdfd62](https://github.com/windmill-labs/windmill/commit/dbdfd626386398180ecba7976714f86365eeccd8))
+
+## [1.77.0](https://github.com/windmill-labs/windmill/compare/v1.76.0...v1.77.0) (2023-03-14)
+
+
+### Features
+
+* **apps:** state can be used as input in apps ([2f0acb9](https://github.com/windmill-labs/windmill/commit/2f0acb9ffa8dace4a886527dcee49809d019b271))
+* **apps:** tabs can be made pages or invisible + better frontend scripts reactivity ([cd645d0](https://github.com/windmill-labs/windmill/commit/cd645d0935f2d06e0ff71f14d2cf63accd378ff3))
+* **deno:** add support for custom npm repo ([#1291](https://github.com/windmill-labs/windmill/issues/1291)) ([944795f](https://github.com/windmill-labs/windmill/commit/944795f6eeaa7d01ab1a35a80570a55c363723e6))
+* **frontend:** add setTab to frontend scripts ([c2a97c5](https://github.com/windmill-labs/windmill/commit/c2a97c53cfff0fdb35dd8bc249490566eebdc1a9))
+* **frontend:** app components output panel ([#1283](https://github.com/windmill-labs/windmill/issues/1283)) ([751edcf](https://github.com/windmill-labs/windmill/commit/751edcf9b8e0976a1d073603c9eff5dc6e714490))
+
+
+### Bug Fixes
+
+* **backend:** do not cache reference to workspace scripts ([eb73f2a](https://github.com/windmill-labs/windmill/commit/eb73f2a687f6faad301b9038ab8585450bec7481))
+* **frontend:** fix app tabs ([#1288](https://github.com/windmill-labs/windmill/issues/1288)) ([c71a577](https://github.com/windmill-labs/windmill/commit/c71a577fead90c9cd01a736b54d859ec4f0b7807))
+* **frontend:** fix container deletion ([#1287](https://github.com/windmill-labs/windmill/issues/1287)) ([bc870bd](https://github.com/windmill-labs/windmill/commit/bc870bd03eb76cb8bc0e0c861f6cd8a9c661186b))
+* **frontend:** Update setting accordion ([#1285](https://github.com/windmill-labs/windmill/issues/1285)) ([dea12e8](https://github.com/windmill-labs/windmill/commit/dea12e8870ece998bb6607723cbaab9b9a958f22))
+
+## [1.76.0](https://github.com/windmill-labs/windmill/compare/v1.75.0...v1.76.0) (2023-03-13)
+
+
+### Features
+
+* **frontend:** add frontend (JS)  scripts to apps ([f0b1b1f](https://github.com/windmill-labs/windmill/commit/f0b1b1f752731ba434b960a75624118152f53c00))
+* **frontend:** Copy, Cut and Paste ([#1279](https://github.com/windmill-labs/windmill/issues/1279)) ([82c139e](https://github.com/windmill-labs/windmill/commit/82c139ed0992be401e250cfb7ecc0fca61b76772))
+* **frontend:** disabled for action buttons can now depend on row ([75f87e7](https://github.com/windmill-labs/windmill/commit/75f87e7e1117a9c12afcf626379e94b134a9a493))
+* **frontend:** improve drag-n-drop behavior ([cfd489a](https://github.com/windmill-labs/windmill/commit/cfd489a55059e7b6843f99bab261c70b3852e6a2))
+
+
+### Bug Fixes
+
+* **backend:** improve worker ping api ([c958480](https://github.com/windmill-labs/windmill/commit/c958480ce83844a989f58dd5a70eb288582e2194))
+* **frontend:** General fixes and updates ([#1281](https://github.com/windmill-labs/windmill/issues/1281)) ([3e5a179](https://github.com/windmill-labs/windmill/commit/3e5a179eb8cd8001f49c92305141dade1571e20f))
+
+## [1.75.0](https://github.com/windmill-labs/windmill/compare/v1.74.2...v1.75.0) (2023-03-11)
+
+
+### Features
+
+* add filter jobs by args or result ([3b44f9a](https://github.com/windmill-labs/windmill/commit/3b44f9a72ca0466a44963a4b9657a0ee59b44753))
+* **apps:** add resource picker ([8681e83](https://github.com/windmill-labs/windmill/commit/8681e83b574141acbf7e5a389a9e8a4f340336d1))
+* **bash:** add default argument handling for bash ([1d5c194](https://github.com/windmill-labs/windmill/commit/1d5c194f09ffba963d52e418c5954843d84ae337))
+* **frontend-apps:** add variable picker for static string input on apps ([bc440f8](https://github.com/windmill-labs/windmill/commit/bc440f8d4154ce464c0e027d93b7a0a3b76d782e))
+* **frontend:** make runs filters synced with query args ([61a5e1f](https://github.com/windmill-labs/windmill/commit/61a5e1f1accc988628b785b3b9be04c4ea719874))
+
+
+### Bug Fixes
+
+* **backend:** add killpill for lines reading ([7c825c2](https://github.com/windmill-labs/windmill/commit/7c825c212dd0f1e8be427eabd9a9756303241d1b))
+* **cli:** many small fixes ([ce32370](https://github.com/windmill-labs/windmill/commit/ce323709a94d27fb24214719180ea1aafc66d646))
+
+## [1.74.2](https://github.com/windmill-labs/windmill/compare/v1.74.1...v1.74.2) (2023-03-09)
+
+
+### Bug Fixes
+
+* **frontend:** fix splitpanes navigation ([#1276](https://github.com/windmill-labs/windmill/issues/1276)) ([8d5c5b8](https://github.com/windmill-labs/windmill/commit/8d5c5b88a35d7a3bad1d8ddf2d940026825241eb))
+
+## [1.74.1](https://github.com/windmill-labs/windmill/compare/v1.74.0...v1.74.1) (2023-03-09)
+
+
+### Bug Fixes
+
+* **apps:** proper reactivity for non rendered static components ([ae53baf](https://github.com/windmill-labs/windmill/commit/ae53bafaf6777f928113f84b2c6ed6a2ed341844))
+* **ci:** make windmill compile again by pinning swc deps ([2ea15d5](https://github.com/windmill-labs/windmill/commit/2ea15d5035e5e15473968db3c0501a4dddff5cd0))
+
+## [1.74.0](https://github.com/windmill-labs/windmill/compare/v1.73.1...v1.74.0) (2023-03-09)
+
+
+### Features
+
+* add delete by path for scripts ([0c2cf92](https://github.com/windmill-labs/windmill/commit/0c2cf92dd3df9610e649f15e23921a4ca0d94e6a))
+* **frontend:** Add color picker input to app ([#1270](https://github.com/windmill-labs/windmill/issues/1270)) ([88e537a](https://github.com/windmill-labs/windmill/commit/88e537ad1fb4c207f38fbe951c82106bef6491a3))
+* **frontend:** add expand ([#1268](https://github.com/windmill-labs/windmill/issues/1268)) ([b854ee3](https://github.com/windmill-labs/windmill/commit/b854ee34393534bde104e2e6f606108fd66d38dc))
+* **frontend:** add hash to ctx in apps ([b1a45b1](https://github.com/windmill-labs/windmill/commit/b1a45b1e708aa6f19f8be9c949507083e044f2d8))
+* **frontend:** Add key navigation in app editor ([#1273](https://github.com/windmill-labs/windmill/issues/1273)) ([6b0fb75](https://github.com/windmill-labs/windmill/commit/6b0fb75d23e2151c88b07814139d203c1bd0578d))
+
+
+### Bug Fixes
+
+* **cli:** improve visibility of the active workspace ([e6344da](https://github.com/windmill-labs/windmill/commit/e6344dac6d1be04b46231fa8ef8579fd12ca8f37))
+* **frontend:** add confirmation modal to delete script/flow/app ([a4adcb5](https://github.com/windmill-labs/windmill/commit/a4adcb5192c11f7bf47a0d259825e474779378d7))
+* **frontend:** Clean up app editor ([#1267](https://github.com/windmill-labs/windmill/issues/1267)) ([0a5e181](https://github.com/windmill-labs/windmill/commit/0a5e181a3aa966fb8211bee0d9174fc16353b31f))
+* **frontend:** Minor changes ([#1272](https://github.com/windmill-labs/windmill/issues/1272)) ([3b6ae0c](https://github.com/windmill-labs/windmill/commit/3b6ae0cc49461b858d9cfff79eae9a7569465235))
+* **frontend:** simplify input bindings ([b2de531](https://github.com/windmill-labs/windmill/commit/b2de531a46e4b120d7106d361b727746bec516dd))
+
+## [1.73.1](https://github.com/windmill-labs/windmill/compare/v1.73.0...v1.73.1) (2023-03-07)
+
+
+### Bug Fixes
+
+* **frontend:** load flow is not initialized ([719d475](https://github.com/windmill-labs/windmill/commit/719d4752621d462b1cfaa0d27930fba7586be779))
+
+## [1.73.0](https://github.com/windmill-labs/windmill/compare/v1.72.0...v1.73.0) (2023-03-07)
+
+
+### Features
+
+* **frontend:** add a way to automatically resize ([#1259](https://github.com/windmill-labs/windmill/issues/1259)) ([24f58ef](https://github.com/windmill-labs/windmill/commit/24f58efd9994a2201c1b1d9bbfb11734c57068e3))
+* **frontend:** add ability to move nodes ([614fb50](https://github.com/windmill-labs/windmill/commit/614fb5022aa7d5428fb96b7ee3a20794edd1e9d3))
+* **frontend:** Add app PDF viewer ([#1254](https://github.com/windmill-labs/windmill/issues/1254)) ([3e5d09e](https://github.com/windmill-labs/windmill/commit/3e5d09ef0b5619186bee5ec6d442cbfd12a6e8d5))
+* **frontend:** add fork/save buttons + consistent styling for slider/range ([9e9f8ef](https://github.com/windmill-labs/windmill/commit/9e9f8efb8ee389ea75e99b67ef720756959ca737))
+* **frontend:** add history to flows and apps ([9e4d90a](https://github.com/windmill-labs/windmill/commit/9e4d90ad37a57ff1f515eea0c82cf603649e915d))
+* **frontend:** Fix object viewer style ([#1255](https://github.com/windmill-labs/windmill/issues/1255)) ([94f1aad](https://github.com/windmill-labs/windmill/commit/94f1aadef2b09ac1962478f11b27cc708b8328f1))
+* **frontend:** refactor entire flow builder UX ([2ac51b0](https://github.com/windmill-labs/windmill/commit/2ac51b0af08bdef7ce3c7e874e9983b9fc00478a))
+
+
+### Bug Fixes
+
+* **frontend:** arginput + apppreview fixes ([e2c4545](https://github.com/windmill-labs/windmill/commit/e2c45452401022b00285b21551ffaf35a114be33))
+* **frontend:** fix app map reactivity ([#1260](https://github.com/windmill-labs/windmill/issues/1260)) ([2557e13](https://github.com/windmill-labs/windmill/commit/2557e136bd0df1a023819b7d9b2235e30d7140b6))
+* **frontend:** fix branch deletion ([#1261](https://github.com/windmill-labs/windmill/issues/1261)) ([a999eb2](https://github.com/windmill-labs/windmill/commit/a999eb21121a7c0010621448324e0c77caf2b3f6))
+* **frontend:** Side menu z-index issue ([#1265](https://github.com/windmill-labs/windmill/issues/1265)) ([c638897](https://github.com/windmill-labs/windmill/commit/c638897fdcd58f55b0929f91641b21a6f9d25ead))
+
+## [1.72.0](https://github.com/windmill-labs/windmill/compare/v1.71.0...v1.72.0) (2023-03-02)
+
+
+### Features
+
+* **backend:** get_result_by_id do a downward pass to find node at any depth ([#1249](https://github.com/windmill-labs/windmill/issues/1249)) ([4c913dc](https://github.com/windmill-labs/windmill/commit/4c913dc4b6be03571a015c97a13829adffb61479))
+* **frontend:** Add app map component ([#1251](https://github.com/windmill-labs/windmill/issues/1251)) ([ed25d9f](https://github.com/windmill-labs/windmill/commit/ed25d9f186d9925f75404cb193a025d8a41c4540))
+* **frontend:** app splitpanes ([#1248](https://github.com/windmill-labs/windmill/issues/1248)) ([f4d79ee](https://github.com/windmill-labs/windmill/commit/f4d79ee2633e6cdab0fa2410108b31cfa77e10da))
+
+
+### Bug Fixes
+
+* **backend:** improve result retrieval ([c4463bb](https://github.com/windmill-labs/windmill/commit/c4463bb029907f3c8d77abb194f872aae7876bf6))
+* **backend:** incorrect get_result_by_id for  list_result job ([2a75cd2](https://github.com/windmill-labs/windmill/commit/2a75cd250ea5e01849fc8bbb69bf44f147d0acb8))
+* **cli:** fix workspace option + run script/flow + whoami ([35ea2b2](https://github.com/windmill-labs/windmill/commit/35ea2b27b12159c68c8507ec1f8686028c975387))
+* **frontend:** background script not showing inputs ([55eb48c](https://github.com/windmill-labs/windmill/commit/55eb48c55332431304cedbf3bcbbbcff61ec3645))
+* **frontend:** fix table bindings ([2679386](https://github.com/windmill-labs/windmill/commit/2679386bf87a56352269911bd89e52df5ee9f314))
+* **frontend:** rework app reactivity ([94b20d2](https://github.com/windmill-labs/windmill/commit/94b20d2f5e3b551974c57ea82b6e3dc16e97b9b8))
+* **frontend:** rework app reactivity ([1753cb7](https://github.com/windmill-labs/windmill/commit/1753cb7da658f47be974c15da82c71a8e19309a6))
+
+## [1.71.0](https://github.com/windmill-labs/windmill/compare/v1.70.1...v1.71.0) (2023-02-28)
+
+
+### Features
+
+* **backend:** use counter for sleep/execution/pull durations ([e568690](https://github.com/windmill-labs/windmill/commit/e56869092a03fec4703ddd9ef65c89edb8122962))
+* **cli:** add autocompletions ([287b2db](https://github.com/windmill-labs/windmill/commit/287b2db22f7b56e90bcd0c4727c00096695c2e0d))
+* **frontend:** App drawer ([#1246](https://github.com/windmill-labs/windmill/issues/1246)) ([8a0d115](https://github.com/windmill-labs/windmill/commit/8a0d1158c4d7e970cb91e1adf4838e5efdbb39ff))
+* **frontend:** drawer for editing workspace scripts in flows ([6adc875](https://github.com/windmill-labs/windmill/commit/6adc87561070d8aceaba1838008cd7e6be2e2660))
+
+
+### Bug Fixes
+
+* **frontend:** Add more app custom css ([#1229](https://github.com/windmill-labs/windmill/issues/1229)) ([a4e4d18](https://github.com/windmill-labs/windmill/commit/a4e4d188ad10443dd0b7f104389594efc768dc59))
+* **frontend:** Add more app custom css ([#1247](https://github.com/windmill-labs/windmill/issues/1247)) ([1bb5ed9](https://github.com/windmill-labs/windmill/commit/1bb5ed9ae01fd7998b06833b6222e5dd5d774d35))
+* **frontend:** display currently selected filter even if not in list ([42d1cd6](https://github.com/windmill-labs/windmill/commit/42d1cd6456620ba917c560c87d736dc93634adff))
+* **frontend:** Fix deeply nested move ([#1245](https://github.com/windmill-labs/windmill/issues/1245)) ([a67f10e](https://github.com/windmill-labs/windmill/commit/a67f10eeb6fdb44bbb3a510badcc5ad0ae187a2b))
+* **frontend:** invisible subgrids have h-0 + app policies fix ([2244e83](https://github.com/windmill-labs/windmill/commit/2244e83b9da803a4cf46ab0825d7cb6cb0e24872))
+
 ## [1.70.1](https://github.com/windmill-labs/windmill/compare/v1.70.0...v1.70.1) (2023-02-27)
 
 

Caddyfile

@@ -11,5 +11,5 @@ http://{$BASE_URL} {
 https://{$BASE_URL}  {  
   bind {$ADDRESS}
   reverse_proxy /ws/* http://lsp:3001
-}
+}
 }

Dockerfile

@@ -73,7 +73,7 @@ ARG features=""
 
 COPY --from=planner /windmill/recipe.json recipe.json
 
-RUN CARGO_NET_GIT_FETCH_WITH_CLI=true cargo chef cook --release --features "$features" --recipe-path recipe.json
+RUN CARGO_NET_GIT_FETCH_WITH_CLI=true RUST_BACKTRACE=1 cargo chef cook --release --features "$features" --recipe-path recipe.json
 
 COPY ./openflow.openapi.yaml /openflow.openapi.yaml
 COPY ./backend ./
@@ -86,6 +86,7 @@ RUN CARGO_NET_GIT_FETCH_WITH_CLI=true cargo build --release --features "$feature
 
 
 FROM python:3.11.2-slim-buster
+ARG TARGETPLATFORM
 
 ARG APP=/usr/src/app
 
@@ -129,6 +130,10 @@ COPY --from=nsjail /nsjail/nsjail /bin/nsjail
 
 COPY --from=denoland/deno:latest /usr/bin/deno /usr/bin/deno
 
+# docker does not support conditional COPY and we want to use the same Dockerfile for both amd64 and arm64 and privilege the official image
+COPY --from=lukechannings/deno:latest /usr/bin/deno /usr/bin/deno-arm
+RUN if [ "$TARGETPLATFORM" = "linux/amd64" ]; then rm /usr/bin/deno-arm; elif [ "$TARGETPLATFORM" = "linux/arm64" ]; then mv /usr/bin/deno-arm /usr/bin/deno; fi
+
 RUN mkdir -p ${APP}
 
 WORKDIR ${APP}

LICENSE

@@ -8,5 +8,9 @@ or belonging to one of the below cases:
 
 The files under backend/ are AGPL Licensed.
 The files under frontend/ are AGPL Licensed.
-The files under python-client/ are Apache 2.0 Licensed.
-The files under community/ are Apache 2.0 Licensed.
+The files under python-client/ deno-client/ go-client/ are Apache 2.0 Licensed.
+
+The openapi files, including the OpenFlow spec is Apache 2.0 Licensed.
+
+All third party components incorporated into the Windmill Software are licensed under the
+original license provided by the owner of the applicable component.

README.md

@@ -26,7 +26,8 @@ Open-source developer infrastructure for internal tools. Self-hostable alternati
 
 # Windmill - Turn scripts into workflows and UIs that you can share and run at scale
 
-Windmill is <b>fully open-sourced (AGPLv3)</b> and Windmill Labs offers dedicated instance and commercial support and licenses.
+Windmill is <b>fully open-sourced (AGPLv3)</b> and Windmill Labs offers
+dedicated instance and commercial support and licenses.
 
 ![Windmill Diagram](/imgs/stacks.svg)
 
@@ -74,7 +75,7 @@ https://user-images.githubusercontent.com/275584/218350457-bc2fdc3b-e667-4da5-a2
 4. Build complex UI on top of your scripts and flows.
    ![Step 5](./imgs/windmill-builder.png)
 
-Scripts and flows can also be triggered by a cron schedule '*/5 * * * *' or
+Scripts and flows can also be triggered by a cron schedule '_/5 _ \* \* \*' or
 through webhooks.
 
 You can build your entire infra on top of Windmill!
@@ -82,46 +83,49 @@ You can build your entire infra on top of Windmill!
 ## Show me some actual script code
 
 ```typescript
-import * as wmill from "https://deno.land/x/windmill@v1.62.0/mod.ts"
+import * as wmill from "https://deno.land/x/windmill@v1.62.0/mod.ts";
 //import any dependency  from npm
 
-import cowsay from 'npm:cowsay@1.5.0'
+import cowsay from "npm:cowsay@1.5.0";
 
 export async function main(
-    a: number,
-    // unions generate enums
-    b: "my" | "enum",
-    // default parameters prefill the field 
-    d = "default arg",
-    // nested objects work c = { nested: "object" }, 
-    // permissioned and typed json
-    db: wmill.Resource<"postgresql">) {
+  a: number,
+  // unions generate enums
+  b: "my" | "enum",
+  // default parameters prefill the field
+  d = "default arg",
+  // nested objects work c = { nested: "object" },
+  // permissioned and typed json
+  db: wmill.Resource<"postgresql">
+) {
+  const email = Deno.env.get("WM_EMAIL");
+  // variables are permissioned and by path
+  let variable = await wmill.getVariable("f/company-folder/my_secret");
+  const lastTimeRun = await wmill.getState();
+  // logs are printed and always inspectable
+  console.log(cowsay.say({ text: "hello " + email + " " + lastTimeRun }));
+  await wmill.setState(Date.now());
 
-    const email = Deno.env.get('WM_EMAIL')
-    // variables are permissioned and by path
-    let variable = await wmill.getVariable('f/company-folder/my_secret')
-    const lastTimeRun = await wmill.getState()
-    // logs are printed and always inspectable
-    console.log(cowsay.say({ text: "hello " + email + " " + lastTimeRun }))
-    await wmill.setState(Date.now())
-
-    // return is serialized as JSON
-    return { foo: d, variable };
+  // return is serialized as JSON
+  return { foo: d, variable };
 }
 ```
 
 ## CLI
 
 We have a powerful CLI to interact with the windmill platform and sync your
-scripts from local files, github repos and to run scripts and flows on the instance from local commands. See
+scripts from local files, github repos and to run scripts and flows on the
+instance from local commands. See
 [more details](https://github.com/windmill-labs/windmill/tree/main/cli)
 
 ![CLI Screencast](./cli/vhs/output/setup.gif)
 
-
 ### Running scripts locally
 
-You can run your script locally easily, you simply need to pass the right environment variables for the `wmill` client library to fetch resource and variables from your instance if necessary. See more: <https://docs.windmill.dev/docs/advanced/local_development/>
+You can run your script locally easily, you simply need to pass the right
+environment variables for the `wmill` client library to fetch resource and
+variables from your instance if necessary. See more:
+<https://docs.windmill.dev/docs/advanced/local_development/>
 
 ## Stack
 
@@ -187,7 +191,6 @@ compiling from source or using without a postgres super user, see
 
 Go to http://localhost et voilà :)
 
-
 The default super-admin user is: admin@windmill.dev / changeme
 
 From there, you can create other users (do not forget to change the password!)
@@ -199,9 +202,9 @@ We publish helm charts at:
 
 ### Postgres without superuser
 
-If you do not want, or cannot (for instance, in AWS Aurora or Cloud sql) use a postgres superuser,
-you can run `./init-db-as-superuser.sql` to init the required users for windmill.
-
+If you do not want, or cannot (for instance, in AWS Aurora or Cloud sql) use a
+postgres superuser, you can run `./init-db-as-superuser.sql` to init the
+required users for windmill.
 
 ### Commercial license
 
@@ -275,8 +278,8 @@ You may also add your own custom OAuth2 IdP and OAuth2 Resource provider:
 ### Resource types
 
 You will also want to import all the approved resource types from
-[WindmillHub](https://hub.windmill.dev). A setup script will prompt
-you to have it being synced automatically everyday.
+[WindmillHub](https://hub.windmill.dev). A setup script will prompt you to have
+it being synced automatically everyday.
 
 ## Environment Variables
 
@@ -284,14 +287,17 @@ you to have it being synced automatically everyday.
 | ------------------------- | ---------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------- |
 | DATABASE_URL              |                        | The Postgres database url.                                                                                                                                                                         | All                   |
 | DISABLE_NSJAIL            | true                   | Disable Nsjail Sandboxing                                                                                                                                                                          | Worker                |
-| PORT                      | 8000                   | Exposed port                                                                                                                                                                                       | Server                |  |
+| SERVER_BIND_ADDR          | 0.0.0.0                | IP Address on which to bind listening socket                                                                                                                                                       | Server                |
+| PORT                      | 8000                   | Exposed port                                                                                                                                                                                       | Server                |
 | NUM_WORKERS               | 3                      | The number of worker per Worker instance (set to 1 on Eks to have 1 pod = 1 worker, set to 0 for an API only instance)                                                                             | Worker                |
 | DISABLE_SERVER            | false                  | Binary would operate as a worker only instance                                                                                                                                                     | Worker                |
 | METRICS_ADDR              | None                   | The socket addr at which to expose Prometheus metrics at the /metrics path. Set to "true" to expose it on port 8001                                                                                | All                   |
 | JSON_FMT                  | false                  | Output the logs in json format instead of logfmt                                                                                                                                                   | All                   |
 | BASE_URL                  | http://localhost:8000  | The base url that is exposed publicly to access your instance                                                                                                                                      | Server                |
 | BASE_INTERNAL_URL         | http://localhost:8000  | The base url that is reachable by your workers to talk to the Servers. This help avoiding going through the external load balancer for VPC-internal requests.                                      | Worker                |
-| TIMEOUT                   | 300                    | The timeout in seconds for the execution of a script                                                                                                                                               | Worker                |
+| TIMEOUT                   | 300                    | The maximum time of execution of a script. When reached, the job is failed as having timedout.                                                                                                     | Worker                |
+| ZOMBIE_JOB_TIMEOUT        | 30                     | The timeout after which a job is considered to be zombie if the worker did not send pings about processing the job (every server check for zombie jobs every 30s)                                  | Server                |
+| RESTART_ZOMBIE_JOBS       | true                   | If true then a zombie job is restarted (in-place with the same uuid and some logs), if false the zombie job is failed                                                                              | Server                |
 | SLEEP_QUEUE               | 50                     | The number of ms to sleep in between the last check for new jobs in the DB. It is multiplied by NUM_WORKERS such that in average, for one worker instance, there is one pull every SLEEP_QUEUE ms. | Worker                |
 | MAX_LOG_SIZE              | 500000                 | The maximum number of characters a job can emit (log + result)                                                                                                                                     | Worker                |
 | DISABLE_NUSER             | false                  | If Nsjail is enabled, disable the nsjail's `clone_newuser` setting                                                                                                                                 | Worker                |
@@ -300,7 +306,7 @@ you to have it being synced automatically everyday.
 | S3_CACHE_BUCKET (EE only) | None                   | The S3 bucket to sync the cache of the workers to                                                                                                                                                  | Worker                |
 | TAR_CACHE_RATE (EE only)  | 100                    | The rate at which to tar the cache of the workers. 100 means every 100th job in average (uniformly randomly distributed).                                                                          | Worker                |
 | SLACK_SIGNING_SECRET      | None                   | The signing secret of your Slack app. See [Slack documentation](https://api.slack.com/authentication/verifying-requests-from-slack)                                                                | Server                |
-| COOKIE_DOMAIN             | None                   | The domain of the cookie. If not set, the cookie will be set by the browser based on the full origin                                                                                               | Server                |  |
+| COOKIE_DOMAIN             | None                   | The domain of the cookie. If not set, the cookie will be set by the browser based on the full origin                                                                                               | Server                |
 | DENO_PATH                 | /usr/bin/deno          | The path to the deno binary.                                                                                                                                                                       | Worker                |
 | PYTHON_PATH               | /usr/local/bin/python3 | The path to the python binary.                                                                                                                                                                     | Worker                |
 | GO_PATH                   | /usr/bin/go            | The path to the go binary.                                                                                                                                                                         | Worker                |
@@ -315,6 +321,7 @@ you to have it being synced automatically everyday.
 | QUEUE_LIMIT_WAIT_RESULT   | None                   | The number of max jobs in the queue before rejecting immediately the request in 'run_wait_result' endpoint. Takes precedence on the query arg. If none is specified, there are no limit.           | Worker                |
 | DENO_AUTH_TOKENS          | None                   | Custom DENO_AUTH_TOKENS to pass to worker to allow the use of private modules                                                                                                                      | Worker                |
 | DENO_FLAGS                | None                   | Override the flags passed to deno (default --allow-all) to tighten permissions. Minimum permissions needed are "--allow-read=args.json --allow-write=result.json"                                  | Worker                |
+| NPM_CONFIG_REGISTRY       | None                   | Registry to use for NPM dependencies, set if you have a private repository you need to use instead of the default public NPM registry                                                              | Worker                |
 | PIP_LOCAL_DEPENDENCIES    | None                   | Specify dependencies that are installed locally and do not need to be solved nor installed again                                                                                                   |                       |
 | ADDITIONAL_PYTHON_PATHS   | None                   | Specify python paths (separated by a :) to be appended to the PYTHONPATH of the python jobs. To be used with PIP_LOCAL_DEPENDENCIES to use python codebases within Windmill                        | Worker                |
 | INCLUDE_HEADERS           | None                   | Whitelist of headers that are passed to jobs as args (separated by a comma)                                                                                                                        | Server                |

backend/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "windmill"
-version = "1.70.1"
+version = "1.83.1"
 authors.workspace = true
 edition.workspace = true
 
@@ -19,7 +19,7 @@ members = [
 ]
 
 [workspace.package]
-version = "1.70.1"
+version = "1.83.1"
 authors = ["Ruben Fiszel <ruben@windmill.dev>"]
 edition = "2021"
 
@@ -122,9 +122,9 @@ itertools = "^0"
 regex = "^1"
 deno_core = "^0"
 async-recursion = "^1"
-swc_common = "^0"
-swc_ecma_parser = "^0"
-swc_ecma_ast = "^0"
+swc_common = "0.29.39"
+swc_ecma_parser = "0.128.2"
+swc_ecma_ast = "0.98.1"
 base64 = "0.21.0"
 unicode-general-category = "^0"
 hmac = "0.12.1"
@@ -153,3 +153,4 @@ async-stripe = { version = "0.14", features = [
     "checkout",
 ] }
 async_zip = { version = "0.0.11", features = ["full"] }
+once_cell = "1.17.1"

backend/migrations/20230303064952_add_root_leaf_jobs.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230303064952_add_root_leaf_jobs.up.sql

@@ -0,0 +1,3 @@
+-- Add up migration script here
+ALTER TABLE queue ADD COLUMN root_job uuid;
+ALTER TABLE queue ADD COLUMN leaf_jobs jsonb;

backend/migrations/20230327115727_update_app_setup.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230327201258_add_granular_share_apps.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230327201258_add_granular_share_apps.up.sql

@@ -0,0 +1,11 @@
+-- Add up migration script here
+CREATE POLICY see_extra_perms_user ON app FOR ALL
+USING (extra_perms ? CONCAT('u/', current_setting('session.user')))
+WITH CHECK ((extra_perms ->> CONCAT('u/', current_setting('session.user')))::boolean);
+
+CREATE POLICY see_extra_perms_groups ON app FOR ALL
+USING (extra_perms ?| regexp_split_to_array(current_setting('session.pgroups'), ',')::text[])
+WITH CHECK (exists(
+    SELECT key, value FROM jsonb_each_text(extra_perms) 
+    WHERE SPLIT_PART(key, '/', 1) = 'g' AND key = ANY(regexp_split_to_array(current_setting('session.pgroups'), ',')::text[])
+    AND value::boolean));

backend/parsers/windmill-parser-bash/Cargo.toml

@@ -16,4 +16,5 @@ unicode-general-category.workspace = true
 itertools.workspace = true
 anyhow.workspace = true
 regex.workspace = true
-lazy_static.workspace = true
+lazy_static.workspace = true
+serde_json.workspace = true

backend/parsers/windmill-parser-bash/src/lib.rs

@@ -1,6 +1,8 @@
 #![allow(non_snake_case)] // TODO: switch to parse_* function naming
 
+use anyhow::anyhow;
 use regex::Regex;
+use serde_json::json;
 
 use std::collections::HashMap;
 use windmill_parser::{Arg, MainArgSignature, Typ};
@@ -17,19 +19,32 @@ pub fn parse_bash_sig(code: &str) -> windmill_common::error::Result<MainArgSigna
     }
 }
 
+lazy_static::lazy_static! {
+    static ref RE: Regex = Regex::new(r#"(?m)^(\w+)="\$(?:(\d+)|\{(\d+):-(.*)\})"$"#).unwrap();
+}
+
 fn parse_file(code: &str) -> anyhow::Result<Option<Vec<Arg>>> {
-    let mut hm = HashMap::new();
-    let re = Regex::new(r#"(?m)^(\w+)="\$(\d+)"$"#).unwrap();
-    for cap in re.captures_iter(code) {
-        hm.insert(cap[2].parse::<i32>()?, cap[1].to_string());
+    let mut hm: HashMap<i32, (String, Option<String>)> = HashMap::new();
+    for cap in RE.captures_iter(code) {
+        hm.insert(
+            cap.get(2)
+                .or(cap.get(3))
+                .and_then(|x| x.as_str().parse::<i32>().ok())
+                .ok_or_else(|| anyhow!("Impossible to parse arg digit"))?,
+            (
+                cap[1].to_string(),
+                cap.get(4).map(|x| x.as_str().to_string()),
+            ),
+        );
     }
     let mut args = vec![];
     for i in 1..20 {
         if hm.contains_key(&i) {
+            let (name, default) = hm.get(&i).unwrap();
             args.push(Arg {
-                name: hm[&i].clone(),
+                name: name.clone(),
                 typ: Typ::Str(None),
-                default: None,
+                default: default.clone().map(|x| json!(x)),
                 otyp: None,
                 has_default: false,
             });
@@ -43,6 +58,8 @@ fn parse_file(code: &str) -> anyhow::Result<Option<Vec<Arg>>> {
 #[cfg(test)]
 mod tests {
 
+    use serde_json::json;
+
     use super::*;
 
     #[test]
@@ -50,8 +67,7 @@ mod tests {
         let code = r#"
 token="$1"
 image="$2"
-digest="${3:-latest}"
-foo="$4"
+digest="${3:-latest with spaces}"
 
 "#;
         //println!("{}", serde_json::to_string()?);
@@ -74,6 +90,13 @@ foo="$4"
                         typ: Typ::Str(None),
                         default: None,
                         has_default: false
+                    },
+                    Arg {
+                        otyp: None,
+                        name: "digest".to_string(),
+                        typ: Typ::Str(None),
+                        default: Some(json!("latest with spaces")),
+                        has_default: false
                     }
                 ]
             }

backend/parsers/windmill-parser-go/src/lib.rs

@@ -58,6 +58,9 @@ fn parse_go_typ(typ: &parser_go_ast::Expr) -> (Option<String>, Typ) {
             Some((*name).to_string()),
             match *name {
                 "int" => Typ::Int,
+                "int16" => Typ::Int,
+                "int32" => Typ::Int,
+                "int64" => Typ::Int,
                 "string" => Typ::Str(None),
                 "bool" => Typ::Bool,
                 _ => Typ::Unknown,

backend/parsers/windmill-parser-py/src/lib.rs

@@ -181,6 +181,7 @@ static PYTHON_IMPORTS_REPLACEMENT: phf::Map<&'static str, &'static str> = phf_ma
     "git" => "GitPython",
     "u" => "requests",
     "f" => "requests",
+    "." => "requests",
     "shopify" => "ShopifyAPI",
     "seleniumwire" => "selenium-wire",
     "openbb-terminal" => "openbb[all]",
@@ -224,12 +225,23 @@ pub fn parse_python_imports(code: &str) -> error::Result<Vec<String>> {
                     StmtKind::Import { names } => Some(
                         names
                             .into_iter()
-                            .map(|x| x.node.name.split('.').next().unwrap_or("").to_string())
+                            .map(|x| {
+                                let name = x.node.name;
+                                if name.starts_with('.') {
+                                    ".".to_string()
+                                } else {
+                                    name.split('.').next().unwrap_or("").to_string()
+                                }
+                            })
                             .map(replace_import)
                             .collect::<Vec<String>>(),
                     ),
-                    StmtKind::ImportFrom { level: _, module: Some(mod_), names: _ } => {
-                        let imprt = mod_.split('.').next().unwrap_or("").replace("_", "-");
+                    StmtKind::ImportFrom { level, module: Some(mod_), names: _ } => {
+                        let imprt = if level.is_some() && level.unwrap() > 0 {
+                            ".".to_string()
+                        } else {
+                            mod_.split('.').next().unwrap_or("").replace("_", "-")
+                        };
 
                         Some(vec![replace_import(imprt)])
                     }
@@ -442,6 +454,7 @@ import os
 import wmill
 from zanzibar.estonie import talin
 import matplotlib.pyplot as plt
+from . import tests
 
 def main():
     pass
@@ -449,7 +462,7 @@ def main():
 ";
         let r = parse_python_imports(code)?;
         // println!("{}", serde_json::to_string(&r)?);
-        assert_eq!(r, vec!["wmill", "zanzibar", "matplotlib"]);
+        assert_eq!(r, vec!["wmill", "zanzibar", "matplotlib", "requests"]);
         Ok(())
     }
 

backend/parsers/windmill-parser-ts/src/lib.rs

@@ -246,7 +246,7 @@ fn tstype_to_typ(ts_type: &TsType) -> (Typ, bool) {
 pub fn eval_sync(code: &str) -> Result<serde_json::Value, String> {
     let mut context = JsRuntime::new(RuntimeOptions::default());
     let code = format!("let x = {}; x", code);
-    let res = context.execute_script("<anon>", &code);
+    let res = context.execute_script("<anon>", code);
     match res {
         Ok(global) => {
             let scope = &mut context.handle_scope();

backend/sqlx-data.json

@@ -468,6 +468,27 @@
     },
     "query": "DELETE FROM workspace_settings WHERE workspace_id = $1"
   },
+  "0e7d95f4913e5775651971d741a3b5c1ef5dfe079be5325abe2866d39a7fe5fb": {
+    "describe": {
+      "columns": [
+        {
+          "name": "path",
+          "ordinal": 0,
+          "type_info": "Varchar"
+        }
+      ],
+      "nullable": [
+        false
+      ],
+      "parameters": {
+        "Left": [
+          "Text",
+          "Text"
+        ]
+      }
+    },
+    "query": "DELETE FROM script WHERE path = $1 AND workspace_id = $2 RETURNING path"
+  },
   "11b1586acdfc180c5a077861ee1f7201fcbcec9d0ebada464f9d952c9c3e400d": {
     "describe": {
       "columns": [],
@@ -550,21 +571,6 @@
     },
     "query": "SELECT * FROM workspace LIMIT $1 OFFSET $2"
   },
-  "15de975d9be141c9ed9647935a508492aabbbddbf986d5c5c0f0c415293c432d": {
-    "describe": {
-      "columns": [],
-      "nullable": [],
-      "parameters": {
-        "Left": [
-          "Varchar",
-          "Varchar",
-          "Varchar",
-          "Varchar"
-        ]
-      }
-    },
-    "query": "INSERT INTO variable\n            (workspace_id, path, value, is_secret, description)\n            VALUES ($1, 'g/all/pretty_secret', $2, true, 'This item is secret'), \n                ($3, 'g/all/not_secret', $4, false, 'This item is not secret')"
-  },
   "163f00eb8b1a489d5f382cdba22a5744e88a8e6f1532d7cb02af560f5f5d49f7": {
     "describe": {
       "columns": [
@@ -820,75 +826,6 @@
     },
     "query": "\n        SELECT id, flow_status, suspend, script_path\n        FROM queue\n        WHERE id = $1\n        "
   },
-  "1e35c39bc786d638252e5483ca4efae9a041f7e845341f8bfd715ddd9e899499": {
-    "describe": {
-      "columns": [
-        {
-          "name": "id",
-          "ordinal": 0,
-          "type_info": "Uuid"
-        }
-      ],
-      "nullable": [
-        false
-      ],
-      "parameters": {
-        "Left": [
-          "Varchar",
-          "Uuid",
-          "Bool",
-          "Uuid",
-          "Varchar",
-          "Varchar",
-          "Timestamptz",
-          "Int8",
-          "Varchar",
-          "Text",
-          "Text",
-          "Jsonb",
-          {
-            "Custom": {
-              "kind": {
-                "Enum": [
-                  "script",
-                  "preview",
-                  "flow",
-                  "dependencies",
-                  "flowpreview",
-                  "script_hub",
-                  "identity",
-                  "flowdependencies"
-                ]
-              },
-              "name": "job_kind"
-            }
-          },
-          "Varchar",
-          "Jsonb",
-          "Jsonb",
-          "Bool",
-          {
-            "Custom": {
-              "kind": {
-                "Enum": [
-                  "python3",
-                  "deno",
-                  "go",
-                  "bash"
-                ]
-              },
-              "name": "script_lang"
-            }
-          },
-          "Bool",
-          "Text",
-          "Varchar",
-          "Bool"
-        ]
-      }
-    },
-    "query": "INSERT INTO queue\n            (workspace_id, id, running, parent_job, created_by, permissioned_as, scheduled_for, \n                script_hash, script_path, raw_code, raw_lock, args, job_kind, schedule_path, raw_flow, flow_status, is_flow_step, language, started_at, same_worker, pre_run_error, email, visible_to_owner)\n            VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, CASE WHEN $3 THEN now() END, $19, $20, $21, $22) RETURNING id"
-  },
   "1eaf8d677d520c7f2f303a731de6b6d939918e41ad0d1c748d80db3fd33cb9d3": {
     "describe": {
       "columns": [],
@@ -1170,6 +1107,20 @@
     },
     "query": "SELECT set_config('session.folders_read', $1, true)"
   },
+  "2a3ebe1b0eae5b2164894321e138cc4dc0293788aeb98d05d95d18dfc708d6a6": {
+    "describe": {
+      "columns": [],
+      "nullable": [],
+      "parameters": {
+        "Left": [
+          "Text",
+          "Jsonb",
+          "Uuid"
+        ]
+      }
+    },
+    "query": "\n            UPDATE queue\n               SET leaf_jobs = JSONB_SET(coalesce(leaf_jobs, '{}'::jsonb), ARRAY[$1::TEXT], $2)\n             WHERE COALESCE((SELECT root_job FROM queue WHERE id = $3), $3) = id\n            "
+  },
   "2a4be8334db7d39f3d954193a8b0169cc4a4a07e081d2fa61d8764879d6a8ff5": {
     "describe": {
       "columns": [],
@@ -1183,33 +1134,6 @@
     },
     "query": "UPDATE script SET archived = true WHERE hash = $1 AND workspace_id = $2"
   },
-  "2be0cfd075df9624ccbcbe5fd645e0a5c25460c2d01493f86dcdd9b2b71f6181": {
-    "describe": {
-      "columns": [
-        {
-          "name": "flow_status",
-          "ordinal": 0,
-          "type_info": "Jsonb"
-        },
-        {
-          "name": "parent_job",
-          "ordinal": 1,
-          "type_info": "Uuid"
-        }
-      ],
-      "nullable": [
-        null,
-        null
-      ],
-      "parameters": {
-        "Left": [
-          "Uuid",
-          "Text"
-        ]
-      }
-    },
-    "query": "SELECT flow_status, parent_job FROM completed_job WHERE id = $1 AND workspace_id = $2 UNION ALL SELECT flow_status, parent_job FROM queue WHERE id = $1 AND workspace_id = $2 "
-  },
   "2e4115bb2e6c8c85ad1492ad135d6b0454b342126cb5fa17e58caf71b32ee755": {
     "describe": {
       "columns": [],
@@ -2577,28 +2501,7 @@
     },
     "query": "SELECT set_config('session.user', $1, true)"
   },
-  "6dfd1ac7168f098a9e7c92d530a470c0a156041d5081053d43d9efb00d81c464": {
-    "describe": {
-      "columns": [
-        {
-          "name": "policy",
-          "ordinal": 0,
-          "type_info": "Jsonb"
-        }
-      ],
-      "nullable": [
-        false
-      ],
-      "parameters": {
-        "Left": [
-          "Text",
-          "Text"
-        ]
-      }
-    },
-    "query": "SELECT policy from app WHERE path = $1 AND workspace_id = $2"
-  },
-  "6fc2cfae9df83eb24ea33e4c9567740100f4dd2285afc3ef474fc70041b0567b": {
+  "6d50a8dc9cfc040b6f37b58053daa0709671e008a10ab4189114ceff66efe603": {
     "describe": {
       "columns": [
         {
@@ -2612,9 +2515,9 @@
           "type_info": "Varchar"
         },
         {
-          "name": "ping_at",
+          "name": "last_ping",
           "ordinal": 2,
-          "type_info": "Timestamptz"
+          "type_info": "Int4"
         },
         {
           "name": "started_at",
@@ -2635,7 +2538,7 @@
       "nullable": [
         false,
         false,
-        false,
+        null,
         false,
         false,
         false
@@ -2647,7 +2550,28 @@
         ]
       }
     },
-    "query": "SELECT * FROM worker_ping ORDER BY ping_at desc LIMIT $1 OFFSET $2"
+    "query": "SELECT worker, worker_instance,  EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed FROM worker_ping ORDER BY ping_at desc LIMIT $1 OFFSET $2"
+  },
+  "6dfd1ac7168f098a9e7c92d530a470c0a156041d5081053d43d9efb00d81c464": {
+    "describe": {
+      "columns": [
+        {
+          "name": "policy",
+          "ordinal": 0,
+          "type_info": "Jsonb"
+        }
+      ],
+      "nullable": [
+        false
+      ],
+      "parameters": {
+        "Left": [
+          "Text",
+          "Text"
+        ]
+      }
+    },
+    "query": "SELECT policy from app WHERE path = $1 AND workspace_id = $2"
   },
   "701f215eb14ba67a79afea15d7effc0dd394ba6c4a72c95d4560c5a377015d4e": {
     "describe": {
@@ -3060,20 +2984,6 @@
     },
     "query": "UPDATE group_ SET summary = $1 WHERE name = $2 AND workspace_id = $3"
   },
-  "83fc9bf32b9b40987feff61fea82e20613e74efb7bf8ba52485566667ccf2284": {
-    "describe": {
-      "columns": [],
-      "nullable": [],
-      "parameters": {
-        "Left": [
-          "Varchar",
-          "Text",
-          "Text"
-        ]
-      }
-    },
-    "query": "UPDATE resource SET path = $1 WHERE path = $2 AND workspace_id = $3"
-  },
   "8543f029d9784234e4c6a6dcd7b03e62d544b98be261334ee210594e0bb839f2": {
     "describe": {
       "columns": [
@@ -3244,6 +3154,21 @@
     },
     "query": "INSERT INTO usage (id, is_workspace, month_, usage) \n                    VALUES ($1, false, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), 0) \n                    ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + 1 \n                    RETURNING usage.usage"
   },
+  "8c0131a9cc61f2daa258d49767242bcaab6bb34a977ff7fb0c18aa9202d11f47": {
+    "describe": {
+      "columns": [],
+      "nullable": [],
+      "parameters": {
+        "Left": [
+          "Varchar",
+          "Jsonb",
+          "Text",
+          "Text"
+        ]
+      }
+    },
+    "query": "UPDATE resource SET path = $1, value = $2 WHERE path = $3 AND workspace_id = $4"
+  },
   "8c11511a74a41a65f448249a00ebe6964a61d00c2f7b4875a55e64741bf1f0ca": {
     "describe": {
       "columns": [
@@ -4003,6 +3928,76 @@
     },
     "query": "SELECT content FROM script WHERE path = $1 AND workspace_id = $2 AND\n         created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND archived = false AND workspace_id = $2)"
   },
+  "a1c41bbeb2d64fa1e7dfd2ed053191a1de5d786ae8c22e225e450865ecac94e9": {
+    "describe": {
+      "columns": [
+        {
+          "name": "id",
+          "ordinal": 0,
+          "type_info": "Uuid"
+        }
+      ],
+      "nullable": [
+        false
+      ],
+      "parameters": {
+        "Left": [
+          "Varchar",
+          "Uuid",
+          "Bool",
+          "Uuid",
+          "Varchar",
+          "Varchar",
+          "Timestamptz",
+          "Int8",
+          "Varchar",
+          "Text",
+          "Text",
+          "Jsonb",
+          {
+            "Custom": {
+              "kind": {
+                "Enum": [
+                  "script",
+                  "preview",
+                  "flow",
+                  "dependencies",
+                  "flowpreview",
+                  "script_hub",
+                  "identity",
+                  "flowdependencies"
+                ]
+              },
+              "name": "job_kind"
+            }
+          },
+          "Varchar",
+          "Jsonb",
+          "Jsonb",
+          "Bool",
+          {
+            "Custom": {
+              "kind": {
+                "Enum": [
+                  "python3",
+                  "deno",
+                  "go",
+                  "bash"
+                ]
+              },
+              "name": "script_lang"
+            }
+          },
+          "Bool",
+          "Text",
+          "Varchar",
+          "Bool",
+          "Uuid"
+        ]
+      }
+    },
+    "query": "INSERT INTO queue\n            (workspace_id, id, running, parent_job, created_by, permissioned_as, scheduled_for, \n                script_hash, script_path, raw_code, raw_lock, args, job_kind, schedule_path, raw_flow, flow_status, is_flow_step, language, started_at, same_worker, pre_run_error, email, visible_to_owner, root_job)\n            VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, CASE WHEN $3 THEN now() END, $19, $20, $21, $22, $23) RETURNING id"
+  },
   "a227548b6604c56bfc15eb780bd8ee72a89dc6701a50f5048e928bd87baa7b9a": {
     "describe": {
       "columns": [
@@ -4113,6 +4108,28 @@
     },
     "query": "UPDATE flow SET dependency_job = $1 WHERE path = $2 AND workspace_id = $3"
   },
+  "a5f9fb82791103e2bbaf9cb6d87e8c50495d12d87f8ed83382068203a8dd7a67": {
+    "describe": {
+      "columns": [
+        {
+          "name": "?column?",
+          "ordinal": 0,
+          "type_info": "Jsonb"
+        }
+      ],
+      "nullable": [
+        null
+      ],
+      "parameters": {
+        "Left": [
+          "Text",
+          "Uuid",
+          "Text"
+        ]
+      }
+    },
+    "query": "SELECT leaf_jobs->$1::text FROM queue WHERE COALESCE((SELECT root_job FROM queue WHERE id = $2), $2) = id AND workspace_id = $3"
+  },
   "a6145b0482c9e5da245059a80b1563cad20318fd2dd8aef33f9ca97de1826b8b": {
     "describe": {
       "columns": [],
@@ -4681,6 +4698,27 @@
     },
     "query": "SELECT null FROM queue WHERE id = $1 FOR UPDATE"
   },
+  "be1de9116b18a40681cb8c5ec7578fd7c5f7ce77f7af63f766c2899d85fe0bef": {
+    "describe": {
+      "columns": [
+        {
+          "name": "value",
+          "ordinal": 0,
+          "type_info": "Jsonb"
+        }
+      ],
+      "nullable": [
+        true
+      ],
+      "parameters": {
+        "Left": [
+          "Text",
+          "Text"
+        ]
+      }
+    },
+    "query": "SELECT value FROM resource  WHERE path = $1 AND workspace_id = $2"
+  },
   "be7a99a5bb6858323ca61dd51077010f51ba58ae76b9a413339255024dcb524d": {
     "describe": {
       "columns": [],
@@ -4869,27 +4907,6 @@
     },
     "query": "SELECT result FROM completed_job WHERE id = $1"
   },
-  "c2d0e44faab6981a21ca28dfd6f4eef9dfcafb471852e701c0bbc8ae11344325": {
-    "describe": {
-      "columns": [
-        {
-          "name": "parent_job",
-          "ordinal": 0,
-          "type_info": "Uuid"
-        }
-      ],
-      "nullable": [
-        null
-      ],
-      "parameters": {
-        "Left": [
-          "Uuid",
-          "Text"
-        ]
-      }
-    },
-    "query": "SELECT parent_job FROM completed_job WHERE id = $1 AND workspace_id = $2 UNION ALL SELECT parent_job FROM queue WHERE id = $1 AND workspace_id = $2"
-  },
   "c2d6cb56c1dea4498e2aab9ea9301dbbaa127602a38f57f5add4108fdc209b1a": {
     "describe": {
       "columns": [
@@ -5045,6 +5062,18 @@
     },
     "query": "\n        SELECT id, flow_status, suspend, script_path\n        FROM queue\n        WHERE id = ( SELECT parent_job FROM queue WHERE id = $1 UNION ALL SELECT parent_job FROM completed_job WHERE id = $1)\n        FOR UPDATE\n        "
   },
+  "c9d97800eb0ec87df8e8959b283dacb2c6cce422365ed394375641488ceb6b65": {
+    "describe": {
+      "columns": [],
+      "nullable": [],
+      "parameters": {
+        "Left": [
+          "Text"
+        ]
+      }
+    },
+    "query": "UPDATE worker_ping SET ping_at = now() WHERE worker = $1"
+  },
   "cac594031a21b4806de9c4616317d3541522ef9712a83ecff7bd8b5f6e870748": {
     "describe": {
       "columns": [],
@@ -5093,55 +5122,6 @@
     },
     "query": "select hash from script where path = $1 AND workspace_id = $2 AND\n    created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2) AND\n    deleted = false"
   },
-  "cc34f0298cfdaffe892a286e6b4390c287d33298e3342ccd1e10d48a9c3cc5c0": {
-    "describe": {
-      "columns": [
-        {
-          "name": "id",
-          "ordinal": 0,
-          "type_info": "Uuid"
-        },
-        {
-          "name": "workspace_id",
-          "ordinal": 1,
-          "type_info": "Varchar"
-        },
-        {
-          "name": "last_ping",
-          "ordinal": 2,
-          "type_info": "Timestamptz"
-        }
-      ],
-      "nullable": [
-        false,
-        false,
-        false
-      ],
-      "parameters": {
-        "Left": [
-          "Text",
-          {
-            "Custom": {
-              "kind": {
-                "Enum": [
-                  "script",
-                  "preview",
-                  "flow",
-                  "dependencies",
-                  "flowpreview",
-                  "script_hub",
-                  "identity",
-                  "flowdependencies"
-                ]
-              },
-              "name": "job_kind"
-            }
-          }
-        ]
-      }
-    },
-    "query": "UPDATE queue SET running = false WHERE last_ping < now() - ($1 || ' seconds')::interval AND running = true AND job_kind != $2 AND same_worker = false RETURNING id, workspace_id, last_ping"
-  },
   "d0308abac80575038203b60bb66d3b39b586939da0421a595e47c7a759616431": {
     "describe": {
       "columns": [],
@@ -6429,6 +6409,72 @@
     },
     "query": "SELECT client, refresh_token FROM account WHERE workspace_id = $1 AND id = $2"
   },
+  "fc2ec6fc4e22e46cc35b00503eff1eac35a6fdbc82a634b7eb9da4d7931f3582": {
+    "describe": {
+      "columns": [
+        {
+          "name": "id",
+          "ordinal": 0,
+          "type_info": "Uuid"
+        },
+        {
+          "name": "workspace_id",
+          "ordinal": 1,
+          "type_info": "Varchar"
+        },
+        {
+          "name": "last_ping",
+          "ordinal": 2,
+          "type_info": "Timestamptz"
+        }
+      ],
+      "nullable": [
+        false,
+        false,
+        false
+      ],
+      "parameters": {
+        "Left": [
+          "Text",
+          {
+            "Custom": {
+              "kind": {
+                "Enum": [
+                  "script",
+                  "preview",
+                  "flow",
+                  "dependencies",
+                  "flowpreview",
+                  "script_hub",
+                  "identity",
+                  "flowdependencies"
+                ]
+              },
+              "name": "job_kind"
+            }
+          },
+          {
+            "Custom": {
+              "kind": {
+                "Enum": [
+                  "script",
+                  "preview",
+                  "flow",
+                  "dependencies",
+                  "flowpreview",
+                  "script_hub",
+                  "identity",
+                  "flowdependencies"
+                ]
+              },
+              "name": "job_kind"
+            }
+          }
+        ]
+      }
+    },
+    "query": "UPDATE queue SET running = false, started_at = null, logs = logs || '\nRestarted job after not receiving job''s ping for too long the ' || now() || '\n\n' WHERE last_ping < now() - ($1 || ' seconds')::interval AND running = true AND job_kind != $2 AND job_kind != $3 AND same_worker = false RETURNING id, workspace_id, last_ping"
+  },
   "fcb6ce4fab662602827281e3e85fffaa48542f8d590100d0e7eead2ad4ad4873": {
     "describe": {
       "columns": [],

backend/src/main.rs

@@ -6,7 +6,7 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
-use std::net::SocketAddr;
+use std::net::{IpAddr, Ipv4Addr, SocketAddr};
 
 use git_version::git_version;
 use sqlx::{Pool, Postgres};
@@ -15,6 +15,7 @@ use windmill_common::utils::rd_string;
 const GIT_VERSION: &str = git_version!(args = ["--tag", "--always"], fallback = "unknown-version");
 const DEFAULT_NUM_WORKERS: usize = 3;
 const DEFAULT_PORT: u16 = 8000;
+const DEFAULT_SERVER_BIND_ADDR: Ipv4Addr = Ipv4Addr::new(0, 0, 0, 0);
 
 mod ee;
 
@@ -39,6 +40,11 @@ async fn main() -> anyhow::Result<()> {
         .transpose()?
         .flatten();
 
+    let server_bind_address: IpAddr = std::env::var("SERVER_BIND_ADDR")
+        .ok()
+        .and_then(|x| x.parse().ok())
+        .unwrap_or(IpAddr::from(DEFAULT_SERVER_BIND_ADDR));
+
     let port: u16 = std::env::var("PORT")
         .ok()
         .and_then(|x| x.parse::<u16>().ok())
@@ -60,8 +66,66 @@ async fn main() -> anyhow::Result<()> {
     let (tx, rx) = tokio::sync::broadcast::channel::<()>(3);
     let shutdown_signal = windmill_common::shutdown_signal(tx);
 
+    #[cfg(feature = "enterprise")]
+    tracing::info!(
+        "
+##############################
+Windmill Enterprise Edition {GIT_VERSION}
+##############################"
+    );
+
+    #[cfg(not(feature = "enterprise"))]
+    tracing::info!(
+        "
+##############################
+Windmill Community Edition {GIT_VERSION}
+##############################"
+    );
+
+    display_config(vec![
+        "DISABLE_NSJAIL",
+        "DISABLE_SERVER",
+        "NUM_WORKERS",
+        "METRICS_ADDR",
+        "JSON_FMT",
+        "BASE_URL",
+        "BASE_INTERNAL_URL",
+        "TIMEOUT",
+        "ZOMBIE_JOB_TIMEOUT",
+        "RESTART_ZOMBIE_JOBS",
+        "SLEEP_QUEUE",
+        "MAX_LOG_SIZE",
+        "SERVER_BIND_ADDR",
+        "PORT",
+        "KEEP_JOB_DIR",
+        "S3_CACHE_BUCKET",
+        "TAR_CACHE_RATE",
+        "COOKIE_DOMAIN",
+        "PYTHON_PATH",
+        "DENO_PATH",
+        "GO_PATH",
+        "PIP_INDEX_URL",
+        "PIP_EXTRA_INDEX_URL",
+        "PIP_TRUSTED_HOST",
+        "PATH",
+        "HOME",
+        "DATABASE_CONNECTIONS",
+        "TIMEOUT_WAIT_RESULT",
+        "QUEUE_LIMIT_WAIT_RESULT",
+        "DENO_AUTH_TOKENS",
+        "DENO_FLAGS",
+        "NPM_CONFIG_REGISTRY",
+        "PIP_LOCAL_DEPENDENCIES",
+        "ADDITIONAL_PYTHON_PATHS",
+        "INCLUDE_HEADERS",
+        "WHITELIST_WORKSPACES",
+        "BLACKLIST_WORKSPACES",
+        "NEW_USER_WEBHOOK",
+        "CLOUD_HOSTED",
+    ]);
+
     if server_mode || num_workers > 0 {
-        let addr = SocketAddr::from(([0, 0, 0, 0], port));
+        let addr = SocketAddr::from((server_bind_address, port));
 
         let server_f = async {
             if server_mode {
@@ -72,60 +136,6 @@ async fn main() -> anyhow::Result<()> {
 
         let workers_f = async {
             if num_workers > 0 {
-                #[cfg(feature = "enterprise")]
-                tracing::info!(
-                    "
-##############################
-Windmill Enterprise Edition {GIT_VERSION}
-##############################"
-                );
-
-                #[cfg(not(feature = "enterprise"))]
-                tracing::info!(
-                    "
-##############################
-Windmill Community Edition {GIT_VERSION}
-##############################"
-                );
-
-                display_config(vec![
-                    "DISABLE_NSJAIL",
-                    "DISABLE_SERVER",
-                    "NUM_WORKERS",
-                    "METRICS_ADDR",
-                    "JSON_FMT",
-                    "BASE_URL",
-                    "BASE_INTERNAL_URL",
-                    "TIMEOUT",
-                    "SLEEP_QUEUE",
-                    "MAX_LOG_SIZE",
-                    "PORT",
-                    "KEEP_JOB_DIR",
-                    "S3_CACHE_BUCKET",
-                    "TAR_CACHE_RATE",
-                    "COOKIE_DOMAIN",
-                    "PYTHON_PATH",
-                    "DENO_PATH",
-                    "GO_PATH",
-                    "PIP_INDEX_URL",
-                    "PIP_EXTRA_INDEX_URL",
-                    "PIP_TRUSTED_HOST",
-                    "PATH",
-                    "HOME",
-                    "DATABASE_CONNECTIONS",
-                    "TIMEOUT_WAIT_RESULT",
-                    "QUEUE_LIMIT_WAIT_RESULT",
-                    "DENO_AUTH_TOKENS",
-                    "DENO_FLAGS",
-                    "PIP_LOCAL_DEPENDENCIES",
-                    "ADDITIONAL_PYTHON_PATHS",
-                    "INCLUDE_HEADERS",
-                    "WHITELIST_WORKSPACES",
-                    "BLACKLIST_WORKSPACES",
-                    "NEW_USER_WEBHOOK",
-                    "CLOUD_HOSTED",
-                ]);
-
                 run_workers(
                     db.clone(),
                     rx.resubscribe(),
@@ -154,6 +164,8 @@ Windmill Community Edition {GIT_VERSION}
         };
 
         futures::try_join!(shutdown_signal, server_f, metrics_f, workers_f, monitor_f)?;
+    } else {
+        tracing::info!("Nothing to do, exiting.");
     }
     Ok(())
 }

backend/tests/worker.rs

@@ -686,7 +686,7 @@ def main(error, port):
             result,
             json!({
                 "recv": 42,
-                "from failure module": {"error": {"name": "IndexError", "stack": "  File \"/tmp/inner.py\", line 5, in main\n    return sock.recv(1)[0]\n", "message": "index out of range"}},
+                "from failure module": {"error": {"name": "IndexError", "stack": "  File \"/tmp/tmp/main/step_0.py\", line 5, in main\n    return sock.recv(1)[0]\n", "message": "index out of range"}},
             })
         );
     }
@@ -836,6 +836,7 @@ impl RunJob {
             /* scheduled_for_o */ None,
             /* schedule_path */ None,
             /* parent_job */ None,
+            /* root job  */ None,
             /* is_flow_step */ false,
             /* running */ false,
             None,
@@ -2068,7 +2069,7 @@ async fn test_branchall_skip_failure(db: Pool<Postgres>) {
 
     assert_eq!(
         result,
-        serde_json::json!([{"error": {"name": "Error", "stack": "Error: failure\n    at main (file:///tmp/inner.ts:1:31)\n    at run (file:///tmp/main.ts:9:26)\n    at file:///tmp/main.ts:14:1", "message": "failure"}}, [1,3]])
+        serde_json::json!([{"error": {"name": "Error", "stack": "Error: failure\n    at main (file:///tmp/main.ts:1:31)\n    at run (file:///tmp/wrapper.ts:9:26)\n    at file:///tmp/wrapper.ts:14:1", "message": "failure"}}, [1,3]])
     );
 
     let flow: FlowValue = serde_json::from_value(json!({
@@ -2102,7 +2103,7 @@ async fn test_branchall_skip_failure(db: Pool<Postgres>) {
 
     assert_eq!(
         result,
-        serde_json::json!([ {"error": {"name": "Error", "stack": "Error: failure\n    at main (file:///tmp/inner.ts:1:31)\n    at run (file:///tmp/main.ts:9:26)\n    at file:///tmp/main.ts:14:1", "message": "failure"}}, [1, 2]])
+        serde_json::json!([ {"error": {"name": "Error", "stack": "Error: failure\n    at main (file:///tmp/main.ts:1:31)\n    at run (file:///tmp/wrapper.ts:9:26)\n    at file:///tmp/wrapper.ts:14:1", "message": "failure"}}, [1, 2]])
     );
 }
 

backend/windmill-api/Cargo.toml

@@ -69,3 +69,4 @@ async-stripe = { workspace = true, optional = true }
 lazy_static.workspace = true
 prometheus.workspace = true
 async_zip.workspace = true
+regex.workspace = true

backend/windmill-api/openapi.yaml

@@ -1,7 +1,7 @@
 openapi: "3.0.3"
 
 info:
-  version: 1.70.1
+  version: 1.83.1
   title: Windmill API
 
   contact:
@@ -204,7 +204,6 @@ paths:
               schema:
                 type: string
 
-
   /w/{workspace}/users/is_owner/{path}:
     get:
       summary: is owner of path
@@ -1549,7 +1548,7 @@ paths:
         required: true
         content:
           application/json:
-            schema: 
+            schema:
               type: object
               properties:
                 value: {}
@@ -2103,7 +2102,6 @@ paths:
                 items:
                   type: string
 
-
   /w/{workspace}/scripts/create:
     post:
       summary: create script
@@ -2278,7 +2276,7 @@ paths:
 
   /w/{workspace}/scripts/delete/h/{hash}:
     post:
-      summary: delete script by hash (erase content but keep hash)
+      summary: delete script by hash (erase content but keep hash, require admin)
       operationId: deleteScriptByHash
       tags:
         - script
@@ -2293,6 +2291,23 @@ paths:
               schema:
                 $ref: "#/components/schemas/Script"
 
+  /w/{workspace}/scripts/delete/p/{path}:
+    post:
+      summary: delete all scripts at a given path (require admin)
+      operationId: deleteScriptByPath
+      tags:
+        - script
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/ScriptPath"
+      responses:
+        "200":
+          description: script path
+          content:
+            application/json:
+              schema:
+                type: string
+
   /w/{workspace}/scripts/get/p/{path}:
     get:
       summary: get script by path
@@ -2538,11 +2553,6 @@ paths:
           required: true
           schema:
             type: string
-        - name: skip_direct
-          description: Skip checking that the node is part of the given flow.
-          in: query
-          schema:
-            type: boolean
       responses:
         "200":
           description: job result
@@ -2823,7 +2833,6 @@ paths:
               schema:
                 $ref: "#/components/schemas/AppWithLastVersion"
 
-
   /w/{workspace}/apps/secret_of/{path}:
     get:
       summary: get public secret of app
@@ -3156,12 +3165,14 @@ paths:
         - $ref: "#/components/parameters/ScriptExactPath"
         - $ref: "#/components/parameters/ScriptStartPath"
         - $ref: "#/components/parameters/ScriptExactHash"
-        - $ref: "#/components/parameters/CreatedBefore"
-        - $ref: "#/components/parameters/CreatedAfter"
+        - $ref: "#/components/parameters/StartedBefore"
+        - $ref: "#/components/parameters/StartedAfter"
         - $ref: "#/components/parameters/Success"
         - $ref: "#/components/parameters/JobKinds"
         - $ref: "#/components/parameters/Suspended"
         - $ref: "#/components/parameters/Running"
+        - $ref: "#/components/parameters/ArgsFilter"
+        - $ref: "#/components/parameters/ResultFilter"
       responses:
         "200":
           description: All available queued jobs
@@ -3186,10 +3197,12 @@ paths:
         - $ref: "#/components/parameters/ScriptExactPath"
         - $ref: "#/components/parameters/ScriptStartPath"
         - $ref: "#/components/parameters/ScriptExactHash"
-        - $ref: "#/components/parameters/CreatedBefore"
-        - $ref: "#/components/parameters/CreatedAfter"
+        - $ref: "#/components/parameters/StartedBefore"
+        - $ref: "#/components/parameters/StartedAfter"
         - $ref: "#/components/parameters/Success"
         - $ref: "#/components/parameters/JobKinds"
+        - $ref: "#/components/parameters/ArgsFilter"
+        - $ref: "#/components/parameters/ResultFilter"
         - name: is_skipped
           description: is the job skipped
           in: query
@@ -3223,9 +3236,11 @@ paths:
         - $ref: "#/components/parameters/ScriptExactPath"
         - $ref: "#/components/parameters/ScriptStartPath"
         - $ref: "#/components/parameters/ScriptExactHash"
-        - $ref: "#/components/parameters/CreatedBefore"
-        - $ref: "#/components/parameters/CreatedAfter"
+        - $ref: "#/components/parameters/StartedBefore"
+        - $ref: "#/components/parameters/StartedAfter"
         - $ref: "#/components/parameters/JobKinds"
+        - $ref: "#/components/parameters/ArgsFilter"
+        - $ref: "#/components/parameters/ResultFilter"
         - name: is_skipped
           description: is the job skipped
           in: query
@@ -3337,6 +3352,22 @@ paths:
               schema:
                 $ref: "#/components/schemas/CompletedJob"
 
+  /w/{workspace}/jobs/completed/get_result/{id}:
+    get:
+      summary: get completed job result
+      operationId: getCompletedJobResult
+      tags:
+        - job
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/JobId"
+      responses:
+        "200":
+          description: result
+          content:
+            application/json:
+              schema: {}
+
   /w/{workspace}/jobs/completed/delete/{id}:
     post:
       summary: delete completed job (erase content but keep run id)
@@ -3410,7 +3441,7 @@ paths:
 
   /w/{workspace}/jobs/resume_urls/{id}/{resume_id}:
     get:
-      summary:  get resume urls given a job_id, resume_id and a nonce to resume a flow
+      summary: get resume urls given a job_id, resume_id and a nonce to resume a flow
       operationId: getResumeUrls
       tags:
         - job
@@ -3436,7 +3467,7 @@ paths:
                 properties:
                   approvalPage:
                     type: string
-                  resume: 
+                  resume:
                     type: string
                   cancel:
                     type: string
@@ -4081,7 +4112,7 @@ paths:
                   type: string
                 owners:
                   type: array
-                  items: 
+                  items:
                     type: string
                 extra_perms:
                   additionalProperties:
@@ -4115,7 +4146,7 @@ paths:
               properties:
                 owners:
                   type: array
-                  items: 
+                  items:
                     type: string
                 extra_perms:
                   additionalProperties:
@@ -4286,7 +4317,8 @@ paths:
           required: true
           schema:
             type: string
-            enum: [script, group_, resource, schedule, variable, flow, folder, app]
+            enum:
+              [script, group_, resource, schedule, variable, flow, folder, app]
       responses:
         "200":
           description: acls
@@ -4311,7 +4343,8 @@ paths:
           required: true
           schema:
             type: string
-            enum: [script, group_, resource, schedule, variable, flow, folder, app]
+            enum:
+              [script, group_, resource, schedule, variable, flow, folder, app]
       requestBody:
         description: acl to add
         required: true
@@ -4347,7 +4380,8 @@ paths:
           required: true
           schema:
             type: string
-            enum: [script, group_, resource, schedule, variable, flow, folder, app]
+            enum:
+              [script, group_, resource, schedule, variable, flow, folder, app]
       requestBody:
         description: acl to add
         required: true
@@ -4367,19 +4401,18 @@ paths:
               schema:
                 type: string
 
-
   /w/{workspace}/capture_u/{path}:
-      post:
-        summary: update flow preview capture
-        operationId: updateCapture
-        tags:
-          - capture
-        parameters:
-          - $ref: "#/components/parameters/WorkspaceId"
-          - $ref: "#/components/parameters/Path"
-        responses:
-          "204":
-            description: flow preview captured
+    post:
+      summary: update flow preview capture
+      operationId: updateCapture
+      tags:
+        - capture
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/Path"
+      responses:
+        "204":
+          description: flow preview captured
 
   /w/{workspace}/capture/{path}:
     put:
@@ -4586,15 +4619,15 @@ components:
       in: query
       schema:
         type: string
-    CreatedBefore:
-      name: created_before
+    StartedBefore:
+      name: started_before
       description: filter on created before (inclusive) timestamp
       in: query
       schema:
         type: string
         format: date-time
-    CreatedAfter:
-      name: created_after
+    StartedAfter:
+      name: started_after
       description: filter on created after (exclusive) timestamp
       in: query
       schema:
@@ -4618,6 +4651,19 @@ components:
       in: query
       schema:
         type: boolean
+    ArgsFilter:
+      name: args
+      description: filter on jobs containing those args as a json subset (@> in postgres)
+      in: query
+      schema:
+        type: string
+
+    ResultFilter:
+      name: result
+      description: filter on jobs containing those result as a json subset (@> in postgres)
+      in: query
+      schema:
+        type: string
     After:
       name: after
       description: filter on created after (exclusive) timestamp
@@ -5522,9 +5568,8 @@ components:
           type: string
         worker_instance:
           type: string
-        ping_at:
-          type: string
-          format: date-time
+        last_ping:
+          type: number
         started_at:
           type: string
           format: date-time

backend/windmill-api/src/apps.rs

@@ -313,10 +313,17 @@ async fn create_app(
     Extension(user_db): Extension<UserDB>,
     Extension(webhook): Extension<WebhookShared>,
     Path(w_id): Path<String>,
-    Json(app): Json<CreateApp>,
+    Json(mut app): Json<CreateApp>,
 ) -> Result<(StatusCode, String)> {
     let mut tx = user_db.begin(&authed).await?;
 
+    app.policy.on_behalf_of = Some(username_to_permissioned_as(&authed.username));
+    app.policy.on_behalf_of_email = Some(authed.email);
+
+    if &app.path == "" {
+        return Err(Error::BadRequest("App path cannot be empty".to_string()));
+    }
+
     let id = sqlx::query_scalar!(
         "INSERT INTO app
             (workspace_id, path, summary, policy, versions)
@@ -463,7 +470,9 @@ async fn update_app(
             sqlb.set_str("summary", nsummary);
         }
 
-        if let Some(npolicy) = ns.policy {
+        if let Some(mut npolicy) = ns.policy {
+            npolicy.on_behalf_of = Some(username_to_permissioned_as(&authed.username));
+            npolicy.on_behalf_of_email = Some(authed.email);
             sqlb.set(
                 "policy",
                 &format!(
@@ -670,6 +679,7 @@ async fn execute_component(
         None,
         None,
         None,
+        None,
         false,
         false,
         None,
@@ -727,6 +737,9 @@ fn build_args(
     path: String,
     args: &Map<String, Value>,
 ) -> Result<Map<String, Value>> {
+    // disallow var and res access in args coming from the user for security reasons
+    args.into_iter()
+        .try_for_each(|x| disallow_var_res_access(x.1))?;
     let static_args = policy
         .triggerables
         .get(&path)
@@ -747,3 +760,20 @@ fn build_args(
     }
     Ok(args)
 }
+
+fn disallow_var_res_access(args: &serde_json::Value) -> Result<()> {
+    match args {
+        Value::Object(v) => v.into_iter().try_for_each(|x| disallow_var_res_access(x.1)),
+        Value::Array(arr) => arr.into_iter().try_for_each(|v| disallow_var_res_access(v)),
+        Value::String(s) => {
+            if s.starts_with("$var:") || s.starts_with("$res:") {
+                Err(Error::BadRequest(format!(
+                    "For security reasons, variable or resource access is not allowed as dynamic argument"
+                )))
+            } else {
+                Ok(())
+            }
+        }
+        _ => Ok(()),
+    }
+}

backend/windmill-api/src/capture.rs

@@ -7,11 +7,12 @@
  */
 
 use axum::{
-    extract::{Extension, Path},
+    extract::{Extension, Path, Query},
     routing::{get, post, put},
     Json, Router,
 };
-use hyper::StatusCode;
+use hyper::{HeaderMap, StatusCode};
+use serde::Deserialize;
 use windmill_common::{
     error::{JsonResult, Result},
     utils::{not_found_if_none, StripPath},
@@ -19,6 +20,7 @@ use windmill_common::{
 
 use crate::{
     db::{UserDB, DB},
+    jobs::add_include_headers,
     users::Authed,
 };
 
@@ -83,13 +85,21 @@ pub async fn new_payload(
     Ok(StatusCode::CREATED)
 }
 
+#[derive(Deserialize, Clone)]
+pub struct IncludeHeaderQuery {
+    include_header: Option<String>,
+}
+
 pub async fn update_payload(
     Extension(db): Extension<DB>,
     Path((w_id, path)): Path<(String, StripPath)>,
-    Json(payload): Json<serde_json::Value>,
+    Query(run_query): Query<IncludeHeaderQuery>,
+    headers: HeaderMap,
+    Json(args): Json<Option<serde_json::Map<String, serde_json::Value>>>,
 ) -> Result<StatusCode> {
     let mut tx = db.begin().await?;
 
+    let args = add_include_headers(&run_query.include_header, headers, args.unwrap_or_default());
     sqlx::query!(
         "
        UPDATE capture
@@ -99,7 +109,7 @@ pub async fn update_payload(
         ",
         &w_id,
         &path.to_path(),
-        &payload,
+        serde_json::json!(args),
     )
     .execute(&mut tx)
     .await?;

backend/windmill-api/src/flows.rs

@@ -237,6 +237,7 @@ async fn create_flow(
         None,
         None,
         None,
+        None,
         false,
         false,
         None,
@@ -393,6 +394,7 @@ async fn update_flow(
         None,
         None,
         None,
+        None,
         false,
         false,
         None,
@@ -638,8 +640,6 @@ mod tests {
                 "type": "script",
                 "path": "test"
               },
-              "stop_after_if": null,
-              "summary": null
             },
             {
               "id": "b",
@@ -648,15 +648,12 @@ mod tests {
                 "input_transforms": {},
                 "type": "rawscript",
                 "content": "test",
-                "lock": null,
-                "path": null,
                 "language": "deno"
               },
               "stop_after_if": {
                   "expr": "foo = 'bar'",
                   "skip_if_stopped": false
-              },
-              "summary": null
+              }
             },
             {
               "id": "c",
@@ -678,8 +675,7 @@ mod tests {
               "stop_after_if": {
                   "expr": "previous.isEmpty()",
                   "skip_if_stopped": false,
-              },
-              "summary": null
+              }
             }
           ],
           "failure_module": {
@@ -693,8 +689,7 @@ mod tests {
             "stop_after_if": {
                 "expr": "previous.isEmpty()",
                 "skip_if_stopped": false
-            },
-            "summary": null
+            }
           }
         });
         assert_eq!(dbg!(serde_json::json!(fv)), dbg!(expect));

backend/windmill-api/src/folders.rs

@@ -16,10 +16,11 @@ use axum::{
     routing::{delete, get, post},
     Json, Router,
 };
-use itertools::Itertools;
+use lazy_static::lazy_static;
+use regex::Regex;
 use windmill_audit::{audit_log, ActionKind};
 use windmill_common::{
-    error::{self, Error, JsonResult, Result},
+    error::{self, to_anyhow, Error, JsonResult, Result},
     users::username_to_permissioned_as,
     utils::{not_found_if_none, paginate, Pagination},
 };
@@ -137,6 +138,10 @@ async fn check_name_conflict<'c>(
     return Ok(());
 }
 
+lazy_static! {
+    static ref VALID_FOLDER_NAME: Regex = Regex::new(r#"^[a-zA-Z_0-9]+$"#).unwrap();
+}
+
 async fn create_folder(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
@@ -146,6 +151,12 @@ async fn create_folder(
 ) -> Result<String> {
     let mut tx = user_db.begin(&authed).await?;
 
+    if !VALID_FOLDER_NAME.is_match(&ng.name) {
+        return Err(windmill_common::error::Error::BadRequest(format!(
+            "Folder name can only contain alphanumeric characters, underscores"
+        )));
+    }
+
     check_name_conflict(&mut tx, &w_id, &ng.name).await?;
     let owner = username_to_permissioned_as(&authed.username);
     let owners = &ng.owners.unwrap_or(vec![owner.clone()]);
@@ -262,13 +273,26 @@ async fn update_folder(
     sqlb.and_where_eq("workspace_id", "?".bind(&w_id));
 
     if let Some(display_name) = ng.display_name {
-        sqlb.set("display_name", display_name);
+        sqlb.set("display_name", "?".bind(&display_name));
     }
     if let Some(owners) = ng.owners {
-        sqlb.set_str("owners", format!("{{{}}}", owners.into_iter().join(",")));
+        sqlb.set(
+            "owners",
+            "?".bind(&format!(
+                "{{{}}}",
+                owners
+                    .iter()
+                    .map(|x| format!("\"{x}\""))
+                    .collect::<Vec<_>>()
+                    .join(","),
+            )),
+        );
     }
     if let Some(extra_perms) = ng.extra_perms {
-        sqlb.set_str("extra_perms", extra_perms.to_string());
+        sqlb.set(
+            "extra_perms",
+            "?".bind(&serde_json::to_string(&extra_perms).map_err(to_anyhow)?),
+        );
     }
 
     sqlb.returning("*");

backend/windmill-api/src/jobs.rs

@@ -34,7 +34,7 @@ use windmill_queue::{get_queued_job, push, JobKind, JobPayload, QueuedJob, RawCo
 
 use crate::{
     db::{UserDB, DB},
-    users::{require_owner_of_path, Authed},
+    users::{require_owner_of_path, Authed, OptAuthed},
     variables::get_workspace_key,
     BASE_URL,
 };
@@ -60,7 +60,6 @@ pub fn workspaced_service() -> Router {
         .route("/run/preview_flow", post(run_preview_flow_job))
         .route("/list", get(list_jobs))
         .route("/queue/list", get(list_queue_jobs))
-        .route("/queue/cancel/:id", post(cancel_job_api))
         .route("/completed/list", get(list_completed_jobs))
         .route("/completed/get/:id", get(get_completed_job))
         .route("/completed/get_result/:id", get(get_completed_job_result))
@@ -98,32 +97,36 @@ pub fn global_service() -> Router {
         )
         .route("/get/:id", get(get_job))
         .route("/getupdate/:id", get(get_job_update))
+        .route("/queue/cancel/:id", post(cancel_job_api))
 }
 
 async fn get_result_by_id(
     Extension(db): Extension<DB>,
-    Query(ResultByIdQuery { skip_direct }): Query<ResultByIdQuery>,
-    Path((w_id, flow_id, node_id)): Path<(String, String, String)>,
+    Path((w_id, flow_id, node_id)): Path<(String, Uuid, String)>,
 ) -> windmill_common::error::JsonResult<serde_json::Value> {
-    let res = windmill_queue::get_result_by_id(db, skip_direct, w_id, flow_id, node_id).await?;
+    let res = windmill_queue::get_result_by_id(db, w_id, flow_id, node_id).await?;
     Ok(Json(res))
 }
 
 async fn cancel_job_api(
-    authed: Authed,
-    Extension(user_db): Extension<UserDB>,
+    OptAuthed(opt_authed): OptAuthed,
+    Extension(db): Extension<DB>,
     Path((w_id, id)): Path<(String, Uuid)>,
     Json(CancelJob { reason }): Json<CancelJob>,
 ) -> error::Result<String> {
-    let tx = user_db.begin(&authed).await?;
+    let tx = db.begin().await?;
 
-    let (mut tx, job_option) =
-        windmill_queue::cancel_job(&authed.username, reason, id, &w_id, tx).await?;
+    let username = match opt_authed {
+        Some(authed) => authed.username,
+        None => "anonymous".to_string(),
+    };
+
+    let (mut tx, job_option) = windmill_queue::cancel_job(&username, reason, id, &w_id, tx).await?;
 
     if let Some(id) = job_option {
         audit_log(
             &mut tx,
-            &authed.username,
+            &username,
             "jobs.cancel",
             ActionKind::Delete,
             &w_id,
@@ -177,11 +180,6 @@ async fn get_job(
     Ok(Json(job))
 }
 
-#[derive(Deserialize)]
-pub struct ResultByIdQuery {
-    pub skip_direct: bool,
-}
-
 pub async fn get_job_by_id<'c>(
     mut tx: Transaction<'c, Postgres>,
     w_id: &str,
@@ -295,41 +293,51 @@ impl RunJobQuery {
     fn add_include_headers(
         &self,
         headers: HeaderMap,
-        mut args: serde_json::Map<String, serde_json::Value>,
+        args: serde_json::Map<String, serde_json::Value>,
     ) -> serde_json::Map<String, serde_json::Value> {
-        let whitelist = self
-            .include_header
-            .as_ref()
-            .map(|s| s.split(",").map(|s| s.to_string()).collect::<Vec<_>>())
-            .unwrap_or_default();
-        whitelist
-            .iter()
-            .chain(INCLUDE_HEADERS.iter())
-            .for_each(|h| {
-                if let Some(v) = headers.get(h) {
-                    args.insert(
-                        h.to_string().to_lowercase().replace('-', "_"),
-                        serde_json::Value::String(v.to_str().unwrap().to_string()),
-                    );
-                }
-            });
-        args
+        return add_include_headers(&self.include_header, headers, args);
     }
 }
 
+pub fn add_include_headers(
+    include_header: &Option<String>,
+    headers: HeaderMap,
+    mut args: serde_json::Map<String, serde_json::Value>,
+) -> serde_json::Map<String, serde_json::Value> {
+    let whitelist = include_header
+        .as_ref()
+        .map(|s| s.split(",").map(|s| s.to_string()).collect::<Vec<_>>())
+        .unwrap_or_default();
+
+    whitelist
+        .iter()
+        .chain(INCLUDE_HEADERS.iter())
+        .for_each(|h| {
+            if let Some(v) = headers.get(h) {
+                args.insert(
+                    h.to_string().to_lowercase().replace('-', "_"),
+                    serde_json::Value::String(v.to_str().unwrap().to_string()),
+                );
+            }
+        });
+    args
+}
+
 #[derive(Deserialize)]
 pub struct ListQueueQuery {
     pub script_path_start: Option<String>,
     pub script_path_exact: Option<String>,
     pub script_hash: Option<String>,
     pub created_by: Option<String>,
-    pub created_before: Option<chrono::DateTime<chrono::Utc>>,
-    pub created_after: Option<chrono::DateTime<chrono::Utc>>,
+    pub started_before: Option<chrono::DateTime<chrono::Utc>>,
+    pub started_after: Option<chrono::DateTime<chrono::Utc>>,
     pub running: Option<bool>,
     pub parent_job: Option<String>,
     pub order_desc: Option<bool>,
     pub job_kinds: Option<String>,
     pub suspended: Option<bool>,
+    // filter by matching a subset of the args using base64 encoded json subset
+    pub args: Option<String>,
 }
 
 fn list_queue_jobs_query(w_id: &str, lq: &ListQueueQuery, fields: &[&str]) -> SqlBuilder {
@@ -358,11 +366,11 @@ fn list_queue_jobs_query(w_id: &str, lq: &ListQueueQuery, fields: &[&str]) -> Sq
     if let Some(pj) = &lq.parent_job {
         sqlb.and_where_eq("parent_job", "?".bind(pj));
     }
-    if let Some(dt) = &lq.created_before {
-        sqlb.and_where_lt("created_at", format!("to_timestamp({})", dt.timestamp()));
+    if let Some(dt) = &lq.started_before {
+        sqlb.and_where_le("started_at", format!("to_timestamp({})", dt.timestamp()));
     }
-    if let Some(dt) = &lq.created_after {
-        sqlb.and_where_gt("created_at", format!("to_timestamp({})", dt.timestamp()));
+    if let Some(dt) = &lq.started_after {
+        sqlb.and_where_ge("started_at", format!("to_timestamp({})", dt.timestamp()));
     }
 
     if let Some(s) = &lq.suspended {
@@ -372,6 +380,7 @@ fn list_queue_jobs_query(w_id: &str, lq: &ListQueueQuery, fields: &[&str]) -> Sq
             sqlb.and_where_eq("suspend", 0);
         }
     }
+
     if let Some(jk) = &lq.job_kinds {
         sqlb.and_where_in(
             "job_kind",
@@ -379,6 +388,10 @@ fn list_queue_jobs_query(w_id: &str, lq: &ListQueueQuery, fields: &[&str]) -> Sq
         );
     }
 
+    if let Some(args) = &lq.args {
+        sqlb.and_where("args @> ?".bind(&args.replace("'", "''")));
+    }
+
     sqlb
 }
 
@@ -451,13 +464,14 @@ async fn list_jobs(
             script_path_exact: lq.script_path_exact,
             script_hash: lq.script_hash,
             created_by: lq.created_by,
-            created_before: lq.created_before,
-            created_after: lq.created_after,
+            started_before: lq.started_before,
+            started_after: lq.started_after,
             running: None,
             parent_job: lq.parent_job,
             order_desc: Some(true),
             job_kinds: lq.job_kinds,
             suspended: lq.suspended,
+            args: lq.args,
         },
         &[
             "'QueuedJob' as typ",
@@ -1068,6 +1082,8 @@ impl From<UnifiedJob> for Job {
                 visible_to_owner: uj.visible_to_owner,
                 suspend: uj.suspend,
                 mem_peak: uj.mem_peak,
+                root_job: None,
+                leaf_jobs: None,
             }),
             t => panic!("job type {} not valid", t),
         }
@@ -1130,15 +1146,15 @@ where
         struct InPayload {
             payload: Option<String>,
         }
-
-        fn decode_payload<D: DeserializeOwned, T: AsRef<[u8]>>(t: T) -> anyhow::Result<D> {
-            let vec = base64::engine::general_purpose::URL_SAFE
-                .decode(t)
-                .context("invalid base64")?;
-            serde_json::from_slice(vec.as_slice()).context("invalid json")
-        }
     }
 }
+
+fn decode_payload<D: DeserializeOwned, T: AsRef<[u8]>>(t: T) -> anyhow::Result<D> {
+    let vec = base64::engine::general_purpose::URL_SAFE
+        .decode(t)
+        .context("invalid base64")?;
+    serde_json::from_slice(vec.as_slice()).context("invalid json")
+}
 pub async fn run_flow_by_path(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
@@ -1163,6 +1179,7 @@ pub async fn run_flow_by_path(
         scheduled_for,
         None,
         run_query.parent_job,
+        run_query.parent_job,
         false,
         false,
         None,
@@ -1198,6 +1215,7 @@ pub async fn run_job_by_path(
         scheduled_for,
         None,
         run_query.parent_job,
+        run_query.parent_job,
         false,
         false,
         None,
@@ -1350,6 +1368,7 @@ pub async fn run_wait_result_job_by_path(
         scheduled_for,
         None,
         run_query.parent_job,
+        run_query.parent_job,
         false,
         false,
         None,
@@ -1396,6 +1415,7 @@ pub async fn run_wait_result_job_by_hash(
         scheduled_for,
         None,
         run_query.parent_job,
+        run_query.parent_job,
         false,
         false,
         None,
@@ -1441,6 +1461,7 @@ pub async fn run_wait_result_flow_by_path(
         scheduled_for,
         None,
         run_query.parent_job,
+        run_query.parent_job,
         false,
         false,
         None,
@@ -1503,6 +1524,7 @@ async fn run_preview_job(
         scheduled_for,
         None,
         None,
+        None,
         false,
         false,
         None,
@@ -1536,6 +1558,7 @@ async fn run_preview_flow_job(
         scheduled_for,
         None,
         None,
+        None,
         false,
         false,
         None,
@@ -1571,6 +1594,7 @@ pub async fn run_job_by_hash(
         scheduled_for,
         None,
         run_query.parent_job,
+        run_query.parent_job,
         false,
         false,
         None,
@@ -1673,11 +1697,11 @@ fn list_completed_jobs_query(
     if let Some(pj) = &lq.parent_job {
         sqlb.and_where_eq("parent_job", "?".bind(pj));
     }
-    if let Some(dt) = &lq.created_before {
-        sqlb.and_where_lt("created_at", format!("to_timestamp({})", dt.timestamp()));
+    if let Some(dt) = &lq.started_before {
+        sqlb.and_where_le("started_at", format!("to_timestamp({})", dt.timestamp()));
     }
-    if let Some(dt) = &lq.created_after {
-        sqlb.and_where_gt("created_at", format!("to_timestamp({})", dt.timestamp()));
+    if let Some(dt) = &lq.started_after {
+        sqlb.and_where_ge("started_at", format!("to_timestamp({})", dt.timestamp()));
     }
     if let Some(sk) = &lq.is_skipped {
         sqlb.and_where_eq("is_skipped", sk);
@@ -1692,6 +1716,14 @@ fn list_completed_jobs_query(
         );
     }
 
+    if let Some(args) = &lq.args {
+        sqlb.and_where("args @> ?".bind(&args.replace("'", "''")));
+    }
+
+    if let Some(result) = &lq.result {
+        sqlb.and_where("result @> ?".bind(&result.replace("'", "''")));
+    }
+
     sqlb
 }
 #[derive(Deserialize, Clone)]
@@ -1700,8 +1732,8 @@ pub struct ListCompletedQuery {
     pub script_path_exact: Option<String>,
     pub script_hash: Option<String>,
     pub created_by: Option<String>,
-    pub created_before: Option<chrono::DateTime<chrono::Utc>>,
-    pub created_after: Option<chrono::DateTime<chrono::Utc>>,
+    pub started_before: Option<chrono::DateTime<chrono::Utc>>,
+    pub started_after: Option<chrono::DateTime<chrono::Utc>>,
     pub success: Option<bool>,
     pub parent_job: Option<String>,
     pub order_desc: Option<bool>,
@@ -1709,6 +1741,10 @@ pub struct ListCompletedQuery {
     pub is_skipped: Option<bool>,
     pub is_flow_step: Option<bool>,
     pub suspended: Option<bool>,
+    // filter by matching a subset of the args using base64 encoded json subset
+    pub args: Option<String>,
+    // filter by matching a subset of the result using base64 encoded json subset
+    pub result: Option<String>,
 }
 
 async fn list_completed_jobs(
@@ -1775,6 +1811,7 @@ async fn get_completed_job(
     .fetch_optional(&db)
     .await?;
 
+    tracing::info!("job_o: {:?}", job_o);
     let job = not_found_if_none(job_o, "Completed Job", id.to_string())?;
     Ok(Json(job))
 }

backend/windmill-api/src/oauth2.rs

@@ -734,14 +734,10 @@ async fn slack_command(
         .map_err(|_| error::Error::BadRequest("invalid payload".to_string()))?;
 
     let body = String::from_utf8_lossy(&body);
-    if SLACK_SIGNING_SECRET
-        .as_ref()
-        .as_ref()
-        .map(|sv| sv.verify(&ts, &body, &sig).ok())
-        .flatten()
-        .is_none()
-    {
-        return Err(error::Error::BadRequest("verification failed".to_owned()));
+    if let Some(sv) = SLACK_SIGNING_SECRET.as_ref() {
+        if sv.verify(&ts, &body, &sig).ok().is_none() {
+            return Err(error::Error::BadRequest("verification failed".to_owned()));
+        }
     }
 
     let mut tx = db.begin().await?;
@@ -785,6 +781,7 @@ async fn slack_command(
                 None,
                 None,
                 None,
+                None,
                 false,
                 false,
                 None,

backend/windmill-api/src/scripts.rs

@@ -70,6 +70,7 @@ pub fn workspaced_service() -> Router {
         .route("/exists/p/*path", get(exists_script_by_path))
         .route("/archive/h/:hash", post(archive_script_by_hash))
         .route("/delete/h/:hash", post(delete_script_by_hash))
+        .route("/delete/p/*path", post(delete_script_by_path))
         .route("/get/h/:hash", get(get_script_by_hash))
         .route("/raw/h/:hash", get(raw_script_by_hash))
         .route("/deployment_status/h/:hash", get(get_deployment_status))
@@ -123,6 +124,7 @@ async fn list_scripts(
             AND workspace_id = ?)"
                 .bind(&w_id),
         );
+        sqlb.and_where_eq("archived", true);
     } else {
         sqlb.and_where_eq("archived", false);
     }
@@ -377,6 +379,7 @@ async fn create_script(
             None,
             None,
             None,
+            None,
             false,
             false,
             None,
@@ -718,6 +721,46 @@ async fn delete_script_by_hash(
     Ok(Json(script))
 }
 
+async fn delete_script_by_path(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    Extension(webhook): Extension<WebhookShared>,
+    Extension(db): Extension<DB>,
+    Path((w_id, path)): Path<(String, StripPath)>,
+) -> JsonResult<String> {
+    let mut tx = user_db.begin(&authed).await?;
+    let path = path.to_path();
+
+    require_admin(authed.is_admin, &authed.username)?;
+    let script = sqlx::query_scalar!(
+        "DELETE FROM script WHERE path = $1 AND workspace_id = $2 RETURNING path",
+        path,
+        w_id
+    )
+    .fetch_one(&db)
+    .await
+    .map_err(|e| Error::InternalErr(format!("deleting script by path {w_id}: {e}")))?;
+
+    audit_log(
+        &mut tx,
+        &authed.username,
+        "scripts.delete",
+        ActionKind::Delete,
+        &w_id,
+        Some(&path),
+        Some([("workspace", w_id.as_str())].into()),
+    )
+    .await?;
+    tx.commit().await?;
+
+    webhook.send_message(
+        w_id.clone(),
+        WebhookMessage::DeleteScriptPath { workspace: w_id, path: path.to_string() },
+    );
+
+    Ok(Json(script))
+}
+
 async fn parse_python_code_to_jsonschema(
     Json(code): Json<String>,
 ) -> JsonResult<windmill_parser::MainArgSignature> {

backend/windmill-api/src/users.rs

@@ -25,7 +25,9 @@ use axum::{
     Json, Router,
 };
 use hyper::{header::LOCATION, StatusCode};
+use lazy_static::lazy_static;
 use rand::rngs::OsRng;
+use regex::Regex;
 use retainer::Cache;
 use serde::{Deserialize, Serialize};
 use sqlx::FromRow;
@@ -35,6 +37,7 @@ use tracing::{Instrument, Span};
 use windmill_audit::{audit_log, ActionKind};
 use windmill_common::{
     error::{self, Error, JsonResult, Result},
+    users::SUPERADMIN_SECRET_EMAIL,
     utils::{not_found_if_none, rd_string, require_admin, Pagination, StripPath},
 };
 use windmill_queue::CLOUD_HOSTED;
@@ -266,7 +269,7 @@ impl AuthCache {
                     .unwrap_or(false)
                 {
                     Some(Authed {
-                        email: "superadmin_secret@windmill.dev".to_string(),
+                        email: SUPERADMIN_SECRET_EMAIL.to_string(),
                         username: "superadmin_secret".to_string(),
                         is_admin: true,
                         groups: Vec::new(),
@@ -997,6 +1000,10 @@ async fn decline_invite(
     }
 }
 
+lazy_static! {
+    pub static ref VALID_USERNAME: Regex = Regex::new(r#"^[a-zA-Z_0-9]+$"#).unwrap();
+}
+
 async fn accept_invite(
     Authed { email, .. }: Authed,
     Extension(db): Extension<DB>,
@@ -1005,6 +1012,13 @@ async fn accept_invite(
     if &nu.username == "bot" {
         return Err(Error::BadRequest("bot is a reserved username".to_string()));
     }
+
+    if !VALID_USERNAME.is_match(&nu.username) {
+        return Err(windmill_common::error::Error::BadRequest(format!(
+            "Usermame can only contain alphanumeric characters and underscores"
+        )));
+    }
+
     let mut tx = db.begin().await?;
 
     let r = sqlx::query!(
@@ -1077,6 +1091,12 @@ async fn add_user_to_workspace<'c>(
         )));
     }
 
+    if !VALID_USERNAME.is_match(username) {
+        return Err(windmill_common::error::Error::BadRequest(format!(
+            "Usermame can only contain alphanumeric characters and underscores"
+        )));
+    }
+
     let already_exists_email = sqlx::query_scalar!(
         "SELECT EXISTS(SELECT 1 FROM usr WHERE workspace_id = $1 AND email = $2)",
         &w_id,
@@ -1282,6 +1302,12 @@ async fn create_user(
 
     require_super_admin(&mut tx, &email).await?;
 
+    if nu.email == SUPERADMIN_SECRET_EMAIL {
+        return Err(Error::BadRequest(
+            "The superadmin email is a reserved email".into(),
+        ));
+    }
+
     sqlx::query!(
         "INSERT INTO password(email, verified, password_hash, login_type, super_admin, name, \
          company)
@@ -1311,7 +1337,9 @@ async fn create_user(
     if let Some(new_user_webhook) = NEW_USER_WEBHOOK.clone() {
         let _ = HTTP_CLIENT
             .post(&new_user_webhook)
-            .json(&serde_json::json!({"email" : &nu.email, "name": &nu.name, "event": "new_user"}))
+            .json(
+                &serde_json::json!({"email" : &nu.email, "name": &nu.name, "event": "global_add"}),
+            )
             .send()
             .await
             .map_err(|e| tracing::error!("Error sending new user webhook: {}", e.to_string()));

backend/windmill-api/src/utils.rs

@@ -7,16 +7,24 @@
  */
 
 use sqlx::{Postgres, Transaction};
-use windmill_common::error::{self, Error};
+use windmill_common::{
+    error::{self, Error},
+    users::SUPERADMIN_SECRET_EMAIL,
+};
 
 pub async fn require_super_admin<'c>(
     db: &mut Transaction<'c, Postgres>,
     email: &str,
 ) -> error::Result<()> {
+    if email == SUPERADMIN_SECRET_EMAIL {
+        return Ok(());
+    }
     let is_admin = sqlx::query_scalar!("SELECT super_admin FROM password WHERE email = $1", email)
-        .fetch_one(db)
+        .fetch_optional(db)
         .await
-        .map_err(|e| Error::InternalErr(format!("fetching super admin: {e}")))?;
+        .map_err(|e| Error::InternalErr(format!("fetching super admin: {e}")))?
+        .unwrap_or(false);
+
     if !is_admin {
         Err(Error::NotAuthorized(
             "This endpoint require caller to be a super admin".to_owned(),

backend/windmill-api/src/variables.rs

@@ -26,6 +26,7 @@ use axum::{
     Json, Router,
 };
 use hyper::StatusCode;
+use serde_json::Value;
 use windmill_audit::{audit_log, ActionKind};
 use windmill_common::{
     error::{Error, JsonResult, Result},
@@ -378,9 +379,27 @@ async fn update_variable(
             if !authed.is_admin {
                 require_owner_of_path(&w_id, &authed.username, &authed.groups, &path, &db).await?;
             }
+            let mut v = sqlx::query_scalar!(
+                "SELECT value FROM resource  WHERE path = $1 AND workspace_id = $2",
+                path,
+                w_id
+            )
+            .fetch_optional(&mut tx)
+            .await?
+            .flatten();
+
+            if let Some(old_v) = v {
+                v = Some(replace_path(
+                    old_v,
+                    &format!("$var:{path}"),
+                    &format!("$var:{npath}"),
+                ))
+            }
+
             sqlx::query!(
-                "UPDATE resource SET path = $1 WHERE path = $2 AND workspace_id = $3",
+                "UPDATE resource SET path = $1, value = $2 WHERE path = $3 AND workspace_id = $4",
                 npath,
+                v,
                 path,
                 w_id
             )
@@ -419,6 +438,23 @@ async fn update_variable(
     Ok(format!("variable {} updated (npath: {:?})", path, npath))
 }
 
+fn replace_path(v: serde_json::Value, path: &str, npath: &str) -> Value {
+    match v {
+        Value::Object(v) => Value::Object(
+            v.into_iter()
+                .map(|(k, v)| (k, replace_path(v, path, npath)))
+                .collect(),
+        ),
+        Value::Array(arr) => Value::Array(
+            arr.into_iter()
+                .map(|v| replace_path(v, path, npath))
+                .collect(),
+        ),
+        Value::String(s) if s == path => Value::String(npath.to_owned()),
+        _ => v,
+    }
+}
+
 pub async fn build_crypt<'c>(
     db: &mut Transaction<'c, Postgres>,
     w_id: &str,

backend/windmill-api/src/webhook_util.rs

@@ -37,6 +37,7 @@ pub enum WebhookMessage {
     CreateScript { workspace: String, path: String, hash: String },
     UpdateScript { workspace: String, path: String, hash: String },
     DeleteScript { workspace: String, hash: String },
+    DeleteScriptPath { workspace: String, path: String },
     CreateVariable { workspace: String, path: String },
     UpdateVariable { workspace: String, old_path: String, new_path: String },
     DeleteVariable { workspace: String, path: String },

backend/windmill-api/src/worker_ping.rs

@@ -28,7 +28,7 @@ pub fn global_service() -> Router {
 struct WorkerPing {
     worker: String,
     worker_instance: String,
-    ping_at: chrono::DateTime<chrono::Utc>,
+    last_ping: Option<i32>,
     started_at: chrono::DateTime<chrono::Utc>,
     ip: String,
     jobs_executed: i32,
@@ -45,7 +45,7 @@ async fn list_worker_pings(
 
     let rows = sqlx::query_as!(
         WorkerPing,
-        "SELECT * FROM worker_ping ORDER BY ping_at desc LIMIT $1 OFFSET $2",
+        "SELECT worker, worker_instance,  EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed FROM worker_ping ORDER BY ping_at desc LIMIT $1 OFFSET $2",
         per_page as i64,
         offset as i64
     )

backend/windmill-api/src/workspaces.rs

@@ -16,8 +16,9 @@ use crate::{
     db::{UserDB, DB},
     folders::Folder,
     resources::{Resource, ResourceType},
-    users::{Authed, WorkspaceInvite, NEW_USER_WEBHOOK},
+    users::{Authed, WorkspaceInvite, NEW_USER_WEBHOOK, VALID_USERNAME},
     utils::require_super_admin,
+    variables::build_crypt,
     HTTP_CLIENT,
 };
 #[cfg(feature = "enterprise")]
@@ -30,6 +31,7 @@ use axum::{
     routing::{delete, get, post},
     Json, Router,
 };
+use magic_crypt::MagicCryptTrait;
 #[cfg(feature = "enterprise")]
 use stripe::CustomerId;
 use windmill_audit::{audit_log, ActionKind};
@@ -965,7 +967,7 @@ async fn invite_user(
     if let Some(new_user_webhook) = NEW_USER_WEBHOOK.clone() {
         let _ = &HTTP_CLIENT
             .post(&new_user_webhook)
-            .json(&serde_json::json!({"email" : &nu.email, "event": "new_invite"}))
+            .json(&serde_json::json!({"email" : &nu.email, "event": "workspace_invite"}))
             .send()
             .await
             .map_err(|e| tracing::error!("Error sending new user webhook: {}", e.to_string()));
@@ -986,6 +988,11 @@ async fn add_user(
     require_admin(is_admin, &username)?;
 
     let mut tx = db.begin().await?;
+    if !VALID_USERNAME.is_match(&nu.username) {
+        return Err(windmill_common::error::Error::BadRequest(format!(
+            "Usermame can only contain alphanumeric characters and underscores"
+        )));
+    }
 
     sqlx::query!(
         "INSERT INTO usr
@@ -1000,8 +1007,27 @@ async fn add_user(
     .execute(&mut tx)
     .await?;
 
+    sqlx::query_as!(
+        Group,
+        "INSERT INTO usr_to_group (workspace_id, usr, group_) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING",
+        &w_id,
+        nu.username,
+        "all",
+    )
+    .execute(&mut tx)
+    .await?;
+
     tx.commit().await?;
 
+    if let Some(new_user_webhook) = NEW_USER_WEBHOOK.clone() {
+        let _ = HTTP_CLIENT
+            .post(&new_user_webhook)
+            .json(&serde_json::json!({"email" : &nu.email, "event": "workspace_add"}))
+            .send()
+            .await
+            .map_err(|e| tracing::error!("Error sending new user webhook: {}", e.to_string()));
+    }
+
     Ok((
         StatusCode::CREATED,
         format!("user with email {} added", nu.email),
@@ -1064,6 +1090,7 @@ struct ScriptMetadata {
     schema: Option<Schema>,
     is_template: bool,
     lock: Vec<String>,
+    kind: String,
 }
 
 enum ArchiveImpl {
@@ -1115,6 +1142,7 @@ impl ArchiveImpl {
 #[derive(Deserialize)]
 struct ArchiveQueryParams {
     archive_type: Option<String>,
+    plain_secret: Option<bool>,
 }
 
 #[inline]
@@ -1160,7 +1188,7 @@ async fn tarball_workspace(
     authed: Authed,
     Extension(db): Extension<DB>,
     Path(w_id): Path<String>,
-    Query(ArchiveQueryParams { archive_type }): Query<ArchiveQueryParams>,
+    Query(ArchiveQueryParams { archive_type, plain_secret }): Query<ArchiveQueryParams>,
 ) -> Result<([(headers::HeaderName, String); 2], impl IntoResponse)> {
     require_admin(authed.is_admin, &authed.username)?;
 
@@ -1226,6 +1254,7 @@ async fn tarball_workspace(
                 description: script.description,
                 schema: script.schema,
                 is_template: script.is_template,
+                kind: script.kind.to_string(),
                 lock,
             };
             let metadata_str = serde_json::to_string_pretty(&metadata).unwrap();
@@ -1296,7 +1325,15 @@ async fn tarball_workspace(
         .fetch_all(&db)
         .await?;
 
-        for var in variables {
+        let mc = build_crypt(&mut db.begin().await?, &w_id).await?;
+
+        for mut var in variables {
+            if plain_secret.unwrap_or(false) && var.value.is_some() {
+                var.value = Some(
+                    mc.decrypt_base64_to_string(var.value.unwrap())
+                        .map_err(|e| Error::InternalErr(e.to_string()))?,
+                );
+            }
             let var_str = &to_string_without_metadata(&var, false).unwrap();
             archive
                 .write_to_archive(&var_str, &format!("{}.variable.json", var.path))

backend/windmill-common/src/flow_status.rs

@@ -125,7 +125,7 @@ pub enum FlowStatusModule {
     },
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub enum JobResult {
     SingleJob(Uuid),
     ListJob(Vec<Uuid>),

backend/windmill-common/src/scripts.rs

@@ -7,7 +7,7 @@
  */
 
 use std::{
-    fmt::Display,
+    fmt::{self, Display},
     hash::{Hash, Hasher},
 };
 
@@ -103,6 +103,18 @@ pub enum ScriptKind {
     Approval,
 }
 
+impl Display for ScriptKind {
+    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
+        fmt.write_str(match self {
+            ScriptKind::Trigger => "trigger",
+            ScriptKind::Failure => "failure",
+            ScriptKind::Script => "script",
+            ScriptKind::Approval => "approval",
+        })?;
+        Ok(())
+    }
+}
+
 #[derive(Serialize)]
 #[cfg_attr(feature = "sqlx", derive(sqlx::FromRow))]
 pub struct Script {

backend/windmill-common/src/users.rs

@@ -6,6 +6,8 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
+pub const SUPERADMIN_SECRET_EMAIL: &str = "superadmin_secret@windmill.dev";
+
 pub fn username_to_permissioned_as(user: &str) -> String {
     if user.contains('@') {
         user.to_string()

backend/windmill-queue/src/jobs.rs

@@ -6,7 +6,7 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
-use std::{collections::HashMap, str::FromStr};
+use std::collections::HashMap;
 
 use anyhow::Context;
 use reqwest::Client;
@@ -152,55 +152,24 @@ pub async fn pull(
 
 pub async fn get_result_by_id(
     db: Pool<Postgres>,
-    mut skip_direct: bool,
     w_id: String,
-    flow_id: String,
+    flow_id: Uuid,
     node_id: String,
 ) -> error::Result<serde_json::Value> {
-    let mut result_id: Option<JobResult> = None;
-    let mut parent_id = Uuid::from_str(&flow_id).ok();
-    while result_id.is_none() && parent_id.is_some() {
-        if !skip_direct {
-            let r = sqlx::query!(
-                "SELECT flow_status, parent_job FROM completed_job WHERE id = $1 AND workspace_id = $2 UNION ALL SELECT flow_status, parent_job FROM queue WHERE id = $1 AND workspace_id = $2 ",
-                parent_id.unwrap(),
-                w_id,
-            )
-            .fetch_optional(&db)
-            .await?;
-            if let Some(r) = r {
-                let value = r
-                    .flow_status
-                    .as_ref()
-                    .ok_or_else(|| Error::InternalErr(format!("requiring a flow status value")))?
-                    .to_owned();
-                parent_id = r.parent_job;
-                let status_o = serde_json::from_value::<FlowStatus>(value).ok();
-                result_id = status_o.and_then(|status| {
-                    status
-                        .modules
-                        .iter()
-                        .find(|m| m.id() == node_id)
-                        .and_then(|m| m.job_result())
-                });
-            } else {
-                parent_id = None;
-            }
-        } else {
-            let q_parent = sqlx::query_scalar!(
-                "SELECT parent_job FROM completed_job WHERE id = $1 AND workspace_id = $2 UNION ALL SELECT parent_job FROM queue WHERE id = $1 AND workspace_id = $2",
-                parent_id.unwrap(),
-                w_id,
-            )
-            .fetch_optional(&db)
-            .await?
-            .flatten();
-            parent_id = q_parent;
-            skip_direct = false
-        }
-    }
+    let job_result: Option<JobResult> = sqlx::query_scalar!(
+        "SELECT leaf_jobs->$1::text FROM queue WHERE COALESCE((SELECT root_job FROM queue WHERE id = $2), $2) = id AND workspace_id = $3",
+        node_id,
+        flow_id,
+        w_id,
+    )
+    .fetch_optional(&db)
+    .await?
+    .flatten()
+    .map(|x| serde_json::from_value(x).ok())
+    .flatten();
+
     let result_id = windmill_common::utils::not_found_if_none(
-        result_id,
+        job_result,
         "Flow result by id",
         format!("{}, {}", flow_id, node_id),
     )?;
@@ -282,6 +251,7 @@ pub async fn push<'c>(
     scheduled_for_o: Option<chrono::DateTime<chrono::Utc>>,
     schedule_path: Option<String>,
     parent_job: Option<Uuid>,
+    root_job: Option<Uuid>,
     is_flow_step: bool,
     mut same_worker: bool,
     pre_run_error: Option<&windmill_common::error::Error>,
@@ -534,12 +504,13 @@ pub async fn push<'c>(
         .unwrap_or_else(|| (None, None));
 
     let flow_status = raw_flow.as_ref().map(FlowStatus::new);
+
     let uuid = sqlx::query_scalar!(
         "INSERT INTO queue
             (workspace_id, id, running, parent_job, created_by, permissioned_as, scheduled_for, 
                 script_hash, script_path, raw_code, raw_lock, args, job_kind, schedule_path, raw_flow, \
-         flow_status, is_flow_step, language, started_at, same_worker, pre_run_error, email, visible_to_owner)
-            VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, CASE WHEN $3 THEN now() END, $19, $20, $21, $22) \
+         flow_status, is_flow_step, language, started_at, same_worker, pre_run_error, email, visible_to_owner, root_job)
+            VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, CASE WHEN $3 THEN now() END, $19, $20, $21, $22, $23) \
          RETURNING id",
         workspace_id,
         job_id,
@@ -562,7 +533,8 @@ pub async fn push<'c>(
         same_worker,
         pre_run_error.map(|e| e.to_string()),
         email,
-        visible_to_owner
+        visible_to_owner,
+        root_job
     )
     .fetch_one(&mut tx)
     .await
@@ -675,6 +647,10 @@ pub struct QueuedJob {
     pub suspend: Option<i32>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub mem_peak: Option<i32>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub root_job: Option<Uuid>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub leaf_jobs: Option<serde_json::Value>,
 }
 
 impl QueuedJob {
@@ -682,7 +658,7 @@ impl QueuedJob {
         self.script_path
             .as_ref()
             .map(String::as_str)
-            .unwrap_or("NO_FLOW_PATH")
+            .unwrap_or("tmp/main")
     }
 }
 

backend/windmill-queue/src/schedule.rs

@@ -85,6 +85,7 @@ pub async fn push_scheduled_job<'c>(
         Some(next),
         Some(schedule.path.clone()),
         None,
+        None,
         false,
         false,
         None,

backend/windmill-worker/Cargo.toml

@@ -48,3 +48,4 @@ deno_core.workspace = true
 const_format.workspace = true
 git-version.workspace = true
 dyn-iter.workspace = true
+once_cell.workspace = true

backend/windmill-worker/nsjail/run.deno.config.proto

@@ -61,8 +61,8 @@ mount {
 
 
 mount {
-    src: "{JOB_DIR}/inner.ts"
-    dst: "/tmp/inner.ts"
+    src: "{JOB_DIR}/wrapper.ts"
+    dst: "/tmp/wrapper.ts"
     is_bind: true
 }
 

backend/windmill-worker/nsjail/run.python3.config.proto

@@ -66,8 +66,8 @@ mount {
 }
 
 mount {
-    src: "{JOB_DIR}/inner.py"
-    dst: "/tmp/inner.py"
+    src: "{JOB_DIR}/{MAIN}.py"
+    dst: "/tmp/{MAIN}.py"
     is_bind: true
 }
 
@@ -79,8 +79,8 @@ mount {
 }
 
 mount {
-    src: "{JOB_DIR}/main.py"
-    dst: "/tmp/main.py"
+    src: "{JOB_DIR}/wrapper.py"
+    dst: "/tmp/wrapper.py"
     is_bind: true
 }
 

backend/windmill-worker/src/js_eval.rs

@@ -6,9 +6,11 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
-use std::collections::HashMap;
+use std::{cell::RefCell, collections::HashMap, rc::Rc};
 
-use deno_core::{op, serde_v8, v8, v8::IsolateHandle, Extension, JsRuntime, RuntimeOptions};
+use deno_core::{
+    op, serde_v8, v8, v8::IsolateHandle, Extension, JsRuntime, OpState, RuntimeOptions,
+};
 use itertools::Itertools;
 use lazy_static::lazy_static;
 use regex::Regex;
@@ -17,10 +19,7 @@ use tokio::{sync::oneshot, time::timeout};
 use uuid::Uuid;
 use windmill_common::{error::Error, flow_status::JobResult};
 
-pub struct EvalCreds {
-    pub workspace: String,
-    pub token: String,
-}
+use crate::AuthedClient;
 
 #[derive(Debug, Clone)]
 pub struct IdContext {
@@ -29,22 +28,23 @@ pub struct IdContext {
     pub previous_id: String,
 }
 
+pub struct OptAuthedClient(Option<AuthedClient>);
 pub async fn eval_timeout(
     expr: String,
     env: Vec<(String, serde_json::Value)>,
-    creds: Option<EvalCreds>,
+    authed_client: Option<&AuthedClient>,
     by_id: Option<IdContext>,
-    base_internal_url: &str,
 ) -> anyhow::Result<serde_json::Value> {
     let expr2 = expr.clone();
     let (sender, mut receiver) = oneshot::channel::<IsolateHandle>();
-    let base_internal_url: String = base_internal_url.to_string();
+    let has_client = authed_client.is_some();
+    let authed_client = authed_client.cloned();
     timeout(
-        std::time::Duration::from_millis(2000),
+        std::time::Duration::from_millis(10000),
         tokio::task::spawn_blocking(move || {
             let mut ops = vec![];
 
-            if creds.is_some() {
+            if authed_client.is_some() {
                 ops.extend([
                     // An op for summing an array of numbers
                     // The op-layer automatically deserializes inputs
@@ -54,12 +54,12 @@ pub async fn eval_timeout(
                 ])
             }
 
-            if by_id.is_some() {
+            if by_id.is_some() && authed_client.is_some() {
                 ops.push(op_get_result::decl());
                 ops.push(op_get_id::decl());
             }
 
-            let ext = Extension::builder().ops(ops).build();
+            let ext = Extension::builder("js_eval").ops(ops).build();
             // Use our snapshot to provision our new runtime
             let options = RuntimeOptions {
                 extensions: vec![ext],
@@ -68,6 +68,11 @@ pub async fn eval_timeout(
             };
 
             let mut js_runtime = JsRuntime::new(options);
+            {
+                let op_state = js_runtime.op_state();
+                let mut op_state = op_state.borrow_mut();
+                op_state.put(OptAuthedClient(authed_client.clone()));
+            }
 
             sender
                 .send(js_runtime.v8_isolate().thread_safe_handle())
@@ -86,14 +91,7 @@ pub async fn eval_timeout(
 
             let expr = replace_with_await_result(expr);
 
-            let r = runtime.block_on(eval(
-                &mut js_runtime,
-                &expr,
-                env,
-                creds,
-                by_id,
-                &base_internal_url,
-            ))?;
+            let r = runtime.block_on(eval(&mut js_runtime, &expr, env, by_id, has_client))?;
 
             Ok(r) as anyhow::Result<Value>
         }),
@@ -104,7 +102,7 @@ pub async fn eval_timeout(
             isolate.terminate_execution();
         };
         Error::ExecutionErr(format!(
-            "The expression of evaluation `{expr2}` took too long to execute (>2000ms)"
+            "The expression of evaluation `{expr2}` took too long to execute (>10000ms)"
         ))
     })??
 }
@@ -145,24 +143,30 @@ fn add_closing_bracket(s: &str) -> String {
     s
 }
 
-const SPLIT_PAT: &str = ";\n";
+const SPLIT_PAT: &str = ";";
 async fn eval(
     context: &mut JsRuntime,
     expr: &str,
     env: Vec<(String, serde_json::Value)>,
-    creds: Option<EvalCreds>,
     by_id: Option<IdContext>,
-    base_internal_url: &str,
+    has_client: bool,
 ) -> anyhow::Result<serde_json::Value> {
-    let expr = expr.trim();
-    let expr = format!(
-        "{}\nreturn {};",
-        expr.split(SPLIT_PAT)
-            .take(expr.split(SPLIT_PAT).count() - 1)
-            .join("\n"),
-        expr.split(SPLIT_PAT).last().unwrap_or_else(|| "")
-    );
-    let (api_code, by_id_code) = if let Some(EvalCreds { workspace, token }) = creds {
+    let exprs = expr
+        .trim()
+        .split(SPLIT_PAT)
+        .map(|x| x.trim())
+        .filter(|x| !x.is_empty())
+        .collect::<Vec<&str>>();
+    let expr = if exprs.is_empty() {
+        "return undefined;".to_string()
+    } else {
+        format!(
+            "{};\n    return {};",
+            exprs.iter().take(exprs.len() - 1).join(";\n"),
+            exprs.last().unwrap()
+        )
+    };
+    let (api_code, by_id_code) = if has_client {
         let by_id_code = if let Some(by_id) = by_id {
             format!(
                 r#"
@@ -179,12 +183,12 @@ async function result_by_id(node_id) {{
         }}
     }} else {{
         let flow_job_id = "{}";
-        return await Deno.core.opAsync("op_get_id", [workspace, flow_job_id, token, base_url, node_id]);
+        return await Deno.core.opAsync("op_get_id", [ flow_job_id, node_id]);
     }}
 }}
 
 async function get_result(id) {{
-    return await Deno.core.opAsync("op_get_result", [workspace, id, token, base_url]);
+    return await Deno.core.opAsync("op_get_result", [id]);
 }}
 const results = new Proxy({{}}, {{
     get: function(target, name, receiver) {{
@@ -198,12 +202,12 @@ const results = new Proxy({{}}, {{
                     .into_iter()
                     .map(|(k, v)| {
                         let v_str = match v {
-                            JobResult::SingleJob(x) => x.to_string(),
+                            JobResult::SingleJob(x) => format!("\"{x}\""),
                             JobResult::ListJob(x) => {
-                                format!("[{}]", x.iter().map(|x| x.to_string()).join(","))
+                                format!("[{}]", x.iter().map(|x| format!("\"{x}\"")).join(","))
                             }
                         };
-                        format!("\"{k}\": \"{v_str}\"")
+                        format!("\"{k}\": {v_str}")
                     })
                     .join(","),
                 by_id.previous_id,
@@ -215,17 +219,13 @@ const results = new Proxy({{}}, {{
 
         let api_code = format!(
             r#"
-let workspace = "{workspace}";
-let base_url = "{}";
-let token = "{token}";
 async function variable(path) {{
-    return await Deno.core.opAsync("op_variable", [workspace, path, token, base_url]);
+    return await Deno.core.opAsync("op_variable", [path]);
 }}
 async function resource(path) {{
-    return await Deno.core.opAsync("op_resource", [workspace, path, token, base_url]);
+    return await Deno.core.opAsync("op_resource", [path]);
 }}
         "#,
-            base_internal_url,
         );
         (api_code, by_id_code)
     } else {
@@ -252,7 +252,7 @@ async function resource(path) {{
             .join(""),
     );
     tracing::debug!("{}", code);
-    let global = context.execute_script("<anon>", &code)?;
+    let global = context.execute_script("<anon>", code)?;
     let global = context.resolve_value(global).await?;
 
     let scope = &mut context.handle_scope();
@@ -274,60 +274,83 @@ async function resource(path) {{
 
 // TODO: Can we a) share the api configuration here somehow or b) just implement this natively in deno, via the deno client?
 #[op]
-async fn op_variable(args: Vec<String>) -> Result<String, anyhow::Error> {
-    let workspace = &args[0];
-    let path = &args[1];
-    let token = &args[2];
-    let base_url = &args[3];
-    let client = windmill_api_client::create_client(base_url, token.clone());
-    let result = client.get_variable(workspace, path, None).await?;
-    Ok(result.into_inner().value.unwrap_or_else(|| "".to_owned()))
+async fn op_variable(
+    op_state: Rc<RefCell<OpState>>,
+    args: Vec<String>,
+) -> Result<String, anyhow::Error> {
+    let path = &args[0];
+    let client = op_state.borrow().borrow::<OptAuthedClient>().0.clone();
+    if let Some(client) = client {
+        let result = client
+            .get_client()
+            .get_variable(&client.workspace, path, None)
+            .await?;
+        Ok(result.into_inner().value.unwrap_or_else(|| "".to_owned()))
+    } else {
+        anyhow::bail!("No client found in op state");
+    }
 }
 
 #[op]
-async fn op_get_result(args: Vec<String>) -> Result<serde_json::Value, anyhow::Error> {
-    let workspace = &args[0];
-    let id = &args[1];
-    let token = &args[2];
-    let base_url = &args[3];
-    let client = windmill_api_client::create_client(base_url, token.clone());
-    let result = client
-        .get_completed_job(workspace, &id.parse()?)
-        .await?
-        .result
-        .clone();
-    Ok(serde_json::json!(result))
+async fn op_get_result(
+    op_state: Rc<RefCell<OpState>>,
+    args: Vec<String>,
+) -> Result<serde_json::Value, anyhow::Error> {
+    let id = &args[0];
+    let client = op_state.borrow().borrow::<OptAuthedClient>().0.clone();
+    if let Some(client) = client {
+        let result = client
+            .get_client()
+            .get_completed_job_result(&client.workspace, &id.parse()?)
+            .await?
+            .clone();
+        Ok(serde_json::json!(result))
+    } else {
+        anyhow::bail!("No client found in op state");
+    }
 }
 
 #[op]
-async fn op_get_id(args: Vec<String>) -> Result<Option<serde_json::Value>, anyhow::Error> {
-    let workspace = &args[0];
-    let flow_job_id = &args[1];
-    let token = &args[2];
-    let base_url = &args[3];
-    let node_id = &args[4];
+async fn op_get_id(
+    op_state: Rc<RefCell<OpState>>,
+    args: Vec<String>,
+) -> Result<Option<serde_json::Value>, anyhow::Error> {
+    let flow_job_id = &args[0];
+    let node_id = &args[1];
 
-    let client = windmill_api_client::create_client(base_url, token.clone());
-    let result = client
-        .result_by_id(workspace, flow_job_id, node_id, Some(true))
-        .await
-        .map_or(None, |e| Some(e.into_inner()));
-
-    Ok(result)
+    let client = op_state.borrow().borrow::<OptAuthedClient>().0.clone();
+    if let Some(client) = client {
+        let result = client
+            .get_client()
+            .result_by_id(&client.workspace, flow_job_id, node_id)
+            .await
+            .map_or(None, |e| Some(e.into_inner()));
+        Ok(result)
+    } else {
+        anyhow::bail!("No client found in op state");
+    }
 }
 
 #[op]
-async fn op_resource(args: Vec<String>) -> Result<serde_json::Value, anyhow::Error> {
-    let workspace = &args[0];
-    let path = &args[1];
-    let token = &args[2];
-    let base_url = &args[3];
-    let client = windmill_api_client::create_client(base_url, token.clone());
-    let result = client.get_resource(workspace, path).await?;
-    Ok(result
-        .into_inner()
-        .value
-        .unwrap_or_else(|| serde_json::json!({})))
+async fn op_resource(
+    op_state: Rc<RefCell<OpState>>,
+    args: Vec<String>,
+) -> Result<serde_json::Value, anyhow::Error> {
+    let path = &args[0];
+
+    let client = op_state.borrow().borrow::<OptAuthedClient>().0.clone();
+    if let Some(client) = client {
+        let result = client
+            .get_client()
+            .get_resource(&client.workspace, path)
+            .await?;
+        Ok(result
+            .into_inner()
+            .value
+            .unwrap_or_else(|| serde_json::json!({})))
+    } else {
+        anyhow::bail!("No client found in op state");
+    }
 }
 
 #[cfg(test)]
@@ -347,7 +370,7 @@ mod tests {
         let code = "value.test + params.test";
 
         let mut runtime = JsRuntime::new(RuntimeOptions::default());
-        let res = eval(&mut runtime, code, env, None, None, String::new().as_str()).await?;
+        let res = eval(&mut runtime, code, env, None, false).await?;
         assert_eq!(res, json!(4));
         Ok(())
     }
@@ -360,7 +383,7 @@ mod tests {
 multiline template`";
 
         let mut runtime = JsRuntime::new(RuntimeOptions::default());
-        let res = eval(&mut runtime, code, env, None, None, String::new().as_str()).await?;
+        let res = eval(&mut runtime, code, env, None, false).await?;
         assert_eq!(res, json!("my 5\nmultiline template"));
         Ok(())
     }
@@ -373,7 +396,7 @@ multiline template`";
         ];
         let code = r#"params.test"#;
 
-        let res = eval_timeout(code.to_string(), env, None, None, String::new().as_str()).await?;
+        let res = eval_timeout(code.to_string(), env, None, None).await?;
         assert_eq!(res, json!(2));
         Ok(())
     }

backend/windmill-worker/src/worker.rs

@@ -9,8 +9,10 @@
 use const_format::concatcp;
 use itertools::Itertools;
 use lazy_static::lazy_static;
+use once_cell::sync::OnceCell;
 use regex::Regex;
 use sqlx::{Pool, Postgres, Transaction};
+use windmill_api_client::Client;
 use std::{
     borrow::Borrow, collections::HashMap, io, os::unix::process::ExitStatusExt, panic,
     process::Stdio, time::Duration,
@@ -23,7 +25,7 @@ use windmill_common::{
     flows::{FlowModuleValue, FlowValue},
     scripts::{ScriptHash, ScriptLang},
     utils::rd_string,
-    variables, BASE_URL,
+    variables, BASE_URL, users::SUPERADMIN_SECRET_EMAIL,
 };
 use windmill_queue::{canceled_job_to_result, get_queued_job, pull, JobKind, QueuedJob, CLOUD_HOSTED};
 
@@ -34,7 +36,7 @@ use tokio::{
     io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader},
     process::{Child, Command},
     sync::{
-        mpsc::{self, Sender},  watch,
+        mpsc::{self, Sender},  watch, broadcast
     },
     time::{interval, sleep, Instant, MissedTickBehavior},
 };
@@ -236,11 +238,10 @@ pub async fn create_token_for_owner<'c>(
 ) -> error::Result<(Transaction<'c, Postgres>, String)> {
     // TODO: Bad implementation. We should not have access to this DB here.
     let token: String = rd_string(30);
-    let is_super_admin = owner.contains('@')
-        && sqlx::query_scalar!("SELECT super_admin FROM password WHERE email = $1", owner)
+    let is_super_admin = sqlx::query_scalar!("SELECT super_admin FROM password WHERE email = $1", email)
             .fetch_optional(&mut tx)
             .await?
-            .unwrap_or(false);
+            .unwrap_or(false) || email == SUPERADMIN_SECRET_EMAIL;
 
     sqlx::query_scalar!(
         "INSERT INTO token
@@ -264,7 +265,7 @@ const ROOT_CACHE_DIR: &str = "/tmp/windmill/cache/";
 const PIP_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "pip");
 const DENO_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "deno");
 const GO_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "go");
-const NUM_SECS_ENV_CHECK: u64 = 15;
+const NUM_SECS_PING: u64 = 5;
 const NUM_SECS_SYNC: u64 = 60 * 10;
 
 const INCLUDE_DEPS_PY_SH_CONTENT: &str = include_str!("../nsjail/download_deps.py.sh");
@@ -331,8 +332,7 @@ lazy_static::lazy_static! {
         .map(|x| format!(";{x}"))
         .unwrap_or_else(|| String::new());
 
-    
-
+    static ref NPM_CONFIG_REGISTRY: Option<String> = std::env::var("NPM_CONFIG_REGISTRY").ok();
 
     static ref DENO_FLAGS: Option<Vec<String>> = std::env::var("DENO_FLAGS")
         .ok()
@@ -390,10 +390,19 @@ lazy_static::lazy_static! {
         .ok()
         .and_then(|x| x.parse::<u16>().ok())
         .unwrap_or(DEFAULT_TIMEOUT as u16);
+
     static ref TIMEOUT_DURATION: Duration = Duration::from_secs(*TIMEOUT as u64);
 
+    static ref ZOMBIE_JOB_TIMEOUT: String = std::env::var("ZOMBIE_JOB_TIMEOUT")
+        .ok()
+        .and_then(|x| x.parse::<String>().ok())
+        .unwrap_or_else(|| "30".to_string());
 
-    static ref ZOMBIE_JOB_TIMEOUT: String = (*TIMEOUT as u32 * 5).to_string();
+
+    pub static ref RESTART_ZOMBIE_JOBS: bool = std::env::var("RESTART_ZOMBIE_JOBS")
+        .ok()
+        .and_then(|x| x.parse::<bool>().ok())
+        .unwrap_or(true);
 
     static ref SESSION_TOKEN_EXPIRY: i32 = (*TIMEOUT as i32) * 2;
 }
@@ -401,6 +410,21 @@ lazy_static::lazy_static! {
 //only matter if CLOUD_HOSTED
 const MAX_RESULT_SIZE: usize = 1024 * 1024 * 2; // 2MB
 
+#[derive(Clone)]
+pub struct AuthedClient {
+    pub base_internal_url: String,
+    pub workspace: String,
+    pub token: String,
+    pub client: OnceCell<Client>
+}
+
+impl AuthedClient {
+    pub fn get_client(&self) -> &Client {
+        return self.client.get_or_init(|| {
+            windmill_api_client::create_client(&self.base_internal_url, self.token.clone())
+        });
+    }
+}
 
 
 #[tracing::instrument(level = "trace")]
@@ -553,7 +577,7 @@ pub async fn run_worker(
 
 
         let do_break = async {
-            if last_ping.elapsed().as_secs() > NUM_SECS_ENV_CHECK {
+            if last_ping.elapsed().as_secs() > NUM_SECS_PING {
                 sqlx::query!(
                     "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1 WHERE worker = $2",
                     jobs_executed,
@@ -627,7 +651,8 @@ pub async fn run_worker(
                             .with_label_values(label_values.as_slice()),
                     };
 
-                    tracing::info!(worker = %worker_name, id = %job.id, "fetched job {}", job.id);
+                    let job_root = job.root_job.map(|x| x.to_string()).unwrap_or_else(|| "none".to_string());
+                    tracing::info!(worker = %worker_name, id = %job.id, root_id = %job_root, "fetched job {}, root job: {}", job.id, job_root);
 
                     let job_dir = format!("{worker_dir}/{}", job.id);
 
@@ -670,13 +695,13 @@ pub async fn run_worker(
                     )
                     .await.expect("could not create job token");
                     tx.commit().await.expect("could not commit job token");
-                    let job_client = windmill_api_client::create_client(base_internal_url, token.clone());
+                    let authed_client = AuthedClient { base_internal_url: base_internal_url.to_string(), token: token.clone(), workspace: job.workspace_id.to_string(), client: OnceCell::new() };
                     let is_flow = job.job_kind == JobKind::Flow || job.job_kind == JobKind::FlowPreview || job.job_kind == JobKind::FlowDependencies;
 
                     if let Some(err) = handle_queued_job(
                         job.clone(),
                         db,
-                        &job_client,
+                        &authed_client,
                         token,
                         &worker_name,
                         &worker_dir,
@@ -690,7 +715,7 @@ pub async fn run_worker(
                     {
                         handle_job_error(
                             db,
-                            &job_client,
+                            &authed_client,
                             job,
                             err,
                             Some(metrics),
@@ -734,7 +759,7 @@ pub async fn run_worker(
 
 async fn handle_job_error(
     db: &Pool<Postgres>,
-    client: &windmill_api_client::Client,
+    client: &AuthedClient,
     job: QueuedJob,
     err: Error,
     metrics: Option<Metrics>,
@@ -833,14 +858,14 @@ fn extract_error_value(log_lines: &str) -> serde_json::Value {
 async fn handle_queued_job(
     job: QueuedJob,
     db: &sqlx::Pool<sqlx::Postgres>,
-    client: &windmill_api_client::Client,
+    client: &AuthedClient,
     token: String,
     worker_name: &str,
     worker_dir: &str,
     job_dir: &str,
     metrics: Metrics,
     same_worker_tx: Sender<Uuid>,
-    base_internal_url: &str
+    base_internal_url: &str,
 ) -> windmill_common::error::Result<()> {
     if job.canceled {
         return Err(Error::JsonErr(canceled_job_to_result(&job)))?;
@@ -865,6 +890,9 @@ async fn handle_queued_job(
         }
         _ => {
             let mut logs = "".to_string();
+            if let Some(log_str) = &job.logs {
+                logs.push_str(&log_str);
+            }
 
             if job.is_flow_step {
                 update_flow_status_in_progress(
@@ -888,10 +916,10 @@ async fn handle_queued_job(
             logs.push_str(&format!("job {} on worker {}\n", &job.id, &worker_name));
             let result = match job.job_kind {
                 JobKind::Dependencies => {
-                    handle_dependency_job(&job, &mut logs, job_dir, db).await
+                    handle_dependency_job(&job, &mut logs, job_dir, db, worker_name).await
                 }
                 JobKind::FlowDependencies => {
-                    handle_flow_dependency_job(&job, &mut logs, job_dir, db)
+                    handle_flow_dependency_job(&job, &mut logs, job_dir, db, worker_name)
                         .await
                         .map(|()| Value::Null)
                 }
@@ -912,7 +940,8 @@ async fn handle_queued_job(
                         job_dir,
                         worker_dir,
                         &mut logs,
-                        base_internal_url
+                        base_internal_url,
+                        worker_name
                     )
                     .await
                 }
@@ -1012,14 +1041,14 @@ async fn write_file(dir: &str, path: &str, content: &str) -> error::Result<File>
 #[async_recursion]
 async fn transform_json_value(
     name: &str,
-    client: &windmill_api_client::Client,
+    client: &AuthedClient,
     workspace: &str,
     v: Value,
 ) -> error::Result<Value> {
     match v {
         Value::String(y) if y.starts_with("$var:") => {
             let path = y.strip_prefix("$var:").unwrap();
-            let v = client
+            let v = client.get_client()
                 .get_variable(workspace, path, Some(true))
                 .await
                 .map_err(|_| Error::NotFound(format!("Variable {path} not found for `{name}`")))
@@ -1035,7 +1064,7 @@ async fn transform_json_value(
                     "Argument `{name}` is an invalid resource path: {path}",
                 )));
             }
-            let v = client
+            let v = client.get_client()
                 .get_resource_value(workspace, path)
                 .await
                 .map_err(|_| Error::NotFound(format!("Resource {path} not found for `{name}`")))?
@@ -1059,12 +1088,14 @@ async fn transform_json_value(
 async fn handle_code_execution_job(
     job: &QueuedJob,
     db: &sqlx::Pool<sqlx::Postgres>,
-    client: &windmill_api_client::Client,
+    client: &AuthedClient,
     token: String,
     job_dir: &str,
     worker_dir: &str,
     logs: &mut String,
-    base_internal_url: &str
+    base_internal_url: &str,
+    worker_name: &str
+
 ) -> error::Result<serde_json::Value> {
     let (inner_content, requirements_o, language) = match job.job_kind {
         JobKind::Preview | JobKind::Script_Hub => (
@@ -1086,7 +1117,6 @@ async fn handle_code_execution_job(
             "handle_code_execution_job should never be reachable with a non-code execution job"
         ),
     };
-    let worker_name = worker_dir.split("/").last().unwrap_or("unknown");
     let lang_str = job
         .language
         .as_ref()
@@ -1136,7 +1166,7 @@ mount {{
                 token,
                 &inner_content,
                 &shared_mount,
-                base_internal_url
+                base_internal_url,
             )
             .await
         }
@@ -1151,7 +1181,8 @@ mount {{
                 &inner_content,
                 &shared_mount,
                 requirements_o,
-                base_internal_url
+                base_internal_url,
+                worker_name
             )
             .await
         }
@@ -1166,7 +1197,8 @@ mount {{
                 job_dir,
                 requirements_o,
                 &shared_mount,
-                base_internal_url
+                base_internal_url,
+                worker_name
             )
             .await
         }
@@ -1179,7 +1211,8 @@ mount {{
                 &inner_content,
                 job_dir,
                 &shared_mount,
-                base_internal_url
+                base_internal_url,
+                worker_name
             )
             .await
         }
@@ -1201,13 +1234,14 @@ async fn handle_go_job(
     logs: &mut String,
     job: &QueuedJob,
     db: &sqlx::Pool<sqlx::Postgres>,
-    client: &windmill_api_client::Client,
+    client: &AuthedClient,
     token: String,
     inner_content: &str,
     job_dir: &str,
     requirements_o: Option<String>,
     shared_mount: &str,
     base_internal_url: &str,
+    worker_name: &str,
 ) -> Result<serde_json::Value, Error> {
     //go does not like executing modules at temp root
     let job_dir = &format!("{job_dir}/go");
@@ -1236,6 +1270,8 @@ async fn handle_go_job(
         db,
         true,
         skip_go_mod,
+        worker_name,
+        &job.workspace_id,
     )
     .await?;
 
@@ -1353,7 +1389,7 @@ func Run(req Req) (interface{{}}, error){{
             .stdout(Stdio::piped())
             .stderr(Stdio::piped())
             .spawn()?;
-        handle_child(&job.id, db, logs,  build_go, false).await?;
+        handle_child(&job.id, db, logs,  build_go, false, worker_name, &job.workspace_id).await?;
 
         Command::new(NSJAIL_PATH.as_str())
             .current_dir(job_dir)
@@ -1379,7 +1415,7 @@ func Run(req Req) (interface{{}}, error){{
             .stderr(Stdio::piped())
             .spawn()?
     };
-    handle_child(&job.id, db, logs, child, !*DISABLE_NSJAIL).await?;
+    handle_child(&job.id, db, logs, child, !*DISABLE_NSJAIL, worker_name, &job.workspace_id).await?;
     read_result(job_dir).await
 }
 
@@ -1393,6 +1429,7 @@ async fn handle_bash_job(
     job_dir: &str,
     shared_mount: &str,
     base_internal_url: &str,
+    worker_name: &str,
 ) -> Result<serde_json::Value, Error> {
     logs.push_str("\n\n--- BASH CODE EXECUTION ---\n");
     set_logs(logs, &job.id, db).await;
@@ -1429,7 +1466,7 @@ async fn handle_bash_job(
                 .replace("{SHARED_MOUNT}", shared_mount),
         )
         .await?;
-        let mut cmd_args = vec!["--config", "run.config.proto", "--", "/bin/sh", "main.sh"];
+        let mut cmd_args = vec!["--config", "run.config.proto", "--", "/bin/bash", "main.sh"];
         cmd_args.extend(args);
         Command::new(NSJAIL_PATH.as_str())
             .current_dir(job_dir)
@@ -1444,7 +1481,7 @@ async fn handle_bash_job(
     } else {
         let mut cmd_args = vec!["main.sh"];
         cmd_args.extend(&args);
-        Command::new("/bin/sh")
+        Command::new("/bin/bash")
             .current_dir(job_dir)
             .env_clear()
             .envs(reserved_variables)
@@ -1456,7 +1493,7 @@ async fn handle_bash_job(
             .stderr(Stdio::piped())
             .spawn()?
     };
-    handle_child(&job.id, db, logs,  child, !*DISABLE_NSJAIL).await?;
+    handle_child(&job.id, db, logs,  child, !*DISABLE_NSJAIL, worker_name, &job.workspace_id).await?;
     //for now bash jobs have an empty result object
     Ok(serde_json::json!(logs
         .lines()
@@ -1473,18 +1510,39 @@ fn capitalize(s: &str) -> String {
     }
 }
 
+fn get_common_deno_proc_envs(token: &str, base_internal_url: &str) -> HashMap<String, String> {
+    let hostname_base = BASE_URL.split("://").last().unwrap_or("localhost");
+    let hostname_internal = base_internal_url.split("://").last().unwrap_or("localhost");
+    let deno_auth_tokens_base = DENO_AUTH_TOKENS.as_str();
+    let deno_auth_tokens =
+        format!("{token}@{hostname_base};{token}@{hostname_internal}{deno_auth_tokens_base}",);
+
+    let mut deno_envs: HashMap<String, String> = HashMap::from([
+        (String::from("PATH"), PATH_ENV.clone()),
+        (String::from("DENO_AUTH_TOKENS"), deno_auth_tokens),
+        (String::from("BASE_INTERNAL_URL"), base_internal_url.to_string()),
+        (String::from("NO_COLOR"), String::from("true")),
+    ]);
+
+    if let Some(ref s) = *NPM_CONFIG_REGISTRY {
+        deno_envs.insert(String::from("NPM_CONFIG_REGISTRY"), s.clone());
+    }
+    return deno_envs;
+}
+
 #[tracing::instrument(level = "trace", skip_all)]
 async fn handle_deno_job(
     logs: &mut String,
     job: &QueuedJob,
     db: &sqlx::Pool<sqlx::Postgres>,
-    client: &windmill_api_client::Client,
+    client: &AuthedClient,
     token: String,
     job_dir: &str,
     inner_content: &String,
     shared_mount: &str,
     lockfile: Option<String>,
-    base_internal_url: &str
+    base_internal_url: &str,
+    worker_name: &str
 ) -> error::Result<serde_json::Value> {
     logs.push_str("\n\n--- DENO CODE EXECUTION ---\n");
     set_logs(logs, &job.id, db).await;
@@ -1498,14 +1556,14 @@ async fn handle_deno_job(
     // TODO: Separately cache dependencies here using `deno cache --reload --lock=lock.json src/deps.ts` (https://deno.land/manual@v1.27.0/linking_to_external_code/integrity_checking)
     // Then require caching below using --cached-only. This makes it so we require zero network interaction when running the process below
 
-    let _ = write_file(job_dir, "inner.ts", inner_content).await?;
+    let _ = write_file(job_dir, "main.ts", inner_content).await?;
     let sig = trace_span!("parse_deno_signature")
         .in_scope(|| windmill_parser_ts::parse_deno_signature(inner_content))?;
     create_args_and_out_file(client, job, job_dir).await?;
     let spread = sig.args.into_iter().map(|x| x.name).join(",");
     let wrapper_content: String = format!(
         r#"
-import {{ main }} from "./inner.ts";
+import {{ main }} from "./main.ts";
 
 const args = await Deno.readTextFile("args.json")
     .then(JSON.parse)
@@ -1523,25 +1581,35 @@ run().catch(async (e) => {{
 }});
 "#,
     );
-    write_file(job_dir, "main.ts", &wrapper_content).await?;
     let w_id = job.workspace_id.clone();
+    let script_path_split = job.script_path().split("/");
+    let script_path_parts_len = script_path_split.clone().count();
+    let mut relative_mounts = "".to_string();
+    for c in 0..script_path_parts_len {
+        relative_mounts += ",\n          ";
+        relative_mounts += &format!("\"./{}\": \"{base_internal_url}/api/w/{w_id}/scripts/raw/p/{}{}\"",
+            (0..c).map(|_| "../").join(""),
+            &script_path_split.clone().take(script_path_parts_len - c - 1).join("/"),
+            if c == script_path_parts_len - 1 { "" } else { "/" },
+        );
+    }
+    write_file(job_dir, "wrapper.ts", &wrapper_content).await?;
     let import_map = format!(
         r#"{{
         "imports": {{
           "/": "{base_internal_url}/api/w/{w_id}/scripts/raw/p/",
-          "./": "./"
+          "./wrapper.ts": "./wrapper.ts",
+          "./main.ts": "./main.ts"{relative_mounts}
         }}
-      }}"#
+      }}"#,
     );
     write_file(job_dir, "import_map.json", &import_map).await?;
     let mut reserved_variables = get_reserved_variables(job, &token, db).await?;
     reserved_variables.insert("RUST_LOG".to_string(), "info".to_string());
 
-    let hostname_base = BASE_URL.split("://").last().unwrap_or("localhost");
-    let hostname_internal = base_internal_url.split("://").last().unwrap_or("localhost");
-    let deno_auth_tokens_base = DENO_AUTH_TOKENS.as_str();
-    let deno_auth_tokens =
-        format!("{token}@{hostname_base};{token}@{hostname_internal}{deno_auth_tokens_base}",);
+    let common_deno_proc_envs = get_common_deno_proc_envs(&token, base_internal_url);
+    //do not cache local dependencies
+    let reload = format!("--reload={base_internal_url}");
     let child = async {
         Ok(if !*DISABLE_NSJAIL {
             let _ = write_file(
@@ -1565,6 +1633,7 @@ run().catch(async (e) => {{
             }
             args.push("--import-map");
             args.push("/tmp/import_map.json");
+            args.push(&reload);
             args.push("--unstable");
             if let Some(deno_flags) = DENO_FLAGS.as_ref() {
                 for flag in deno_flags {
@@ -1573,26 +1642,25 @@ run().catch(async (e) => {{
             } else {
                 args.push("-A");
             }
-            args.push("/tmp/main.ts");
+            args.push("/tmp/wrapper.ts");
 
             Command::new(NSJAIL_PATH.as_str())
                 .current_dir(job_dir)
                 .env_clear()
                 .envs(reserved_variables)
-                .env("PATH", PATH_ENV.as_str())
-                .env("DENO_AUTH_TOKENS", deno_auth_tokens)
-                .env("BASE_INTERNAL_URL", base_internal_url)
+                .envs(common_deno_proc_envs)
                 .args(args)
                 .stdout(Stdio::piped())
                 .stderr(Stdio::piped())
                 .spawn()?
         } else {
             let mut args = Vec::new();
-            let script_path = format!("{job_dir}/main.ts");
+            let script_path = format!("{job_dir}/wrapper.ts");
             let import_map_path = format!("{job_dir}/import_map.json");
             args.push("run");
             args.push("--import-map");
             args.push(&import_map_path);
+            args.push(&reload);
             args.push("--unstable");
             if let Some(deno_flags) = DENO_FLAGS.as_ref() {
                 for flag in deno_flags {
@@ -1606,11 +1674,8 @@ run().catch(async (e) => {{
                 .current_dir(job_dir)
                 .env_clear()
                 .envs(reserved_variables)
-                .env("PATH", PATH_ENV.as_str())
+                .envs(common_deno_proc_envs)
                 .env("DENO_DIR", DENO_CACHE_DIR)
-                .env("DENO_AUTH_TOKENS", deno_auth_tokens)
-                .env("BASE_INTERNAL_URL", base_internal_url)
-                .env("NO_COLOR", "true")
                 .args(args)
                 .stdout(Stdio::piped())
                 .stderr(Stdio::piped())
@@ -1619,13 +1684,13 @@ run().catch(async (e) => {{
     }
     .instrument(trace_span!("create_deno_jail"))
     .await?;
-    handle_child(&job.id, db, logs, child, !*DISABLE_NSJAIL).await?;
+    handle_child(&job.id, db, logs, child, !*DISABLE_NSJAIL, worker_name, &job.workspace_id).await?;
     read_result(job_dir).await
 }
 
 #[tracing::instrument(level = "trace", skip_all)]
 async fn create_args_and_out_file(
-    client: &windmill_api_client::Client,
+    client: &AuthedClient,
     job: &QueuedJob,
     job_dir: &str,
 ) -> Result<(), Error> {
@@ -1641,7 +1706,7 @@ async fn create_args_and_out_file(
 }
 
 lazy_static! {
-    static ref RELATIVE_IMPORT_REGEX: Regex = Regex::new(r#"(import|from)\s(u|f)\."#).unwrap();
+    static ref RELATIVE_IMPORT_REGEX: Regex = Regex::new(r#"(import|from)\s(((u|f)\.)|\.)"#).unwrap();
 }
 
 #[tracing::instrument(level = "trace", skip_all)]
@@ -1653,7 +1718,7 @@ async fn handle_python_job(
     job: &QueuedJob,
     logs: &mut String,
     db: &sqlx::Pool<sqlx::Postgres>,
-    client: &windmill_api_client::Client,
+    client: &AuthedClient,
     token: String,
     inner_content: &String,
     shared_mount: &str,
@@ -1671,7 +1736,7 @@ async fn handle_python_job(
             if requirements.is_empty() {
                 "".to_string()
             } else {
-                pip_compile(&job.id, &requirements, logs, job_dir, db)
+                pip_compile(&job.id, &requirements, logs, job_dir, db, worker_name, &job.workspace_id)
                     .await
                     .map_err(|e| {
                         Error::ExecutionErr(format!("pip compile failed: {}", e.to_string()))
@@ -1712,9 +1777,14 @@ async fn handle_python_job(
 
     let relative_imports = RELATIVE_IMPORT_REGEX.is_match(&inner_content);
 
-    let _ = write_file(job_dir, "inner.py", inner_content).await?;
+    let script_path_splitted = &job.script_path().split("/");
+    let dirs = script_path_splitted.clone().take(script_path_splitted.clone().count() - 1).join("/").replace("-", "_");
+    let last = script_path_splitted.clone().last().unwrap().replace("-", "_");
+    let module_dir = format!("{}/{}", job_dir, dirs );
+    tokio::fs::create_dir_all(format!("{module_dir}/")).await?;
+    let _ = write_file(&module_dir, &format!("{last}.py"), inner_content).await?;
     if relative_imports {
-        let _ = write_file(job_dir, "loader.py", RELATIVE_PYTHON_LOADER).await?;
+        let _ = write_file(&job_dir, "loader.py", RELATIVE_PYTHON_LOADER).await?;
     }
 
     let sig = windmill_parser_py::parse_python_signature(inner_content)?;
@@ -1775,6 +1845,7 @@ async fn handle_python_job(
             .join("\n")
     };
 
+    let module_dir_dot = dirs.replace("/", ".").replace("-", "_");
     let wrapper_content: String = format!(
         r#"
 import json
@@ -1783,8 +1854,8 @@ import json
 {import_datetime}
 import traceback
 import sys
+from {module_dir_dot} import {last} as inner_script
 
-inner_script = __import__("inner")
 
 with open("args.json") as f:
     kwargs = json.load(f, strict=False)
@@ -1811,7 +1882,7 @@ except Exception as e:
         sys.exit(1)
 "#,
     );
-    write_file(job_dir, "main.py", &wrapper_content).await?;
+    write_file(job_dir, "wrapper.py", &wrapper_content).await?;
 
     let mut reserved_variables = get_reserved_variables(job, &token, db).await?;
     let additional_python_paths_folders = additional_python_paths.iter().join(":");
@@ -1839,6 +1910,7 @@ mount {{
                 .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string())
                 .replace("{SHARED_MOUNT}", shared_mount)
                 .replace("{SHARED_DEPENDENCIES}", shared_deps.as_str())
+                .replace("{MAIN}", format!("{dirs}/{last}").as_str())
                 .replace(
                     "{ADDITIONAL_PYTHON_PATHS}",
                     additional_python_paths_folders.as_str(),
@@ -1870,7 +1942,8 @@ mount {{
                 "--",
                 PYTHON_PATH.as_str(),
                 "-u",
-                "/tmp/main.py",
+                "-m",
+                "wrapper",
             ])
             .stdout(Stdio::piped())
             .stderr(Stdio::piped())
@@ -1882,13 +1955,13 @@ mount {{
             .envs(reserved_variables)
             .env("PATH", PATH_ENV.as_str())
             .env("BASE_INTERNAL_URL", base_internal_url)
-            .args(vec!["-u", "main.py"])
+            .args(vec!["-u", "-m", "wrapper"])
             .stdout(Stdio::piped())
             .stderr(Stdio::piped())
             .spawn()?
     };
 
-    handle_child(&job.id, db, logs, child, !*DISABLE_NSJAIL).await?;
+    handle_child(&job.id, db, logs, child, !*DISABLE_NSJAIL, worker_name, &job.workspace_id).await?;
     read_result(job_dir).await
 }
 
@@ -1918,6 +1991,7 @@ async fn handle_dependency_job(
     logs: &mut String,
     job_dir: &str,
     db: &sqlx::Pool<sqlx::Postgres>,
+    worker_name: &str,
 ) -> error::Result<serde_json::Value> {
     let content = capture_dependency_job(
         &job.id,
@@ -1932,7 +2006,9 @@ async fn handle_dependency_job(
             .unwrap_or_else(|| "no raw code"),
         logs,
         job_dir,
-        db
+        db,
+        worker_name,
+        &job.workspace_id,
     )
     .await;
     match content {
@@ -1967,6 +2043,7 @@ async fn handle_flow_dependency_job(
     logs: &mut String,
     job_dir: &str,
     db: &sqlx::Pool<sqlx::Postgres>,
+    worker_name: &str,
 ) -> error::Result<()> {
     let path = job.script_path.clone().ok_or_else(|| {
         error::Error::InternalErr(
@@ -1997,6 +2074,8 @@ async fn handle_flow_dependency_job(
             logs,
             job_dir,
             db,
+            worker_name,
+            &job.workspace_id,
         )
         .await;
         match new_lock {
@@ -2111,12 +2190,14 @@ async fn capture_dependency_job(
     job_raw_code: &str,
     logs: &mut String,
     job_dir: &str,
-    db: &sqlx::Pool<sqlx::Postgres>
+    db: &sqlx::Pool<sqlx::Postgres>,
+    worker_name: &str,
+    w_id: &str,
 ) -> error::Result<String> {
     match job_language {
         ScriptLang::Python3 => {
             create_dependencies_dir(job_dir).await;
-            pip_compile(job_id, job_raw_code, logs, job_dir, db ).await
+            pip_compile(job_id, job_raw_code, logs, job_dir, db, worker_name, w_id).await
         }
         ScriptLang::Go => {
             install_go_dependencies(
@@ -2127,6 +2208,8 @@ async fn capture_dependency_job(
                 db,
                 false,
                 false,
+                worker_name,
+                w_id
             )
             .await
         }
@@ -2144,6 +2227,8 @@ async fn pip_compile(
     logs: &mut String,
     job_dir: &str,
     db: &Pool<Postgres>,
+    worker_name: &str,
+    w_id: &str,
 ) -> error::Result<String> {
     logs.push_str(&format!("\nresolving dependencies..."));
     set_logs(logs, job_id, db).await;
@@ -2176,7 +2261,7 @@ async fn pip_compile(
         .stdout(Stdio::piped())
         .stderr(Stdio::piped())
         .spawn()?;
-    handle_child(job_id, db, logs,  child, false)
+    handle_child(job_id, db, logs,  child, false, worker_name, &w_id)
         .await
         .map_err(|e| Error::ExecutionErr(format!("Lock file generation failed: {e:?}")))?;
     let path_lock = format!("{job_dir}/requirements.txt");
@@ -2199,6 +2284,8 @@ async fn install_go_dependencies(
     db: &sqlx::Pool<sqlx::Postgres>,
     preview: bool,
     skip_go_mod: bool,
+    worker_name: &str,
+    w_id: &str
 ) -> error::Result<String> {
     if !skip_go_mod {
         gen_go_mymod(code, job_dir).await?;
@@ -2209,7 +2296,7 @@ async fn install_go_dependencies(
             .stderr(Stdio::piped())
             .spawn()?;
 
-        handle_child(job_id, db, logs,   child, false).await?;
+        handle_child(job_id, db, logs,   child, false, worker_name, w_id).await?;
     }
     let child = Command::new(GO_PATH.as_str())
         .current_dir(job_dir)
@@ -2217,7 +2304,7 @@ async fn install_go_dependencies(
         .stdout(Stdio::piped())
         .stderr(Stdio::piped())
         .spawn()?;
-    handle_child(job_id, db, logs,  child, false)
+    handle_child(job_id, db, logs,  child, false, worker_name, &w_id)
         .await
         .map_err(|e| Error::ExecutionErr(format!("Lock file generation failed: {e:?}")))?;
 
@@ -2323,6 +2410,8 @@ async fn handle_child(
     logs: &mut String,
     mut child: Child,
     nsjail: bool,
+    worker_name: &str,
+    w_id: &str,
 ) -> error::Result<()> {
     let update_job_interval = Duration::from_millis(500);
     let write_logs_delay = Duration::from_millis(500);
@@ -2336,11 +2425,14 @@ async fn handle_child(
         tracing::info!("could not get child pid");
     }
     let (set_too_many_logs, mut too_many_logs) = watch::channel::<bool>(false);
+    let (tx, mut rx) = broadcast::channel::<()>(3);
+    let mut rx2 = tx.subscribe();
+
 
     let output = child_joined_output_stream(&mut child);
+
     let job_id = job_id.clone();
 
-    let (tx, mut rx) = mpsc::channel::<()>(1);
 
     /* the cancellation future is polled on by `wait_on_child` while
      * waiting for the child to exit normally */
@@ -2350,10 +2442,22 @@ async fn handle_child(
         let mut interval = interval(update_job_interval);
         interval.set_missed_tick_behavior(MissedTickBehavior::Skip);
 
+        let mut i = 1;
         loop {
             tokio::select!(
                 _ = rx.recv() => break,
                 _ = interval.tick() => {
+                    // update the last_ping column every 5 seconds
+                    i+=1;
+                    if i % 10 == 0 {
+                        sqlx::query!(
+                            "UPDATE worker_ping SET ping_at = now() WHERE worker = $1",
+                            &worker_name
+                        )
+                        .execute(&db)
+                        .await
+                        .expect("update worker ping");
+                    }
                     let mem_peak = get_mem_peak(pid, nsjail).await;
                     tracing::info!("{job_id} still running. mem peak: {}kB", mem_peak);
                     let mem_peak = if mem_peak > 0 { Some(mem_peak) } else { None };
@@ -2383,14 +2487,32 @@ async fn handle_child(
     let wait_on_child = async {
         let db = db.clone();
 
+        #[cfg(not(feature = "enterprise"))]
+        let timeout_duration = *TIMEOUT_DURATION;
+
+        #[cfg(feature = "enterprise")]
+        let premium_workspace = sqlx::query_scalar!("SELECT premium FROM workspace WHERE id = $1", w_id)
+            .fetch_one(&db)
+            .await
+            .map_err(|e| {
+                tracing::error!(%e, "error getting premium workspace for job {job_id}: {e}");
+            }).unwrap_or(false);
+        
+        #[cfg(feature = "enterprise")]
+        let timeout_duration = if premium_workspace {
+            *TIMEOUT_DURATION*6 //30mins
+        } else {
+            *TIMEOUT_DURATION
+        };
+
         let kill_reason = tokio::select! {
             biased;
             result = child.wait() => return result.map(Ok),
             Ok(()) = too_many_logs.changed() => KillReason::TooManyLogs,
+            _ = sleep(timeout_duration) => KillReason::Timeout,
             _ = update_job => KillReason::Cancelled,
-            _ = sleep(*TIMEOUT_DURATION) => KillReason::Timeout,
         };
-        tx.send(()).await.expect("rx should never be dropped");
+        tx.send(()).expect("rx should never be dropped");
         drop(tx);
 
         let set_reason = async {
@@ -2413,7 +2535,7 @@ async fn handle_child(
                 }
             }
         };
-
+        
         /* send SIGKILL and reap child process */
         let (_, kill) = future::join(set_reason, child.kill()).await;
         kill.map(|()| Err(kill_reason))
@@ -2429,12 +2551,13 @@ async fn handle_child(
         /* log_remaining is zero when output limit was reached */
         let mut log_remaining = max_log_size.saturating_sub(logs.chars().count());
         let mut result = io::Result::Ok(());
-        let mut output = output;
+        let mut output = output.take_until(rx2.recv()).boxed();
         /* `do_write` resolves the task, but does not contain the Result.
          * It's useful to know if the task completed. */
         let (mut do_write, mut write_result) = tokio::spawn(ready(())).remote_handle();
 
-        while let Some(line) = output.by_ref().next().await {
+        while let Some(line) =  output.by_ref().next().await {
+
             let do_write_ = do_write.shared();
 
             let mut read_lines = stream::once(async { line })
@@ -2449,11 +2572,14 @@ async fn handle_child(
             let mut joined = String::new();
 
             while let Some(line) = read_lines.next().await {
+
                 match line {
                     Ok(_) if log_remaining == 0 => (),
                     Ok(line) => {
+                        if line.is_empty() {
+                            continue;
+                        }
                         append_with_limit(&mut joined, &line, &mut log_remaining);
-
                         if log_remaining == 0 {
                             tracing::info!(%job_id, "Too many logs lines for job {job_id}");
                             let _ = set_too_many_logs.send(true);
@@ -2473,6 +2599,7 @@ async fn handle_child(
 
             logs.push_str(&joined);
 
+
             /* Ensure the last flush completed before starting a new one.
              *
              * This shouldn't pause since `take_until()` reads lines until `do_write`
@@ -2489,8 +2616,7 @@ async fn handle_child(
                 panic::resume_unwind(p);
             }
 
-            (do_write, write_result) =
-                tokio::spawn(append_logs(job_id, joined, db.clone())).remote_handle();
+            (do_write, write_result) = tokio::spawn(append_logs(job_id, joined, db.clone())).remote_handle();
 
             if let Err(err) = result {
                 tracing::error!(%job_id, %err, "error reading output for job {job_id}: {err}");
@@ -2500,6 +2626,7 @@ async fn handle_child(
             if *set_too_many_logs.borrow() {
                 break;
             }
+
         }
 
         /* drop our end of the pipe */
@@ -2641,7 +2768,7 @@ pub async fn handle_zombie_jobs_periodically(
         handle_zombie_jobs(db, base_internal_url).await;
 
         tokio::select! {
-            _ = tokio::time::sleep(Duration::from_secs(60))    => (),
+            _ = tokio::time::sleep(Duration::from_secs(30))    => (),
             _ = rx.recv() => {
                     println!("received killpill for monitor job");
                     break;
@@ -2651,28 +2778,35 @@ pub async fn handle_zombie_jobs_periodically(
 }
 
 async fn handle_zombie_jobs(db: &Pool<Postgres>, base_internal_url: &str) {
-    let restarted = sqlx::query!(
-            "UPDATE queue SET running = false WHERE last_ping < now() - ($1 || ' seconds')::interval AND running = true AND job_kind != $2 AND same_worker = false RETURNING id, workspace_id, last_ping",
-            *ZOMBIE_JOB_TIMEOUT,
-            JobKind::Flow: JobKind,
-        )
-        .fetch_all(db)
-        .await
-        .ok()
-        .unwrap_or_else(|| vec![]);
+    if *RESTART_ZOMBIE_JOBS {
+        let restarted = sqlx::query!(
+                "UPDATE queue SET running = false, started_at = null, logs = logs || '\nRestarted job after not receiving job''s ping for too long the ' || now() || '\n\n' WHERE last_ping < now() - ($1 || ' seconds')::interval AND running = true AND job_kind != $2 AND job_kind != $3 AND same_worker = false RETURNING id, workspace_id, last_ping",
+                *ZOMBIE_JOB_TIMEOUT,
+                JobKind::Flow: JobKind,
+                JobKind::FlowPreview: JobKind,
+            )
+            .fetch_all(db)
+            .await
+            .ok()
+            .unwrap_or_else(|| vec![]);
 
-    QUEUE_ZOMBIE_RESTART_COUNT.inc_by(restarted.len() as _);
-    for r in restarted {
-        tracing::info!(
-            "restarted zombie job {} {} {}",
-            r.id,
-            r.workspace_id,
-            r.last_ping
-        );
+        QUEUE_ZOMBIE_RESTART_COUNT.inc_by(restarted.len() as _);
+        for r in restarted {
+            tracing::info!(
+                "restarted zombie job {} {} {}",
+                r.id,
+                r.workspace_id,
+                r.last_ping
+            );
+        }
     }
 
+    let mut timeout_query = "SELECT * FROM queue WHERE last_ping < now() - ($1 || ' seconds')::interval AND running = true AND job_kind != $2".to_string();
+    if *RESTART_ZOMBIE_JOBS  {
+        timeout_query.push_str(" AND same_worker = true");
+    };
     let timeouts = sqlx::query_as::<_, QueuedJob>(
-            "SELECT * FROM queue WHERE last_ping < now() - ($1 || ' seconds')::interval AND running = true AND job_kind != $2 AND same_worker = true",
+            &timeout_query
         )
         .bind(ZOMBIE_JOB_TIMEOUT.as_str())
         .bind(JobKind::Flow)
@@ -2684,7 +2818,7 @@ async fn handle_zombie_jobs(db: &Pool<Postgres>, base_internal_url: &str) {
     QUEUE_ZOMBIE_DELETE_COUNT.inc_by(timeouts.len() as _);
     for job in timeouts {
         tracing::info!(
-            "timedouts zombie same_worker job {} {}",
+            "timedout zombie job {} {}",
             job.id,
             job.workspace_id,
         );
@@ -2704,13 +2838,15 @@ async fn handle_zombie_jobs(db: &Pool<Postgres>, base_internal_url: &str) {
         .await
         .expect("could not create job token");
         tx.commit().await.expect("could not commit job token");
-        let client = windmill_api_client::create_client(base_internal_url, token.clone());
+        let client = AuthedClient { base_internal_url: base_internal_url.to_string(), token: token.clone(), workspace: job.workspace_id.to_string(), client: OnceCell::new() };
 
+        let last_ping = job.last_ping.clone();
         let _ = handle_job_error(
             db,
             &client,
             job,
-            error::Error::ExecutionErr("Same worker job timed out".to_string()),
+            error::Error::ExecutionErr(format!("Job timed out after no ping from job since {} (ZOMBIE_JOB_TIMEOUT: {})",
+                last_ping.map(|x| x.to_string()).unwrap_or_else(|| "no ping".to_string()), *ZOMBIE_JOB_TIMEOUT)),
             None,
             true,
             same_worker_tx_never_used,
@@ -2813,7 +2949,7 @@ async fn handle_python_reqs(
                 .spawn()?
         };
 
-        let child = handle_child(&job.id, db, logs, child, false).await;
+        let child = handle_child(&job.id, db, logs, child, false, worker_name, &job.workspace_id).await;
         tracing::info!(
             worker_name = %worker_name,
             job_id = %job.id,

backend/windmill-worker/src/worker_flow.rs

@@ -11,8 +11,8 @@ use std::iter;
 use std::time::Duration;
 
 use crate::jobs::{add_completed_job, add_completed_job_error, schedule_again_if_scheduled};
-use crate::js_eval::{eval_timeout, EvalCreds, IdContext};
-use crate::{worker, KEEP_JOB_DIR};
+use crate::js_eval::{eval_timeout, IdContext};
+use crate::{worker, AuthedClient, KEEP_JOB_DIR};
 use anyhow::Context;
 use async_recursion::async_recursion;
 use dyn_iter::DynIter;
@@ -40,7 +40,7 @@ use windmill_queue::{
 // #[instrument(level = "trace", skip_all)]
 pub async fn update_flow_status_after_job_completion(
     db: &DB,
-    client: &windmill_api_client::Client,
+    client: &AuthedClient,
     flow: uuid::Uuid,
     job_id_for_status: &Uuid,
     w_id: &str,
@@ -120,8 +120,7 @@ pub async fn update_flow_status_after_job_completion(
 
         let stop_early = success
             && if let Some(expr) = r.stop_early_expr.clone() {
-                compute_bool_from_expr(expr, &r.args, result.clone(), None, None, base_internal_url)
-                    .await?
+                compute_bool_from_expr(expr, &r.args, result.clone(), None, Some(client)).await?
             } else {
                 false
             };
@@ -326,6 +325,21 @@ pub async fn update_flow_status_after_job_completion(
             )
             .execute(&mut tx)
             .await?;
+
+            if let Some(job_result) = new_status.job_result() {
+                sqlx::query!(
+                    "
+            UPDATE queue
+               SET leaf_jobs = JSONB_SET(coalesce(leaf_jobs, '{}'::jsonb), ARRAY[$1::TEXT], $2)
+             WHERE COALESCE((SELECT root_job FROM queue WHERE id = $3), $3) = id
+            ",
+                    new_status.id(),
+                    json!(job_result),
+                    flow
+                )
+                .execute(&mut tx)
+                .await?;
+            }
         }
     }
 
@@ -381,6 +395,12 @@ pub async fn update_flow_status_after_job_completion(
         .await?
         .ok_or_else(|| Error::InternalErr(format!("requiring flow to be in the queue")))?;
 
+    let job_root = flow_job
+        .root_job
+        .map(|x| x.to_string())
+        .unwrap_or_else(|| "none".to_string());
+    tracing::info!(id = %flow_job.id, root_id = %job_root, "update flow status");
+
     let raw_flow = flow_job.parse_raw_flow();
     let module = raw_flow.as_ref().and_then(|module| {
         module_index.and_then(|i| module.modules.get(i).or(module.failure_module.as_ref()))
@@ -583,8 +603,7 @@ async fn compute_bool_from_expr(
     flow_args: &Option<serde_json::Value>,
     result: serde_json::Value,
     by_id: Option<IdContext>,
-    creds: Option<EvalCreds>,
-    base_internal_url: &str,
+    client: Option<&AuthedClient>,
 ) -> error::Result<bool> {
     let flow_input = flow_args.clone().unwrap_or_else(|| json!({}));
     match eval_timeout(
@@ -595,9 +614,8 @@ async fn compute_bool_from_expr(
             ("previous_result".to_string(), result),
         ]
         .into(),
-        creds,
+        client,
         by_id,
-        base_internal_url,
     )
     .await?
     {
@@ -672,12 +690,10 @@ async fn transform_input(
     flow_args: &Option<serde_json::Value>,
     last_result: serde_json::Value,
     input_transforms: &HashMap<String, InputTransform>,
-    workspace: &str,
-    token: &str,
     resumes: &[Value],
     approvers: Vec<String>,
     by_id: &IdContext,
-    base_internal_url: &str,
+    client: &AuthedClient,
 ) -> windmill_common::error::Result<Map<String, serde_json::Value>> {
     let mut mapped = serde_json::Map::new();
 
@@ -716,19 +732,13 @@ async fn transform_input(
                     context.push(("error".to_string(), error.unwrap().clone()));
                 }
 
-                let v = eval_timeout(
-                    expr.to_string(),
-                    context,
-                    Some(EvalCreds { workspace: workspace.to_string(), token: token.to_string() }),
-                    Some(by_id.clone()),
-                    base_internal_url,
-                )
-                .await
-                .map_err(|e| {
-                    Error::ExecutionErr(format!(
-                        "Error during isolated evaluation of expression `{expr}`:\n{e}"
-                    ))
-                })?;
+                let v = eval_timeout(expr.to_string(), context, Some(client), Some(by_id.clone()))
+                    .await
+                    .map_err(|e| {
+                        Error::ExecutionErr(format!(
+                            "Error during isolated evaluation of expression `{expr}`:\n{e}"
+                        ))
+                    })?;
                 mapped.insert(key.to_string(), v);
                 ()
             }
@@ -742,7 +752,7 @@ async fn transform_input(
 pub async fn handle_flow(
     flow_job: &QueuedJob,
     db: &sqlx::Pool<sqlx::Postgres>,
-    client: &windmill_api_client::Client,
+    client: &AuthedClient,
     last_result: serde_json::Value,
     same_worker_tx: Sender<Uuid>,
     worker_dir: &str,
@@ -782,12 +792,18 @@ async fn push_next_flow_job(
     mut status: FlowStatus,
     flow: FlowValue,
     db: &sqlx::Pool<sqlx::Postgres>,
-    client: &windmill_api_client::Client,
+    client: &AuthedClient,
     mut last_result: serde_json::Value,
     same_worker_tx: Sender<Uuid>,
     worker_dir: &str,
     base_internal_url: &str,
 ) -> error::Result<()> {
+    let job_root = flow_job
+        .root_job
+        .map(|x| x.to_string())
+        .unwrap_or_else(|| "none".to_string());
+    tracing::info!(id = %flow_job.id, root_id = %job_root, "pushing next flow job");
+
     let mut i = usize::try_from(status.step)
         .with_context(|| format!("invalid module index {}", status.step))?;
 
@@ -856,7 +872,6 @@ async fn push_next_flow_job(
                         .into(),
                         None,
                         None,
-                        base_internal_url,
                     )
                     .await
                     .map_err(|e| {
@@ -1099,15 +1114,15 @@ async fn push_next_flow_job(
         _ => (),
     }
 
-    let mut transform_context: Option<TransformContext> = None;
+    let mut transform_context: Option<IdContext> = None;
 
     let args: windmill_common::error::Result<_> = match &module.value {
         FlowModuleValue::Script { input_transforms, .. }
         | FlowModuleValue::RawScript { input_transforms, .. }
         | FlowModuleValue::Flow { input_transforms, .. } => {
-            let ctx = get_transform_context(db, &flow_job, previous_id.clone(), &status).await?;
+            let ctx = get_transform_context(&flow_job, previous_id.clone(), &status).await?;
             transform_context = Some(ctx);
-            let (token, by_id) = transform_context.as_ref().unwrap();
+            let by_id = transform_context.as_ref().unwrap();
             transform_input(
                 &flow_job.args,
                 last_result.clone(),
@@ -1116,12 +1131,10 @@ async fn push_next_flow_job(
                 } else {
                     &module.input_transforms
                 },
-                &flow_job.workspace_id,
-                &token,
                 resume_messages.as_slice(),
                 approvers,
                 by_id,
-                base_internal_url,
+                client,
             )
             .await
         }
@@ -1154,17 +1167,17 @@ async fn push_next_flow_job(
         flow_job,
         &flow,
         transform_context,
-        db,
         tx,
         &module,
         &status,
         &status_module,
         last_result.clone(),
         previous_id,
-        base_internal_url,
+        client,
     )
     .await?;
     tx.commit().await?;
+    tracing::info!(id = %flow_job.id, root_id = %job_root, "next flow transform computed");
 
     let (job_payloads, next_status) = match next_flow_transform {
         NextFlowTransform::Continue(job_payload, next_state) => (job_payload, next_state),
@@ -1237,6 +1250,7 @@ async fn push_next_flow_job(
             Ok(v) => (Some(v), None),
             Err(e) => (None, Some(e)),
         };
+        let root_job = flow_job.root_job.or_else(|| Some(flow_job.id));
         let (uuid, inner_tx) = push(
             tx,
             &flow_job.workspace_id,
@@ -1248,6 +1262,7 @@ async fn push_next_flow_job(
             scheduled_for_o,
             flow_job.schedule_path.clone(),
             Some(flow_job.id),
+            root_job,
             true,
             continue_on_same_worker,
             err,
@@ -1359,6 +1374,7 @@ async fn push_next_flow_job(
     };
 
     tx.commit().await?;
+    tracing::info!(id = %flow_job.id, root_id = %job_root, "all next flow jobs pushed");
 
     if continue_on_same_worker {
         if !is_one_uuid {
@@ -1490,20 +1506,17 @@ async fn script_path_to_payload<'c>(
     Ok(job_payload)
 }
 
-type TransformContext = (String, IdContext);
-
 async fn compute_next_flow_transform<'c>(
     flow_job: &QueuedJob,
     flow: &FlowValue,
-    transform_context: Option<TransformContext>,
-    db: &DB,
+    by_id: Option<IdContext>,
     mut tx: sqlx::Transaction<'c, sqlx::Postgres>,
     module: &FlowModule,
     status: &FlowStatus,
     status_module: &FlowStatusModule,
     last_result: serde_json::Value,
     previous_id: String,
-    base_internal_url: &str,
+    client: &AuthedClient,
 ) -> error::Result<(sqlx::Transaction<'c, sqlx::Postgres>, NextFlowTransform)> {
     match &module.value {
         FlowModuleValue::Identity => Ok((
@@ -1534,7 +1547,7 @@ async fn compute_next_flow_transform<'c>(
         FlowModuleValue::RawScript { path, content, language, lock, .. } => {
             let path = path
                 .clone()
-                .or_else(|| Some(format!("{}/{}", flow_job.script_path(), status.step)));
+                .or_else(|| Some(format!("{}/step-{}", flow_job.script_path(), status.step)));
             Ok((
                 tx,
                 NextFlowTransform::Continue(
@@ -1556,10 +1569,10 @@ async fn compute_next_flow_transform<'c>(
                 FlowStatusModule::WaitingForPriorSteps { .. }
                 | FlowStatusModule::WaitingForEvents { .. }
                 | FlowStatusModule::WaitingForExecutor { .. } => {
-                    let (token, by_id) = if let Some(x) = transform_context {
+                    let by_id = if let Some(x) = by_id {
                         x
                     } else {
-                        get_transform_context(db, &flow_job, previous_id, &status).await?
+                        get_transform_context(&flow_job, previous_id, &status).await?
                     };
                     let flow_input = flow_job.args.clone().unwrap_or_else(|| json!({}));
                     /* Iterator is an InputTransform, evaluate it into an array. */
@@ -1572,10 +1585,8 @@ async fn compute_next_flow_transform<'c>(
                                 ("previous_result".to_string(), last_result.clone()),
                             ]
                         },
-                        token,
-                        flow_job.workspace_id.clone(),
+                        Some(client),
                         Some(by_id),
-                        base_internal_url,
                     )
                     .await?
                     .into_array()
@@ -1692,19 +1703,14 @@ async fn compute_next_flow_transform<'c>(
                 | FlowStatusModule::WaitingForEvents { .. }
                 | FlowStatusModule::WaitingForExecutor { .. } => {
                     let mut branch_chosen = BranchChosen::Default;
-                    let (token, idcontext) =
-                        get_transform_context(db, &flow_job, previous_id, &status).await?;
+                    let idcontext = get_transform_context(&flow_job, previous_id, &status).await?;
                     for (i, b) in branches.iter().enumerate() {
                         let pred = compute_bool_from_expr(
                             b.expr.to_string(),
                             &flow_job.args,
                             last_result.clone(),
                             Some(idcontext.clone()),
-                            Some(EvalCreds {
-                                workspace: flow_job.workspace_id.clone(),
-                                token: token.to_string(),
-                            }),
-                            base_internal_url,
+                            Some(client),
                         )
                         .await?;
 
@@ -1869,59 +1875,31 @@ async fn compute_next_flow_transform<'c>(
 }
 
 async fn get_transform_context(
-    db: &DB,
     flow_job: &QueuedJob,
     previous_id: String,
     status: &FlowStatus,
-) -> error::Result<TransformContext> {
-    let tx = db.begin().await?;
-    let (tx, new_token) = crate::create_token_for_owner(
-        tx,
-        &flow_job.workspace_id,
-        &flow_job.permissioned_as,
-        "transform-input",
-        10,
-        &flow_job.email,
-    )
-    .await?;
-    //we need to commit asap otherwise the token won't be valid for auth to check outside of this transaction
-    //which will happen with client http calls
-    tx.commit().await?;
+) -> error::Result<IdContext> {
     let steps_results: HashMap<String, JobResult> = status
         .modules
         .iter()
         .filter_map(|x| x.job_result().map(|y| (x.id(), y)))
         .collect();
 
-    Ok((
-        new_token,
-        IdContext { flow_job: flow_job.id, steps_results, previous_id },
-    ))
+    Ok(IdContext { flow_job: flow_job.id, steps_results, previous_id })
 }
 
 async fn evaluate_with<F>(
     transform: InputTransform,
     vars: F,
-    token: String,
-    workspace: String,
+    client: Option<&AuthedClient>,
     by_id: Option<IdContext>,
-    base_internal_url: &str,
 ) -> anyhow::Result<serde_json::Value>
 where
     F: FnOnce() -> Vec<(String, serde_json::Value)>,
 {
     match transform {
         InputTransform::Static { value } => Ok(value),
-        InputTransform::Javascript { expr } => {
-            eval_timeout(
-                expr,
-                vars(),
-                Some(EvalCreds { workspace, token }),
-                by_id,
-                base_internal_url,
-            )
-            .await
-        }
+        InputTransform::Javascript { expr } => eval_timeout(expr, vars(), client, by_id).await,
     }
 }
 trait IntoArray: Sized {

cli/apps.ts

@@ -1,12 +1,22 @@
-import { Any, model, property } from "./decoverto.ts";
+import { requireLogin, resolveWorkspace, validatePath } from "./context.ts";
+import { Any, decoverto, model, property } from "./decoverto.ts";
 import {
   AppService,
   AppWithLastVersion,
   colors,
+  Command,
+  ListableApp,
   microdiff,
   Policy,
+  Table,
 } from "./deps.ts";
-import { Difference, PushDiffs, Resource, setValueByPath } from "./types.ts";
+import {
+  Difference,
+  GlobalOptions,
+  PushDiffs,
+  Resource,
+  setValueByPath,
+} from "./types.ts";
 
 @model()
 export class AppFile implements Resource, PushDiffs {
@@ -17,7 +27,6 @@ export class AppFile implements Resource, PushDiffs {
   @property(Any)
   policy: Policy;
 
-
   constructor(value: string, summary: string, policy: Policy) {
     this.value = value;
     this.summary = summary;
@@ -26,16 +35,20 @@ export class AppFile implements Resource, PushDiffs {
   async pushDiffs(
     workspace: string,
     remotePath: string,
-    diffs: Difference[],
+    diffs: Difference[]
   ): Promise<void> {
-    if (await AppService.existsApp({ workspace, path: remotePath })) {
+    let app: AppWithLastVersion | undefined = undefined;
+    try {
+      app = await AppService.getAppByPath({ workspace, path: remotePath });
+    } catch (e) {}
+
+    if (app) {
       console.log(
         colors.bold.yellow(
-          `Applying ${diffs.length} diffs to existing app...`,
-        ),
+          `Applying ${diffs.length} diffs to existing app... ${remotePath}`
+        )
       );
       const changeset: {
-        path?: string | undefined;
         summary?: string | undefined;
         value?: any;
         policy?: Policy | undefined;
@@ -43,14 +56,10 @@ export class AppFile implements Resource, PushDiffs {
       for (const diff of diffs) {
         if (
           diff.type !== "REMOVE" &&
-          (
-            diff.path[0] !== "value" && diff.path[0] !== "policy" && (
-              diff.path.length !== 1 ||
-              !["path", "summary"].includes(
-                diff.path[0] as string,
-              )
-            )
-          )
+          diff.path[0] !== "value" &&
+          diff.path[0] !== "policy" &&
+          (diff.path.length !== 1 ||
+            !["summary"].includes(diff.path[0] as string))
         ) {
           throw new Error("Invalid app diff with path " + diff.path);
         }
@@ -61,8 +70,21 @@ export class AppFile implements Resource, PushDiffs {
         }
       }
 
-      const hasChanges = Object.values(changeset).some((v) =>
-        v !== null && typeof v !== "undefined"
+      if (
+        (!changeset?.policy ||
+          JSON.stringify(changeset?.policy) == JSON.stringify(app.policy)) &&
+        (!changeset?.value ||
+          JSON.stringify(changeset?.value) == JSON.stringify(app.value)) &&
+        (!changeset?.summary || changeset.summary == app.summary)
+      ) {
+        console.log(
+          colors.yellow(`No changes to push for app ${remotePath}, skipping`)
+        );
+        return;
+      }
+
+      const hasChanges = Object.values(changeset).some(
+        (v) => v !== null && typeof v !== "undefined"
       );
       if (!hasChanges) {
         return;
@@ -75,6 +97,7 @@ export class AppFile implements Resource, PushDiffs {
       });
     } else {
       console.log(colors.yellow.bold("Creating new app..."));
+
       await AppService.createApp({
         workspace,
         requestBody: {
@@ -87,19 +110,70 @@ export class AppFile implements Resource, PushDiffs {
     }
   }
   async push(workspace: string, remotePath: string): Promise<void> {
-    let existing: AppWithLastVersion | undefined;
-    try {
-      existing = await AppService.getAppByPath({
-        workspace: workspace,
-        path: remotePath,
-      });
-    } catch {
-      existing = undefined;
-    }
     await this.pushDiffs(
       workspace,
       remotePath,
-      microdiff(existing ?? {}, this, { cyclesFix: false }),
+      microdiff({}, this, { cyclesFix: false })
     );
   }
 }
+
+async function list(opts: GlobalOptions) {
+  const workspace = await resolveWorkspace(opts);
+  await requireLogin(opts);
+
+  let page = 0;
+  const perPage = 10;
+  const total: ListableApp[] = [];
+  while (true) {
+    const res = await AppService.listApps({
+      workspace: workspace.workspaceId,
+      page,
+      perPage,
+    });
+    page += 1;
+    total.push(...res);
+    if (res.length < perPage) {
+      break;
+    }
+  }
+
+  new Table()
+    .header(["path", "summary"])
+    .padding(2)
+    .border(true)
+    .body(total.map((x) => [x.path, x.summary]))
+    .render();
+}
+
+async function push(opts: GlobalOptions, filePath: string) {
+  const remotePath = filePath.split(".")[0];
+  if (!validatePath(remotePath)) {
+    return;
+  }
+  const workspace = await resolveWorkspace(opts);
+  await requireLogin(opts);
+
+  await pushApp(filePath, workspace.workspaceId, remotePath);
+  console.log(colors.bold.underline.green("App pushed"));
+}
+
+export async function pushApp(
+  filePath: string,
+  workspace: string,
+  remotePath: string
+) {
+  const data = decoverto
+    .type(AppFile)
+    .rawToInstance(await Deno.readTextFile(filePath));
+  await data.push(workspace, remotePath);
+}
+
+const command = new Command()
+  .description("app related commands")
+  .action(list as any)
+  .command("push", "push a local app ")
+  .arguments("<file_path:file>")
+  .action(push as any);
+
+export default command;

cli/context.ts

@@ -18,12 +18,12 @@ export type Context = {
 };
 
 async function tryResolveWorkspace(
-  opts: GlobalOptions,
+  opts: GlobalOptions
 ): Promise<
   { isError: false; value: Workspace } | { isError: true; error: string }
 > {
   const cache = (opts as any).__secret_workspace;
-  if (cache) return cache;
+  if (cache) return { isError: false, value: cache };
 
   if (opts.workspace) {
     const e = await getWorkspaceByName(opts.workspace);
@@ -49,21 +49,23 @@ async function tryResolveWorkspace(
 }
 
 export async function resolveWorkspace(
-  opts: GlobalOptions,
+  opts: GlobalOptions
 ): Promise<Workspace> {
   const res = await tryResolveWorkspace(opts);
   if (res.isError) {
-    console.log(res.error);
+    console.log(colors.red.bold(res.error));
     return Deno.exit(-1);
   } else {
     return res.value;
   }
 }
 
-export async function requireLogin(opts: GlobalOptions): Promise<GlobalUserInfo> {
+export async function requireLogin(
+  opts: GlobalOptions
+): Promise<GlobalUserInfo> {
   const workspace = await resolveWorkspace(opts);
-
   let token = await tryGetLoginInfo(opts);
+
   if (!token) {
     token = workspace.token;
   }
@@ -74,26 +76,26 @@ export async function requireLogin(opts: GlobalOptions): Promise<GlobalUserInfo>
     return await UserService.globalWhoami();
   } catch {
     console.log(
-      "! Could not reach API given existing credentials. Attempting to reauth...",
+      "! Could not reach API given existing credentials. Attempting to reauth..."
     );
     const newToken = await loginInteractive(workspace.remote);
     if (!newToken) {
       throw new Error("Could not reauth");
     }
-    removeWorkspace(workspace.name);
+    removeWorkspace(workspace.name, false, opts);
     workspace.token = newToken;
-    addWorkspace(workspace);
+    addWorkspace(workspace, opts);
 
     setClient(
       token,
-      workspace.remote.substring(0, workspace.remote.length - 1),
+      workspace.remote.substring(0, workspace.remote.length - 1)
     );
     return await UserService.globalWhoami();
   }
 }
 
 export async function tryResolveVersion(
-  opts: GlobalOptions,
+  opts: GlobalOptions
 ): Promise<number | undefined> {
   if ((opts as any).__cache_version) {
     return (opts as any).__cache_version;
@@ -103,41 +105,24 @@ export async function tryResolveVersion(
   if (workspaceRes.isError) return undefined;
 
   const response = await fetch(
-    new URL(new URL(workspaceRes.value.remote).origin + "/api/version"),
+    new URL(new URL(workspaceRes.value.remote).origin + "/api/version")
   );
   const version = await response.text();
   try {
     return Number.parseInt(
-      version.split("-", 1)[0].replaceAll(".", "").replace("v", ""),
+      version.split("-", 1)[0].replaceAll(".", "").replace("v", "")
     );
   } catch {
     return undefined;
   }
 }
 
-export async function validatePath(
-  opts: GlobalOptions,
-  path: string,
-): Promise<boolean> {
-  const backendVersion = await tryResolveVersion(opts);
-  if (path.startsWith("f")) {
-    if (!backendVersion || backendVersion >= 1550) {
-      return true;
-    }
-    console.log(
-      `Attempting to use folders, but the current remote does not have support. Remote version is ${backendVersion} but folders are supported from 1560.`,
-    );
-    return false;
-  }
-
-  if (
-    !(path.startsWith("g") ||
-      path.startsWith("u"))
-  ) {
+export function validatePath(path: string): boolean {
+  if (!(path.startsWith("g") || path.startsWith("u") || path.startsWith("f"))) {
     console.log(
       colors.red(
-        "Given remote path looks invalid. Remote paths are typically of the form <u|g|f>/<username|group|folder>/...",
-      ),
+        "Given remote path looks invalid. Remote paths are typically of the form <u|g|f>/<username|group|folder>/..."
+      )
     );
     return false;
   }

cli/deps.ts

@@ -33,9 +33,7 @@ export { passwordGenerator } from "https://deno.land/x/password_generator@latest
 export { nanoid } from "https://deno.land/x/nanoid@v3.0.0/mod.ts";
 export * as cbor from "https://deno.land/x/cbor@v1.4.1/index.js";
 export { default as Murmurhash3 } from "https://deno.land/x/murmurhash@v1.0.0/mod.ts";
-export {
-  default as microdiff,
-} from "https://deno.land/x/microdiff@v1.3.1/index.ts";
+export { default as microdiff } from "https://deno.land/x/microdiff@v1.3.1/index.ts";
 export { default as objectHash } from "https://deno.land/x/object_hash@2.0.3.1/mod.ts";
 export { default as gitignore_parser } from "npm:gitignore-parser";
 export { default as JSZip } from "npm:jszip@3.7.1";

cli/flow.ts

@@ -20,7 +20,6 @@ import { requireLogin, resolveWorkspace, validatePath } from "./context.ts";
 import { resolve, track_job } from "./script.ts";
 import { Any, decoverto, model, property } from "./decoverto.ts";
 
-
 // this is effectively "OpenFlow" but a copy as it is accepted by the CLI
 @model()
 export class FlowFile implements Resource, PushDiffs {
@@ -40,7 +39,7 @@ export class FlowFile implements Resource, PushDiffs {
   async pushDiffs(
     workspace: string,
     remotePath: string,
-    diffs: Difference[],
+    diffs: Difference[]
   ): Promise<void> {
     if (
       await FlowService.existsFlowByPath({
@@ -50,8 +49,8 @@ export class FlowFile implements Resource, PushDiffs {
     ) {
       console.log(
         colors.bold.yellow(
-          `Applying ${diffs.length} diffs to existing flow... ${remotePath}`,
-        ),
+          `Applying ${diffs.length} diffs to existing flow... ${remotePath}`
+        )
       );
 
       // TODO: Make these optional in backend (not path ofc)
@@ -66,14 +65,11 @@ export class FlowFile implements Resource, PushDiffs {
       for (const diff of diffs) {
         if (
           diff.type !== "REMOVE" &&
-          (
-            diff.path[0] !== "value" && (
-              diff.path.length !== 1 ||
-              !["summary", "description", "schema"].includes(
-                diff.path[0] as string,
-              )
-            )
-          )
+          diff.path[0] !== "value" &&
+          (diff.path.length !== 1 ||
+            !["summary", "description", "schema"].includes(
+              diff.path[0] as string
+            ))
         ) {
           throw new Error("Invalid flow diff with path " + diff.path);
         }
@@ -83,8 +79,8 @@ export class FlowFile implements Resource, PushDiffs {
           setValueByPath(changeset, diff.path, null);
         }
       }
-      const hasChanges = Object.values(changeset).some((v) =>
-        v !== null && typeof v !== "undefined"
+      const hasChanges = Object.values(changeset).some(
+        (v) => v !== null && typeof v !== "undefined"
       );
       if (!hasChanges) {
         return;
@@ -93,7 +89,8 @@ export class FlowFile implements Resource, PushDiffs {
       const update = {
         ...changeset,
         ...base_changeset,
-      }
+      };
+
       await FlowService.updateFlow({
         workspace: workspace,
         path: remotePath,
@@ -114,20 +111,10 @@ export class FlowFile implements Resource, PushDiffs {
     }
   }
   async push(workspace: string, remotePath: string): Promise<void> {
-    let remote: Flow | undefined;
-    try {
-      remote = await FlowService.getFlowByPath({
-        workspace,
-        path: remotePath,
-      });
-    } catch {
-
-      remote = undefined;
-    }
     await this.pushDiffs(
       workspace,
       remotePath,
-      microdiff(remote ?? {}, this, { cyclesFix: false }),
+      microdiff({}, this, { cyclesFix: false })
     );
   }
 }
@@ -135,24 +122,24 @@ export class FlowFile implements Resource, PushDiffs {
 type Options = GlobalOptions;
 
 async function push(opts: Options, filePath: string, remotePath: string) {
-  if (!await validatePath(opts, remotePath)) {
+  if (!validatePath(remotePath)) {
     return;
   }
   const workspace = await resolveWorkspace(opts);
   await requireLogin(opts);
 
-  await pushFlow(filePath, workspace.remote, remotePath);
+  await pushFlow(filePath, workspace.workspaceId, remotePath);
   console.log(colors.bold.underline.green("Flow pushed"));
 }
 
 export async function pushFlow(
   filePath: string,
   workspace: string,
-  remotePath: string,
+  remotePath: string
 ) {
-  const data = decoverto.type(FlowFile).rawToInstance(
-    await Deno.readTextFile(filePath),
-  );
+  const data = decoverto
+    .type(FlowFile)
+    .rawToInstance(await Deno.readTextFile(filePath));
   await data.push(workspace, remotePath);
 }
 
@@ -181,26 +168,20 @@ async function list(opts: GlobalOptions & { showArchived?: boolean }) {
     .header(["path", "summary", "edited by"])
     .padding(2)
     .border(true)
-    .body(
-      total.map((x) => [
-        x.path,
-        x.summary,
-        x.edited_by,
-      ]),
-    )
+    .body(total.map((x) => [x.path, x.summary, x.edited_by]))
     .render();
 }
 async function run(
   opts: GlobalOptions & {
-    input: string[];
+    data?: string;
     silent: boolean;
   },
-  path: string,
+  path: string
 ) {
   const workspace = await resolveWorkspace(opts);
   await requireLogin(opts);
 
-  const input = await resolve(opts.input);
+  const input = opts.data ? await resolve(opts.data) : {};
 
   const id = await JobService.runFlowByPath({
     workspace: workspace.workspaceId,
@@ -236,6 +217,7 @@ async function run(
 
   if (!opts.silent) {
     console.log(colors.green.underline.bold("Flow ran to completion"));
+    console.log();
   }
   const jobInfo = await JobService.getCompletedJob({
     workspace: workspace.workspaceId,
@@ -250,19 +232,19 @@ const command = new Command()
   .action(list as any)
   .command(
     "push",
-    "push a local flow spec. This overrides any remote versions.",
+    "push a local flow spec. This overrides any remote versions."
   )
   .arguments("<file_path:string> <remote_path:string>")
   .action(push as any)
   .command("run", "run a flow by path.")
   .arguments("<path:string>")
   .option(
-    "-i --input [inputs...:string]",
-    "Inputs specified as JSON objects or simply as <name>=<value>. Supports file inputs using @<filename> and stdin using @- these also need to be formatted as JSON. Later inputs override earlier ones.",
+    "-d --data <data:string>",
+    "Inputs specified as a JSON string or a file using @<filename> or stdin using @-."
   )
   .option(
     "-s --silent",
-    "Do not ouput anything other then the final output. Useful for scripting.",
+    "Do not ouput anything other then the final output. Useful for scripting."
   )
   .action(run as any);
 

cli/folder.ts

@@ -1,4 +1,4 @@
-import { colors, Command, Folder, FolderService, microdiff } from "./deps.ts";
+import { colors, Command, FolderService, microdiff } from "./deps.ts";
 import { requireLogin, resolveWorkspace, validatePath } from "./context.ts";
 import {
   Difference,
@@ -20,8 +20,16 @@ import {
 export class FolderFile implements Resource, PushDiffs {
   @property(array(() => String))
   owners: Array<string> | undefined;
-  @property(map(() => String, () => Boolean, { shape: MapShape.Object }))
+  @property(
+    map(
+      () => String,
+      () => Boolean,
+      { shape: MapShape.Object }
+    )
+  )
   extra_perms: Map<string, boolean> | undefined;
+  @property(() => String)
+  display_name: string | undefined;
 
   async push(workspace: string, remotePath: string): Promise<void> {
     if (remotePath.startsWith("/")) {
@@ -31,23 +39,17 @@ export class FolderFile implements Resource, PushDiffs {
       remotePath = remotePath.substring(2);
     }
 
-    let existing: Folder | undefined;
-    try {
-      existing = await FolderService.getFolder({ workspace, name: remotePath });
-    } catch {
-      existing = undefined;
-    }
     await this.pushDiffs(
       workspace,
       remotePath,
-      microdiff(existing ?? {}, this, { cyclesFix: false }),
+      microdiff({}, this, { cyclesFix: false })
     );
   }
 
   async pushDiffs(
     workspace: string,
     remotePath: string,
-    diffs: Difference[],
+    diffs: Difference[]
   ): Promise<void> {
     if (remotePath.startsWith("/")) {
       remotePath = remotePath.substring(1);
@@ -59,29 +61,34 @@ export class FolderFile implements Resource, PushDiffs {
     // TODO: Support this in backend
     let exists: boolean;
     try {
-      exists = !!await FolderService.getFolder({ workspace, name: remotePath });
+      exists = !!(await FolderService.getFolder({
+        workspace,
+        name: remotePath,
+      }));
     } catch {
       exists = false;
     }
     if (exists) {
       console.log(
         colors.bold.yellow(
-          `Applying ${diffs.length} diffs to existing folder...`,
-        ),
+          `Applying ${diffs.length} diffs to existing folder... ${remotePath}`
+        )
       );
 
       const changeset: {
         owners?: string[] | undefined;
         extra_perms?: any;
+        display_name?: string | undefined;
       } = {};
       for (const diff of diffs) {
         if (
           diff.type !== "REMOVE" &&
-          (
-            diff.path.length !== 1 ||
-            !["owners", "extra_perms"].includes(diff.path[0] as string)
-          )
+          (diff.path.length !== 1 ||
+            !["owners", "extra_perms", "display_name"].includes(
+              diff.path[0] as string
+            ))
         ) {
+          console.log(diff.path);
           throw new Error("Invalid folder diff with path " + diff.path);
         }
         if (diff.type === "CREATE" || diff.type === "CHANGE") {
@@ -91,18 +98,27 @@ export class FolderFile implements Resource, PushDiffs {
         }
       }
 
-      const hasChanges = Object.values(changeset).some((v) =>
-        v !== null && typeof v !== "undefined"
+      const hasChanges = Object.values(changeset).some(
+        (v) => v !== null && typeof v !== "undefined"
       );
       if (!hasChanges) {
         return;
       }
-
-      await FolderService.updateFolder({
-        workspace: workspace,
-        name: remotePath,
-        requestBody: changeset,
-      });
+      try {
+        await FolderService.updateFolder({
+          workspace: workspace,
+          name: remotePath,
+          requestBody: {
+            ...changeset,
+            extra_perms: changeset.extra_perms
+              ? Object.fromEntries(this.extra_perms?.entries() ?? [])
+              : undefined,
+          },
+        });
+      } catch (e) {
+        console.error(colors.red.bold(e.body));
+        throw e;
+      }
     } else {
       console.log(colors.bold.yellow("Creating new folder: " + remotePath));
       await FolderService.createFolder({
@@ -121,7 +137,7 @@ async function push(opts: GlobalOptions, filePath: string, remotePath: string) {
   const workspace = await resolveWorkspace(opts);
   await requireLogin(opts);
 
-  if (!await validatePath(opts, remotePath)) {
+  if (!validatePath(remotePath)) {
     return;
   }
 
@@ -139,11 +155,11 @@ async function push(opts: GlobalOptions, filePath: string, remotePath: string) {
 export async function pushFolder(
   workspace: string,
   filePath: string,
-  remotePath: string,
+  remotePath: string
 ) {
-  const data = decoverto.type(FolderFile).rawToInstance(
-    await Deno.readTextFile(filePath),
-  );
+  const data = decoverto
+    .type(FolderFile)
+    .rawToInstance(await Deno.readTextFile(filePath));
   data.push(workspace, remotePath);
 }
 
@@ -151,7 +167,7 @@ const command = new Command()
   .description("resource related commands")
   .command(
     "push",
-    "push a local folder spec. This overrides any remote versions.",
+    "push a local folder spec. This overrides any remote versions."
   )
   .arguments("<file_path:string> <remote_path:string>")
   .action(push as any);

cli/main.ts

@@ -1,5 +1,11 @@
-import { Command, CompletionsCommand, DenoLandProvider, UpgradeCommand } from "./deps.ts";
+import {
+  Command,
+  CompletionsCommand,
+  DenoLandProvider,
+  UpgradeCommand,
+} from "./deps.ts";
 import flow from "./flow.ts";
+import app from "./apps.ts";
 import script from "./script.ts";
 import workspace from "./workspace.ts";
 import resource from "./resource.ts";
@@ -13,21 +19,21 @@ import sync from "./sync.ts";
 import { tryResolveVersion } from "./context.ts";
 import { GlobalOptions } from "./types.ts";
 
-const VERSION = "v1.70.1";
+export const VERSION = "v1.83.1";
 
 let command: any = new Command()
   .name("wmill")
   .description("A simple CLI tool for windmill.")
-  .action(() => command.showHelp())
   .globalOption(
     "--workspace <workspace:string>",
-    "Specify the target workspace. This overrides the default workspace.",
+    "Specify the target workspace. This overrides the default workspace."
   )
   .globalOption(
     "--token <token:string>",
-    "Specify an API token. This will override any stored token.",
+    "Specify an API token. This will override any stored token."
   )
   .version(VERSION)
+  .command("app", app)
   .command("flow", flow)
   .command("script", script)
   .command("workspace", workspace)
@@ -59,14 +65,12 @@ let command: any = new Command()
         "--unstable",
       ],
       provider: new DenoLandProvider({ name: "wmill" }),
-    }),
+    })
   )
   .command("completions", new CompletionsCommand());
 
 if (Number.parseInt(VERSION.replace("v", "").replace(".", "")) > 1700) {
-  command = command
-    .command("push", push)
-    .command("pull", pull);
+  command = command.command("push", push).command("pull", pull);
 }
 
 try {

cli/pull.ts

@@ -5,25 +5,27 @@ import { Workspace } from "./workspace.ts";
 
 export async function downloadZip(
   workspace: Workspace,
+  plainSecrets: boolean | undefined
 ): Promise<JSZip | undefined> {
   const requestHeaders: HeadersInit = new Headers();
   requestHeaders.set("Authorization", "Bearer " + workspace.token);
   requestHeaders.set("Content-Type", "application/octet-stream");
 
   const zipResponse = await fetch(
-    workspace.remote + "api/w/" + workspace.workspaceId +
-    "/workspaces/tarball?archive_type=zip",
+    workspace.remote +
+      "api/w/" +
+      workspace.workspaceId +
+      "/workspaces/tarball?archive_type=zip&plain_secret=" +
+      (plainSecrets ?? false),
     {
       headers: requestHeaders,
       method: "GET",
-    },
+    }
   );
 
   if (!zipResponse.ok) {
     console.log(
-      colors.red(
-        "Failed to request tarball from API " + zipResponse.statusText,
-      ),
+      colors.red("Failed to request tarball from API " + zipResponse.statusText)
     );
     throw new Error(await zipResponse.text());
   }
@@ -33,18 +35,18 @@ export async function downloadZip(
 
 async function stub(
   _opts: GlobalOptions & { override: boolean },
-  _dir: string,
+  _dir: string
 ) {
   console.log(
     colors.red.underline(
-      'Pull is deprecated. Use "sync pull --raw" instead. See <TODO_LINK_HERE> for more information.',
-    ),
+      'Pull is deprecated. Use "sync pull --raw" instead. See <TODO_LINK_HERE> for more information.'
+    )
   );
 }
 
 const command = new Command()
   .description(
-    "Pull all definitions in the current workspace from the API and write them to disk.",
+    "Pull all definitions in the current workspace from the API and write them to disk."
   )
   .arguments("<dir:string>")
   .action(stub as any);

cli/resource-type.ts

@@ -13,7 +13,6 @@ import {
   EditResourceType,
   microdiff,
   ResourceService,
-  ResourceType,
   Table,
 } from "./deps.ts";
 import { Any, decoverto, model, property } from "./decoverto.ts";
@@ -26,19 +25,10 @@ export class ResourceTypeFile implements ResourceI, PushDiffs {
   description?: string;
 
   async push(workspace: string, remotePath: string): Promise<void> {
-    let existing: ResourceType | undefined;
-    try {
-      existing = await ResourceService.getResourceType({
-        workspace,
-        path: remotePath,
-      });
-    } catch {
-      existing = undefined;
-    }
-    this.pushDiffs(
+    await this.pushDiffs(
       workspace,
       remotePath,
-      microdiff(existing ?? {}, this, { cyclesFix: false }),
+      microdiff({}, this, { cyclesFix: false }),
     );
   }
 
@@ -65,8 +55,8 @@ export class ResourceTypeFile implements ResourceI, PushDiffs {
         return;
       }
       console.log(
-        colors.yellow(
-          `Applying ${diffs.length} diffs to existing resource type...`,
+        colors.yellow.bold(
+          `Applying ${diffs.length} diffs to existing resource type... ${remotePath}`,
         ),
       );
       const changeset: EditResourceType = {};

cli/resource.ts

@@ -34,7 +34,7 @@ export class ResourceFile implements Resource2, PushDiffs {
   async pushDiffs(
     workspace: string,
     remotePath: string,
-    diffs: Difference[],
+    diffs: Difference[]
   ): Promise<void> {
     if (
       await ResourceService.existsResource({
@@ -43,7 +43,9 @@ export class ResourceFile implements Resource2, PushDiffs {
       })
     ) {
       console.log(
-        colors.yellow(`Applying ${diffs.length} diffs to existing resource...`),
+        colors.yellow.bold(
+          `Applying ${diffs.length} diffs to existing resource... ${remotePath}`
+        )
       );
 
       const changeset: EditResource = {
@@ -51,22 +53,18 @@ export class ResourceFile implements Resource2, PushDiffs {
       };
       for (const diff of diffs) {
         if (diff.path[0] === "is_oauth") {
-          console.log(
-            colors.yellow(
-              "! is_oauth has been removed in newer versions. Ignoring.",
-            ),
-          );
+          //is_oauth is not updatable
           continue;
         }
         if (
           diff.type !== "REMOVE" &&
-          (
-            diff.path[0] !== "value" && (
-              diff.path.length !== 1 ||
-              diff.path[0] !== "description"
-            )
-          )
+          diff.path[0] !== "value" &&
+          (diff.path.length !== 1 || diff.path[0] !== "description") &&
+          diff.path[0] !== "resource_type"
         ) {
+          console.log(
+            colors.red("Invalid variable diff with path " + diff.path)
+          );
           throw new Error("Invalid folder diff with path " + diff.path);
         }
         if (diff.type === "CREATE" || diff.type === "CHANGE") {
@@ -76,8 +74,8 @@ export class ResourceFile implements Resource2, PushDiffs {
         }
       }
 
-      const hasChanges = Object.values(changeset).some((v) =>
-        v !== null && typeof v !== "undefined"
+      const hasChanges = Object.values(changeset).some(
+        (v) => v !== null && typeof v !== "undefined"
       );
       if (!hasChanges) {
         return;
@@ -92,8 +90,8 @@ export class ResourceFile implements Resource2, PushDiffs {
       if (typeof this.is_oauth !== "undefined") {
         console.log(
           colors.yellow(
-            "! is_oauth has been removed in newer versions. Ignoring.",
-          ),
+            "! is_oauth has been removed in newer versions. Ignoring."
+          )
         );
       }
 
@@ -110,19 +108,10 @@ export class ResourceFile implements Resource2, PushDiffs {
     }
   }
   async push(workspace: string, remotePath: string): Promise<void> {
-    let existing: Resource | undefined;
-    try {
-      existing = await ResourceService.getResource({
-        workspace,
-        path: remotePath,
-      });
-    } catch {
-      existing = undefined;
-    }
     await this.pushDiffs(
       workspace,
       remotePath,
-      microdiff(existing ?? {}, this, { cyclesFix: false }),
+      microdiff({}, this, { cyclesFix: false })
     );
   }
 }
@@ -130,11 +119,11 @@ export class ResourceFile implements Resource2, PushDiffs {
 export async function pushResource(
   workspace: string,
   filePath: string,
-  remotePath: string,
+  remotePath: string
 ) {
-  const data = decoverto.type(ResourceFile).rawToInstance(
-    await Deno.readTextFile(filePath),
-  );
+  const data = decoverto
+    .type(ResourceFile)
+    .rawToInstance(await Deno.readTextFile(filePath));
   await data.push(workspace, remotePath);
 }
 
@@ -143,7 +132,7 @@ async function push(opts: PushOptions, filePath: string, remotePath: string) {
   const workspace = await resolveWorkspace(opts);
   await requireLogin(opts);
 
-  if (!await validatePath(opts, remotePath)) {
+  if (!validatePath(remotePath)) {
     return;
   }
 
@@ -190,7 +179,7 @@ const command = new Command()
   .action(list as any)
   .command(
     "push",
-    "push a local resource spec. This overrides any remote versions.",
+    "push a local resource spec. This overrides any remote versions."
   )
   .arguments("<file_path:string> <remote_path:string>")
   .action(push as any);

cli/script.ts

@@ -11,6 +11,7 @@ import {
   Table,
 } from "./deps.ts";
 import { Any, array, decoverto, model, property } from "./decoverto.ts";
+import { writeAllSync } from "https://deno.land/std@0.176.0/streams/mod.ts";
 
 @model()
 export class ScriptFile {
@@ -31,8 +32,11 @@ export class ScriptFile {
       if (data == null) return data;
 
       if (
-        data === "script" || data === "failure" || data === "trigger" ||
-        data === "command" || data === "approvial"
+        data === "script" ||
+        data === "failure" ||
+        data === "trigger" ||
+        data === "command" ||
+        data === "approvial"
       ) {
         return data;
       }
@@ -41,6 +45,7 @@ export class ScriptFile {
     },
     toPlain: (data) => data,
   })
+  @property(() => String)
   kind?: "script" | "failure" | "trigger" | "command" | "approval";
 
   constructor(summary: string, description: string) {
@@ -50,14 +55,11 @@ export class ScriptFile {
 }
 
 type PushOptions = GlobalOptions;
-async function push(
-  opts: PushOptions,
-  filePath: string,
-  remotePath: string,
-  contentPath?: string,
-) {
+async function push(opts: PushOptions, filePath: string) {
   const workspace = await resolveWorkspace(opts);
-  if (!await validatePath(opts, remotePath)) {
+  const remotePath = filePath.split(".")[0];
+
+  if (!validatePath(remotePath)) {
     return;
   }
 
@@ -65,89 +67,136 @@ async function push(
   if (!fstat.isFile) {
     throw new Error("file path must refer to a file.");
   }
-  if (!contentPath) {
+  let contentPath: string;
+  let metaPath: string | undefined;
+  if (filePath.endsWith(".script.json")) {
+    metaPath = filePath;
     contentPath = await findContentFile(filePath);
   } else {
-    const fstat = await Deno.stat(filePath);
-    if (!fstat.isFile) {
-      throw new Error("content path must refer to a file.");
-    }
+    contentPath = filePath;
+    metaPath = undefined;
   }
 
   await requireLogin(opts);
-  await pushScript(filePath, contentPath, workspace.workspaceId, remotePath);
+  await pushScript(metaPath, contentPath, workspace.workspaceId, remotePath);
   console.log(colors.bold.underline.green(`Script ${remotePath} pushed`));
 }
 
-export async function handleScriptMetadata(path: string, workspace: string, alreadySynced: string[]): Promise<boolean> {
+export async function handleScriptMetadata(
+  path: string,
+  workspace: string,
+  alreadySynced: string[]
+): Promise<boolean> {
   if (path.endsWith(".script.json")) {
-    const contentPath = await findContentFile(path)
-    return handleFile(contentPath, await Deno.readTextFile(contentPath), workspace, alreadySynced)
+    const contentPath = await findContentFile(path);
+    return handleFile(
+      contentPath,
+      await Deno.readTextFile(contentPath),
+      workspace,
+      alreadySynced
+    );
   } else {
-    return false
+    return false;
   }
 }
 
-export async function handleFile(path: string, content: string, workspace: string, alreadySynced: string[]): Promise<boolean> {
-  if (path.endsWith(".ts") || path.endsWith(".py") || path.endsWith(".go") || path.endsWith(".sh")) {
+export async function handleFile(
+  path: string,
+  content: string,
+  workspace: string,
+  alreadySynced: string[]
+): Promise<boolean> {
+  if (
+    path.endsWith(".ts") ||
+    path.endsWith(".py") ||
+    path.endsWith(".go") ||
+    path.endsWith(".sh")
+  ) {
     if (alreadySynced.includes(path)) {
-      return true
+      return true;
     }
-    alreadySynced.push(path)
+    alreadySynced.push(path);
     const remotePath = path.substring(0, path.length - 3);
     const metaPath = remotePath + ".script.json";
-    let typed = undefined
+    let typed = undefined;
     try {
-      await Deno.stat(metaPath)
-      typed = JSON.parse(await Deno.readTextFile(metaPath))
+      await Deno.stat(metaPath);
+      typed = JSON.parse(await Deno.readTextFile(metaPath));
       typed = decoverto.type(ScriptFile).plainToInstance(typed);
-    } catch { }
+    } catch {}
     const language = inferContentTypeFromFilePath(path);
 
+    let remote = undefined;
     try {
-      const remote = await ScriptService.getScriptByPath({
+      remote = await ScriptService.getScriptByPath({
         workspace,
         path: remotePath,
       });
+    } catch {}
+
+    if (remote) {
+      if (content === remote.content) {
+        if (
+          typed == undefined ||
+          (typed.description === remote.description &&
+            typed.summary === remote.summary &&
+            typed.is_template === remote.is_template &&
+            typed.kind == remote.kind &&
+            remote?.lock == typed.lock?.join("\n") &&
+            JSON.stringify(typed.schema) == JSON.stringify(remote.schema))
+        ) {
+          console.log(
+            colors.yellow(
+              `No change to push for script ${remotePath}, skipping`
+            )
+          );
+          return true;
+        }
+      }
+
       await ScriptService.createScript({
         workspace,
         requestBody: {
           content,
-          description: typed.description,
+          description: typed?.description ?? "",
           language,
           path: remotePath,
-          summary: typed.summary,
-          is_template: typed.is_template,
-          kind: typed.kind,
-          lock: typed.lock,
+          summary: typed?.summary ?? "",
+          is_template: typed?.is_template,
+          kind: typed?.kind,
+          lock: typed?.lock,
           parent_hash: remote.hash,
-          schema: typed.schema,
+          schema: typed?.schema,
         },
       });
-      console.log(colors.yellow.bold(`Creating script with a parent ${remotePath}`))
-    } catch {
+
+      console.log(
+        colors.yellow.bold(`Creating script with a parent ${remotePath}`)
+      );
+    } else {
       // no parent hash
       await ScriptService.createScript({
         workspace: workspace,
         requestBody: {
           content,
-          description: typed.description,
+          description: typed?.description ?? "",
           language,
           path: remotePath,
-          summary: typed.summary,
-          is_template: typed.is_template,
-          kind: typed.kind,
-          lock: typed.lock,
+          summary: typed?.summary ?? "",
+          is_template: typed?.is_template,
+          kind: typed?.kind,
+          lock: typed?.lock,
           parent_hash: undefined,
-          schema: typed.schema,
+          schema: typed?.schema,
         },
       });
-      console.log(colors.yellow.bold(`Creating script without parent ${remotePath}`))
-
+      console.log(
+        colors.yellow.bold(`Creating script without parent ${remotePath}`)
+      );
     }
-    return true
+    return true;
   }
-  return false
+  return false;
 }
 
 export async function findContentFile(filePath: string) {
@@ -166,7 +215,7 @@ export async function findContentFile(filePath: string) {
           .then((e) => {
             return { path: x, file: e };
           });
-      }),
+      })
     )
   )
     .filter((x) => x.file)
@@ -174,7 +223,7 @@ export async function findContentFile(filePath: string) {
   if (validCandidates.length > 1) {
     throw new Error(
       "No content path given and more then one candidate found: " +
-      validCandidates.join(", "),
+        validCandidates.join(", ")
     );
   }
   if (validCandidates.length < 1) {
@@ -184,7 +233,7 @@ export async function findContentFile(filePath: string) {
 }
 
 export function inferContentTypeFromFilePath(
-  contentPath: string,
+  contentPath: string
 ): "python3" | "deno" | "go" | "bash" {
   let language = contentPath.substring(contentPath.lastIndexOf("."));
   if (language == ".ts") language = "deno";
@@ -192,7 +241,9 @@ export function inferContentTypeFromFilePath(
   if (language == ".sh") language = "bash";
   if (language == ".go") language = "go";
   if (
-    language != "python3" && language != "deno" && language != "go" &&
+    language != "python3" &&
+    language != "deno" &&
+    language != "go" &&
     language != "bash"
   ) {
     throw new Error("Invalid language: " + language);
@@ -201,18 +252,20 @@ export function inferContentTypeFromFilePath(
 }
 
 export async function pushScript(
-  filePath: string,
+  filePath: string | undefined,
   contentPath: string,
   workspace: string,
-  remotePath: string,
+  remotePath: string
 ) {
-  const data = decoverto.type(ScriptFile).rawToInstance(
-    await Deno.readTextFile(filePath),
-  );
+  const data = filePath
+    ? decoverto
+        .type(ScriptFile)
+        .rawToInstance(await Deno.readTextFile(filePath))
+    : undefined;
   const content = await Deno.readTextFile(contentPath);
 
   const language = inferContentTypeFromFilePath(contentPath);
-  let parent_hash = data.parent_hash;
+  let parent_hash = data?.parent_hash;
   if (!parent_hash) {
     try {
       parent_hash = (
@@ -231,15 +284,15 @@ export async function pushScript(
     workspace: workspace,
     requestBody: {
       path: remotePath,
-      summary: data.summary,
+      summary: data?.summary ?? "",
       content: content,
-      description: data.description,
+      description: data?.description ?? "",
       language: language,
-      is_template: data.is_template,
-      kind: data.kind,
-      lock: data.lock,
+      is_template: data?.is_template,
+      kind: data?.kind,
+      lock: data?.lock,
       parent_hash: parent_hash,
-      schema: data.schema,
+      schema: data?.schema,
     },
   });
 }
@@ -269,69 +322,40 @@ async function list(opts: GlobalOptions & { showArchived?: boolean }) {
     .header(["path", "summary", "language", "created by"])
     .padding(2)
     .border(true)
-    .body(
-      total.map((x) => [
-        x.path,
-        x.summary,
-        x.language,
-        x.created_by,
-      ]),
-    )
+    .body(total.map((x) => [x.path, x.summary, x.language, x.created_by]))
     .render();
 }
 
-export async function resolve(inputs: string[]): Promise<Record<string, any>> {
-  let result = {};
-
-  if (!inputs) {
-    return result;
+export async function resolve(input: string): Promise<Record<string, any>> {
+  if (!input) {
+    throw new Error("No data given");
   }
 
-  for (const input of inputs) {
-    let data: string;
-    if (input.startsWith("@")) {
-      if (input == "@-") {
-        data = new TextDecoder().decode(await readAll(Deno.stdin));
-      } else {
-        data = await Deno.readTextFile(input.substring(1));
-      }
-    } else {
-      if (input.startsWith("{")) {
-        data = input;
-      } else {
-        const key = input.split("=", 1)[0];
-        const value = input.substring(key.length + 1);
-        let o;
-        try {
-          o = JSON.parse(value);
-        } catch {
-          o = value;
-        }
-        data = JSON.stringify(Object.fromEntries([[key, o]]));
-      }
-    }
-    let jsonObj;
-    try {
-      jsonObj = JSON.parse(data);
-    } catch {
-      jsonObj = data;
-    }
-    result = { ...result, ...jsonObj };
+  if (input == "@-") {
+    input = new TextDecoder().decode(await readAll(Deno.stdin));
+  }
+  if (input[0] == "@") {
+    input = await Deno.readTextFile(input.substring(1));
+  }
+  try {
+    return JSON.parse(input);
+  } catch (e) {
+    console.error("Impossible to parse input as JSON", input);
+    throw e;
   }
-  return result;
 }
 
 async function run(
   opts: GlobalOptions & {
-    input: string[];
+    data?: string;
     silent: boolean;
   },
-  path: string,
+  path: string
 ) {
   const workspace = await resolveWorkspace(opts);
   await requireLogin(opts);
 
-  const input = await resolve(opts.input);
+  const input = opts.data ? await resolve(opts.data) : {};
   const id = await JobService.runScriptByPath({
     workspace: workspace.workspaceId,
     path,
@@ -344,12 +368,13 @@ async function run(
 
   while (true) {
     try {
-      const result = (
-        await JobService.getCompletedJob({
-          workspace: workspace.workspaceId,
-          id,
-        })
-      ).result ?? {};
+      const result =
+        (
+          await JobService.getCompletedJob({
+            workspace: workspace.workspaceId,
+            id,
+          })
+        ).result ?? {};
       console.log(result);
 
       break;
@@ -364,7 +389,9 @@ export async function track_job(workspace: string, id: string) {
     const result = await JobService.getCompletedJob({ workspace, id });
 
     console.log(result.logs);
+    console.log();
     console.log(colors.bold.underline.green("Job Completed"));
+    console.log();
     return;
   } catch {
     /* ignore */
@@ -403,7 +430,7 @@ export async function track_job(workspace: string, id: string) {
     }
 
     if (updates.new_logs) {
-      console.log(updates.new_logs);
+      writeAllSync(Deno.stdout, new TextEncoder().encode(updates.new_logs));
       logOffset += updates.new_logs.length;
     }
 
@@ -415,7 +442,7 @@ export async function track_job(workspace: string, id: string) {
     if (running && updates.running === false) {
       running = false;
       console.log(
-        colors.yellow("Job suspended. Waiting for it to continue..."),
+        colors.yellow("Job suspended. Waiting for it to continue...")
       );
     }
   }
@@ -426,12 +453,13 @@ export async function track_job(workspace: string, id: string) {
     if ((final_job.logs?.length ?? -1) > logOffset) {
       console.log(final_job.logs!.substring(logOffset));
     }
-
+    console.log("\n");
     if (final_job.success) {
       console.log(colors.bold.underline.green("Job Completed"));
     } else {
       console.log(colors.bold.underline.red("Job Completed"));
     }
+    console.log();
   } catch {
     console.log("Job appears to have completed, but no data can be retrieved");
   }
@@ -456,22 +484,22 @@ const command = new Command()
   .action(list as any)
   .command(
     "push",
-    "push a local script spec. This overrides any remote versions.",
+    "push a local script spec. This overrides any remote versions. Can use a script file (.ts, .js, .py, .sh) or a script spec file (.json). "
   )
-  .arguments("<file_path:string> <remote_path:string> [content_path:string]")
+  .arguments("<file_path:file>")
   .action(push as any)
   .command("show", "show a scripts content")
-  .arguments("<path:string>")
+  .arguments("<path:file>")
   .action(show as any)
   .command("run", "run a script by path")
-  .arguments("<path:string>")
+  .arguments("<path:file>")
   .option(
-    "-i --input [inputs...:string]",
-    "Inputs specified as JSON objects or simply as <name>=<value>. Supports file inputs using @<filename> and stdin using @- these also need to be formatted as JSON. Later inputs override earlier ones.",
+    "-d --data <data:file>",
+    "Inputs specified as a JSON string or a file using @<filename> or stdin using @-."
   )
   .option(
     "-s --silent",
-    "Do not ouput anything other then the final output. Useful for scripting.",
+    "Do not ouput anything other then the final output. Useful for scripting."
   )
   .action(run as any);
 

cli/sync.ts

@@ -13,7 +13,7 @@ import {
   ResourceService,
   VariableService,
   AppService,
-  FlowService
+  FlowService,
 } from "./deps.ts";
 import {
   Difference,
@@ -25,16 +25,14 @@ import {
 import { downloadZip } from "./pull.ts";
 import { FolderFile } from "./folder.ts";
 import { ResourceTypeFile } from "./resource-type.ts";
-import {
-  handleScriptMetadata,
-  ScriptFile,
-} from "./script.ts";
+import { handleScriptMetadata, ScriptFile } from "./script.ts";
 import { ResourceFile } from "./resource.ts";
 import { FlowFile } from "./flow.ts";
 import { VariableFile } from "./variable.ts";
 import { handleFile } from "./script.ts";
-import { equal } from "https://deno.land/x/equal/mod.ts";
-import { diffCharacters } from "https://deno.land/x/diff/mod.ts";
+import { equal } from "https://deno.land/x/equal@v1.5.0/mod.ts";
+import * as Diff from "npm:diff";
+
 type DynFSElement = {
   isDirectory: boolean;
   path: string;
@@ -44,10 +42,7 @@ type DynFSElement = {
 };
 
 async function FSFSElement(p: string): Promise<DynFSElement> {
-  function _internal_element(
-    localP: string,
-    isDir: boolean,
-  ): DynFSElement {
+  function _internal_element(localP: string, isDir: boolean): DynFSElement {
     return {
       isDirectory: isDir,
       path: localP.substring(p.length + 1),
@@ -113,16 +108,14 @@ function ZipFSElement(zip: JSZip): DynFSElement {
 
 async function* readDirRecursiveWithIgnore(
   ignore: (path: string, isDirectory: boolean) => boolean,
-  root: DynFSElement,
-): AsyncGenerator<
-  {
-    path: string;
-    ignored: boolean;
-    isDirectory: boolean;
-    getContentBytes(): Promise<Uint8Array>;
-    getContentText(): Promise<string>;
-  }
-> {
+  root: DynFSElement
+): AsyncGenerator<{
+  path: string;
+  ignored: boolean;
+  isDirectory: boolean;
+  getContentBytes(): Promise<Uint8Array>;
+  getContentText(): Promise<string>;
+}> {
   const stack: {
     path: string;
     isDirectory: boolean;
@@ -130,18 +123,20 @@ async function* readDirRecursiveWithIgnore(
     c(): AsyncIterable<DynFSElement>;
     getContentBytes(): Promise<Uint8Array>;
     getContentText(): Promise<string>;
-  }[] = [{
-    path: root.path,
-    ignored: ignore(root.path, root.isDirectory),
-    isDirectory: root.isDirectory,
-    c: root.getChildren,
-    getContentBytes(): Promise<Uint8Array> {
-      throw undefined;
+  }[] = [
+    {
+      path: root.path,
+      ignored: ignore(root.path, root.isDirectory),
+      isDirectory: root.isDirectory,
+      c: root.getChildren,
+      getContentBytes(): Promise<Uint8Array> {
+        throw undefined;
+      },
+      getContentText(): Promise<string> {
+        throw undefined;
+      },
     },
-    getContentText(): Promise<string> {
-      throw undefined;
-    },
-  }];
+  ];
 
   while (stack.length > 0) {
     const e = stack.pop()!;
@@ -161,17 +156,17 @@ async function* readDirRecursiveWithIgnore(
 }
 
 type Added = { name: "added"; path: string; content: string };
-type Deleted = { name: "deleted"; path: string; };
-type Edit = { name: "edited"; path: string; before: string; after: string; };
+type Deleted = { name: "deleted"; path: string };
+type Edit = { name: "edited"; path: string; before: string; after: string };
 
 type Change = Added | Deleted | Edit;
 
-async function elementsToMap(els: DynFSElement, ignore: (path: string, isDirectory: boolean) => boolean): Promise<{ [key: string]: string }> {
+async function elementsToMap(
+  els: DynFSElement,
+  ignore: (path: string, isDirectory: boolean) => boolean
+): Promise<{ [key: string]: string }> {
   const map: { [key: string]: string } = {};
-  for await (const entry of readDirRecursiveWithIgnore(
-    ignore,
-    els,
-  )) {
+  for await (const entry of readDirRecursiveWithIgnore(ignore, els)) {
     if (entry.isDirectory || entry.ignored) continue;
     const content = await entry.getContentText();
     map[entry.path] = content;
@@ -179,20 +174,26 @@ async function elementsToMap(els: DynFSElement, ignore: (path: string, isDirecto
   return map;
 }
 async function compareDynFSElement(
-  els1: DynFSElement, els2: DynFSElement,
-  ignore: (path: string, isDirectory: boolean) => boolean,
-  raw: boolean
+  els1: DynFSElement,
+  els2: DynFSElement | undefined,
+  ignore: (path: string, isDirectory: boolean) => boolean
 ): Promise<Change[]> {
-
-  const [m1, m2] = raw ? [await elementsToMap(els1, ignore), {}] :
-    await Promise.all([elementsToMap(els1, ignore), elementsToMap(els2, ignore)]);
+  const [m1, m2] = els2
+    ? await Promise.all([
+        elementsToMap(els1, ignore),
+        elementsToMap(els2, ignore),
+      ])
+    : [await elementsToMap(els1, ignore), {}];
 
   const changes: Change[] = [];
 
   for (const [k, v] of Object.entries(m1)) {
     if (m2[k] === undefined) {
       changes.push({ name: "added", path: k, content: v });
-    } else if (m2[k] != v && (!k.endsWith(".json") || !equal(JSON.parse(v), JSON.parse(m2[k])))) {
+    } else if (
+      m2[k] != v &&
+      (!k.endsWith(".json") || !equal(JSON.parse(v), JSON.parse(m2[k])))
+    ) {
       // await Deno.writeTextFile("/tmp/k", m2[k])
       // await Deno.writeTextFile("/tmp/v", v)
       // console.log(k)
@@ -208,54 +209,59 @@ async function compareDynFSElement(
     }
   }
 
-  return changes
+  return changes;
 }
 
 const isNotWmillFile = (p: string, isDirectory: boolean) => {
   if (p.endsWith("/")) {
-    return false
+    return false;
   }
   if (isDirectory) {
-    return !p.startsWith("u/") && !p.startsWith("f/") && !p.startsWith("g/")
+    return !p.startsWith("u/") && !p.startsWith("f/") && !p.startsWith("g/");
   }
 
   try {
-    const typ = getTypeStrFromPath(p)
-    if (typ == 'resource-type') {
-      return p.includes('/')
+    const typ = getTypeStrFromPath(p);
+    if (typ == "resource-type") {
+      return p.includes("/");
     } else {
-      return !p.startsWith("u/") && !p.startsWith("f/") && !p.startsWith("g/")
+      return !p.startsWith("u/") && !p.startsWith("f/") && !p.startsWith("g/");
     }
   } catch {
-    return true
+    return true;
   }
-}
+};
 
 const isWhitelisted = (p: string) => {
-  return p == "./" || p == "" || p == "u" || p == "f" || p == "g"
-}
+  return p == "./" || p == "" || p == "u" || p == "f" || p == "g";
+};
 async function ignoreF() {
   try {
     const ignore: {
       accepts(file: string): boolean;
       denies(file: string): boolean;
-    } = gitignore_parser.compile(
-      await Deno.readTextFile(".wmillignore"),
-    );
+    } = gitignore_parser.compile(await Deno.readTextFile(".wmillignore"));
 
     return (p: string, isDirectory: boolean) => {
-      return !isWhitelisted(p) && (isNotWmillFile(p, isDirectory) || ignore.denies(p));
-    }
+      return (
+        !isWhitelisted(p) &&
+        (isNotWmillFile(p, isDirectory) || ignore.denies(p))
+      );
+    };
   } catch (e) {
-    return (p: string, isDirectory: boolean) => !isWhitelisted(p) && isNotWmillFile(p, isDirectory)
+    return (p: string, isDirectory: boolean) =>
+      !isWhitelisted(p) && isNotWmillFile(p, isDirectory);
   }
 }
 
 async function pull(
-  opts: GlobalOptions & { raw: boolean; yes: boolean, failConflicts: boolean },
+  opts: GlobalOptions & {
+    raw: boolean;
+    yes: boolean;
+    failConflicts: boolean;
+    plainSecrets?: boolean;
+  }
 ) {
-
-
   if (!opts.raw) {
     await ensureDir(path.join(Deno.cwd(), ".wmill"));
   }
@@ -263,74 +269,98 @@ async function pull(
   const workspace = await resolveWorkspace(opts);
   await requireLogin(opts);
 
-
-  console.log(colors.gray("Computing the files to update locally to match remote (taking .wmillignore into account)"));
-  const remote = ZipFSElement((await downloadZip(workspace))!)
-  const local = await FSFSElement(path.join(Deno.cwd(), opts.raw ? "" : ".wmill"))
-  const changes = await compareDynFSElement(remote, local, await ignoreF(), opts.raw)
-
+  console.log(
+    colors.gray(
+      "Computing the files to update locally to match remote (taking .wmillignore into account)"
+    )
+  );
+  const remote = ZipFSElement(
+    (await downloadZip(workspace, opts.plainSecrets))!
+  );
+  const local = opts.raw
+    ? undefined
+    : await FSFSElement(path.join(Deno.cwd(), opts.raw ? "" : ".wmill"));
+  const changes = await compareDynFSElement(remote, local, await ignoreF());
 
   console.log(`remote -> local: ${changes.length} changes to apply`);
   if (changes.length > 0) {
-
-    prettyChanges(changes)
+    prettyChanges(changes);
     if (
-      !opts.yes && !opts.raw && !(await Confirm.prompt({ message: `Do you want to apply these ${changes.length} changes?`, default: true }))
+      !opts.yes &&
+      !opts.raw &&
+      !(await Confirm.prompt({
+        message: `Do you want to apply these ${changes.length} changes?`,
+        default: true,
+      }))
     ) {
-      return
+      return;
     }
 
-    const conflicts = []
+    const conflicts = [];
     console.log(colors.gray(`Applying changes to files ...`));
     for await (const change of changes) {
       const target = path.join(Deno.cwd(), change.path);
-      const stateTarget = path.join(Deno.cwd(), ".wmill", change.path)
+      const stateTarget = path.join(Deno.cwd(), ".wmill", change.path);
       if (change.name === "edited") {
-
         try {
-          const currentLocal = await Deno.readTextFile(target)
-          if (currentLocal !== change.before) {
-            console.log(colors.red(`Conflict detected on ${change.path}\nBoth local and remote have been modified.`))
+          const currentLocal = await Deno.readTextFile(target);
+          if (currentLocal !== change.before && currentLocal !== change.after) {
+            console.log(
+              colors.red(
+                `Conflict detected on ${change.path}\nBoth local and remote have been modified.`
+              )
+            );
             if (opts.failConflicts) {
-              conflicts.push({ local: currentLocal, change, path: change.path })
+              conflicts.push({
+                local: currentLocal,
+                change,
+                path: change.path,
+              });
               continue;
             } else if (opts.yes) {
-              console.log(colors.red(`Override local version with remote since --yes was passed and no --fail-conflicts.`))
-            }
-            else {
-              showConflict(change.path, currentLocal, change.after)
-              if (await Confirm.prompt("Preserve local (push to change remote and avoid seeing this again)?")) {
+              console.log(
+                colors.red(
+                  `Override local version with remote since --yes was passed and no --fail-conflicts.`
+                )
+              );
+            } else {
+              showConflict(change.path, currentLocal, change.after);
+              if (
+                await Confirm.prompt(
+                  "Preserve local (push to change remote and avoid seeing this again)?"
+                )
+              ) {
                 continue;
               }
             }
           }
-        } catch { }
+        } catch {}
         if (change.path.endsWith(".json")) {
-          const diffs =
-            microdiff(
-              JSON.parse(change.before),
-              JSON.parse(change.after),
-              { cyclesFix: false },
-            )
-
-          console.log(`Editing ${getTypeStrFromPath(change.path)} json ${change.path}`)
-          await applyDiff(
-            diffs,
-            target,
+          const diffs = microdiff(
+            JSON.parse(change.before),
+            JSON.parse(change.after),
+            { cyclesFix: false }
           );
+
+          console.log(
+            `Editing ${getTypeStrFromPath(change.path)} json ${change.path}`
+          );
+          await applyDiff(diffs, target);
         } else {
-          console.log(`Editing script ${change.path}`)
+          console.log(`Editing script ${change.path}`);
           await Deno.writeTextFile(target, change.after);
         }
         if (!opts.raw) {
-          await ensureDir(path.dirname(stateTarget))
+          await ensureDir(path.dirname(stateTarget));
           await Deno.copyFile(target, stateTarget);
         }
       } else if (change.name === "added") {
-        await ensureDir(path.dirname(target))
+        await ensureDir(path.dirname(target));
         if (!opts.raw) {
-          await ensureDir(path.dirname(stateTarget))
-          console.log(`Adding ${getTypeStrFromPath(change.path)} ${change.path}`)
+          await ensureDir(path.dirname(stateTarget));
+          console.log(
+            `Adding ${getTypeStrFromPath(change.path)} ${change.path}`
+          );
         }
         await Deno.writeTextFile(target, change.content);
         if (!opts.raw) {
@@ -338,8 +368,10 @@ async function pull(
         }
       } else if (change.name === "deleted") {
         try {
-          console.log(`Deleting ${getTypeStrFromPath(change.path)} ${change.path}`)
-          await Deno.remove(target)
+          console.log(
+            `Deleting ${getTypeStrFromPath(change.path)} ${change.path}`
+          );
+          await Deno.remove(target);
           if (!opts.raw) {
             await Deno.remove(stateTarget);
           }
@@ -352,41 +384,46 @@ async function pull(
     }
     if (opts.failConflicts) {
       if (conflicts.length > 0) {
-        console.error(colors.red(`Conflicts were found`))
-        console.log("Conflicts:")
+        console.error(colors.red(`Conflicts were found`));
+        console.log("Conflicts:");
         for (const conflict of conflicts) {
-          showConflict(conflict.path, conflict.local, conflict.change.after)
+          showConflict(conflict.path, conflict.local, conflict.change.after);
         }
-        console.log(colors.red(`Please resolve theses conflicts manually by either:
+        console.log(
+          colors.red(`Please resolve theses conflicts manually by either:
   - reverting the content back to its remote (\`wmill pull\` and refuse to preserve local when prompted)
   - pushing the changes with \`wmill push --skip-pull\` to override wmill with all your local changes
-`))
-        Deno.exit(1)
+`)
+        );
+        Deno.exit(1);
       }
     }
-    console.log(colors.green.underline(`Done! All ${changes.length} changes applied locally.`));
+    console.log(
+      colors.green.underline(
+        `Done! All ${changes.length} changes applied locally.`
+      )
+    );
   }
 
-
   function showConflict(path: string, local: string, remote: string) {
-    console.log(colors.yellow(`- ${path}`))
+    console.log(colors.yellow(`- ${path}`));
 
     let finalString = "";
-    for (const character of diffCharacters(local, remote)) {
-      if (character.wasRemoved) {
+    for (const part of Diff.diffLines(local, remote)) {
+      if (part.removed) {
         // print red if removed without newline
-        finalString += `\x1b[31m${character.character}\x1b[0m`;
-      } else if (character.wasAdded) {
+        finalString += `\x1b[31m${part.value}\x1b[0m`;
+      } else if (part.added) {
         // print green if added
-        finalString += `\x1b[32m${character.character}\x1b[0m`;
+        finalString += `\x1b[32m${part.value}\x1b[0m`;
       } else {
         // print white if unchanged
-        finalString += `\x1b[37m${character.character}\x1b[0m`;
+        finalString += `\x1b[37m${part.value}\x1b[0m`;
       }
     }
     console.log(finalString);
-    console.log("\x1b[31mlocal\x1b[31m - \x1b[32mremote\x1b[32m")
-    console.log()
+    console.log("\x1b[31mlocal\x1b[31m - \x1b[32mremote\x1b[32m");
+    console.log();
   }
   async function applyDiff(diffs: Difference[], file: string) {
     ensureDir(path.dirname(file));
@@ -415,175 +452,258 @@ async function pull(
   }
 }
 
-
-
-
 function prettyChanges(changes: Change[]) {
   for (const change of changes) {
     if (change.name === "added") {
-      console.log(colors.green(`+ ${getTypeStrFromPath(change.path)} ` + change.path));
+      console.log(
+        colors.green(`+ ${getTypeStrFromPath(change.path)} ` + change.path)
+      );
     } else if (change.name === "deleted") {
-      console.log(colors.red(`- ${getTypeStrFromPath(change.path)} ` + change.path));
+      console.log(
+        colors.red(`- ${getTypeStrFromPath(change.path)} ` + change.path)
+      );
     } else if (change.name === "edited") {
-      console.log(colors.yellow(`~ ${getTypeStrFromPath(change.path)} ` + change.path));
-    }
-  }
-}
-
-function prettyDiff(diffs: Difference[]) {
-  for (const diff of diffs) {
-    let pathString = "";
-    for (const pathSegment of diff.path) {
-      if (typeof pathSegment === "string") {
-        pathString += ".";
-        pathString += pathSegment;
-      } else {
-        pathString += "[";
-        pathString += pathSegment;
-        pathString += "]";
-      }
-    }
-    if (diff.type === "REMOVE" || diff.type === "CHANGE") {
-      console.log(colors.red("- " + pathString + " = " + diff.oldValue));
-    }
-    if (diff.type === "CREATE" || diff.type === "CHANGE") {
-      console.log(colors.green("+ " + pathString + " = " + diff.value));
+      console.log(
+        colors.yellow(`~ ${getTypeStrFromPath(change.path)} ` + change.path)
+      );
     }
   }
 }
 
+// function prettyDiff(diffs: Difference[]) {
+//   for (const diff of diffs) {
+//     let pathString = "";
+//     for (const pathSegment of diff.path) {
+//       if (typeof pathSegment === "string") {
+//         pathString += ".";
+//         pathString += pathSegment;
+//       } else {
+//         pathString += "[";
+//         pathString += pathSegment;
+//         pathString += "]";
+//       }
+//     }
+//     if (diff.type === "REMOVE" || diff.type === "CHANGE") {
+//       console.log(colors.red("- " + pathString + " = " + diff.oldValue));
+//     }
+//     if (diff.type === "CREATE" || diff.type === "CHANGE") {
+//       console.log(colors.green("+ " + pathString + " = " + diff.value));
+//     }
+//   }
+// }
 
 function removeSuffix(str: string, suffix: string) {
   return str.slice(0, str.length - suffix.length);
 }
 
-async function push(opts: GlobalOptions & { raw: boolean, yes: boolean, skipPull: boolean, failConflicts: boolean }) {
-
-
+async function push(
+  opts: GlobalOptions & {
+    raw: boolean;
+    yes: boolean;
+    skipPull: boolean;
+    failConflicts: boolean;
+    plainSecrets?: boolean;
+  }
+) {
   if (!opts.raw) {
     if (!opts.skipPull) {
-      console.log(colors.gray("You need to be up-to-date before pushing, pulling first."))
-      await pull(opts)
-      console.log(colors.green("Pull done, now pushing."))
-      console.log()
+      console.log(
+        colors.gray("You need to be up-to-date before pushing, pulling first.")
+      );
+      await pull(opts);
+      console.log(colors.green("Pull done, now pushing."));
+      console.log();
     }
   }
 
   const workspace = await resolveWorkspace(opts);
   await requireLogin(opts);
 
-
-  console.log(colors.gray("Computing the files to update on the remote to match local (taking .wmillignore into account)"));
-  const remote = ZipFSElement((await downloadZip(workspace))!)
-  const local = await FSFSElement(path.join(Deno.cwd(), ""))
-  const changes = await compareDynFSElement(local, remote, await ignoreF(), opts.raw)
+  console.log(
+    colors.gray(
+      "Computing the files to update on the remote to match local (taking .wmillignore into account)"
+    )
+  );
+  const remote = opts.raw
+    ? undefined
+    : ZipFSElement((await downloadZip(workspace, opts.plainSecrets))!);
+  const local = await FSFSElement(path.join(Deno.cwd(), ""));
+  const changes = await compareDynFSElement(local, remote, await ignoreF());
 
   console.log(`remote <- local: ${changes.length} changes to apply`);
   if (changes.length > 0) {
-
-    prettyChanges(changes)
+    prettyChanges(changes);
     if (
-      !opts.yes && !(await Confirm.prompt({ message: `Do you want to apply these ${changes.length} changes?`, default: true }))
+      !opts.yes &&
+      !(await Confirm.prompt({
+        message: `Do you want to apply these ${changes.length} changes?`,
+        default: true,
+      }))
     ) {
-      return
+      return;
     }
     console.log(colors.gray(`Applying changes to files ...`));
-    const alreadySynced: string[] = []
+    const alreadySynced: string[] = [];
     for await (const change of changes) {
-      const stateTarget = path.join(Deno.cwd(), ".wmill", change.path)
+      const stateTarget = path.join(Deno.cwd(), ".wmill", change.path);
+      let stateExists = true;
+      try {
+        await Deno.stat(stateTarget);
+      } catch {
+        stateExists = false;
+      }
+
       if (change.name === "edited") {
-        if (await handleScriptMetadata(change.path, workspace.workspaceId, alreadySynced)) {
-          if (!opts.raw) {
+        if (
+          await handleScriptMetadata(
+            change.path,
+            workspace.workspaceId,
+            alreadySynced
+          )
+        ) {
+          if (!opts.raw && stateExists) {
             await Deno.writeTextFile(stateTarget, change.after);
           }
-          continue
-        } else if (await handleFile(change.path, change.after, workspace.workspaceId, alreadySynced)) {
-          if (!opts.raw) {
+          continue;
+        } else if (
+          await handleFile(
+            change.path,
+            change.after,
+            workspace.workspaceId,
+            alreadySynced
+          )
+        ) {
+          if (!opts.raw && stateExists) {
             await Deno.writeTextFile(stateTarget, change.after);
           }
-          continue
+          continue;
         }
         if (!opts.raw) {
-          await ensureDir(path.dirname(stateTarget))
-          console.log(`Editing ${getTypeStrFromPath(change.path)} ${change.path}`)
+          await ensureDir(path.dirname(stateTarget));
+          console.log(
+            `Editing ${getTypeStrFromPath(change.path)} ${change.path}`
+          );
         }
-        const obj = inferTypeFromPath(change.path, JSON.parse(change.after))
+        const obj = inferTypeFromPath(change.path, JSON.parse(change.after));
 
-        const diff = microdiff(inferTypeFromPath(change.path, JSON.parse(change.before)), obj, { cyclesFix: false });
+        const diff = microdiff(
+          inferTypeFromPath(change.path, JSON.parse(change.before)),
+          obj,
+          { cyclesFix: false }
+        );
         await applyDiff(
           workspace.workspaceId,
           change.path.split(".")[0],
           obj,
           diff,
+          opts.plainSecrets
         );
-        if (!opts.raw) {
+        if (!opts.raw && stateExists) {
           await Deno.writeTextFile(stateTarget, change.after);
         }
       } else if (change.name === "added") {
         if (change.path.endsWith(".script.json")) {
-          continue
-        } else if (await handleFile(change.path, change.content, workspace.workspaceId, alreadySynced)) {
-          continue
+          continue;
+        } else if (
+          await handleFile(
+            change.path,
+            change.content,
+            workspace.workspaceId,
+            alreadySynced
+          )
+        ) {
+          continue;
         }
-        if (!opts.raw) {
-          await ensureDir(path.dirname(stateTarget))
-          console.log(`Adding ${getTypeStrFromPath(change.path)} ${change.path}`)
+        if (!opts.raw && stateExists) {
+          await ensureDir(path.dirname(stateTarget));
+          console.log(
+            `Adding ${getTypeStrFromPath(change.path)} ${change.path}`
+          );
         }
-        const obj = inferTypeFromPath(change.path, JSON.parse(change.content))
+        const obj = inferTypeFromPath(change.path, JSON.parse(change.content));
         const diff = microdiff({}, obj, { cyclesFix: false });
         await applyDiff(
           workspace.workspaceId,
           change.path.split(".")[0],
           obj,
           diff,
+          opts.plainSecrets
         );
-        if (!opts.raw) {
+        if (!opts.raw && stateExists) {
           await Deno.writeTextFile(stateTarget, change.content);
         }
       } else if (change.name === "deleted") {
         if (!change.path.includes(".json")) {
-          continue
+          continue;
         }
-        console.log(`Deleting ${getTypeStrFromPath(change.path)} ${change.path}`)
-        const typ = getTypeStrFromPath(change.path)
+        console.log(
+          `Deleting ${getTypeStrFromPath(change.path)} ${change.path}`
+        );
+        const typ = getTypeStrFromPath(change.path);
         const workspaceId = workspace.workspaceId;
         switch (typ) {
           case "script": {
-            const script = await ScriptService.getScriptByPath({ workspace: workspaceId, path: removeSuffix(change.path, ".script.json") })
-            await ScriptService.deleteScriptByHash({ workspace: workspaceId, hash: script.hash })
+            const script = await ScriptService.getScriptByPath({
+              workspace: workspaceId,
+              path: removeSuffix(change.path, ".script.json"),
+            });
+            await ScriptService.deleteScriptByHash({
+              workspace: workspaceId,
+              hash: script.hash,
+            });
             break;
           }
           case "folder":
-            await FolderService.deleteFolder({ workspace: workspaceId, name: change.path.split('/')[1] })
+            await FolderService.deleteFolder({
+              workspace: workspaceId,
+              name: change.path.split("/")[1],
+            });
             break;
           case "resource":
-            await ResourceService.deleteResource({ workspace: workspaceId, path: removeSuffix(change.path, ".resource.json") })
+            await ResourceService.deleteResource({
+              workspace: workspaceId,
+              path: removeSuffix(change.path, ".resource.json"),
+            });
             break;
           case "resource-type":
-            await ResourceService.deleteResourceType({ workspace: workspaceId, path: removeSuffix(change.path, ".resource-type.json") })
-            break
+            await ResourceService.deleteResourceType({
+              workspace: workspaceId,
+              path: removeSuffix(change.path, ".resource-type.json"),
+            });
+            break;
           case "flow":
-            await FlowService.deleteFlowByPath({ workspace: workspaceId, path: removeSuffix(change.path, ".flow.json") })
-            break
+            await FlowService.deleteFlowByPath({
+              workspace: workspaceId,
+              path: removeSuffix(change.path, ".flow.json"),
+            });
+            break;
           case "app":
-            await AppService.deleteApp({ workspace: workspaceId, path: removeSuffix(change.path, ".app.json") })
-            break
+            await AppService.deleteApp({
+              workspace: workspaceId,
+              path: removeSuffix(change.path, ".app.json"),
+            });
+            break;
           case "variable":
-            await VariableService.deleteVariable({ workspace: workspaceId, path: removeSuffix(change.path, ".variable.json") })
-            break
+            await VariableService.deleteVariable({
+              workspace: workspaceId,
+              path: removeSuffix(change.path, ".variable.json"),
+            });
+            break;
           default:
             break;
         }
         try {
-          Deno.remove(stateTarget)
-        } catch { }
+          await Deno.remove(stateTarget);
+        } catch {}
       }
     }
-    console.log(colors.green.underline(`Done! All ${changes.length} changes pushed to the remote workspace.`));
-
+    console.log(
+      colors.green.underline(
+        `Done! All ${changes.length} changes pushed to the remote workspace.`
+      )
+    );
   }
 
-
   async function applyDiff(
     workspace: string,
     remotePath: string,
@@ -595,10 +715,11 @@ async function push(opts: GlobalOptions & { raw: boolean, yes: boolean, skipPull
       | ResourceTypeFile
       | FolderFile,
     diffs: Difference[],
+    plainSecrets?: boolean
   ) {
     if (file instanceof ScriptFile) {
       throw new Error(
-        "This code path should be unreachable - we should never generate diffs for scripts",
+        "This code path should be unreachable - we should never generate diffs for scripts"
       );
     } else if (file instanceof FolderFile) {
       const parts = remotePath.split("/");
@@ -609,14 +730,14 @@ async function push(opts: GlobalOptions & { raw: boolean, yes: boolean, skipPull
       }
     }
     if (diffs.length === 0) {
-      console.log("No diffs to apply to " + remotePath)
+      console.log("No diffs to apply to " + remotePath);
       return;
     }
     try {
-      await file.pushDiffs(workspace, remotePath, diffs);
+      await file.pushDiffs(workspace, remotePath, diffs, plainSecrets);
     } catch (e) {
-      console.error("Failing to apply diffs to " + remotePath)
-      console.error(e.body)
+      console.error("Failing to apply diffs to " + remotePath);
+      console.error(JSON.stringify(e));
     }
   }
 }
@@ -624,20 +745,28 @@ async function push(opts: GlobalOptions & { raw: boolean, yes: boolean, skipPull
 const command = new Command()
   .command("pull")
   .description(
-    "Pull any remote changes and apply them locally. Use --raw for usage without local state tracking.",
+    "Pull any remote changes and apply them locally. Use --raw for usage without local state tracking."
+  )
+  .option(
+    "--fail-conflicts",
+    "Error on conflicts (both remote and local have changes on the same item)"
   )
-  .option("--fail-conflicts", "Error on conflicts (both remote and local have changes on the same item)")
   .option("--yes", "Pull without needing confirmation")
   .option("--raw", "Pull without using state, just overwrite.")
+  .option("--plain-secrets", "Pull secrets as plain text")
   .action(pull as any)
   .command("push")
   .description(
-    "Push any local changes and apply them remotely. Use --raw for usage without local state tracking.",
+    "Push any local changes and apply them remotely. Use --raw for usage without local state tracking."
   )
-  .option("--fail-conflicts", "Error on conflicts (both remote and local have changes on the same item)")
-  .option("--skip-pull", "Push without pulling first")
+  .option(
+    "--fail-conflicts",
+    "Error on conflicts (both remote and local have changes on the same item)"
+  )
+  .option("--skip-pull", "Push without pulling first (you have pulled prior)")
   .option("--yes", "Push without needing confirmation")
   .option("--raw", "Push without using state, just overwrite.")
+  .option("--plain-secrets", "Push secrets as plain text")
   .action(push as any);
 
 export default command;

cli/types.ts

@@ -11,7 +11,11 @@ import { AppFile } from "./apps.ts";
 // TODO: Remove this & replace with a "pull" that lets the object either pull the remote version or return undefined.
 // Then combine those with diffing, which then gives the new push impl
 export interface Resource {
-  push(workspace: string, remotePath: string): Promise<void>;
+  push(
+    workspace: string,
+    remotePath: string,
+    plainSecrets?: boolean
+  ): Promise<void>;
 }
 
 export interface PushDiffs {
@@ -19,6 +23,7 @@ export interface PushDiffs {
     workspace: string,
     remotePath: string,
     diffs: Difference[],
+    plainSecrets?: boolean
   ): Promise<void>;
 }
 
@@ -46,7 +51,7 @@ export type Difference = DifferenceCreate | DifferenceRemove | DifferenceChange;
 export function setValueByPath(
   obj: any,
   path: (string | number)[],
-  value: any,
+  value: any
 ) {
   let i;
   let lastObj = undefined;
@@ -84,7 +89,7 @@ export type GlobalOptions = {
 
 export function inferTypeFromPath(
   p: string,
-  obj: any,
+  obj: any
 ):
   | ScriptFile
   | VariableFile
@@ -115,7 +120,7 @@ export function inferTypeFromPath(
 }
 
 export function getTypeStrFromPath(
-  p: string,
+  p: string
 ):
   | "script"
   | "variable"
@@ -125,8 +130,13 @@ export function getTypeStrFromPath(
   | "folder"
   | "app" {
   const parsed = path.parse(p);
-  if (parsed.ext == ".go" || parsed.ext == ".ts" || parsed.ext == ".sh" || parsed.ext == ".py") {
-    return 'script'
+  if (
+    parsed.ext == ".go" ||
+    parsed.ext == ".ts" ||
+    parsed.ext == ".sh" ||
+    parsed.ext == ".py"
+  ) {
+    return "script";
   }
 
   if (parsed.name === "folder.meta") {
@@ -135,9 +145,12 @@ export function getTypeStrFromPath(
 
   const typeEnding = parsed.name.split(".").at(-1);
   if (
-    typeEnding === "script" || typeEnding === "variable" ||
-    typeEnding === "flow" || typeEnding === "resource" ||
-    typeEnding === "resource-type" || typeEnding === "app"
+    typeEnding === "script" ||
+    typeEnding === "variable" ||
+    typeEnding === "flow" ||
+    typeEnding === "resource" ||
+    typeEnding === "resource-type" ||
+    typeEnding === "app"
   ) {
     return typeEnding;
   } else {

cli/variable.ts

@@ -11,7 +11,6 @@ import {
   colors,
   Command,
   EditVariable,
-  ListableVariable,
   microdiff,
   Table,
   VariableService,
@@ -36,7 +35,7 @@ async function list(opts: GlobalOptions) {
         x.is_secret ? "true" : "false",
         x.account ?? "-",
         x.value ?? "-",
-      ]),
+      ])
     )
     .render();
 }
@@ -63,24 +62,31 @@ export class VariableFile implements Resource, PushDiffs {
     workspace: string,
     remotePath: string,
     diffs: Difference[],
+    plainSecrets?: boolean
   ): Promise<void> {
     if (await VariableService.existsVariable({ workspace, path: remotePath })) {
       console.log(
         colors.bold.yellow(
-          `Applying ${diffs.length} diffs to existing variable...`,
-        ),
+          `Applying ${diffs.length} diffs to existing variable... ${remotePath}`
+        )
       );
       const changeset: EditVariable = {};
       for (const diff of diffs) {
         if (
           diff.type !== "REMOVE" &&
-          (
-            diff.path.length !== 1 ||
-            !["path", "value", "is_secret", "description"].includes(
-              diff.path[0] as string,
-            )
-          )
+          (diff.path.length !== 1 ||
+            ![
+              "path",
+              "value",
+              "is_secret",
+              "description",
+              "account",
+              "is_oauth",
+            ].includes(diff.path[0] as string))
         ) {
+          console.log(
+            colors.red("Invalid variable diff with path " + diff.path)
+          );
           throw new Error("Invalid variable diff with path " + diff.path);
         }
         if (diff.type === "CREATE" || diff.type === "CHANGE") {
@@ -90,25 +96,24 @@ export class VariableFile implements Resource, PushDiffs {
         }
       }
 
-      const hasChanges = Object.values(changeset).some((v) =>
-        v !== null && typeof v !== "undefined"
+      const hasChanges = Object.values(changeset).some(
+        (v) => v !== null && typeof v !== "undefined"
       );
       if (!hasChanges) {
         return;
       }
+
       await VariableService.updateVariable({
         workspace,
         path: remotePath,
-        alreadyEncrypted: true,
+        alreadyEncrypted: !plainSecrets,
         requestBody: changeset,
       });
-
-      console.log(changeset);
     } else {
       console.log(colors.yellow.bold("Creating new variable..."));
       await VariableService.createVariable({
         workspace,
-        alreadyEncrypted: true,
+        alreadyEncrypted: !plainSecrets,
         requestBody: {
           path: remotePath,
           description: this.description,
@@ -120,29 +125,29 @@ export class VariableFile implements Resource, PushDiffs {
       });
     }
   }
-  async push(workspace: string, remotePath: string): Promise<void> {
-    let existing: ListableVariable | undefined;
-    try {
-      existing = await VariableService.getVariable({
-        workspace: workspace,
-        path: remotePath,
-      });
-    } catch {
-      existing = undefined;
-    }
+  async push(
+    workspace: string,
+    remotePath: string,
+    plainSecrets?: boolean
+  ): Promise<void> {
     await this.pushDiffs(
       workspace,
       remotePath,
-      microdiff(existing ?? {}, this, { cyclesFix: false }),
+      microdiff({}, this, { cyclesFix: false }),
+      plainSecrets
     );
   }
 }
 
-async function push(opts: GlobalOptions, filePath: string, remotePath: string) {
+async function push(
+  opts: GlobalOptions & { plainSecrets: boolean },
+  filePath: string,
+  remotePath: string
+) {
   const workspace = await resolveWorkspace(opts);
   await requireLogin(opts);
 
-  if (!await validatePath(opts, remotePath)) {
+  if (!validatePath(remotePath)) {
     return;
   }
 
@@ -153,7 +158,12 @@ async function push(opts: GlobalOptions, filePath: string, remotePath: string) {
 
   console.log(colors.bold.yellow("Pushing variable..."));
 
-  await pushVariable(workspace.workspaceId, filePath, remotePath);
+  await pushVariable(
+    workspace.workspaceId,
+    filePath,
+    remotePath,
+    opts.plainSecrets
+  );
   console.log(colors.bold.underline.green(`Variable ${remotePath} pushed`));
 }
 
@@ -161,11 +171,12 @@ export async function pushVariable(
   workspace: string,
   filePath: string,
   remotePath: string,
+  plainSecrets: boolean
 ) {
-  const data = decoverto.type(VariableFile).rawToInstance(
-    await Deno.readTextFile(filePath),
-  );
-  await data.push(workspace, remotePath);
+  const data = decoverto
+    .type(VariableFile)
+    .rawToInstance(await Deno.readTextFile(filePath));
+  await data.push(workspace, remotePath, plainSecrets);
 }
 
 const command = new Command()
@@ -173,9 +184,10 @@ const command = new Command()
   .action(list as any)
   .command(
     "push",
-    "Push a local variable spec. This overrides any remote versions.",
+    "Push a local variable spec. This overrides any remote versions."
   )
   .arguments("<file_path:string> <remote_path:string>")
+  .option("--plain-secrets", "Push secrets as plain text")
   .action(push as any);
 
 export default command;

cli/workspace.ts

@@ -9,9 +9,11 @@ import {
   Input,
   setClient,
   Table,
+  UserService,
   WorkspaceService,
 } from "./deps.ts";
 import { decoverto, model, property } from "./decoverto.ts";
+import { requireLogin } from "./context.ts";
 
 @model()
 export class Workspace {
@@ -28,7 +30,7 @@ export class Workspace {
     remote: string,
     workspaceId: string,
     name: string,
-    token: string,
+    token: string
   ) {
     this.remote = remote;
     this.workspaceId = workspaceId;
@@ -38,7 +40,7 @@ export class Workspace {
 }
 
 function makeWorkspaceStream(
-  readable: ReadableStream<Uint8Array>,
+  readable: ReadableStream<Uint8Array>
 ): ReadableStream<Workspace> {
   return readable
     .pipeThrough(new DelimiterStream(new TextEncoder().encode("\n")))
@@ -57,7 +59,7 @@ function makeWorkspaceStream(
             /* ignore */
           }
         },
-      }),
+      })
     );
 }
 
@@ -84,7 +86,7 @@ export async function allWorkspaces(): Promise<Workspace[]> {
 }
 
 async function getActiveWorkspaceName(
-  opts: GlobalOptions,
+  opts: GlobalOptions
 ): Promise<string | undefined> {
   if (opts.workspace) {
     return opts.workspace;
@@ -97,7 +99,7 @@ async function getActiveWorkspaceName(
 }
 
 export async function getActiveWorkspace(
-  opts: GlobalOptions,
+  opts: GlobalOptions
 ): Promise<Workspace | undefined> {
   const name = await getActiveWorkspaceName(opts);
   if (!name) {
@@ -107,7 +109,7 @@ export async function getActiveWorkspace(
 }
 
 export async function getWorkspaceByName(
-  workspaceName: string,
+  workspaceName: string
 ): Promise<Workspace | undefined> {
   const workspaceStream = await getWorkspaceStream();
   for await (const workspace of workspaceStream) {
@@ -134,34 +136,40 @@ async function list(opts: GlobalOptions) {
         } else {
           return a;
         }
-      }),
+      })
     )
     .render();
+
+  console.log("Active: " + colors.green.bold(activeName || "none"));
 }
 
 async function switchC(opts: GlobalOptions, workspaceName: string) {
   if (opts.workspace) {
     console.log(
       colors.red.bold(
-        "! Workspace needs to be specified as positional argument, not as option.",
-      ),
+        "! Workspace needs to be specified as positional argument, not as option."
+      )
     );
     return;
   }
 
   const all = await allWorkspaces();
   if (all.findIndex((x) => x.name === workspaceName) === -1) {
-    console.log(colors.red.bold(`! This workspace profile ${workspaceName} does not exist locally.`));
-    console.log("available workspace profiles:")
+    console.log(
+      colors.red.bold(
+        `! This workspace profile ${workspaceName} does not exist locally.`
+      )
+    );
+    console.log("available workspace profiles:");
     for (const w of all) {
-      console.log('  - ' + w.name)
+      console.log("  - " + w.name);
     }
     return;
   }
 
   return await Deno.writeTextFile(
     (await getRootStore()) + "/activeWorkspace",
-    workspaceName,
+    workspaceName
   );
 }
 
@@ -173,13 +181,13 @@ export async function add(
   },
   workspaceName: string | undefined,
   workspaceId: string | undefined,
-  remote: string | undefined,
+  remote: string | undefined
 ) {
   if (opts.workspace) {
     console.log(
       colors.red.bold(
-        "! Workspace needs to be specified as positional argument, not as option.",
-      ),
+        "! Workspace needs to be specified as positional argument, not as option."
+      )
     );
     return;
   }
@@ -189,7 +197,11 @@ export async function add(
   }
 
   if (!workspaceId) {
-    workspaceId = await Input.prompt({ message: "Enter the ID of this workspace", default: workspaceName, suggestions: [workspaceName] });
+    workspaceId = await Input.prompt({
+      message: "Enter the ID of this workspace",
+      default: workspaceName,
+      suggestions: [workspaceName],
+    });
   }
 
   if (!remote) {
@@ -205,8 +217,8 @@ export async function add(
         await Input.prompt({
           message: "Enter the Remote URL",
           suggestions: ["https://app.windmill.dev/"],
-          default: "https://app.windmill.dev/"
-        }),
+          default: "https://app.windmill.dev/",
+        })
       ).toString();
     }
   }
@@ -219,22 +231,26 @@ export async function add(
 
   setClient(
     token,
-    remote.endsWith("/") ? remote.substring(0, remote.length - 1) : remote,
+    remote.endsWith("/") ? remote.substring(0, remote.length - 1) : remote
   );
-  let alreadyExists = false
+  let alreadyExists = false;
   try {
     alreadyExists = await WorkspaceService.existsWorkspace({
       requestBody: { id: workspaceId },
-    })
+    });
   } catch (e) {
-    console.log(colors.red.bold("! Credentials or instance is invalid. Aborting."));
-    throw e
+    console.log(
+      colors.red.bold("! Credentials or instance is invalid. Aborting.")
+    );
+    throw e;
   }
   if (opts.create) {
-    if (
-      !alreadyExists
-    ) {
-      console.log(colors.yellow(`Workspace at id ${workspaceId} on ${remote} does not exist. Creating...`));
+    if (!alreadyExists) {
+      console.log(
+        colors.yellow(
+          `Workspace at id ${workspaceId} on ${remote} does not exist. Creating...`
+        )
+      );
       await WorkspaceService.createWorkspace({
         requestBody: {
           id: workspaceId,
@@ -244,63 +260,87 @@ export async function add(
       });
     }
   } else if (!alreadyExists) {
-    console.log(colors.red.bold(`! Workspace at id ${workspaceId} on ${remote} does not exist. Re-run with --create to create it. Aborting.`));
-    console.log("On that instance and with those credentials, the workspaces that you can access are:")
-    const workspaces = await WorkspaceService.listWorkspaces()
+    console.log(
+      colors.red.bold(
+        `! Workspace at id ${workspaceId} on ${remote} does not exist. Re-run with --create to create it. Aborting.`
+      )
+    );
+    console.log(
+      "On that instance and with those credentials, the workspaces that you can access are:"
+    );
+    const workspaces = await WorkspaceService.listWorkspaces();
     for (const workspace of workspaces) {
-      console.log(`- ${workspace.id} (name: ${workspace.name})`)
+      console.log(`- ${workspace.id} (name: ${workspace.name})`);
     }
     Deno.exit(1);
   }
 
-  await addWorkspace({
-    name: workspaceName,
-    remote: remote,
-    workspaceId: workspaceId,
-    token: token,
-  }, opts);
+  await addWorkspace(
+    {
+      name: workspaceName,
+      remote: remote,
+      workspaceId: workspaceId,
+      token: token,
+    },
+    opts
+  );
   await Deno.writeTextFile(
     (await getRootStore()) + "/activeWorkspace",
-    workspaceName,
+    workspaceName
+  );
+  console.log(
+    colors.green.underline(
+      `Added workspace ${workspaceName} for ${workspaceId} on ${remote}!`
+    )
   );
-  console.log(colors.green.underline(`Added workspace ${workspaceName} for ${workspaceId} on ${remote}!`));
 }
 
 export async function addWorkspace(workspace: Workspace, opts: any) {
   workspace.remote = new URL(workspace.remote).toString(); // add trailing slash in all cases!
+  await removeWorkspace(workspace.name, true, opts);
   const file = await Deno.open((await getRootStore()) + "remotes.ndjson", {
     append: true,
     write: true,
     read: true,
     create: true,
   });
-  await removeWorkspace(workspace.name, true, opts);
   await file.write(new TextEncoder().encode(JSON.stringify(workspace) + "\n"));
+
   file.close();
 }
 
-export async function removeWorkspace(name: string, silent: boolean, opts: any) {
+export async function removeWorkspace(
+  name: string,
+  silent: boolean,
+  opts: any
+) {
   const orgWorkspaces = await allWorkspaces();
   if (orgWorkspaces.findIndex((x) => x.name === name) === -1) {
     if (!silent) {
-      console.log(colors.red.bold(`! Workspace profile ${name} does not exist locally`));
-      console.log("available workspace profiles:")
-      await list(opts)
+      console.log(
+        colors.red.bold(`! Workspace profile ${name} does not exist locally`)
+      );
+      console.log("available workspace profiles:");
+      await list(opts);
     }
     return;
   }
   if (silent) {
-    console.log(colors.yellow(`Replacing existing workspace ${name}`))
+    console.log(colors.yellow(`Replacing existing workspace ${name}`));
   }
+
   await Deno.writeTextFile(
     (await getRootStore()) + "remotes.ndjson",
     orgWorkspaces
       .filter((x) => x.name !== name)
       .map((x) => JSON.stringify(x))
-      .join("\n"),
+      .join("\n") + "\n"
   );
+
   if (!silent) {
-    console.log(colors.green.underline(`Succesfully removed workspace ${name}!`));
+    console.log(
+      colors.green.underline(`Succesfully removed workspace ${name}!`)
+    );
   }
 }
 
@@ -308,6 +348,13 @@ async function remove(_opts: GlobalOptions, name: string) {
   await removeWorkspace(name, false, _opts);
 }
 
+async function whoami(_opts: GlobalOptions) {
+  await requireLogin(_opts);
+  console.log(await UserService.globalWhoami());
+  const activeName = await getActiveWorkspaceName(_opts);
+  console.log("Active: " + colors.green.bold(activeName || "none"));
+}
+
 const command = new Command()
   .description("workspace related commands")
   .action(list as any)
@@ -322,19 +369,22 @@ const command = new Command()
   .option("-c --create", "Create the workspace if it does not exist")
   .option(
     "--create-workspace-name <workspace_name:string>",
-    "Specify the workspace name. Ignored if --create is not specified or the workspace already exists. Will default to the workspace id.",
+    "Specify the workspace name. Ignored if --create is not specified or the workspace already exists. Will default to the workspace id."
   )
   .option(
     "--create-username <username:string>",
     "Specify your own username in the newly created workspace. Ignored if --create is not specified or the workspace already exists.",
     {
       default: "admin",
-    },
+    }
   )
   .action(add as any)
   .command("remove")
   .description("Remove a workspace")
   .arguments("<workspace_name:string>")
-  .action(remove as any);
+  .action(remove as any)
+  .command("whoami")
+  .description("Show the currently active user")
+  .action(whoami as any);
 
 export default command;

deno-client/mod.ts

@@ -1,41 +1,59 @@
-import { JobService, ResourceService, VariableService } from './windmill-api/index.ts'
-import { OpenAPI } from './windmill-api/index.ts'
+import {
+  JobService,
+  ResourceService,
+  VariableService,
+} from "./windmill-api/index.ts";
+import { OpenAPI } from "./windmill-api/index.ts";
 
 export {
-    AdminService, AuditService, FlowService, GranularAclService, GroupService,
-    JobService, ResourceService, VariableService, ScriptService, ScheduleService, SettingsService,
-    UserService, WorkspaceService
-} from './windmill-api/index.ts'
-
+  AdminService,
+  AuditService,
+  FlowService,
+  GranularAclService,
+  GroupService,
+  JobService,
+  ResourceService,
+  VariableService,
+  ScriptService,
+  ScheduleService,
+  SettingsService,
+  UserService,
+  WorkspaceService,
+} from "./windmill-api/index.ts";
 
 // @ts-ignore: Otherwise BigInt is not supported for export
 BigInt.prototype.toJSON = function () {
-    return this.toString();
+  return this.toString();
 };
 
-export { pgSql, pgClient } from './pg.ts'
+export { pgSql, pgClient } from "./pg.ts";
 
-export type Sql = string
-export type Email = string
-export type Base64 = string
-export type Resource<S extends string> = any
+export type Sql = string;
+export type Email = string;
+export type Base64 = string;
+export type Resource<S extends string> = any;
 
-export const SHARED_FOLDER = '/shared'
+export const SHARED_FOLDER = "/shared";
 
 export function setClient(token: string, baseUrl: string) {
-    OpenAPI.WITH_CREDENTIALS = true
-    OpenAPI.TOKEN = token
-    OpenAPI.BASE = baseUrl + '/api'
+  OpenAPI.WITH_CREDENTIALS = true;
+  OpenAPI.TOKEN = token;
+  OpenAPI.BASE = baseUrl + "/api";
 }
 
-setClient(Deno.env.get("WM_TOKEN") ?? 'no_token', Deno.env.get("BASE_INTERNAL_URL") ?? Deno.env.get("BASE_URL") ?? 'http://localhost:8000')
+setClient(
+  Deno.env.get("WM_TOKEN") ?? "no_token",
+  Deno.env.get("BASE_INTERNAL_URL") ??
+    Deno.env.get("BASE_URL") ??
+    "http://localhost:8000"
+);
 
 /**
  * Create a client configuration from env variables
  * @returns client configuration
  */
 export function getWorkspace(): string {
-    return Deno.env.get("WM_WORKSPACE") ?? 'no_workspace'
+  return Deno.env.get("WM_WORKSPACE") ?? "no_workspace";
 }
 
 /**
@@ -44,27 +62,30 @@ export function getWorkspace(): string {
  * @param undefinedIfEmpty if the resource does not exist, return undefined instead of throwing an error
  * @returns resource value
  */
-export async function getResource(path?: string, undefinedIfEmpty?: boolean): Promise<any> {
-    const workspace = getWorkspace()
-    path = path ?? getStatePath()
-    try {
-        const resource = await ResourceService.getResource({ workspace, path })
-        return await _transformLeaf(resource.value)
-    } catch (e: any) {
-        if (undefinedIfEmpty && e.status === 404) {
-            return undefined
-        } else {
-            throw Error(`Resource not found at ${path} or not visible to you`)
-        }
+export async function getResource(
+  path?: string,
+  undefinedIfEmpty?: boolean
+): Promise<any> {
+  const workspace = getWorkspace();
+  path = path ?? getStatePath();
+  try {
+    const resource = await ResourceService.getResource({ workspace, path });
+    return await _transformLeaf(resource.value);
+  } catch (e: any) {
+    if (undefinedIfEmpty && e.status === 404) {
+      return undefined;
+    } else {
+      throw Error(`Resource not found at ${path} or not visible to you`);
     }
+  }
 }
 
 export function getStatePath(): string {
-    const state_path = Deno.env.get("WM_STATE_PATH")
-    if (state_path === undefined) {
-        throw Error("State path not set")
-    }
-    return state_path
+  const state_path = Deno.env.get("WM_STATE_PATH");
+  if (state_path === undefined) {
+    throw Error("State path not set");
+  }
+  return state_path;
 }
 
 /**
@@ -73,16 +94,29 @@ export function getStatePath(): string {
  * @param value new value of the resource to set
  * @param initializeToTypeIfNotExist if the resource does not exist, initialize it with this type
  */
-export async function setResource(value: any, path?: string, initializeToTypeIfNotExist?: string): Promise<void> {
-    path = path ?? getStatePath()
-    const workspace = getWorkspace()
-    if (await ResourceService.existsResource({ workspace, path })) {
-        await ResourceService.updateResourceValue({ workspace, path, requestBody: { value } })
-    } else if (initializeToTypeIfNotExist) {
-        await ResourceService.createResource({ workspace, requestBody: { path, value, resource_type: initializeToTypeIfNotExist } })
-    } else {
-        throw Error(`Resource at path ${path} does not exist and no type was provided to initialize it`)
-    }
+export async function setResource(
+  value: any,
+  path?: string,
+  initializeToTypeIfNotExist?: string
+): Promise<void> {
+  path = path ?? getStatePath();
+  const workspace = getWorkspace();
+  if (await ResourceService.existsResource({ workspace, path })) {
+    await ResourceService.updateResourceValue({
+      workspace,
+      path,
+      requestBody: { value },
+    });
+  } else if (initializeToTypeIfNotExist) {
+    await ResourceService.createResource({
+      workspace,
+      requestBody: { path, value, resource_type: initializeToTypeIfNotExist },
+    });
+  } else {
+    throw Error(
+      `Resource at path ${path} does not exist and no type was provided to initialize it`
+    );
+  }
 }
 
 /**
@@ -91,7 +125,7 @@ export async function setResource(value: any, path?: string, initializeToTypeIfN
  * @deprecated use setState instead
  */
 export async function setInternalState(state: any): Promise<void> {
-    await setResource(state, undefined, 'state')
+  await setResource(state, undefined, "state");
 }
 
 /**
@@ -99,39 +133,41 @@ export async function setInternalState(state: any): Promise<void> {
  * @param state state to set
  */
 export async function setState(state: any): Promise<void> {
-    await setResource(state, undefined, 'state')
+  await setResource(state, undefined, "state");
 }
 
 /**
  * Set the shared state
  * @param state state to set
  */
-export async function setSharedState(state: any, path = 'state.json'): Promise<void> {
-    await Deno.writeTextFile(SHARED_FOLDER + '/' + path, JSON.stringify(state))
+export async function setSharedState(
+  state: any,
+  path = "state.json"
+): Promise<void> {
+  await Deno.writeTextFile(SHARED_FOLDER + "/" + path, JSON.stringify(state));
 }
 
 /**
  * Get the shared state
  * @param state state to set
  */
-export async function getSharedState(path = 'state.json'): Promise<any> {
-    return JSON.parse(await Deno.readTextFile(SHARED_FOLDER + '/' + path))
+export async function getSharedState(path = "state.json"): Promise<any> {
+  return JSON.parse(await Deno.readTextFile(SHARED_FOLDER + "/" + path));
 }
 
-
 /**
  * Get the internal state
  * @deprecated use getState instead
  */
 export async function getInternalState(): Promise<any> {
-    return await getResource(getStatePath(), true)
+  return await getResource(getStatePath(), true);
 }
 
 /**
  * Get the state shared across executions
  */
 export async function getState(): Promise<any> {
-    return await getResource(getStatePath(), true)
+  return await getResource(getStatePath(), true);
 }
 
 /**
@@ -140,38 +176,43 @@ export async function getState(): Promise<any> {
  * @returns variable value
  */
 export async function getVariable(path: string): Promise<string | undefined> {
-    const workspace = getWorkspace()
-    try {
-        const variable = await VariableService.getVariable({ workspace, path })
-        return variable.value
-    } catch (e: any) {
-        throw Error(`Variable not found at ${path} or not visible to you`)
-    }
+  const workspace = getWorkspace();
+  try {
+    const variable = await VariableService.getVariable({ workspace, path });
+    return variable.value;
+  } catch (e: any) {
+    throw Error(`Variable not found at ${path} or not visible to you`);
+  }
 }
 
-async function transformLeaves(d: { [key: string]: any }): Promise<{ [key: string]: any }> {
-    for (const k in d) {
-        d[k] = await _transformLeaf(d[k])
-    }
-    return d
+async function transformLeaves(d: {
+  [key: string]: any;
+}): Promise<{ [key: string]: any }> {
+  for (const k in d) {
+    d[k] = await _transformLeaf(d[k]);
+  }
+  return d;
 }
 
-const VAR_RESOURCE_PREFIX = "$var:"
+const VAR_RESOURCE_PREFIX = "$var:";
+const RES_RESOURCE_PREFIX = "$res:";
 async function _transformLeaf(v: any): Promise<any> {
-    if (typeof v === 'object') {
-        return transformLeaves(v)
-    }
-    else if (typeof v === 'string' && v.startsWith(VAR_RESOURCE_PREFIX)) {
-        const varName = v.substring(VAR_RESOURCE_PREFIX.length)
-        return await getVariable(varName)
-    } else {
-        return v
-    }
+  if (typeof v === "object") {
+    return transformLeaves(v);
+  } else if (typeof v === "string" && v.startsWith(VAR_RESOURCE_PREFIX)) {
+    const varName = v.substring(VAR_RESOURCE_PREFIX.length);
+    return await getVariable(varName);
+  } else if (typeof v === "string" && v.startsWith(RES_RESOURCE_PREFIX)) {
+    const resName = v.substring(RES_RESOURCE_PREFIX.length);
+    return await getResource(resName);
+  } else {
+    return v;
+  }
 }
 
 export async function databaseUrlFromResource(path: string): Promise<string> {
-    const resource = await getResource(path)
-    return `postgresql://${resource.user}:${resource.password}@${resource.host}:${resource.port}/${resource.dbname}?sslmode=${resource.sslmode}`
+  const resource = await getResource(path);
+  return `postgresql://${resource.user}:${resource.password}@${resource.host}:${resource.port}/${resource.dbname}?sslmode=${resource.sslmode}`;
 }
 
 /**
@@ -180,78 +221,84 @@ export async function databaseUrlFromResource(path: string): Promise<string> {
  * @returns approval page UI URL, resume and cancel API URLs for resumeing the flow
  */
 export async function getResumeUrls(approver?: string): Promise<{
-    approvalPage: string;
-    resume: string;
-    cancel: string;
+  approvalPage: string;
+  resume: string;
+  cancel: string;
 }> {
-    const nonce = Math.floor(Math.random() * 4294967295);
-    const workspace = getWorkspace()
-    return await JobService.getResumeUrls({ workspace, resumeId: nonce, approver, id: Deno.env.get("WM_JOB_ID") ?? 'NO_JOB_ID' })
+  const nonce = Math.floor(Math.random() * 4294967295);
+  const workspace = getWorkspace();
+  return await JobService.getResumeUrls({
+    workspace,
+    resumeId: nonce,
+    approver,
+    id: Deno.env.get("WM_JOB_ID") ?? "NO_JOB_ID",
+  });
 }
 
 /**
  * @deprecated use getResumeUrls instead
  */
 export function getResumeEndpoints(approver?: string): Promise<{
-    approvalPage: string;
-    resume: string;
-    cancel: string;
+  approvalPage: string;
+  resume: string;
+  cancel: string;
 }> {
-    return getResumeUrls(approver)
+  return getResumeUrls(approver);
 }
 
 export function base64ToUint8Array(data: string): Uint8Array {
-    return Uint8Array.from(atob(data), c => c.charCodeAt(0))
+  return Uint8Array.from(atob(data), (c) => c.charCodeAt(0));
 }
 
 export function uint8ArrayToBase64(arrayBuffer: Uint8Array): string {
-    let base64 = ''
-    const encodings = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'
+  let base64 = "";
+  const encodings =
+    "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
 
-    const bytes = new Uint8Array(arrayBuffer)
-    const byteLength = bytes.byteLength
-    const byteRemainder = byteLength % 3
-    const mainLength = byteLength - byteRemainder
+  const bytes = new Uint8Array(arrayBuffer);
+  const byteLength = bytes.byteLength;
+  const byteRemainder = byteLength % 3;
+  const mainLength = byteLength - byteRemainder;
 
-    let a, b, c, d
-    let chunk
+  let a, b, c, d;
+  let chunk;
 
-    // Main loop deals with bytes in chunks of 3
-    for (let i = 0; i < mainLength; i = i + 3) {
-        // Combine the three bytes into a single integer
-        chunk = (bytes[i] << 16) | (bytes[i + 1] << 8) | bytes[i + 2]
+  // Main loop deals with bytes in chunks of 3
+  for (let i = 0; i < mainLength; i = i + 3) {
+    // Combine the three bytes into a single integer
+    chunk = (bytes[i] << 16) | (bytes[i + 1] << 8) | bytes[i + 2];
 
-        // Use bitmasks to extract 6-bit segments from the triplet
-        a = (chunk & 16515072) >> 18 // 16515072 = (2^6 - 1) << 18
-        b = (chunk & 258048) >> 12 // 258048   = (2^6 - 1) << 12
-        c = (chunk & 4032) >> 6 // 4032     = (2^6 - 1) << 6
-        d = chunk & 63               // 63       = 2^6 - 1
+    // Use bitmasks to extract 6-bit segments from the triplet
+    a = (chunk & 16515072) >> 18; // 16515072 = (2^6 - 1) << 18
+    b = (chunk & 258048) >> 12; // 258048   = (2^6 - 1) << 12
+    c = (chunk & 4032) >> 6; // 4032     = (2^6 - 1) << 6
+    d = chunk & 63; // 63       = 2^6 - 1
 
-        // Convert the raw binary segments to the appropriate ASCII encoding
-        base64 += encodings[a] + encodings[b] + encodings[c] + encodings[d]
-    }
+    // Convert the raw binary segments to the appropriate ASCII encoding
+    base64 += encodings[a] + encodings[b] + encodings[c] + encodings[d];
+  }
 
-    // Deal with the remaining bytes and padding
-    if (byteRemainder == 1) {
-        chunk = bytes[mainLength]
+  // Deal with the remaining bytes and padding
+  if (byteRemainder == 1) {
+    chunk = bytes[mainLength];
 
-        a = (chunk & 252) >> 2 // 252 = (2^6 - 1) << 2
+    a = (chunk & 252) >> 2; // 252 = (2^6 - 1) << 2
 
-        // Set the 4 least significant bits to zero
-        b = (chunk & 3) << 4 // 3   = 2^2 - 1
+    // Set the 4 least significant bits to zero
+    b = (chunk & 3) << 4; // 3   = 2^2 - 1
 
-        base64 += encodings[a] + encodings[b] + '=='
-    } else if (byteRemainder == 2) {
-        chunk = (bytes[mainLength] << 8) | bytes[mainLength + 1]
+    base64 += encodings[a] + encodings[b] + "==";
+  } else if (byteRemainder == 2) {
+    chunk = (bytes[mainLength] << 8) | bytes[mainLength + 1];
 
-        a = (chunk & 64512) >> 10 // 64512 = (2^6 - 1) << 10
-        b = (chunk & 1008) >> 4 // 1008  = (2^6 - 1) << 4
+    a = (chunk & 64512) >> 10; // 64512 = (2^6 - 1) << 10
+    b = (chunk & 1008) >> 4; // 1008  = (2^6 - 1) << 4
 
-        // Set the 2 least significant bits to zero
-        c = (chunk & 15) << 2 // 15    = 2^4 - 1
+    // Set the 2 least significant bits to zero
+    c = (chunk & 15) << 2; // 15    = 2^4 - 1
 
-        base64 += encodings[a] + encodings[b] + encodings[c] + '='
-    }
+    base64 += encodings[a] + encodings[b] + encodings[c] + "=";
+  }
 
-    return base64
+  return base64;
 }

docker-compose.yml

@@ -38,7 +38,9 @@ services:
     depends_on:
       db:
         condition: service_healthy
-
+    # volumes:
+      # - ./oauth.json/:/usr/src/app/oauth.json
+      
   windmill_worker:
     image: ghcr.io/windmill-labs/windmill:main
     deploy:
@@ -61,7 +63,6 @@ services:
     # to mount the worker folder to debug,, KEEP_JOB_DIR=true and mount /tmp/windmill
     volumes:
       - worker_dependency_cache:/tmp/windmill/cache
-      # - ./oauth.json/:/usr/src/app/oauth.json
 
   lsp:
     image: ghcr.io/windmill-labs/windmill-lsp:latest

frontend/.env

@@ -0,0 +1 @@
+VITE_DEFAULT_WEBHOOK_TYPE=async 

frontend/.gitignore

@@ -7,3 +7,4 @@ node_modules
 CaddyfileRemoteRuben
 tests-out/
 storageState.json
+.env.production

frontend/README_DEV.md

@@ -1,6 +1,6 @@
 # Developing
 
-## Starting the Development Server
+## Starting The Development Server
 
 Once you've created a project and installed dependencies with `npm install` (or
 `pnpm install` or `yarn`), start a development server:
@@ -17,7 +17,7 @@ In the root folder:
 
 ```bash
 docker build . -t windmill
-docker-compose up db server
+docker compose up db windmill_server windmill_worker
 ```
 
 ### 2. Backend is run by cargo
@@ -25,9 +25,9 @@ docker-compose up db server
 **Prerequisites**
 
 - Install Rust [as explained on the website](https://www.rust-lang.org/tools/install).
-- Install llvm 
+- Install llvm
 
-  **on OSX:**
+  **On OSX:**
   ```bash
   brew install llvm caddy gsed
   
@@ -36,7 +36,20 @@ docker-compose up db server
   
   # now, restart your shell. You should now have the `lld` binary on your PATH.
   ```
-  
+- To test that you have Rust and Cargo installed run `cargo --version`
+
+- In your terminal, go to the backend directory and run `cargo build`
+- Run `cargo run`
+
+**Known issue on M1 Mac while running `cargo build`**
+- You may encounter `linking with cc failed` build time error.
+- To solve this run:
+  ```bash
+  echo 'export RUSTFLAGS="-L/opt/homebrew/opt/libomp/lib"' >> ~/.zshrc
+  source ~/.zshrc
+  ```
+
+
 **Do a Frontend Build**
 
 In order to run the backend, you need to have a frontend build inside `frontend/build/`.
@@ -56,6 +69,14 @@ npm run build
 # now, you'll have a `frontend/build` folder.
 ```
 
+**Known issue while running `npm run build`**
+- You may encounter `FATAL ERROR: Ineffective mark-compacts near heap limit Allocation failed - JavaScript heap out of memory` error.
+- To solve this run:
+```bash
+export NODE_OPTIONS=--max_old_space_size=8096
+```
+- run `npm run build` again
+
 In the root folder:
 
 ```bash
@@ -82,7 +103,7 @@ sudo caddy run --config ./Caddyfile
 
 and then go to <http://localhost>
 
-### Backend is run by remote!
+### 3. Backend is run by remote!
 
 ```bash
 sudo caddy run --config ./CaddyfileRemote
@@ -119,8 +140,30 @@ Recommended config for VS Code:
 
 ```json
 "[svelte]": {
-        "editor.defaultFormatter": "esbenp.prettier-vscode"
-    }
+"editor.defaultFormatter": "esbenp.prettier-vscode"
+}
 ```
 
 - turn _format on save_ on
+
+
+## Building
+
+The project is built with [SvelteKit](https://kit.svelte.dev/) and uses as output static files.
+There are others adapters for sveltekit, but we use the static adapter.
+
+To build the frontend as static assets, use:
+```
+npm run build
+```
+
+The output is in the `build` folder.
+
+The default build assume you serve every non static files as the 200.html file which is catchall. If you prefer a normal layout, you can use:
+```
+NOTCATCHALL=true npm run build
+```
+which will generate an index.html and allow you to serve the frontend with any static server.
+
+Env variables used for build are set in .env file. See [https://vitejs.dev/guide/env-and-mode.html#env-files](https://vitejs.dev/guide/env-and-mode.html#env-files) for more details.
+

frontend/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "windmill",
-	"version": "1.70.1",
+	"version": "1.83.1",
 	"scripts": {
 		"dev": "vite dev",
 		"build": "vite build",
@@ -21,12 +21,13 @@
 		"@sveltejs/package": "^1.0.2",
 		"@tailwindcss/forms": "^0.5.3",
 		"@tailwindcss/typography": "^0.5.8",
+		"@types/d3": "^7.4.0",
+		"@types/d3-zoom": "^3.0.2",
+		"@types/lodash": "^4.14.191",
 		"@types/node": "^18.11.18",
 		"@types/vscode": "~1.74.0",
 		"@typescript-eslint/eslint-plugin": "^5.49.0",
 		"@typescript-eslint/parser": "^5.48.0",
-		"@windmill-labs/svelte-grid": "^5.1.4",
-		"@windmill-labs/svelvet": "^4.0.20",
 		"@zerodevx/svelte-toast": "^0.8.1",
 		"autoprefixer": "^10.4.13",
 		"cssnano": "^5.1.14",
@@ -34,16 +35,20 @@
 		"eslint": "^8.28.0",
 		"eslint-config-prettier": "^8.6.0",
 		"eslint-plugin-svelte3": "^4.0.0",
+		"ol": "^7.2.2",
 		"openapi-typescript-codegen": "^0.23.0",
 		"path-browserify": "^1.0.1",
+		"pdfjs-dist": "^3.4.120",
 		"postcss": "^8.4.18",
 		"postcss-load-config": "^4.0.1",
 		"prettier": "^2.8.3",
 		"prettier-plugin-svelte": "^2.9.0",
 		"simple-svelte-autocomplete": "^2.5.1",
+		"style-to-object": "^0.4.1",
 		"stylelint-config-recommended": "^9.0.0",
 		"svelte": "^3.55.1",
 		"svelte-awesome": "^3.0.0",
+		"svelte-awesome-color-picker": "^2.4.1",
 		"svelte-check": "^3.0.2",
 		"svelte-highlight": "^6.2.1",
 		"svelte-overlay": "^1.4.1",
@@ -51,7 +56,7 @@
 		"svelte-preprocess": "^5.0.1",
 		"svelte-range-slider-pips": "^2.1.1",
 		"svelte-splitpanes": "^0.7.3",
-		"svelte2tsx": "^0.6.1",
+		"svelte2tsx": "^0.6.10",
 		"tailwindcss": "^3.2.2",
 		"tslib": "^2.4.1",
 		"typescript": "^4.9.3",
@@ -71,9 +76,11 @@
 		"chart.js": "^3.9.1",
 		"chartjs-adapter-date-fns": "^3.0.0",
 		"chartjs-plugin-zoom": "^2.0.0",
+		"d3-zoom": "^3.0.0",
 		"date-fns": "^2.29.3",
 		"fast-equals": "^4.0.3",
 		"highlight.js": "^11.7.0",
+		"lodash": "^4.17.21",
 		"lucide-svelte": "^0.115.0",
 		"monaco-editor": "0.33.0",
 		"monaco-editor-workers": "0.33.0",

frontend/src/global.d.ts

@@ -1,78 +1,2 @@
 /// <reference types="@sveltejs/kit" />
 
-declare type Item = import('svelte-dnd-action').Item
-declare type DndEvent<ItemType = Item> = import('svelte-dnd-action').DndEvent<ItemType>
-declare namespace svelte.JSX {
-	interface HTMLAttributes<T> {
-		onconsider?: (event: CustomEvent<DndEvent<ItemType>> & { target: EventTarget & T }) => void
-		onfinalize?: (event: CustomEvent<DndEvent<ItemType>> & { target: EventTarget & T }) => void
-	}
-}
-
-declare module '@windmill-labs/svelte-grid' {
-	import type { SvelteComponentTyped } from 'svelte'
-
-	export interface Size {
-		w: number
-		h: number
-	}
-
-	export interface Positon {
-		x: number
-		y: number
-	}
-
-	interface ItemLayout extends Size, Positon {
-		fixed?: boolean
-		resizable?: boolean
-		draggable?: boolean
-		customDragger?: boolean
-		customResizer?: boolean
-		min?: Size
-		max?: Size
-	}
-
-	export type Item<T> = T & { [width: number]: ItemLayout; data: any }
-	export type FilledItem<T> = T & { [width: number]: Required<ItemLayout>; data: any }
-
-	export interface Props<T> {
-		fillSpace?: boolean
-		items: FilledItem<T>[]
-		rowHeight: number
-		cols: [number, number][]
-		gap?: [number, number]
-		fastStart?: boolean
-		throttleUpdate?: number
-		throttleResize?: number
-		onTopId?: string
-		scroller?: undefined
-		sensor?: number
-	}
-
-	export interface Slots<T> {
-		default: { item: ItemLayout; dataItem: Item<T> }
-	}
-
-	export default class Grid<T = {}> extends SvelteComponentTyped<
-		Props<T>,
-		{
-			pointerup: CustomEvent<{ id: string }>
-			mount: CustomEvent<>
-		},
-		Slots<T>
-	> { }
-}
-
-declare module '@windmill-labs/svelte-grid/build/helper/index.mjs' {
-	import { ItemLayout } from '@windmill-labs/svelte-grid'
-
-	const x: {
-		normalize(items: any[], col: any): unknown[]
-		adjust(items: any[], col: any): unknown[]
-		findSpace(item: any, items: any, cols: any): unknown
-
-		item<T>(obj: ItemLayout): Required<ItemLayout>
-	}
-
-	export default x
-}

frontend/src/lib/assets/app.css

@@ -50,12 +50,23 @@
 	.Template-editor span.mtk20 {
 		color: black !important;
 	}
+
+	::-webkit-scrollbar {
+		width: 9px;
+		height: 9px;
+	}
+	::-webkit-scrollbar-track {
+		background: transparent;
+	}
+	::-webkit-scrollbar-thumb {
+		background-color: rgba(155, 155, 155, 0.5);
+		border: transparent;
+	}
 }
 
 @layer components {
 	/* Flow graph viewer -> Svelvet library internal class overwrite */
 	.Node {
 		display: flex !important;
-		cursor: pointer !important;
 	}
 }

frontend/src/lib/common.ts

@@ -1,4 +1,4 @@
-import type { Script } from "./gen"
+import type { Script } from './gen'
 
 export type OwnerKind = 'group' | 'user' | 'folder'
 
@@ -14,7 +14,7 @@ export interface SchemaProperty {
 	enum?: string[]
 	contentEncoding?: 'base64' | 'binary'
 	format?: string
-	items?: { type?: 'string' | 'number' | 'bytes'; contentEncoding?: 'base64' }
+	items?: { type?: 'string' | 'number' | 'bytes' | 'object'; contentEncoding?: 'base64' }
 	properties?: { [name: string]: SchemaProperty }
 }
 
@@ -28,11 +28,14 @@ export type Schema = {
 export type Meta = { ownerKind: OwnerKind; owner: string; name: string }
 
 type Enumerate<N extends number, Acc extends number[] = []> = Acc['length'] extends N
-  ? Acc[number]
-  : Enumerate<N, [...Acc, Acc['length']]>
+	? Acc[number]
+	: Enumerate<N, [...Acc, Acc['length']]>
 
 /** An inclusive range of integer numbers */
-export type IntRange<F extends number, T extends number> = F | Exclude<Enumerate<T>, Enumerate<F>> | T
+export type IntRange<F extends number, T extends number> =
+	| F
+	| Exclude<Enumerate<T>, Enumerate<F>>
+	| T
 
 export function pathToMeta(path: string): Meta {
 	const splitted = path.split('/')

frontend/src/lib/components/AppConnect.svelte

@@ -81,7 +81,7 @@
 	$: key =
 		apiTokenApps[resource_type]?.key ??
 		(args != undefined
-			? Object.keys(args).filter((x) => ['token', 'password', 'api_key'].includes(x))[0]
+			? Object.keys(args).filter((x) => ['token', 'password', 'api_key', 'key'].includes(x))[0]
 			: undefined)
 
 	let scopes: string[] = []
@@ -259,6 +259,7 @@
 			args['token'] == '' &&
 			args['password'] == '' &&
 			args['api_key'] == '' &&
+			args['key'] == '' &&
 			key != undefined) ||
 		(step == 3 && pathError != '') ||
 		!isValid
@@ -396,7 +397,7 @@
 				bind:error={pathError}
 				bind:path
 				initialPath=""
-				namePlaceholder="{resource_type}"
+				namePlaceholder={resource_type}
 				kind="resource"
 			/>
 
@@ -433,7 +434,7 @@
 		{:else}
 			<Path
 				initialPath=""
-				namePlaceholder="{resource_type}"
+				namePlaceholder={resource_type}
 				bind:error={pathError}
 				bind:path
 				kind="resource"

frontend/src/lib/components/ArgInput.svelte

@@ -7,7 +7,7 @@
 		faPlus
 	} from '@fortawesome/free-solid-svg-icons'
 
-	import { setInputCat as computeInputCat, type InputCat } from '$lib/utils'
+	import { setInputCat as computeInputCat } from '$lib/utils'
 	import { Badge, Button } from './common'
 	import { createEventDispatcher } from 'svelte'
 	import Icon from 'svelte-awesome'
@@ -26,6 +26,9 @@
 	import type ItemPicker from './ItemPicker.svelte'
 	import NumberTypeNarrowing from './NumberTypeNarrowing.svelte'
 	import Range from './Range.svelte'
+	import JsonEditor from './apps/editor/settingsPanel/inputEditor/JsonEditor.svelte'
+	import { fade } from 'svelte/transition'
+	import { X } from 'lucide-svelte'
 
 	export let label: string = ''
 	export let value: any
@@ -44,7 +47,7 @@
 	export let disabled = false
 	export let editableSchema = false
 	export let itemsType:
-		| { type?: 'string' | 'number' | 'bytes'; contentEncoding?: 'base64' }
+		| { type?: 'string' | 'number' | 'bytes' | 'object'; contentEncoding?: 'base64' }
 		| undefined = undefined
 	export let displayHeader = true
 	export let properties: { [name: string]: SchemaProperty } | undefined = undefined
@@ -93,7 +96,7 @@
 
 	export function evalValueToRaw() {
 		if (value) {
-			rawValue = JSON.stringify(value, null, 4)
+			rawValue = JSON.stringify(value, null, 2)
 		}
 	}
 
@@ -152,18 +155,21 @@
 					value = enum_?.[0]
 				} else if (inputCat == 'boolean') {
 					value = false
+				} else if (inputCat == 'list') {
+					value = []
 				}
 			}
 		}
 	}
 
 	$: inputCat = computeInputCat(type, format, itemsType?.type, enum_, contentEncoding)
+	let redraw = 0
 </script>
 
 <div class="flex flex-col w-full min-w-[250px]">
 	<div>
 		{#if displayHeader}
-			<FieldHeader {label} {required} {type} {contentEncoding} {format} {itemsType} />
+			<FieldHeader {label} {required} {type} {contentEncoding} {format} />
 		{/if}
 		{#if editableSchema}
 			<div class="p-2 my-1 text-xs border-solid border border-gray-400">
@@ -186,6 +192,7 @@
 								use:autosize
 								rows="1"
 								bind:value={description}
+								on:keydown|stopPropagation
 								placeholder="Field description"
 							/>
 							{#if type == 'string' && format != 'date-time'}
@@ -198,6 +205,7 @@
 								<select bind:value={itemsType}>
 									<option value={undefined}>No specific item type</option>
 									<option value={{ type: 'string' }}> Items are strings</option>
+									<option value={{ type: 'object' }}> Items are objects (JSON)</option>
 									<option value={{ type: 'number' }}>Items are numbers</option>
 									<option value={{ type: 'string', contentEncoding: 'base64' }}
 										>Items are bytes</option
@@ -232,7 +240,6 @@
 					<input
 						{autofocus}
 						on:focus={(e) => {
-							window.dispatchEvent(new Event('pointerup'))
 							dispatch('focus')
 						}}
 						{disabled}
@@ -244,14 +251,12 @@
 						bind:value
 						min={extra['min']}
 						max={extra['max']}
-						on:input={() => dispatch('input', { value, isRaw: true })}
 					/>
 				{/if}
 			{:else if inputCat == 'boolean'}
 				<Toggle
 					on:pointerdown={(e) => {
 						e?.stopPropagation()
-						window.dispatchEvent(new Event('pointerup'))
 					}}
 					{disabled}
 					class={valid
@@ -263,44 +268,46 @@
 					<span>&nbsp; Not set</span>
 				{/if}
 			{:else if inputCat == 'list'}
-				<div>
-					<div>
-						{#each value ?? [] as v, i}
-							<div class="flex flex-row max-w-md mt-1">
-								{#if itemsType?.type == 'number'}
-									<input autofocus={autofocus && i == 0} type="number" bind:value={v} />
-								{:else if itemsType?.type == 'string' && itemsType?.contentEncoding == 'base64'}
-									<input
-										autofocus={autofocus && i == 0}
-										type="file"
-										class="my-6"
-										on:change={(x) => fileChanged(x, (val) => (value[i] = val))}
-										multiple={false}
-									/>
-								{:else}
-									<input autofocus={autofocus && i == 0} type="text" bind:value={v} />
-								{/if}
-								<Button
-									variant="border"
-									color="red"
-									size="sm"
-									btnClasses="mx-6"
-									on:click={() => {
-										value = value.filter((el) => el != v)
-										if (value.length == 0) {
-											value = undefined
-										}
-									}}
-								>
-									<Icon data={faMinus} />
-								</Button>
-							</div>
-						{/each}
+				<div class="w-full">
+					<div class="w-full">
+						{#key redraw}
+							{#if Array.isArray(value)}
+								{#each value ?? [] as v, i}
+									<div class="flex max-w-md mt-1 w-full items-center">
+										{#if itemsType?.type == 'number'}
+											<input type="number" bind:value={v} />
+										{:else if itemsType?.type == 'string' && itemsType?.contentEncoding == 'base64'}
+											<input
+												type="file"
+												class="my-6"
+												on:change={(x) => fileChanged(x, (val) => (value[i] = val))}
+												multiple={false}
+											/>
+										{:else if itemsType?.type == 'object'}
+											<JsonEditor code={JSON.stringify(v, null, 2)} bind:value={v} />
+										{:else}
+											<input type="text" bind:value={v} />
+										{/if}
+										<button
+											transition:fade|local={{ duration: 100 }}
+											class="rounded-full p-1 bg-white/60 duration-200 hover:bg-gray-200"
+											aria-label="Clear"
+											on:click={() => {
+												value.splice(i, 1)
+												redraw += 1
+											}}
+										>
+											<X size={14} />
+										</button>
+									</div>
+								{/each}
+							{/if}
+						{/key}
 					</div>
 					<Button
 						variant="border"
-						color="blue"
-						size="sm"
+						color="dark"
+						size="xs"
 						btnClasses="mt-1"
 						on:click={() => {
 							if (value == undefined || !Array.isArray(value)) {
@@ -312,9 +319,6 @@
 						<Icon data={faPlus} class="mr-2" />
 						Add item
 					</Button>
-					<span class="ml-2">
-						{(value ?? []).length} item{(value ?? []).length > 1 ? 's' : ''}
-					</span>
 				</div>
 			{:else if inputCat == 'resource-object'}
 				<ObjectResourceInput {format} bind:value />
@@ -327,31 +331,21 @@
 							bind:args={value}
 						/>
 					</div>
+				{:else if disabled}
+					<textarea disabled />
 				{:else}
-					<textarea
-						bind:this={el}
+					<JsonEditor
+						bind:editor
 						on:focus={(e) => {
-							window.dispatchEvent(new Event('pointerup'))
 							dispatch('focus')
 						}}
-						{autofocus}
-						{disabled}
-						use:autosize
-						style="max-height: {maxHeight}"
-						on:input={() => {
-							dispatch('input', { rawValue: value, isRaw: false })
-						}}
-						class="col-span-10 {valid
-							? ''
-							: 'border border-red-700 border-opacity-30 focus:border-red-700 focus:border-opacity-30 bg-red-100'}"
-						placeholder={defaultValue ? JSON.stringify(defaultValue, null, 4) : ''}
-						bind:value={rawValue}
+						code={rawValue}
+						bind:value
 					/>
 				{/if}
 			{:else if inputCat == 'enum'}
 				<select
 					on:focus={(e) => {
-						window.dispatchEvent(new Event('pointerup'))
 						dispatch('focus')
 					}}
 					{disabled}
@@ -368,16 +362,11 @@
 				<div class="border my-1 mb-4 w-full border-gray-400">
 					<SimpleEditor
 						on:focus={(e) => {
-							window.dispatchEvent(new Event('pointerup'))
 							dispatch('focus')
 						}}
-						on:blur={() => dispatch('blur')}
 						bind:this={editor}
 						lang={inputCat}
 						bind:code={value}
-						on:change={async () => {
-							dispatch('input', { rawValue: value, isRaw: false })
-						}}
 						autoHeight
 					/>
 				</div>
@@ -407,11 +396,10 @@
 								rows="1"
 								bind:this={el}
 								on:focus={(e) => {
-									window.dispatchEvent(new Event('pointerup'))
 									dispatch('focus')
 								}}
-								on:blur={() => dispatch('blur')}
 								use:autosize
+								on:keydown|stopPropagation
 								type="text"
 								{disabled}
 								class="col-span-10 {valid
@@ -419,9 +407,6 @@
 									: 'border border-red-700 border-opacity-30 focus:border-red-700 focus:border-opacity-30 bg-red-100'}"
 								placeholder={defaultValue ?? ''}
 								bind:value
-								on:input={() => {
-									dispatch('input', { rawValue: value, isRaw: false })
-								}}
 							/>
 							{#if itemPicker}
 								<div class="ml-1 relative">
@@ -450,19 +435,6 @@
 					{/if}
 				</div>
 			{/if}
-			{#if !required && inputCat != 'resource-object'}
-				<!-- <Tooltip placement="bottom" content="Reset to default value">
-					<Button
-						on:click={() => (value = undefined)}
-						{disabled}
-						color="alternative"
-						size="sm"
-						class="h-8"
-					>
-						<Icon data={faArrowRotateLeft} />
-					</Button>
-				</Tooltip> -->
-			{/if}
 			<slot name="actions" />
 		</div>
 		{#if !compact || (error && error != '')}

frontend/src/lib/components/DisplayResult.svelte

@@ -6,6 +6,7 @@
 	import { Button, Drawer, DrawerContent } from './common'
 	import autosize from 'svelte-autosize'
 	import { ClipboardCopy } from 'lucide-svelte'
+	import Portal from 'svelte-portal'
 
 	export let result: any
 	export let requireHtmlApproval = false
@@ -91,28 +92,13 @@
 	let jsonViewer: Drawer
 </script>
 
-<Drawer bind:this={jsonViewer} size="900px">
-	<DrawerContent title="Expanded Result" on:close={jsonViewer.closeDrawer}>
-		<svelte:fragment slot="actions">
-			<Button
-				on:click={() => copyToClipboard(JSON.stringify(result, null, 4))}
-				color="light"
-				size="xs"
-			>
-				<div class="flex gap-2 items-center">Copy to clipboard <ClipboardCopy /> </div>
-			</Button>
-		</svelte:fragment>
-		<Highlight language={json} code={JSON.stringify(result, null, 4).replace(/\\n/g, '\n')} />
-	</DrawerContent>
-</Drawer>
-
 <div class="inline-highlight">
 	{#if result != undefined}
 		{#if resultKind && resultKind != 'json'}
 			<div class="mb-2 text-gray-500 text-sm bg-gray-50/20">
-				as JSON&nbsp;<input type="checkbox" bind:checked={forceJson} /></div
+				as JSON&nbsp;<input class="windmillapp" type="checkbox" bind:checked={forceJson} /></div
 			>{/if}{#if typeof result == 'object' && Object.keys(result).length > 0}<div
-				class="mb-2 text-sm text-gray-700 relative"
+				class="mb-2 min-w-[360px] text-sm text-gray-700 relative"
 				>The result keys are: <b>{truncate(Object.keys(result).join(', '), 50)}</b>
 				<div class="text-gray-500 text-sm  absolute top-0 right-2">
 					<button on:click={jsonViewer.openDrawer}>Expand JSON</button>
@@ -248,6 +234,23 @@
 			<Highlight language={json} code={JSON.stringify(result, null, 4).replace(/\\n/g, '\n')} />
 		{/if}
 	{:else}
-		<div class="text-gray-500 text-sm">No result</div>
+		<div class="text-gray-500 text-sm">No result: {JSON.stringify(result)}</div>
 	{/if}
 </div>
+
+<Portal>
+	<Drawer bind:this={jsonViewer} size="900px">
+		<DrawerContent title="Expanded Result" on:close={jsonViewer.closeDrawer}>
+			<svelte:fragment slot="actions">
+				<Button
+					on:click={() => copyToClipboard(JSON.stringify(result, null, 4))}
+					color="light"
+					size="xs"
+				>
+					<div class="flex gap-2 items-center">Copy to clipboard <ClipboardCopy /> </div>
+				</Button>
+			</svelte:fragment>
+			<Highlight language={json} code={JSON.stringify(result, null, 4).replace(/\\n/g, '\n')} />
+		</DrawerContent>
+	</Drawer>
+</Portal>

frontend/src/lib/components/Dropdown.svelte

@@ -2,9 +2,10 @@
 	import type { DropdownItem } from '$lib/utils'
 	import { createEventDispatcher } from 'svelte'
 	import Icon from 'svelte-awesome'
-	import { MoreHorizontal } from 'lucide-svelte'
+	import { MoreVertical } from 'lucide-svelte'
 	import { Button, Menu } from './common'
 	import { goto } from '$app/navigation'
+	import { twMerge } from 'tailwind-merge'
 
 	type Alignment = 'start' | 'end'
 	type Side = 'top' | 'bottom'
@@ -13,15 +14,24 @@
 	export let dropdownItems: DropdownItem[]
 	export let name: string | undefined = undefined
 	export let placement: Placement = 'bottom-start'
-	export let btnClasses = '!text-blue-500 bg-transparent'
+	export let btnClasses = ''
+	$: buttonClass = twMerge('!border-0 bg-transparent !p-[6px]', btnClasses)
 
 	const dispatch = createEventDispatcher()
 </script>
 
 <Menu {placement} let:close>
-	<Button nonCaptureEvent color="light" size="xs" {btnClasses} {...$$restProps} slot="trigger">
+	<Button
+		nonCaptureEvent
+		color="dark"
+		variant="border"
+		size="xs"
+		btnClasses={buttonClass}
+		{...$$restProps}
+		slot="trigger"
+	>
 		{#if !$$slots.default}
-			<MoreHorizontal size={20} />
+			<MoreVertical size={20} />
 		{:else}
 			<slot />
 		{/if}

frontend/src/lib/components/Editor.svelte

@@ -461,10 +461,6 @@
 			}
 		})
 
-		editor.onDidBlurEditorText(() => {
-			dispatch('blur')
-		})
-
 		reloadWebsocket()
 
 		return () => {

frontend/src/lib/components/FieldHeader.svelte

@@ -6,22 +6,23 @@
 	export let contentEncoding = ''
 	export let type: string | undefined = undefined
 	export let required = false
-	export let itemsType:
-		| { type?: 'string' | 'number' | 'bytes'; contentEncoding?: 'base64' }
-		| undefined = undefined
 </script>
 
-<div class="inline-flex flex-row items-center">
+<div class="inline-flex flex-row items-center truncated">
 	<span class="font-semibold">
 		{label}
 	</span>
 	<Required {required} class="!ml-0" />
 
-	<span class="text-sm italic ml-1 text-indigo-800">
-		({type ?? 'any'}{contentEncoding && contentEncoding != ''
-			? `, encoding: ${contentEncoding}`
-			: ''}{format && format != '' ? `, format: ${format}` : ''}{itemsType?.type
-			? ` of ${itemsType?.type}s`
-			: ''})</span
-	>
+	{#if format && format != ''}
+		<span class="text-sm italic ml-1 text-indigo-800">
+			({format})
+		</span>
+	{:else}
+		<span class="text-sm italic ml-1 text-indigo-800">
+			({type ?? 'any'}{contentEncoding && contentEncoding != ''
+				? `, encoding: ${contentEncoding}`
+				: ''})</span
+		>
+	{/if}
 </div>

frontend/src/lib/components/FlowBuilder.svelte

@@ -1,22 +1,20 @@
 <script lang="ts">
 	import { goto } from '$app/navigation'
-	import { page } from '$app/stores'
-	import { FlowService, ScheduleService, type Flow } from '$lib/gen'
+	import { FlowService, ScheduleService, type Flow, type FlowModule } from '$lib/gen'
+	import { initHistory, redo, undo } from '$lib/history'
 	import { userStore, workspaceStore } from '$lib/stores'
 	import { encodeState, formatCron, loadHubScripts, sendUserToast } from '$lib/utils'
-	import { faCalendarAlt, faEye, faPen, faSave } from '@fortawesome/free-solid-svg-icons'
+	import { faCalendarAlt, faPen, faSave } from '@fortawesome/free-solid-svg-icons'
 	import { setContext } from 'svelte'
-	import { writable } from 'svelte/store'
+	import { writable, type Writable } from 'svelte/store'
 	import CenteredPage from './CenteredPage.svelte'
-	import { Button, Drawer, DrawerContent } from './common'
+	import { Button, ButtonPopup, ButtonPopupItem, UndoRedo } from './common'
 	import { dirtyStore } from './common/confirmationModal/dirtyStore'
-	import UnsavedConfirmationModal from './common/confirmationModal/UnsavedConfirmationModal.svelte'
 	import { OFFSET } from './CronInput.svelte'
-	import FlowGraphViewer from './FlowGraphViewer.svelte'
 	import ScriptEditorDrawer from './flows/content/ScriptEditorDrawer.svelte'
 	import FlowEditor from './flows/FlowEditor.svelte'
-	import { flowStateStore } from './flows/flowState'
-	import { flowStore } from './flows/flowStore'
+	import type { FlowState } from './flows/flowState'
+	import { dfs } from './flows/flowStore'
 	import FlowImportExportMenu from './flows/header/FlowImportExportMenu.svelte'
 	import FlowPreviewButtons from './flows/header/FlowPreviewButtons.svelte'
 	import { loadFlowSchedule, type Schedule } from './flows/scheduleUtils'
@@ -27,8 +25,8 @@
 	export let selectedId: string | undefined
 	export let initialArgs: Record<string, any> = {}
 	export let loading = false
-
-	let pathError = ''
+	export let flowStore: Writable<Flow>
+	export let flowStateStore: Writable<FlowState>
 
 	async function createSchedule(path: string) {
 		const { cron, args, enabled } = $scheduleStore
@@ -51,7 +49,10 @@
 		}
 	}
 
-	async function saveFlow(): Promise<void> {
+	let loadingSave = false
+
+	async function saveFlow(leave: boolean): Promise<void> {
+		loadingSave = true
 		const flow = cleanInputs($flowStore)
 		const { cron, args, enabled } = $scheduleStore
 		$dirtyStore = false
@@ -113,8 +114,13 @@
 				await createSchedule(flow.path)
 			}
 		}
-		sendUserToast(`Flow saved at ${$flowStore.path}`)
-		goto(`/flows/get/${$flowStore.path}?workspace_id=${$workspaceStore}`)
+		loadingSave = false
+		if (leave) {
+			goto(`/flows/get/${$flowStore.path}?workspace_id=${$workspaceStore}`)
+		} else if (initialPath !== $flowStore.path) {
+			initialPath = $flowStore.path
+			goto(`/flows/edit/${$flowStore.path}?workspace_id=${$workspaceStore}`)
+		}
 	}
 
 	let timeout: NodeJS.Timeout | undefined = undefined
@@ -142,11 +148,15 @@
 		}, 500)
 	}
 
-	const selectedIdStore = writable<string>(selectedId)
+	const selectedIdStore = writable<string>(selectedId ?? 'settings-metadata')
 
 	const scheduleStore = writable<Schedule>({ args: {}, cron: '', enabled: false })
 	const previewArgsStore = writable<Record<string, any>>(initialArgs)
 	const scriptEditorDrawer = writable<ScriptEditorDrawer | undefined>(undefined)
+	const moving = writable<{ module: FlowModule; modules: FlowModule[] } | undefined>(undefined)
+	const history = initHistory($flowStore)
+
+	const testStepStore = writable<Record<string, any>>({})
 
 	function select(selectedId: string) {
 		selectedIdStore.set(selectedId)
@@ -155,9 +165,13 @@
 	setContext<FlowEditorContext>('FlowEditorContext', {
 		selectedId: selectedIdStore,
 		schedule: scheduleStore,
-		select,
 		previewArgs: previewArgsStore,
-		scriptEditorDrawer
+		scriptEditorDrawer,
+		moving,
+		history,
+		flowStateStore,
+		flowStore,
+		testStepStore
 	})
 
 	async function loadSchedule() {
@@ -180,39 +194,90 @@
 
 	loadHubScripts()
 
-	let flowViewer: Drawer
+	function onKeyDown(event: KeyboardEvent) {
+		let classes = event.target?.['className']
+		if (
+			(typeof classes === 'string' && classes.includes('inputarea')) ||
+			['INPUT', 'TEXTAREA'].includes(document.activeElement?.tagName!)
+		) {
+			return
+		}
+
+		switch (event.key) {
+			case 'Z':
+				if (event.ctrlKey) {
+					$flowStore = redo(history)
+					event.preventDefault()
+				}
+				break
+			case 'z':
+				if (event.ctrlKey) {
+					$flowStore = undo(history, $flowStore)
+					$selectedIdStore = 'Input'
+					event.preventDefault()
+				}
+				break
+			case 's':
+				if (event.ctrlKey) {
+					saveFlow(false)
+					event.preventDefault()
+				}
+				break
+			case 'ArrowDown': {
+				let ids = generateIds()
+				let idx = ids.indexOf($selectedIdStore)
+				if (idx > -1 && idx < ids.length - 1) {
+					$selectedIdStore = ids[idx + 1]
+					event.preventDefault()
+				}
+				break
+			}
+			case 'ArrowUp': {
+				let ids = generateIds()
+				let idx = ids.indexOf($selectedIdStore)
+				if (idx > 0 && idx < ids.length) {
+					$selectedIdStore = ids[idx - 1]
+					event.preventDefault()
+				}
+				break
+			}
+		}
+	}
+
+	function generateIds() {
+		return [
+			'settings-metadata',
+			'constants',
+			...dfs($flowStore.value.modules, (module) => module.id)
+		]
+	}
 </script>
 
+<svelte:window on:keydown={onKeyDown} />
+
 {#if !$userStore?.operator}
 	<ScriptEditorDrawer bind:this={$scriptEditorDrawer} />
-	<UnsavedConfirmationModal />
-
-	<Drawer bind:this={flowViewer} size="75%">
-		<DrawerContent title="View Graph" on:close={flowViewer.closeDrawer} noPadding>
-			<div class="overflow-hidden h-full w-full">
-				<FlowGraphViewer flow={$flowStore} />
-			</div>
-		</DrawerContent>
-	</Drawer>
 
 	<div class="flex flex-col flex-1 h-screen">
 		<!-- Nav between steps-->
 		<div
 			class="justify-between flex flex-row w-full items-center pl-2.5 pr-6  space-x-4 overflow-x-auto scrollbar-hidden max-h-12 h-full"
 		>
-			<div class="flex flex-row">
+			<div class="flex flex-row gap-4 items-center">
 				<FlowImportExportMenu />
-				<Button
-					btnClasses="inline-flex"
-					startIcon={{ icon: faEye }}
-					variant="border"
-					color="light"
-					size="sm"
-					on:click={flowViewer.openDrawer}
-				>
-					Graph
-				</Button>
+				<UndoRedo
+					undoProps={{ disabled: $history.index === 0 }}
+					redoProps={{ disabled: $history.index === $history.history.length - 1 }}
+					on:undo={() => {
+						$flowStore = undo(history, $flowStore)
+						$selectedIdStore = 'Input'
+					}}
+					on:redo={() => {
+						$flowStore = redo(history)
+					}}
+				/>
 			</div>
+
 			<div class="gap-1 flex-row hidden md:flex shrink overflow-hidden">
 				{#if $scheduleStore.enabled}
 					<Button
@@ -259,12 +324,22 @@
 			<div class="flex flex-row space-x-2">
 				<FlowPreviewButtons />
 				<div class="center-center">
-					<Button
-						disabled={pathError != ''}
-						startIcon={{ icon: faSave }}
+					<ButtonPopup
+						loading={loadingSave}
 						size="sm"
-						on:click={saveFlow}>Save</Button
+						startIcon={{ icon: faSave }}
+						on:click={() => saveFlow(false)}
 					>
+						<svelte:fragment slot="main">Save</svelte:fragment>
+						<ButtonPopupItem on:click={() => saveFlow(true)}>Save and exit</ButtonPopupItem>
+						{#if initialPath != ''}
+							<ButtonPopupItem
+								on:click={() => {
+									window.open(`/flows/add?template=${initialPath}`)
+								}}>Fork</ButtonPopupItem
+							>
+						{/if}
+					</ButtonPopup>
 				</div>
 			</div>
 		</div>

frontend/src/lib/components/FlowGraphViewer.svelte

@@ -79,17 +79,16 @@
 </Drawer>
 <div class="grid grid-cols-3 w-full">
 	<div
-		bind:clientHeight={topHeight}
 		class="{noSide
 			? 'col-span-3'
-			: 'sm:col-span-2 col-span-3'} w-full border border-gray-400 h-screen"
+			: 'sm:col-span-2 col-span-3'} w-full border border-gray-400 max-h-screen"
 		class:overflow-auto={overflowAuto}
 	>
 		<FlowGraph
-			minHeight={topHeight}
+			minHeight={400}
 			modules={flow?.value?.modules}
 			failureModule={flow?.value?.failure_module}
-			on:click={(e) => (stepDetail = e.detail)}
+			on:select={(e) => (stepDetail = e.detail)}
 		/>
 	</div>
 	{#if !noSide}
@@ -97,7 +96,9 @@
 			class="w-full border-r border-b border-t border-gray-400 min-h-[150px] p-2 overflow-auto hidden sm:block"
 		>
 			{#if stepDetail == undefined}
-				<span class="font-black text-lg w-full my-4">
+				<SchemaViewer schema={flow?.schema} />
+
+				<span class="font-black text-lg w-full my-4 mt-14">
 					<span>Click on a step to see its details</span>
 				</span>
 			{:else if stepDetail == 'Input'}

frontend/src/lib/components/FlowJobResult.svelte

@@ -12,9 +12,9 @@
 
 <div
 	class:border={!noBorder}
-	class="grid {!col ? 'grid-cols-2' : 'grid-rows-2'} shadow  border-gray-400 h-full"
+	class="grid {!col ? 'grid-cols-2' : 'grid-rows-2'} shadow  border-gray-400 h-full max-h-screen"
 >
-	<div class="bg-white max-h-80 h-full p-1 overflow-auto relative">
+	<div class="bg-white {col ? '' : 'max-h-80'} h-full p-1 overflow-auto relative">
 		<span class="text-gray-500">Result</span>
 		{#if result}
 			<DisplayResult {result} />
@@ -24,7 +24,7 @@
 			<div class="text-gray-400">No result (result is undefined)</div>
 		{/if}
 	</div>
-	<div class="overflow-auto max-h-80 h-full relative">
+	<div class="overflow-auto {col ? '' : 'max-h-80'} h-full relative">
 		<LogViewer content={logs ?? ''} isLoading={false} />
 	</div>
 </div>

frontend/src/lib/components/FlowPreviewContent.svelte

@@ -5,13 +5,12 @@
 	import { Button, Kbd } from './common'
 	import { createEventDispatcher, getContext } from 'svelte'
 	import Icon from 'svelte-awesome'
-	import { dfs, flowStore } from './flows/flowStore'
+	import { dfs } from './flows/flowStore'
 	import type { FlowEditorContext } from './flows/types'
 	import { runFlowPreview } from './flows/utils'
 	import SchemaForm from './SchemaForm.svelte'
 	import FlowStatusViewer from '../components/FlowStatusViewer.svelte'
 	import FlowProgressBar from './flows/FlowProgressBar.svelte'
-	import { flowStateStore } from './flows/flowState'
 	import CapturePayload from './flows/content/CapturePayload.svelte'
 	import { Loader2 } from 'lucide-svelte'
 
@@ -21,11 +20,11 @@
 
 	export let jobId: string | undefined = undefined
 	export let job: Job | undefined = undefined
-	let isValid: boolean = true
 	let isRunning: boolean = false
 	let jobProgressReset: () => void
 
-	const { selectedId, previewArgs } = getContext<FlowEditorContext>('FlowEditorContext')
+	const { selectedId, previewArgs, flowStateStore, flowStore } =
+		getContext<FlowEditorContext>('FlowEditorContext')
 	const dispatch = createEventDispatcher()
 
 	function sliceModules(modules: FlowModule[], upTo: number, idOrders: string[]): FlowModule[] {
@@ -105,7 +104,6 @@
 
 		{#if isRunning}
 			<Button
-				disabled={!isValid}
 				color="red"
 				on:click={async () => {
 					isRunning = false
@@ -132,7 +130,6 @@
 				color="blue"
 				size="sm"
 				btnClasses="w-full max-w-lg"
-				disabled={!isValid}
 				on:click={() => runPreview($previewArgs)}
 			>
 				Test flow <Kbd class="ml-2">Ctrl+Enter</Kbd>
@@ -156,7 +153,6 @@
 				compact
 				class="py-4 max-w-3xl"
 				schema={$flowStore.schema}
-				bind:isValid
 				bind:args={$previewArgs}
 			/>
 		</div>

frontend/src/lib/components/FlowStatusViewer.svelte

@@ -22,7 +22,7 @@
 	const dispatch = createEventDispatcher()
 
 	export let jobId: string
-
+	export let workspaceId: string | undefined = undefined
 	export let flowState: FlowState | undefined = undefined
 	export let flowJobIds:
 		| {
@@ -102,7 +102,7 @@
 				localFlowModuleStates[mod.id ?? '']?.scheduled_for == undefined
 			) {
 				JobService.getJob({
-					workspace: $workspaceStore ?? '',
+					workspace: workspaceId ?? $workspaceStore ?? '',
 					id: mod.job ?? ''
 				}).then((job) => {
 					localFlowModuleStates[mod.id ?? ''] = {
@@ -122,7 +122,7 @@
 		if (jobId != '00000000-0000-0000-0000-000000000000') {
 			try {
 				const newJob = await JobService.getJob({
-					workspace: $workspaceStore ?? '',
+					workspace: workspaceId ?? $workspaceStore ?? '',
 					id: jobId ?? ''
 				})
 				if (JSON.stringify(newJob) !== JSON.stringify(job)) {
@@ -159,7 +159,7 @@
 	})
 
 	async function loadOwner(path: string) {
-		is_owner = await isOwner(path, $userStore!, $workspaceStore!)
+		is_owner = await isOwner(path, $userStore!, workspaceId ?? $workspaceStore!)
 	}
 
 	let selected: 'graph' | 'sequence' = 'graph'
@@ -207,7 +207,7 @@
 											variant="border"
 											on:click={async () =>
 												await JobService.resumeSuspendedFlowAsOwner({
-													workspace: $workspaceStore ?? '',
+													workspace: workspaceId ?? $workspaceStore ?? '',
 													id: job?.id ?? '',
 													requestBody: JSON.parse(payload)
 												})}
@@ -305,6 +305,7 @@
 					</Button>
 					<div class="border p-6" class:hidden={forloop_selected != loopJobId}>
 						<svelte:self
+							{workspaceId}
 							bind:suspend_status
 							bind:retry_status
 							bind:flowState
@@ -379,6 +380,7 @@
 						<li class="w-full border border-gray-600 p-6 space-y-2 bg-blue-50/50">
 							{#if [FlowStatusModule.type.IN_PROGRESS, FlowStatusModule.type.SUCCESS, FlowStatusModule.type.FAILURE].includes(mod.type)}
 								<svelte:self
+									{workspaceId}
 									bind:suspend_status
 									bind:retry_status
 									bind:flowState
@@ -452,13 +454,17 @@
 					<FlowGraph
 						success={isSuccess(job?.['success'])}
 						flowModuleStates={localFlowModuleStates}
-						on:click={(e) => {
-							if (e.detail.id) {
+						on:select={(e) => {
+							if (typeof e.detail == 'string') {
+								if (e.detail == 'Input') {
+									selectedNode = 'start'
+								} else if (e.detail == 'Result') {
+									selectedNode = 'end'
+								} else {
+									selectedNode = e.detail
+								}
+							} else {
 								selectedNode = e.detail.id
-							} else if (e.detail == 'Result') {
-								selectedNode = 'end'
-							} else if (e.detail == 'Input') {
-								selectedNode = 'start'
 							}
 						}}
 						modules={job.raw_flow?.modules ?? []}
@@ -505,6 +511,7 @@
 							<div class="px-1 border-b border-black">
 								<JobArgs args={node.args} />
 							</div>
+
 							<FlowJobResult
 								loading={job['running'] == true}
 								noBorder

frontend/src/lib/components/FlowViewer.svelte

@@ -64,7 +64,6 @@
 									type={toAny(v)?.type}
 									contentEncoding={toAny(v)?.contentEncoding}
 									format={toAny(v)?.format}
-									itemsType={toAny(v)?.itemsType}
 								/><span class="ml-4 mt-2 text-xs"
 									>{toAny(v)?.default != undefined
 										? 'default: ' + JSON.stringify(toAny(v)?.default)

frontend/src/lib/components/FolderUsageInfo.svelte

@@ -3,6 +3,8 @@
 	import { workspaceStore } from '$lib/stores'
 
 	export let name: string
+	export let tabular = false
+	export let order = ['scripts', 'flows', 'apps', 'schedules', 'variables', 'resources']
 
 	$: $workspaceStore && loadUsage()
 
@@ -13,10 +15,18 @@
 	}
 </script>
 
-<div class="flex flex-col text-xs text-gray-600">
-	{#each Object.entries(usage) as [k, v]}
-		<div>
-			{k}: {v}
-		</div>
+{#if tabular}
+	{#each order as key}
+		<td class="text-center">
+			{usage[key] ?? ''}
+		</td>
 	{/each}
-</div>
+{:else}
+	<div class="flex flex-col text-xs text-gray-600">
+		{#each Object.entries(usage) as [k, v]}
+			<div>
+				{k}: {v}
+			</div>
+		{/each}
+	</div>
+{/if}

frontend/src/lib/components/GroupEditor.svelte

@@ -234,6 +234,14 @@
 											name,
 											requestBody: { username: member_name }
 										})
+										await GranularAclService.removeGranularAcls({
+											workspace: $workspaceStore ?? '',
+											path: name,
+											kind: 'group_',
+											requestBody: {
+												owner: 'u/' + member_name
+											}
+										})
 										loadGroup()
 									}}>remove</button
 								>

frontend/src/lib/components/InputTransformForm.svelte

@@ -82,16 +82,10 @@
 	}
 
 	function connectProperty(rawValue: string) {
-		if (isStaticTemplate(inputCat)) {
-			arg.value = `\$\{${rawValue}}`
-			setPropertyType(arg.value)
-			monacoTemplate?.setCode(arg.value)
-		} else {
-			arg.expr = getDefaultExpr(undefined, previousModuleId, rawValue)
-			arg.type = 'javascript'
-			propertyType = 'javascript'
-			monaco?.setCode(arg.expr)
-		}
+		arg.expr = getDefaultExpr(undefined, previousModuleId, rawValue)
+		arg.type = 'javascript'
+		propertyType = 'javascript'
+		monaco?.setCode(arg.expr)
 	}
 
 	function onFocus() {
@@ -139,7 +133,6 @@
 				contentEncoding={schema.properties[argName].contentEncoding}
 				required={schema.required.includes(argName)}
 				type={schema.properties[argName].type}
-				itemsType={schema.properties[argName].items}
 			/>
 
 			{#if isStaticTemplate(inputCat)}
@@ -154,7 +147,7 @@
 			{/if}
 		</div>
 		{#if !noDynamicToggle}
-			<div class="flex flex-row gap-x-4 gap-y-1 flex-wrap">
+			<div class="flex flex-row gap-x-4 gap-y-1 flex-wrap z-10">
 				<ToggleButtonGroup
 					bind:selected={propertyType}
 					on:selected={(e) => {
@@ -196,10 +189,10 @@
 				>
 					{#if isStaticTemplate(inputCat)}
 						<ToggleButton light position="left" value="static" size="xs">
-							{'${} '}Template &nbsp; <Tooltip
-								>Write javascript expressions between "{openBracket}" and "{closeBracket}". You may
-								refer to contextual objects like 'flow_input', or 'result' or functions like
-								'resource' and 'variable'
+							{'${} '}&nbsp;
+							<Tooltip
+								>Write text or surround javascript with "{openBracket}" and "{closeBracket}". Use
+								`result` to connect to another node's output.
 							</Tooltip></ToggleButton
 						>
 					{:else}
@@ -212,9 +205,11 @@
 						value="javascript"
 						startIcon={{ icon: faCode }}
 						size="xs"
+						>&nbsp;<Tooltip
+							>Write javascript expressions directly, using 'flow_input' or 'result'. You can use
+							multiline javascript.
+						</Tooltip></ToggleButton
 					>
-						Dynamic (JS)
-					</ToggleButton>
 				</ToggleButtonGroup>
 				<Button
 					variant="contained"
@@ -305,6 +300,4 @@
 			Not recognized input type {argName}
 		{/if}
 	</div>
-{:else}
-	<p class="text-sm text-gray-700">Argument at {argName} is undefined</p>
 {/if}

frontend/src/lib/components/InputTransformSchemaForm.svelte

@@ -0,0 +1,114 @@
+<script lang="ts">
+	import type { Schema } from '$lib/common'
+	import { VariableService, type InputTransform } from '$lib/gen'
+	import { workspaceStore } from '$lib/stores'
+	import { allTrue } from '$lib/utils'
+	import { faPlus } from '@fortawesome/free-solid-svg-icons'
+	import { Button } from './common'
+	import InputTransformForm from './InputTransformForm.svelte'
+	import ItemPicker from './ItemPicker.svelte'
+	import VariableEditor from './VariableEditor.svelte'
+
+	export let schema: Schema
+	export let args: Record<string, InputTransform | any> = {}
+
+	export let isValid: boolean = true
+	export let extraLib: string = 'missing extraLib'
+	export let previousModuleId: string | undefined = undefined
+
+	export let filter: string[] | undefined = undefined
+	export let noDynamicToggle = false
+
+	let clazz: string = ''
+	export { clazz as class }
+
+	let inputCheck: { [id: string]: boolean } = {}
+	$: isValid = allTrue(inputCheck) ?? false
+
+	$: if (args == undefined || typeof args !== 'object') {
+		args = {}
+	}
+
+	function removeExtraKey() {
+		const nargs = {}
+		Object.keys(args ?? {}).forEach((key) => {
+			if (keys.includes(key)) {
+				nargs[key] = args[key]
+			}
+		})
+		args = nargs
+	}
+
+	let pickForField: string | undefined
+	let itemPicker: ItemPicker | undefined = undefined
+	let variableEditor: VariableEditor | undefined = undefined
+
+	let keys: string[] = []
+	$: {
+		let lkeys = Object.keys(schema?.properties ?? {})
+		if (schema?.properties && JSON.stringify(lkeys) != JSON.stringify(keys)) {
+			keys = lkeys
+			removeExtraKey()
+		}
+	}
+</script>
+
+<div class="w-full {clazz}">
+	{#if keys.length > 0}
+		{#each keys as argName, i (argName)}
+			{#if (!filter || filter.includes(argName)) && Object.keys(schema.properties ?? {}).includes(argName)}
+				<div class="z-10">
+					<InputTransformForm
+						{previousModuleId}
+						bind:arg={args[argName]}
+						bind:schema
+						bind:argName
+						bind:inputCheck={inputCheck[argName]}
+						bind:extraLib
+						{variableEditor}
+						{itemPicker}
+						bind:pickForField
+						{noDynamicToggle}
+					/>
+				</div>
+			{/if}
+		{/each}
+	{:else}
+		<div class="text-gray-500 text-sm">No inputs</div>
+	{/if}
+</div>
+
+<ItemPicker
+	bind:this={itemPicker}
+	pickCallback={(path, _) => {
+		if (pickForField) {
+			args[pickForField].value = '$var:' + path
+		}
+	}}
+	itemName="Variable"
+	extraField="path"
+	loadItems={async () =>
+		(await VariableService.listVariable({ workspace: $workspaceStore ?? '' })).map((x) => ({
+			name: x.path,
+			...x
+		}))}
+>
+	<div
+		slot="submission"
+		class="flex flex-row-reverse w-full bg-white border-t border-gray-200 rounded-bl-lg rounded-br-lg"
+	>
+		<Button
+			variant="border"
+			color="blue"
+			size="sm"
+			startIcon={{ icon: faPlus }}
+			on:click={() => {
+				variableEditor?.initNew?.()
+			}}
+		>
+			New variable
+		</Button>
+	</div>
+</ItemPicker>
+
+<VariableEditor bind:this={variableEditor} />

frontend/src/lib/components/LightweightArgInput.svelte

@@ -0,0 +1,318 @@
+<script lang="ts">
+	import { faMinus, faPlus } from '@fortawesome/free-solid-svg-icons'
+
+	import { setInputCat as computeInputCat } from '$lib/utils'
+	import { Badge, Button } from './common'
+	import { createEventDispatcher } from 'svelte'
+	import Icon from 'svelte-awesome'
+	import FieldHeader from './FieldHeader.svelte'
+	import ObjectResourceInput from './ObjectResourceInput.svelte'
+	import ResourcePicker from './ResourcePicker.svelte'
+	import type { SchemaProperty } from '$lib/common'
+	import autosize from 'svelte-autosize'
+	import Toggle from './Toggle.svelte'
+	import Range from './Range.svelte'
+	import LightweightSchemaForm from './LightweightSchemaForm.svelte'
+
+	export let label: string = ''
+	export let value: any
+
+	export let defaultValue: any = undefined
+
+	export let description: string = ''
+	export let format: string = ''
+	export let contentEncoding: 'base64' | 'binary' | undefined = undefined
+	export let type: string | undefined = undefined
+	export let required = false
+	export let pattern: undefined | string = undefined
+	export let valid = required ? false : true
+	export let maxRows = 10
+	export let enum_: string[] | undefined = undefined
+	export let itemsType:
+		| { type?: 'string' | 'number' | 'bytes' | 'object'; contentEncoding?: 'base64' }
+		| undefined = undefined
+	export let displayHeader = true
+	export let properties: { [name: string]: SchemaProperty } | undefined = undefined
+	export let extra: Record<string, any> = {}
+
+	const dispatch = createEventDispatcher()
+
+	$: maxHeight = maxRows ? `${1 + maxRows * 1.2}em` : `auto`
+
+	$: validateInput(pattern, value)
+
+	let error: string = ''
+
+	let el: HTMLTextAreaElement | undefined = undefined
+
+	let rawValue: string | undefined = undefined
+
+	$: {
+		if (rawValue) {
+			try {
+				value = JSON.parse(rawValue)
+				error = ''
+			} catch (err) {
+				error = err.toString()
+			}
+		}
+	}
+
+	$: {
+		error = ''
+		if (inputCat === 'object') {
+			evalValueToRaw()
+			validateInput(pattern, value)
+		}
+	}
+
+	export function evalValueToRaw() {
+		if (value) {
+			rawValue = JSON.stringify(value, null, 4)
+		}
+	}
+
+	function fileChanged(e: any, cb: (v: string | undefined) => void) {
+		let t = e.target
+		if (t && 'files' in t && t.files.length > 0) {
+			let reader = new FileReader()
+			reader.onload = (e: any) => {
+				cb(e.target.result.split('base64,')[1])
+			}
+			reader.readAsDataURL(t.files[0])
+		} else {
+			cb(undefined)
+		}
+	}
+
+	export function focus() {
+		el?.focus()
+		if (el) {
+			el.style.height = '5px'
+			el.style.height = el.scrollHeight + 50 + 'px'
+		}
+	}
+
+	function validateInput(pattern: string | undefined, v: any): void {
+		if (required && (v == undefined || v == null || v === '')) {
+			error = 'This field is required'
+			valid = false
+		} else {
+			if (pattern && !testRegex(pattern, v)) {
+				error = `Should match ${pattern}`
+				valid = false
+			} else {
+				error = ''
+				valid = true
+			}
+		}
+	}
+
+	function testRegex(pattern: string, value: any): boolean {
+		try {
+			const regex = new RegExp(pattern)
+			return regex.test(value)
+		} catch (err) {
+			return false
+		}
+	}
+
+	$: {
+		if (value == undefined || value == null) {
+			value = defaultValue
+			if (defaultValue === undefined || defaultValue === null) {
+				if (inputCat === 'string') {
+					value = ''
+				} else if (inputCat == 'enum') {
+					value = enum_?.[0]
+				} else if (inputCat == 'boolean') {
+					value = false
+				}
+			}
+		}
+	}
+
+	$: inputCat = computeInputCat(type, format, itemsType?.type, enum_, contentEncoding)
+</script>
+
+<div class="flex flex-col w-full min-w-[250px]">
+	<div>
+		{#if displayHeader}
+			<FieldHeader {label} {required} {type} {contentEncoding} {format} />
+		{/if}
+
+		{#if description}
+			<div class="text-sm italic pb-1">
+				{description}
+			</div>
+		{/if}
+
+		<div class="flex space-x-1">
+			{#if inputCat == 'number'}
+				{#if extra['min'] != undefined && extra['max'] != undefined}
+					<div class="flex w-full gap-1">
+						<span>{extra['min']}</span>
+						<div class="grow">
+							<Range bind:value min={extra['min']} max={extra['max']} />
+						</div>
+						<span>{extra['max']}</span>
+						<span class="mx-2"><Badge large color="blue">{value}</Badge></span>
+					</div>
+				{:else}
+					<input
+						on:focus={(e) => {
+							dispatch('focus')
+						}}
+						type="number"
+						class={valid
+							? ''
+							: 'border border-red-700 border-opacity-30 focus:border-red-700 focus:border-opacity-30 bg-red-100'}
+						placeholder={defaultValue ?? ''}
+						bind:value
+						min={extra['min']}
+						max={extra['max']}
+					/>
+				{/if}
+			{:else if inputCat == 'boolean'}
+				<Toggle
+					on:pointerdown={(e) => {
+						e?.stopPropagation()
+					}}
+					class={valid
+						? ''
+						: 'border border-red-700 border-opacity-30 focus:border-red-700 focus:border-opacity-30 bg-red-100'}
+					bind:checked={value}
+				/>
+				{#if type == 'boolean' && value == undefined}
+					<span>&nbsp; Not set</span>
+				{/if}
+			{:else if inputCat == 'list'}
+				<div>
+					<div>
+						{#each value ?? [] as v, i}
+							<div class="flex flex-row max-w-md mt-1">
+								{#if itemsType?.type == 'number'}
+									<input type="number" bind:value={v} />
+								{:else if itemsType?.type == 'string' && itemsType?.contentEncoding == 'base64'}
+									<input
+										type="file"
+										class="my-6"
+										on:change={(x) => fileChanged(x, (val) => (value[i] = val))}
+										multiple={false}
+									/>
+								{:else}
+									<input type="text" bind:value={v} />
+								{/if}
+								<Button
+									variant="border"
+									color="red"
+									size="sm"
+									btnClasses="mx-6"
+									on:click={() => {
+										value = value.filter((el) => el != v)
+										if (value.length == 0) {
+											value = undefined
+										}
+									}}
+								>
+									<Icon data={faMinus} />
+								</Button>
+							</div>
+						{/each}
+					</div>
+					<Button
+						variant="border"
+						color="blue"
+						size="sm"
+						btnClasses="mt-1"
+						on:click={() => {
+							if (value == undefined || !Array.isArray(value)) {
+								value = []
+							}
+							value = value.concat('')
+						}}
+					>
+						<Icon data={faPlus} class="mr-2" />
+						Add item
+					</Button>
+					<span class="ml-2">
+						{(value ?? []).length} item{(value ?? []).length > 1 ? 's' : ''}
+					</span>
+				</div>
+			{:else if inputCat == 'resource-object'}
+				<ObjectResourceInput {format} bind:value />
+			{:else if inputCat == 'object'}
+				{#if properties && Object.keys(properties).length > 0}
+					<div class="p-4 pl-8 border rounded w-full">
+						<LightweightSchemaForm
+							schema={{ properties, $schema: '', required: [], type: 'object' }}
+							bind:args={value}
+						/>
+					</div>
+				{:else}
+					<textarea
+						bind:this={el}
+						on:focus={(e) => {
+							dispatch('focus')
+						}}
+						use:autosize
+						style="max-height: {maxHeight}"
+						class="col-span-10 {valid
+							? ''
+							: 'border border-red-700 border-opacity-30 focus:border-red-700 focus:border-opacity-30 bg-red-100'}"
+						placeholder={defaultValue ? JSON.stringify(defaultValue, null, 4) : ''}
+						bind:value={rawValue}
+					/>
+				{/if}
+			{:else if inputCat == 'enum'}
+				<select
+					on:focus={(e) => {
+						dispatch('focus')
+					}}
+					class="px-6"
+					bind:value
+				>
+					{#each enum_ ?? [] as e}
+						<option>{e}</option>
+					{/each}
+				</select>
+			{:else if inputCat == 'date'}
+				<input class="inline-block" type="datetime-local" bind:value />
+			{:else if inputCat == 'base64'}
+				<input
+					type="file"
+					class="my-6"
+					on:change={(x) => fileChanged(x, (val) => (value = val))}
+					multiple={false}
+				/>
+			{:else if inputCat == 'resource-string'}
+				<ResourcePicker
+					bind:value
+					resourceType={format.split('-').length > 1
+						? format.substring('resource-'.length)
+						: undefined}
+				/>
+			{:else if inputCat == 'string'}
+				<div class="flex flex-col w-full">
+					<div class="flex flex-row w-full items- justify-between">
+						<textarea
+							rows="1"
+							bind:this={el}
+							on:focus={(e) => {
+								dispatch('focus')
+							}}
+							use:autosize
+							type="text"
+							class="col-span-10 {valid
+								? ''
+								: 'border border-red-700 border-opacity-30 focus:border-red-700 focus:border-opacity-30 bg-red-100'}"
+							placeholder={defaultValue ?? ''}
+							bind:value
+						/>
+					</div>
+				</div>
+			{/if}
+			<slot name="actions" />
+		</div>
+	</div>
+</div>

frontend/src/lib/components/LightweightSchemaForm.svelte

@@ -0,0 +1,35 @@
+<script lang="ts">
+	import type { Schema } from '$lib/common'
+	import LightweightArgInput from './LightweightArgInput.svelte'
+
+	export let schema: Schema
+	export let args: Record<string, any> | undefined = undefined
+
+	$: if (args === undefined) {
+		args = {}
+	}
+</script>
+
+<div class="w-full">
+	{#each Object.keys(schema.properties ?? {}) as argName, i (argName)}
+		<div>
+			{#if typeof args == 'object' && schema?.properties[argName] && args}
+				<LightweightArgInput
+					label={argName}
+					description={schema.properties[argName].description}
+					bind:value={args[argName]}
+					type={schema.properties[argName].type}
+					required={schema.required.includes(argName)}
+					pattern={schema.properties[argName].pattern}
+					defaultValue={schema.properties[argName].default}
+					enum_={schema.properties[argName].enum}
+					format={schema.properties[argName].format}
+					contentEncoding={schema.properties[argName].contentEncoding}
+					properties={schema.properties[argName].properties}
+					itemsType={schema.properties[argName].items}
+					extra={schema.properties[argName]}
+				/>
+			{/if}
+		</div>
+	{/each}
+</div>

frontend/src/lib/components/LogViewer.svelte

@@ -42,7 +42,7 @@
 				<button on:click={logViewer.openDrawer}>Expand</button>
 				<div class="py-2 pr-2 text-xs flex gap-2 items-center">
 					Auto scroll
-					<input type="checkbox" bind:checked={scroll} />
+					<input class="windmillapp" type="checkbox" bind:checked={scroll} />
 				</div>
 			</div>
 		</div>

frontend/src/lib/components/ModulePreview.svelte

@@ -8,14 +8,17 @@
 	import LogViewer from './LogViewer.svelte'
 	import DisplayResult from './DisplayResult.svelte'
 	import Button from './common/button/Button.svelte'
-	import { flowStateStore, testStepStore } from './flows/flowState'
-	import { flowStore } from './flows/flowStore'
 	import { workspaceStore } from '$lib/stores'
 	import { Loader2 } from 'lucide-svelte'
+	import { getContext } from 'svelte'
+	import type { FlowEditorContext } from './flows/types'
 
 	export let mod: FlowModule
 	export let schema: Schema
 
+	const { flowStore, flowStateStore, testStepStore } =
+		getContext<FlowEditorContext>('FlowEditorContext')
+
 	// Test
 	let testJobLoader: TestJobLoader
 	let testIsLoading = false
@@ -57,6 +60,7 @@
 		if (testJob && !testJob.canceled && testJob.type == 'CompletedJob' && `result` in testJob) {
 			if ($flowStateStore[mod.id]?.previewResult) {
 				$flowStateStore[mod.id].previewResult = testJob.result
+				$flowStateStore = $flowStateStore
 			}
 		}
 	}
@@ -86,6 +90,7 @@
 			detailed={false}
 			topButton
 			bind:args={stepArgs}
+			isFlow={false}
 		/>
 		{#if testIsLoading}
 			<Button on:click={testJobLoader?.cancelJob} btnClasses="w-full mt-4" color="red" size="sm">

frontend/src/lib/components/Multiselect.svelte

@@ -167,7 +167,7 @@
 	{#if showOptions}
 		<ul
 			class="options"
-			transition:fly={{ duration: 200, y: 5 }}
+			transition:fly|local={{ duration: 200, y: 5 }}
 			on:mousedown|preventDefault={handleOptionMousedown}
 		>
 			{#each filtered as option}

frontend/src/lib/components/PageHeader.svelte

@@ -11,7 +11,7 @@
 		<span class="flex items-center space-x-2 mb-2">
 			<h1>{title}</h1>
 			{#if tooltip}
-				<Tooltip>{tooltip}</Tooltip>
+				<Tooltip scale={0.9}>{tooltip}</Tooltip>
 			{/if}
 		</span>
 	{:else}

frontend/src/lib/components/Path.svelte

@@ -23,6 +23,7 @@
 	import FolderEditor from './FolderEditor.svelte'
 	import GroupEditor from './GroupEditor.svelte'
 	import { random_adj } from './random_positive_adjetive'
+	import Badge from './common/badge/Badge.svelte'
 
 	type PathKind = 'resource' | 'script' | 'variable' | 'flow' | 'schedule' | 'app'
 	let meta: Meta | undefined = undefined
@@ -199,6 +200,10 @@
 	}
 
 	function initPath() {
+		if (path != undefined && path != '') {
+			meta = pathToMeta(path)
+			return
+		}
 		if (initialPath == undefined || initialPath == '') {
 			reset()
 		} else {
@@ -277,15 +282,19 @@
 			folderCreated = undefined
 		}}
 	>
-		<div class="flex flex-row">
-			<input class="mr-2" placeholder="New folder name" bind:value={newFolderName} />
-			<Button size="md" startIcon={{ icon: faPlus }} disabled={!newFolderName} on:click={addFolder}>
-				New&nbsp;folder
-			</Button>
-		</div>
-
-		{#if folderCreated}
-			<div class="mt-8" />
+		{#if !folderCreated}
+			<div class="flex flex-row">
+				<input class="mr-2" placeholder="New folder name" bind:value={newFolderName} />
+				<Button
+					size="md"
+					startIcon={{ icon: faPlus }}
+					disabled={!newFolderName}
+					on:click={addFolder}
+				>
+					New&nbsp;folder
+				</Button>
+			</div>
+		{:else}
 			<FolderEditor name={folderCreated} />
 		{/if}
 	</DrawerContent>
@@ -302,7 +311,7 @@
 		{#if meta != undefined}
 			<div class="flex gap-4 shrink">
 				<label class="block">
-					<span class="text-gray-700 text-sm whitespace-nowrap">&nbsp;</span>
+					<span class="text-gray-700 text-sm whitespace-nowrap">Owner</span>
 
 					<ToggleButtonGroup
 						class="mt-0.5"
@@ -405,8 +414,26 @@
 		{/if}
 	</div>
 
-	<div class="flex-row flex justify-between">
-		<div><span class="font-mono text-sm">{path}</span></div>
+	<div class="flex flex-col w-full mt-4">
+		<div class="flex justify-start w-full">
+			<Badge
+				color="gray"
+				class="center-center !bg-gray-300 !text-gray-600 !w-[70px] !h-[24px] rounded-r-none"
+			>
+				Full path
+			</Badge>
+			<input
+				type="text"
+				readonly
+				value={path}
+				size={path?.length || 50}
+				class="font-mono !text-xs max-w-[calc(100%-70px)] !w-auto !h-[24px] !py-0 !border-l-0 !rounded-l-none"
+				on:focus={({ currentTarget }) => {
+					currentTarget.select()
+				}}
+			/>
+			<!-- <span class="font-mono text-sm break-all">{path}</span> -->
+		</div>
 		<div class="text-red-600 text-2xs">{error}</div>
 	</div>
 </div>