chore(main): release 1.76.0 (#1280)

* chore(main): release 1.76.0

* Apply automatic changes

---------

Co-authored-by: rubenfiszel <rubenfiszel@users.noreply.github.com>

.github/workflows/backend-test.yml

@@ -40,4 +40,4 @@ jobs:
             backend -> target
       - name: cargo test
         timeout-minutes: 10
-        run: mkdir frontend/build && cd backend && touch windmill-api/openapi-deref.yaml && DATABASE_URL=postgres://postgres:changeme@postgres:5432/windmill cargo test --all -- --nocapture
+        run: mkdir frontend/build && cd backend && touch windmill-api/openapi-deref.yaml && DATABASE_URL=postgres://postgres:changeme@postgres:5432/windmill DISABLE_NSJAIL=false cargo test --all -- --nocapture

.github/workflows/change-versions.yml

@@ -10,6 +10,7 @@ jobs:
     container: node:18
     steps:
       - uses: actions/checkout@v3
+      - run: git config --system --add safe.directory /__w/windmill/windmill
       - name: Change versions
         run: ./.github/change-versions.sh "$(cat version.txt)"
       - uses: actions-rs/toolchain@v1

.github/workflows/deploy_to_windmill.yml

@@ -1,20 +0,0 @@
-name: Deploy to windmill.dev
-
-on:
-  push:
-    branches: [main]
-    paths:
-      - "community/**"
-
-jobs:
-  deploy:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: Deploy to windmill.dev
-        uses: windmill-labs/windmill-gh-action-deploy@v2.0.0
-        with:
-          dry_run: false
-          input_dir: community
-          windmill_workspace: starter
-          windmill_token: ${{ secrets.WINDMILL_API_TOKEN }}

.github/workflows/docker-image.yml

@@ -109,38 +109,38 @@ jobs:
             ${{ steps.meta-ee-public.outputs.labels }}
             org.opencontainers.image.licenses=Windmill-Enterprise-License
 
-
-  playwright:
-    runs-on: [self-hosted, new]
-    needs: [build]
-    services:
-      postgres:
-        image: postgres
-        env:
-          POSTGRES_DB: windmill
-          POSTGRES_USER: admin
-          POSTGRES_PASSWORD: changeme
-        ports:
-          - 5432:5432
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-    steps:
-      - uses: actions/checkout@v3
-      - name: "Docker"
-        run: echo "::set-output name=id::$(docker run --network=host --rm -d -p 8000:8000 --privileged -it -e DATABASE_URL=postgres://admin:changeme@localhost:5432/windmill -e BASE_INTERNAL_URL=http://localhost:8000 ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest)"
-        id: docker-container
-      - uses: actions/setup-node@v3
-        with:
-          node-version: 16
-      - name: "Playwright run"
-        timeout-minutes: 2
-        run: cd frontend && npm ci @playwright/test && npx playwright install && export BASE_URL=http://localhost:8000 && npm run test
-      - name: "Clean up"
-        run: docker kill ${{ steps.docker-container.outputs.id }}
-        if: always()
+  # disabled until we make it 100% reliable and add more meaningful tests
+  # playwright:
+  #   runs-on: [self-hosted, new]
+  #   needs: [build]
+  #   services:
+  #     postgres:
+  #       image: postgres
+  #       env:
+  #         POSTGRES_DB: windmill
+  #         POSTGRES_USER: admin
+  #         POSTGRES_PASSWORD: changeme
+  #       ports:
+  #         - 5432:5432
+  #       options: >-
+  #         --health-cmd pg_isready
+  #         --health-interval 10s
+  #         --health-timeout 5s
+  #         --health-retries 5
+  #   steps:
+  #     - uses: actions/checkout@v3
+  #     - name: "Docker"
+  #       run: echo "::set-output name=id::$(docker run --network=host --rm -d -p 8000:8000 --privileged -it -e DATABASE_URL=postgres://admin:changeme@localhost:5432/windmill -e BASE_INTERNAL_URL=http://localhost:8000 ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest)"
+  #       id: docker-container
+  #     - uses: actions/setup-node@v3
+  #       with:
+  #         node-version: 16
+  #     - name: "Playwright run"
+  #       timeout-minutes: 2
+  #       run: cd frontend && npm ci @playwright/test && npx playwright install && export BASE_URL=http://localhost:8000 && npm run test
+  #     - name: "Clean up"
+  #       run: docker kill ${{ steps.docker-container.outputs.id }}
+  #       if: always()
 
 
   publish_privately_heavy:
@@ -182,7 +182,7 @@ jobs:
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Build and push privately
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v4
         if: github.event_name != 'pull_request'
         with:
           context: .
@@ -222,7 +222,7 @@ jobs:
           password: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           
       - name: Build and push privately
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v4
         if: github.event_name != 'pull_request'
         with:
           context: .

.github/workflows/pypi_on_release.yml

@@ -65,7 +65,7 @@ jobs:
           password: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
 
       - name: Build and push publicly
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v4
         with:
           context: "{{defaultContext}}:lsp"
           push: true

.github/workflows/uffizzi-build.yml.archived

@@ -25,12 +25,12 @@ jobs:
         run: echo "UUID_TAG_APP=$(uuidgen)" >> $GITHUB_ENV
       - name: Docker metadata
         id: meta
-        uses: docker/metadata-action@v3
+        uses: docker/metadata-action@v4
         with:
           images: registry.uffizzi.com/${{ env.UUID_TAG_APP }}
           tags: type=raw,value=60d
       - name: Build and Push Image to registry.uffizzi.com ephemeral registry
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v4
         with:
           push: true
           context: ./

CHANGELOG.md

@@ -1,6 +1,339 @@
 # Changelog
 
 
+## [1.76.0](https://github.com/windmill-labs/windmill/compare/v1.75.0...v1.76.0) (2023-03-13)
+
+
+### Features
+
+* **frontend:** add frontend (JS)  scripts to apps ([f0b1b1f](https://github.com/windmill-labs/windmill/commit/f0b1b1f752731ba434b960a75624118152f53c00))
+* **frontend:** Copy, Cut and Paste ([#1279](https://github.com/windmill-labs/windmill/issues/1279)) ([82c139e](https://github.com/windmill-labs/windmill/commit/82c139ed0992be401e250cfb7ecc0fca61b76772))
+* **frontend:** disabled for action buttons can now depend on row ([75f87e7](https://github.com/windmill-labs/windmill/commit/75f87e7e1117a9c12afcf626379e94b134a9a493))
+* **frontend:** improve drag-n-drop behavior ([cfd489a](https://github.com/windmill-labs/windmill/commit/cfd489a55059e7b6843f99bab261c70b3852e6a2))
+
+
+### Bug Fixes
+
+* **backend:** improve worker ping api ([c958480](https://github.com/windmill-labs/windmill/commit/c958480ce83844a989f58dd5a70eb288582e2194))
+* **frontend:** General fixes and updates ([#1281](https://github.com/windmill-labs/windmill/issues/1281)) ([3e5a179](https://github.com/windmill-labs/windmill/commit/3e5a179eb8cd8001f49c92305141dade1571e20f))
+
+## [1.75.0](https://github.com/windmill-labs/windmill/compare/v1.74.2...v1.75.0) (2023-03-11)
+
+
+### Features
+
+* add filter jobs by args or result ([3b44f9a](https://github.com/windmill-labs/windmill/commit/3b44f9a72ca0466a44963a4b9657a0ee59b44753))
+* **apps:** add resource picker ([8681e83](https://github.com/windmill-labs/windmill/commit/8681e83b574141acbf7e5a389a9e8a4f340336d1))
+* **bash:** add default argument handling for bash ([1d5c194](https://github.com/windmill-labs/windmill/commit/1d5c194f09ffba963d52e418c5954843d84ae337))
+* **frontend-apps:** add variable picker for static string input on apps ([bc440f8](https://github.com/windmill-labs/windmill/commit/bc440f8d4154ce464c0e027d93b7a0a3b76d782e))
+* **frontend:** make runs filters synced with query args ([61a5e1f](https://github.com/windmill-labs/windmill/commit/61a5e1f1accc988628b785b3b9be04c4ea719874))
+
+
+### Bug Fixes
+
+* **backend:** add killpill for lines reading ([7c825c2](https://github.com/windmill-labs/windmill/commit/7c825c212dd0f1e8be427eabd9a9756303241d1b))
+* **cli:** many small fixes ([ce32370](https://github.com/windmill-labs/windmill/commit/ce323709a94d27fb24214719180ea1aafc66d646))
+
+## [1.74.2](https://github.com/windmill-labs/windmill/compare/v1.74.1...v1.74.2) (2023-03-09)
+
+
+### Bug Fixes
+
+* **frontend:** fix splitpanes navigation ([#1276](https://github.com/windmill-labs/windmill/issues/1276)) ([8d5c5b8](https://github.com/windmill-labs/windmill/commit/8d5c5b88a35d7a3bad1d8ddf2d940026825241eb))
+
+## [1.74.1](https://github.com/windmill-labs/windmill/compare/v1.74.0...v1.74.1) (2023-03-09)
+
+
+### Bug Fixes
+
+* **apps:** proper reactivity for non rendered static components ([ae53baf](https://github.com/windmill-labs/windmill/commit/ae53bafaf6777f928113f84b2c6ed6a2ed341844))
+* **ci:** make windmill compile again by pinning swc deps ([2ea15d5](https://github.com/windmill-labs/windmill/commit/2ea15d5035e5e15473968db3c0501a4dddff5cd0))
+
+## [1.74.0](https://github.com/windmill-labs/windmill/compare/v1.73.1...v1.74.0) (2023-03-09)
+
+
+### Features
+
+* add delete by path for scripts ([0c2cf92](https://github.com/windmill-labs/windmill/commit/0c2cf92dd3df9610e649f15e23921a4ca0d94e6a))
+* **frontend:** Add color picker input to app ([#1270](https://github.com/windmill-labs/windmill/issues/1270)) ([88e537a](https://github.com/windmill-labs/windmill/commit/88e537ad1fb4c207f38fbe951c82106bef6491a3))
+* **frontend:** add expand ([#1268](https://github.com/windmill-labs/windmill/issues/1268)) ([b854ee3](https://github.com/windmill-labs/windmill/commit/b854ee34393534bde104e2e6f606108fd66d38dc))
+* **frontend:** add hash to ctx in apps ([b1a45b1](https://github.com/windmill-labs/windmill/commit/b1a45b1e708aa6f19f8be9c949507083e044f2d8))
+* **frontend:** Add key navigation in app editor ([#1273](https://github.com/windmill-labs/windmill/issues/1273)) ([6b0fb75](https://github.com/windmill-labs/windmill/commit/6b0fb75d23e2151c88b07814139d203c1bd0578d))
+
+
+### Bug Fixes
+
+* **cli:** improve visibility of the active workspace ([e6344da](https://github.com/windmill-labs/windmill/commit/e6344dac6d1be04b46231fa8ef8579fd12ca8f37))
+* **frontend:** add confirmation modal to delete script/flow/app ([a4adcb5](https://github.com/windmill-labs/windmill/commit/a4adcb5192c11f7bf47a0d259825e474779378d7))
+* **frontend:** Clean up app editor ([#1267](https://github.com/windmill-labs/windmill/issues/1267)) ([0a5e181](https://github.com/windmill-labs/windmill/commit/0a5e181a3aa966fb8211bee0d9174fc16353b31f))
+* **frontend:** Minor changes ([#1272](https://github.com/windmill-labs/windmill/issues/1272)) ([3b6ae0c](https://github.com/windmill-labs/windmill/commit/3b6ae0cc49461b858d9cfff79eae9a7569465235))
+* **frontend:** simplify input bindings ([b2de531](https://github.com/windmill-labs/windmill/commit/b2de531a46e4b120d7106d361b727746bec516dd))
+
+## [1.73.1](https://github.com/windmill-labs/windmill/compare/v1.73.0...v1.73.1) (2023-03-07)
+
+
+### Bug Fixes
+
+* **frontend:** load flow is not initialized ([719d475](https://github.com/windmill-labs/windmill/commit/719d4752621d462b1cfaa0d27930fba7586be779))
+
+## [1.73.0](https://github.com/windmill-labs/windmill/compare/v1.72.0...v1.73.0) (2023-03-07)
+
+
+### Features
+
+* **frontend:** add a way to automatically resize ([#1259](https://github.com/windmill-labs/windmill/issues/1259)) ([24f58ef](https://github.com/windmill-labs/windmill/commit/24f58efd9994a2201c1b1d9bbfb11734c57068e3))
+* **frontend:** add ability to move nodes ([614fb50](https://github.com/windmill-labs/windmill/commit/614fb5022aa7d5428fb96b7ee3a20794edd1e9d3))
+* **frontend:** Add app PDF viewer ([#1254](https://github.com/windmill-labs/windmill/issues/1254)) ([3e5d09e](https://github.com/windmill-labs/windmill/commit/3e5d09ef0b5619186bee5ec6d442cbfd12a6e8d5))
+* **frontend:** add fork/save buttons + consistent styling for slider/range ([9e9f8ef](https://github.com/windmill-labs/windmill/commit/9e9f8efb8ee389ea75e99b67ef720756959ca737))
+* **frontend:** add history to flows and apps ([9e4d90a](https://github.com/windmill-labs/windmill/commit/9e4d90ad37a57ff1f515eea0c82cf603649e915d))
+* **frontend:** Fix object viewer style ([#1255](https://github.com/windmill-labs/windmill/issues/1255)) ([94f1aad](https://github.com/windmill-labs/windmill/commit/94f1aadef2b09ac1962478f11b27cc708b8328f1))
+* **frontend:** refactor entire flow builder UX ([2ac51b0](https://github.com/windmill-labs/windmill/commit/2ac51b0af08bdef7ce3c7e874e9983b9fc00478a))
+
+
+### Bug Fixes
+
+* **frontend:** arginput + apppreview fixes ([e2c4545](https://github.com/windmill-labs/windmill/commit/e2c45452401022b00285b21551ffaf35a114be33))
+* **frontend:** fix app map reactivity ([#1260](https://github.com/windmill-labs/windmill/issues/1260)) ([2557e13](https://github.com/windmill-labs/windmill/commit/2557e136bd0df1a023819b7d9b2235e30d7140b6))
+* **frontend:** fix branch deletion ([#1261](https://github.com/windmill-labs/windmill/issues/1261)) ([a999eb2](https://github.com/windmill-labs/windmill/commit/a999eb21121a7c0010621448324e0c77caf2b3f6))
+* **frontend:** Side menu z-index issue ([#1265](https://github.com/windmill-labs/windmill/issues/1265)) ([c638897](https://github.com/windmill-labs/windmill/commit/c638897fdcd58f55b0929f91641b21a6f9d25ead))
+
+## [1.72.0](https://github.com/windmill-labs/windmill/compare/v1.71.0...v1.72.0) (2023-03-02)
+
+
+### Features
+
+* **backend:** get_result_by_id do a downward pass to find node at any depth ([#1249](https://github.com/windmill-labs/windmill/issues/1249)) ([4c913dc](https://github.com/windmill-labs/windmill/commit/4c913dc4b6be03571a015c97a13829adffb61479))
+* **frontend:** Add app map component ([#1251](https://github.com/windmill-labs/windmill/issues/1251)) ([ed25d9f](https://github.com/windmill-labs/windmill/commit/ed25d9f186d9925f75404cb193a025d8a41c4540))
+* **frontend:** app splitpanes ([#1248](https://github.com/windmill-labs/windmill/issues/1248)) ([f4d79ee](https://github.com/windmill-labs/windmill/commit/f4d79ee2633e6cdab0fa2410108b31cfa77e10da))
+
+
+### Bug Fixes
+
+* **backend:** improve result retrieval ([c4463bb](https://github.com/windmill-labs/windmill/commit/c4463bb029907f3c8d77abb194f872aae7876bf6))
+* **backend:** incorrect get_result_by_id for  list_result job ([2a75cd2](https://github.com/windmill-labs/windmill/commit/2a75cd250ea5e01849fc8bbb69bf44f147d0acb8))
+* **cli:** fix workspace option + run script/flow + whoami ([35ea2b2](https://github.com/windmill-labs/windmill/commit/35ea2b27b12159c68c8507ec1f8686028c975387))
+* **frontend:** background script not showing inputs ([55eb48c](https://github.com/windmill-labs/windmill/commit/55eb48c55332431304cedbf3bcbbbcff61ec3645))
+* **frontend:** fix table bindings ([2679386](https://github.com/windmill-labs/windmill/commit/2679386bf87a56352269911bd89e52df5ee9f314))
+* **frontend:** rework app reactivity ([94b20d2](https://github.com/windmill-labs/windmill/commit/94b20d2f5e3b551974c57ea82b6e3dc16e97b9b8))
+* **frontend:** rework app reactivity ([1753cb7](https://github.com/windmill-labs/windmill/commit/1753cb7da658f47be974c15da82c71a8e19309a6))
+
+## [1.71.0](https://github.com/windmill-labs/windmill/compare/v1.70.1...v1.71.0) (2023-02-28)
+
+
+### Features
+
+* **backend:** use counter for sleep/execution/pull durations ([e568690](https://github.com/windmill-labs/windmill/commit/e56869092a03fec4703ddd9ef65c89edb8122962))
+* **cli:** add autocompletions ([287b2db](https://github.com/windmill-labs/windmill/commit/287b2db22f7b56e90bcd0c4727c00096695c2e0d))
+* **frontend:** App drawer ([#1246](https://github.com/windmill-labs/windmill/issues/1246)) ([8a0d115](https://github.com/windmill-labs/windmill/commit/8a0d1158c4d7e970cb91e1adf4838e5efdbb39ff))
+* **frontend:** drawer for editing workspace scripts in flows ([6adc875](https://github.com/windmill-labs/windmill/commit/6adc87561070d8aceaba1838008cd7e6be2e2660))
+
+
+### Bug Fixes
+
+* **frontend:** Add more app custom css ([#1229](https://github.com/windmill-labs/windmill/issues/1229)) ([a4e4d18](https://github.com/windmill-labs/windmill/commit/a4e4d188ad10443dd0b7f104389594efc768dc59))
+* **frontend:** Add more app custom css ([#1247](https://github.com/windmill-labs/windmill/issues/1247)) ([1bb5ed9](https://github.com/windmill-labs/windmill/commit/1bb5ed9ae01fd7998b06833b6222e5dd5d774d35))
+* **frontend:** display currently selected filter even if not in list ([42d1cd6](https://github.com/windmill-labs/windmill/commit/42d1cd6456620ba917c560c87d736dc93634adff))
+* **frontend:** Fix deeply nested move ([#1245](https://github.com/windmill-labs/windmill/issues/1245)) ([a67f10e](https://github.com/windmill-labs/windmill/commit/a67f10eeb6fdb44bbb3a510badcc5ad0ae187a2b))
+* **frontend:** invisible subgrids have h-0 + app policies fix ([2244e83](https://github.com/windmill-labs/windmill/commit/2244e83b9da803a4cf46ab0825d7cb6cb0e24872))
+
+## [1.70.1](https://github.com/windmill-labs/windmill/compare/v1.70.0...v1.70.1) (2023-02-27)
+
+
+### Bug Fixes
+
+* **cli:** make cli resilient to systems without openable browsers ([c051ffe](https://github.com/windmill-labs/windmill/commit/c051ffeb42c1cff609f93da7745036ea722e17d4))
+* **frontend:** Disable move in nested subgrid ([#1238](https://github.com/windmill-labs/windmill/issues/1238)) ([70eab30](https://github.com/windmill-labs/windmill/commit/70eab303bd45111ae198d9b710bfd6f9f59e53b0))
+* **frontend:** Fix inline scripts list ([#1240](https://github.com/windmill-labs/windmill/issues/1240)) ([97602ac](https://github.com/windmill-labs/windmill/commit/97602ac6db1404d36d160a431ffcea6c0f567a48))
+* **frontend:** Fix subgrid lock ([#1232](https://github.com/windmill-labs/windmill/issues/1232)) ([8ee9d67](https://github.com/windmill-labs/windmill/commit/8ee9d67f4faa91446338b41c664ef91913eb8b81))
+
+## [1.70.1](https://github.com/windmill-labs/windmill/compare/v1.70.0...v1.70.1) (2023-02-27)
+
+
+### Bug Fixes
+
+* **cli:** make cli resilient to systems without openable browsers ([c051ffe](https://github.com/windmill-labs/windmill/commit/c051ffeb42c1cff609f93da7745036ea722e17d4))
+* **frontend:** Disable move in nested subgrid ([#1238](https://github.com/windmill-labs/windmill/issues/1238)) ([70eab30](https://github.com/windmill-labs/windmill/commit/70eab303bd45111ae198d9b710bfd6f9f59e53b0))
+* **frontend:** Fix subgrid lock ([#1232](https://github.com/windmill-labs/windmill/issues/1232)) ([8ee9d67](https://github.com/windmill-labs/windmill/commit/8ee9d67f4faa91446338b41c664ef91913eb8b81))
+
+## [1.70.0](https://github.com/windmill-labs/windmill/compare/v1.69.3...v1.70.0) (2023-02-27)
+
+
+### Features
+
+* **apps:** add ag grid ([b690d80](https://github.com/windmill-labs/windmill/commit/b690d801d4aa5695ee558e81d1ed114074dfcb83))
+* **frontend:** move to other grid ([#1230](https://github.com/windmill-labs/windmill/issues/1230)) ([104e4ac](https://github.com/windmill-labs/windmill/commit/104e4ac5e790c30e6fb6b27726776693038d4f19))
+
+
+### Bug Fixes
+
+* app setup and sync now uses 1.69.3 ([d38aff2](https://github.com/windmill-labs/windmill/commit/d38aff2fe228f23eb18c3991392928c064e6aca2))
+* **frontend:** Fix duplication ([#1237](https://github.com/windmill-labs/windmill/issues/1237)) ([e87f4fc](https://github.com/windmill-labs/windmill/commit/e87f4fc44b847a573f5acafc0348fbcbfcb2258f))
+* **frontend:** fix graph viewer id assignment ([e1f686d](https://github.com/windmill-labs/windmill/commit/e1f686d8508cfc1f73c43be08facc44217ca8de0))
+
+## [1.69.3](https://github.com/windmill-labs/windmill/compare/v1.69.2...v1.69.3) (2023-02-24)
+
+
+### Bug Fixes
+
+* **deno:** fix denoify buffer handling ([c2e5afd](https://github.com/windmill-labs/windmill/commit/c2e5afd4e07fb63375832f308da8c744616ee188))
+
+## [1.69.2](https://github.com/windmill-labs/windmill/compare/v1.69.1...v1.69.2) (2023-02-24)
+
+
+### Bug Fixes
+
+* **app:** fix all nested behavior ([dd28308](https://github.com/windmill-labs/windmill/commit/dd28308c3cf1877ba3f19dcd2bd20bf1c7896a99))
+* **frontend:** delete grid item ([008c30f](https://github.com/windmill-labs/windmill/commit/008c30fcaad64af512407f9889a9881fafac0868))
+* **frontend:** duplicate ([483407c](https://github.com/windmill-labs/windmill/commit/483407cdf0e1ed61de180a904934e950fed4adc3))
+* **frontend:** Fix findGridItem ([a8295d0](https://github.com/windmill-labs/windmill/commit/a8295d0b5acd08cec42b7939d907df5c25132644))
+* **frontend:** Fix findGridItem ([5bb77ed](https://github.com/windmill-labs/windmill/commit/5bb77edf45740a75e969b1bef31580271c9d5505))
+* **frontend:** Fix next id ([8ddcf4d](https://github.com/windmill-labs/windmill/commit/8ddcf4d9c1a8d6dd20ee241a3f308811c49e58f1))
+* **frontend:** gridtab ([fa105b4](https://github.com/windmill-labs/windmill/commit/fa105b4caeaa2d0e9704a48f6caf8d846839c23e))
+* **frontend:** rewrote utils ([ea1b2c2](https://github.com/windmill-labs/windmill/commit/ea1b2c29b95282df347ef9c5973917fa3880e843))
+* **frontend:** wip ([33ebe2d](https://github.com/windmill-labs/windmill/commit/33ebe2da8e81476be62a2567d5012573a8a010b6))
+
+## [1.69.1](https://github.com/windmill-labs/windmill/compare/v1.69.0...v1.69.1) (2023-02-24)
+
+
+### Bug Fixes
+
+* **deno:** remove mysql support waiting for deno fix ([dd7e8c7](https://github.com/windmill-labs/windmill/commit/dd7e8c742c83f6a1d13e4343ca626c0b5efc06fb))
+* **deno:** remove mysql support waiting for deno fix ([2f78132](https://github.com/windmill-labs/windmill/commit/2f78132e081bdf3d7468e022f0e981ebfa52cfb3))
+* **frontend:** containers and tab fixes v1 ([27cac3f](https://github.com/windmill-labs/windmill/commit/27cac3ffe69c4dac160e9e55ffd1eb8ea348d487))
+* **frontend:** containers and tab fixes v1 ([705703a](https://github.com/windmill-labs/windmill/commit/705703a5e2f2dc7ceb4c215221f72bf624799841))
+* **frontend:** containers and tab fixes v1 ([fac31c6](https://github.com/windmill-labs/windmill/commit/fac31c6628b289ad6aae92434e312c4be281a4d2))
+
+## [1.69.0](https://github.com/windmill-labs/windmill/compare/v1.68.0...v1.69.0) (2023-02-23)
+
+
+### Features
+
+* **frontend:** Duplicate component ([#1228](https://github.com/windmill-labs/windmill/issues/1228)) ([089a6b6](https://github.com/windmill-labs/windmill/commit/089a6b6ae52e8d28dd15e2f9a6ad900c5853d0a1))
+* **frontend:** Properly delete tab content ([#1227](https://github.com/windmill-labs/windmill/issues/1227)) ([857ee5f](https://github.com/windmill-labs/windmill/commit/857ee5f318466d12bf0d41515451798df087ab74))
+* **frontend:** Support deeply nested components ([#1225](https://github.com/windmill-labs/windmill/issues/1225)) ([6ad876e](https://github.com/windmill-labs/windmill/commit/6ad876ebb45a934b7a4dc980cf38a5228d7d11f1))
+
+
+### Bug Fixes
+
+* **cli:** .wmillignore whitelist behavior ([d543650](https://github.com/windmill-labs/windmill/commit/d543650b313c434e794ad800aefe4aeda83c0fed))
+
+## [1.68.0](https://github.com/windmill-labs/windmill/compare/v1.67.4...v1.68.0) (2023-02-23)
+
+
+### Features
+
+* **frontend:** Add more app component CSS customisation ([#1218](https://github.com/windmill-labs/windmill/issues/1218)) ([6044e3b](https://github.com/windmill-labs/windmill/commit/6044e3b6ef92e89b8f15f38bc2d0986ec64105d5))
+
+
+### Bug Fixes
+
+* **cli:** better ergonomics around workspace add ([40c12e6](https://github.com/windmill-labs/windmill/commit/40c12e6139c7b42d7ab169bab2dd37f8b43bea06))
+* **cli:** better ergonomics around workspaces ([3b7160e](https://github.com/windmill-labs/windmill/commit/3b7160e84aa454bdb5f343da99cfd97a6b319937))
+
+## [1.67.4](https://github.com/windmill-labs/windmill/compare/v1.67.3...v1.67.4) (2023-02-23)
+
+
+### Bug Fixes
+
+* **backend:** workflow check for has_failure_module ([e54dc3f](https://github.com/windmill-labs/windmill/commit/e54dc3ff97e4454a15b9efe25cc12f6c9e1e176b))
+
+## [1.67.3](https://github.com/windmill-labs/windmill/compare/v1.67.2...v1.67.3) (2023-02-23)
+
+
+### Bug Fixes
+
+* **cli:** ignone non wmill looking files ([ec57c59](https://github.com/windmill-labs/windmill/commit/ec57c5977f122b629a07e05bc3551662d518ce30))
+
+## [1.67.2](https://github.com/windmill-labs/windmill/compare/v1.67.1...v1.67.2) (2023-02-23)
+
+
+### Bug Fixes
+
+* **cli:** ignone non wmill looking files ([969e89f](https://github.com/windmill-labs/windmill/commit/969e89f8bbc10f6712920321b70ede35f19ab9ed))
+
+## [1.67.1](https://github.com/windmill-labs/windmill/compare/v1.67.0...v1.67.1) (2023-02-22)
+
+
+### Bug Fixes
+
+* **cli:** coloring nits ([3fa24ad](https://github.com/windmill-labs/windmill/commit/3fa24adad0a07ba2f469c545b28251b035efdf90))
+
+## [1.67.0](https://github.com/windmill-labs/windmill/compare/v1.66.1...v1.67.0) (2023-02-22)
+
+
+### Features
+
+* **frontend:** Add app sub grids ([#1208](https://github.com/windmill-labs/windmill/issues/1208)) ([dbc59e9](https://github.com/windmill-labs/windmill/commit/dbc59e952143ee5813780ad13794cef4e036911c))
+
+
+### Bug Fixes
+
+* **cli:** add --fail-conflicts to ci push ([0085b46](https://github.com/windmill-labs/windmill/commit/0085b46c1e3b8267fcafcb06ce72b4d820e49df5))
+
+## [1.66.1](https://github.com/windmill-labs/windmill/compare/v1.66.0...v1.66.1) (2023-02-22)
+
+
+### Bug Fixes
+
+* **cli:** delete workspace instead of archiving them ([70dfc8b](https://github.com/windmill-labs/windmill/commit/70dfc8b8d0293d80da7db14caa1b9eb0ed67653d))
+
+## [1.66.0](https://github.com/windmill-labs/windmill/compare/v1.65.0...v1.66.0) (2023-02-22)
+
+
+### Features
+
+* add delete flows ([e81f7bd](https://github.com/windmill-labs/windmill/commit/e81f7bd7239b73710da2a4ddec0da7805c13da06))
+* CLI refactor v1 ([e31d2ae](https://github.com/windmill-labs/windmill/commit/e31d2ae27f886e774ffc429eea80057f4f9f4213))
+* **frontend:** Add image app component ([#1213](https://github.com/windmill-labs/windmill/issues/1213)) ([a4b773a](https://github.com/windmill-labs/windmill/commit/a4b773af294554c5787f02ebda363c8d9a3eff1b))
+
+## [1.65.0](https://github.com/windmill-labs/windmill/compare/v1.64.0...v1.65.0) (2023-02-21)
+
+
+### Features
+
+* **apps:** add asJson for customcss ([71d6dad](https://github.com/windmill-labs/windmill/commit/71d6dad37cc239952ce7799609c02474b0b1fc81))
+* **apps:** add custom css for apps ([7f00e1c](https://github.com/windmill-labs/windmill/commit/7f00e1c1a8f2e905b0677d82ba547f55dc23b3e0))
+* **backend:** Zip Workspace Export ([#1201](https://github.com/windmill-labs/windmill/issues/1201)) ([5d109b3](https://github.com/windmill-labs/windmill/commit/5d109b3cd4b7749788f9cb9fcbe1949c45eedf1f))
+* **frontend:** Add divider app component ([#1209](https://github.com/windmill-labs/windmill/issues/1209)) ([c33e79e](https://github.com/windmill-labs/windmill/commit/c33e79e0b8d5ba1103d87fdd47fcd0e1071e19de))
+* **frontend:** Add file input app component ([#1211](https://github.com/windmill-labs/windmill/issues/1211)) ([d4b6d69](https://github.com/windmill-labs/windmill/commit/d4b6d691264bf21e4e2c97548aaad9aa80678a6b))
+* **frontend:** Add icon app component ([#1207](https://github.com/windmill-labs/windmill/issues/1207)) ([e4791c2](https://github.com/windmill-labs/windmill/commit/e4791c2b7e3a0e6b90c37bc1200f9cd0ab3b6845))
+
+## [1.64.0](https://github.com/windmill-labs/windmill/compare/v1.63.2...v1.64.0) (2023-02-16)
+
+
+### Features
+
+* **frontend:** Trigger settings drawer with URL hash ([#1185](https://github.com/windmill-labs/windmill/issues/1185)) ([8445697](https://github.com/windmill-labs/windmill/commit/8445697e31394ac11f3b8aa10af1546cc9c0041c))
+
+## [1.63.2](https://github.com/windmill-labs/windmill/compare/v1.63.1...v1.63.2) (2023-02-15)
+
+
+### Bug Fixes
+
+* **psql:** update pg client ([a2fbc57](https://github.com/windmill-labs/windmill/commit/a2fbc5702509bb259bae106baa9a6146360ec5dd))
+
+## [1.63.1](https://github.com/windmill-labs/windmill/compare/v1.63.0...v1.63.1) (2023-02-14)
+
+
+### Bug Fixes
+
+* update hub sync script ([03eb144](https://github.com/windmill-labs/windmill/commit/03eb1444c4a5dfbd170ba8d200784e530ca2f771))
+
+## [1.63.0](https://github.com/windmill-labs/windmill/compare/v1.62.0...v1.63.0) (2023-02-14)
+
+
+### Features
+
+* add mem peak info ([f584062](https://github.com/windmill-labs/windmill/commit/f584062f13aa7da8e767fd35de1aef7bbb67c3c8))
+* **frontend:** Minimal support for custom filenames ([#1190](https://github.com/windmill-labs/windmill/issues/1190)) ([b03b3be](https://github.com/windmill-labs/windmill/commit/b03b3be154efb0984f9623c27acc05617f125bc5))
+* **worker:** set oom_adj to 1000 to prioritize killing subprocess ([265fbc5](https://github.com/windmill-labs/windmill/commit/265fbc5835d029d510a794e171392884cb20bdae))
+
+
+### Bug Fixes
+
+* **python:** return none if argument is missing ([3f2754b](https://github.com/windmill-labs/windmill/commit/3f2754b3305f6cb65373d532ff0db6020bf07e45))
+* Update references to the docs ([#1191](https://github.com/windmill-labs/windmill/issues/1191)) ([a574270](https://github.com/windmill-labs/windmill/commit/a574270bc259f423c984259cd7d9a6d91b77815c))
+
 ## [1.62.0](https://github.com/windmill-labs/windmill/compare/v1.61.1...v1.62.0) (2023-02-03)
 
 

Dockerfile

@@ -73,7 +73,7 @@ ARG features=""
 
 COPY --from=planner /windmill/recipe.json recipe.json
 
-RUN CARGO_NET_GIT_FETCH_WITH_CLI=true cargo chef cook --release --features "$features" --recipe-path recipe.json
+RUN CARGO_NET_GIT_FETCH_WITH_CLI=true RUST_BACKTRACE=1 cargo chef cook --release --features "$features" --recipe-path recipe.json
 
 COPY ./openflow.openapi.yaml /openflow.openapi.yaml
 COPY ./backend ./
@@ -85,7 +85,8 @@ COPY .git/ .git/
 RUN CARGO_NET_GIT_FETCH_WITH_CLI=true cargo build --release --features "$features"
 
 
-FROM python:3.11.1-slim-buster
+FROM python:3.11.2-slim-buster
+ARG TARGETPLATFORM
 
 ARG APP=/usr/src/app
 
@@ -129,6 +130,10 @@ COPY --from=nsjail /nsjail/nsjail /bin/nsjail
 
 COPY --from=denoland/deno:latest /usr/bin/deno /usr/bin/deno
 
+# docker does not support conditional COPY and we want to use the same Dockerfile for both amd64 and arm64 and privilege the official image
+COPY --from=lukechannings/deno:latest /usr/bin/deno /usr/bin/deno-arm
+RUN if [ "$TARGETPLATFORM" = "linux/amd64" ]; then rm /usr/bin/deno-arm; elif [ "$TARGETPLATFORM" = "linux/arm64" ]; then mv /usr/bin/deno-arm /usr/bin/deno; fi
+
 RUN mkdir -p ${APP}
 
 WORKDIR ${APP}

LICENSE

@@ -8,5 +8,9 @@ or belonging to one of the below cases:
 
 The files under backend/ are AGPL Licensed.
 The files under frontend/ are AGPL Licensed.
-The files under python-client/ are Apache 2.0 Licensed.
-The files under community/ are Apache 2.0 Licensed.
+The files under python-client/ deno-client/ go-client/ are Apache 2.0 Licensed.
+
+The openapi files, including the OpenFlow spec is Apache 2.0 Licensed.
+
+All third party components incorporated into the Windmill Software are licensed under the
+original license provided by the owner of the applicable component.

README.md

@@ -5,7 +5,7 @@
     <em>.</em>
 </p>
 <p align=center>
-Open-source developer infrastructure for internal tools. Self-hostable alternative to Airplane, Pipedream, Superblocks and a simplified Temporal with autogenerated UIs to trigger workflows and scripts as internal apps. Scripts are turned into UIs and no-code modules, no-code modules can be composed into very rich flows, and script and flows can be triggered from internal UIs made with a low-code builder. The script languages supported are: Python, Typescript, Go, Bash.
+Open-source developer infrastructure for internal tools. Self-hostable alternative to Airplane, Pipedream, Superblocks and a simplified Temporal with autogenerated UIs to trigger workflows and scripts as internal apps. Scripts are turned into UIs and no-code modules, no-code modules can be composed into very rich flows, and script and flows can be triggered from internal UIs made with a low-code builder. The script languages supported are: Python, Typescript, Go, Bash, SQL.
 </p>
 
 <p align="center">
@@ -20,55 +20,35 @@ Open-source developer infrastructure for internal tools. Self-hostable alternati
 </a>
 </p>
 
----
-
-**Try it (personal workspaces are free forever)**: <https://app.windmill.dev>
-
-**Documentation**: <https://docs.windmill.dev>
-
-**Discord**: <https://discord.gg/V7PM2YHsPB>
-
-**Hub**: <https://hub.windmill.dev>
-
-**Contributor's guide**: <https://docs.windmill.dev/docs/contributors_guide>
-
-**Roadmap**: <https://github.com/orgs/windmill-labs/projects/2>
-
-You can show your support for the project by starring this repo.
-
-Windmill Labs offers commercial licenses, an enterprise edition, local hub
-mirrors, and support: contact ruben@windmill.dev.
-
----
-
-# Windmill
-
 <p align="center">
-<b>Disclaimer: </b>Windmill is in <b>BETA</b>. It is secure to run in production but we are still <a href="https://github.com/orgs/windmill-labs/projects/2">improving the product fast<a/>.
+  <a href="https://app.windmill.dev">Try it</a> - <a href="https://docs.windmill.dev/docs/intro/">Docs</a> - <a href="https://discord.gg/V7PM2YHsPB">Discord</a> - <a href="https://hub.windmill.dev">Hub</a> - <a href="https://docs.windmill.dev/docs/misc/contributing">Contributor's guide</a>
 </p>
 
-![Windmill Screenshot](./imgs/windmill-flow.png)
-![Windmill Screenshot](./imgs/windmill.png)
+# Windmill - Turn scripts into workflows and UIs that you can share and run at scale
 
-Windmill is <b>fully open-sourced (AGPLv3)</b>:
+Windmill is <b>fully open-sourced (AGPLv3)</b> and Windmill Labs offers dedicated instance and commercial support and licenses.
 
-- [Windmill](#windmill)
+![Windmill Diagram](/imgs/stacks.svg)
+
+https://user-images.githubusercontent.com/275584/218350457-bc2fdc3b-e667-4da5-a2bd-3bacc1f0ec79.mp4
+
+- [Windmill - Turn scripts into workflows and UIs that you can share and run at scale](#windmill---turn-scripts-into-workflows-and-uis-that-you-can-share-and-run-at-scale)
   - [Main Concepts](#main-concepts)
+  - [Show me some actual script code](#show-me-some-actual-script-code)
   - [CLI](#cli)
-  - [Layout](#layout)
+    - [Running scripts locally](#running-scripts-locally)
   - [Stack](#stack)
   - [Security](#security)
-    - [Sandboxing and workload isolation](#sandboxing-and-workload-isolation)
+    - [Sandboxing](#sandboxing)
     - [Secrets, credentials and sensitive values](#secrets-credentials-and-sensitive-values)
   - [Performance](#performance)
   - [Architecture](#architecture)
-    - [Big-picture Architecture](#big-picture-architecture)
-    - [Technical Architecture](#technical-architecture)
   - [How to self-host](#how-to-self-host)
     - [Docker compose](#docker-compose)
     - [Kubernetes (k8s) and Helm charts](#kubernetes-k8s-and-helm-charts)
+    - [Postgres without superuser](#postgres-without-superuser)
     - [Commercial license](#commercial-license)
-    - [OAuth for self-hosting (very optional)](#oauth-for-self-hosting-very-optional)
+    - [OAuth for self-hosting](#oauth-for-self-hosting)
     - [Resource types](#resource-types)
   - [Environment Variables](#environment-variables)
   - [Run a local dev setup](#run-a-local-dev-setup)
@@ -99,20 +79,49 @@ through webhooks.
 
 You can build your entire infra on top of Windmill!
 
+## Show me some actual script code
+
+```typescript
+import * as wmill from "https://deno.land/x/windmill@v1.62.0/mod.ts"
+//import any dependency  from npm
+
+import cowsay from 'npm:cowsay@1.5.0'
+
+export async function main(
+    a: number,
+    // unions generate enums
+    b: "my" | "enum",
+    // default parameters prefill the field 
+    d = "default arg",
+    // nested objects work c = { nested: "object" }, 
+    // permissioned and typed json
+    db: wmill.Resource<"postgresql">) {
+
+    const email = Deno.env.get('WM_EMAIL')
+    // variables are permissioned and by path
+    let variable = await wmill.getVariable('f/company-folder/my_secret')
+    const lastTimeRun = await wmill.getState()
+    // logs are printed and always inspectable
+    console.log(cowsay.say({ text: "hello " + email + " " + lastTimeRun }))
+    await wmill.setState(Date.now())
+
+    // return is serialized as JSON
+    return { foo: d, variable };
+}
+```
+
 ## CLI
 
 We have a powerful CLI to interact with the windmill platform and sync your
-scripts from your own github repo. See
+scripts from local files, github repos and to run scripts and flows on the instance from local commands. See
 [more details](https://github.com/windmill-labs/windmill/tree/main/cli)
 
 ![CLI Screencast](./cli/vhs/output/setup.gif)
 
-## Layout
 
-- `backend/`: Rust backend
-- `frontend`: Svelte frontend
-- `lsp/`: Lsp asssistant for the monaco editor
-- `<lang>-client/`: Windmill client for the given `<lang>`
+### Running scripts locally
+
+You can run your script locally easily, you simply need to pass the right environment variables for the `wmill` client library to fetch resource and variables from your instance if necessary. See more: <https://docs.windmill.dev/docs/advanced/local_development/>
 
 ## Stack
 
@@ -135,7 +144,7 @@ scripts from your own github repo. See
 
 ## Security
 
-### Sandboxing and workload isolation
+### Sandboxing
 
 Windmill uses [nsjail](https://github.com/google/nsjail) on top of the deno
 sandboxing. It is production multi-tenant grade secure. Do not take our word for
@@ -161,21 +170,15 @@ back to the database is ~50ms. A typical lightweight deno job will take around
 
 <p align="center">
 
-### Big-picture Architecture
-
 <img src="./imgs/diagram.svg">
 
-### Technical Architecture
-
-<img src="./imgs/architecture.svg">
-
 </p>
 
 ## How to self-host
 
 We only provide docker-compose setup here. For more advanced setups, like
 compiling from source or using without a postgres super user, see
-[documentation](https://docs.windmill.dev/docs/how-tos/self_host)
+[documentation](https://docs.windmill.dev/docs/advanced/self_host)
 
 ### Docker compose
 
@@ -184,10 +187,6 @@ compiling from source or using without a postgres super user, see
 
 Go to http://localhost et voilà :)
 
-For older kernels < 4.18, set `DISABLE_NUSER=true` as env variable, otherwise
-nsjail will not be able to launch the isolated scripts.
-
-To disable nsjail altogether, set `DISABLE_NSJAIL=true`.
 
 The default super-admin user is: admin@windmill.dev / changeme
 
@@ -195,7 +194,14 @@ From there, you can create other users (do not forget to change the password!)
 
 ### Kubernetes (k8s) and Helm charts
 
-We publish helm charts at: <https://github.com/windmill-labs/windmill-helm-charts>
+We publish helm charts at:
+<https://github.com/windmill-labs/windmill-helm-charts>
+
+### Postgres without superuser
+
+If you do not want, or cannot (for instance, in AWS Aurora or Cloud sql) use a postgres superuser,
+you can run `./init-db-as-superuser.sql` to init the required users for windmill.
+
 
 ### Commercial license
 
@@ -207,14 +213,14 @@ comfortable with AGPLv3.
 To re-expose any Windmill parts to your users as a feature of your product, or
 to build a feature on top of Windmill, to comply with AGPLv3 your product must
 be AGPLv3 or you must get a commercial license. Contact us at
-<license@windmill.dev> if you have any doubts.
+<ruben@windmill.dev> if you have any doubts.
 
 In addition, a commercial license grants you a dedicated engineer to transition
 your current infrastructure to Windmill, support with tight SLA, audit logs
 export features, SSO, unlimited users creation, advanced permission managing
 features such as groups and the ability to create more than one workspace.
 
-### OAuth for self-hosting (very optional)
+### OAuth for self-hosting
 
 To get the same oauth integrations as Windmill Cloud, mount `oauth.json` with
 the following format:
@@ -231,12 +237,13 @@ the following format:
 
 and mount it at `/usr/src/app/oauth.json`.
 
-The redirect url for the oauth clients is: `<instance_url>/user/login_callback/<client>`
+The redirect url for the oauth clients is:
+`<instance_url>/user/login_callback/<client>`
 
 [The list of all possible "connect an app" oauth clients](https://github.com/windmill-labs/windmill/blob/main/backend/oauth_connect.json)
 
 To add more "connect an app" OAuth clients to the Windmill project, read the
-[Contributor's guide](https://docs.windmill.dev/docs/contributors_guide). We
+[Contributor's guide](https://docs.windmill.dev/docs/misc/contributing). We
 welcome contributions!
 
 You may also add your own custom OAuth2 IdP and OAuth2 Resource provider:
@@ -268,16 +275,17 @@ You may also add your own custom OAuth2 IdP and OAuth2 Resource provider:
 ### Resource types
 
 You will also want to import all the approved resource types from
-[WindmillHub](https://hub.windmill.dev). There is no automatic way to do this
-automatically currently, but it will be possible using a command with the
-upcoming CLI tool.
+[WindmillHub](https://hub.windmill.dev). A setup script will prompt
+you to have it being synced automatically everyday.
 
 ## Environment Variables
 
 | Environment Variable name | Default                | Description                                                                                                                                                                                        | Api Server/Worker/All |
 | ------------------------- | ---------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------- |
 | DATABASE_URL              |                        | The Postgres database url.                                                                                                                                                                         | All                   |
-| DISABLE_NSJAIL            | true                   | Disable Nsjail Sandboxing                                                                                                                                                                          |                       |
+| DISABLE_NSJAIL            | true                   | Disable Nsjail Sandboxing                                                                                                                                                                          | Worker                |
+| SERVER_BIND_ADDR          | 0.0.0.0                | IP Address on which to bind listening socket                                                                                                                                                       | Server                |
+| PORT                      | 8000                   | Exposed port                                                                                                                                                                                       | Server                |  |
 | NUM_WORKERS               | 3                      | The number of worker per Worker instance (set to 1 on Eks to have 1 pod = 1 worker, set to 0 for an API only instance)                                                                             | Worker                |
 | DISABLE_SERVER            | false                  | Binary would operate as a worker only instance                                                                                                                                                     | Worker                |
 | METRICS_ADDR              | None                   | The socket addr at which to expose Prometheus metrics at the /metrics path. Set to "true" to expose it on port 8001                                                                                | All                   |
@@ -293,8 +301,7 @@ upcoming CLI tool.
 | S3_CACHE_BUCKET (EE only) | None                   | The S3 bucket to sync the cache of the workers to                                                                                                                                                  | Worker                |
 | TAR_CACHE_RATE (EE only)  | 100                    | The rate at which to tar the cache of the workers. 100 means every 100th job in average (uniformly randomly distributed).                                                                          | Worker                |
 | SLACK_SIGNING_SECRET      | None                   | The signing secret of your Slack app. See [Slack documentation](https://api.slack.com/authentication/verifying-requests-from-slack)                                                                | Server                |
-| COOKIE_DOMAIN             | None                   | The domain of the cookie. If not set, the cookie will be set by the browser based on the full origin                                                                                               | Server                |
-| SERVE_CSP                 | None                   | The CSP directives to use when serving the frontend static assets                                                                                                                                  | Server                |
+| COOKIE_DOMAIN             | None                   | The domain of the cookie. If not set, the cookie will be set by the browser based on the full origin                                                                                               | Server                |  |
 | DENO_PATH                 | /usr/bin/deno          | The path to the deno binary.                                                                                                                                                                       | Worker                |
 | PYTHON_PATH               | /usr/local/bin/python3 | The path to the python binary.                                                                                                                                                                     | Worker                |
 | GO_PATH                   | /usr/bin/go            | The path to the go binary.                                                                                                                                                                         | Worker                |
@@ -309,14 +316,12 @@ upcoming CLI tool.
 | QUEUE_LIMIT_WAIT_RESULT   | None                   | The number of max jobs in the queue before rejecting immediately the request in 'run_wait_result' endpoint. Takes precedence on the query arg. If none is specified, there are no limit.           | Worker                |
 | DENO_AUTH_TOKENS          | None                   | Custom DENO_AUTH_TOKENS to pass to worker to allow the use of private modules                                                                                                                      | Worker                |
 | DENO_FLAGS                | None                   | Override the flags passed to deno (default --allow-all) to tighten permissions. Minimum permissions needed are "--allow-read=args.json --allow-write=result.json"                                  | Worker                |
-| PIP_LOCAL_DEPENDENCIES    | None                   | Specify dependencies that are installed locally and do not need to be solved nor installed again                                                                                                   |
+| PIP_LOCAL_DEPENDENCIES    | None                   | Specify dependencies that are installed locally and do not need to be solved nor installed again                                                                                                   |                       |
 | ADDITIONAL_PYTHON_PATHS   | None                   | Specify python paths (separated by a :) to be appended to the PYTHONPATH of the python jobs. To be used with PIP_LOCAL_DEPENDENCIES to use python codebases within Windmill                        | Worker                |
 | INCLUDE_HEADERS           | None                   | Whitelist of headers that are passed to jobs as args (separated by a comma)                                                                                                                        | Server                |
 | WHITELIST_WORKSPACES      | None                   | Whitelist of workspaces this worker takes job from                                                                                                                                                 | Worker                |
 | BLACKLIST_WORKSPACES      | None                   | Blacklist of workspaces this worker takes job from                                                                                                                                                 | Worker                |
-
-
-
+| NEW_USER_WEBHOOK          | None                   | Webhook to notify of a new user added, signup/invite. Can hook back to windmill to send emails                                                                                                     | Server                |
 
 ## Run a local dev setup
 
@@ -363,4 +368,4 @@ running options.
 
 ## Copyright
 
-Windmill Labs, Inc 2022
+Windmill Labs, Inc 2023

backend/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "windmill"
-version = "1.62.0"
+version = "1.76.0"
 authors.workspace = true
 edition.workspace = true
 
@@ -19,7 +19,7 @@ members = [
 ]
 
 [workspace.package]
-version = "1.62.0"
+version = "1.76.0"
 authors = ["Ruben Fiszel <ruben@windmill.dev>"]
 edition = "2021"
 
@@ -28,7 +28,11 @@ name = "windmill"
 path = "./src/main.rs"
 
 [features]
-enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmill-api/enterprise"]
+enterprise = [
+    "windmill-worker/enterprise",
+    "windmill-queue/enterprise",
+    "windmill-api/enterprise",
+]
 
 [dependencies]
 anyhow.workspace = true
@@ -83,7 +87,7 @@ chrono = { version = "^0", features = ["serde"] }
 tracing = "^0"
 tracing-subscriber = { version = "^0", features = ["env-filter", "json"] }
 prometheus = { version = "^0", default-features = false }
-cookie = { version = "0.16.2" }
+cookie = { version = "0.17.0" }
 phf = { version = "0.11", features = ["macros"] }
 rust-embed = "^6"
 mime_guess = "^2"
@@ -119,9 +123,9 @@ regex = "^1"
 deno_core = "^0"
 async-recursion = "^1"
 swc_common = "^0"
-swc_ecma_parser = "^0"
-swc_ecma_ast = "^0"
-base64 = "^0"
+swc_ecma_parser = "0.128.2"
+swc_ecma_ast = "0.98.1"
+base64 = "0.21.0"
 unicode-general-category = "^0"
 hmac = "0.12.1"
 sha2 = "0.10.6"
@@ -144,4 +148,8 @@ serde_derive = "1.0.147"
 const_format = { version = "0.2", features = ["rust_1_64", "rust_1_51"] }
 dyn-iter = "0.2.0"
 rsa = "0.7.2"
-async-stripe = { version = "0.14", features = ["runtime-tokio-hyper", "checkout"] }
+async-stripe = { version = "0.14", features = [
+    "runtime-tokio-hyper",
+    "checkout",
+] }
+async_zip = { version = "0.0.11", features = ["full"] }

backend/migrations/20221214105402_first_time_users.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20221214105402_first_time_users.up.sql

@@ -0,0 +1,12 @@
+-- Add up migration script here
+ALTER TABLE
+    password
+ADD
+    first_time_user boolean NOT NULL DEFAULT (false);
+
+UPDATE
+    password
+SET
+    first_time_user = true
+WHERE
+    email = 'admin@windmill.dev';

backend/migrations/20230210145514_first_time_app.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230213145545_change_admin_app.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230214022802_goto_logout_setup.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230214135931_migrate_hub_sync_script.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230214135931_migrate_hub_sync_script.up.sql

@@ -0,0 +1,16 @@
+-- Add up migration script here
+-- Add up migration script here
+UPDATE script SET content = 'import wmill from "https://deno.land/x/wmill@v1.63.1/main.ts";
+export async function main() {
+  await run(
+    "workspace", "add", "__automation", "admins", Deno.env.get("BASE_INTERNAL_URL") + "/", "--token", Deno.env.get("WM_TOKEN"));
+
+  await run("hub", "pull");
+}
+
+async function run(...cmd: string[]) {
+  console.log("Running \"" + cmd.join('' '') + "\"");
+  await wmill.parse(cmd);
+}', summary = 'Synchronize Hub Resource types with admins workspace',
+description = 'Basic administrative script to sync latest resource types from hub to share to every workspace. Recommended to run at least once. On a schedule by default.'
+WHERE hash = -28028598712388162 AND workspace_id = 'admins';

backend/migrations/20230214144244_improve_app_setup.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230226130307_update_included_scripts_with_deno_changes.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230227100232_update_included_scripts_with_deno_changes.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230227131521_update_included_scripts_with_deno_changes.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230227131521_update_included_scripts_with_deno_changes.up.sql

@@ -0,0 +1,16 @@
+-- Add up migration script here
+
+UPDATE script SET content = 'import wmill from "https://deno.land/x/wmill@v1.70.1/main.ts";
+export async function main() {
+  await run(
+    "workspace", "add", "__automation", "admins", Deno.env.get("BASE_INTERNAL_URL") + "/", "--token", Deno.env.get("WM_TOKEN"));
+
+  await run("hub", "pull");
+}
+
+async function run(...cmd: string[]) {
+  console.log("Running \"" + cmd.join('' '') + "\"");
+  await wmill.parse(cmd);
+}', summary = 'Synchronize Hub Resource types with admins workspace',
+description = 'Basic administrative script to sync latest resource types from hub to share to every workspace. Recommended to run at least once. On a schedule by default.'
+WHERE hash = -28028598712388162 AND workspace_id = 'admins';

backend/migrations/20230303064952_add_root_leaf_jobs.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230303064952_add_root_leaf_jobs.up.sql

@@ -0,0 +1,3 @@
+-- Add up migration script here
+ALTER TABLE queue ADD COLUMN root_job uuid;
+ALTER TABLE queue ADD COLUMN leaf_jobs jsonb;

backend/parsers/windmill-parser-bash/Cargo.toml

@@ -16,4 +16,5 @@ unicode-general-category.workspace = true
 itertools.workspace = true
 anyhow.workspace = true
 regex.workspace = true
-lazy_static.workspace = true
+lazy_static.workspace = true
+serde_json.workspace = true

backend/parsers/windmill-parser-bash/src/lib.rs

@@ -1,6 +1,8 @@
 #![allow(non_snake_case)] // TODO: switch to parse_* function naming
 
+use anyhow::anyhow;
 use regex::Regex;
+use serde_json::json;
 
 use std::collections::HashMap;
 use windmill_parser::{Arg, MainArgSignature, Typ};
@@ -17,19 +19,32 @@ pub fn parse_bash_sig(code: &str) -> windmill_common::error::Result<MainArgSigna
     }
 }
 
+lazy_static::lazy_static! {
+    static ref RE: Regex = Regex::new(r#"(?m)^(\w+)="\$(?:(\d+)|\{(\d+):-(.*)\})"$"#).unwrap();
+}
+
 fn parse_file(code: &str) -> anyhow::Result<Option<Vec<Arg>>> {
-    let mut hm = HashMap::new();
-    let re = Regex::new(r#"(?m)^(\w+)="\$(\d+)"$"#).unwrap();
-    for cap in re.captures_iter(code) {
-        hm.insert(cap[2].parse::<i32>()?, cap[1].to_string());
+    let mut hm: HashMap<i32, (String, Option<String>)> = HashMap::new();
+    for cap in RE.captures_iter(code) {
+        hm.insert(
+            cap.get(2)
+                .or(cap.get(3))
+                .and_then(|x| x.as_str().parse::<i32>().ok())
+                .ok_or_else(|| anyhow!("Impossible to parse arg digit"))?,
+            (
+                cap[1].to_string(),
+                cap.get(4).map(|x| x.as_str().to_string()),
+            ),
+        );
     }
     let mut args = vec![];
     for i in 1..20 {
         if hm.contains_key(&i) {
+            let (name, default) = hm.get(&i).unwrap();
             args.push(Arg {
-                name: hm[&i].clone(),
+                name: name.clone(),
                 typ: Typ::Str(None),
-                default: None,
+                default: default.clone().map(|x| json!(x)),
                 otyp: None,
                 has_default: false,
             });
@@ -43,6 +58,8 @@ fn parse_file(code: &str) -> anyhow::Result<Option<Vec<Arg>>> {
 #[cfg(test)]
 mod tests {
 
+    use serde_json::json;
+
     use super::*;
 
     #[test]
@@ -50,8 +67,7 @@ mod tests {
         let code = r#"
 token="$1"
 image="$2"
-digest="${3:-latest}"
-foo="$4"
+digest="${3:-latest with spaces}"
 
 "#;
         //println!("{}", serde_json::to_string()?);
@@ -74,6 +90,13 @@ foo="$4"
                         typ: Typ::Str(None),
                         default: None,
                         has_default: false
+                    },
+                    Arg {
+                        otyp: None,
+                        name: "digest".to_string(),
+                        typ: Typ::Str(None),
+                        default: Some(json!("latest with spaces")),
+                        has_default: false
                     }
                 ]
             }

backend/parsers/windmill-parser-py/src/lib.rs

@@ -17,10 +17,8 @@ use serde_json::json;
 use windmill_common::error;
 use windmill_parser::{json_to_typ, Arg, MainArgSignature, Typ};
 
-use rustpython_parser::{
-    ast::{Constant, ExprKind, Located, StmtKind},
-    parser,
-};
+use rustpython_parser as parser;
+use rustpython_parser::ast::{Constant, ExprKind, Located, StmtKind};
 
 const DEF_MAIN: &str = "def main(";
 const FUNCTION_CALL: &str = "<function call>";

backend/src/main.rs

@@ -6,14 +6,16 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
-use std::net::SocketAddr;
+use std::net::{IpAddr, Ipv4Addr, SocketAddr};
 
 use git_version::git_version;
 use sqlx::{Pool, Postgres};
 use windmill_common::utils::rd_string;
-use windmill_worker::WorkerConfig;
 
 const GIT_VERSION: &str = git_version!(args = ["--tag", "--always"], fallback = "unknown-version");
+const DEFAULT_NUM_WORKERS: usize = 3;
+const DEFAULT_PORT: u16 = 8000;
+const DEFAULT_SERVER_BIND_ADDR: Ipv4Addr = Ipv4Addr::new(0, 0, 0, 0);
 
 mod ee;
 
@@ -26,7 +28,7 @@ async fn main() -> anyhow::Result<()> {
     let num_workers = std::env::var("NUM_WORKERS")
         .ok()
         .and_then(|x| x.parse::<i32>().ok())
-        .unwrap_or(windmill_common::DEFAULT_NUM_WORKERS as i32);
+        .unwrap_or(DEFAULT_NUM_WORKERS as i32);
 
     let metrics_addr: Option<SocketAddr> = std::env::var("METRICS_ADDR")
         .ok()
@@ -38,6 +40,18 @@ async fn main() -> anyhow::Result<()> {
         .transpose()?
         .flatten();
 
+    let server_bind_address: IpAddr = std::env::var("SERVER_BIND_ADDR")
+        .ok()
+        .and_then(|x| x.parse().ok() )
+        .unwrap_or(IpAddr::from(DEFAULT_SERVER_BIND_ADDR));
+
+    let port: u16 = std::env::var("PORT")
+        .ok()
+        .and_then(|x| x.parse::<u16>().ok())
+        .unwrap_or(DEFAULT_PORT as u16);
+    let base_internal_url: String = std::env::var("BASE_INTERNAL_URL")
+        .unwrap_or_else(|_| format!("http://localhost:{}", port.to_string()));
+
     let server_mode = !std::env::var("DISABLE_SERVER")
         .ok()
         .and_then(|x| x.parse::<bool>().ok())
@@ -52,99 +66,87 @@ async fn main() -> anyhow::Result<()> {
     let (tx, rx) = tokio::sync::broadcast::channel::<()>(3);
     let shutdown_signal = windmill_common::shutdown_signal(tx);
 
-    let base_url = std::env::var("BASE_URL").unwrap_or_else(|_| "http://localhost".to_string());
+    #[cfg(feature = "enterprise")]
+    tracing::info!(
+        "
+##############################
+Windmill Enterprise Edition {GIT_VERSION}
+##############################"
+    );
 
-    let base_internal_url =
-        std::env::var("BASE_INTERNAL_URL").unwrap_or_else(|_| "http://localhost:8000".to_string());
-    let timeout = std::env::var("TIMEOUT")
-        .ok()
-        .and_then(|x| x.parse::<i32>().ok())
-        .unwrap_or(windmill_common::DEFAULT_TIMEOUT);
+    #[cfg(not(feature = "enterprise"))]
+    tracing::info!(
+        "
+##############################
+Windmill Community Edition {GIT_VERSION}
+##############################"
+    );
+
+    display_config(vec![
+        "DISABLE_NSJAIL",
+        "DISABLE_SERVER",
+        "NUM_WORKERS",
+        "METRICS_ADDR",
+        "JSON_FMT",
+        "BASE_URL",
+        "BASE_INTERNAL_URL",
+        "TIMEOUT",
+        "SLEEP_QUEUE",
+        "MAX_LOG_SIZE",
+        "SERVER_BIND_ADDR",
+        "PORT",
+        "KEEP_JOB_DIR",
+        "S3_CACHE_BUCKET",
+        "TAR_CACHE_RATE",
+        "COOKIE_DOMAIN",
+        "PYTHON_PATH",
+        "DENO_PATH",
+        "GO_PATH",
+        "PIP_INDEX_URL",
+        "PIP_EXTRA_INDEX_URL",
+        "PIP_TRUSTED_HOST",
+        "PATH",
+        "HOME",
+        "DATABASE_CONNECTIONS",
+        "TIMEOUT_WAIT_RESULT",
+        "QUEUE_LIMIT_WAIT_RESULT",
+        "DENO_AUTH_TOKENS",
+        "DENO_FLAGS",
+        "PIP_LOCAL_DEPENDENCIES",
+        "ADDITIONAL_PYTHON_PATHS",
+        "INCLUDE_HEADERS",
+        "WHITELIST_WORKSPACES",
+        "BLACKLIST_WORKSPACES",
+        "NEW_USER_WEBHOOK",
+        "CLOUD_HOSTED",
+    ]);
 
     if server_mode || num_workers > 0 {
-        let addr = SocketAddr::from(([0, 0, 0, 0], 8000));
+        let addr = SocketAddr::from((server_bind_address, port));
 
-        let base_url2 = base_url.clone();
         let server_f = async {
             if server_mode {
-                windmill_api::run_server(db.clone(), addr, base_url, rx.resubscribe()).await?;
+                windmill_api::run_server(db.clone(), addr, rx.resubscribe()).await?;
             }
             Ok(()) as anyhow::Result<()>
         };
 
-        let base_url = base_url2.clone();
         let workers_f = async {
             if num_workers > 0 {
-                let sleep_queue = std::env::var("SLEEP_QUEUE")
-                    .ok()
-                    .and_then(|x| x.parse::<u64>().ok())
-                    .unwrap_or(windmill_common::DEFAULT_SLEEP_QUEUE);
-                let disable_nuser = std::env::var("DISABLE_NUSER")
-                    .ok()
-                    .and_then(|x| x.parse::<bool>().ok())
-                    .unwrap_or(false);
-                let disable_nsjail = std::env::var("DISABLE_NSJAIL")
-                    .ok()
-                    .and_then(|x| x.parse::<bool>().ok())
-                    .unwrap_or(true);
-                let keep_job_dir = std::env::var("KEEP_JOB_DIR")
-                    .ok()
-                    .and_then(|x| x.parse::<bool>().ok())
-                    .unwrap_or(false);
-                let license_key = std::env::var("LICENSE_KEY").ok();
-                let sync_bucket = std::env::var("S3_CACHE_BUCKET")
-                    .ok()
-                    .map(|e| Some(e))
-                    .unwrap_or(None);
-
-                #[cfg(feature = "enterprise")]
-                tracing::info!(
-                        "
-##############################
-Windmill Enterprise Edition {GIT_VERSION} LICENSE_KEY: {license_key:?}, S3_CACHE_BUCKET: {sync_bucket:?}
-##############################"
-                    );
-
-                #[cfg(not(feature = "enterprise"))]
-                tracing::info!(
-                    "
-##############################
-Windmill Community Edition {GIT_VERSION}
-##############################"
-                );
-
-                tracing::info!(
-                    "DISABLE_NSJAIL: {disable_nsjail}, DISABLE_NUSER: {disable_nuser}, BASE_URL: \
-                     {base_url}, SLEEP_QUEUE: {sleep_queue}, NUM_WORKERS: {num_workers}, TIMEOUT: \
-                     {timeout}, KEEP_JOB_DIR: {keep_job_dir}"
-                );
-
                 run_workers(
                     db.clone(),
-                    addr,
-                    timeout,
-                    num_workers,
-                    sleep_queue,
-                    WorkerConfig {
-                        disable_nsjail,
-                        disable_nuser,
-                        base_internal_url,
-                        base_url,
-                        keep_job_dir,
-                    },
                     rx.resubscribe(),
-                    sync_bucket,
-                    license_key,
+                    num_workers,
+                    base_internal_url.clone(),
                 )
                 .await?;
             }
             Ok(()) as anyhow::Result<()>
         };
 
-        let base_url = base_url2;
         let monitor_f = async {
             if server_mode {
-                monitor_db(&db, timeout, base_url, rx.resubscribe());
+                monitor_db(&db, rx.resubscribe(), &base_internal_url);
             }
             Ok(()) as anyhow::Result<()>
         };
@@ -163,34 +165,46 @@ Windmill Community Edition {GIT_VERSION}
     Ok(())
 }
 
+fn display_config(envs: Vec<&str>) {
+    tracing::info!(
+        "config: {}",
+        envs.iter()
+            .filter(|env| std::env::var(env).is_ok())
+            .map(|env| {
+                format!(
+                    "{}: {}",
+                    env,
+                    std::env::var(env).unwrap_or_else(|_| "not set".to_string())
+                )
+            })
+            .collect::<Vec<String>>()
+            .join(", ")
+    )
+}
+
 pub fn monitor_db(
     db: &Pool<Postgres>,
-    timeout: i32,
-    base_url: String,
     rx: tokio::sync::broadcast::Receiver<()>,
+    base_internal_url: &str,
 ) {
     let db1 = db.clone();
     let db2 = db.clone();
 
     let rx2 = rx.resubscribe();
-
+    let base_internal_url = base_internal_url.to_string();
     tokio::spawn(async move {
-        windmill_worker::handle_zombie_jobs_periodically(&db1, timeout, &base_url, rx).await
+        windmill_worker::handle_zombie_jobs_periodically(&db1, rx, &base_internal_url).await
     });
     tokio::spawn(async move { windmill_api::delete_expired_items_perdiodically(&db2, rx2).await });
 }
 
 pub async fn run_workers(
     db: Pool<Postgres>,
-    addr: SocketAddr,
-    timeout: i32,
-    num_workers: i32,
-    sleep_queue: u64,
-    worker_config: WorkerConfig,
     rx: tokio::sync::broadcast::Receiver<()>,
-    mut periodic_script: Option<String>,
-    license_key: Option<String>,
+    num_workers: i32,
+    base_internal_url: String,
 ) -> anyhow::Result<()> {
+    let license_key = std::env::var("LICENSE_KEY").ok();
     #[cfg(feature = "enterprise")]
     ee::verify_license_key(license_key)?;
 
@@ -198,12 +212,6 @@ pub async fn run_workers(
     if license_key.is_some() {
         panic!("License key is required ONLY for the enterprise edition");
     }
-    #[cfg(not(feature = "enterprise"))]
-    if !worker_config.disable_nsjail {
-        tracing::warn!(
-            "NSJAIL to sandbox process in untrusted environments is an enterprise feature but allowed to be used for testing purposes"
-        );
-    }
 
     let instance_name = rd_string(5);
     let monitor = tokio_metrics::TaskMonitor::new();
@@ -223,22 +231,17 @@ pub async fn run_workers(
         let worker_name = format!("dt-worker-{}-{}", &instance_name, rd_string(5));
         let ip = ip.clone();
         let rx = rx.resubscribe();
-        let worker_config = worker_config.clone();
-        let wp = periodic_script.take();
+        let base_internal_url = base_internal_url.clone();
         handles.push(tokio::spawn(monitor.instrument(async move {
-            tracing::info!(addr = %addr.to_string(), worker = %worker_name, "starting worker");
+            tracing::info!(worker = %worker_name, "starting worker");
             windmill_worker::run_worker(
                 &db1,
-                timeout,
                 &instance_name,
                 worker_name,
                 i as u64,
-                num_workers as u64,
                 &ip,
-                sleep_queue,
-                worker_config,
-                wp,
                 rx,
+                &base_internal_url,
             )
             .await
         })));

backend/tests/worker.rs

@@ -6,10 +6,8 @@ use windmill_common::{
     flow_status::{FlowStatus, FlowStatusModule},
     flows::{FlowModule, FlowModuleValue, FlowValue, InputTransform},
     scripts::ScriptLang,
-    DEFAULT_SLEEP_QUEUE,
 };
 use windmill_queue::{get_queued_job, JobPayload, RawCode};
-use windmill_worker::WorkerConfig;
 
 async fn initialize_tracing() {
     use std::sync::Once;
@@ -89,14 +87,7 @@ impl ApiServer {
         let addr = sock.local_addr().unwrap();
         drop(sock);
 
-        let task = tokio::task::spawn({
-            windmill_api::run_server(
-                db.clone(),
-                addr,
-                format!("http://localhost:{}", addr.port()),
-                rx,
-            )
-        });
+        let task = tokio::task::spawn(windmill_api::run_server(db.clone(), addr, rx));
 
         return Self { addr, tx, task };
     }
@@ -845,6 +836,7 @@ impl RunJob {
             /* scheduled_for_o */ None,
             /* schedule_path */ None,
             /* parent_job */ None,
+            /* root job  */ None,
             /* is_flow_step */ false,
             /* running */ false,
             None,
@@ -917,43 +909,20 @@ fn spawn_test_worker(
 ) {
     let (tx, rx) = tokio::sync::broadcast::channel(1);
     let db = db.to_owned();
-    let timeout = 4_000;
     let worker_instance: &str = "test worker instance";
     let worker_name: String = next_worker_name();
     let i_worker: u64 = Default::default();
-    let num_workers: u64 = 2;
     let ip: &str = Default::default();
-    let sleep_queue: u64 = DEFAULT_SLEEP_QUEUE / num_workers;
-    let port = port;
-    let worker_config = WorkerConfig {
-        base_internal_url: format!("http://localhost:{port}"),
-        base_url: format!("http://localhost:{port}"),
-        disable_nuser: std::env::var("DISABLE_NUSER")
-            .ok()
-            .and_then(|x| x.parse::<bool>().ok())
-            .unwrap_or(false),
-        disable_nsjail: std::env::var("DISABLE_NSJAIL")
-            .ok()
-            .and_then(|x| x.parse::<bool>().ok())
-            .unwrap_or(false),
-        keep_job_dir: std::env::var("KEEP_JOB_DIR")
-            .ok()
-            .and_then(|x| x.parse::<bool>().ok())
-            .unwrap_or(false),
-    };
     let future = async move {
+        let base_internal_url = format!("http://localhost:{}", port);
         windmill_worker::run_worker(
             &db,
-            timeout,
             worker_instance,
             worker_name,
             i_worker,
-            num_workers,
             ip,
-            sleep_queue,
-            worker_config,
-            None,
             rx,
+            &base_internal_url,
         )
         .await
     };

backend/windmill-api/Cargo.toml

@@ -8,12 +8,8 @@ edition.workspace = true
 name = "windmill_api"
 path = "src/lib.rs"
 
-[[bin]]
-name = "windmill_api"
-path = "src/main.rs"
-
 [features]
-enterprise = ["windmill-queue/enterprise"]
+enterprise = ["windmill-queue/enterprise", "async-stripe"]
 
 [dependencies]
 windmill-queue.workspace = true
@@ -69,6 +65,7 @@ hmac.workspace = true
 cookie.workspace = true
 sha2.workspace = true
 urlencoding.workspace = true
-async-stripe.workspace = true
+async-stripe = { workspace = true, optional = true }
 lazy_static.workspace = true
-prometheus.workspace = true
+prometheus.workspace = true
+async_zip.workspace = true

backend/windmill-api/openapi.yaml

@@ -1,7 +1,7 @@
 openapi: "3.0.3"
 
 info:
-  version: 1.62.0
+  version: 1.76.0
   title: Windmill API
 
   contact:
@@ -204,7 +204,6 @@ paths:
               schema:
                 type: string
 
-
   /w/{workspace}/users/is_owner/{path}:
     get:
       summary: is owner of path
@@ -1092,6 +1091,10 @@ paths:
         - variable
       parameters:
         - $ref: "#/components/parameters/WorkspaceId"
+        - name: already_encrypted
+          in: query
+          schema:
+            type: boolean
       requestBody:
         description: new variable
         required: true
@@ -1133,6 +1136,10 @@ paths:
       parameters:
         - $ref: "#/components/parameters/WorkspaceId"
         - $ref: "#/components/parameters/Path"
+        - name: already_encrypted
+          in: query
+          schema:
+            type: boolean
       requestBody:
         description: updated variable
         required: true
@@ -1541,7 +1548,7 @@ paths:
         required: true
         content:
           application/json:
-            schema: 
+            schema:
               type: object
               properties:
                 value: {}
@@ -2095,7 +2102,6 @@ paths:
                 items:
                   type: string
 
-
   /w/{workspace}/scripts/create:
     post:
       summary: create script
@@ -2270,7 +2276,7 @@ paths:
 
   /w/{workspace}/scripts/delete/h/{hash}:
     post:
-      summary: delete script by hash (erase content but keep hash)
+      summary: delete script by hash (erase content but keep hash, require admin)
       operationId: deleteScriptByHash
       tags:
         - script
@@ -2285,6 +2291,23 @@ paths:
               schema:
                 $ref: "#/components/schemas/Script"
 
+  /w/{workspace}/scripts/delete/p/{path}:
+    post:
+      summary: delete all scripts at a given path (require admin)
+      operationId: deleteScriptByPath
+      tags:
+        - script
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/ScriptPath"
+      responses:
+        "200":
+          description: script path
+          content:
+            application/json:
+              schema:
+                type: string
+
   /w/{workspace}/scripts/get/p/{path}:
     get:
       summary: get script by path
@@ -2530,11 +2553,6 @@ paths:
           required: true
           schema:
             type: string
-        - name: skip_direct
-          description: Skip checking that the node is part of the given flow.
-          in: query
-          schema:
-            type: boolean
       responses:
         "200":
           description: job result
@@ -2708,6 +2726,23 @@ paths:
               schema:
                 type: string
 
+  /w/{workspace}/flows/delete/{path}:
+    delete:
+      summary: delete flow by path
+      operationId: deleteFlowByPath
+      tags:
+        - flow
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/ScriptPath"
+      responses:
+        "200":
+          description: flow delete
+          content:
+            text/plain:
+              schema:
+                type: string
+
   /w/{workspace}/apps/list:
     get:
       summary: list all available apps
@@ -2798,7 +2833,6 @@ paths:
               schema:
                 $ref: "#/components/schemas/AppWithLastVersion"
 
-
   /w/{workspace}/apps/secret_of/{path}:
     get:
       summary: get public secret of app
@@ -3131,12 +3165,14 @@ paths:
         - $ref: "#/components/parameters/ScriptExactPath"
         - $ref: "#/components/parameters/ScriptStartPath"
         - $ref: "#/components/parameters/ScriptExactHash"
-        - $ref: "#/components/parameters/CreatedBefore"
-        - $ref: "#/components/parameters/CreatedAfter"
+        - $ref: "#/components/parameters/StartedBefore"
+        - $ref: "#/components/parameters/StartedAfter"
         - $ref: "#/components/parameters/Success"
         - $ref: "#/components/parameters/JobKinds"
         - $ref: "#/components/parameters/Suspended"
         - $ref: "#/components/parameters/Running"
+        - $ref: "#/components/parameters/ArgsFilter"
+        - $ref: "#/components/parameters/ResultFilter"
       responses:
         "200":
           description: All available queued jobs
@@ -3161,10 +3197,12 @@ paths:
         - $ref: "#/components/parameters/ScriptExactPath"
         - $ref: "#/components/parameters/ScriptStartPath"
         - $ref: "#/components/parameters/ScriptExactHash"
-        - $ref: "#/components/parameters/CreatedBefore"
-        - $ref: "#/components/parameters/CreatedAfter"
+        - $ref: "#/components/parameters/StartedBefore"
+        - $ref: "#/components/parameters/StartedAfter"
         - $ref: "#/components/parameters/Success"
         - $ref: "#/components/parameters/JobKinds"
+        - $ref: "#/components/parameters/ArgsFilter"
+        - $ref: "#/components/parameters/ResultFilter"
         - name: is_skipped
           description: is the job skipped
           in: query
@@ -3198,9 +3236,11 @@ paths:
         - $ref: "#/components/parameters/ScriptExactPath"
         - $ref: "#/components/parameters/ScriptStartPath"
         - $ref: "#/components/parameters/ScriptExactHash"
-        - $ref: "#/components/parameters/CreatedBefore"
-        - $ref: "#/components/parameters/CreatedAfter"
+        - $ref: "#/components/parameters/StartedBefore"
+        - $ref: "#/components/parameters/StartedAfter"
         - $ref: "#/components/parameters/JobKinds"
+        - $ref: "#/components/parameters/ArgsFilter"
+        - $ref: "#/components/parameters/ResultFilter"
         - name: is_skipped
           description: is the job skipped
           in: query
@@ -3312,6 +3352,22 @@ paths:
               schema:
                 $ref: "#/components/schemas/CompletedJob"
 
+  /w/{workspace}/jobs/completed/get_result/{id}:
+    get:
+      summary: get completed job result
+      operationId: getCompletedJobResult
+      tags:
+        - job
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/JobId"
+      responses:
+        "200":
+          description: result
+          content:
+            application/json:
+              schema: {}
+
   /w/{workspace}/jobs/completed/delete/{id}:
     post:
       summary: delete completed job (erase content but keep run id)
@@ -3385,7 +3441,7 @@ paths:
 
   /w/{workspace}/jobs/resume_urls/{id}/{resume_id}:
     get:
-      summary:  get resume urls given a job_id, resume_id and a nonce to resume a flow
+      summary: get resume urls given a job_id, resume_id and a nonce to resume a flow
       operationId: getResumeUrls
       tags:
         - job
@@ -3411,7 +3467,7 @@ paths:
                 properties:
                   approvalPage:
                     type: string
-                  resume: 
+                  resume:
                     type: string
                   cancel:
                     type: string
@@ -4056,7 +4112,7 @@ paths:
                   type: string
                 owners:
                   type: array
-                  items: 
+                  items:
                     type: string
                 extra_perms:
                   additionalProperties:
@@ -4090,7 +4146,7 @@ paths:
               properties:
                 owners:
                   type: array
-                  items: 
+                  items:
                     type: string
                 extra_perms:
                   additionalProperties:
@@ -4261,7 +4317,8 @@ paths:
           required: true
           schema:
             type: string
-            enum: [script, group_, resource, schedule, variable, flow, folder, app]
+            enum:
+              [script, group_, resource, schedule, variable, flow, folder, app]
       responses:
         "200":
           description: acls
@@ -4286,7 +4343,8 @@ paths:
           required: true
           schema:
             type: string
-            enum: [script, group_, resource, schedule, variable, flow, folder, app]
+            enum:
+              [script, group_, resource, schedule, variable, flow, folder, app]
       requestBody:
         description: acl to add
         required: true
@@ -4322,7 +4380,8 @@ paths:
           required: true
           schema:
             type: string
-            enum: [script, group_, resource, schedule, variable, flow, folder, app]
+            enum:
+              [script, group_, resource, schedule, variable, flow, folder, app]
       requestBody:
         description: acl to add
         required: true
@@ -4342,19 +4401,18 @@ paths:
               schema:
                 type: string
 
-
   /w/{workspace}/capture_u/{path}:
-      post:
-        summary: update flow preview capture
-        operationId: updateCapture
-        tags:
-          - capture
-        parameters:
-          - $ref: "#/components/parameters/WorkspaceId"
-          - $ref: "#/components/parameters/Path"
-        responses:
-          "204":
-            description: flow preview captured
+    post:
+      summary: update flow preview capture
+      operationId: updateCapture
+      tags:
+        - capture
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/Path"
+      responses:
+        "204":
+          description: flow preview captured
 
   /w/{workspace}/capture/{path}:
     put:
@@ -4561,15 +4619,15 @@ components:
       in: query
       schema:
         type: string
-    CreatedBefore:
-      name: created_before
+    StartedBefore:
+      name: started_before
       description: filter on created before (inclusive) timestamp
       in: query
       schema:
         type: string
         format: date-time
-    CreatedAfter:
-      name: created_after
+    StartedAfter:
+      name: started_after
       description: filter on created after (exclusive) timestamp
       in: query
       schema:
@@ -4593,6 +4651,19 @@ components:
       in: query
       schema:
         type: boolean
+    ArgsFilter:
+      name: args
+      description: filter on jobs containing those args as a json subset (@> in postgres)
+      in: query
+      schema:
+        type: string
+
+    ResultFilter:
+      name: result
+      description: filter on jobs containing those result as a json subset (@> in postgres)
+      in: query
+      schema:
+        type: string
     After:
       name: after
       description: filter on created after (exclusive) timestamp
@@ -5497,9 +5568,8 @@ components:
           type: string
         worker_instance:
           type: string
-        ping_at:
-          type: string
-          format: date-time
+        last_ping:
+          type: number
         started_at:
           type: string
           format: date-time

backend/windmill-api/src/apps.rs

@@ -13,6 +13,7 @@ use crate::{
     users::{require_owner_of_path, Authed, OptAuthed},
     variables::build_crypt,
     webhook_util::{WebhookMessage, WebhookShared},
+    HTTP_CLIENT,
 };
 use axum::{
     extract::{Extension, Json, Path, Query},
@@ -21,7 +22,6 @@ use axum::{
 };
 use hyper::StatusCode;
 use magic_crypt::MagicCryptTrait;
-use reqwest::Client;
 use serde::{Deserialize, Serialize};
 use serde_json::{json, Map, Value};
 use sha2::{Digest, Sha256};
@@ -170,7 +170,7 @@ async fn list_apps(
         )
         .order_desc("favorite.path IS NOT NULL")
         .order_by("app_version.created_at", true)
-        .and_where("app.workspace_id = ? OR app.workspace_id = 'starter'".bind(&w_id))
+        .and_where("app.workspace_id = ?".bind(&w_id))
         .offset(offset)
         .limit(per_page)
         .clone();
@@ -313,10 +313,13 @@ async fn create_app(
     Extension(user_db): Extension<UserDB>,
     Extension(webhook): Extension<WebhookShared>,
     Path(w_id): Path<String>,
-    Json(app): Json<CreateApp>,
+    Json(mut app): Json<CreateApp>,
 ) -> Result<(StatusCode, String)> {
     let mut tx = user_db.begin(&authed).await?;
 
+    app.policy.on_behalf_of = Some(username_to_permissioned_as(&authed.username));
+    app.policy.on_behalf_of_email = Some(authed.email);
+
     let id = sqlx::query_scalar!(
         "INSERT INTO app
             (workspace_id, path, summary, policy, versions)
@@ -368,12 +371,9 @@ async fn create_app(
     Ok((StatusCode::CREATED, app.path))
 }
 
-async fn list_hub_apps(
-    Authed { email, .. }: Authed,
-    Extension(http_client): Extension<Client>,
-) -> JsonResult<serde_json::Value> {
+async fn list_hub_apps(Authed { email, .. }: Authed) -> JsonResult<serde_json::Value> {
     let flows = list_elems_from_hub(
-        http_client,
+        &HTTP_CLIENT,
         "https://hub.windmill.dev/searchUiData?approved=true",
         &email,
     )
@@ -384,10 +384,9 @@ async fn list_hub_apps(
 pub async fn get_hub_app_by_id(
     Authed { email, .. }: Authed,
     Path(id): Path<i32>,
-    Extension(http_client): Extension<Client>,
 ) -> JsonResult<serde_json::Value> {
     let value = http_get_from_hub(
-        http_client,
+        &HTTP_CLIENT,
         &format!("https://hub.windmill.dev/apps/{id}/json"),
         &email,
         false,
@@ -467,7 +466,9 @@ async fn update_app(
             sqlb.set_str("summary", nsummary);
         }
 
-        if let Some(npolicy) = ns.policy {
+        if let Some(mut npolicy) = ns.policy {
+            npolicy.on_behalf_of = Some(username_to_permissioned_as(&authed.username));
+            npolicy.on_behalf_of_email = Some(authed.email);
             sqlb.set(
                 "policy",
                 &format!(
@@ -674,6 +675,7 @@ async fn execute_component(
         None,
         None,
         None,
+        None,
         false,
         false,
         None,
@@ -731,6 +733,9 @@ fn build_args(
     path: String,
     args: &Map<String, Value>,
 ) -> Result<Map<String, Value>> {
+    // disallow var and res access in args coming from the user for security reasons
+    args.into_iter()
+        .try_for_each(|x| disallow_var_res_access(x.1))?;
     let static_args = policy
         .triggerables
         .get(&path)
@@ -751,3 +756,20 @@ fn build_args(
     }
     Ok(args)
 }
+
+fn disallow_var_res_access(args: &serde_json::Value) -> Result<()> {
+    match args {
+        Value::Object(v) => v.into_iter().try_for_each(|x| disallow_var_res_access(x.1)),
+        Value::Array(arr) => arr.into_iter().try_for_each(|v| disallow_var_res_access(v)),
+        Value::String(s) => {
+            if s.starts_with("$var:") || s.starts_with("$res:") {
+                Err(Error::BadRequest(format!(
+                    "For security reasons, variable or resource access is not allowed as dynamic argument"
+                )))
+            } else {
+                Ok(())
+            }
+        }
+        _ => Ok(()),
+    }
+}

backend/windmill-api/src/capture.rs

@@ -7,11 +7,12 @@
  */
 
 use axum::{
-    extract::{Extension, Path},
+    extract::{Extension, Path, Query},
     routing::{get, post, put},
     Json, Router,
 };
-use hyper::StatusCode;
+use hyper::{HeaderMap, StatusCode};
+use serde::Deserialize;
 use windmill_common::{
     error::{JsonResult, Result},
     utils::{not_found_if_none, StripPath},
@@ -19,6 +20,7 @@ use windmill_common::{
 
 use crate::{
     db::{UserDB, DB},
+    jobs::add_include_headers,
     users::Authed,
 };
 
@@ -83,13 +85,21 @@ pub async fn new_payload(
     Ok(StatusCode::CREATED)
 }
 
+#[derive(Deserialize, Clone)]
+pub struct IncludeHeaderQuery {
+    include_header: Option<String>,
+}
+
 pub async fn update_payload(
     Extension(db): Extension<DB>,
     Path((w_id, path)): Path<(String, StripPath)>,
-    Json(payload): Json<serde_json::Value>,
+    Query(run_query): Query<IncludeHeaderQuery>,
+    headers: HeaderMap,
+    Json(args): Json<Option<serde_json::Map<String, serde_json::Value>>>,
 ) -> Result<StatusCode> {
     let mut tx = db.begin().await?;
 
+    let args = add_include_headers(&run_query.include_header, headers, args.unwrap_or_default());
     sqlx::query!(
         "
        UPDATE capture
@@ -99,7 +109,7 @@ pub async fn update_payload(
         ",
         &w_id,
         &path.to_path(),
-        &payload,
+        serde_json::json!(args),
     )
     .execute(&mut tx)
     .await?;

backend/windmill-api/src/flows.rs

@@ -7,12 +7,11 @@
  */
 
 use hyper::StatusCode;
-use reqwest::Client;
 use sql_builder::prelude::*;
 
 use axum::{
     extract::{Extension, Path, Query},
-    routing::{get, post},
+    routing::{delete, get, post},
     Json, Router,
 };
 use sql_builder::SqlBuilder;
@@ -33,6 +32,7 @@ use crate::{
     schedule::clear_schedule,
     users::{require_owner_of_path, Authed},
     webhook_util::{WebhookMessage, WebhookShared},
+    HTTP_CLIENT,
 };
 
 pub fn workspaced_service() -> Router {
@@ -41,6 +41,7 @@ pub fn workspaced_service() -> Router {
         .route("/create", post(create_flow))
         .route("/update/*path", post(update_flow))
         .route("/archive/*path", post(archive_flow_by_path))
+        .route("/delete/*path", delete(delete_flow_by_path))
         .route("/get/*path", get(get_flow_by_path))
         .route("/exists/*path", get(exists_flow_by_path))
         .route("/list_paths", get(list_paths))
@@ -81,7 +82,7 @@ async fn list_flows(
         )
         .order_desc("favorite.path IS NOT NULL")
         .order_by("edited_at", lq.order_desc.unwrap_or(true))
-        .and_where("o.workspace_id = ? OR o.workspace_id = 'starter'".bind(&w_id))
+        .and_where("o.workspace_id = ?".bind(&w_id))
         .offset(offset)
         .limit(per_page)
         .clone();
@@ -111,12 +112,9 @@ async fn list_flows(
     Ok(Json(rows))
 }
 
-async fn list_hub_flows(
-    Authed { email, .. }: Authed,
-    Extension(http_client): Extension<Client>,
-) -> JsonResult<serde_json::Value> {
+async fn list_hub_flows(Authed { email, .. }: Authed) -> JsonResult<serde_json::Value> {
     let flows = list_elems_from_hub(
-        http_client,
+        &HTTP_CLIENT,
         "https://hub.windmill.dev/searchFlowData?approved=true",
         &email,
     )
@@ -145,10 +143,9 @@ async fn list_paths(
 pub async fn get_hub_flow_by_id(
     Authed { email, .. }: Authed,
     Path(id): Path<i32>,
-    Extension(http_client): Extension<Client>,
 ) -> JsonResult<serde_json::Value> {
     let value = http_get_from_hub(
-        http_client,
+        &HTTP_CLIENT,
         &format!("https://hub.windmill.dev/flows/{id}/json"),
         &email,
         false,
@@ -240,6 +237,7 @@ async fn create_flow(
         None,
         None,
         None,
+        None,
         false,
         false,
         None,
@@ -396,6 +394,7 @@ async fn update_flow(
         None,
         None,
         None,
+        None,
         false,
         false,
         None,
@@ -430,13 +429,12 @@ async fn get_flow_by_path(
     let path = path.to_path();
     let mut tx = user_db.begin(&authed).await?;
 
-    let flow_o = sqlx::query_as::<_, Flow>(
-        "SELECT * FROM flow WHERE path = $1 AND (workspace_id = $2 OR workspace_id = 'starter')",
-    )
-    .bind(path)
-    .bind(w_id)
-    .fetch_optional(&mut tx)
-    .await?;
+    let flow_o =
+        sqlx::query_as::<_, Flow>("SELECT * FROM flow WHERE path = $1 AND workspace_id = $2")
+            .bind(path)
+            .bind(w_id)
+            .fetch_optional(&mut tx)
+            .await?;
     tx.commit().await?;
 
     let flow = not_found_if_none(flow_o, "Flow", path)?;
@@ -450,8 +448,7 @@ async fn exists_flow_by_path(
     let path = path.to_path();
 
     let exists = sqlx::query_scalar!(
-        "SELECT EXISTS(SELECT 1 FROM flow WHERE path = $1 AND (workspace_id = $2 OR workspace_id \
-         = 'starter'))",
+        "SELECT EXISTS(SELECT 1 FROM flow WHERE path = $1 AND workspace_id = $2)",
         path,
         w_id
     )
@@ -498,6 +495,42 @@ async fn archive_flow_by_path(
     Ok(format!("Flow {path} archived"))
 }
 
+async fn delete_flow_by_path(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    Extension(webhook): Extension<WebhookShared>,
+    Path((w_id, path)): Path<(String, StripPath)>,
+) -> Result<String> {
+    let path = path.to_path();
+    let mut tx = user_db.begin(&authed).await?;
+
+    sqlx::query!(
+        "DELETE FROM flow WHERE path = $1 AND workspace_id = $2",
+        path,
+        &w_id
+    )
+    .execute(&mut tx)
+    .await?;
+
+    audit_log(
+        &mut tx,
+        &authed.username,
+        "flows.delete",
+        ActionKind::Delete,
+        &w_id,
+        Some(path),
+        Some([("workspace", w_id.as_str())].into()),
+    )
+    .await?;
+    tx.commit().await?;
+    webhook.send_message(
+        w_id.clone(),
+        WebhookMessage::DeleteFlow { workspace: w_id, path: path.to_owned() },
+    );
+
+    Ok(format!("Flow {path} deleted"))
+}
+
 #[cfg(test)]
 mod tests {
 
@@ -607,8 +640,6 @@ mod tests {
                 "type": "script",
                 "path": "test"
               },
-              "stop_after_if": null,
-              "summary": null
             },
             {
               "id": "b",
@@ -617,15 +648,12 @@ mod tests {
                 "input_transforms": {},
                 "type": "rawscript",
                 "content": "test",
-                "lock": null,
-                "path": null,
                 "language": "deno"
               },
               "stop_after_if": {
                   "expr": "foo = 'bar'",
                   "skip_if_stopped": false
-              },
-              "summary": null
+              }
             },
             {
               "id": "c",
@@ -647,8 +675,7 @@ mod tests {
               "stop_after_if": {
                   "expr": "previous.isEmpty()",
                   "skip_if_stopped": false,
-              },
-              "summary": null
+              }
             }
           ],
           "failure_module": {
@@ -662,8 +689,7 @@ mod tests {
             "stop_after_if": {
                 "expr": "previous.isEmpty()",
                 "skip_if_stopped": false
-            },
-            "summary": null
+            }
           }
         });
         assert_eq!(dbg!(serde_json::json!(fv)), dbg!(expect));

backend/windmill-api/src/folders.rs

@@ -16,10 +16,9 @@ use axum::{
     routing::{delete, get, post},
     Json, Router,
 };
-use itertools::Itertools;
 use windmill_audit::{audit_log, ActionKind};
 use windmill_common::{
-    error::{self, Error, JsonResult, Result},
+    error::{self, to_anyhow, Error, JsonResult, Result},
     users::username_to_permissioned_as,
     utils::{not_found_if_none, paginate, Pagination},
 };
@@ -262,13 +261,26 @@ async fn update_folder(
     sqlb.and_where_eq("workspace_id", "?".bind(&w_id));
 
     if let Some(display_name) = ng.display_name {
-        sqlb.set("display_name", display_name);
+        sqlb.set("display_name", "?".bind(&display_name));
     }
     if let Some(owners) = ng.owners {
-        sqlb.set_str("owners", format!("{{{}}}", owners.into_iter().join(",")));
+        sqlb.set(
+            "owners",
+            "?".bind(&format!(
+                "{{{}}}",
+                owners
+                    .iter()
+                    .map(|x| format!("\"{x}\""))
+                    .collect::<Vec<_>>()
+                    .join(","),
+            )),
+        );
     }
     if let Some(extra_perms) = ng.extra_perms {
-        sqlb.set_str("extra_perms", extra_perms.to_string());
+        sqlb.set(
+            "extra_perms",
+            "?".bind(&serde_json::to_string(&extra_perms).map_err(to_anyhow)?),
+        );
     }
 
     sqlb.returning("*");

backend/windmill-api/src/jobs.rs

@@ -6,8 +6,6 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
-use std::sync::Arc;
-
 use anyhow::Context;
 use axum::{
     extract::{FromRequest, Json, Path, Query},
@@ -38,7 +36,7 @@ use crate::{
     db::{UserDB, DB},
     users::{require_owner_of_path, Authed},
     variables::get_workspace_key,
-    BaseUrl, QueueLimitWaitResult, TimeoutWaitResult,
+    BASE_URL,
 };
 
 pub fn workspaced_service() -> Router {
@@ -104,10 +102,9 @@ pub fn global_service() -> Router {
 
 async fn get_result_by_id(
     Extension(db): Extension<DB>,
-    Query(ResultByIdQuery { skip_direct }): Query<ResultByIdQuery>,
-    Path((w_id, flow_id, node_id)): Path<(String, String, String)>,
+    Path((w_id, flow_id, node_id)): Path<(String, Uuid, String)>,
 ) -> windmill_common::error::JsonResult<serde_json::Value> {
-    let res = windmill_queue::get_result_by_id(db, skip_direct, w_id, flow_id, node_id).await?;
+    let res = windmill_queue::get_result_by_id(db, w_id, flow_id, node_id).await?;
     Ok(Json(res))
 }
 
@@ -154,8 +151,7 @@ pub async fn get_path_for_hash<'c>(
     hash: i64,
 ) -> error::Result<String> {
     let path = sqlx::query_scalar!(
-        "select path from script where hash = $1 AND (workspace_id = $2 OR workspace_id = \
-         'starter')",
+        "select path from script where hash = $1 AND workspace_id = $2",
         hash,
         w_id
     )
@@ -180,11 +176,6 @@ async fn get_job(
     Ok(Json(job))
 }
 
-#[derive(Deserialize)]
-pub struct ResultByIdQuery {
-    pub skip_direct: bool,
-}
-
 pub async fn get_job_by_id<'c>(
     mut tx: Transaction<'c, Postgres>,
     w_id: &str,
@@ -298,41 +289,53 @@ impl RunJobQuery {
     fn add_include_headers(
         &self,
         headers: HeaderMap,
-        mut args: serde_json::Map<String, serde_json::Value>,
+        args: serde_json::Map<String, serde_json::Value>,
     ) -> serde_json::Map<String, serde_json::Value> {
-        let whitelist = self
-            .include_header
-            .as_ref()
-            .map(|s| s.split(",").map(|s| s.to_string()).collect::<Vec<_>>())
-            .unwrap_or_default();
-        whitelist
-            .iter()
-            .chain(INCLUDE_HEADERS.iter())
-            .for_each(|h| {
-                if let Some(v) = headers.get(h) {
-                    args.insert(
-                        h.to_string().to_lowercase().replace('-', "_"),
-                        serde_json::Value::String(v.to_str().unwrap().to_string()),
-                    );
-                }
-            });
-        args
+        return add_include_headers(&self.include_header, headers, args);
     }
 }
 
+pub fn add_include_headers(
+    include_header: &Option<String>,
+    headers: HeaderMap,
+    mut args: serde_json::Map<String, serde_json::Value>,
+) -> serde_json::Map<String, serde_json::Value> {
+    if include_header.is_none() {
+        return args;
+    }
+    let whitelist = include_header
+        .as_ref()
+        .map(|s| s.split(",").map(|s| s.to_string()).collect::<Vec<_>>())
+        .unwrap_or_default();
+
+    whitelist
+        .iter()
+        .chain(INCLUDE_HEADERS.iter())
+        .for_each(|h| {
+            if let Some(v) = headers.get(h) {
+                args.insert(
+                    h.to_string().to_lowercase().replace('-', "_"),
+                    serde_json::Value::String(v.to_str().unwrap().to_string()),
+                );
+            }
+        });
+    args
+}
 #[derive(Deserialize)]
 pub struct ListQueueQuery {
     pub script_path_start: Option<String>,
     pub script_path_exact: Option<String>,
     pub script_hash: Option<String>,
     pub created_by: Option<String>,
-    pub created_before: Option<chrono::DateTime<chrono::Utc>>,
-    pub created_after: Option<chrono::DateTime<chrono::Utc>>,
+    pub started_before: Option<chrono::DateTime<chrono::Utc>>,
+    pub started_after: Option<chrono::DateTime<chrono::Utc>>,
     pub running: Option<bool>,
     pub parent_job: Option<String>,
     pub order_desc: Option<bool>,
     pub job_kinds: Option<String>,
     pub suspended: Option<bool>,
+    // filter by matching a subset of the args using base64 encoded json subset
+    pub args: Option<String>,
 }
 
 fn list_queue_jobs_query(w_id: &str, lq: &ListQueueQuery, fields: &[&str]) -> SqlBuilder {
@@ -361,11 +364,11 @@ fn list_queue_jobs_query(w_id: &str, lq: &ListQueueQuery, fields: &[&str]) -> Sq
     if let Some(pj) = &lq.parent_job {
         sqlb.and_where_eq("parent_job", "?".bind(pj));
     }
-    if let Some(dt) = &lq.created_before {
-        sqlb.and_where_lt("created_at", format!("to_timestamp({})", dt.timestamp()));
+    if let Some(dt) = &lq.started_before {
+        sqlb.and_where_le("started_at", format!("to_timestamp({})", dt.timestamp()));
     }
-    if let Some(dt) = &lq.created_after {
-        sqlb.and_where_gt("created_at", format!("to_timestamp({})", dt.timestamp()));
+    if let Some(dt) = &lq.started_after {
+        sqlb.and_where_ge("started_at", format!("to_timestamp({})", dt.timestamp()));
     }
 
     if let Some(s) = &lq.suspended {
@@ -375,6 +378,7 @@ fn list_queue_jobs_query(w_id: &str, lq: &ListQueueQuery, fields: &[&str]) -> Sq
             sqlb.and_where_eq("suspend", 0);
         }
     }
+
     if let Some(jk) = &lq.job_kinds {
         sqlb.and_where_in(
             "job_kind",
@@ -382,6 +386,10 @@ fn list_queue_jobs_query(w_id: &str, lq: &ListQueueQuery, fields: &[&str]) -> Sq
         );
     }
 
+    if let Some(args) = &lq.args {
+        sqlb.and_where("args @> ?".bind(&args.replace("'", "''")));
+    }
+
     sqlb
 }
 
@@ -454,13 +462,14 @@ async fn list_jobs(
             script_path_exact: lq.script_path_exact,
             script_hash: lq.script_hash,
             created_by: lq.created_by,
-            created_before: lq.created_before,
-            created_after: lq.created_after,
+            started_before: lq.started_before,
+            started_after: lq.started_after,
             running: None,
             parent_job: lq.parent_job,
             order_desc: Some(true),
             job_kinds: lq.job_kinds,
             suspended: lq.suspended,
+            args: lq.args,
         },
         &[
             "'QueuedJob' as typ",
@@ -925,16 +934,16 @@ pub async fn get_resume_urls(
     Extension(user_db): Extension<UserDB>,
     Path((w_id, job_id, resume_id)): Path<(String, Uuid, u32)>,
     Query(approver): Query<QueryApprover>,
-    Extension(base_url): Extension<Arc<BaseUrl>>,
 ) -> error::JsonResult<ResumeUrls> {
     let key = get_workspace_key(&w_id, &mut user_db.begin(&authed).await?).await?;
     let signature = create_signature(key, job_id, resume_id, approver.approver.clone())?;
-    let base_url = base_url.0.clone();
     let approver = approver
         .approver
         .as_ref()
         .map(|x| format!("?approver={}", encode(x)))
         .unwrap_or_else(String::new);
+
+    let base_url = BASE_URL.as_str();
     let res = ResumeUrls {
         approvalPage: format!(
             "{base_url}/approve/{w_id}/{job_id}/{resume_id}/{signature}{approver}"
@@ -1071,6 +1080,8 @@ impl From<UnifiedJob> for Job {
                 visible_to_owner: uj.visible_to_owner,
                 suspend: uj.suspend,
                 mem_peak: uj.mem_peak,
+                root_job: None,
+                leaf_jobs: None,
             }),
             t => panic!("job type {} not valid", t),
         }
@@ -1133,15 +1144,15 @@ where
         struct InPayload {
             payload: Option<String>,
         }
-
-        fn decode_payload<D: DeserializeOwned, T: AsRef<[u8]>>(t: T) -> anyhow::Result<D> {
-            let vec = base64::engine::general_purpose::URL_SAFE
-                .decode(t)
-                .context("invalid base64")?;
-            serde_json::from_slice(vec.as_slice()).context("invalid json")
-        }
     }
 }
+
+fn decode_payload<D: DeserializeOwned, T: AsRef<[u8]>>(t: T) -> anyhow::Result<D> {
+    let vec = base64::engine::general_purpose::URL_SAFE
+        .decode(t)
+        .context("invalid base64")?;
+    serde_json::from_slice(vec.as_slice()).context("invalid json")
+}
 pub async fn run_flow_by_path(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
@@ -1166,6 +1177,7 @@ pub async fn run_flow_by_path(
         scheduled_for,
         None,
         run_query.parent_job,
+        run_query.parent_job,
         false,
         false,
         None,
@@ -1201,6 +1213,7 @@ pub async fn run_job_by_path(
         scheduled_for,
         None,
         run_query.parent_job,
+        run_query.parent_job,
         false,
         false,
         None,
@@ -1315,18 +1328,26 @@ pub async fn check_queue_too_long(db: DB, queue_limit: Option<i64>) -> error::Re
     }
     Ok(())
 }
+
+lazy_static::lazy_static! {
+    pub static ref QUEUE_LIMIT_WAIT_RESULT: Option<i64> = std::env::var("QUEUE_LIMIT_WAIT_RESULT")
+        .ok()
+        .and_then(|x| x.parse().ok());
+    pub static ref TIMEOUT_WAIT_RESULT: i32 = std::env::var("TIMEOUT_WAIT_RESULT")
+        .ok()
+        .and_then(|x| x.parse().ok())
+        .unwrap_or(20);
+}
 pub async fn run_wait_result_job_by_path(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
     Extension(db): Extension<DB>,
-    Extension(timeout): Extension<Arc<TimeoutWaitResult>>,
-    Extension(queue_limit): Extension<Arc<QueueLimitWaitResult>>,
     Path((w_id, script_path)): Path<(String, StripPath)>,
     Query(run_query): Query<RunJobQuery>,
     headers: HeaderMap,
     Json(args): Json<Option<serde_json::Map<String, serde_json::Value>>>,
 ) -> error::JsonResult<serde_json::Value> {
-    check_queue_too_long(db, queue_limit.0.or(run_query.queue_limit)).await?;
+    check_queue_too_long(db, QUEUE_LIMIT_WAIT_RESULT.or(run_query.queue_limit)).await?;
     let script_path = script_path.to_path();
     let mut tx = user_db.clone().begin(&authed).await?;
     let job_payload = script_path_to_payload(script_path, &mut tx, &w_id).await?;
@@ -1345,6 +1366,7 @@ pub async fn run_wait_result_job_by_path(
         scheduled_for,
         None,
         run_query.parent_job,
+        run_query.parent_job,
         false,
         false,
         None,
@@ -1356,7 +1378,7 @@ pub async fn run_wait_result_job_by_path(
     run_wait_result(
         authed,
         Extension(user_db),
-        timeout.0,
+        *TIMEOUT_WAIT_RESULT,
         uuid,
         Path((w_id, script_path)),
     )
@@ -1367,7 +1389,6 @@ pub async fn run_wait_result_job_by_hash(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
     Extension(db): Extension<DB>,
-    Extension(timeout): Extension<Arc<TimeoutWaitResult>>,
     Path((w_id, script_hash)): Path<(String, ScriptHash)>,
     Query(run_query): Query<RunJobQuery>,
     headers: HeaderMap,
@@ -1392,6 +1413,7 @@ pub async fn run_wait_result_job_by_hash(
         scheduled_for,
         None,
         run_query.parent_job,
+        run_query.parent_job,
         false,
         false,
         None,
@@ -1403,7 +1425,7 @@ pub async fn run_wait_result_job_by_hash(
     run_wait_result(
         authed,
         Extension(user_db),
-        timeout.0,
+        *TIMEOUT_WAIT_RESULT,
         uuid,
         Path((w_id, script_hash)),
     )
@@ -1414,7 +1436,6 @@ pub async fn run_wait_result_flow_by_path(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
     Extension(db): Extension<DB>,
-    Extension(timeout): Extension<Arc<TimeoutWaitResult>>,
     Path((w_id, flow_path)): Path<(String, StripPath)>,
     Query(run_query): Query<RunJobQuery>,
     headers: HeaderMap,
@@ -1438,6 +1459,7 @@ pub async fn run_wait_result_flow_by_path(
         scheduled_for,
         None,
         run_query.parent_job,
+        run_query.parent_job,
         false,
         false,
         None,
@@ -1450,7 +1472,7 @@ pub async fn run_wait_result_flow_by_path(
     run_wait_result(
         authed,
         Extension(user_db),
-        timeout.0,
+        *TIMEOUT_WAIT_RESULT,
         uuid,
         Path((w_id, flow_path)),
     )
@@ -1500,6 +1522,7 @@ async fn run_preview_job(
         scheduled_for,
         None,
         None,
+        None,
         false,
         false,
         None,
@@ -1533,6 +1556,7 @@ async fn run_preview_flow_job(
         scheduled_for,
         None,
         None,
+        None,
         false,
         false,
         None,
@@ -1568,6 +1592,7 @@ pub async fn run_job_by_hash(
         scheduled_for,
         None,
         run_query.parent_job,
+        run_query.parent_job,
         false,
         false,
         None,
@@ -1670,11 +1695,11 @@ fn list_completed_jobs_query(
     if let Some(pj) = &lq.parent_job {
         sqlb.and_where_eq("parent_job", "?".bind(pj));
     }
-    if let Some(dt) = &lq.created_before {
-        sqlb.and_where_lt("created_at", format!("to_timestamp({})", dt.timestamp()));
+    if let Some(dt) = &lq.started_before {
+        sqlb.and_where_le("started_at", format!("to_timestamp({})", dt.timestamp()));
     }
-    if let Some(dt) = &lq.created_after {
-        sqlb.and_where_gt("created_at", format!("to_timestamp({})", dt.timestamp()));
+    if let Some(dt) = &lq.started_after {
+        sqlb.and_where_ge("started_at", format!("to_timestamp({})", dt.timestamp()));
     }
     if let Some(sk) = &lq.is_skipped {
         sqlb.and_where_eq("is_skipped", sk);
@@ -1689,6 +1714,14 @@ fn list_completed_jobs_query(
         );
     }
 
+    if let Some(args) = &lq.args {
+        sqlb.and_where("args @> ?".bind(&args.replace("'", "''")));
+    }
+
+    if let Some(result) = &lq.result {
+        sqlb.and_where("result @> ?".bind(&result.replace("'", "''")));
+    }
+
     sqlb
 }
 #[derive(Deserialize, Clone)]
@@ -1697,8 +1730,8 @@ pub struct ListCompletedQuery {
     pub script_path_exact: Option<String>,
     pub script_hash: Option<String>,
     pub created_by: Option<String>,
-    pub created_before: Option<chrono::DateTime<chrono::Utc>>,
-    pub created_after: Option<chrono::DateTime<chrono::Utc>>,
+    pub started_before: Option<chrono::DateTime<chrono::Utc>>,
+    pub started_after: Option<chrono::DateTime<chrono::Utc>>,
     pub success: Option<bool>,
     pub parent_job: Option<String>,
     pub order_desc: Option<bool>,
@@ -1706,6 +1739,10 @@ pub struct ListCompletedQuery {
     pub is_skipped: Option<bool>,
     pub is_flow_step: Option<bool>,
     pub suspended: Option<bool>,
+    // filter by matching a subset of the args using base64 encoded json subset
+    pub args: Option<String>,
+    // filter by matching a subset of the result using base64 encoded json subset
+    pub result: Option<String>,
 }
 
 async fn list_completed_jobs(
@@ -1772,6 +1809,7 @@ async fn get_completed_job(
     .fetch_optional(&db)
     .await?;
 
+    tracing::info!("job_o: {:?}", job_o);
     let job = not_found_if_none(job_o, "Completed Job", id.to_string())?;
     Ok(Json(job))
 }

backend/windmill-api/src/lib.rs

@@ -6,15 +6,17 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
+use crate::oauth2::AllClients;
 use argon2::Argon2;
 use axum::{middleware::from_extractor, routing::get, Extension, Router};
 use db::DB;
 use git_version::git_version;
+use reqwest::Client;
 use std::{net::SocketAddr, sync::Arc};
 use tower::ServiceBuilder;
 use tower_cookies::CookieManagerLayer;
 use tower_http::trace::TraceLayer;
-use windmill_common::{error::to_anyhow, utils::rd_string};
+use windmill_common::utils::rd_string;
 
 use crate::{
     db::UserDB,
@@ -50,20 +52,32 @@ mod workspaces;
 pub const GIT_VERSION: &str =
     git_version!(args = ["--tag", "--always"], fallback = "unknown-version");
 
-pub struct BaseUrl(String);
-pub struct IsSecure(bool);
-pub struct CookieDomain(Option<String>);
-pub struct CloudHosted(bool);
-pub struct ContentSecurityPolicy(String);
-pub struct TimeoutWaitResult(i32);
-pub struct QueueLimitWaitResult(Option<i64>);
-
 pub use users::delete_expired_items_perdiodically;
 
+lazy_static::lazy_static! {
+    pub static ref BASE_URL: String = std::env::var("BASE_URL").unwrap_or_else(|_| "http://localhost".to_string());
+
+
+    pub static ref COOKIE_DOMAIN: Option<String> = std::env::var("COOKIE_DOMAIN").ok();
+
+    pub static ref SLACK_SIGNING_SECRET: Option<SlackVerifier> = std::env::var("SLACK_SIGNING_SECRET")
+        .ok()
+        .map(|x| SlackVerifier::new(x).unwrap());
+
+        static ref IS_SECURE: bool = BASE_URL.starts_with("https://");
+
+    pub static ref HTTP_CLIENT: Client = reqwest::ClientBuilder::new()
+        .user_agent("windmill/beta")
+        .build().unwrap();
+
+    pub static ref OAUTH_CLIENTS: AllClients = build_oauth_clients(&BASE_URL)
+        .map_err(|e| tracing::error!("Error building oauth clients: {}", e))
+        .unwrap();
+}
+
 pub async fn run_server(
     db: DB,
     addr: SocketAddr,
-    base_url: String,
     mut rx: tokio::sync::broadcast::Receiver<()>,
 ) -> anyhow::Result<()> {
     let user_db = UserDB::new(db.clone());
@@ -73,16 +87,7 @@ pub async fn run_server(
         std::env::var("SUPERADMIN_SECRET").ok(),
     ));
     let argon2 = Arc::new(Argon2::default());
-    let basic_clients = Arc::new(build_oauth_clients(&base_url).await?);
-    let slack_verifier = Arc::new(
-        std::env::var("SLACK_SIGNING_SECRET")
-            .ok()
-            .map(|x| SlackVerifier::new(x).unwrap()),
-    );
-    let http_client = reqwest::ClientBuilder::new()
-        .user_agent("windmill/beta")
-        .build()
-        .map_err(to_anyhow)?;
+
     let middleware_stack = ServiceBuilder::new()
         .layer(
             TraceLayer::new_for_http()
@@ -93,21 +98,6 @@ pub async fn run_server(
         .layer(Extension(db.clone()))
         .layer(Extension(user_db))
         .layer(Extension(auth_cache.clone()))
-        .layer(Extension(basic_clients))
-        .layer(Extension(Arc::new(BaseUrl(base_url.to_string()))))
-        .layer(Extension(Arc::new(ContentSecurityPolicy(
-            std::env::var("SERVE_CSP").unwrap_or("".to_owned()),
-        ))))
-        .layer(Extension(Arc::new(CloudHosted(
-            std::env::var("CLOUD_HOSTED").is_ok(),
-        ))))
-        .layer(Extension(Arc::new(IsSecure(
-            base_url.starts_with("https://"),
-        ))))
-        .layer(Extension(Arc::new(CookieDomain(
-            std::env::var("COOKIE_DOMAIN").ok(),
-        ))))
-        .layer(Extension(http_client))
         .layer(CookieManagerLayer::new())
         .layer(Extension(WebhookShared::new(rx.resubscribe(), db.clone())));
     // build our application with a route
@@ -119,21 +109,7 @@ pub async fn run_server(
                     "/w/:workspace_id",
                     Router::new()
                         .nest("/scripts", scripts::workspaced_service())
-                        .nest(
-                            "/jobs",
-                            jobs::workspaced_service()
-                                .layer(Extension(Arc::new(TimeoutWaitResult(
-                                    std::env::var("TIMEOUT_WAIT_RESULT")
-                                        .ok()
-                                        .and_then(|x| x.parse().ok())
-                                        .unwrap_or(20),
-                                ))))
-                                .layer(Extension(Arc::new(QueueLimitWaitResult(
-                                    std::env::var("QUEUE_LIMIT_WAIT_RESULT")
-                                        .ok()
-                                        .and_then(|x| x.parse().ok()),
-                                )))),
-                        )
+                        .nest("/jobs", jobs::workspaced_service())
                         .nest(
                             "/users",
                             users::workspaced_service().layer(Extension(argon2.clone())),
@@ -174,10 +150,7 @@ pub async fn run_server(
                     "/auth",
                     users::make_unauthed_service().layer(Extension(argon2)),
                 )
-                .nest(
-                    "/oauth",
-                    oauth2::global_service().layer(Extension(slack_verifier)),
-                )
+                .nest("/oauth", oauth2::global_service())
                 .route("/version", get(git_v))
                 .route("/openapi.yaml", get(openapi)),
         )

backend/windmill-api/src/main.rs

@@ -1,70 +0,0 @@
-/*
- * Author: Ruben Fiszel
- * Copyright: Windmill Labs, Inc 2022
- * This file and its contents are licensed under the AGPLv3 License.
- * Please see the included NOTICE for copyright information and
- * LICENSE-AGPL for a copy of the license.
- */
-
-use std::net::SocketAddr;
-
-use anyhow::Ok;
-
-#[tokio::main]
-async fn main() -> anyhow::Result<()> {
-    windmill_common::tracing_init::initialize_tracing();
-
-    let db = windmill_common::connect_db(true).await?;
-
-    let num_workers = std::env::var("NUM_WORKERS")
-        .ok()
-        .and_then(|x| x.parse::<i32>().ok())
-        .unwrap_or(windmill_common::DEFAULT_NUM_WORKERS as i32);
-
-    let metrics_addr: Option<SocketAddr> = std::env::var("METRICS_ADDR")
-        .ok()
-        .map(|s| {
-            s.parse::<bool>()
-                .map(|b| b.then(|| SocketAddr::from(([0, 0, 0, 0], 8001))))
-                .or_else(|_| s.parse::<SocketAddr>().map(Some))
-        })
-        .transpose()?
-        .flatten();
-
-    let server_mode = !std::env::var("DISABLE_SERVER")
-        .ok()
-        .and_then(|x| x.parse::<bool>().ok())
-        .unwrap_or(false);
-
-    if server_mode {
-        windmill_api::migrate_db(&db).await?;
-    }
-
-    let (tx, rx) = tokio::sync::broadcast::channel::<()>(3);
-    let shutdown_signal = windmill_common::shutdown_signal(tx);
-
-    let base_url = std::env::var("BASE_URL").unwrap_or_else(|_| "http://localhost".to_string());
-
-    if server_mode || num_workers > 0 {
-        let addr = SocketAddr::from(([0, 0, 0, 0], 8000));
-
-        let server_f = async {
-            if server_mode {
-                windmill_api::run_server(db.clone(), addr, base_url, rx.resubscribe()).await?;
-            }
-            Ok(()) as anyhow::Result<()>
-        };
-
-        let metrics_f = async {
-            match metrics_addr {
-                Some(addr) => windmill_common::serve_metrics(addr, rx.resubscribe())
-                    .await
-                    .map_err(anyhow::Error::from),
-                None => Ok(()),
-            }
-        };
-
-        futures::try_join!(shutdown_signal, server_f, metrics_f)?;
-    }
-    Ok(())
-}

backend/windmill-api/src/main3.rs

@@ -0,0 +1,69 @@
+// /*
+//  * Author: Ruben Fiszel
+//  * Copyright: Windmill Labs, Inc 2022
+//  * This file and its contents are licensed under the AGPLv3 License.
+//  * Please see the included NOTICE for copyright information and
+//  * LICENSE-AGPL for a copy of the license.
+//  */
+// use std::net::SocketAddr;
+
+// use anyhow::Ok;
+
+// pub const DEFAULT_NUM_WORKERS: usize = 3;
+
+// #[tokio::main]
+// async fn main() -> anyhow::Result<()> {
+//     windmill_common::tracing_init::initialize_tracing();
+
+//     let db = windmill_common::connect_db(true).await?;
+
+//     let num_workers = std::env::var("NUM_WORKERS")
+//         .ok()
+//         .and_then(|x| x.parse::<i32>().ok())
+//         .unwrap_or(DEFAULT_NUM_WORKERS as i32);
+
+//     let metrics_addr: Option<SocketAddr> = std::env::var("METRICS_ADDR")
+//         .ok()
+//         .map(|s| {
+//             s.parse::<bool>()
+//                 .map(|b| b.then(|| SocketAddr::from(([0, 0, 0, 0], 8001))))
+//                 .or_else(|_| s.parse::<SocketAddr>().map(Some))
+//         })
+//         .transpose()?
+//         .flatten();
+
+//     let server_mode = !std::env::var("DISABLE_SERVER")
+//         .ok()
+//         .and_then(|x| x.parse::<bool>().ok())
+//         .unwrap_or(false);
+
+//     if server_mode {
+//         windmill_api::migrate_db(&db).await?;
+//     }
+
+//     let (tx, rx) = tokio::sync::broadcast::channel::<()>(3);
+//     let shutdown_signal = windmill_common::shutdown_signal(tx);
+
+//     if server_mode || num_workers > 0 {
+//         let addr = SocketAddr::from(([0, 0, 0, 0], 8000));
+
+//         let server_f = async {
+//             if server_mode {
+//                 windmill_api::run_server(db.clone(), addr, rx.resubscribe()).await?;
+//             }
+//             Ok(()) as anyhow::Result<()>
+//         };
+
+//         let metrics_f = async {
+//             match metrics_addr {
+//                 Some(addr) => windmill_common::serve_metrics(addr, rx.resubscribe())
+//                     .await
+//                     .map_err(anyhow::Error::from),
+//                 None => Ok(()),
+//             }
+//         };
+
+//         futures::try_join!(shutdown_signal, server_f, metrics_f)?;
+//     }
+//     Ok(())
+// }

backend/windmill-api/src/oauth2.rs

@@ -8,8 +8,6 @@
 
 use std::{collections::HashMap, fmt::Debug};
 
-use std::sync::Arc;
-
 use anyhow::Context;
 use axum::extract::FromRequestParts;
 use axum::http::request::Parts;
@@ -29,26 +27,25 @@ use oauth2::{Client as OClient, *};
 use reqwest::Client;
 use serde::{de::DeserializeOwned, Deserialize, Serialize};
 use sqlx::{Postgres, Transaction};
-use tokio::{fs::File, io::AsyncReadExt};
 use tower_cookies::{Cookie, Cookies};
 use windmill_audit::{audit_log, ActionKind};
+use windmill_common::users::username_to_permissioned_as;
 use windmill_common::utils::{not_found_if_none, now_from_db};
 
-use crate::users::{truncate_token, Authed};
+use crate::users::{truncate_token, Authed, NEW_USER_WEBHOOK};
 use crate::workspaces::invite_user_to_all_auto_invite_worspaces;
 use crate::{
     db::{UserDB, DB},
     variables::{build_crypt, encrypt},
     workspaces::WorkspaceSettings,
-    BaseUrl,
 };
-use crate::{CookieDomain, IsSecure};
+use crate::{BASE_URL, HTTP_CLIENT, IS_SECURE, OAUTH_CLIENTS, SLACK_SIGNING_SECRET};
 use windmill_common::error::{self, to_anyhow, Error};
 use windmill_common::oauth2::*;
 
 use windmill_queue::JobPayload;
 
-use std::str;
+use std::{fs, str};
 
 pub fn global_service() -> Router {
     Router::new()
@@ -111,7 +108,7 @@ pub struct AllClients {
     pub slack: Option<OClient>,
 }
 
-pub async fn build_oauth_clients(base_url: &str) -> anyhow::Result<AllClients> {
+pub fn build_oauth_clients(base_url: &str) -> anyhow::Result<AllClients> {
     let connect_configs = serde_json::from_str::<HashMap<String, OAuthConfig>>(include_str!(
         "../../oauth_connect.json"
     ))?;
@@ -119,14 +116,12 @@ pub async fn build_oauth_clients(base_url: &str) -> anyhow::Result<AllClients> {
         "../../oauth_login.json"
     ))?;
 
-    let mut content = String::new();
     let path = "./oauth.json";
-    if std::path::Path::new(path).exists() {
-        let mut file = File::open(path).await?;
-        file.read_to_string(&mut content).await?;
+    let content = if std::path::Path::new(path).exists() {
+        fs::read_to_string(path).map_err(to_anyhow)?
     } else {
-        content.push_str("{}");
-    }
+        "{}".to_string()
+    };
 
     let oauths: HashMap<String, OAuthClient> =
         match serde_json::from_str::<HashMap<String, OAuthClient>>(&content) {
@@ -288,12 +283,10 @@ pub struct SlackBotToken {
 async fn connect(
     Path(client_name): Path<String>,
     Query(query): Query<HashMap<String, String>>,
-    Extension(clients): Extension<Arc<AllClients>>,
-    Extension(is_secure): Extension<Arc<IsSecure>>,
     cookies: Cookies,
 ) -> error::Result<Redirect> {
     let mut query = query.clone();
-    let connects = &clients.connects;
+    let connects = &OAUTH_CLIENTS.connects;
     let scopes = query
         .get("scopes")
         .map(|x| x.split('+').map(|x| x.to_owned()).collect());
@@ -309,7 +302,7 @@ async fn connect(
         cookies,
         scopes,
         extra_params,
-        is_secure.0,
+        *IS_SECURE,
     )
 }
 
@@ -376,11 +369,9 @@ async fn delete_account(
     Ok(format!("Deleted account id {id}"))
 }
 
-async fn list_logins(
-    Extension(clients): Extension<Arc<AllClients>>,
-) -> error::JsonResult<Vec<String>> {
+async fn list_logins() -> error::JsonResult<Vec<String>> {
     Ok(Json(
-        clients
+        OAUTH_CLIENTS
             .logins
             .keys()
             .map(|x| x.to_owned())
@@ -393,11 +384,9 @@ struct ScopesAndParams {
     scopes: Vec<String>,
     extra_params: Option<HashMap<String, String>>,
 }
-async fn list_connects(
-    Extension(clients): Extension<Arc<AllClients>>,
-) -> error::JsonResult<HashMap<String, ScopesAndParams>> {
+async fn list_connects() -> error::JsonResult<HashMap<String, ScopesAndParams>> {
     Ok(Json(
-        (&clients.connects)
+        (&OAUTH_CLIENTS.connects)
             .into_iter()
             .map(|(k, v)| {
                 (
@@ -412,12 +401,8 @@ async fn list_connects(
     ))
 }
 
-async fn connect_slack(
-    Extension(clients): Extension<Arc<AllClients>>,
-    Extension(is_secure): Extension<Arc<IsSecure>>,
-    cookies: Cookies,
-) -> error::Result<Redirect> {
-    let mut client = clients
+async fn connect_slack(cookies: Cookies) -> error::Result<Redirect> {
+    let mut client = OAUTH_CLIENTS
         .slack
         .as_ref()
         .ok_or_else(|| error::Error::BadRequest("slack client not setup".to_string()))?
@@ -428,7 +413,7 @@ async fn connect_slack(
     client.add_scope("commands");
     let url = client.authorize_url(&state);
 
-    set_cookie(&state, cookies, is_secure.0);
+    set_cookie(&state, cookies, *IS_SECURE);
     Ok(Redirect::to(url.as_str()))
 }
 
@@ -470,14 +455,9 @@ async fn disconnect_slack(
     Ok(format!("slack disconnected"))
 }
 
-async fn login(
-    Extension(clients): Extension<Arc<AllClients>>,
-    Extension(is_secure): Extension<Arc<IsSecure>>,
-    Path(client_name): Path<String>,
-    cookies: Cookies,
-) -> error::Result<Redirect> {
-    let clients = &clients.logins;
-    oauth_redirect(clients, client_name, cookies, None, None, is_secure.0)
+async fn login(Path(client_name): Path<String>, cookies: Cookies) -> error::Result<Redirect> {
+    let clients = &OAUTH_CLIENTS.logins;
+    oauth_redirect(clients, client_name, cookies, None, None, *IS_SECURE)
 }
 
 #[derive(Deserialize)]
@@ -488,13 +468,11 @@ async fn refresh_token(
     authed: Authed,
     Path((w_id, id)): Path<(String, i32)>,
     Extension(user_db): Extension<UserDB>,
-    Extension(clients): Extension<Arc<AllClients>>,
-    Extension(http_client): Extension<Client>,
     Json(VariablePath { path }): Json<VariablePath>,
 ) -> error::Result<String> {
     let tx = user_db.begin(&authed).await?;
 
-    _refresh_token(tx, &path, w_id, id, clients, http_client).await?;
+    _refresh_token(tx, &path, w_id, id).await?;
 
     Ok(format!("Token at path {path} refreshed"))
 }
@@ -504,8 +482,6 @@ pub async fn _refresh_token<'c>(
     path: &str,
     w_id: String,
     id: i32,
-    clients: Arc<AllClients>,
-    http_client: Client,
 ) -> error::Result<String> {
     let account = sqlx::query!(
         "SELECT client, refresh_token FROM account WHERE workspace_id = $1 AND id = $2",
@@ -515,14 +491,14 @@ pub async fn _refresh_token<'c>(
     .fetch_optional(&mut tx)
     .await?;
     let account = not_found_if_none(account, "Account", &id.to_string())?;
-    let client = (&clients
+    let client = (&OAUTH_CLIENTS
         .connects
         .get(&account.client)
         .ok_or_else(|| error::Error::BadRequest("invalid client".to_string()))?
         .client)
         .to_owned();
 
-    let token = _exchange_token(client, &account.refresh_token, http_client).await;
+    let token = _exchange_token(client, &account.refresh_token).await;
 
     if let Err(token_err) = token {
         sqlx::query!(
@@ -580,14 +556,10 @@ pub async fn _refresh_token<'c>(
     Ok(token_str)
 }
 
-async fn _exchange_token(
-    client: OClient,
-    refresh_token: &str,
-    http_client: Client,
-) -> Result<TokenResponse, Error> {
+async fn _exchange_token(client: OClient, refresh_token: &str) -> Result<TokenResponse, Error> {
     let token_json = client
         .exchange_refresh_token(&RefreshToken::from(refresh_token.clone()))
-        .with_client(&http_client)
+        .with_client(&HTTP_CLIENT)
         .execute::<serde_json::Value>()
         .await
         .map_err(to_anyhow)?;
@@ -608,11 +580,9 @@ pub struct OAuthCallback {
 async fn connect_callback(
     cookies: Cookies,
     Path(client_name): Path<String>,
-    Extension(clients): Extension<Arc<AllClients>>,
-    Extension(http_client): Extension<Client>,
     Json(callback): Json<OAuthCallback>,
 ) -> error::JsonResult<TokenResponse> {
-    let client_w_scopes = &clients
+    let client_w_scopes = OAUTH_CLIENTS
         .connects
         .get(&client_name)
         .ok_or_else(|| error::Error::BadRequest("invalid client".to_string()))?;
@@ -620,7 +590,7 @@ async fn connect_callback(
     let client = client_w_scopes.client.to_owned();
     let extra_params = client_w_scopes.extra_params_callback.clone();
     let token_response =
-        exchange_code::<TokenResponse>(callback, &cookies, client, &http_client, extra_params)
+        exchange_code::<TokenResponse>(callback, &cookies, client, &HTTP_CLIENT, extra_params)
             .await?;
 
     Ok(Json(token_response))
@@ -631,17 +601,15 @@ async fn connect_slack_callback(
     authed: Authed,
     cookies: Cookies,
     Extension(user_db): Extension<UserDB>,
-    Extension(clients): Extension<Arc<AllClients>>,
-    Extension(http_client): Extension<Client>,
     Json(callback): Json<OAuthCallback>,
 ) -> error::Result<String> {
-    let client = clients
+    let client = OAUTH_CLIENTS
         .slack
         .as_ref()
         .ok_or_else(|| error::Error::BadRequest("slack client not setup".to_string()))?
         .to_owned();
     let token =
-        exchange_code::<SlackTokenResponse>(callback, &cookies, client, &http_client, None).await?;
+        exchange_code::<SlackTokenResponse>(callback, &cookies, client, &HTTP_CLIENT, None).await?;
 
     let mut tx = user_db.begin(&authed).await?;
 
@@ -657,14 +625,26 @@ async fn connect_slack_callback(
     )
     .execute(&mut tx)
     .await?;
+    sqlx::query_as!(
+        Group,
+        "INSERT INTO group_ (workspace_id, name, summary, extra_perms) VALUES ($1, $2, $3, $4) ON CONFLICT DO NOTHING",
+        w_id,
+        "slack",
+        "The group slack commands act on belhalf of",
+        serde_json::json!({username_to_permissioned_as(&authed.username): true})
+    )
+    .execute(&mut tx)
+    .await?;
+
     sqlx::query!(
         "INSERT INTO folder
-                (workspace_id, name, owners, extra_perms)
-                VALUES ($1, $2, $3, $4) ON CONFLICT DO NOTHING",
+                (workspace_id, name, display_name, owners, extra_perms)
+                VALUES ($1, $2, $3, $4, $5) ON CONFLICT DO NOTHING",
         &w_id,
         "slack_bot",
-        &[],
-        serde_json::json!({})
+        "Slack bot",
+        &["g/slack".to_string()],
+        serde_json::json!({"g/slack": true})
     )
     .execute(&mut tx)
     .await?;
@@ -747,23 +727,17 @@ where
 
 async fn slack_command(
     SlackSig { sig, ts }: SlackSig,
-    Extension(slack_verifier): Extension<Arc<Option<SlackVerifier>>>,
     Extension(db): Extension<DB>,
-    Extension(base_url): Extension<Arc<BaseUrl>>,
     body: Bytes,
 ) -> error::Result<String> {
     let form: SlackCommand = serde_urlencoded::from_bytes(&body)
         .map_err(|_| error::Error::BadRequest("invalid payload".to_string()))?;
 
     let body = String::from_utf8_lossy(&body);
-    if slack_verifier
-        .as_ref()
-        .as_ref()
-        .map(|sv| sv.verify(&ts, &body, &sig).ok())
-        .flatten()
-        .is_none()
-    {
-        return Err(error::Error::BadRequest("verification failed".to_owned()));
+    if let Some(sv) = SLACK_SIGNING_SECRET.as_ref() {
+        if sv.verify(&ts, &body, &sig).ok().is_none() {
+            return Err(error::Error::BadRequest("verification failed".to_owned()));
+        }
     }
 
     let mut tx = db.begin().await?;
@@ -807,6 +781,7 @@ async fn slack_command(
                 None,
                 None,
                 None,
+                None,
                 false,
                 false,
                 None,
@@ -814,7 +789,7 @@ async fn slack_command(
             )
             .await?;
             tx.commit().await?;
-            let url = base_url.0.to_owned();
+            let url = BASE_URL.to_owned();
             return Ok(format!(
                 "Job launched. See details at {url}/run/{uuid}?workspace={}",
                 &settings.workspace_id
@@ -839,31 +814,27 @@ pub struct UserInfo {
 async fn login_callback(
     Path(client_name): Path<String>,
     cookies: Cookies,
-    Extension(clients): Extension<Arc<AllClients>>,
     Extension(db): Extension<DB>,
-    Extension(http_client): Extension<Client>,
-    Extension(is_secure): Extension<Arc<IsSecure>>,
-    Extension(cookie_domain): Extension<Arc<CookieDomain>>,
     Json(callback): Json<OAuthCallback>,
 ) -> error::Result<String> {
-    let client_w_config = &clients
+    let client_w_config = &OAUTH_CLIENTS
         .logins
         .get(&client_name)
         .ok_or_else(|| error::Error::BadRequest("invalid client".to_string()))?;
     let client = client_w_config.client.to_owned();
     let token_res =
-        exchange_code::<TokenResponse>(callback, &cookies, client, &http_client, None).await;
+        exchange_code::<TokenResponse>(callback, &cookies, client, &HTTP_CLIENT, None).await;
 
     if let Ok(token) = token_res {
         let token = &token.access_token.to_string();
         let userinfo_url = client_w_config.userinfo_url.as_ref().ok_or_else(|| {
             Error::BadConfig(format!("Missing userinfo_url in client {client_name}"))
         })?;
-        let user = http_get_user_info::<UserInfo>(&http_client, userinfo_url, token).await?;
+        let user = http_get_user_info::<UserInfo>(&HTTP_CLIENT, userinfo_url, token).await?;
 
         let email = match client_name.as_str() {
             "github" => http_get_user_info::<Vec<GHEmailInfo>>(
-                &http_client,
+                &HTTP_CLIENT,
                 "https://api.github.com/user/emails",
                 token,
             )
@@ -899,15 +870,7 @@ async fn login_callback(
         if let Some((email, login_type, super_admin)) = login {
             let login_type = serde_json::json!(login_type);
             if login_type == client_name {
-                crate::users::create_session_token(
-                    &email,
-                    super_admin,
-                    &mut tx,
-                    cookies,
-                    is_secure.0,
-                    &cookie_domain.as_ref().0,
-                )
-                .await?;
+                crate::users::create_session_token(&email, super_admin, &mut tx, cookies).await?;
             } else {
                 return Err(error::Error::BadRequest(format!(
                     "an user with the email associated to this login exists but with a different \
@@ -942,15 +905,7 @@ async fn login_callback(
             tx.commit().await?;
             invite_user_to_all_auto_invite_worspaces(&db, &email).await?;
             tx = db.begin().await?;
-            crate::users::create_session_token(
-                &email,
-                false,
-                &mut tx,
-                cookies,
-                is_secure.0,
-                &cookie_domain.as_ref().0,
-            )
-            .await?;
+            crate::users::create_session_token(&email, false, &mut tx, cookies).await?;
             audit_log(
                 &mut tx,
                 &email,
@@ -961,6 +916,7 @@ async fn login_callback(
                 Some([("method", &client_name[..])].into()),
             )
             .await?;
+
             let demo_exists =
                 sqlx::query_scalar!("SELECT EXISTS(SELECT 1 FROM workspace WHERE id = 'demo')")
                     .fetch_one(&mut tx)
@@ -982,6 +938,16 @@ async fn login_callback(
             }
         }
         tx.commit().await?;
+
+        if let Some(new_user_webhook) = NEW_USER_WEBHOOK.clone() {
+            let _ = HTTP_CLIENT
+                .post(&new_user_webhook)
+                .json(&serde_json::json!({"email" : &email, "event": "oauth_signup"}))
+                .send()
+                .await
+                .map_err(|e| tracing::error!("Error sending new user webhook: {}", e.to_string()));
+        }
+
         Ok("Successfully logged in".to_string())
     } else {
         Err(error::Error::BadRequest(format!(
@@ -1103,7 +1069,7 @@ fn set_cookie(state: &State, cookies: Cookies, is_secure: bool) {
     let csrf = state.to_base64();
     let mut cookie = Cookie::new("csrf", csrf);
     cookie.set_secure(is_secure);
-    cookie.set_same_site(cookie::SameSite::Lax);
+    cookie.set_same_site(Some(cookie::SameSite::Lax));
     cookie.set_http_only(true);
     cookie.set_path("/");
     cookies.add(cookie);

backend/windmill-api/src/resources.rs

@@ -142,7 +142,7 @@ async fn list_resources(
         .join("account")
         .on("variable.account = account.id AND account.workspace_id = variable.workspace_id")
         .order_by("path", true)
-        .and_where("resource.workspace_id = ? OR resource.workspace_id = 'starter'".bind(&w_id))
+        .and_where("resource.workspace_id = ?".bind(&w_id))
         .offset(offset)
         .limit(per_page)
         .clone();
@@ -187,7 +187,7 @@ async fn get_resource(
         FROM resource
         LEFT JOIN variable ON variable.path = resource.path AND variable.workspace_id = resource.workspace_id
         LEFT JOIN account ON variable.account = account.id AND account.workspace_id = resource.workspace_id
-        WHERE resource.path = $1 AND (resource.workspace_id = $2 OR resource.workspace_id = 'starter')",
+        WHERE resource.path = $1 AND resource.workspace_id = $2",
         path.to_owned(),
         &w_id
     )
@@ -226,8 +226,7 @@ async fn get_resource_value(
     let mut tx = user_db.begin(&authed).await?;
 
     let value_o = sqlx::query_scalar!(
-        "SELECT value from resource WHERE path = $1 AND (workspace_id = $2 OR workspace_id = \
-         'starter')",
+        "SELECT value from resource WHERE path = $1 AND workspace_id = $2",
         path.to_owned(),
         &w_id
     )
@@ -478,7 +477,7 @@ async fn list_resource_types(
 ) -> JsonResult<Vec<ResourceType>> {
     let rows = sqlx::query_as!(
         ResourceType,
-        "SELECT * from resource_type WHERE (workspace_id = $1 OR workspace_id = 'starter' OR workspace_id = 'admins') ORDER \
+        "SELECT * from resource_type WHERE (workspace_id = $1 OR workspace_id = 'admins') ORDER \
          BY name",
         &w_id
     )
@@ -493,7 +492,7 @@ async fn list_resource_types_names(
     Path(w_id): Path<String>,
 ) -> JsonResult<Vec<String>> {
     let rows = sqlx::query_scalar!(
-        "SELECT name from resource_type WHERE (workspace_id = $1 OR workspace_id = 'starter' OR workspace_id = 'admins') \
+        "SELECT name from resource_type WHERE (workspace_id = $1 OR workspace_id = 'admins') \
          ORDER BY name",
         &w_id
     )
@@ -512,8 +511,7 @@ async fn get_resource_type(
 
     let resource_type_o = sqlx::query_as!(
         ResourceType,
-        "SELECT * from resource_type WHERE name = $1 AND (workspace_id = $2 OR workspace_id = \
-         'starter' OR workspace_id = 'admins')",
+        "SELECT * from resource_type WHERE name = $1 AND (workspace_id = $2 OR workspace_id = 'admins')",
         &name,
         &w_id
     )
@@ -530,8 +528,7 @@ async fn exists_resource_type(
     Path((w_id, name)): Path<(String, String)>,
 ) -> JsonResult<bool> {
     let exists = sqlx::query_scalar!(
-        "SELECT EXISTS(SELECT 1 FROM resource_type WHERE name = $1 AND (workspace_id = $2 OR workspace_id = \
-            'starter' OR workspace_id = 'admins'))",
+        "SELECT EXISTS(SELECT 1 FROM resource_type WHERE name = $1 AND (workspace_id = $2 OR workspace_id = 'admins'))",
         name,
         w_id
     )

backend/windmill-api/src/scripts.rs

@@ -6,7 +6,6 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
-use reqwest::Client;
 use sql_builder::prelude::*;
 use windmill_audit::{audit_log, ActionKind};
 
@@ -15,6 +14,7 @@ use crate::{
     schedule::clear_schedule,
     users::{require_owner_of_path, Authed},
     webhook_util::{WebhookMessage, WebhookShared},
+    HTTP_CLIENT,
 };
 use axum::{
     extract::{Extension, Path, Query},
@@ -70,6 +70,7 @@ pub fn workspaced_service() -> Router {
         .route("/exists/p/*path", get(exists_script_by_path))
         .route("/archive/h/:hash", post(archive_script_by_hash))
         .route("/delete/h/:hash", post(delete_script_by_hash))
+        .route("/delete/p/*path", post(delete_script_by_path))
         .route("/get/h/:hash", get(get_script_by_hash))
         .route("/raw/h/:hash", get(raw_script_by_hash))
         .route("/deployment_status/h/:hash", get(get_deployment_status))
@@ -111,7 +112,7 @@ async fn list_scripts(
         )
         .order_desc("favorite.path IS NOT NULL")
         .order_by("created_at", lq.order_desc.unwrap_or(true))
-        .and_where("o.workspace_id = ? OR o.workspace_id = 'starter'".bind(&w_id))
+        .and_where("o.workspace_id = ?".bind(&w_id))
         .offset(offset)
         .limit(per_page)
         .clone();
@@ -120,9 +121,10 @@ async fn list_scripts(
         sqlb.and_where_eq(
             "created_at",
             "(select max(created_at) from script where o.path = path 
-            AND (workspace_id = ? OR workspace_id = 'starter'))"
+            AND workspace_id = ?)"
                 .bind(&w_id),
         );
+        sqlb.and_where_eq("archived", true);
     } else {
         sqlb.and_where_eq("archived", false);
     }
@@ -163,12 +165,9 @@ async fn list_scripts(
     Ok(Json(rows))
 }
 
-async fn list_hub_scripts(
-    Authed { email, .. }: Authed,
-    Extension(http_client): Extension<Client>,
-) -> JsonResult<serde_json::Value> {
+async fn list_hub_scripts(Authed { email, .. }: Authed) -> JsonResult<serde_json::Value> {
     let asks = list_elems_from_hub(
-        http_client,
+        &HTTP_CLIENT,
         "https://hub.windmill.dev/searchData?approved=true",
         &email,
     )
@@ -380,6 +379,7 @@ async fn create_script(
             None,
             None,
             None,
+            None,
             false,
             false,
             None,
@@ -442,21 +442,16 @@ async fn create_script(
     Ok((StatusCode::CREATED, format!("{}", hash)))
 }
 
-pub async fn get_hub_script_by_path(
-    authed: Authed,
-    Path(path): Path<StripPath>,
-    Extension(http_client): Extension<Client>,
-) -> Result<String> {
-    windmill_common::scripts::get_hub_script_by_path(&authed.email, path, http_client).await
+pub async fn get_hub_script_by_path(authed: Authed, Path(path): Path<StripPath>) -> Result<String> {
+    windmill_common::scripts::get_hub_script_by_path(&authed.email, path, &HTTP_CLIENT).await
 }
 
 pub async fn get_full_hub_script_by_path(
     Authed { email, .. }: Authed,
     Path(path): Path<StripPath>,
-    Extension(http_client): Extension<Client>,
 ) -> JsonResult<HubScript> {
     Ok(Json(
-        windmill_common::scripts::get_full_hub_script_by_path(&email, path, http_client).await?,
+        windmill_common::scripts::get_full_hub_script_by_path(&email, path, &HTTP_CLIENT).await?,
     ))
 }
 
@@ -469,9 +464,9 @@ async fn get_script_by_path(
     let mut tx = user_db.begin(&authed).await?;
 
     let script_o = sqlx::query_as::<_, Script>(
-        "SELECT * FROM script WHERE path = $1 AND (workspace_id = $2 OR workspace_id = 'starter') \
+        "SELECT * FROM script WHERE path = $1 AND workspace_id = $2 \
          AND created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND \
-         (workspace_id = $2 OR workspace_id = 'starter'))",
+         workspace_id = $2)",
     )
     .bind(path)
     .bind(w_id)
@@ -510,11 +505,12 @@ async fn raw_script_by_path(
     let mut tx = user_db.begin(&authed).await?;
 
     let content_o = sqlx::query_scalar!(
-        "SELECT content FROM script WHERE path = $1 AND (workspace_id = $2 OR workspace_id = 'starter') \
+        "SELECT content FROM script WHERE path = $1 AND workspace_id = $2 \
          AND
          created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND archived = false AND \
-         (workspace_id = $2 OR workspace_id = 'starter'))",
-         path, w_id
+         workspace_id = $2)",
+        path,
+        w_id
     )
     .fetch_optional(&mut tx)
     .await?;
@@ -531,10 +527,8 @@ async fn exists_script_by_path(
     let path = path.to_path();
 
     let exists = sqlx::query_scalar!(
-        "SELECT EXISTS(SELECT 1 FROM script WHERE path = $1 AND (workspace_id = $2 OR \
-         workspace_id = 'starter') AND
-         created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND (workspace_id = $2 \
-         OR workspace_id = 'starter')))",
+        "SELECT EXISTS(SELECT 1 FROM script WHERE path = $1 AND workspace_id = $2 AND
+         created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2))",
         path,
         w_id
     )
@@ -550,13 +544,12 @@ async fn get_script_by_hash_internal<'c>(
     workspace_id: &str,
     hash: &ScriptHash,
 ) -> Result<Script> {
-    let script_o = sqlx::query_as::<_, Script>(
-        "SELECT * FROM script WHERE hash = $1 AND (workspace_id = $2 OR workspace_id = 'starter')",
-    )
-    .bind(hash)
-    .bind(workspace_id)
-    .fetch_optional(db)
-    .await?;
+    let script_o =
+        sqlx::query_as::<_, Script>("SELECT * FROM script WHERE hash = $1 AND workspace_id = $2")
+            .bind(hash)
+            .bind(workspace_id)
+            .fetch_optional(db)
+            .await?;
 
     let script = not_found_if_none(script_o, "Script", hash.to_string())?;
     Ok(script)
@@ -602,8 +595,7 @@ async fn get_deployment_status(
     let mut tx = user_db.begin(&authed).await?;
     let status_o: Option<DeploymentStatus> = sqlx::query_as!(
         DeploymentStatus,
-        "SELECT lock, lock_error_logs FROM script WHERE hash = $1 AND (workspace_id = $2 OR \
-         workspace_id = 'starter')",
+        "SELECT lock, lock_error_logs FROM script WHERE hash = $1 AND workspace_id = $2",
         hash.0,
         w_id,
     )
@@ -729,6 +721,46 @@ async fn delete_script_by_hash(
     Ok(Json(script))
 }
 
+async fn delete_script_by_path(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    Extension(webhook): Extension<WebhookShared>,
+    Extension(db): Extension<DB>,
+    Path((w_id, path)): Path<(String, StripPath)>,
+) -> JsonResult<String> {
+    let mut tx = user_db.begin(&authed).await?;
+    let path = path.to_path();
+
+    require_admin(authed.is_admin, &authed.username)?;
+    let script = sqlx::query_scalar!(
+        "DELETE FROM script WHERE path = $1 AND workspace_id = $2 RETURNING path",
+        path,
+        w_id
+    )
+    .fetch_one(&db)
+    .await
+    .map_err(|e| Error::InternalErr(format!("deleting script by path {w_id}: {e}")))?;
+
+    audit_log(
+        &mut tx,
+        &authed.username,
+        "scripts.delete",
+        ActionKind::Delete,
+        &w_id,
+        Some(&path),
+        Some([("workspace", w_id.as_str())].into()),
+    )
+    .await?;
+    tx.commit().await?;
+
+    webhook.send_message(
+        w_id.clone(),
+        WebhookMessage::DeleteScriptPath { workspace: w_id, path: path.to_string() },
+    );
+
+    Ok(Json(script))
+}
+
 async fn parse_python_code_to_jsonschema(
     Json(code): Json<String>,
 ) -> JsonResult<windmill_parser::MainArgSignature> {

backend/windmill-api/src/static_assets.rs

@@ -9,58 +9,41 @@
 use axum::{
     body::{self, BoxBody},
     extract::OriginalUri,
-    http::{header, response::Builder, Response},
+    http::{header, Response},
     response::IntoResponse,
-    Extension,
 };
 
-use crate::{CloudHosted, ContentSecurityPolicy, IsSecure};
+use hyper::Uri;
 use mime_guess::mime;
 use rust_embed::RustEmbed;
-use std::sync::Arc;
 
 // static_handler is a handler that serves static files from the
-pub async fn static_handler(
-    Extension(is_secure): Extension<Arc<IsSecure>>,
-    Extension(is_cloud_hosted): Extension<Arc<CloudHosted>>,
-    Extension(csp): Extension<Arc<ContentSecurityPolicy>>,
-    OriginalUri(original_uri): OriginalUri,
-) -> StaticFile {
-    let path = original_uri.path().trim_start_matches('/').to_string();
-    StaticFile(path, is_secure.0, is_cloud_hosted.0, csp)
+pub async fn static_handler(OriginalUri(original_uri): OriginalUri) -> StaticFile {
+    StaticFile(original_uri)
 }
 
 #[derive(RustEmbed)]
 #[folder = "../../frontend/build/"]
 struct Asset;
-pub struct StaticFile(
-    pub String,
-    pub bool,
-    pub bool,
-    pub Arc<ContentSecurityPolicy>,
-);
+pub struct StaticFile(Uri);
 
 impl IntoResponse for StaticFile {
     fn into_response(self) -> Response<BoxBody> {
-        let path = self.0;
-        let can_set_security_headers = self.1 && self.2;
-        let csp = self.3;
-        serve_path(path, can_set_security_headers, csp)
+        let path = self.0.path().trim_start_matches('/');
+        serve_path(path)
     }
 }
 
-fn serve_path(
-    path: String,
-    can_set_security_headers: bool,
-    csp: Arc<ContentSecurityPolicy>,
-) -> Response<BoxBody> {
+const TWO_HUNDRED: &str = "200.html";
+
+fn serve_path(path: &str) -> Response<BoxBody> {
     if path.starts_with("api/") {
         return Response::builder()
             .status(404)
             .body(body::boxed(body::Empty::new()))
             .unwrap();
     }
-    match Asset::get(path.as_str()) {
+    match Asset::get(path) {
         Some(content) => {
             let body = body::boxed(body::Full::from(content.data));
             let mime = mime_guess::from_path(path).first_or_octet_stream();
@@ -75,26 +58,12 @@ fn serve_path(
                 res = res.header(header::CACHE_CONTROL, "no-cache, no-store, must-revalidate");
             }
 
-            if can_set_security_headers {
-                res = set_security_headers(res, csp);
-            }
             res.body(body).unwrap()
         }
-        None if path.as_str().starts_with("_app/") => Response::builder()
+        None if path.starts_with("_app/") => Response::builder()
             .status(404)
             .body(body::boxed(body::Empty::new()))
             .unwrap(),
-        None => serve_path("200.html".to_owned(), can_set_security_headers, csp),
+        None => serve_path(TWO_HUNDRED),
     }
 }
-
-fn set_security_headers(mut res: Builder, csp: Arc<ContentSecurityPolicy>) -> Builder {
-    res = res.header("X-Frame-Options", "DENY");
-    res = res.header("X-Content-Type-Options", "nosniff");
-
-    if !csp.0.is_empty() {
-        res = res.header("Content-Security-Policy", &csp.0);
-    }
-
-    res
-}

backend/windmill-api/src/users.rs

@@ -13,7 +13,7 @@ use crate::{
     folders::get_folders_for_user,
     utils::require_super_admin,
     workspaces::invite_user_to_all_auto_invite_worspaces,
-    CookieDomain, IsSecure,
+    COOKIE_DOMAIN, HTTP_CLIENT, IS_SECURE,
 };
 use argon2::{password_hash::SaltString, Argon2, PasswordHash, PasswordHasher, PasswordVerifier};
 use axum::{
@@ -720,14 +720,12 @@ async fn logout(
     Tokened { token }: Tokened,
     cookies: Cookies,
     Extension(db): Extension<DB>,
-    Extension(cookie_domain): Extension<Arc<CookieDomain>>,
     Query(LogoutQuery { rd }): Query<LogoutQuery>,
 ) -> Result<Response> {
     let mut cookie = Cookie::new(COOKIE_NAME, "");
     cookie.set_path(COOKIE_PATH);
-    let domain = cookie_domain.0.clone();
-    if domain.is_some() {
-        cookie.set_domain(domain.clone().unwrap());
+    if COOKIE_DOMAIN.is_some() {
+        cookie.set_domain(COOKIE_DOMAIN.clone().unwrap());
     }
     cookies.remove(cookie);
     let mut tx = db.begin().await?;
@@ -1270,6 +1268,10 @@ async fn delete_user(
     Ok(format!("email {} deleted", &email_to_delete))
 }
 
+lazy_static::lazy_static! {
+    pub static ref NEW_USER_WEBHOOK: Option<String> = std::env::var("NEW_USER_WEBHOOK").ok();
+}
+
 async fn create_user(
     Authed { email, .. }: Authed,
     Extension(db): Extension<DB>,
@@ -1305,6 +1307,18 @@ async fn create_user(
     )
     .await?;
     tx.commit().await?;
+
+    if let Some(new_user_webhook) = NEW_USER_WEBHOOK.clone() {
+        let _ = HTTP_CLIENT
+            .post(&new_user_webhook)
+            .json(
+                &serde_json::json!({"email" : &nu.email, "name": &nu.name, "event": "global_add"}),
+            )
+            .send()
+            .await
+            .map_err(|e| tracing::error!("Error sending new user webhook: {}", e.to_string()));
+    }
+
     invite_user_to_all_auto_invite_worspaces(&db, &nu.email).await?;
 
     Ok((StatusCode::CREATED, format!("email {} created", nu.email)))
@@ -1549,21 +1563,19 @@ async fn login(
     cookies: Cookies,
     Extension(db): Extension<DB>,
     Extension(argon2): Extension<Arc<Argon2<'_>>>,
-    Extension(is_secure): Extension<Arc<IsSecure>>,
-    Extension(cookie_domain): Extension<Arc<CookieDomain>>,
     Json(Login { email, password }): Json<Login>,
 ) -> Result<String> {
     let mut tx = db.begin().await?;
 
-    let email_w_h: Option<(String, String, bool)> = sqlx::query_as(
-        "SELECT email, password_hash, super_admin FROM password WHERE email = $1 AND login_type = \
+    let email_w_h: Option<(String, String, bool, bool)> = sqlx::query_as(
+        "SELECT email, password_hash, super_admin, first_time_user FROM password WHERE email = $1 AND login_type = \
          'password'",
     )
     .bind(&email)
     .fetch_optional(&mut tx)
     .await?;
 
-    if let Some((email, hash, super_admin)) = email_w_h {
+    if let Some((email, hash, super_admin, first_time_user)) = email_w_h {
         let parsed_hash =
             PasswordHash::new(&hash).map_err(|e| Error::InternalErr(e.to_string()))?;
         if argon2
@@ -1572,15 +1584,27 @@ async fn login(
         {
             Err(Error::BadRequest("Invalid login".to_string()))
         } else {
-            let token = create_session_token(
-                &email,
-                super_admin,
-                &mut tx,
-                cookies,
-                is_secure.0,
-                &cookie_domain.as_ref().0,
-            )
-            .await?;
+            if first_time_user {
+                sqlx::query_scalar!(
+                    "UPDATE password SET first_time_user = false WHERE email = $1",
+                    &email
+                )
+                .execute(&mut tx)
+                .await?;
+                let mut c = Cookie::new("first_time", "1");
+                if let Some(domain) = COOKIE_DOMAIN.as_ref() {
+                    c.set_domain(domain);
+                }
+                c.set_secure(false);
+                c.set_expires(time::OffsetDateTime::now_utc() + time::Duration::minutes(15));
+                c.set_http_only(false);
+                c.set_path("/");
+
+                cookies.add(c);
+            }
+
+            let token = create_session_token(&email, super_admin, &mut tx, cookies).await?;
+
             tx.commit().await?;
             Ok(token)
         }
@@ -1594,8 +1618,6 @@ pub async fn create_session_token<'c>(
     super_admin: bool,
     tx: &mut sqlx::Transaction<'c, sqlx::Postgres>,
     cookies: Cookies,
-    is_secure: bool,
-    domain: &Option<String>,
 ) -> Result<String> {
     let token = rd_string(30);
     sqlx::query!(
@@ -1611,12 +1633,12 @@ pub async fn create_session_token<'c>(
     .execute(tx)
     .await?;
     let mut cookie = Cookie::new(COOKIE_NAME, token.clone());
-    cookie.set_secure(is_secure);
-    cookie.set_same_site(cookie::SameSite::Lax);
+    cookie.set_secure(*IS_SECURE);
+    cookie.set_same_site(Some(cookie::SameSite::Lax));
     cookie.set_http_only(true);
     cookie.set_path(COOKIE_PATH);
-    if domain.is_some() {
-        cookie.set_domain(domain.clone().unwrap());
+    if COOKIE_DOMAIN.is_some() {
+        cookie.set_domain(COOKIE_DOMAIN.clone().unwrap());
     }
     let mut expire: OffsetDateTime = time::OffsetDateTime::now_utc();
     expire += time::Duration::days(3);

backend/windmill-api/src/variables.rs

@@ -6,14 +6,11 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
-use std::sync::Arc;
-
 use crate::{
     db::{UserDB, DB},
-    oauth2::{AllClients, _refresh_token},
+    oauth2::_refresh_token,
     users::{require_owner_of_path, Authed},
     webhook_util::{WebhookMessage, WebhookShared},
-    BaseUrl,
 };
 /*
  * Author: Ruben Fiszel
@@ -29,6 +26,7 @@ use axum::{
     Json, Router,
 };
 use hyper::StatusCode;
+use serde_json::Value;
 use windmill_audit::{audit_log, ActionKind};
 use windmill_common::{
     error::{Error, JsonResult, Result},
@@ -37,7 +35,6 @@ use windmill_common::{
 };
 
 use magic_crypt::{MagicCrypt256, MagicCryptTrait};
-use reqwest::Client;
 use serde::Deserialize;
 use sqlx::{Postgres, Transaction};
 
@@ -54,7 +51,6 @@ pub fn workspaced_service() -> Router {
 
 async fn list_contextual_variables(
     Path(w_id): Path<String>,
-    Extension(base_url): Extension<Arc<BaseUrl>>,
     Authed { username, email, .. }: Authed,
 ) -> JsonResult<Vec<ContextualVariable>> {
     Ok(Json(
@@ -65,7 +61,6 @@ async fn list_contextual_variables(
             &username,
             "017e0ad5-f499-73b6-5488-92a61c5196dd",
             format!("u/{username}").as_str(),
-            &base_url.0,
             Some("u/user/script_path".to_string()),
             Some("017e0ad5-f499-73b6-5488-92a61c5196dd".to_string()),
             Some("u/user/encapsulating_flow_path".to_string()),
@@ -91,7 +86,7 @@ async fn list_variables(
          from variable
          LEFT JOIN account ON variable.account = account.id AND account.workspace_id = variable.workspace_id
          LEFT JOIN resource ON resource.path = variable.path AND resource.workspace_id = variable.workspace_id
-         WHERE variable.workspace_id = $1 OR (is_secret IS NOT TRUE AND variable.workspace_id = 'starter') ORDER BY path",
+         WHERE variable.workspace_id = $1 ORDER BY path",
     )
     .bind(&w_id)
     .fetch_all(&mut tx)
@@ -111,8 +106,6 @@ async fn get_variable(
     Extension(user_db): Extension<UserDB>,
     Query(q): Query<GetVariableQuery>,
     Path((w_id, path)): Path<(String, StripPath)>,
-    Extension(clients): Extension<Arc<AllClients>>,
-    Extension(http_client): Extension<Client>,
 ) -> JsonResult<ListableVariable> {
     let path = path.to_path();
     let mut tx = user_db.begin(&authed).await?;
@@ -124,8 +117,7 @@ async fn get_variable(
         from variable
         LEFT JOIN account ON variable.account = account.id
         LEFT JOIN resource ON resource.path = variable.path AND resource.workspace_id = variable.workspace_id
-        WHERE variable.path = $1 AND (variable.workspace_id = $2 OR (is_secret IS NOT TRUE AND \
-         variable.workspace_id = 'starter')) 
+        WHERE variable.path = $1 AND variable.workspace_id = $2
         LIMIT 1",
     )
     .bind(&path)
@@ -151,17 +143,7 @@ async fn get_variable(
         let value = variable.value.unwrap_or_else(|| "".to_string());
         ListableVariable {
             value: if variable.is_expired.unwrap_or(false) && variable.account.is_some() {
-                Some(
-                    _refresh_token(
-                        tx,
-                        &variable.path,
-                        w_id,
-                        variable.account.unwrap(),
-                        clients,
-                        http_client,
-                    )
-                    .await?,
-                )
+                Some(_refresh_token(tx, &variable.path, w_id, variable.account.unwrap()).await?)
             } else if !value.is_empty() && decrypt_secret {
                 let mc = build_crypt(&mut tx, &w_id).await?;
                 tx.commit().await?;
@@ -227,12 +209,13 @@ async fn create_variable(
     Extension(user_db): Extension<UserDB>,
     Extension(webhook): Extension<WebhookShared>,
     Path(w_id): Path<String>,
+    Query(AlreadyEncrypted { already_encrypted }): Query<AlreadyEncrypted>,
     Json(variable): Json<CreateVariable>,
 ) -> Result<(StatusCode, String)> {
     let mut tx = user_db.begin(&authed).await?;
 
     check_path_conflict(&mut tx, &w_id, &variable.path).await?;
-    let value = if variable.is_secret {
+    let value = if variable.is_secret && !already_encrypted.unwrap_or(false) {
         let mc = build_crypt(&mut tx, &w_id).await?;
         encrypt(&mc, &variable.value)
     } else {
@@ -330,12 +313,18 @@ struct EditVariable {
     description: Option<String>,
 }
 
+#[derive(Deserialize)]
+struct AlreadyEncrypted {
+    already_encrypted: Option<bool>,
+}
+
 async fn update_variable(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
     Extension(webhook): Extension<WebhookShared>,
     Extension(db): Extension<DB>,
     Path((w_id, path)): Path<(String, StripPath)>,
+    Query(AlreadyEncrypted { already_encrypted }): Query<AlreadyEncrypted>,
     Json(ns): Json<EditVariable>,
 ) -> Result<String> {
     use sql_builder::prelude::*;
@@ -361,7 +350,7 @@ async fn update_variable(
         .await?
         .unwrap_or(false);
 
-        let value = if is_secret {
+        let value = if is_secret && !already_encrypted.unwrap_or(false) {
             let mc = build_crypt(&mut tx, &w_id).await?;
             encrypt(&mc, &nvalue)
         } else {
@@ -390,9 +379,26 @@ async fn update_variable(
             if !authed.is_admin {
                 require_owner_of_path(&w_id, &authed.username, &authed.groups, &path, &db).await?;
             }
+            let mut v = sqlx::query_scalar!(
+                "SELECT value FROM resource  WHERE path = $1 AND workspace_id = $2",
+                path,
+                w_id
+            )
+            .fetch_one(&mut tx)
+            .await?;
+
+            if let Some(old_v) = v {
+                v = Some(replace_path(
+                    old_v,
+                    &format!("$var:{path}"),
+                    &format!("$var:{npath}"),
+                ))
+            }
+
             sqlx::query!(
-                "UPDATE resource SET path = $1 WHERE path = $2 AND workspace_id = $3",
+                "UPDATE resource SET path = $1, value = $2 WHERE path = $3 AND workspace_id = $4",
                 npath,
+                v,
                 path,
                 w_id
             )
@@ -431,6 +437,23 @@ async fn update_variable(
     Ok(format!("variable {} updated (npath: {:?})", path, npath))
 }
 
+fn replace_path(v: serde_json::Value, path: &str, npath: &str) -> Value {
+    match v {
+        Value::Object(v) => Value::Object(
+            v.into_iter()
+                .map(|(k, v)| (k, replace_path(v, path, npath)))
+                .collect(),
+        ),
+        Value::Array(arr) => Value::Array(
+            arr.into_iter()
+                .map(|v| replace_path(v, path, npath))
+                .collect(),
+        ),
+        Value::String(s) if s == path => Value::String(npath.to_owned()),
+        _ => v,
+    }
+}
+
 pub async fn build_crypt<'c>(
     db: &mut Transaction<'c, Postgres>,
     w_id: &str,

backend/windmill-api/src/webhook_util.rs

@@ -24,6 +24,7 @@ pub enum WebhookMessage {
     CreateFlow { workspace: String, path: String },
     UpdateFlow { workspace: String, old_path: String, new_path: String },
     ArchiveFlow { workspace: String, path: String },
+    DeleteFlow { workspace: String, path: String },
     CreateFolder { workspace: String, name: String },
     UpdateFolder { workspace: String, name: String },
     DeleteFolder { workspace: String, name: String },
@@ -36,6 +37,7 @@ pub enum WebhookMessage {
     CreateScript { workspace: String, path: String, hash: String },
     UpdateScript { workspace: String, path: String, hash: String },
     DeleteScript { workspace: String, hash: String },
+    DeleteScriptPath { workspace: String, path: String },
     CreateVariable { workspace: String, path: String },
     UpdateVariable { workspace: String, old_path: String, new_path: String },
     DeleteVariable { workspace: String, path: String },

backend/windmill-api/src/worker_ping.rs

@@ -28,7 +28,7 @@ pub fn global_service() -> Router {
 struct WorkerPing {
     worker: String,
     worker_instance: String,
-    ping_at: chrono::DateTime<chrono::Utc>,
+    last_ping: Option<i32>,
     started_at: chrono::DateTime<chrono::Utc>,
     ip: String,
     jobs_executed: i32,
@@ -45,7 +45,7 @@ async fn list_worker_pings(
 
     let rows = sqlx::query_as!(
         WorkerPing,
-        "SELECT * FROM worker_ping ORDER BY ping_at desc LIMIT $1 OFFSET $2",
+        "SELECT worker, worker_instance,  EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed FROM worker_ping ORDER BY ping_at desc LIMIT $1 OFFSET $2",
         per_page as i64,
         offset as i64
     )

backend/windmill-api/src/workspaces.rs

@@ -6,24 +6,31 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
-use std::{str::FromStr, sync::Arc};
+#[cfg(feature = "enterprise")]
+use std::str::FromStr;
 
+#[cfg(feature = "enterprise")]
+use crate::BASE_URL;
 use crate::{
+    apps::AppWithLastVersion,
     db::{UserDB, DB},
     folders::Folder,
     resources::{Resource, ResourceType},
-    users::{Authed, WorkspaceInvite},
+    users::{Authed, WorkspaceInvite, NEW_USER_WEBHOOK},
     utils::require_super_admin,
-    BaseUrl,
+    HTTP_CLIENT,
 };
+#[cfg(feature = "enterprise")]
+use axum::response::Redirect;
 use axum::{
     body::StreamBody,
     extract::{Extension, Path, Query},
     headers,
-    response::{IntoResponse, Redirect},
+    response::IntoResponse,
     routing::{delete, get, post},
     Json, Router,
 };
+#[cfg(feature = "enterprise")]
 use stripe::CustomerId;
 use windmill_audit::{audit_log, ActionKind};
 use windmill_common::{
@@ -42,7 +49,7 @@ use tokio::fs::File;
 use tokio_util::io::ReaderStream;
 
 pub fn workspaced_service() -> Router {
-    Router::new()
+    let router = Router::new()
         .route("/list_pending_invites", get(list_pending_invites))
         .route("/update", post(edit_workspace))
         .route("/archive", post(archive_workspace))
@@ -54,9 +61,17 @@ pub fn workspaced_service() -> Router {
         .route("/edit_webhook", post(edit_webhook))
         .route("/edit_auto_invite", post(edit_auto_invite))
         .route("/tarball", get(tarball_workspace))
-        .route("/premium_info", get(premium_info))
+        .route("/premium_info", get(premium_info));
+
+    #[cfg(feature = "enterprise")]
+    tracing::info!("stripe enabled");
+
+    #[cfg(feature = "enterprise")]
+    let router = router
         .route("/checkout", get(stripe_checkout))
-        .route("/billing_portal", get(stripe_portal))
+        .route("/billing_portal", get(stripe_portal));
+
+    router
 }
 pub fn global_service() -> Router {
     Router::new()
@@ -216,24 +231,25 @@ async fn premium_info(
     Ok(Json(row))
 }
 
+#[cfg(feature = "enterprise")]
 #[derive(Deserialize)]
 struct PlanQuery {
     plan: String,
 }
 
+#[cfg(feature = "enterprise")]
 async fn stripe_checkout(
     authed: Authed,
     Path(w_id): Path<String>,
     Query(plan): Query<PlanQuery>,
-    Extension(base_url): Extension<Arc<BaseUrl>>,
 ) -> Result<Redirect> {
     // #[cfg(feature = "enterprise")]
     {
         require_admin(authed.is_admin, &authed.username)?;
 
         let client = stripe::Client::new(std::env::var("STRIPE_KEY").expect("STRIPE_KEY"));
-        let success_rd = format!("{}/workspace_settings/checkout?success=true", base_url.0);
-        let failure_rd = format!("{}/workspace_settings/checkout?success=false", base_url.0);
+        let success_rd = format!("{}/workspace_settings/checkout?success=true", *BASE_URL);
+        let failure_rd = format!("{}/workspace_settings/checkout?success=false", *BASE_URL);
         let checkout_session = {
             let mut params = stripe::CreateCheckoutSession::new(&failure_rd, &success_rd);
             params.mode = Some(stripe::CheckoutSessionMode::Subscription);
@@ -287,11 +303,11 @@ async fn stripe_checkout(
     }
 }
 
+#[cfg(feature = "enterprise")]
 async fn stripe_portal(
     authed: Authed,
     Path(w_id): Path<String>,
     Extension(db): Extension<DB>,
-    Extension(base_url): Extension<Arc<BaseUrl>>,
 ) -> Result<Redirect> {
     require_admin(authed.is_admin, &authed.username)?;
     let customer_id = sqlx::query_scalar!(
@@ -302,7 +318,7 @@ async fn stripe_portal(
     .await?
     .ok_or_else(|| Error::InternalErr(format!("no customer id for workspace {}", w_id)))?;
     let client = stripe::Client::new(std::env::var("STRIPE_KEY").expect("STRIPE_KEY"));
-    let success_rd = format!("{}/workspace_settings?tab=premium", base_url.0);
+    let success_rd = format!("{}/workspace_settings?tab=premium", *BASE_URL);
     let portal_session = {
         let customer_id = CustomerId::from_str(&customer_id).unwrap();
         let mut params = stripe::CreateBillingPortalSession::new(customer_id);
@@ -661,19 +677,19 @@ async fn create_workspace(
     .execute(&mut tx)
     .await?;
 
-    let mc = magic_crypt::new_magic_crypt!(key, 256);
-    sqlx::query!(
-        "INSERT INTO variable
-            (workspace_id, path, value, is_secret, description)
-            VALUES ($1, 'g/all/pretty_secret', $2, true, 'This item is secret'), 
-                ($3, 'g/all/not_secret', $4, false, 'This item is not secret')",
-        nw.id,
-        crate::variables::encrypt(&mc, "pretty secret value"),
-        nw.id,
-        "finland does not actually exist",
-    )
-    .execute(&mut tx)
-    .await?;
+    // let mc = magic_crypt::new_magic_crypt!(key, 256);
+    // sqlx::query!(
+    //     "INSERT INTO variable
+    //         (workspace_id, path, value, is_secret, description)
+    //         VALUES ($1, 'g/all/pretty_secret', $2, true, 'This item is secret'),
+    //             ($3, 'g/all/not_secret', $4, false, 'This item is not secret')",
+    //     nw.id,
+    //     crate::variables::encrypt(&mc, "pretty secret value"),
+    //     nw.id,
+    //     "finland does not actually exist",
+    // )
+    // .execute(&mut tx)
+    // .await?;
 
     sqlx::query!(
         "INSERT INTO usr
@@ -946,6 +962,15 @@ async fn invite_user(
 
     tx.commit().await?;
 
+    if let Some(new_user_webhook) = NEW_USER_WEBHOOK.clone() {
+        let _ = &HTTP_CLIENT
+            .post(&new_user_webhook)
+            .json(&serde_json::json!({"email" : &nu.email, "event": "workspace_invite"}))
+            .send()
+            .await
+            .map_err(|e| tracing::error!("Error sending new user webhook: {}", e.to_string()));
+    }
+
     Ok((
         StatusCode::CREATED,
         format!("user with email {} invited", nu.email),
@@ -977,6 +1002,15 @@ async fn add_user(
 
     tx.commit().await?;
 
+    if let Some(new_user_webhook) = NEW_USER_WEBHOOK.clone() {
+        let _ = HTTP_CLIENT
+            .post(&new_user_webhook)
+            .json(&serde_json::json!({"email" : &nu.email, "event": "workspace_add"}))
+            .send()
+            .await
+            .map_err(|e| tracing::error!("Error sending new user webhook: {}", e.to_string()));
+    }
+
     Ok((
         StatusCode::CREATED,
         format!("user with email {} added", nu.email),
@@ -1039,22 +1073,121 @@ struct ScriptMetadata {
     schema: Option<Schema>,
     is_template: bool,
     lock: Vec<String>,
+    kind: String,
+}
+
+enum ArchiveImpl {
+    Zip(async_zip::write::ZipFileWriter<File>),
+    Tar(tokio_tar::Builder<File>),
+}
+
+impl ArchiveImpl {
+    async fn write_to_archive(&mut self, content: &str, path: &str) -> Result<()> {
+        match self {
+            ArchiveImpl::Tar(t) => {
+                let bytes = content.as_bytes();
+                let mut header = tokio_tar::Header::new_gnu();
+                header.set_size(bytes.len() as u64);
+                header.set_mtime(0);
+                header.set_uid(0);
+                header.set_gid(0);
+                header.set_mode(0o777);
+                header.set_cksum();
+                t.append_data(&mut header, path, bytes).await?;
+            }
+            ArchiveImpl::Zip(z) => {
+                let header = async_zip::ZipEntryBuilder::new(
+                    path.to_owned(),
+                    async_zip::Compression::Deflate,
+                )
+                .last_modification_date(Default::default())
+                .unix_permissions(0o777)
+                .build();
+                z.write_entry_whole(header, content.as_bytes())
+                    .await
+                    .map_err(to_anyhow)?;
+            }
+        }
+        Ok(())
+    }
+    async fn finish(self) -> Result<()> {
+        match self {
+            ArchiveImpl::Tar(t) => t.into_inner().await?,
+            ArchiveImpl::Zip(z) => z.close().await.map_err(to_anyhow)?,
+        }
+        .sync_all()
+        .await?;
+
+        Ok(())
+    }
+}
+
+#[derive(Deserialize)]
+struct ArchiveQueryParams {
+    archive_type: Option<String>,
+}
+
+#[inline]
+pub fn to_string_without_metadata<T>(value: &T, preserve_extra_perms: bool) -> Result<String>
+where
+    T: ?Sized + Serialize,
+{
+    let value = serde_json::to_value(value).map_err(to_anyhow)?;
+    value
+        .as_object()
+        .map(|obj| {
+            let mut obj = obj.clone();
+            for key in [
+                "workspace_id",
+                "path",
+                "name",
+                "versions",
+                "id",
+                "created_at",
+                "updated_at",
+                "created_by",
+                "updated_by",
+                "edited_at",
+                "edited_by",
+                "archived",
+            ] {
+                if obj.contains_key(key) {
+                    obj.remove(key);
+                }
+            }
+
+            if !preserve_extra_perms && obj.contains_key("extra_perms") {
+                obj.remove("extra_perms");
+            }
+
+            serde_json::to_string_pretty(&obj).ok()
+        })
+        .flatten()
+        .ok_or_else(|| Error::BadRequest("Impossible to serialize value".to_string()))
 }
 
 async fn tarball_workspace(
     authed: Authed,
     Extension(db): Extension<DB>,
     Path(w_id): Path<String>,
+    Query(ArchiveQueryParams { archive_type }): Query<ArchiveQueryParams>,
 ) -> Result<([(headers::HeaderName, String); 2], impl IntoResponse)> {
     require_admin(authed.is_admin, &authed.username)?;
 
     let tmp_dir = TempDir::new_in(".")?;
 
-    let name = format!("windmill-{w_id}.tar");
+    let name = match archive_type.as_deref() {
+        Some("tar") | None => Ok(format!("windmill-{w_id}.tar")),
+        Some("zip") => Ok(format!("windmill-{w_id}.zip")),
+        Some(t) => Err(Error::BadRequest(format!("Invalid Archive Type {t}"))),
+    }?;
     let file_path = tmp_dir.path().join(&name);
     let file = File::create(&file_path).await?;
-    let mut a = tokio_tar::Builder::new(file);
-
+    let mut archive = match archive_type.as_deref() {
+        Some("tar") | None => Ok(ArchiveImpl::Tar(tokio_tar::Builder::new(file))),
+        Some("zip") => Ok(ArchiveImpl::Zip(async_zip::write::ZipFileWriter::new(file))),
+        Some(t) => Err(Error::BadRequest(format!("Invalid Archive Type {t}"))),
+    }?;
     {
         let folders = sqlx::query_as::<_, Folder>("SELECT * FROM folder WHERE workspace_id = $1")
             .bind(&w_id)
@@ -1062,12 +1195,12 @@ async fn tarball_workspace(
             .await?;
 
         for folder in folders {
-            write_to_archive(
-                serde_json::to_string_pretty(&folder).unwrap(),
-                format!("f/{}/folder.meta.json", folder.name),
-                &mut a,
-            )
-            .await?;
+            archive
+                .write_to_archive(
+                    &to_string_without_metadata(&folder, true).unwrap(),
+                    &format!("f/{}/folder.meta.json", folder.name),
+                )
+                .await?;
         }
     }
 
@@ -1088,7 +1221,9 @@ async fn tarball_workspace(
                 ScriptLang::Go => "go",
                 ScriptLang::Bash => "sh",
             };
-            write_to_archive(script.content, format!("{}.{}", script.path, ext), &mut a).await?;
+            archive
+                .write_to_archive(&script.content, &format!("{}.{}", script.path, ext))
+                .await?;
 
             let lock = script
                 .lock
@@ -1101,10 +1236,13 @@ async fn tarball_workspace(
                 description: script.description,
                 schema: script.schema,
                 is_template: script.is_template,
+                kind: script.kind.to_string(),
                 lock,
             };
             let metadata_str = serde_json::to_string_pretty(&metadata).unwrap();
-            write_to_archive(metadata_str, format!("{}.script.json", script.path), &mut a).await?;
+            archive
+                .write_to_archive(&metadata_str, &format!("{}.script.json", script.path))
+                .await?;
         }
     }
 
@@ -1118,13 +1256,10 @@ async fn tarball_workspace(
         .await?;
 
         for resource in resources {
-            let resource_str = serde_json::to_string_pretty(&resource).unwrap();
-            write_to_archive(
-                resource_str,
-                format!("{}.resource.json", resource.path),
-                &mut a,
-            )
-            .await?;
+            let resource_str = &to_string_without_metadata(&resource, false).unwrap();
+            archive
+                .write_to_archive(&resource_str, &format!("{}.resource.json", resource.path))
+                .await?;
         }
     }
 
@@ -1138,13 +1273,13 @@ async fn tarball_workspace(
         .await?;
 
         for resource_type in resource_types {
-            let resource_str = serde_json::to_string_pretty(&resource_type).unwrap();
-            write_to_archive(
-                resource_str,
-                format!("{}.resource-type.json", resource_type.name),
-                &mut a,
-            )
-            .await?;
+            let resource_str = &to_string_without_metadata(&resource_type, false).unwrap();
+            archive
+                .write_to_archive(
+                    &resource_str,
+                    &format!("{}.resource-type.json", resource_type.name),
+                )
+                .await?;
         }
     }
 
@@ -1157,25 +1292,49 @@ async fn tarball_workspace(
         .await?;
 
         for flow in flows {
-            let flow_str = serde_json::to_string_pretty(&flow).unwrap();
-            write_to_archive(flow_str, format!("{}.flow.json", flow.path), &mut a).await?;
+            let flow_str = &to_string_without_metadata(&flow, false).unwrap();
+            archive
+                .write_to_archive(&flow_str, &format!("{}.flow.json", flow.path))
+                .await?;
         }
     }
 
     {
         let variables = sqlx::query_as::<_, ExportableListableVariable>(
-            "SELECT *, false as is_expired FROM variable WHERE workspace_id = $1 AND is_secret = false",
+            "SELECT *, false as is_expired FROM variable WHERE workspace_id = $1",
         )
         .bind(&w_id)
         .fetch_all(&db)
         .await?;
 
         for var in variables {
-            let flow_str = serde_json::to_string_pretty(&var).unwrap();
-            write_to_archive(flow_str, format!("{}.variable.json", var.path), &mut a).await?;
+            let var_str = &to_string_without_metadata(&var, false).unwrap();
+            archive
+                .write_to_archive(&var_str, &format!("{}.variable.json", var.path))
+                .await?;
         }
     }
-    a.into_inner().await?;
+
+    {
+        let apps = sqlx::query_as!(
+            AppWithLastVersion,
+            "SELECT app.id, app.path, app.summary, app.versions, app.policy,
+            app.extra_perms, app_version.value, 
+            app_version.created_at, app_version.created_by from app, app_version 
+            WHERE app.workspace_id = $1 AND app_version.id = app.versions[array_upper(app.versions, 1)]",
+            &w_id
+        )
+        .fetch_all(&db)
+        .await?;
+
+        for app in apps {
+            let app_str = &to_string_without_metadata(&app, false).unwrap();
+            archive
+                .write_to_archive(&app_str, &format!("{}.app.json", app.path))
+                .await?;
+        }
+    }
+    archive.finish().await?;
 
     let file = tokio::fs::File::open(file_path).await?;
 
@@ -1192,20 +1351,3 @@ async fn tarball_workspace(
 
     Ok((headers, body))
 }
-
-async fn write_to_archive(
-    content: String,
-    path: String,
-    a: &mut tokio_tar::Builder<File>,
-) -> Result<()> {
-    let bytes = content.as_bytes();
-    let mut header = tokio_tar::Header::new_gnu();
-    header.set_size(bytes.len() as u64);
-    header.set_mtime(0);
-    header.set_uid(0);
-    header.set_gid(0);
-    header.set_mode(0o777);
-    header.set_cksum();
-    a.append_data(&mut header, path, bytes).await?;
-    Ok(())
-}

backend/windmill-common/Cargo.toml

@@ -41,3 +41,4 @@ hyper = { workspace = true, optional = true }
 tokio = { workspace = true, optional = true }
 reqwest = { workspace = true, optional = true }
 tracing-subscriber = { workspace = true, optional = true }
+lazy_static.workspace = true

backend/windmill-common/src/flow_status.rs

@@ -125,7 +125,7 @@ pub enum FlowStatusModule {
     },
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub enum JobResult {
     SingleJob(Uuid),
     ListJob(Vec<Uuid>),

backend/windmill-common/src/flows.rs

@@ -59,6 +59,7 @@ pub struct NewFlow {
 #[derive(Deserialize, Serialize, Debug, Clone, Default)]
 pub struct FlowValue {
     pub modules: Vec<FlowModule>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     #[serde(default)]
     pub failure_module: Option<FlowModule>,
     #[serde(default)]
@@ -152,7 +153,9 @@ pub struct FlowModule {
     #[serde(alias = "input_transform")]
     pub input_transforms: HashMap<String, InputTransform>,
     pub value: FlowModuleValue,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub stop_after_if: Option<StopAfterIf>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub summary: Option<String>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub suspend: Option<Suspend>,
@@ -245,7 +248,9 @@ pub enum FlowModuleValue {
         #[serde(alias = "input_transform")]
         input_transforms: HashMap<String, InputTransform>,
         content: String,
+        #[serde(skip_serializing_if = "Option::is_none")]
         lock: Option<String>,
+        #[serde(skip_serializing_if = "Option::is_none")]
         path: Option<String>,
         language: ScriptLang,
     },

backend/windmill-common/src/lib.rs

@@ -26,12 +26,13 @@ pub mod variables;
 #[cfg(feature = "tracing_init")]
 pub mod tracing_init;
 
-pub const DEFAULT_NUM_WORKERS: usize = 3;
-pub const DEFAULT_TIMEOUT: i32 = 300;
-pub const DEFAULT_SLEEP_QUEUE: u64 = 50;
 pub const DEFAULT_MAX_CONNECTIONS_SERVER: u32 = 50;
 pub const DEFAULT_MAX_CONNECTIONS_WORKER: u32 = 3;
 
+lazy_static::lazy_static! {
+    pub static ref BASE_URL: String = std::env::var("BASE_URL").unwrap_or_else(|_| "http://localhost".to_string());
+}
+
 #[cfg(feature = "tokio")]
 pub async fn shutdown_signal(tx: tokio::sync::broadcast::Sender<()>) -> anyhow::Result<()> {
     use std::io;
@@ -123,10 +124,8 @@ pub async fn get_latest_hash_for_path<'c>(
     script_path: &str,
 ) -> error::Result<scripts::ScriptHash> {
     let script_hash_o = sqlx::query_scalar!(
-        "select hash from script where path = $1 AND (workspace_id = $2 OR workspace_id = \
-         'starter') AND
-    created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND (workspace_id = $2 OR \
-         workspace_id = 'starter')) AND
+        "select hash from script where path = $1 AND workspace_id = $2 AND
+    created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2) AND
     deleted = false",
         script_path,
         w_id

backend/windmill-common/src/scripts.rs

@@ -7,7 +7,7 @@
  */
 
 use std::{
-    fmt::Display,
+    fmt::{self, Display},
     hash::{Hash, Hasher},
 };
 
@@ -103,6 +103,18 @@ pub enum ScriptKind {
     Approval,
 }
 
+impl Display for ScriptKind {
+    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
+        fmt.write_str(match self {
+            ScriptKind::Trigger => "trigger",
+            ScriptKind::Failure => "failure",
+            ScriptKind::Script => "script",
+            ScriptKind::Approval => "approval",
+        })?;
+        Ok(())
+    }
+}
+
 #[derive(Serialize)]
 #[cfg_attr(feature = "sqlx", derive(sqlx::FromRow))]
 pub struct Script {
@@ -210,7 +222,7 @@ pub fn to_hex_string(i: &i64) -> String {
 pub async fn get_hub_script_by_path(
     email: &str,
     path: StripPath,
-    http_client: reqwest::Client,
+    http_client: &reqwest::Client,
 ) -> crate::error::Result<String> {
     use crate::{
         error::{to_anyhow, Error},
@@ -239,7 +251,7 @@ pub async fn get_hub_script_by_path(
 pub async fn get_full_hub_script_by_path(
     email: &str,
     path: StripPath,
-    http_client: reqwest::Client,
+    http_client: &reqwest::Client,
 ) -> crate::error::Result<HubScript> {
     use crate::{
         error::{to_anyhow, Error},

backend/windmill-common/src/utils.rs

@@ -74,7 +74,7 @@ pub fn not_found_if_none<T, U: AsRef<str>>(opt: Option<T>, kind: &str, name: U)
 
 #[cfg(feature = "reqwest")]
 pub async fn list_elems_from_hub(
-    http_client: reqwest::Client,
+    http_client: &reqwest::Client,
     url: &str,
     email: &str,
 ) -> Result<serde_json::Value> {
@@ -88,7 +88,7 @@ pub async fn list_elems_from_hub(
 
 #[cfg(feature = "reqwest")]
 pub async fn http_get_from_hub(
-    http_client: reqwest::Client,
+    http_client: &reqwest::Client,
     url: &str,
     email: &str,
     plain: bool,

backend/windmill-common/src/variables.rs

@@ -8,6 +8,8 @@
 
 use serde::{Deserialize, Serialize};
 
+use crate::BASE_URL;
+
 #[derive(Serialize, Clone)]
 
 pub struct ContextualVariable {
@@ -66,7 +68,6 @@ pub fn get_reserved_variables(
     username: &str,
     job_id: &str,
     permissioned_as: &str,
-    base_url: &str,
     path: Option<String>,
     flow_id: Option<String>,
     flow_path: Option<String>,
@@ -114,7 +115,7 @@ pub fn get_reserved_variables(
         },
         ContextualVariable {
             name: "WM_BASE_URL".to_string(),
-            value: base_url.to_string(),
+            value: BASE_URL.clone(),
             description: "base url of this instance".to_string(),
         },
         ContextualVariable {

backend/windmill-queue/src/jobs.rs

@@ -6,9 +6,10 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
-use std::{collections::HashMap, str::FromStr};
+use std::collections::HashMap;
 
 use anyhow::Context;
+use reqwest::Client;
 use serde::{Deserialize, Serialize};
 use sqlx::{Pool, Postgres, Transaction};
 use tracing::{instrument, Instrument};
@@ -16,7 +17,7 @@ use ulid::Ulid;
 use uuid::Uuid;
 use windmill_audit::{audit_log, ActionKind};
 use windmill_common::{
-    error::{self, to_anyhow, Error},
+    error::{self, Error},
     flow_status::{FlowStatus, JobResult, MAX_RETRY_ATTEMPTS, MAX_RETRY_INTERVAL},
     flows::{FlowModule, FlowModuleValue, FlowValue},
     scripts::{get_full_hub_script_by_path, HubScript, ScriptHash, ScriptLang},
@@ -24,6 +25,10 @@ use windmill_common::{
 };
 
 lazy_static::lazy_static! {
+    pub static ref HTTP_CLIENT: Client = reqwest::ClientBuilder::new()
+        .user_agent("windmill/beta")
+        .build().unwrap();
+
     // TODO: these aren't synced, they should be moved into the queue abstraction once/if that happens.
     static ref QUEUE_PUSH_COUNT: prometheus::IntCounter = prometheus::register_int_counter!(
         "queue_push_count",
@@ -147,55 +152,24 @@ pub async fn pull(
 
 pub async fn get_result_by_id(
     db: Pool<Postgres>,
-    mut skip_direct: bool,
     w_id: String,
-    flow_id: String,
+    flow_id: Uuid,
     node_id: String,
 ) -> error::Result<serde_json::Value> {
-    let mut result_id: Option<JobResult> = None;
-    let mut parent_id = Uuid::from_str(&flow_id).ok();
-    while result_id.is_none() && parent_id.is_some() {
-        if !skip_direct {
-            let r = sqlx::query!(
-                "SELECT flow_status, parent_job FROM completed_job WHERE id = $1 AND workspace_id = $2 UNION ALL SELECT flow_status, parent_job FROM queue WHERE id = $1 AND workspace_id = $2 ",
-                parent_id.unwrap(),
-                w_id,
-            )
-            .fetch_optional(&db)
-            .await?;
-            if let Some(r) = r {
-                let value = r
-                    .flow_status
-                    .as_ref()
-                    .ok_or_else(|| Error::InternalErr(format!("requiring a flow status value")))?
-                    .to_owned();
-                parent_id = r.parent_job;
-                let status_o = serde_json::from_value::<FlowStatus>(value).ok();
-                result_id = status_o.and_then(|status| {
-                    status
-                        .modules
-                        .iter()
-                        .find(|m| m.id() == node_id)
-                        .and_then(|m| m.job_result())
-                });
-            } else {
-                parent_id = None;
-            }
-        } else {
-            let q_parent = sqlx::query_scalar!(
-                "SELECT parent_job FROM completed_job WHERE id = $1 AND workspace_id = $2 UNION ALL SELECT parent_job FROM queue WHERE id = $1 AND workspace_id = $2",
-                parent_id.unwrap(),
-                w_id,
-            )
-            .fetch_optional(&db)
-            .await?
-            .flatten();
-            parent_id = q_parent;
-            skip_direct = false
-        }
-    }
+    let job_result: Option<JobResult> = sqlx::query_scalar!(
+        "SELECT leaf_jobs->$1::text FROM queue WHERE COALESCE((SELECT root_job FROM queue WHERE id = $2), $2) = id AND workspace_id = $3",
+        node_id,
+        flow_id,
+        w_id,
+    )
+    .fetch_optional(&db)
+    .await?
+    .flatten()
+    .map(|x| serde_json::from_value(x).ok())
+    .flatten();
+
     let result_id = windmill_common::utils::not_found_if_none(
-        result_id,
+        job_result,
         "Flow result by id",
         format!("{}, {}", flow_id, node_id),
     )?;
@@ -277,6 +251,7 @@ pub async fn push<'c>(
     scheduled_for_o: Option<chrono::DateTime<chrono::Utc>>,
     schedule_path: Option<String>,
     parent_job: Option<Uuid>,
+    root_job: Option<Uuid>,
     is_flow_step: bool,
     mut same_worker: bool,
     pre_run_error: Option<&windmill_common::error::Error>,
@@ -346,7 +321,10 @@ pub async fn push<'c>(
                     .unwrap_or(false);
 
             if !is_super_admin {
-                if usage > MAX_FREE_EXECS {
+                if usage > MAX_FREE_EXECS
+                    && !matches!(job_payload, JobPayload::Dependencies { .. })
+                    && !matches!(job_payload, JobPayload::FlowDependencies { .. })
+                {
                     return Err(error::Error::BadRequest(format!(
                     "User {email} has exceeded the free usage limit of {MAX_FREE_EXECS} that applies outside of premium workspaces."
                 )));
@@ -384,8 +362,7 @@ pub async fn push<'c>(
         match job_payload {
             JobPayload::ScriptHash { hash, path } => {
                 let language = sqlx::query_scalar!(
-                    "SELECT language as \"language: ScriptLang\" FROM script WHERE hash = $1 AND \
-                 (workspace_id = $2 OR workspace_id = 'starter')",
+                    "SELECT language as \"language: ScriptLang\" FROM script WHERE hash = $1 AND workspace_id = $2",
                     hash.0,
                     workspace_id
                 )
@@ -406,7 +383,7 @@ pub async fn push<'c>(
                 )
             }
             JobPayload::ScriptHub { path } => {
-                let script = get_hub_script(path.clone(), email)
+                let script = get_hub_script(&HTTP_CLIENT, path.clone(), email)
                     .await
                     .context("error fetching hub script")?;
                 (
@@ -436,11 +413,10 @@ pub async fn push<'c>(
             ),
             JobPayload::FlowDependencies { path } => {
                 let value_json = sqlx::query_scalar!(
-                "SELECT value FROM flow WHERE path = $1 AND (workspace_id = $2 OR workspace_id = \
-                 'starter')",
-                path,
-                workspace_id
-            )
+                    "SELECT value FROM flow WHERE path = $1 AND workspace_id = $2",
+                    path,
+                    workspace_id
+                )
                 .fetch_optional(&mut tx)
                 .await?
                 .ok_or_else(|| Error::InternalErr(format!("not found flow at path {:?}", path)))?;
@@ -463,11 +439,10 @@ pub async fn push<'c>(
             }
             JobPayload::Flow(flow) => {
                 let value_json = sqlx::query_scalar!(
-                "SELECT value FROM flow WHERE path = $1 AND (workspace_id = $2 OR workspace_id = \
-                 'starter')",
-                flow,
-                workspace_id
-            )
+                    "SELECT value FROM flow WHERE path = $1 AND workspace_id = $2",
+                    flow,
+                    workspace_id
+                )
                 .fetch_optional(&mut tx)
                 .await?
                 .ok_or_else(|| Error::InternalErr(format!("not found flow at path {:?}", flow)))?;
@@ -529,12 +504,13 @@ pub async fn push<'c>(
         .unwrap_or_else(|| (None, None));
 
     let flow_status = raw_flow.as_ref().map(FlowStatus::new);
+
     let uuid = sqlx::query_scalar!(
         "INSERT INTO queue
             (workspace_id, id, running, parent_job, created_by, permissioned_as, scheduled_for, 
                 script_hash, script_path, raw_code, raw_lock, args, job_kind, schedule_path, raw_flow, \
-         flow_status, is_flow_step, language, started_at, same_worker, pre_run_error, email, visible_to_owner)
-            VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, CASE WHEN $3 THEN now() END, $19, $20, $21, $22) \
+         flow_status, is_flow_step, language, started_at, same_worker, pre_run_error, email, visible_to_owner, root_job)
+            VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, CASE WHEN $3 THEN now() END, $19, $20, $21, $22, $23) \
          RETURNING id",
         workspace_id,
         job_id,
@@ -557,7 +533,8 @@ pub async fn push<'c>(
         same_worker,
         pre_run_error.map(|e| e.to_string()),
         email,
-        visible_to_owner
+        visible_to_owner,
+        root_job
     )
     .fetch_one(&mut tx)
     .await
@@ -610,17 +587,14 @@ pub fn canceled_job_to_result(job: &QueuedJob) -> serde_json::Value {
     serde_json::json!({"message": format!("Job canceled: {reason} by {canceler}"), "name": "Canceled", "reason": reason, "canceler": canceler})
 }
 
-pub async fn get_hub_script(path: String, email: &str) -> error::Result<HubScript> {
-    get_full_hub_script_by_path(
-        email,
-        StripPath(path),
-        reqwest::ClientBuilder::new()
-            .user_agent("windmill/beta")
-            .build()
-            .map_err(to_anyhow)?,
-    )
-    .await
-    .map(|e| e)
+pub async fn get_hub_script(
+    client: &reqwest::Client,
+    path: String,
+    email: &str,
+) -> error::Result<HubScript> {
+    get_full_hub_script_by_path(email, StripPath(path), client)
+        .await
+        .map(|e| e)
 }
 
 #[derive(Debug, sqlx::FromRow, Serialize, Clone)]
@@ -673,6 +647,10 @@ pub struct QueuedJob {
     pub suspend: Option<i32>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub mem_peak: Option<i32>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub root_job: Option<Uuid>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub leaf_jobs: Option<serde_json::Value>,
 }
 
 impl QueuedJob {

backend/windmill-queue/src/schedule.rs

@@ -85,6 +85,7 @@ pub async fn push_scheduled_job<'c>(
         Some(next),
         Some(schedule.path.clone()),
         None,
+        None,
         false,
         false,
         None,

backend/windmill-worker/Cargo.toml

@@ -3,11 +3,10 @@ name = "windmill-worker"
 version.workspace = true
 authors.workspace = true
 edition.workspace = true
-default-run = "worker"
 
-[[bin]]
-name = "worker"
-path = "./src/main.rs"
+[lib]
+name = "windmill_worker"
+path = "src/lib.rs"
 
 [features]
 default = []

backend/windmill-worker/src/js_eval.rs

@@ -34,12 +34,13 @@ pub async fn eval_timeout(
     env: Vec<(String, serde_json::Value)>,
     creds: Option<EvalCreds>,
     by_id: Option<IdContext>,
-    base_internal_url: String,
+    base_internal_url: &str,
 ) -> anyhow::Result<serde_json::Value> {
     let expr2 = expr.clone();
     let (sender, mut receiver) = oneshot::channel::<IsolateHandle>();
+    let base_internal_url: String = base_internal_url.to_string();
     timeout(
-        std::time::Duration::from_millis(2000),
+        std::time::Duration::from_millis(3000),
         tokio::task::spawn_blocking(move || {
             let mut ops = vec![];
 
@@ -103,7 +104,7 @@ pub async fn eval_timeout(
             isolate.terminate_execution();
         };
         Error::ExecutionErr(format!(
-            "The expression of evaluation `{expr2}` took too long to execute (>2000ms)"
+            "The expression of evaluation `{expr2}` took too long to execute (>3000ms)"
         ))
     })??
 }
@@ -144,7 +145,7 @@ fn add_closing_bracket(s: &str) -> String {
     s
 }
 
-const SPLIT_PAT: &str = ";\n";
+const SPLIT_PAT: &str = ";";
 async fn eval(
     context: &mut JsRuntime,
     expr: &str,
@@ -153,14 +154,21 @@ async fn eval(
     by_id: Option<IdContext>,
     base_internal_url: &str,
 ) -> anyhow::Result<serde_json::Value> {
-    let expr = expr.trim();
-    let expr = format!(
-        "{}\nreturn {};",
-        expr.split(SPLIT_PAT)
-            .take(expr.split(SPLIT_PAT).count() - 1)
-            .join("\n"),
-        expr.split(SPLIT_PAT).last().unwrap_or_else(|| "")
-    );
+    let exprs = expr
+        .trim()
+        .split(SPLIT_PAT)
+        .map(|x| x.trim())
+        .filter(|x| !x.is_empty())
+        .collect::<Vec<&str>>();
+    let expr = if exprs.is_empty() {
+        "return undefined;".to_string()
+    } else {
+        format!(
+            "{};\n    return {};",
+            exprs.iter().take(exprs.len() - 1).join(";\n"),
+            exprs.last().unwrap()
+        )
+    };
     let (api_code, by_id_code) = if let Some(EvalCreds { workspace, token }) = creds {
         let by_id_code = if let Some(by_id) = by_id {
             format!(
@@ -197,12 +205,12 @@ const results = new Proxy({{}}, {{
                     .into_iter()
                     .map(|(k, v)| {
                         let v_str = match v {
-                            JobResult::SingleJob(x) => x.to_string(),
+                            JobResult::SingleJob(x) => format!("\"{x}\""),
                             JobResult::ListJob(x) => {
-                                format!("[{}]", x.iter().map(|x| x.to_string()).join(","))
+                                format!("[{}]", x.iter().map(|x| format!("\"{x}\"")).join(","))
                             }
                         };
-                        format!("\"{k}\": \"{v_str}\"")
+                        format!("\"{k}\": {v_str}")
                     })
                     .join(","),
                 by_id.previous_id,
@@ -291,9 +299,8 @@ async fn op_get_result(args: Vec<String>) -> Result<serde_json::Value, anyhow::E
     let base_url = &args[3];
     let client = windmill_api_client::create_client(base_url, token.clone());
     let result = client
-        .get_completed_job(workspace, &id.parse()?)
+        .get_completed_job_result(workspace, &id.parse()?)
         .await?
-        .result
         .clone();
     Ok(serde_json::json!(result))
 }
@@ -308,7 +315,7 @@ async fn op_get_id(args: Vec<String>) -> Result<Option<serde_json::Value>, anyho
 
     let client = windmill_api_client::create_client(base_url, token.clone());
     let result = client
-        .result_by_id(workspace, flow_job_id, node_id, Some(true))
+        .result_by_id(workspace, flow_job_id, node_id)
         .await
         .map_or(None, |e| Some(e.into_inner()));
 
@@ -346,7 +353,7 @@ mod tests {
         let code = "value.test + params.test";
 
         let mut runtime = JsRuntime::new(RuntimeOptions::default());
-        let res = eval(&mut runtime, code, env, None, None, "").await?;
+        let res = eval(&mut runtime, code, env, None, None, String::new().as_str()).await?;
         assert_eq!(res, json!(4));
         Ok(())
     }
@@ -359,7 +366,7 @@ mod tests {
 multiline template`";
 
         let mut runtime = JsRuntime::new(RuntimeOptions::default());
-        let res = eval(&mut runtime, code, env, None, None, "").await?;
+        let res = eval(&mut runtime, code, env, None, None, String::new().as_str()).await?;
         assert_eq!(res, json!("my 5\nmultiline template"));
         Ok(())
     }
@@ -372,7 +379,7 @@ multiline template`";
         ];
         let code = r#"params.test"#;
 
-        let res = eval_timeout(code.to_string(), env, None, None, "".to_string()).await?;
+        let res = eval_timeout(code.to_string(), env, None, None, String::new().as_str()).await?;
         assert_eq!(res, json!(2));
         Ok(())
     }

backend/windmill-worker/src/main.rs

@@ -1,138 +0,0 @@
-/*
- * Author: Ruben Fiszel
- * Copyright: Windmill Labs, Inc 2022
- * This file and its contents are licensed under the AGPLv3 License.
- * Please see the included NOTICE for copyright information and
- * LICENSE-AGPL for a copy of the license.
- */
-
-use std::{net::SocketAddr, time::Duration};
-
-use anyhow::Context;
-use sqlx::{postgres::PgPoolOptions, Pool, Postgres};
-use windmill_common::{
-    error::{self, Error},
-    utils::rd_string,
-};
-
-#[tokio::main]
-async fn main() -> anyhow::Result<()> {
-    // dotenv().ok();
-
-    windmill_common::tracing_init::initialize_tracing();
-
-    let db = async {
-        let database_url = std::env::var("DATABASE_URL")
-            .map_err(|_| Error::BadConfig("DATABASE_URL env var is missing".to_string()))?;
-
-        let max_connections = match std::env::var("DATABASE_CONNECTIONS") {
-            Ok(n) => n.parse::<u32>().context("invalid DATABASE_CONNECTIONS")?,
-            Err(_) => 10,
-        };
-
-        Ok::<Pool<Postgres>, error::Error>(
-            PgPoolOptions::new()
-                .max_connections(max_connections)
-                .max_lifetime(Duration::from_secs(30 * 60)) // 30 mins
-                .connect(&database_url)
-                .await
-                .map_err(|err| Error::ConnectingToDatabase(err.to_string()))?,
-        )
-    }
-    .await?;
-
-    let metrics_addr: Option<SocketAddr> = std::env::var("METRICS_ADDR")
-        .ok()
-        .map(|s| {
-            s.parse::<bool>()
-                .map(|b| b.then(|| SocketAddr::from(([0, 0, 0, 0], 8001))))
-                .or_else(|_| s.parse::<SocketAddr>().map(Some))
-        })
-        .transpose()?
-        .flatten();
-
-    let (tx, rx) = tokio::sync::broadcast::channel::<()>(3);
-    let shutdown_signal = windmill_common::shutdown_signal(tx);
-
-    let base_internal_url =
-        std::env::var("BASE_INTERNAL_URL").unwrap_or_else(|_| "http://localhost:8000".to_string());
-
-    let base_url = std::env::var("BASE_URL").unwrap_or_else(|_| "http://localhost".to_string());
-
-    let timeout = std::env::var("TIMEOUT")
-        .ok()
-        .and_then(|x| x.parse::<i32>().ok())
-        .unwrap_or(windmill_common::DEFAULT_TIMEOUT);
-
-    let workers_f = async {
-        let sleep_queue = std::env::var("SLEEP_QUEUE")
-            .ok()
-            .and_then(|x| x.parse::<u64>().ok())
-            .unwrap_or(windmill_common::DEFAULT_SLEEP_QUEUE);
-        let disable_nuser = std::env::var("DISABLE_NUSER")
-            .ok()
-            .and_then(|x| x.parse::<bool>().ok())
-            .unwrap_or(false);
-        let disable_nsjail = std::env::var("DISABLE_NSJAIL")
-            .ok()
-            .and_then(|x| x.parse::<bool>().ok())
-            .unwrap_or(true);
-        let keep_job_dir = std::env::var("KEEP_JOB_DIR")
-            .ok()
-            .and_then(|x| x.parse::<bool>().ok())
-            .unwrap_or(false);
-        let sync_bucket = std::env::var("S3_CACHE_BUCKET")
-            .ok()
-            .map(|e| Some(e))
-            .unwrap_or(None);
-
-        tracing::info!(
-            "DISABLE_NSJAIL: {disable_nsjail}, DISABLE_NUSER: {disable_nuser}, BASE_URL: \
-                     {base_url}, SLEEP_QUEUE: {sleep_queue}, TIMEOUT: \
-                     {timeout}, KEEP_JOB_DIR: {keep_job_dir}"
-        );
-        let instance_name = rd_string(5);
-
-        let ip = windmill_common::external_ip::get_ip()
-            .await
-            .unwrap_or_else(|e| {
-                tracing::warn!(error = e.to_string(), "failed to get external IP");
-                "unretrievable IP".to_string()
-            });
-        let worker_name = format!("dt-worker-{}-{}", &instance_name, rd_string(5));
-        windmill_worker::run_worker(
-            &db.clone(),
-            timeout,
-            &instance_name,
-            worker_name,
-            1,
-            1,
-            &ip,
-            sleep_queue,
-            windmill_worker::WorkerConfig {
-                disable_nsjail,
-                disable_nuser,
-                base_internal_url,
-                base_url,
-                keep_job_dir,
-            },
-            sync_bucket,
-            rx.resubscribe(),
-        )
-        .await;
-        Ok(()) as anyhow::Result<()>
-    };
-
-    let metrics_f = async {
-        match metrics_addr {
-            Some(addr) => windmill_common::serve_metrics(addr, rx.resubscribe())
-                .await
-                .map_err(anyhow::Error::from),
-            None => Ok(()),
-        }
-    };
-
-    futures::try_join!(shutdown_signal, workers_f, metrics_f)?;
-
-    Ok(())
-}

backend/windmill-worker/src/main2.rs

@@ -0,0 +1,92 @@
+/*
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
+ * This file and its contents are licensed under the AGPLv3 License.
+ * Please see the included NOTICE for copyright information and
+ * LICENSE-AGPL for a copy of the license.
+ */
+
+// use std::{net::SocketAddr, time::Duration};
+
+// use anyhow::Context;
+// use sqlx::{postgres::PgPoolOptions, Pool, Postgres};
+// use windmill_common::{
+//     error::{self, Error},
+//     utils::rd_string,
+// };
+
+// #[tokio::main]
+// async fn main() -> anyhow::Result<()> {
+//     // dotenv().ok();
+
+//     windmill_common::tracing_init::initialize_tracing();
+
+//     let db = async {
+//         let database_url = std::env::var("DATABASE_URL")
+//             .map_err(|_| Error::BadConfig("DATABASE_URL env var is missing".to_string()))?;
+
+//         let max_connections = match std::env::var("DATABASE_CONNECTIONS") {
+//             Ok(n) => n.parse::<u32>().context("invalid DATABASE_CONNECTIONS")?,
+//             Err(_) => 10,
+//         };
+
+//         Ok::<Pool<Postgres>, error::Error>(
+//             PgPoolOptions::new()
+//                 .max_connections(max_connections)
+//                 .max_lifetime(Duration::from_secs(30 * 60)) // 30 mins
+//                 .connect(&database_url)
+//                 .await
+//                 .map_err(|err| Error::ConnectingToDatabase(err.to_string()))?,
+//         )
+//     }
+//     .await?;
+
+//     let metrics_addr: Option<SocketAddr> = std::env::var("METRICS_ADDR")
+//         .ok()
+//         .map(|s| {
+//             s.parse::<bool>()
+//                 .map(|b| b.then(|| SocketAddr::from(([0, 0, 0, 0], 8001))))
+//                 .or_else(|_| s.parse::<SocketAddr>().map(Some))
+//         })
+//         .transpose()?
+//         .flatten();
+
+//     let (tx, rx) = tokio::sync::broadcast::channel::<()>(3);
+//     let shutdown_signal = windmill_common::shutdown_signal(tx);
+
+//     let workers_f = async {
+//         let instance_name = rd_string(5);
+
+//         let ip = windmill_common::external_ip::get_ip()
+//             .await
+//             .unwrap_or_else(|e| {
+//                 tracing::warn!(error = e.to_string(), "failed to get external IP");
+//                 "unretrievable IP".to_string()
+//             });
+//         let worker_name = format!("dt-worker-{}-{}", &instance_name, rd_string(5));
+//         windmill_worker::run_worker(
+//             &db.clone(),
+//             &instance_name,
+//             worker_name,
+//             1,
+//             1,
+//             &ip,
+//             rx.resubscribe(),
+//         )
+//         .await;
+//         Ok(()) as anyhow::Result<()>
+//     };
+
+//     let metrics_f = async {
+//         match metrics_addr {
+//             Some(addr) => windmill_common::serve_metrics(addr, rx.resubscribe())
+//                 .await
+//                 .map_err(anyhow::Error::from),
+//             None => Ok(()),
+//         }
+//     };
+
+//     futures::try_join!(shutdown_signal, workers_f, metrics_f)?;
+
+//     Ok(())
+// }

backend/windmill-worker/src/worker_flow.rs

@@ -12,7 +12,7 @@ use std::time::Duration;
 
 use crate::jobs::{add_completed_job, add_completed_job_error, schedule_again_if_scheduled};
 use crate::js_eval::{eval_timeout, EvalCreds, IdContext};
-use crate::worker;
+use crate::{worker, KEEP_JOB_DIR};
 use anyhow::Context;
 use async_recursion::async_recursion;
 use dyn_iter::DynIter;
@@ -50,9 +50,8 @@ pub async fn update_flow_status_after_job_completion(
     unrecoverable: bool,
     same_worker_tx: Sender<Uuid>,
     worker_dir: &str,
-    keep_job_dir: bool,
-    base_internal_url: &str,
     stop_early_override: Option<bool>,
+    base_internal_url: &str,
 ) -> error::Result<()> {
     tracing::debug!("UPDATE FLOW STATUS: {flow:?} {success} {result:?} {w_id}");
 
@@ -121,7 +120,7 @@ pub async fn update_flow_status_after_job_completion(
 
         let stop_early = success
             && if let Some(expr) = r.stop_early_expr.clone() {
-                compute_bool_from_expr(expr, &r.args, result.clone(), base_internal_url, None, None)
+                compute_bool_from_expr(expr, &r.args, result.clone(), None, None, base_internal_url)
                     .await?
             } else {
                 false
@@ -327,6 +326,21 @@ pub async fn update_flow_status_after_job_completion(
             )
             .execute(&mut tx)
             .await?;
+
+            if let Some(job_result) = new_status.job_result() {
+                sqlx::query!(
+                    "
+            UPDATE queue
+               SET leaf_jobs = JSONB_SET(coalesce(leaf_jobs, '{}'::jsonb), ARRAY[$1::TEXT], $2)
+             WHERE COALESCE((SELECT root_job FROM queue WHERE id = $3), $3) = id
+            ",
+                    new_status.id(),
+                    json!(job_result),
+                    flow
+                )
+                .execute(&mut tx)
+                .await?;
+            }
         }
     }
 
@@ -481,7 +495,7 @@ pub async fn update_flow_status_after_job_completion(
     };
 
     if done {
-        if flow_job.same_worker && !keep_job_dir {
+        if flow_job.same_worker && !*KEEP_JOB_DIR {
             let _ = tokio::fs::remove_dir_all(format!("{worker_dir}/{}", flow_job.id)).await;
         }
 
@@ -498,13 +512,12 @@ pub async fn update_flow_status_after_job_completion(
                 false,
                 same_worker_tx.clone(),
                 worker_dir,
-                keep_job_dir,
-                base_internal_url,
                 if stop_early {
                     Some(skip_if_stop_early)
                 } else {
                     None
                 },
+                base_internal_url,
             )
             .await?);
         }
@@ -559,7 +572,7 @@ async fn has_failure_module<'c>(
     flow: Uuid,
     tx: &mut sqlx::Transaction<'c, sqlx::Postgres>,
 ) -> Result<bool, Error> {
-    sqlx::query_scalar(
+    sqlx::query_scalar::<_, Option<bool>>(
         "
     SELECT raw_flow->'failure_module' != 'null'::jsonb
       FROM queue
@@ -570,6 +583,7 @@ async fn has_failure_module<'c>(
     .fetch_one(tx)
     .await
     .map_err(|e| Error::InternalErr(format!("error during retrieval of has_failure_module: {e}")))
+    .map(|v| v.unwrap_or(false))
 }
 
 fn next_retry(retry: &Retry, status: &RetryStatus) -> Option<(u16, Duration)> {
@@ -583,9 +597,9 @@ async fn compute_bool_from_expr(
     expr: String,
     flow_args: &Option<serde_json::Value>,
     result: serde_json::Value,
-    base_internal_url: &str,
     by_id: Option<IdContext>,
     creds: Option<EvalCreds>,
+    base_internal_url: &str,
 ) -> error::Result<bool> {
     let flow_input = flow_args.clone().unwrap_or_else(|| json!({}));
     match eval_timeout(
@@ -598,7 +612,7 @@ async fn compute_bool_from_expr(
         .into(),
         creds,
         by_id,
-        base_internal_url.to_string(),
+        base_internal_url,
     )
     .await?
     {
@@ -722,7 +736,7 @@ async fn transform_input(
                     context,
                     Some(EvalCreds { workspace: workspace.to_string(), token: token.to_string() }),
                     Some(by_id.clone()),
-                    base_internal_url.to_string(),
+                    base_internal_url,
                 )
                 .await
                 .map_err(|e| {
@@ -769,8 +783,8 @@ pub async fn handle_flow(
         client,
         last_result,
         same_worker_tx,
-        base_internal_url,
         worker_dir,
+        base_internal_url,
     )
     .await?;
     Ok(())
@@ -786,8 +800,8 @@ async fn push_next_flow_job(
     client: &windmill_api_client::Client,
     mut last_result: serde_json::Value,
     same_worker_tx: Sender<Uuid>,
-    base_internal_url: &str,
     worker_dir: &str,
+    base_internal_url: &str,
 ) -> error::Result<()> {
     let mut i = usize::try_from(status.step)
         .with_context(|| format!("invalid module index {}", status.step))?;
@@ -816,9 +830,8 @@ async fn push_next_flow_job(
             true,
             same_worker_tx,
             worker_dir,
-            false,
-            base_internal_url,
             None,
+            base_internal_url,
         )
         .await;
     }
@@ -858,7 +871,7 @@ async fn push_next_flow_job(
                         .into(),
                         None,
                         None,
-                        "".to_string(),
+                        base_internal_url,
                     )
                     .await
                     .map_err(|e| {
@@ -1162,8 +1175,8 @@ async fn push_next_flow_job(
         &status,
         &status_module,
         last_result.clone(),
-        base_internal_url,
         previous_id,
+        base_internal_url,
     )
     .await?;
     tx.commit().await?;
@@ -1239,6 +1252,7 @@ async fn push_next_flow_job(
             Ok(v) => (Some(v), None),
             Err(e) => (None, Some(e)),
         };
+        let root_job = flow_job.root_job.or_else(|| Some(flow_job.id));
         let (uuid, inner_tx) = push(
             tx,
             &flow_job.workspace_id,
@@ -1250,6 +1264,7 @@ async fn push_next_flow_job(
             scheduled_for_o,
             flow_job.schedule_path.clone(),
             Some(flow_job.id),
+            root_job,
             true,
             continue_on_same_worker,
             err,
@@ -1504,8 +1519,8 @@ async fn compute_next_flow_transform<'c>(
     status: &FlowStatus,
     status_module: &FlowStatusModule,
     last_result: serde_json::Value,
-    base_internal_url: &str,
     previous_id: String,
+    base_internal_url: &str,
 ) -> error::Result<(sqlx::Transaction<'c, sqlx::Postgres>, NextFlowTransform)> {
     match &module.value {
         FlowModuleValue::Identity => Ok((
@@ -1701,12 +1716,12 @@ async fn compute_next_flow_transform<'c>(
                             b.expr.to_string(),
                             &flow_job.args,
                             last_result.clone(),
-                            base_internal_url,
                             Some(idcontext.clone()),
                             Some(EvalCreds {
                                 workspace: flow_job.workspace_id.clone(),
                                 token: token.to_string(),
                             }),
+                            base_internal_url,
                         )
                         .await?;
 
@@ -1920,7 +1935,7 @@ where
                 vars(),
                 Some(EvalCreds { workspace, token }),
                 by_id,
-                base_internal_url.to_string(),
+                base_internal_url,
             )
             .await
         }

cli/README.md

@@ -28,12 +28,25 @@ Flow Steps and Logs will be streamed during execution automatically.
 The CLI can push specifications to a windmill instance. See the
 [examples/](./examples/) folder for formats.
 
-### Pushing a folder
+## Switch to a different workspace
 
-You can push all files in a folder at once using `wmill push` Files MUST be
-named resource_name.\<type\>.json. They will be pushed to the remote path they
-are in, for example the file `u/admin/fib/fib.script.json` will be pushed as a
-script to u/admin/fib/fib.
+```
+wmill workspace switch <workspace_name>
+```
+
+## Sync a workspace
+
+### Pull
+
+```
+wmill sync pull
+```
+
+### Push
+
+```
+wmill sync push
+```
 
 ### Pushing individual files
 

cli/apps.ts

@@ -0,0 +1,107 @@
+import { Any, model, property } from "./decoverto.ts";
+import {
+  AppService,
+  AppWithLastVersion,
+  colors,
+  microdiff,
+  Policy,
+} from "./deps.ts";
+import { Difference, PushDiffs, Resource, setValueByPath } from "./types.ts";
+
+@model()
+export class AppFile implements Resource, PushDiffs {
+  @property(Any)
+  value: any;
+  @property(() => String)
+  summary: string;
+  @property(Any)
+  policy: Policy;
+
+
+  constructor(value: string, summary: string, policy: Policy) {
+    this.value = value;
+    this.summary = summary;
+    this.policy = policy;
+  }
+  async pushDiffs(
+    workspace: string,
+    remotePath: string,
+    diffs: Difference[],
+  ): Promise<void> {
+    let app: AppWithLastVersion | undefined = undefined;
+    try {
+      app = await AppService.getAppByPath({ workspace, path: remotePath });
+    } catch (e) {}
+
+    if (app) {
+      console.log(
+        colors.bold.yellow(
+          `Applying ${diffs.length} diffs to existing app... ${remotePath}`,
+        ),
+      );
+      const changeset: {
+        summary?: string | undefined;
+        value?: any;
+        policy?: Policy | undefined;
+      } = {};
+      for (const diff of diffs) {
+        if (
+          diff.type !== "REMOVE" &&
+          (
+            diff.path[0] !== "value" && diff.path[0] !== "policy" && (
+              diff.path.length !== 1 ||
+              !["summary"].includes(
+                diff.path[0] as string,
+              )
+            )
+          )
+        ) {
+          throw new Error("Invalid app diff with path " + diff.path);
+        }
+        if (diff.type === "CREATE" || diff.type === "CHANGE") {
+          setValueByPath(changeset, diff.path, diff.value);
+        } else if (diff.type === "REMOVE") {
+          setValueByPath(changeset, diff.path, null);
+        }
+      }
+
+      if ((!changeset?.policy || JSON.stringify(changeset?.policy) == JSON.stringify(app.policy)) 
+        && (!changeset?.value || JSON.stringify(changeset?.value) == JSON.stringify(app.value))
+        && (!changeset?.summary || changeset.summary == app.summary)) {
+        console.log(colors.yellow(`No changes to push for app ${remotePath}, skipping`))
+        return;
+      }
+
+      const hasChanges = Object.values(changeset).some((v) =>
+        v !== null && typeof v !== "undefined"
+      );
+      if (!hasChanges) {
+        return;
+      }
+
+      await AppService.updateApp({
+        workspace,
+        path: remotePath,
+        requestBody: changeset,
+      });
+    } else {
+      console.log(colors.yellow.bold("Creating new app..."));
+      await AppService.createApp({
+        workspace,
+        requestBody: {
+          path: remotePath,
+          policy: this.policy,
+          summary: this.summary,
+          value: this.value,
+        },
+      });
+    }
+  }
+  async push(workspace: string, remotePath: string): Promise<void> {
+    await this.pushDiffs(
+      workspace,
+      remotePath,
+      microdiff({}, this, { cyclesFix: false }),
+    );
+  }
+}

cli/context.ts

@@ -1,5 +1,5 @@
 // deno-lint-ignore-file no-explicit-any
-import { colors, setClient, UserService } from "./deps.ts";
+import { colors, GlobalUserInfo, setClient, UserService } from "./deps.ts";
 import { loginInteractive, tryGetLoginInfo } from "./login.ts";
 import { GlobalOptions } from "./types.ts";
 import {
@@ -18,12 +18,12 @@ export type Context = {
 };
 
 async function tryResolveWorkspace(
-  opts: GlobalOptions,
+  opts: GlobalOptions
 ): Promise<
   { isError: false; value: Workspace } | { isError: true; error: string }
 > {
   const cache = (opts as any).__secret_workspace;
-  if (cache) return cache;
+  if (cache) return { isError: false, value: cache };
 
   if (opts.workspace) {
     const e = await getWorkspaceByName(opts.workspace);
@@ -49,21 +49,23 @@ async function tryResolveWorkspace(
 }
 
 export async function resolveWorkspace(
-  opts: GlobalOptions,
+  opts: GlobalOptions
 ): Promise<Workspace> {
   const res = await tryResolveWorkspace(opts);
   if (res.isError) {
-    console.log(res.error);
+    console.log(colors.red.bold(res.error));
     return Deno.exit(-1);
   } else {
     return res.value;
   }
 }
 
-export async function requireLogin(opts: GlobalOptions) {
+export async function requireLogin(
+  opts: GlobalOptions
+): Promise<GlobalUserInfo> {
   const workspace = await resolveWorkspace(opts);
-
   let token = await tryGetLoginInfo(opts);
+
   if (!token) {
     token = workspace.token;
   }
@@ -71,29 +73,29 @@ export async function requireLogin(opts: GlobalOptions) {
   setClient(token, workspace.remote.substring(0, workspace.remote.length - 1));
 
   try {
-    await UserService.globalWhoami();
+    return await UserService.globalWhoami();
   } catch {
     console.log(
-      "! Could not reach API given existing credentials. Attempting to reauth...",
+      "! Could not reach API given existing credentials. Attempting to reauth..."
     );
     const newToken = await loginInteractive(workspace.remote);
     if (!newToken) {
       throw new Error("Could not reauth");
     }
-    removeWorkspace(workspace.name);
+    removeWorkspace(workspace.name, false, opts);
     workspace.token = newToken;
-    addWorkspace(workspace);
+    addWorkspace(workspace, opts);
 
     setClient(
       token,
-      workspace.remote.substring(0, workspace.remote.length - 1),
+      workspace.remote.substring(0, workspace.remote.length - 1)
     );
-    await UserService.globalWhoami();
+    return await UserService.globalWhoami();
   }
 }
 
 export async function tryResolveVersion(
-  opts: GlobalOptions,
+  opts: GlobalOptions
 ): Promise<number | undefined> {
   if ((opts as any).__cache_version) {
     return (opts as any).__cache_version;
@@ -103,12 +105,12 @@ export async function tryResolveVersion(
   if (workspaceRes.isError) return undefined;
 
   const response = await fetch(
-    new URL(new URL(workspaceRes.value.remote).origin + "/api/version"),
+    new URL(new URL(workspaceRes.value.remote).origin + "/api/version")
   );
   const version = await response.text();
   try {
     return Number.parseInt(
-      version.split("-", 1)[0].replaceAll(".", "").replace("v", ""),
+      version.split("-", 1)[0].replaceAll(".", "").replace("v", "")
     );
   } catch {
     return undefined;
@@ -117,7 +119,7 @@ export async function tryResolveVersion(
 
 export async function validatePath(
   opts: GlobalOptions,
-  path: string,
+  path: string
 ): Promise<boolean> {
   const backendVersion = await tryResolveVersion(opts);
   if (path.startsWith("f")) {
@@ -125,19 +127,16 @@ export async function validatePath(
       return true;
     }
     console.log(
-      `Attempting to use folders, but the current remote does not have support. Remote version is ${backendVersion} but folders are supported from 1560.`,
+      `Attempting to use folders, but the current remote does not have support. Remote version is ${backendVersion} but folders are supported from 1560.`
     );
     return false;
   }
 
-  if (
-    !(path.startsWith("g") ||
-      path.startsWith("u"))
-  ) {
+  if (!(path.startsWith("g") || path.startsWith("u"))) {
     console.log(
       colors.red(
-        "Given remote path looks invalid. Remote paths are typically of the form <u|g|f>/<username|group|folder>/...",
-      ),
+        "Given remote path looks invalid. Remote paths are typically of the form <u|g|f>/<username|group|folder>/..."
+      )
     );
     return false;
   }

cli/deps.ts

@@ -1,6 +1,6 @@
 // windmill
-export { setClient } from "https://deno.land/x/windmill@v1.61.1/mod.ts";
-export * from "https://deno.land/x/windmill@v1.61.1/windmill-api/index.ts";
+export { setClient } from "https://deno.land/x/windmill@v1.69.3/mod.ts";
+export * from "https://deno.land/x/windmill@v1.69.3/windmill-api/index.ts";
 
 // cliffy
 export { Command } from "https://deno.land/x/cliffy@v0.25.7/command/command.ts";
@@ -14,9 +14,8 @@ export {
   DenoLandProvider,
   UpgradeCommand,
 } from "https://deno.land/x/cliffy@v0.25.7/command/upgrade/mod.ts";
-
+export { CompletionsCommand } from "https://deno.land/x/cliffy@v0.25.7/command/completions/mod.ts";
 // std
-export { Untar } from "https://deno.land/std@0.176.0/archive/untar.ts";
 export * as path from "https://deno.land/std@0.176.0/path/mod.ts";
 export { ensureDir } from "https://deno.land/std@0.176.0/fs/ensure_dir.ts";
 export {
@@ -38,3 +37,5 @@ export {
   default as microdiff,
 } from "https://deno.land/x/microdiff@v1.3.1/index.ts";
 export { default as objectHash } from "https://deno.land/x/object_hash@2.0.3.1/mod.ts";
+export { default as gitignore_parser } from "npm:gitignore-parser";
+export { default as JSZip } from "npm:jszip@3.7.1";

cli/flow.ts

@@ -10,7 +10,6 @@ import {
   colors,
   Command,
   Flow,
-  FlowModule,
   FlowService,
   JobService,
   microdiff,
@@ -19,21 +18,8 @@ import {
 } from "./deps.ts";
 import { requireLogin, resolveWorkspace, validatePath } from "./context.ts";
 import { resolve, track_job } from "./script.ts";
-import { Any, array, decoverto, model, property } from "./decoverto.ts";
+import { Any, decoverto, model, property } from "./decoverto.ts";
 
-@model()
-export class FlowValueFilePart {
-  @property(array(Any))
-  modules: Array<FlowModule>;
-  @property(Any)
-  failure_module?: FlowModule;
-  @property(() => Boolean)
-  same_worker?: boolean;
-
-  constructor(modules: Array<FlowModule>) {
-    this.modules = modules;
-  }
-}
 
 // this is effectively "OpenFlow" but a copy as it is accepted by the CLI
 @model()
@@ -42,13 +28,13 @@ export class FlowFile implements Resource, PushDiffs {
   summary: string;
   @property(() => String)
   description?: string;
-  @property(() => FlowValueFilePart)
-  value: FlowValueFilePart;
+  @property(Any)
+  value: any;
   @property(Any)
   schema?: any;
 
-  constructor(summary: string, value: FlowValueFilePart) {
-    this.summary = summary;
+  constructor(value: any, summary?: string) {
+    this.summary = summary ?? "";
     this.value = value;
   }
   async pushDiffs(
@@ -64,7 +50,7 @@ export class FlowFile implements Resource, PushDiffs {
     ) {
       console.log(
         colors.bold.yellow(
-          `Applying ${diffs.length} diffs to existing flow...`,
+          `Applying ${diffs.length} diffs to existing flow... ${remotePath}`,
         ),
       );
 
@@ -76,6 +62,7 @@ export class FlowFile implements Resource, PushDiffs {
         description: this.description, // This is OpenAPIed as optional, but isn't
         schema: this.schema, // Same
       };
+      const base_changeset = { ...changeset };
       for (const diff of diffs) {
         if (
           diff.type !== "REMOVE" &&
@@ -100,14 +87,18 @@ export class FlowFile implements Resource, PushDiffs {
         v !== null && typeof v !== "undefined"
       );
       if (!hasChanges) {
-        console.log(colors.yellow("! Skipping empty changeset"));
         return;
       }
 
+      const update = {
+        ...changeset,
+        ...base_changeset,
+      }
+
       await FlowService.updateFlow({
         workspace: workspace,
         path: remotePath,
-        requestBody: changeset,
+        requestBody: update,
       });
     } else {
       console.log(colors.bold.yellow("Creating new flow..."));
@@ -124,19 +115,11 @@ export class FlowFile implements Resource, PushDiffs {
     }
   }
   async push(workspace: string, remotePath: string): Promise<void> {
-    let remote: Flow | undefined;
-    try {
-      remote = await FlowService.getFlowByPath({
-        workspace,
-        path: remotePath,
-      });
-    } catch {
-      remote = undefined;
-    }
+
     await this.pushDiffs(
       workspace,
       remotePath,
-      microdiff(remote ?? {}, this, { cyclesFix: false }),
+      microdiff({}, this, { cyclesFix: false }),
     );
   }
 }
@@ -151,7 +134,7 @@ async function push(opts: Options, filePath: string, remotePath: string) {
   await requireLogin(opts);
 
   await pushFlow(filePath, workspace.remote, remotePath);
-  console.log(colors.bold.underline.green("Flow successfully pushed"));
+  console.log(colors.bold.underline.green("Flow pushed"));
 }
 
 export async function pushFlow(
@@ -187,23 +170,21 @@ async function list(opts: GlobalOptions & { showArchived?: boolean }) {
   }
 
   new Table()
-    .header(["path", "summary", "edited at", "edited by"])
+    .header(["path", "summary", "edited by"])
     .padding(2)
     .border(true)
     .body(
       total.map((x) => [
         x.path,
         x.summary,
-        x.edited_at,
         x.edited_by,
-        x.description ?? "-",
       ]),
     )
     .render();
 }
 async function run(
   opts: GlobalOptions & {
-    input: string[];
+    data?: string;
     silent: boolean;
   },
   path: string,
@@ -211,7 +192,8 @@ async function run(
   const workspace = await resolveWorkspace(opts);
   await requireLogin(opts);
 
-  const input = await resolve(opts.input);
+  const input = opts.data ? await resolve(opts.data) : {};
+
 
   const id = await JobService.runFlowByPath({
     workspace: workspace.workspaceId,
@@ -247,6 +229,7 @@ async function run(
 
   if (!opts.silent) {
     console.log(colors.green.underline.bold("Flow ran to completion"));
+    console.log()
   }
   const jobInfo = await JobService.getCompletedJob({
     workspace: workspace.workspaceId,
@@ -268,8 +251,8 @@ const command = new Command()
   .command("run", "run a flow by path.")
   .arguments("<path:string>")
   .option(
-    "-i --input [inputs...:string]",
-    "Inputs specified as JSON objects or simply as <name>=<value>. Supports file inputs using @<filename> and stdin using @- these also need to be formatted as JSON. Later inputs override earlier ones.",
+    "-d --data <data:string>",
+    "Inputs specified as a JSON string or a file using @<filename> or stdin using @-.",
   )
   .option(
     "-s --silent",

cli/folder.ts

@@ -1,4 +1,4 @@
-import { colors, Command, Folder, FolderService, microdiff } from "./deps.ts";
+import { colors, Command, FolderService, microdiff } from "./deps.ts";
 import { requireLogin, resolveWorkspace, validatePath } from "./context.ts";
 import {
   Difference,
@@ -22,7 +22,9 @@ export class FolderFile implements Resource, PushDiffs {
   owners: Array<string> | undefined;
   @property(map(() => String, () => Boolean, { shape: MapShape.Object }))
   extra_perms: Map<string, boolean> | undefined;
-
+  @property(() => String)
+  display_name: string| undefined;
+  
   async push(workspace: string, remotePath: string): Promise<void> {
     if (remotePath.startsWith("/")) {
       remotePath = remotePath.substring(1);
@@ -31,16 +33,10 @@ export class FolderFile implements Resource, PushDiffs {
       remotePath = remotePath.substring(2);
     }
 
-    let existing: Folder | undefined;
-    try {
-      existing = await FolderService.getFolder({ workspace, name: remotePath });
-    } catch {
-      existing = undefined;
-    }
     await this.pushDiffs(
       workspace,
       remotePath,
-      microdiff(existing ?? {}, this, { cyclesFix: false }),
+      microdiff({}, this, { cyclesFix: false }),
     );
   }
 
@@ -66,22 +62,24 @@ export class FolderFile implements Resource, PushDiffs {
     if (exists) {
       console.log(
         colors.bold.yellow(
-          `Applying ${diffs.length} diffs to existing folder...`,
+          `Applying ${diffs.length} diffs to existing folder... ${remotePath}`,
         ),
       );
 
       const changeset: {
         owners?: string[] | undefined;
         extra_perms?: any;
+        display_name?: string | undefined;
       } = {};
       for (const diff of diffs) {
         if (
           diff.type !== "REMOVE" &&
           (
             diff.path.length !== 1 ||
-            !["owners", "extra_perms"].includes(diff.path[0] as string)
+            !["owners", "extra_perms", "display_name"].includes(diff.path[0] as string)
           )
         ) {
+          console.log(diff.path)
           throw new Error("Invalid folder diff with path " + diff.path);
         }
         if (diff.type === "CREATE" || diff.type === "CHANGE") {
@@ -95,22 +93,20 @@ export class FolderFile implements Resource, PushDiffs {
         v !== null && typeof v !== "undefined"
       );
       if (!hasChanges) {
-        console.log(colors.yellow("! Skipping empty changeset"));
         return;
       }
-
       await FolderService.updateFolder({
         workspace: workspace,
         name: remotePath,
-        requestBody: changeset,
+        requestBody: {...changeset, extra_perms: changeset.extra_perms ? Object.fromEntries(this.extra_perms?.entries() ?? []) : undefined}
       });
     } else {
-      console.log(colors.bold.yellow("Creating new folder..."));
+      console.log(colors.bold.yellow("Creating new folder: " + remotePath));
       await FolderService.createFolder({
         workspace: workspace,
         requestBody: {
           name: remotePath,
-          extra_perms: this.extra_perms,
+          extra_perms: Object.fromEntries(this.extra_perms?.entries() ?? []),
           owners: this.owners,
         },
       });
@@ -134,7 +130,7 @@ async function push(opts: GlobalOptions, filePath: string, remotePath: string) {
   console.log(colors.bold.yellow("Pushing resource..."));
 
   await pushFolder(workspace.workspaceId, filePath, remotePath);
-  console.log(colors.bold.underline.green("Resource successfully pushed"));
+  console.log(colors.bold.underline.green("Resource pushed"));
 }
 
 export async function pushFolder(

cli/hub.ts

@@ -13,7 +13,7 @@ async function pull(opts: GlobalOptions) {
     return;
   }
 
-  await requireLogin(opts);
+  const userInfo = await requireLogin(opts);
   const list: {
     id: number;
     name: string;
@@ -26,6 +26,12 @@ async function pull(opts: GlobalOptions) {
     comments: never[];
   }[] = await fetch(
     "https://hub.windmill.dev/resource_types/list",
+    {
+      headers: {
+        "Accept": "application/json",
+        "X-email": userInfo.email,
+      },
+    },
   )
     .then((r) => r.json())
     .then((list: { id: number; name: string }[]) =>

cli/login.ts

@@ -1,5 +1,6 @@
 import { GlobalOptions } from "./types.ts";
 import { colors, getAvailablePort, Secret, Select } from "./deps.ts";
+import { open } from 'https://deno.land/x/open/index.ts';
 
 export async function loginInteractive(remote: string) {
   let token: string | undefined;
@@ -47,13 +48,20 @@ export async function browserLogin(
   }
 
   const server = Deno.listen({ transport: "tcp", port });
-  console.log(`Login by going to ${baseUrl}user/cli?port=${port}`);
+  const url = `${baseUrl}user/cli?port=${port}`
+  console.log(`Login by going to ${url}`);
+  try {
+    await open(url)
+    console.log("Opened browser for you");
+  } catch {
+    console.error(`Failed to open browser, please navigate to ${url}`)
+  }
   const firstConnection = await server.accept();
   const httpFirstConnection = Deno.serveHttp(firstConnection);
   const firstRequest = (await httpFirstConnection.nextRequest())!;
   const params = new URL(firstRequest.request.url!).searchParams;
   const token = params.get("token");
-  const _workspace = params.get("workspace");
+  // const _workspace = params.get("workspace");
   await firstRequest?.respondWith(
     Response.redirect(baseUrl + "user/cli-success", 302),
   );

cli/main.ts

@@ -1,4 +1,4 @@
-import { Command, DenoLandProvider, UpgradeCommand } from "./deps.ts";
+import { Command, CompletionsCommand, DenoLandProvider, UpgradeCommand } from "./deps.ts";
 import flow from "./flow.ts";
 import script from "./script.ts";
 import workspace from "./workspace.ts";
@@ -13,11 +13,12 @@ import sync from "./sync.ts";
 import { tryResolveVersion } from "./context.ts";
 import { GlobalOptions } from "./types.ts";
 
-const VERSION = "v1.62.0";
+const VERSION = "v1.76.0";
 
 let command: any = new Command()
   .name("wmill")
   .description("A simple CLI tool for windmill.")
+  .action(() => command.showHelp())
   .globalOption(
     "--workspace <workspace:string>",
     "Specify the target workspace. This overrides the default workspace.",
@@ -59,7 +60,8 @@ let command: any = new Command()
       ],
       provider: new DenoLandProvider({ name: "wmill" }),
     }),
-  );
+  )
+  .command("completions", new CompletionsCommand());
 
 if (Number.parseInt(VERSION.replace("v", "").replace(".", "")) > 1700) {
   command = command

cli/pull.ts

@@ -1,43 +1,34 @@
 // deno-lint-ignore-file no-explicit-any
 import { GlobalOptions } from "./types.ts";
-import { colors, Command, readerFromStreamReader, Untar } from "./deps.ts";
+import { colors, Command, JSZip } from "./deps.ts";
 import { Workspace } from "./workspace.ts";
 
-export async function downloadTar(
+export async function downloadZip(
   workspace: Workspace,
-): Promise<Untar | undefined> {
+): Promise<JSZip | undefined> {
   const requestHeaders: HeadersInit = new Headers();
   requestHeaders.set("Authorization", "Bearer " + workspace.token);
   requestHeaders.set("Content-Type", "application/octet-stream");
 
-  const tarResponse = await fetch(
+  const zipResponse = await fetch(
     workspace.remote + "api/w/" + workspace.workspaceId +
-      "/workspaces/tarball",
+    "/workspaces/tarball?archive_type=zip",
     {
       headers: requestHeaders,
       method: "GET",
     },
   );
 
-  if (!tarResponse.ok) {
+  if (!zipResponse.ok) {
     console.log(
       colors.red(
-        "Failed to request tarball from API " + tarResponse.statusText,
+        "Failed to request tarball from API " + zipResponse.statusText,
       ),
     );
-    console.log(await tarResponse.text());
-    return undefined;
+    throw new Error(await zipResponse.text());
   }
-
-  const streamReader = tarResponse.body?.getReader();
-  if (!streamReader) {
-    console.log(colors.red("Failed to read tar request body"));
-    return undefined;
-  }
-  console.log(colors.yellow("Streaming tarball to disk..."));
-  const denoReader = readerFromStreamReader(streamReader);
-  const untar = new Untar(denoReader);
-  return untar;
+  const blob = await zipResponse.blob();
+  return await JSZip.loadAsync(blob);
 }
 
 async function stub(

cli/resource-type.ts

@@ -13,7 +13,6 @@ import {
   EditResourceType,
   microdiff,
   ResourceService,
-  ResourceType,
   Table,
 } from "./deps.ts";
 import { Any, decoverto, model, property } from "./decoverto.ts";
@@ -26,19 +25,10 @@ export class ResourceTypeFile implements ResourceI, PushDiffs {
   description?: string;
 
   async push(workspace: string, remotePath: string): Promise<void> {
-    let existing: ResourceType | undefined;
-    try {
-      existing = await ResourceService.getResourceType({
-        workspace,
-        path: remotePath,
-      });
-    } catch {
-      existing = undefined;
-    }
-    this.pushDiffs(
+    await this.pushDiffs(
       workspace,
       remotePath,
-      microdiff(existing ?? {}, this, { cyclesFix: false }),
+      microdiff({}, this, { cyclesFix: false }),
     );
   }
 
@@ -60,13 +50,13 @@ export class ResourceTypeFile implements ResourceI, PushDiffs {
       ) {
         console.log(
           "Resource type " + remotePath +
-            " is already taken for the current workspace, but cannot be updated. Is this a conflict with starter?",
+          " is already taken for the current workspace, but cannot be updated. Is this a conflict with starter?",
         );
         return;
       }
       console.log(
-        colors.yellow(
-          `Applying ${diffs.length} diffs to existing resource type...`,
+        colors.yellow.bold(
+          `Applying ${diffs.length} diffs to existing resource type... ${remotePath}`,
         ),
       );
       const changeset: EditResourceType = {};
@@ -91,7 +81,6 @@ export class ResourceTypeFile implements ResourceI, PushDiffs {
         v !== null && typeof v !== "undefined"
       );
       if (!hasChanges) {
-        console.log(colors.yellow("! Skipping empty changeset"));
         return;
       }
 
@@ -101,7 +90,7 @@ export class ResourceTypeFile implements ResourceI, PushDiffs {
         requestBody: changeset,
       });
     } else {
-      console.log(colors.yellow("Creating new resource type..."));
+      console.log(colors.yellow.bold("Creating new resource type..."));
       await ResourceService.createResourceType({
         workspace: workspace,
         requestBody: {
@@ -138,7 +127,7 @@ async function push(opts: PushOptions, filePath: string, name: string) {
   console.log(colors.bold.yellow("Pushing resource..."));
 
   await pushResourceType(workspace.workspaceId, filePath, name);
-  console.log(colors.bold.underline.green("Resource successfully pushed"));
+  console.log(colors.bold.underline.green("Resource pushed"));
 }
 
 async function list(opts: GlobalOptions) {

cli/resource.ts

@@ -43,7 +43,7 @@ export class ResourceFile implements Resource2, PushDiffs {
       })
     ) {
       console.log(
-        colors.yellow(`Applying ${diffs.length} diffs to existing resource...`),
+        colors.yellow.bold(`Applying ${diffs.length} diffs to existing resource... ${remotePath}`),
       );
 
       const changeset: EditResource = {
@@ -51,11 +51,7 @@ export class ResourceFile implements Resource2, PushDiffs {
       };
       for (const diff of diffs) {
         if (diff.path[0] === "is_oauth") {
-          console.log(
-            colors.yellow(
-              "! is_oauth has been removed in newer versions. Ignoring.",
-            ),
-          );
+          //is_oauth is not updatable
           continue;
         }
         if (
@@ -64,9 +60,10 @@ export class ResourceFile implements Resource2, PushDiffs {
             diff.path[0] !== "value" && (
               diff.path.length !== 1 ||
               diff.path[0] !== "description"
-            )
+            ) && diff.path[0] !== "resource_type"
           )
         ) {
+          console.log(colors.red("Invalid variable diff with path " + diff.path));
           throw new Error("Invalid folder diff with path " + diff.path);
         }
         if (diff.type === "CREATE" || diff.type === "CHANGE") {
@@ -80,7 +77,6 @@ export class ResourceFile implements Resource2, PushDiffs {
         v !== null && typeof v !== "undefined"
       );
       if (!hasChanges) {
-        console.log(colors.yellow("! Skipping empty changeset"));
         return;
       }
 
@@ -98,7 +94,7 @@ export class ResourceFile implements Resource2, PushDiffs {
         );
       }
 
-      console.log(colors.yellow("Creating new resource..."));
+      console.log(colors.yellow.bold("Creating new resource..."));
       await ResourceService.createResource({
         workspace: workspace,
         requestBody: {
@@ -111,19 +107,10 @@ export class ResourceFile implements Resource2, PushDiffs {
     }
   }
   async push(workspace: string, remotePath: string): Promise<void> {
-    let existing: Resource | undefined;
-    try {
-      existing = await ResourceService.getResource({
-        workspace,
-        path: remotePath,
-      });
-    } catch {
-      existing = undefined;
-    }
     await this.pushDiffs(
       workspace,
       remotePath,
-      microdiff(existing ?? {}, this, { cyclesFix: false }),
+      microdiff({}, this, { cyclesFix: false }),
     );
   }
 }
@@ -156,7 +143,7 @@ async function push(opts: PushOptions, filePath: string, remotePath: string) {
   console.log(colors.bold.yellow("Pushing resource..."));
 
   await pushResource(workspace.workspaceId, filePath, remotePath);
-  console.log(colors.bold.underline.green("Resource successfully pushed"));
+  console.log(colors.bold.underline.green(`Resource ${remotePath} pushed`));
 }
 
 async function list(opts: GlobalOptions) {

cli/script.ts

@@ -11,6 +11,7 @@ import {
   Table,
 } from "./deps.ts";
 import { Any, array, decoverto, model, property } from "./decoverto.ts";
+import { writeAllSync } from "https://deno.land/std@0.176.0/streams/mod.ts";
 
 @model()
 export class ScriptFile {
@@ -41,6 +42,7 @@ export class ScriptFile {
     },
     toPlain: (data) => data,
   })
+  @property(() => String)
   kind?: "script" | "failure" | "trigger" | "command" | "approval";
 
   constructor(summary: string, description: string) {
@@ -76,11 +78,91 @@ async function push(
 
   await requireLogin(opts);
   await pushScript(filePath, contentPath, workspace.workspaceId, remotePath);
-  console.log(colors.bold.underline.green("Script successfully pushed"));
+  console.log(colors.bold.underline.green(`Script ${remotePath} pushed`));
+}
+
+export async function handleScriptMetadata(path: string, workspace: string, alreadySynced: string[]): Promise<boolean> {
+  if (path.endsWith(".script.json")) {
+    const contentPath = await findContentFile(path)
+    return handleFile(contentPath, await Deno.readTextFile(contentPath), workspace, alreadySynced)
+  } else {
+    return false
+  }
+}
+
+export async function handleFile(path: string, content: string, workspace: string, alreadySynced: string[]): Promise<boolean> {
+  if (path.endsWith(".ts") || path.endsWith(".py") || path.endsWith(".go") || path.endsWith(".sh")) {
+    if (alreadySynced.includes(path)) {
+      return true
+    }
+    alreadySynced.push(path)
+    const remotePath = path.substring(0, path.length - 3);
+    const metaPath = remotePath + ".script.json";
+    let typed = undefined
+    try {
+      await Deno.stat(metaPath)
+      typed = JSON.parse(await Deno.readTextFile(metaPath))
+      typed = decoverto.type(ScriptFile).plainToInstance(typed);
+    } catch { }
+    const language = inferContentTypeFromFilePath(path);
+
+    let remote = undefined
+    try {
+      remote = await ScriptService.getScriptByPath({
+        workspace,
+        path: remotePath,
+      });
+    } catch { }
+    
+    if (remote) {
+      if (typed.description === remote.description && content === remote.content && typed.summary === remote.summary && typed.is_template === remote.is_template && typed.kind == remote.kind && remote?.lock == typed.lock?.join('\n') && JSON.stringify(typed.schema) == JSON.stringify(remote.schema)) {
+        console.log(colors.yellow(`No change to push for script ${remotePath}, skipping`))
+        return true
+      }
+
+
+      await ScriptService.createScript({
+        workspace,
+        requestBody: {
+          content,
+          description: typed.description,
+          language,
+          path: remotePath,
+          summary: typed.summary,
+          is_template: typed.is_template,
+          kind: typed.kind,
+          lock: typed.lock,
+          parent_hash: remote.hash,
+          schema: typed.schema,
+        }
+      });
+
+      console.log(colors.yellow.bold(`Creating script with a parent ${remotePath}`))
+    } else {
+      // no parent hash
+      await ScriptService.createScript({
+        workspace: workspace,
+        requestBody: {
+          content,
+          description: typed.description,
+          language,
+          path: remotePath,
+          summary: typed.summary,
+          is_template: typed.is_template,
+          kind: typed.kind,
+          lock: typed.lock,
+          parent_hash: undefined,
+          schema: typed.schema,
+        },
+      });
+      console.log(colors.yellow.bold(`Creating script without parent ${remotePath}`))
+    }
+    return true
+  }
+  return false
 }
 
 export async function findContentFile(filePath: string) {
-  console.log("Searching " + filePath);
   const candidates = [
     filePath.replace(".script.json", ".ts"),
     filePath.replace(".script.json", ".py"),
@@ -104,7 +186,7 @@ export async function findContentFile(filePath: string) {
   if (validCandidates.length > 1) {
     throw new Error(
       "No content path given and more then one candidate found: " +
-        validCandidates.join(", "),
+      validCandidates.join(", "),
     );
   }
   if (validCandidates.length < 1) {
@@ -196,66 +278,41 @@ async function list(opts: GlobalOptions & { showArchived?: boolean }) {
   }
 
   new Table()
-    .header(["path", "hash", "kind", "language", "created at", "created by"])
+    .header(["path", "summary", "language", "created by"])
     .padding(2)
     .border(true)
     .body(
       total.map((x) => [
         x.path,
-        x.hash,
-        x.kind,
+        x.summary,
         x.language,
-        x.created_at,
         x.created_by,
       ]),
     )
     .render();
 }
 
-export async function resolve(inputs: string[]): Promise<Record<string, any>> {
-  let result = {};
-
-  if (!inputs) {
-    return result;
+export async function resolve(input: string): Promise<Record<string, any>> {
+  if (!input) {
+    throw new Error("No data given");
   }
 
-  for (const input of inputs) {
-    let data: string;
-    if (input.startsWith("@")) {
-      if (input == "@-") {
-        data = new TextDecoder().decode(await readAll(Deno.stdin));
-      } else {
-        data = await Deno.readTextFile(input.substring(1));
-      }
-    } else {
-      if (input.startsWith("{")) {
-        data = input;
-      } else {
-        const key = input.split("=", 1)[0];
-        const value = input.substring(key.length + 1);
-        let o;
-        try {
-          o = JSON.parse(value);
-        } catch {
-          o = value;
-        }
-        data = JSON.stringify(Object.fromEntries([[key, o]]));
-      }
-    }
-    let jsonObj;
-    try {
-      jsonObj = JSON.parse(data);
-    } catch {
-      jsonObj = data;
-    }
-    result = { ...result, ...jsonObj };
+  if (input == "@-") {
+    input = new TextDecoder().decode(await readAll(Deno.stdin));
+  } if (input[0] == "@") {
+    input = await Deno.readTextFile(input.substring(1));
+  }
+  try {
+    return JSON.parse(input);
+  } catch (e) {
+    console.error("Impossible to parse input as JSON", input)
+    throw e
   }
-  return result;
 }
 
 async function run(
   opts: GlobalOptions & {
-    input: string[];
+    data?: string;
     silent: boolean;
   },
   path: string,
@@ -263,7 +320,8 @@ async function run(
   const workspace = await resolveWorkspace(opts);
   await requireLogin(opts);
 
-  const input = await resolve(opts.input);
+
+  const input = opts.data ? await resolve(opts.data) : {};
   const id = await JobService.runScriptByPath({
     workspace: workspace.workspaceId,
     path,
@@ -296,7 +354,9 @@ export async function track_job(workspace: string, id: string) {
     const result = await JobService.getCompletedJob({ workspace, id });
 
     console.log(result.logs);
+    console.log()
     console.log(colors.bold.underline.green("Job Completed"));
+    console.log()
     return;
   } catch {
     /* ignore */
@@ -335,7 +395,7 @@ export async function track_job(workspace: string, id: string) {
     }
 
     if (updates.new_logs) {
-      console.log(updates.new_logs);
+      writeAllSync(Deno.stdout, new TextEncoder().encode(updates.new_logs));
       logOffset += updates.new_logs.length;
     }
 
@@ -358,12 +418,15 @@ export async function track_job(workspace: string, id: string) {
     if ((final_job.logs?.length ?? -1) > logOffset) {
       console.log(final_job.logs!.substring(logOffset));
     }
-
+    console.log("\n")
     if (final_job.success) {
       console.log(colors.bold.underline.green("Job Completed"));
+
     } else {
       console.log(colors.bold.underline.red("Job Completed"));
     }
+    console.log()
+
   } catch {
     console.log("Job appears to have completed, but no data can be retrieved");
   }
@@ -398,8 +461,8 @@ const command = new Command()
   .command("run", "run a script by path")
   .arguments("<path:string>")
   .option(
-    "-i --input [inputs...:string]",
-    "Inputs specified as JSON objects or simply as <name>=<value>. Supports file inputs using @<filename> and stdin using @- these also need to be formatted as JSON. Later inputs override earlier ones.",
+    "-d --data <data:string>",
+    "Inputs specified as a JSON string or a file using @<filename> or stdin using @-.",
   )
   .option(
     "-s --silent",

cli/types.ts

@@ -6,6 +6,7 @@ import { ScriptFile } from "./script.ts";
 import { VariableFile } from "./variable.ts";
 import { path } from "./deps.ts";
 import { FolderFile } from "./folder.ts";
+import { AppFile } from "./apps.ts";
 
 // TODO: Remove this & replace with a "pull" that lets the object either pull the remote version or return undefined.
 // Then combine those with diffing, which then gives the new push impl
@@ -90,7 +91,8 @@ export function inferTypeFromPath(
   | FlowFile
   | ResourceFile
   | ResourceTypeFile
-  | FolderFile {
+  | FolderFile
+  | AppFile {
   const typeEnding = getTypeStrFromPath(p);
 
   if (typeEnding === "folder") {
@@ -105,6 +107,8 @@ export function inferTypeFromPath(
     return decoverto.type(ResourceFile).plainToInstance(obj);
   } else if (typeEnding === "resource-type") {
     return decoverto.type(ResourceTypeFile).plainToInstance(obj);
+  } else if (typeEnding === "app") {
+    return decoverto.type(AppFile).plainToInstance(obj);
   } else {
     throw new Error("infer type unreachable");
   }
@@ -112,12 +116,17 @@ export function inferTypeFromPath(
 
 export function getTypeStrFromPath(
   p: string,
-): "script" | "variable" | "flow" | "resource" | "resource-type" | "folder" {
+):
+  | "script"
+  | "variable"
+  | "flow"
+  | "resource"
+  | "resource-type"
+  | "folder"
+  | "app" {
   const parsed = path.parse(p);
-  if (parsed.ext !== ".json") {
-    throw new Error(
-      "Cannot infer type of non-json file " + JSON.stringify(parsed),
-    );
+  if (parsed.ext == ".go" || parsed.ext == ".ts" || parsed.ext == ".sh" || parsed.ext == ".py") {
+    return 'script'
   }
 
   if (parsed.name === "folder.meta") {
@@ -128,7 +137,7 @@ export function getTypeStrFromPath(
   if (
     typeEnding === "script" || typeEnding === "variable" ||
     typeEnding === "flow" || typeEnding === "resource" ||
-    typeEnding === "resource-type"
+    typeEnding === "resource-type" || typeEnding === "app"
   ) {
     return typeEnding;
   } else {

cli/variable.ts

@@ -11,7 +11,6 @@ import {
   colors,
   Command,
   EditVariable,
-  ListableVariable,
   microdiff,
   Table,
   VariableService,
@@ -67,7 +66,7 @@ export class VariableFile implements Resource, PushDiffs {
     if (await VariableService.existsVariable({ workspace, path: remotePath })) {
       console.log(
         colors.bold.yellow(
-          `Applying ${diffs.length} diffs to existing variable...`,
+          `Applying ${diffs.length} diffs to existing variable... ${remotePath}`,
         ),
       );
       const changeset: EditVariable = {};
@@ -76,11 +75,12 @@ export class VariableFile implements Resource, PushDiffs {
           diff.type !== "REMOVE" &&
           (
             diff.path.length !== 1 ||
-            !["path", "value", "is_secret", "description"].includes(
+            !["path", "value", "is_secret", "description", "account", "is_oauth"].includes(
               diff.path[0] as string,
             )
           )
         ) {
+          console.log(colors.red("Invalid variable diff with path " + diff.path));
           throw new Error("Invalid variable diff with path " + diff.path);
         }
         if (diff.type === "CREATE" || diff.type === "CHANGE") {
@@ -94,19 +94,21 @@ export class VariableFile implements Resource, PushDiffs {
         v !== null && typeof v !== "undefined"
       );
       if (!hasChanges) {
-        console.log(colors.yellow("! Skipping empty changeset"));
         return;
       }
 
       await VariableService.updateVariable({
         workspace,
         path: remotePath,
+        alreadyEncrypted: true,
         requestBody: changeset,
       });
+
     } else {
-      console.log(colors.yellow("Creating new variable..."));
+      console.log(colors.yellow.bold("Creating new variable..."));
       await VariableService.createVariable({
         workspace,
+        alreadyEncrypted: true,
         requestBody: {
           path: remotePath,
           description: this.description,
@@ -119,19 +121,10 @@ export class VariableFile implements Resource, PushDiffs {
     }
   }
   async push(workspace: string, remotePath: string): Promise<void> {
-    let existing: ListableVariable | undefined;
-    try {
-      existing = await VariableService.getVariable({
-        workspace: workspace,
-        path: remotePath,
-      });
-    } catch {
-      existing = undefined;
-    }
     await this.pushDiffs(
       workspace,
       remotePath,
-      microdiff(existing ?? {}, this, { cyclesFix: false }),
+      microdiff({}, this, { cyclesFix: false }),
     );
   }
 }
@@ -152,7 +145,7 @@ async function push(opts: GlobalOptions, filePath: string, remotePath: string) {
   console.log(colors.bold.yellow("Pushing variable..."));
 
   await pushVariable(workspace.workspaceId, filePath, remotePath);
-  console.log(colors.bold.underline.green("Variable successfully pushed"));
+  console.log(colors.bold.underline.green(`Variable ${remotePath} pushed`));
 }
 
 export async function pushVariable(

cli/vhs/example.flow.json

@@ -9,7 +9,7 @@
           "lock": "",
           "path": null,
           "type": "rawscript",
-          "content": "// import * as wmill from \"https://deno.land/x/windmill@v1.50.0/mod.ts\"\n\nexport async function main(x: string) {\n  console.log(\"Hello from Deno! The argument x is \" + x);\n  return x;\n}\n",
+          "content": "// import * as wmill from \"https://deno.land/x/windmill@v1.69.3/mod.ts\"\n\nexport async function main(x: string) {\n  console.log(\"Hello from Deno! The argument x is \" + x);\n  return x;\n}\n",
           "language": "deno",
           "input_transforms": {
             "x": {

cli/workspace.ts

@@ -9,9 +9,11 @@ import {
   Input,
   setClient,
   Table,
+  UserService,
   WorkspaceService,
 } from "./deps.ts";
 import { decoverto, model, property } from "./decoverto.ts";
+import { requireLogin } from "./context.ts";
 
 @model()
 export class Workspace {
@@ -137,6 +139,8 @@ async function list(opts: GlobalOptions) {
       }),
     )
     .render();
+
+  console.log('Active: ' + colors.green.bold(activeName || 'none'))
 }
 
 async function switchC(opts: GlobalOptions, workspaceName: string) {
@@ -151,7 +155,11 @@ async function switchC(opts: GlobalOptions, workspaceName: string) {
 
   const all = await allWorkspaces();
   if (all.findIndex((x) => x.name === workspaceName) === -1) {
-    console.log(colors.red.bold("! This workspace name does not exist."));
+    console.log(colors.red.bold(`! This workspace profile ${workspaceName} does not exist locally.`));
+    console.log("available workspace profiles:")
+    for (const w of all) {
+      console.log('  - ' + w.name)
+    }
     return;
   }
 
@@ -184,14 +192,8 @@ export async function add(
     workspaceName = await Input.prompt("Name this workspace:");
   }
 
-  const all = await allWorkspaces();
-  if (all.findIndex((x) => x.name === workspaceName) !== -1) {
-    console.log(colors.red.bold("! Workspace name already exists"));
-    return;
-  }
-
   if (!workspaceId) {
-    workspaceId = await Input.prompt("Enter the ID of this workspace");
+    workspaceId = await Input.prompt({ message: "Enter the ID of this workspace", default: workspaceName, suggestions: [workspaceName] });
   }
 
   if (!remote) {
@@ -203,7 +205,13 @@ export async function add(
       remote = url.toString();
     } catch {
       // not a url
-      remote = new URL(await Input.prompt("Enter the Remote URL")).toString();
+      remote = new URL(
+        await Input.prompt({
+          message: "Enter the Remote URL",
+          suggestions: ["https://app.windmill.dev/"],
+          default: "https://app.windmill.dev/"
+        }),
+      ).toString();
     }
   }
   remote = new URL(remote).toString(); // add trailing slash in all cases!
@@ -213,18 +221,24 @@ export async function add(
     token = await loginInteractive(remote);
   }
 
+  setClient(
+    token,
+    remote.endsWith("/") ? remote.substring(0, remote.length - 1) : remote,
+  );
+  let alreadyExists = false
+  try {
+    alreadyExists = await WorkspaceService.existsWorkspace({
+      requestBody: { id: workspaceId },
+    })
+  } catch (e) {
+    console.log(colors.red.bold("! Credentials or instance is invalid. Aborting."));
+    throw e
+  }
   if (opts.create) {
-    setClient(
-      token,
-      remote.endsWith("/") ? remote.substring(0, remote.length - 1) : remote,
-    );
-
     if (
-      !await WorkspaceService.existsWorkspace({
-        requestBody: { id: workspaceId },
-      })
+      !alreadyExists
     ) {
-      console.log(colors.yellow("Workspace does not exist. Creating..."));
+      console.log(colors.yellow(`Workspace at id ${workspaceId} on ${remote} does not exist. Creating...`));
       await WorkspaceService.createWorkspace({
         requestBody: {
           id: workspaceId,
@@ -233,6 +247,14 @@ export async function add(
         },
       });
     }
+  } else if (!alreadyExists) {
+    console.log(colors.red.bold(`! Workspace at id ${workspaceId} on ${remote} does not exist. Re-run with --create to create it. Aborting.`));
+    console.log("On that instance and with those credentials, the workspaces that you can access are:")
+    const workspaces = await WorkspaceService.listWorkspaces()
+    for (const workspace of workspaces) {
+      console.log(`- ${workspace.id} (name: ${workspace.name})`)
+    }
+    Deno.exit(1);
   }
 
   await addWorkspace({
@@ -240,28 +262,40 @@ export async function add(
     remote: remote,
     workspaceId: workspaceId,
     token: token,
-  });
+  }, opts);
   await Deno.writeTextFile(
     (await getRootStore()) + "/activeWorkspace",
     workspaceName,
   );
-  console.log(colors.green.underline("Succesfully added workspace!"));
+  console.log(colors.green.underline(`Added workspace ${workspaceName} for ${workspaceId} on ${remote}!`));
 }
 
-export async function addWorkspace(workspace: Workspace) {
+export async function addWorkspace(workspace: Workspace, opts: any) {
   workspace.remote = new URL(workspace.remote).toString(); // add trailing slash in all cases!
   const file = await Deno.open((await getRootStore()) + "remotes.ndjson", {
     append: true,
     write: true,
-    read: false,
+    read: true,
     create: true,
   });
+  await removeWorkspace(workspace.name, true, opts);
   await file.write(new TextEncoder().encode(JSON.stringify(workspace) + "\n"));
   file.close();
 }
 
-export async function removeWorkspace(name: string) {
+export async function removeWorkspace(name: string, silent: boolean, opts: any) {
   const orgWorkspaces = await allWorkspaces();
+  if (orgWorkspaces.findIndex((x) => x.name === name) === -1) {
+    if (!silent) {
+      console.log(colors.red.bold(`! Workspace profile ${name} does not exist locally`));
+      console.log("available workspace profiles:")
+      await list(opts)
+    }
+    return;
+  }
+  if (silent) {
+    console.log(colors.yellow(`Replacing existing workspace ${name}`))
+  }
   await Deno.writeTextFile(
     (await getRootStore()) + "remotes.ndjson",
     orgWorkspaces
@@ -269,19 +303,29 @@ export async function removeWorkspace(name: string) {
       .map((x) => JSON.stringify(x))
       .join("\n"),
   );
+  if (!silent) {
+    console.log(colors.green.underline(`Succesfully removed workspace ${name}!`));
+  }
 }
 
 async function remove(_opts: GlobalOptions, name: string) {
-  await removeWorkspace(name);
-  console.log(colors.green.underline("Succesfully removed workspace!"));
+  await removeWorkspace(name, false, _opts);
+}
+
+async function whoami(_opts: GlobalOptions) {
+  await requireLogin(_opts)
+  console.log(await UserService.globalWhoami())
+  const activeName = await getActiveWorkspaceName(_opts);
+  console.log('Active: ' + colors.green.bold(activeName || 'none'))
 }
 
 const command = new Command()
   .description("workspace related commands")
   .action(list as any)
   .command("switch")
+  .complete("workspace", async () => (await allWorkspaces()).map((x) => x.name))
   .description("Switch to another workspace")
-  .arguments("<workspace_name:string>")
+  .arguments("<workspace_name:string:workspace>")
   .action(switchC as any)
   .command("add")
   .description("Add a workspace")
@@ -302,6 +346,9 @@ const command = new Command()
   .command("remove")
   .description("Remove a workspace")
   .arguments("<workspace_name:string>")
-  .action(remove as any);
+  .action(remove as any)
+  .command("whoami")
+  .description("Show the currently active user")
+  .action(whoami as any);
 
 export default command;

deno-client/build.sh

@@ -4,8 +4,9 @@ set -e
 npm ci --ignore-scripts
 npx --yes openapi-typescript-codegen --input ../backend/windmill-api/openapi.yaml \
  --output ./src --useOptions \
- && sed -i '213 i \\    request.referrerPolicy = \"no-referrer\"\n' src/core/request.ts
-npx --yes denoify
+ && sed -i '213 i \\    request.referrerPolicy = \"no-referrer\"\n' src/core/request.ts 
+ npx --yes denoify
 rm -rf windmill-api
 mv deno_dist windmill-api
+sed -i 's/buffer DENOIFY: UNKNOWN NODE BUILTIN/node:buffer/' ./windmill-api/core/request.ts 
 rm -rf src/

deno-client/mod.ts

@@ -14,7 +14,6 @@ BigInt.prototype.toJSON = function () {
 };
 
 export { pgSql, pgClient } from './pg.ts'
-export { mySql, mysqlClient } from './mysql.ts'
 
 export type Sql = string
 export type Email = string

deno-client/pg.ts

@@ -25,9 +25,8 @@ import { type Resource } from "./mod.ts"
 export function pgClient(
     db: Resource<"postgresql">
 ) {
-    db.database = db.dbname
-    db.hostname = db.host
-    return new Client(db)
+    const databaseUrl = 'postgresql://' + db.user + ':' + db.password + '@' + db.host + ':' + db.port + '/' + db.dbname + '?sslmode=' + db.sslmode
+    return new Client(databaseUrl)
 }
 
 /**

docker-compose.yml

@@ -38,7 +38,9 @@ services:
     depends_on:
       db:
         condition: service_healthy
-
+    # volumes:
+      # - ./oauth.json/:/usr/src/app/oauth.json
+      
   windmill_worker:
     image: ghcr.io/windmill-labs/windmill:main
     deploy:
@@ -47,7 +49,7 @@ services:
     environment:
       - DATABASE_URL=postgres://postgres:${DB_PASSWORD}@db/windmill?sslmode=disable
       - BASE_URL=http://${WM_BASE_URL}
-      - BASE_INTERNAL_URL=http://localhost:8000
+      - BASE_INTERNAL_URL=http://windmill_server:8000
       - RUST_LOG=info
       - NUM_WORKERS=1
       - DISABLE_SERVER=true
@@ -61,7 +63,6 @@ services:
     # to mount the worker folder to debug,, KEEP_JOB_DIR=true and mount /tmp/windmill
     volumes:
       - worker_dependency_cache:/tmp/windmill/cache
-      # - ./oauth.json/:/usr/src/app/oauth.json
 
   lsp:
     image: ghcr.io/windmill-labs/windmill-lsp:latest

frontend/README_DEV.md

@@ -1,6 +1,6 @@
 # Developing
 
-## Starting the Development Server
+## Starting The Development Server
 
 Once you've created a project and installed dependencies with `npm install` (or
 `pnpm install` or `yarn`), start a development server:
@@ -17,7 +17,7 @@ In the root folder:
 
 ```bash
 docker build . -t windmill
-docker-compose up db server
+docker-compose up db windmill_server windmill_worker
 ```
 
 ### 2. Backend is run by cargo
@@ -25,9 +25,9 @@ docker-compose up db server
 **Prerequisites**
 
 - Install Rust [as explained on the website](https://www.rust-lang.org/tools/install).
-- Install llvm 
+- Install llvm
 
-  **on OSX:**
+  **On OSX:**
   ```bash
   brew install llvm caddy gsed
   
@@ -36,7 +36,20 @@ docker-compose up db server
   
   # now, restart your shell. You should now have the `lld` binary on your PATH.
   ```
-  
+- To test that you have Rust and Cargo installed run `cargo --version`
+
+- In your terminal, go to the backend directory and run `cargo build`
+- Run `cargo run`
+
+**Known issue on M1 Mac while running `cargo build`**
+- You may encounter `linking with cc failed` build time error.
+- To solve this run:
+  ```bash
+  echo 'export RUSTFLAGS="-L/opt/homebrew/opt/libomp/lib"' >> ~/.zshrc
+  source ~/.zshrc
+  ```
+
+
 **Do a Frontend Build**
 
 In order to run the backend, you need to have a frontend build inside `frontend/build/`.
@@ -56,6 +69,14 @@ npm run build
 # now, you'll have a `frontend/build` folder.
 ```
 
+**Known issue while running `npm run build`**
+- You may encounter `FATAL ERROR: Ineffective mark-compacts near heap limit Allocation failed - JavaScript heap out of memory` error.
+- To solve this run:
+```bash
+export NODE_OPTIONS=--max_old_space_size=8096
+```
+- run `npm run build` again
+
 In the root folder:
 
 ```bash
@@ -82,7 +103,7 @@ sudo caddy run --config ./Caddyfile
 
 and then go to <http://localhost>
 
-### Backend is run by remote!
+### 3. Backend is run by remote!
 
 ```bash
 sudo caddy run --config ./CaddyfileRemote
@@ -119,8 +140,8 @@ Recommended config for VS Code:
 
 ```json
 "[svelte]": {
-        "editor.defaultFormatter": "esbenp.prettier-vscode"
-    }
+"editor.defaultFormatter": "esbenp.prettier-vscode"
+}
 ```
 
 - turn _format on save_ on

frontend/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "windmill",
-	"version": "1.62.0",
+	"version": "1.76.0",
 	"scripts": {
 		"dev": "vite dev",
 		"build": "vite build",
@@ -15,17 +15,18 @@
 		"test": "playwright test --config=tests-out/playwright.config.js"
 	},
 	"devDependencies": {
-		"@playwright/test": "^1.29.2",
+		"@playwright/test": "^1.31.1",
 		"@sveltejs/adapter-static": "^1.0.0",
 		"@sveltejs/kit": "^1.0.0-next.589",
 		"@sveltejs/package": "^1.0.2",
 		"@tailwindcss/forms": "^0.5.3",
 		"@tailwindcss/typography": "^0.5.8",
+		"@types/d3": "^7.4.0",
+		"@types/d3-zoom": "^3.0.2",
 		"@types/node": "^18.11.18",
 		"@types/vscode": "~1.74.0",
 		"@typescript-eslint/eslint-plugin": "^5.49.0",
 		"@typescript-eslint/parser": "^5.48.0",
-		"@windmill-labs/svelvet": "^4.0.20",
 		"@zerodevx/svelte-toast": "^0.8.1",
 		"autoprefixer": "^10.4.13",
 		"cssnano": "^5.1.14",
@@ -33,44 +34,51 @@
 		"eslint": "^8.28.0",
 		"eslint-config-prettier": "^8.6.0",
 		"eslint-plugin-svelte3": "^4.0.0",
-		"lucide-svelte": "^0.104.0",
+		"ol": "^7.2.2",
 		"openapi-typescript-codegen": "^0.23.0",
 		"path-browserify": "^1.0.1",
+		"pdfjs-dist": "^3.4.120",
 		"postcss": "^8.4.18",
 		"postcss-load-config": "^4.0.1",
 		"prettier": "^2.8.3",
 		"prettier-plugin-svelte": "^2.9.0",
 		"simple-svelte-autocomplete": "^2.5.1",
 		"stylelint-config-recommended": "^9.0.0",
-		"svelte": "^3.53.1",
+		"svelte": "^3.55.1",
 		"svelte-awesome": "^3.0.0",
+		"svelte-awesome-color-picker": "^2.4.1",
 		"svelte-check": "^3.0.2",
-		"svelte-dnd-action": "^0.9.22",
-		"svelte-grid": "^5.1.1",
-		"svelte-heros": "^2.3.5",
 		"svelte-highlight": "^6.2.1",
 		"svelte-overlay": "^1.4.1",
 		"svelte-popperjs": "^1.3.2",
 		"svelte-preprocess": "^5.0.1",
+		"svelte-range-slider-pips": "^2.1.1",
 		"svelte-splitpanes": "^0.7.3",
-		"svelte2tsx": "^0.5.22",
+		"svelte2tsx": "^0.6.1",
 		"tailwindcss": "^3.2.2",
 		"tslib": "^2.4.1",
 		"typescript": "^4.9.3",
-		"vite": "^4.0.4"
+		"vite": "^4.0.4",
+		"yootils": "^0.3.1"
 	},
 	"type": "module",
 	"dependencies": {
 		"@fortawesome/free-brands-svg-icons": "^6.2.1",
 		"@fortawesome/free-solid-svg-icons": "^6.2.1",
 		"@leeoniya/ufuzzy": "^0.9.1",
+		"@popperjs/core": "^2.11.6",
 		"@redocly/json-to-json-schema": "^0.0.1",
 		"@tanstack/svelte-table": "^8.7.6",
+		"ag-grid-svelte": "^0.1.4",
 		"async-mutex": "^0.4.0",
+		"chart.js": "^3.9.1",
 		"chartjs-adapter-date-fns": "^3.0.0",
 		"chartjs-plugin-zoom": "^2.0.0",
+		"d3-zoom": "^3.0.0",
 		"date-fns": "^2.29.3",
+		"fast-equals": "^4.0.3",
 		"highlight.js": "^11.7.0",
+		"lucide-svelte": "^0.115.0",
 		"monaco-editor": "0.33.0",
 		"monaco-editor-workers": "0.33.0",
 		"monaco-languageclient": "2.1.0",
@@ -78,10 +86,11 @@
 		"svelte-autosize": "^1.0.1",
 		"svelte-chartjs": "^3.1.0",
 		"svelte-portal": "^2.2.0",
-		"svelte-select": "^5.0.2",
+		"svelte-select": "^5.3.1",
+		"tailwind-merge": "^1.9.1",
 		"vscode-ws-jsonrpc": "^2.0.1"
 	},
 	"peerDependencies": {
-		"svelte": "^3.53.1"
+		"svelte": "^3.55.1"
 	}
 }

frontend/src/global.d.ts

@@ -1,77 +1,2 @@
 /// <reference types="@sveltejs/kit" />
 
-declare type Item = import('svelte-dnd-action').Item
-declare type DndEvent<ItemType = Item> = import('svelte-dnd-action').DndEvent<ItemType>
-declare namespace svelte.JSX {
-	interface HTMLAttributes<T> {
-		onconsider?: (event: CustomEvent<DndEvent<ItemType>> & { target: EventTarget & T }) => void
-		onfinalize?: (event: CustomEvent<DndEvent<ItemType>> & { target: EventTarget & T }) => void
-	}
-}
-
-declare module 'svelte-grid' {
-	import type { SvelteComponentTyped } from 'svelte'
-
-	export interface Size {
-		w: number
-		h: number
-	}
-
-	export interface Positon {
-		x: number
-		y: number
-	}
-
-	interface ItemLayout extends Size, Positon {
-		fixed?: boolean
-		resizable?: boolean
-		draggable?: boolean
-		customDragger?: boolean
-		customResizer?: boolean
-		min?: Size
-		max?: Size
-	}
-
-	export type Item<T> = T & { [width: number]: ItemLayout; data: any }
-	export type FilledItem<T> = T & { [width: number]: Required<ItemLayout>; data: any }
-
-	export interface Props<T> {
-		fillSpace?: boolean
-		items: FilledItem<T>[],
-		rowHeight: number
-		cols: [number, number][]
-		gap?: [number, number]
-		fastStart?: boolean
-		throttleUpdate?: number
-		throttleResize?: number
-
-		scroller?: undefined
-		sensor?: number
-	}
-
-	export interface Slots<T> {
-		default: { item: ItemLayout; dataItem: Item<T> }
-	}
-
-	export default class Grid<T = {}> extends SvelteComponentTyped<
-		Props<T>,
-		{
-			pointerup: CustomEvent<{ id: string }>
-		},
-		Slots<T>
-	> { }
-}
-
-declare module 'svelte-grid/build/helper/index.mjs' {
-	import { ItemLayout } from 'svelte-grid'
-
-	const x: {
-		normalize(items: any[], col: any): unknown[]
-		adjust(items: any[], col: any): unknown[]
-		findSpace(item: any, items: any, cols: any): unknown
-
-		item<T>(obj: ItemLayout): Required<ItemLayout>
-	}
-
-	export default x
-}