chore(main): release 1.56.0 (#1011)

* chore(main): release 1.56.0

* Apply automatic changes

Co-authored-by: rubenfiszel <rubenfiszel@users.noreply.github.com>

.github/workflows/docker-image-arm.yml

@@ -59,4 +59,4 @@ jobs:
             ${{ steps.meta-slim-public.outputs.labels }}
             org.opencontainers.image.licenses=AGPLv3
           cache-from: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}-arm:buildcache
-          cache-to: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}-arm:buildcache
+          cache-to: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}-arm:buildcache,mode=max

.github/workflows/docker-image.yml

@@ -26,6 +26,13 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
 
+      - name: Login to registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Docker meta
         id: meta-slim
         if: github.event_name == 'pull_request'
@@ -51,16 +58,53 @@ jobs:
           labels: |
             ${{ steps.meta-slim.outputs.labels }}
           cache-from: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}-slim:buildcache
-          cache-to: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}-slim:buildcache
-
+          cache-to: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}-slim:buildcache,mode=max
 
       - name: Docker meta
-        id: meta-slim-public
+        id: meta-public
         if: github.event_name != 'pull_request'
         uses: docker/metadata-action@v4
         with:
           images: |
             ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+
+
+      - name: Build and push publicly
+        if: github.event_name != 'pull_request'
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          push: true
+          tags: |
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
+            ${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}-slim:latest
+            ${{ steps.meta-public.outputs.tags }}
+          labels: |
+            ${{ steps.meta-public.outputs.labels }}
+            org.opencontainers.image.licenses=AGPLv3
+          cache-from: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}-slim:buildcache
+          cache-to: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}-slim:buildcache,mode=max
+
+  build_ee:
+    runs-on: [self-hosted, new]
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Docker meta
+        id: meta-slim
+        if: github.event_name == 'pull_request'
+        uses: docker/metadata-action@v4
+        with:
+          images: |
             ${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}-slim
           tags: |
             type=ref,event=branch
@@ -68,6 +112,20 @@ jobs:
             type=semver,pattern={{version}}
             type=semver,pattern={{major}}.{{minor}}
 
+
+      - name: Docker meta
+        id: meta-ee-public
+        if: github.event_name != 'pull_request'
+        uses: docker/metadata-action@v4
+        with:
+          images: |
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+
       - name: Login to registry
         uses: docker/login-action@v2
         if: github.event_name != 'pull_request'
@@ -76,22 +134,23 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Build and push publicly
+      - name: Build and push publicly ee
         if: github.event_name != 'pull_request'
         uses: docker/build-push-action@v3
         with:
           context: .
           push: true
+          build-args: |
+            features=enterprise
+            nsjail=true
           tags: |
-            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
-            ${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}-slim:latest
-            ${{ steps.meta-slim-public.outputs.tags }}
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest
+            ${{ steps.meta-ee-public.outputs.tags }}
           labels: |
-            ${{ steps.meta-slim-public.outputs.labels }}
-            org.opencontainers.image.licenses=AGPLv3
-          cache-from: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}-slim:buildcache
-          cache-to: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}-slim:buildcache
-
+            ${{ steps.meta-ee-public.outputs.labels }}
+            org.opencontainers.image.licenses=Windmill-Enterprise-License
+          cache-from: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}-ee:buildcache
+          cache-to: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}-ee:buildcache,mode=max
 
   playwright:
     runs-on: [self-hosted, new]
@@ -125,9 +184,10 @@ jobs:
         run: docker kill ${{ steps.docker-container.outputs.id }}
         if: always()
 
+
   publish_privately_heavy:
+    needs: [build_ee]
     runs-on: [self-hosted, new]
-    needs: [build]
     if: github.event_name != 'pull_request'
     steps:
       - uses: actions/checkout@v3
@@ -157,7 +217,14 @@ jobs:
           registry: ${{ env.ECR_REGISTRY }}
           username: ${{ secrets.AWS_ACCESS_KEY_ID }}
           password: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          
+
+      - name: Login to registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Build and push privately
         uses: docker/build-push-action@v3
         if: github.event_name != 'pull_request'
@@ -168,12 +235,12 @@ jobs:
           tags: |
             ${{ steps.meta-heavy.outputs.tags }}
           labels: ${{ steps.meta-heavy.outputs.labels }}
-          cache-from: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache
-          cache-to: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache,mode=max
+          cache-from: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}-heavy:buildcache
+          cache-to: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}-heavy:buildcache,mode=max
 
   publish_privately_helm:
     runs-on: [self-hosted, new]
-    needs: [build]
+    needs: [build_ee]
     if: github.event_name != 'pull_request'
     steps:
       - uses: actions/checkout@v3
@@ -183,6 +250,13 @@ jobs:
         uses: docker/setup-buildx-action@v2
  
 
+      - name: Login to registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Login to ECR
         if: github.event_name != 'pull_request'
         uses: docker/login-action@v2
@@ -200,5 +274,5 @@ jobs:
           file: ./docker/DockerfileHelm
           tags: |
             ${{ env.ECR_REGISTRY }}/${{ env.IMAGE_NAME }}:helm
-          cache-from: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache
-          cache-to: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache,mode=max
+          cache-from: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}-helm:buildcache
+          cache-to: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}-helm:buildcache,mode=max

.github/workflows/pull-hub.yml

@@ -1,20 +0,0 @@
-name: Pull Hub Items
-on:
-  schedule:
-    # * is a special character in YAML so you have to quote this string
-    - cron: "0 0 */1 * *"
-  # Allows you to run this workflow manually from the Actions tab
-  workflow_dispatch:
-
-jobs:
-  change_version:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: Pull hub
-        run: ./.github/pull_hub_items.sh
-      - name: Create Pull Request
-        uses: peter-evans/create-pull-request@v4
-        with:
-          title: sync hub items with community
-          commit-message: sync hub items with community

.github/workflows/pypi_on_release.yml

@@ -28,6 +28,9 @@ jobs:
     needs: [publish_pypi]
     runs-on: [self-hosted, new]
     steps:
+      - name: Sleep for 30 seconds waiting for pypi to update index
+        run: sleep 30s
+        shell: bash
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0

.gitignore

@@ -4,3 +4,4 @@ nohup.out
 local/
 frontend/src/routes/test.svelte
 CaddyfileRemoteMalo
+*.swp

CHANGELOG.md

@@ -1,6 +1,196 @@
 # Changelog
 
 
+## [1.56.0](https://github.com/windmill-labs/windmill/compare/v1.55.0...v1.56.0) (2022-12-23)
+
+
+### Features
+
+* add move to drawer for script and flows ([f73dbd8](https://github.com/windmill-labs/windmill/commit/f73dbd8039b3c987ca94e5b56f0ecdea93cbd1b8))
+* add operator mode ([3485b07](https://github.com/windmill-labs/windmill/commit/3485b07b2548b7ea8fbd2b6b31b91e2d36d072ef))
+* auto-invite from same domain ([2bae50f](https://github.com/windmill-labs/windmill/commit/2bae50f3910a99a87efa402a9eef566320fe1f68))
+* **backend:** add SUPERADMIN_SECRET as an env set superadmin ([c283112](https://github.com/windmill-labs/windmill/commit/c28311242d58af12a039b81a5e5c90688022ce8c))
+* **frontend:** Add an input field to edit inline script name ([#1033](https://github.com/windmill-labs/windmill/issues/1033)) ([95a0b9c](https://github.com/windmill-labs/windmill/commit/95a0b9ceae73e291a0def340e935658b6c2ac3a5))
+* **frontend:** Add app number input ([#1010](https://github.com/windmill-labs/windmill/issues/1010)) ([2fe927f](https://github.com/windmill-labs/windmill/commit/2fe927f7fdc1309c7bad8b90fb7e0cc41d364b3f))
+* **frontend:** Add form component + fix connection bug ([#1012](https://github.com/windmill-labs/windmill/issues/1012)) ([424c31c](https://github.com/windmill-labs/windmill/commit/424c31c54a2652b89f9b06499a5aaf1cc0f00ad9))
+* **frontend:** Add select component to app builder ([#1021](https://github.com/windmill-labs/windmill/issues/1021)) ([08071bb](https://github.com/windmill-labs/windmill/commit/08071bb66b4fc40e3b984ffb459e5d52d5816298))
+* **frontend:** Add the ability to lock components so they don't move around ([#1035](https://github.com/windmill-labs/windmill/issues/1035)) ([26a6de2](https://github.com/windmill-labs/windmill/commit/26a6de247c3566bfa524b8fa4f8fc212ca557874))
+* **frontend:** Align output panel UI ([#1025](https://github.com/windmill-labs/windmill/issues/1025)) ([0e871ca](https://github.com/windmill-labs/windmill/commit/0e871ca8432d4f0bc68543b4a3f3bf8f8af99669))
+* **frontend:** App builder password and date input ([#1022](https://github.com/windmill-labs/windmill/issues/1022)) ([4651c9d](https://github.com/windmill-labs/windmill/commit/4651c9d8cd644e59bfd4f57be0bcecc01962a536))
+* **frontend:** AppTable v2 + Inline script panel ([#1023](https://github.com/windmill-labs/windmill/issues/1023)) ([f6df3ae](https://github.com/windmill-labs/windmill/commit/f6df3ae36748a1271625c3f4b50ca66f604d79f7))
+* **frontend:** Fix component synchro ([#1038](https://github.com/windmill-labs/windmill/issues/1038)) ([cebbc5f](https://github.com/windmill-labs/windmill/commit/cebbc5fbd1b8b855c9b1bcab535cff5b9de8d778))
+* **frontend:** Fix inline script status ([#1034](https://github.com/windmill-labs/windmill/issues/1034)) ([be74311](https://github.com/windmill-labs/windmill/commit/be743117d155afb2a2f0fe33ff610e0f621409f7))
+* **frontend:** Fix UI ([#1009](https://github.com/windmill-labs/windmill/issues/1009)) ([0ceb4ab](https://github.com/windmill-labs/windmill/commit/0ceb4ab1a893fecf9e64497612e6040d0e7bc8cd))
+* **frontend:** Fork + Fix table ([#1037](https://github.com/windmill-labs/windmill/issues/1037)) ([ab13e8c](https://github.com/windmill-labs/windmill/commit/ab13e8cce44ded7e05a8dda3d4d4d1ac696bf739))
+* **frontend:** Small UI fixes ([#1026](https://github.com/windmill-labs/windmill/issues/1026)) ([ebca9f3](https://github.com/windmill-labs/windmill/commit/ebca9f39eab27dda65d0ee5de175a90363bfebae))
+* **frontend:** templatable editor with autocompletion ([e228c64](https://github.com/windmill-labs/windmill/commit/e228c6448ead4a7aef433f4abdfe3c466a0f50f4))
+* implement usage tracker + quotas ([fd87109](https://github.com/windmill-labs/windmill/commit/fd871093f0ea4b2def351857d7d8d7e4e79f9539))
+* introduce folders, deprecate items owned by groups ([4329d25](https://github.com/windmill-labs/windmill/commit/4329d259887da71eb2b2a67f73947b0fbe9f3941))
+* introduce folders, deprecate items owned by groups ([c1b0b64](https://github.com/windmill-labs/windmill/commit/c1b0b64e1728007b364d2a0acc58fc459e49e461))
+* Superadmins workspace ([#1003](https://github.com/windmill-labs/windmill/issues/1003)) ([4004de0](https://github.com/windmill-labs/windmill/commit/4004de06180868af4570668a2040bd711a461e0d))
+
+
+### Bug Fixes
+
+* **frontend:** copy-to-clipnoard url with protocol ([#1027](https://github.com/windmill-labs/windmill/issues/1027)) ([f77fe7b](https://github.com/windmill-labs/windmill/commit/f77fe7b6b321c3d00a51a42a4118fd37f7c9d782))
+* **frontend:** Fix AppTable frontend search ([#1013](https://github.com/windmill-labs/windmill/issues/1013)) ([f7627b5](https://github.com/windmill-labs/windmill/commit/f7627b5f17a9f5a4528715eebb4d207f33609da2))
+
+## [1.55.0](https://github.com/windmill-labs/windmill/compare/v1.54.0...v1.55.0) (2022-12-09)
+
+
+### Features
+
+* **frontend:** Add text input to app builder ([#1008](https://github.com/windmill-labs/windmill/issues/1008)) ([6198383](https://github.com/windmill-labs/windmill/commit/6198383138929237c1eb898954a1fd91bdded08a))
+
+## [1.54.0](https://github.com/windmill-labs/windmill/compare/v1.53.0...v1.54.0) (2022-12-08)
+
+
+### Features
+
+* add lockable version to scripts inside flows ([#972](https://github.com/windmill-labs/windmill/issues/972)) ([799fa92](https://github.com/windmill-labs/windmill/commit/799fa925b39316f6f8232d01959c35c4d6fa9533))
+* **frontend:** Add support for object editor + fix wording ([#1004](https://github.com/windmill-labs/windmill/issues/1004)) ([a562dee](https://github.com/windmill-labs/windmill/commit/a562dee3cebfc07f72f0e952cb102c4c86022937))
+* implement flow as a flow step ([8c1c508](https://github.com/windmill-labs/windmill/commit/8c1c5083585f4882aac3f05f71ad1a6414772082))
+
+## [1.53.0](https://github.com/windmill-labs/windmill/compare/v1.52.0...v1.53.0) (2022-12-05)
+
+
+### Features
+
+* add include_header to pass request headers to script ([31c317b](https://github.com/windmill-labs/windmill/commit/31c317b3581e24aa24fa41a708f080c1d1de7e0c))
+* **cli:** hub sync ([#975](https://github.com/windmill-labs/windmill/issues/975)) ([2265372](https://github.com/windmill-labs/windmill/commit/22653727a4106fa604796b3958efab94762041c2))
+* **frontend:** Add app preview ([#993](https://github.com/windmill-labs/windmill/issues/993)) ([c9ad638](https://github.com/windmill-labs/windmill/commit/c9ad63895891ab3bbaeab43a008573f5bd3681b5))
+* **frontend:** clarified UX for connect step ([e4839e2](https://github.com/windmill-labs/windmill/commit/e4839e21ff5d60bec4499245742f2400168c70ad))
+* **frontend:** introduce mysql as a script language ([#982](https://github.com/windmill-labs/windmill/issues/982)) ([e089109](https://github.com/windmill-labs/windmill/commit/e089109b50bd014c7a4f0fd7f60c53e8be63fb95))
+* refactor favorite menu ([c55fae5](https://github.com/windmill-labs/windmill/commit/c55fae54dd043eb1c01a15c8005e29166a4e992b))
+
+
+### Bug Fixes
+
+* **cli:** Fix cli pull push ([#985](https://github.com/windmill-labs/windmill/issues/985)) ([1bac237](https://github.com/windmill-labs/windmill/commit/1bac23785cb6af255732b1a2551bf9ffa00e24e7))
+* **frontend:** Align hub flow list + fix drawer content everywhere ([#991](https://github.com/windmill-labs/windmill/issues/991)) ([9f59a16](https://github.com/windmill-labs/windmill/commit/9f59a160c39048447ffeefc5070c52e8692c8316))
+* **frontend:** Fix app InputValue sync ([#994](https://github.com/windmill-labs/windmill/issues/994)) ([e217fbf](https://github.com/windmill-labs/windmill/commit/e217fbf071fa834c4b4288f602125164bf1d93bf))
+* **frontend:** fix app preview ([#979](https://github.com/windmill-labs/windmill/issues/979)) ([129a0ad](https://github.com/windmill-labs/windmill/commit/129a0ad56b58840620fdc77e619928e04c67cd1f))
+* **frontend:** fix home ([#981](https://github.com/windmill-labs/windmill/issues/981)) ([fa64e83](https://github.com/windmill-labs/windmill/commit/fa64e83f7ea6bc7786a15db647319d2f2a322b5b))
+* **frontend:** fix home header ([#977](https://github.com/windmill-labs/windmill/issues/977)) ([e9fa0ad](https://github.com/windmill-labs/windmill/commit/e9fa0ad0b75d0678167e7a48f8406639e85986a9))
+* **frontend:** Fix home margins ([#992](https://github.com/windmill-labs/windmill/issues/992)) ([62d2a33](https://github.com/windmill-labs/windmill/commit/62d2a3343dc27317f33446918404373b7d8285f5))
+* **frontend:** Make context clickable ([#984](https://github.com/windmill-labs/windmill/issues/984)) ([9264f4b](https://github.com/windmill-labs/windmill/commit/9264f4b233858537bb344355c5be43be3ec9d8d9))
+* **frontend:** variables and resources uses tab navigation ([90ce431](https://github.com/windmill-labs/windmill/commit/90ce4314181d8e5031c08d5fbb75b920c33b7f75))
+
+## [1.52.0](https://github.com/windmill-labs/windmill/compare/v1.51.0...v1.52.0) (2022-12-02)
+
+
+### Features
+
+* add favorite/star + remove flows/scripts page in favor of unified home page ([#968](https://github.com/windmill-labs/windmill/issues/968)) ([f3f694e](https://github.com/windmill-labs/windmill/commit/f3f694e9251fc62d8e3e10497e8936c588b456ba))
+* **cli:** improved setup & allow workspace in base url & refactor workspaces/remotes to unify ([#966](https://github.com/windmill-labs/windmill/issues/966)) ([d3a171c](https://github.com/windmill-labs/windmill/commit/d3a171c28355c5d452e6e9caa0aa741c1ff23875))
+* **cli:** Login via Frontend ([#956](https://github.com/windmill-labs/windmill/issues/956)) ([2c31a9c](https://github.com/windmill-labs/windmill/commit/2c31a9cbdf84ff2659313df799cbd79f9c167325))
+* **deno-client:** support mysql ([#971](https://github.com/windmill-labs/windmill/issues/971)) ([0e402f6](https://github.com/windmill-labs/windmill/commit/0e402f6a9dfd1b6d00f6d2a951740d7aea0a8b70))
+* **frontend:** Add actions to tables ([#951](https://github.com/windmill-labs/windmill/issues/951)) ([1069105](https://github.com/windmill-labs/windmill/commit/10691054510dd955a6f0d36c0186fdab9ce0facc))
+* **frontend:** Add Mailchimp resource instructions ([#967](https://github.com/windmill-labs/windmill/issues/967)) ([ba90e8c](https://github.com/windmill-labs/windmill/commit/ba90e8c1b8131e1b1e38322d165c04a53a8622b2))
+* **frontend:** flow status viewer include a graph ([02a9c5c](https://github.com/windmill-labs/windmill/commit/02a9c5c4eac557486df6908536a8467d68b92eca))
+* **frontend:** rework script detail ([#952](https://github.com/windmill-labs/windmill/issues/952)) ([6c45fe7](https://github.com/windmill-labs/windmill/commit/6c45fe7344858761422916cc497018b35753e0ce))
+* **frontend:** Update app component list ([#947](https://github.com/windmill-labs/windmill/issues/947)) ([ec1cebc](https://github.com/windmill-labs/windmill/commit/ec1cebc7920350939e365322f77898b31cafd795))
+* overhaul scripts and flows page ([4946093](https://github.com/windmill-labs/windmill/commit/494609364c9d6109c08c7531cf02223793325f88))
+* overhaul scripts and flows page ([c26be86](https://github.com/windmill-labs/windmill/commit/c26be86cef9d6cad44ae7cbbb5e0fd5d147c5c52))
+* **python:** add support for parsing resource type in python ([63d95cf](https://github.com/windmill-labs/windmill/commit/63d95cfbb31a2b599fa9deaee203e1c4c2f0715e))
+* refactor variable + resource linkage + OAuth visibility ([37967a7](https://github.com/windmill-labs/windmill/commit/37967a795006c2eb4e8b218abb3d1b0525c17d5e))
+* unify resources under a single connect API ([539d6be](https://github.com/windmill-labs/windmill/commit/539d6be9088ccb2d18b0d16ca020b23bffaa79b9))
+
+
+### Bug Fixes
+
+* **backend:** support PIP_INDEX_URL ([12f9677](https://github.com/windmill-labs/windmill/commit/12f967726b96cc04e5024134216727ddfcd5fe82))
+* **backend:** support PIP_INDEX_URL ([afcb44a](https://github.com/windmill-labs/windmill/commit/afcb44a12707dc3b0839182479438d2b010362ca))
+* **frontend:** Fix pie animation + actions wrap ([#953](https://github.com/windmill-labs/windmill/issues/953)) ([ed7838d](https://github.com/windmill-labs/windmill/commit/ed7838d6bcf538525f6b3e4257bffe6d51318c8a))
+* **frontend:** psql demo expects integers as a key ([#958](https://github.com/windmill-labs/windmill/issues/958)) ([4d8a5c4](https://github.com/windmill-labs/windmill/commit/4d8a5c4fd927e421825a9d9d2dc5dcfaf8b3949a))
+* **frontend:** Refactor apps to support multiple breakpoints ([#957](https://github.com/windmill-labs/windmill/issues/957)) ([96666af](https://github.com/windmill-labs/windmill/commit/96666af3d9d6f68e4e5bb0f7a748614c9916f394))
+
+## [1.51.0](https://github.com/windmill-labs/windmill/compare/v1.50.0...v1.51.0) (2022-11-26)
+
+
+### Features
+
+* Add notification on app save ([#943](https://github.com/windmill-labs/windmill/issues/943)) ([79cec36](https://github.com/windmill-labs/windmill/commit/79cec368ba643a88a554a88e4bc0500701e2fcc8))
+* **backend:** add configurable custom client ([975a1db](https://github.com/windmill-labs/windmill/commit/975a1db10ea592038cef0c2677e66a8b6d6b8ee5))
+* **cli:** Run flows & scripts ([#940](https://github.com/windmill-labs/windmill/issues/940)) ([cdd3e2c](https://github.com/windmill-labs/windmill/commit/cdd3e2cfc11cd003246643528b950cd0aafe1140))
+* **frontend:** Add guard against script overwrite ([#944](https://github.com/windmill-labs/windmill/issues/944)) ([dd75b37](https://github.com/windmill-labs/windmill/commit/dd75b370afd3d7e6a112e0ec9a6444a82b5620e3))
+* **frontend:** Add inline script picker to apps ([#945](https://github.com/windmill-labs/windmill/issues/945)) ([ddab2df](https://github.com/windmill-labs/windmill/commit/ddab2dffd5459a3e35a368e09a64ebcbceefc87a))
+* **frontend:** flow UX overhaul II + go + python support for trigger scripts ([#928](https://github.com/windmill-labs/windmill/issues/928)) ([802abe7](https://github.com/windmill-labs/windmill/commit/802abe7f901fc93bee1be401a3166fa22b63d00c))
+* **frontend:** login page makeup ([5028d86](https://github.com/windmill-labs/windmill/commit/5028d8603d08f13f4c9ae061b5aa9c6b4b5ea4f4))
+* **frontend:** login page makeup ([ced2678](https://github.com/windmill-labs/windmill/commit/ced2678a21e2078973cfbe506586061f806c2dfe))
+* Update apps button component with colors ([#936](https://github.com/windmill-labs/windmill/issues/936)) ([4b2b346](https://github.com/windmill-labs/windmill/commit/4b2b3467d2bbb204acd5330c4c100d63acb4e40a))
+
+
+### Bug Fixes
+
+* **backend:** bash flow lock & add flow lock tests ([#933](https://github.com/windmill-labs/windmill/issues/933)) ([4ddb3ec](https://github.com/windmill-labs/windmill/commit/4ddb3ec276ef9140e15a8604d796c3a2e6210311))
+* **deno-client:** pg 0.16.1 -> 0.17.0 ([ac6454b](https://github.com/windmill-labs/windmill/commit/ac6454b3835562f70694ce2b935e4b229f9118c6))
+* **frontend:** add checkbox component + fix alignment ([#941](https://github.com/windmill-labs/windmill/issues/941)) ([43a1d7e](https://github.com/windmill-labs/windmill/commit/43a1d7ef2a1c9167262ea7d19cc0fb10d0493eed))
+* **frontend:** Cleanup dead code ([#935](https://github.com/windmill-labs/windmill/issues/935)) ([fa4840a](https://github.com/windmill-labs/windmill/commit/fa4840ad656b2cb592c644193f617b49e53211aa))
+* **frontend:** Fix context panel + delete component ([#937](https://github.com/windmill-labs/windmill/issues/937)) ([ab481b3](https://github.com/windmill-labs/windmill/commit/ab481b3096ae6390e0d08b23a6b18f0f988cf1bd))
+* **frontend:** prevent runnable to run if the script is not defined ([#938](https://github.com/windmill-labs/windmill/issues/938)) ([e64195e](https://github.com/windmill-labs/windmill/commit/e64195e42b940e552d9b89b040dff4a4d0f8be37))
+* **frontend:** properly refresh context panel + Adjust style in the flow editor ([#934](https://github.com/windmill-labs/windmill/issues/934)) ([b59a1de](https://github.com/windmill-labs/windmill/commit/b59a1de93baade3ad576300c07143fbd3f074054))
+
+## [1.50.0](https://github.com/windmill-labs/windmill/compare/v1.49.1...v1.50.0) (2022-11-21)
+
+
+### Features
+
+* **deno,python:** get/set_shared_state ([c8266fb](https://github.com/windmill-labs/windmill/commit/c8266fb8b3262d9e9ec5698f824b2e9df716a228))
+* **frontend:** overhaul the whole flow UX ([d23e218](https://github.com/windmill-labs/windmill/commit/d23e218e1fd9b200aaa3fff12182f18e251da796))
+
+
+### Bug Fixes
+
+* **caching:** preserve permissions ([a352975](https://github.com/windmill-labs/windmill/commit/a3529759ad34db5c8234a7886aba1c3d07a644cf))
+
+## [1.49.1](https://github.com/windmill-labs/windmill/compare/v1.49.0...v1.49.1) (2022-11-20)
+
+
+### Bug Fixes
+
+* **caching:** add a second caching mechanism by tarring the entire cache for fast startup ([7af345e](https://github.com/windmill-labs/windmill/commit/7af345e5e57c6fbc35db9069782432664232851a))
+
+## [1.49.0](https://github.com/windmill-labs/windmill/compare/v1.48.2...v1.49.0) (2022-11-20)
+
+
+### Features
+
+* **go:** improve cold start of 200ms by building outside of nsjail ([838a92a](https://github.com/windmill-labs/windmill/commit/838a92a0dbb75f4e7e32a7541800cbda4808cea7))
+* **python-client:** remove unecessary imports in wmill to speed-up imports ([46fe9ad](https://github.com/windmill-labs/windmill/commit/46fe9ad52594d3a45b7917b91b37a83bc779bb1b))
+
+## [1.48.2](https://github.com/windmill-labs/windmill/compare/v1.48.1...v1.48.2) (2022-11-19)
+
+
+### Bug Fixes
+
+* **go-client:** support setVariable, setResource, setState, getState ([e33bd1e](https://github.com/windmill-labs/windmill/commit/e33bd1e6b25bb9e3a3fe6f2c93d8c686c200b253))
+
+## [1.48.1](https://github.com/windmill-labs/windmill/compare/v1.48.0...v1.48.1) (2022-11-19)
+
+
+### Bug Fixes
+
+* **python-client:** get_state on empty state return None ([968675d](https://github.com/windmill-labs/windmill/commit/968675d8d068b19413a8bca7d4cb80179646c114))
+
+## [1.48.0](https://github.com/windmill-labs/windmill/compare/v1.47.3...v1.48.0) (2022-11-18)
+
+
+### Features
+
+* add slack_bot token on connecting workspace to slack ([b3178d1](https://github.com/windmill-labs/windmill/commit/b3178d1b8aacfa90b8a68554a186f3b26f3190ba))
+* **backend:** sync cache features on all workers [enterprise] ([#907](https://github.com/windmill-labs/windmill/issues/907)) ([bd09884](https://github.com/windmill-labs/windmill/commit/bd09884955bbe04f41fbcce9b978a070145f23a3))
+* **python:** add Resource[resource_type] as a parsed parameter ([9d17abb](https://github.com/windmill-labs/windmill/commit/9d17abbb12463c81de325eef875161cf86449b25))
+* supercache extended to all version ([8846ca5](https://github.com/windmill-labs/windmill/commit/8846ca585699c2ec7b18b4479e895b296774ee95))
+
+
+### Bug Fixes
+
+* **backend:** saving bash script does not require dep job ([381b036](https://github.com/windmill-labs/windmill/commit/381b0368d72ad42501082c91a7c62964593ba3ad))
+* **frontend:** app editor v1 ([#908](https://github.com/windmill-labs/windmill/issues/908)) ([53a8c5e](https://github.com/windmill-labs/windmill/commit/53a8c5e04cc4f407c137b0d621003dbab1bfdc67))
+* **frontend:** Reduce the size of the separator + fix Auto scroll ([#895](https://github.com/windmill-labs/windmill/issues/895)) ([3f8295b](https://github.com/windmill-labs/windmill/commit/3f8295bb0c7d9e9c831e8dbcb7f1e8b944e45c66))
+* support flows to be triggered by slack commands ([199a11a](https://github.com/windmill-labs/windmill/commit/199a11a8cf92691a3ac5aa7ebdc3157d10677139))
+
 ## [1.47.3](https://github.com/windmill-labs/windmill/compare/v1.47.2...v1.47.3) (2022-11-15)
 
 

Dockerfile

@@ -2,7 +2,9 @@ FROM debian:buster-slim as nsjail
 
 WORKDIR /nsjail
 
-RUN apt-get -y update \
+ARG nsjail=""
+
+RUN  if [ "$nsjail" = "true" ]; then apt-get -y update \
     && apt-get install -y \
     bison=2:3.3.* \
     flex=2.6.* \
@@ -13,11 +15,12 @@ RUN apt-get -y update \
     libnl-route-3-dev=3.4.* \
     make=4.2.* \
     pkg-config=0.29-6 \
-    protobuf-compiler=3.6.*
+    protobuf-compiler=3.6.*; fi
 
-RUN git clone -b master --single-branch https://github.com/google/nsjail.git . \
-    && git checkout dccf911fd2659e7b08ce9507c25b2b38ec2c5800
-RUN make
+
+RUN if [ "$nsjail" = "true" ]; then git clone -b master --single-branch https://github.com/google/nsjail.git . \
+    && git checkout dccf911fd2659e7b08ce9507c25b2b38ec2c5800; fi
+RUN if [ "$nsjail" = "true" ]; then make; else touch nsjail; fi
 
 FROM rust:slim-buster AS rust_base
 
@@ -59,13 +62,14 @@ FROM rust_base AS planner
 COPY ./openflow.openapi.yaml /openflow.openapi.yaml
 COPY ./backend ./
 
-RUN CARGO_NET_GIT_FETCH_WITH_CLI=true cargo chef prepare  --recipe-path recipe.json
+RUN CARGO_NET_GIT_FETCH_WITH_CLI=true cargo chef prepare --recipe-path recipe.json
 
 FROM rust_base AS builder
+ARG features=""
 
 COPY --from=planner /windmill/recipe.json recipe.json
 
-RUN CARGO_NET_GIT_FETCH_WITH_CLI=true cargo chef cook --release --recipe-path recipe.json
+RUN CARGO_NET_GIT_FETCH_WITH_CLI=true cargo chef cook --release --features "$features" --recipe-path recipe.json
 
 COPY ./openflow.openapi.yaml /openflow.openapi.yaml
 COPY ./backend ./
@@ -73,32 +77,39 @@ COPY ./backend ./
 COPY --from=frontend /frontend /frontend
 COPY .git/ .git/
 
-RUN CARGO_NET_GIT_FETCH_WITH_CLI=true cargo build --release
+RUN CARGO_NET_GIT_FETCH_WITH_CLI=true cargo build --release --features "$features"
 
 
-FROM python:3.11.0-slim-buster
+FROM python:3.11.1-slim-buster
 
 ARG APP=/usr/src/app
 
 RUN apt-get update \
-    && apt-get install -y ca-certificates wget curl git jq libprotobuf-dev libnl-route-3-dev \
+    && apt-get install -y ca-certificates wget curl git jq libprotobuf-dev libnl-route-3-dev unzip \
+    && apt-get install -y ca-certificates wget curl git jq libprotobuf-dev libnl-route-3-dev unzip build-essential \
     && rm -rf /var/lib/apt/lists/*
 
+RUN arch="$(dpkg --print-architecture)"; arch="${arch##*-}"; \
+    curl -o rclone.zip "https://downloads.rclone.org/v1.60.1/rclone-v1.60.1-linux-$arch.zip"; \
+    unzip -p rclone.zip rclone-v1.60.1-linux-$arch/rclone > /usr/bin/rclone; rm rclone.zip; \
+    chown root:root /usr/bin/rclone; chmod 755 /usr/bin/rclone
+
 RUN set -eux; \
     arch="$(dpkg --print-architecture)"; arch="${arch##*-}"; \
     url=; \
     case "$arch" in \
     'amd64') \
     targz='go1.19.3.linux-amd64.tar.gz'; \
-    sha256='74b9640724fd4e6bb0ed2a1bc44ae813a03f1e72a4c76253e2d5c015494430ba'; \
     ;; \
     'arm64') \
     targz='go1.19.3.linux-arm64.tar.gz'; \
-    sha256='99de2fe112a52ab748fb175edea64b313a0c8d51d6157dba683a6be163fd5eab'; \
+    ;; \
+    'armhf') \
+    targz='go1.19.3.linux-armv6l.tar.gz'; \
     ;; \
     *) echo >&2 "error: unsupported architecture '$arch' (likely packaging update needed)"; exit 1 ;; \
     esac; \
-    wget "https://golang.org/dl/$targz" && tar -C /usr/local -xzf "$targz" && rm "$targz";
+    wget "https://golang.org/dl/$targz" -nv && tar -C /usr/local -xzf "$targz" && rm "$targz";
 
 ENV PATH="${PATH}:/usr/local/go/bin"
 ENV GO_PATH=/usr/local/go/bin/go

README.md

@@ -22,8 +22,7 @@ Open-source developer infrastructure for internal tools. Self-hostable alternati
 
 ---
 
-**Join the beta (personal workspaces are free forever)**:
-<https://app.windmill.dev>
+**Try it (personal workspaces are free forever)**: <https://app.windmill.dev>
 
 **Documentation**: <https://docs.windmill.dev>
 
@@ -35,15 +34,10 @@ Open-source developer infrastructure for internal tools. Self-hostable alternati
 
 **Roadmap**: <https://github.com/orgs/windmill-labs/projects/2>
 
-**[Self-host instruction](#how-to-self-host)**
-
 You can show your support for the project by starring this repo.
 
----
-
-Windmill Labs offers commercial licenses and support to convert your existing
-automation and help you scale it in production. If interested, contact
-ruben@windmill.dev (founder of Windmill).
+Windmill Labs offers commercial licenses, an enterprise edition, local hub
+mirrors, and support: contact ruben@windmill.dev.
 
 ---
 
@@ -53,40 +47,57 @@ ruben@windmill.dev (founder of Windmill).
 <b>Disclaimer: </b>Windmill is in <b>BETA</b>. It is secure to run in production but we are still <a href="https://github.com/orgs/windmill-labs/projects/2">improving the product fast<a/>.
 </p>
 
+![Windmill Screenshot](./imgs/windmill-flow.png)
 ![Windmill Screenshot](./imgs/windmill.png)
 
 Windmill is <b>fully open-sourced (AGPLv3)</b>:
 
-## What is the general idea behind Windmill
+- [Windmill](#windmill)
+  - [Main Concepts](#main-concepts)
+  - [CLI](#cli)
+  - [Layout](#layout)
+  - [Stack](#stack)
+  - [Security](#security)
+    - [Sandboxing and workload isolation](#sandboxing-and-workload-isolation)
+    - [Secrets, credentials and sensitive values](#secrets-credentials-and-sensitive-values)
+  - [Performance](#performance)
+  - [Architecture](#architecture)
+    - [Big-picture Architecture](#big-picture-architecture)
+    - [Technical Architecture](#technical-architecture)
+  - [How to self-host](#how-to-self-host)
+    - [Docker compose](#docker-compose)
+    - [Kubernetes (k8s) and Helm charts](#kubernetes-k8s-and-helm-charts)
+    - [Commercial license](#commercial-license)
+    - [OAuth for self-hosting (very optional)](#oauth-for-self-hosting-very-optional)
+    - [Resource types](#resource-types)
+  - [Environment Variables](#environment-variables)
+  - [Run a local dev setup](#run-a-local-dev-setup)
+    - [only Frontend](#only-frontend)
+    - [Backend + Frontend](#backend--frontend)
+  - [Contributors](#contributors)
+  - [Copyright](#copyright)
 
-1. Define a minimal and generic script in Python, Typescript, Go or Bash that solves a
-   specific task. Here sending an email with SMTP. The code can be defined in
-   the provided Web IDE or synchronized with your own github repo:
-   ![Step 1](./imgs/python-script.png)
+## Main Concepts
 
-2. Your scripts parameters are automatically parsed and generate a frontend. You
-   can narrow down the types during task definition to specify regex for string,
-   an enum or a specific format for objects. Each script correspond to an app by
-   itself: ![Step 2](./imgs/arguments.png)
+1. Define a minimal and generic script in Python, Typescript, Go or Bash that
+   solves a specific task. Here sending an email with SMTP. The code can be
+   defined in the provided Web IDE or synchronized with your own github repo:
+   ![Step 1](./imgs/windmill-editor.png)
+
+2. Your scripts parameters are automatically parsed and generate a frontend.
+   ![Step 2](./imgs/windmill-run.png) ![Step 3](./imgs/windmill-result.png)
 
 3. Make it flow! You can chain your scripts or scripts made by the community
-   shared on [WindmillHub](https://hub.windmill.dev). There is tight integration
-   between Windmill and the hub to make it easy to build flows from a soon-to-be
-   exhaustive library of generic modules. In flows, one can pipe output to input
-   using rich expressions that are just plain Javascript underneath. Flows
-   support for-loops, branching, approval steps. As such and coupled with
-   inputs being able to refer to any step's output, they are actual DAG rather
-   than just linear sequences. They are backed by an open JSON spec we call
-   [OpenFlow](https://docs.windmill.dev/docs/openflow)
+   shared on [WindmillHub](https://hub.windmill.dev).
+   ![Step 4](./imgs/windmill-flow.png)
 
-Both scripts and flows are not restricted to be triggered by the UI. They can be
-triggered by a schedule, watch for changes (using
-[internal states](https://docs.windmill.dev/docs/reference#internal-state)) or
-triggered through API with either an async or sync webhook. The latter kind of
-endpoints make Windmill akin to a self-hostable AWS Lambda. Windmill can be the
-central place to host, build and run all of your integrations, automation and
-internal apps. We include credentials management and OAuth integration, groups
-and much more!
+4. (Coming soon) Build complex UI on top of your scripts and flows.
+   ![Step 5](./imgs/windmill-builder.png)
+
+Scripts and flows can also be triggered by a cron schedule '*/5 * * * *' or
+through webhooks.
+
+You can build your entire infra on top of Windmill!
 
 ## CLI
 
@@ -94,6 +105,8 @@ We have a powerful CLI to interact with the windmill platform and sync your
 scripts from your own github repo. See
 [more details](https://github.com/windmill-labs/windmill/tree/main/cli)
 
+![CLI Screencast](./cli/vhs/output/setup.gif)
+
 ## Layout
 
 - `backend/`: Rust backend
@@ -138,9 +151,11 @@ That is what we do at <https://app.windmill.dev>.
 
 ## Performance
 
-The performances are great, as long as you do not exceed the parallelism of the
-workers, we are
-[worse than AWS Lambda for small workloads but not by that much](https://docs.windmill.dev/docs/benchmark)
+Once a job started, there is no overhead compared to running the same script on
+the node with its corresponding runner (Deno/Go/Python/Bash). The added latency
+from a job being pulled from the queue, started, and then having its result sent
+back to the database is ~50ms. A typical lightweight deno job will take around
+100ms total.
 
 ## Architecture
 
@@ -178,6 +193,10 @@ The default super-admin user is: admin@windmill.dev / changeme
 
 From there, you can create other users (do not forget to change the password!)
 
+### Kubernetes (k8s) and Helm charts
+
+We publish helm charts at: <https://github.com/windmill-labs/windmill-helm-charts>
+
 ### Commercial license
 
 To self-host Windmill, you must respect the terms of the AGPLv3 license which
@@ -192,8 +211,8 @@ be AGPLv3 or you must get a commercial license. Contact us at
 
 In addition, a commercial license grants you a dedicated engineer to transition
 your current infrastructure to Windmill, support with tight SLA, audit logs
-export features, SSO, unlimited users creation, advanced permission managing features
-such as groups and the ability to create more than one workspace.
+export features, SSO, unlimited users creation, advanced permission managing
+features such as groups and the ability to create more than one workspace.
 
 ### OAuth for self-hosting (very optional)
 
@@ -212,12 +231,40 @@ the following format:
 
 and mount it at `/usr/src/app/oauth.json`.
 
+The redirect url for the oauth clients is: `<instance_url>/user/login_callback/<client>`
+
 [The list of all possible "connect an app" oauth clients](https://github.com/windmill-labs/windmill/blob/main/backend/oauth_connect.json)
 
 To add more "connect an app" OAuth clients to the Windmill project, read the
 [Contributor's guide](https://docs.windmill.dev/docs/contributors_guide). We
 welcome contributions!
 
+You may also add your own custom OAuth2 IdP and OAuth2 Resource provider:
+
+```json
+{
+  "<client>": {
+    "id": "<CLIENT_ID>",
+    "secret": "<CLIENT_SECRET>",
+    // To add a new OAuth2 IdP
+    "login_config": {
+      "auth_url": "<auth_endpoint>",
+      "token_url": "<token_endpoint>",
+      "userinfo_url": "<userinfo endpoint>",
+      "scopes": ["scope1", "scope2"],
+      "extra_params": "<if_needed>"
+    },
+    // To add a new OAuth2 Resource
+    "connect_config": {
+      "auth_url": "<auth_endpoint>",
+      "token_url": "<token_endpoint>",
+      "scopes": ["scope1", "scope2"],
+      "extra_params": "<if_needed>"
+    }
+  }
+}
+```
+
 ### Resource types
 
 You will also want to import all the approved resource types from
@@ -225,6 +272,39 @@ You will also want to import all the approved resource types from
 automatically currently, but it will be possible using a command with the
 upcoming CLI tool.
 
+## Environment Variables
+
+| Environment Variable name | Default                | Description                                                                                                                                                                                        | Api Server/Worker/All |
+| ------------------------- | ---------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------- |
+| DATABASE_URL              |                        | The Postgres database url.                                                                                                                                                                         | All                   |
+| DISABLE_NSJAIL            | true                   | Disable Nsjail Sandboxing                                                                                                                                                                          |                       |
+| NUM_WORKERS               | 3                      | The number of worker per Worker instance (set to 1 on Eks to have 1 pod = 1 worker)                                                                                                                | Worker                |
+| METRICS_ADDR              | None                   | The socket addr at which to expose Prometheus metrics at the /metrics path. Set to "true" to expose it on port 8001                                                                                | All                   |
+| JSON_FMT                  | false                  | Output the logs in json format instead of logfmt                                                                                                                                                   | All                   |
+| BASE_URL                  | http://localhost:8000  | The base url that is exposed publicly to access your instance                                                                                                                                      | Server                |
+| BASE_INTERNAL_URL         | http://localhost:8000  | The base url that is reachable by your workers to talk to the Servers. This help avoiding going through the external load balancer for VPC-internal requests.                                      | Worker                |
+| TIMEOUT                   | 300                    | The timeout in seconds for the execution of a script                                                                                                                                               | Worker                |
+| SLEEP_QUEUE               | 50                     | The number of ms to sleep in between the last check for new jobs in the DB. It is multiplied by NUM_WORKERS such that in average, for one worker instance, there is one pull every SLEEP_QUEUE ms. | Worker                |
+| DISABLE_NUSER             | false                  | If Nsjail is enabled, disable the nsjail's `clone_newuser` setting                                                                                                                                 | Worker                |
+| KEEP_JOB_DIR              | false                  | Keep the job directory after the job is done. Useful for debugging.                                                                                                                                | Worker                |
+| LICENSE_KEY (EE only)     | None                   | License key checked at startup for the Enterprise Edition of Windmill                                                                                                                              | Worker                |
+| S3_CACHE_BUCKET (EE only) | None                   | The S3 bucket to sync the cache of the workers to                                                                                                                                                  | Worker                |
+| TAR_CACHE_RATE (EE only)  | 100                    | The rate at which to tar the cache of the workers. 100 means every 100th job in average (uniformly randomly distributed).                                                                          | Worker                |
+| SLACK_SIGNING_SECRET      | None                   | The signing secret of your Slack app. See [Slack documentation](https://api.slack.com/authentication/verifying-requests-from-slack)                                                                | Server                |
+| COOKIE_DOMAIN             | None                   | The domain of the cookie. If not set, the cookie will be set by the browser based on the full origin                                                                                               | Server                |
+| SERVE_CSP                 | None                   | The CSP directives to use when serving the frontend static assets                                                                                                                                  | Server                |
+| DENO_PATH                 | /usr/bin/deno          | The path to the deno binary.                                                                                                                                                                       | Worker                |
+| PYTHON_PATH               | /usr/local/bin/python3 | The path to the python binary.                                                                                                                                                                     | Worker                |
+| GO_PATH                   | /usr/bin/go            | The path to the go binary.                                                                                                                                                                         | Worker                |
+| PIP_INDEX_URL             | None                   | The index url to pass for pip.                                                                                                                                                                     | Worker                |
+| PIP_EXTRA_INDEX_URL       | None                   | The extra index url to pass to pip.                                                                                                                                                                | Worker                |
+| PIP_TRUSTED_HOST          | None                   | The trusted host to pass to pip.                                                                                                                                                                   | Worker                |
+| PATH                      | None                   | The path environment variable, usually inherited                                                                                                                                                   | Worker                |
+| HOME                      | None                   | The home directory to use for Go and Bash , usually inherited                                                                                                                                      | Worker                |
+| DATABASE_CONNECTIONS      | 50 (Server)/3 (Worker) | The max number of connections in the database connection pool                                                                                                                                      | All                   |
+| SUPERADMIN_SECRET         | None                   | A token that would let the caller act as a virtual superadmin superadmin@windmill.dev                                                                                                              | Server                |
+
+
 ## Run a local dev setup
 
 ### only Frontend
@@ -244,9 +324,9 @@ See the [./frontend/README_DEV.md](./frontend/README_DEV.md) file for all
 running options.
 
 1. Create a Postgres Database for Windmill and create an admin role inside your
-   Postgres setup.
-   The easiest way to get a working postgres is running `cargo install sqlx-cli && sqlx migrate run`.
-   This will also avoid compile time issue with sqlx's `query!` macro
+   Postgres setup. The easiest way to get a working postgres is running
+   `cargo install sqlx-cli && sqlx migrate run`. This will also avoid compile
+   time issue with sqlx's `query!` macro
 2. Install [nsjail](https://github.com/google/nsjail) and have it accessible in
    your PATH
 3. Install deno and python3, have the bins at `/usr/bin/deno` and
@@ -255,7 +335,8 @@ running options.
 5. Install the [lld linker](https://lld.llvm.org/)
 6. Go to `frontend/`:
    1. `npm install`, `npm run generate-backend-client` then `npm run dev`
-   2. In another shell `npm run build` otherwise the backend will not find the `frontend/build` folder and will crash
+   2. In another shell `npm run build` otherwise the backend will not find the
+      `frontend/build` folder and will crash
    3. In another shell `sudo caddy run --config Caddyfile`
 7. Go to `backend/`:
    `DATABASE_URL=<DATABASE_URL_TO_YOUR_WINDMILL_DB> RUST_LOG=info cargo run`

backend/Cargo.lock

@@ -437,6 +437,27 @@ dependencies = [
  "unicode-width",
 ]
 
+[[package]]
+name = "const_format"
+version = "0.2.30"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7309d9b4d3d2c0641e018d449232f2e28f1b22933c137f157d3dbc14228b8c0e"
+dependencies = [
+ "const_format_proc_macros",
+ "konst",
+]
+
+[[package]]
+name = "const_format_proc_macros"
+version = "0.2.29"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d897f47bf7270cf70d370f8f98c1abb6d2d4cf60a6845d30e05bfb90c6568650"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-xid",
+]
+
 [[package]]
 name = "convert_case"
 version = "0.4.0"
@@ -781,6 +802,12 @@ version = "1.0.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4f94fa09c2aeea5b8839e414b7b841bf429fd25b9c522116ac97ee87856d88b2"
 
+[[package]]
+name = "dyn-iter"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "131726693bce13b09331bee70734fe266666332b6ddfef23e9dca5b8bf6dea66"
+
 [[package]]
 name = "either"
 version = "1.8.0"
@@ -1371,6 +1398,21 @@ dependencies = [
  "wasm-bindgen",
 ]
 
+[[package]]
+name = "konst"
+version = "0.2.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "330f0e13e6483b8c34885f7e6c9f19b1a7bd449c673fbb948a51c99d66ef74f4"
+dependencies = [
+ "konst_macro_rules",
+]
+
+[[package]]
+name = "konst_macro_rules"
+version = "0.2.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a4933f3f57a8e9d9da04db23fb153356ecaf00cbd14aee46279c33dc80925c37"
+
 [[package]]
 name = "lalrpop"
 version = "0.19.8"
@@ -3793,6 +3835,12 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "urlencoding"
+version = "2.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e8db7427f936968176eaa7cdf81b7f98b980b18495ec28f1b5791ac3bfe3eea9"
+
 [[package]]
 name = "uuid"
 version = "1.2.1"
@@ -4023,13 +4071,14 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
 
 [[package]]
 name = "windmill"
-version = "1.47.3"
+version = "1.56.0"
 dependencies = [
  "anyhow",
  "axum",
  "chrono",
  "dotenv",
  "futures",
+ "git-version",
  "rand",
  "reqwest",
  "serde_json",
@@ -4046,7 +4095,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api"
-version = "1.46.2"
+version = "1.55.0"
 dependencies = [
  "anyhow",
  "argon2",
@@ -4084,6 +4133,7 @@ dependencies = [
  "tower-http",
  "tracing",
  "tracing-subscriber",
+ "urlencoding",
  "windmill-audit",
  "windmill-common",
  "windmill-parser",
@@ -4096,7 +4146,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-client"
-version = "1.46.2"
+version = "1.55.0"
 dependencies = [
  "base64",
  "chrono",
@@ -4111,7 +4161,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-audit"
-version = "1.46.2"
+version = "1.55.0"
 dependencies = [
  "chrono",
  "serde",
@@ -4124,7 +4174,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-common"
-version = "1.46.2"
+version = "1.55.0"
 dependencies = [
  "anyhow",
  "axum",
@@ -4148,7 +4198,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser"
-version = "1.46.2"
+version = "1.55.0"
 dependencies = [
  "serde",
  "serde_json",
@@ -4156,7 +4206,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-bash"
-version = "1.46.2"
+version = "1.55.0"
 dependencies = [
  "anyhow",
  "itertools",
@@ -4170,7 +4220,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-go"
-version = "1.46.2"
+version = "1.55.0"
 dependencies = [
  "anyhow",
  "itertools",
@@ -4182,7 +4232,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-py"
-version = "1.46.2"
+version = "1.55.0"
 dependencies = [
  "anyhow",
  "itertools",
@@ -4197,7 +4247,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-ts"
-version = "1.46.2"
+version = "1.55.0"
 dependencies = [
  "anyhow",
  "deno_core",
@@ -4211,7 +4261,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-queue"
-version = "1.46.2"
+version = "1.55.0"
 dependencies = [
  "anyhow",
  "chrono",
@@ -4234,14 +4284,17 @@ dependencies = [
 
 [[package]]
 name = "windmill-worker"
-version = "1.46.2"
+version = "1.55.0"
 dependencies = [
  "anyhow",
  "async-recursion",
  "chrono",
+ "const_format",
  "deno_core",
  "dotenv",
+ "dyn-iter",
  "futures",
+ "git-version",
  "itertools",
  "lazy_static",
  "prometheus",

backend/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "windmill"
-version = "1.47.3"
+version = "1.56.0"
 authors.workspace = true
 edition.workspace = true
 
@@ -19,7 +19,7 @@ members = [
 ]
 
 [workspace.package]
-version = "1.47.3"
+version = "1.56.0"
 authors = ["Ruben Fiszel <ruben@windmill.dev>"]
 edition = "2021"
 
@@ -27,6 +27,9 @@ edition = "2021"
 name = "windmill"
 path = "./src/main.rs"
 
+[features]
+enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmill-api/enterprise"]
+
 [dependencies]
 anyhow.workspace = true
 tokio.workspace = true
@@ -41,6 +44,7 @@ sqlx.workspace = true
 tokio-metrics.workspace = true
 rand.workspace = true
 chrono.workspace = true
+git-version.workspace = true
 
 [dev-dependencies]
 serde_json.workspace = true
@@ -134,3 +138,5 @@ futures = "^0"
 tokio-metrics = "0.1.0"
 lazy_static = "1.4.0"
 serde_derive = "1.0.147"
+const_format = { version = "0.2", features = ["rust_1_64", "rust_1_51"] }
+dyn-iter = "0.2.0"

backend/build.rs

@@ -0,0 +1,5 @@
+// generated by `sqlx migrate build-script`
+fn main() {
+    // trigger recompilation when a new migration is added
+    println!("cargo:rerun-if-changed=migrations");
+}

backend/migrations/20221123151919_grant_sequence_privileges.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20221123151919_grant_sequence_privileges.up.sql

@@ -0,0 +1,8 @@
+-- Add up migration script here
+
+GRANT ALL
+ON ALL SEQUENCES IN SCHEMA public 
+TO windmill_user;
+GRANT ALL
+ON ALL SEQUENCES IN SCHEMA public 
+TO windmill_admin;

backend/migrations/20221126181131_pg_rt_name.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20221126181131_pg_rt_name.up.sql

@@ -0,0 +1,6 @@
+-- Add up migration script here
+UPDATE resource_type rt SET name = 'postgresql' WHERE name = 'postgres' AND  NOT EXISTS (
+   SELECT 1 FROM resource_type WHERE name = 'postgresql' AND rt.workspace_id = workspace_id
+);
+
+UPDATE resource SET resource_type = 'postgresql' WHERE resource_type = 'postgres';

backend/migrations/20221127093847_add_account_error.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20221127093847_add_account_error.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+ALTER TABLE account ADD COLUMN refresh_error TEXT;

backend/migrations/20221127143814_remove_oauth_from_res.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20221127143814_remove_oauth_from_res.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+ALTER TABLE resource DROP COLUMN is_oauth;

backend/migrations/20221128210151_favorites.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20221128210151_favorites.up.sql

@@ -0,0 +1,11 @@
+-- Add up migration script here
+
+CREATE TYPE FAVORITE_KIND AS ENUM ('app', 'script', 'flow');
+
+CREATE TABLE favorite (
+    usr VARCHAR(50) NOT NULL,
+    workspace_id VARCHAR(50) NOT NULL,
+    path VARCHAR(255) NOT NULL,
+    favorite_kind FAVORITE_KIND NOT NULL,
+    PRIMARY KEY (usr, workspace_id, favorite_kind, path)
+);

backend/migrations/20221203080006_apps_2.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20221203080006_apps_2.up.sql

@@ -0,0 +1,3 @@
+-- Add up migration script here
+ALTER TABLE app ADD CONSTRAINT unique_path_workspace_id UNIQUE (workspace_id, path);
+ALTER TABLE app ENABLE ROW LEVEL SECURITY;

backend/migrations/20221207103910_admins_workspace.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20221207103910_admins_workspace.up.sql

@@ -0,0 +1,9 @@
+INSERT INTO workspace(id, name, owner) VALUES
+	('admins', 'Admins', 'admin@windmill.dev');
+
+INSERT INTO workspace_settings (workspace_id) VALUES
+	('admins');
+
+INSERT INTO workspace_key
+	(workspace_id, kind, key)
+	VALUES ('admins', 'cloud', md5(random()::text) || md5(random()::text))

backend/migrations/20221207125219_admins_hub_sync.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20221207125219_admins_hub_sync.up.sql

@@ -0,0 +1,20 @@
+INSERT INTO script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'admins',
+'system',
+'import wmill from "https://deno.land/x/wmill@v1.55.0/main.ts";
+
+export async function main() {
+  await run(
+    "workspace", "add", "__automation", "starter", Deno.env.get("WM_BASE_URL") + "/", "--token", Deno.env.get("WM_TOKEN"));
+
+  await run("hub", "pull");
+}
+
+async function run(...cmd: string[]) {
+  console.log("Running \"" + cmd.join('' '') + "\"");
+  await wmill.parse(cmd);
+}',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
+'Syncronize Hub Resource types with starter workspace',
+'Basic administrative script to sync latest resource types from hub. Recommended to run at least once. On a schedule by default.',
+'u/admin/hub_sync', -28028598712388162, 'deno', '');

backend/migrations/20221208135921_job_pre_run_error.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20221208135921_job_pre_run_error.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+ALTER TABLE queue ADD COLUMN pre_run_error text;

backend/migrations/20221210125521_increase_variable_length.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20221210125521_increase_variable_length.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+ALTER TABLE variable ALTER COLUMN value TYPE VARCHAR(15000);

backend/migrations/20221210194626_usage_tracker.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20221210194626_usage_tracker.up.sql

@@ -0,0 +1,8 @@
+-- Add up migration script here
+CREATE TABLE usage (
+    id VARCHAR(50) NOT NULL,
+    is_workspace BOOLEAN NOT NULL,
+    month_ INTEGER NOT NULL,
+    usage INTEGER NOT NULL,
+    PRIMARY KEY (id, is_workspace, month_)
+);

backend/migrations/20221211055346_emails_everywhere.down.sql

@@ -0,0 +1,5 @@
+-- Add down migration script here
+ALTER TABLE queue DROP COLUMN email; 
+ALTER TABLE workspace_settings DROP COLUMN slack_email; 
+ALTER TABLE schedule DROP COLUMN email; 
+ALTER TABLE schedule DROP COLUMN error; 

backend/migrations/20221211055346_emails_everywhere.up.sql

@@ -0,0 +1,6 @@
+-- Add up migration script here
+ALTER TABLE queue ADD COLUMN email VARCHAR(50) NOT NULL DEFAULT 'missing@email.xyz'; 
+ALTER TABLE workspace_settings ADD COLUMN slack_email VARCHAR(50) NOT NULL DEFAULT 'missing@email.xyz';;
+ALTER TABLE schedule ADD COLUMN email VARCHAR(50) NOT NULL DEFAULT 'missing@email.xyz';  
+ALTER TABLE schedule ADD COLUMN error TEXT;  
+

backend/migrations/20221211192539_grant_usage.down.sql

@@ -0,0 +1,2 @@
+-- Add down migration script here
+ALTER TABLE workspace_settings DROP COLUMN auto_invite_domain;

backend/migrations/20221211192539_grant_usage.up.sql

@@ -0,0 +1,4 @@
+-- Add up migration script here
+
+GRANT ALL PRIVILEGES ON TABLE usage TO windmill_admin;
+GRANT ALL PRIVILEGES ON TABLE usage TO windmill_user;

backend/migrations/20221211200923_remove_admin_from_starters.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20221211200923_remove_admin_from_starters.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+DELETE FROM usr WHERE workspace_id = 'starter' AND email = 'admin@windmill.dev';

backend/migrations/20221212114409_add_all_group_to_admins.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20221212114409_add_all_group_to_admins.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+INSERT INTO group_ SELECT id, 'all', 'The group that always contains all users of this workspace' FROM workspace ON CONFLICT DO NOTHING;

backend/migrations/20221212130731_add_autoinvite.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20221212130731_add_autoinvite.up.sql

@@ -0,0 +1,3 @@
+-- Add up migration script here
+ALTER TABLE workspace_settings ADD COLUMN auto_invite_domain VARCHAR(50);
+ALTER TABLE workspace DROP COLUMN domain;

backend/migrations/20221213112420_add_operator.down.sql

@@ -0,0 +1,3 @@
+-- Add down migration script here
+ALTER TABLE workspace_invite DROP COLUMN operator;
+ALTER TABLE workspace_settings DROP COLUMN auto_invite_operator;

backend/migrations/20221213112420_add_operator.up.sql

@@ -0,0 +1,4 @@
+-- Add up migration script here
+ALTER TABLE workspace_invite ADD COLUMN operator BOOLEAN NOT NULL DEFAULT false;
+ALTER TABLE workspace_settings ADD COLUMN auto_invite_operator BOOLEAN DEFAULT false;
+ALTER TABLE completed_job ADD COLUMN email VARCHAR(50) NOT NULL DEFAULT 'missing@email.xyz'; 

backend/migrations/20221215101605_share_job_to_owner.down.sql

@@ -0,0 +1,5 @@
+-- Add down migration script here
+ALTER TABLE queue DROP COLUMN visible_to_owner;
+ALTER TABLE completed_job DROP COLUMN visible_to_owner;
+
+DROP POLICY see_own_path ON queue;

backend/migrations/20221215101605_share_job_to_owner.up.sql

@@ -0,0 +1,15 @@
+-- Add up migration script here
+ALTER TABLE queue ADD COLUMN visible_to_owner BOOLEAN DEFAULT true;
+ALTER TABLE completed_job ADD COLUMN visible_to_owner BOOLEAN DEFAULT true;
+
+CREATE POLICY see_own_path ON queue FOR ALL
+USING (queue.visible_to_owner IS true AND SPLIT_PART(queue.script_path, '/', 1) = 'u' AND SPLIT_PART(queue.script_path, '/', 2) = current_setting('session.user'));
+
+CREATE POLICY see_member_path ON queue FOR ALL
+USING (queue.visible_to_owner IS true AND SPLIT_PART(queue.script_path, '/', 1) = 'g' AND SPLIT_PART(queue.script_path, '/', 2) = any(regexp_split_to_array(current_setting('session.groups'), ',')::text[]));
+
+CREATE POLICY see_own_path ON completed_job FOR ALL
+USING (completed_job.visible_to_owner IS true AND SPLIT_PART(completed_job.script_path, '/', 1) = 'u' AND SPLIT_PART(completed_job.script_path, '/', 2) = current_setting('session.user'));
+
+CREATE POLICY see_member_path ON completed_job FOR ALL
+USING (completed_job.visible_to_owner IS true AND SPLIT_PART(completed_job.script_path, '/', 1) = 'g' AND SPLIT_PART(completed_job.script_path, '/', 2) = any(regexp_split_to_array(current_setting('session.groups'), ',')::text[]));

backend/migrations/20221219032311_folders.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20221219032311_folders.up.sql

@@ -0,0 +1,19 @@
+-- Add up migration script here
+ALTER TABLE app_version
+RENAME COLUMN flow_id TO app_id;
+
+CREATE TABLE folder (
+    name VARCHAR(255),
+    workspace_id VARCHAR(50) NOT NULL,
+    display_name VARCHAR(100) NOT NULL,
+    owners VARCHAR(255)[] NOT NULL,
+    extra_perms JSONB NOT NULL DEFAULT '{}',
+    FOREIGN KEY (workspace_id) REFERENCES workspace(id) ON DELETE CASCADE,
+    PRIMARY KEY (workspace_id, name)
+);
+
+CREATE INDEX folder_extra_perms ON folder USING GIN (extra_perms);
+CREATE INDEX folder_owners ON folder USING GIN (owners);
+
+GRANT ALL ON folder TO windmill_user;
+GRANT ALL ON folder TO windmill_admin;

backend/migrations/20221219051235_folders_2.down.sql

@@ -0,0 +1,21 @@
+-- Add down migration script here
+DO
+$do$
+  DECLARE
+    i text;
+    arr text[] := array['resource', 'script', 'variable', 'schedule', 'flow', 'app'];
+  BEGIN
+  FOREACH i IN ARRAY arr
+  LOOP
+    EXECUTE FORMAT(
+      $$
+        DROP POLICY see_folder_extra_perms_user ON %1$I;
+      $$,
+      i
+    );
+  END LOOP;
+  END
+$do$;
+
+DROP POLICY see_extra_perms_user ON folder;
+DROP POLICY see_extra_perms_groups ON folder;

backend/migrations/20221219051235_folders_2.up.sql

@@ -0,0 +1,52 @@
+-- Add up migration script here
+DO
+$do$
+  DECLARE
+    i text;
+    arr text[] := array['resource', 'script', 'variable', 'schedule', 'flow', 'app'];
+  BEGIN
+  FOREACH i IN ARRAY arr
+  LOOP
+    EXECUTE FORMAT(
+      $$
+        CREATE POLICY see_folder_extra_perms_user ON %1$I FOR ALL
+        USING (SPLIT_PART(%1$I.path, '/', 1) = 'f' AND SPLIT_PART(%1$I.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_read'), ',')::text[]))
+        WITH CHECK (SPLIT_PART(%1$I.path, '/', 1) = 'f' AND SPLIT_PART(%1$I.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_write'), ',')::text[]));
+      $$,
+      i
+    );
+  END LOOP;
+  END
+$do$;
+ALTER TABLE folder ENABLE ROW LEVEL SECURITY;
+
+CREATE POLICY see_extra_perms_user ON folder FOR ALL
+USING (extra_perms ? CONCAT('u/', current_setting('session.user')) or (CONCAT('u/', current_setting('session.user')) = ANY(owners)))
+WITH CHECK ((CONCAT('u/', current_setting('session.user')) = ANY(owners)));
+
+CREATE POLICY see_extra_perms_groups ON folder FOR ALL
+USING (extra_perms ?| regexp_split_to_array(current_setting('session.pgroups'), ',')::text[] or (exists(
+    SELECT o FROM unnest(owners) as o
+    WHERE o = ANY(regexp_split_to_array(current_setting('session.pgroups'), ',')::text[]))))
+WITH CHECK (exists(
+    SELECT o FROM unnest(owners) as o
+    WHERE o = ANY(regexp_split_to_array(current_setting('session.pgroups'), ',')::text[])));
+
+DO
+$do$
+  DECLARE
+    i text;
+    arr text[] := array['script', 'flow', 'variable', 'resource', 'schedule'];
+  BEGIN
+  FOREACH i IN ARRAY arr
+  LOOP
+    EXECUTE FORMAT(
+      $$
+        ALTER TABLE %1$I DROP CONSTRAINT proper_id;
+        ALTER TABLE %1$I ADD CONSTRAINT proper_id CHECK (path ~ '^[ufg](\/[\w-]+){2,}$');
+      $$,
+      i
+    );
+  END LOOP;
+  END
+$do$;

backend/oauth_connect.json

@@ -1,96 +1,81 @@
 {
-    "github": {
-        "auth_url": "https://github.com/login/oauth/authorize",
-        "token_url": "https://github.com/login/oauth/access_token",
-        "scopes": [
-            "workflow",
-            "repo"
-        ]
-    },
-    "gitlab": {
-        "auth_url": "https://gitlab.com/oauth/authorize",
-        "token_url": "https://gitlab.com/oauth/token",
-        "scopes": [
-            "api"
-        ]
-    },
-    "bitbucket": {
-        "auth_url": "https://bitbucket.org/site/oauth2/authorize",
-        "token_url": "https://bitbucket.org/site/oauth2/access_token",
-        "scopes": [
-            "repository"
-        ]
-    },
-    "slack": {
-        "auth_url": "https://slack.com/oauth/authorize",
-        "token_url": "https://slack.com/api/oauth.access",
-        "scopes": [
-            "chat:write:user",
-            "users:read",
-            "users:read.email"
-        ]
-    },
-    "gsheets": {
-        "auth_url": "https://accounts.google.com/o/oauth2/v2/auth",
-        "token_url": "https://oauth2.googleapis.com/token",
-        "scopes": [
-            "https://www.googleapis.com/auth/spreadsheets"
-        ],
-        "extra_params": {
-            "access_type": "offline",
-            "consent": "prompt"
-        }
-    },
-    "gdrive": {
-        "auth_url": "https://accounts.google.com/o/oauth2/v2/auth",
-        "token_url": "https://oauth2.googleapis.com/token",
-        "scopes": [
-            "https://www.googleapis.com/auth/drive"
-        ],
-        "extra_params": {
-            "access_type": "offline",
-            "consent": "prompt"
-        }
-    },
-    "gmail": {
-        "auth_url": "https://accounts.google.com/o/oauth2/v2/auth",
-        "token_url": "https://oauth2.googleapis.com/token",
-        "scopes": [
-            "https://www.googleapis.com/auth/gmail.send"
-        ],
-        "extra_params": {
-            "access_type": "offline",
-            "consent": "prompt"
-        }
-    },
-    "gcal": {
-        "auth_url": "https://accounts.google.com/o/oauth2/v2/auth",
-        "token_url": "https://oauth2.googleapis.com/token",
-        "scopes": [
-            "https://www.googleapis.com/auth/calendar.events"
-        ],
-        "extra_params": {
-            "access_type": "offline",
-            "consent": "prompt"
-        }
-    },
-    "gcloud": {
-        "auth_url": "https://accounts.google.com/o/oauth2/v2/auth",
-        "token_url": "https://oauth2.googleapis.com/token",
-        "scopes": [
-            "https://www.googleapis.com/auth/cloud-platform"
-        ],
-        "extra_params": {
-            "access_type": "offline",
-            "consent": "prompt"
-        }
-    },
-    "basecamp": {
-        "auth_url": "https://launchpad.37signals.com/authorization/new",
-        "token_url": "https://launchpad.37signals.com/authorization/token",
-        "scopes": [],
-        "extra_params": {
-            "type": "web_server"
-        }
+  "github": {
+    "auth_url": "https://github.com/login/oauth/authorize",
+    "token_url": "https://github.com/login/oauth/access_token",
+    "scopes": ["workflow", "repo"]
+  },
+  "gitlab": {
+    "auth_url": "https://gitlab.com/oauth/authorize",
+    "token_url": "https://gitlab.com/oauth/token",
+    "scopes": ["api"]
+  },
+  "bitbucket": {
+    "auth_url": "https://bitbucket.org/site/oauth2/authorize",
+    "token_url": "https://bitbucket.org/site/oauth2/access_token",
+    "scopes": ["repository"]
+  },
+  "slack": {
+    "auth_url": "https://slack.com/oauth/authorize",
+    "token_url": "https://slack.com/api/oauth.access",
+    "scopes": ["chat:write:user", "users:read", "users:read.email"]
+  },
+  "gsheets": {
+    "auth_url": "https://accounts.google.com/o/oauth2/v2/auth",
+    "token_url": "https://oauth2.googleapis.com/token",
+    "scopes": ["https://www.googleapis.com/auth/spreadsheets"],
+    "extra_params": {
+      "access_type": "offline",
+      "prompt": "consent"
     }
+  },
+  "gdrive": {
+    "auth_url": "https://accounts.google.com/o/oauth2/v2/auth",
+    "token_url": "https://oauth2.googleapis.com/token",
+    "scopes": ["https://www.googleapis.com/auth/drive"],
+    "extra_params": {
+      "access_type": "offline",
+      "prompt": "consent"
+    }
+  },
+  "gmail": {
+    "auth_url": "https://accounts.google.com/o/oauth2/v2/auth",
+    "token_url": "https://oauth2.googleapis.com/token",
+    "scopes": ["https://www.googleapis.com/auth/gmail.send"],
+    "extra_params": {
+      "access_type": "offline",
+      "prompt": "consent"
+    }
+  },
+  "gcal": {
+    "auth_url": "https://accounts.google.com/o/oauth2/v2/auth",
+    "token_url": "https://oauth2.googleapis.com/token",
+    "scopes": ["https://www.googleapis.com/auth/calendar.events"],
+    "extra_params": {
+      "access_type": "offline",
+      "prompt": "consent"
+    }
+  },
+  "gcloud": {
+    "auth_url": "https://accounts.google.com/o/oauth2/v2/auth",
+    "token_url": "https://oauth2.googleapis.com/token",
+    "scopes": ["https://www.googleapis.com/auth/cloud-platform"],
+    "extra_params": {
+      "access_type": "offline",
+      "prompt": "consent"
+    }
+  },
+  "basecamp": {
+    "auth_url": "https://launchpad.37signals.com/authorization/new",
+    "token_url": "https://launchpad.37signals.com/authorization/token",
+    "scopes": [],
+    "extra_params": {
+      "type": "web_server"
+    }
+  },
+  "linkedin": {
+    "auth_url": "https://www.linkedin.com/oauth/v2/authorization",
+    "token_url": "https://www.linkedin.com/oauth/v2/accessToken",
+    "scopes": ["w_member_social", "r_liteprofile", "r_emailaddress"],
+    "req_body_auth": true
+  }
 }

backend/oauth_login.json

@@ -2,16 +2,25 @@
   "github": {
     "auth_url": "https://github.com/login/oauth/authorize",
     "token_url": "https://github.com/login/oauth/access_token",
+    "userinfo_url": "https://api.github.com/user",
     "scopes": ["user:email"]
   },
   "gitlab": {
     "auth_url": "https://gitlab.com/oauth/authorize",
     "token_url": "https://gitlab.com/oauth/token",
+    "userinfo_url": "https://gitlab.com/api/v4/user",
     "scopes": ["read_user"]
   },
   "google": {
     "auth_url": "https://accounts.google.com/o/oauth2/v2/auth",
     "token_url": "https://oauth2.googleapis.com/token",
+    "userinfo_url": "https://www.googleapis.com/oauth2/v1/userinfo?alt=json",
     "scopes": ["https://www.googleapis.com/auth/userinfo.email"]
+  },
+  "microsoft": {
+    "auth_url": "https://login.microsoftonline.com/common/oauth2/v2.0/authorize",
+    "token_url": "https://login.microsoftonline.com/common/oauth2/v2.0/token",
+    "userinfo_url": "https://graph.microsoft.com/oidc/userinfo",
+    "scopes": ["openid", "profile", "email"]
   }
 }

backend/parsers/windmill-parser-py/src/lib.rs

@@ -104,23 +104,19 @@ pub fn parse_python_signature(code: &str) -> error::Result<MainArgSignature> {
                             "bytes" => Typ::Bytes,
                             "datetime" => Typ::Datetime,
                             "datetime.datetime" => Typ::Datetime,
-                            _ => Typ::Unknown,
+                            _ => Typ::Resource(id),
                         },
                         _ => Typ::Unknown,
                     });
+
                     if typ == Typ::Unknown
                         && default.is_some()
                         && default != Some(json!(FUNCTION_CALL))
                     {
                         typ = json_to_typ(default.as_ref().unwrap());
                     }
-                    Arg {
-                        otyp: None,
-                        name: x.arg,
-                        typ: typ,
-                        has_default: default.is_some(),
-                        default,
-                    }
+
+                    Arg { otyp: None, name: x.arg, typ, has_default: default.is_some(), default }
                 })
                 .collect(),
         })
@@ -330,9 +326,11 @@ def main(test1: str, name: datetime.datetime = datetime.now(), byte: bytes = byt
 
 import os
 
+postgresql = dict
 def main(test1: str,
     name: datetime.datetime = datetime.now(),
-    byte: bytes = bytes(1)):
+    byte: bytes = bytes(1),
+    resource: postgresql = \"$res:g/all/resource\"):
 
 	print(f\"Hello World and a warm welcome especially to {name}\")
 	print(\"The env variable at `all/pretty_secret`: \", os.environ.get(\"ALL_PRETTY_SECRET\"))
@@ -366,6 +364,13 @@ def main(test1: str,
                         typ: Typ::Bytes,
                         default: Some(json!("<function call>")),
                         has_default: true
+                    },
+                    Arg {
+                        otyp: None,
+                        name: "resource".to_string(),
+                        typ: Typ::Resource("postgresql".to_string()),
+                        default: Some(json!("$res:g/all/resource")),
+                        has_default: true
                     }
                 ]
             }
@@ -435,7 +440,7 @@ def main():
 
 ";
         let r = parse_python_imports(code)?;
-        println!("{}", serde_json::to_string(&r)?);
+        // println!("{}", serde_json::to_string(&r)?);
         assert_eq!(r, vec!["wmill", "zanzibar", "matplotlib"]);
         Ok(())
     }

backend/src/main.rs

@@ -8,18 +8,19 @@
 
 use std::net::SocketAddr;
 
+use git_version::git_version;
 use sqlx::{Pool, Postgres};
 use windmill_common::utils::rd_string;
 use windmill_worker::WorkerConfig;
 
+const GIT_VERSION: &str = git_version!(args = ["--tag", "--always"], fallback = "unknown-version");
+
 #[tokio::main]
 async fn main() -> anyhow::Result<()> {
     dotenv::dotenv().ok();
 
     windmill_common::tracing_init::initialize_tracing();
 
-    let db = windmill_common::connect_db().await?;
-
     let num_workers = std::env::var("NUM_WORKERS")
         .ok()
         .and_then(|x| x.parse::<i32>().ok())
@@ -40,6 +41,8 @@ async fn main() -> anyhow::Result<()> {
         .and_then(|x| x.parse::<bool>().ok())
         .unwrap_or(false);
 
+    let db = windmill_common::connect_db(server_mode).await?;
+
     if server_mode {
         windmill_api::migrate_db(&db).await?;
     }
@@ -47,11 +50,10 @@ async fn main() -> anyhow::Result<()> {
     let (tx, rx) = tokio::sync::broadcast::channel::<()>(3);
     let shutdown_signal = windmill_common::shutdown_signal(tx);
 
-    let base_internal_url =
-        std::env::var("BASE_INTERNAL_URL").unwrap_or_else(|_| "http://localhost:8000".to_string());
-
     let base_url = std::env::var("BASE_URL").unwrap_or_else(|_| "http://localhost".to_string());
 
+    let base_internal_url =
+        std::env::var("BASE_INTERNAL_URL").unwrap_or_else(|_| "http://localhost:8000".to_string());
     let timeout = std::env::var("TIMEOUT")
         .ok()
         .and_then(|x| x.parse::<i32>().ok())
@@ -82,17 +84,39 @@ async fn main() -> anyhow::Result<()> {
                 let disable_nsjail = std::env::var("DISABLE_NSJAIL")
                     .ok()
                     .and_then(|x| x.parse::<bool>().ok())
-                    .unwrap_or(false);
+                    .unwrap_or(true);
                 let keep_job_dir = std::env::var("KEEP_JOB_DIR")
                     .ok()
                     .and_then(|x| x.parse::<bool>().ok())
                     .unwrap_or(false);
+                let license_key = std::env::var("LICENSE_KEY").ok();
+                let sync_bucket = std::env::var("S3_CACHE_BUCKET")
+                    .ok()
+                    .map(|e| Some(e))
+                    .unwrap_or(None);
+
+                #[cfg(feature = "enterprise")]
+                tracing::info!(
+                        "
+##############################
+Windmill Enterprise Edition {GIT_VERSION} LICENSE_KEY: {license_key:?}, S3_CACHE_BUCKET: {sync_bucket:?}
+##############################"
+                    );
+
+                #[cfg(not(feature = "enterprise"))]
+                tracing::info!(
+                    "
+##############################
+Windmill Community Edition {GIT_VERSION}
+##############################"
+                );
 
                 tracing::info!(
                     "DISABLE_NSJAIL: {disable_nsjail}, DISABLE_NUSER: {disable_nuser}, BASE_URL: \
                      {base_url}, SLEEP_QUEUE: {sleep_queue}, NUM_WORKERS: {num_workers}, TIMEOUT: \
                      {timeout}, KEEP_JOB_DIR: {keep_job_dir}"
                 );
+
                 run_workers(
                     db.clone(),
                     addr,
@@ -107,6 +131,8 @@ async fn main() -> anyhow::Result<()> {
                         keep_job_dir,
                     },
                     rx.resubscribe(),
+                    sync_bucket,
+                    license_key,
                 )
                 .await?;
             }
@@ -160,7 +186,30 @@ pub async fn run_workers(
     sleep_queue: u64,
     worker_config: WorkerConfig,
     rx: tokio::sync::broadcast::Receiver<()>,
+    mut periodic_script: Option<String>,
+    license_key: Option<String>,
 ) -> anyhow::Result<()> {
+    #[cfg(feature = "enterprise")]
+    if let Some(license_key) = license_key {
+        if license_key != "REQUIRED_DEC1" {
+            panic!("Invalid license key");
+        }
+    } else {
+        panic!("License key is required for the enterprise edition");
+    }
+
+    #[cfg(not(feature = "enterprise"))]
+    if license_key.is_some() {
+        panic!("License key is required ONLY for the enterprise edition");
+    }
+
+    #[cfg(not(feature = "enterprise"))]
+    if !worker_config.disable_nsjail {
+        tracing::warn!(
+            "NSJAIL to sandbox process in untrusted environments is an enterprise feature but allowed to be used for testing purposes"
+        );
+    }
+
     let instance_name = rd_string(5);
     let monitor = tokio_metrics::TaskMonitor::new();
 
@@ -180,6 +229,7 @@ pub async fn run_workers(
         let ip = ip.clone();
         let rx = rx.resubscribe();
         let worker_config = worker_config.clone();
+        let wp = periodic_script.take();
         handles.push(tokio::spawn(monitor.instrument(async move {
             tracing::info!(addr = %addr.to_string(), worker = %worker_name, "starting worker");
             windmill_worker::run_worker(
@@ -192,6 +242,7 @@ pub async fn run_workers(
                 &ip,
                 sleep_queue,
                 worker_config,
+                wp,
                 rx,
             )
             .await

backend/tests/fixtures/base.sql

@@ -2,8 +2,8 @@
 -- https://docs.rs/sqlx/latest/sqlx/attr.test.html
 
 INSERT INTO workspace
-            (id,               name,             owner,       domain)
-     VALUES ('test-workspace', 'test-workspace', 'test-user', null);
+            (id,               name,             owner)
+     VALUES ('test-workspace', 'test-workspace', 'test-user');
 
 INSERT INTO usr(workspace_id, email, username, is_admin, role) VALUES
 	('test-workspace', 'test@windmill.dev', 'test-user', true, 'Admin');
@@ -11,6 +11,8 @@ INSERT INTO usr(workspace_id, email, username, is_admin, role) VALUES
 INSERT INTO workspace_key(workspace_id, kind, key) VALUES
 	('test-workspace', 'cloud', 'test-key');
 
+insert INTO token(token, email, label, super_admin) VALUES ('SECRET_TOKEN', 'test@windmill.dev', 'test token', true);
+
 GRANT ALL PRIVILEGES ON TABLE workspace_key TO windmill_admin;
 GRANT ALL PRIVILEGES ON TABLE workspace_key TO windmill_user;
 
@@ -45,4 +47,4 @@ EXECUTE FUNCTION "notify_queue" ();
  AFTER UPDATE ON "queue"
     FOR EACH ROW
             WHEN (NEW.flow_status IS DISTINCT FROM OLD.flow_status)
-EXECUTE FUNCTION "notify_queue" ();
+EXECUTE FUNCTION "notify_queue" ();

backend/tests/worker.rs

@@ -821,14 +821,17 @@ impl RunJob {
             tx,
             "test-workspace",
             payload,
-            Some(args),
+            args,
             /* user */ "test-user",
+            /* email  */ "test@windmill.dev",
             /* permissioned_as */ "u/admin".to_string(),
             /* scheduled_for_o */ None,
             /* schedule_path */ None,
             /* parent_job */ None,
             /* is_flow_step */ false,
             /* running */ false,
+            None,
+            true,
         )
         .await
         .expect("push has to succeed");
@@ -932,6 +935,7 @@ fn spawn_test_worker(
             ip,
             sleep_queue,
             worker_config,
+            None,
             rx,
         )
         .await
@@ -1115,7 +1119,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
     let server = ApiServer::start(db.clone()).await;
 
     let write_file = r#"export async function main(loop: boolean, i: number, path: string) {  
-            await Deno.writeTextFile(`/shared/${path}`, `${loop} ${i}`);
+            await Deno.writeTextFile(`./shared/${path}`, `${loop} ${i}`);
         }"#
     .to_string();
 
@@ -1200,7 +1204,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
                                     .into(),
                                     language: ScriptLang::Deno,
                                     content: r#"export async function main(path: string, path2: string) {  
-                                        return await Deno.readTextFile(`/shared/${path}`) + "," + await Deno.readTextFile(`/shared/${path2}`);
+                                        return await Deno.readTextFile(`./shared/${path}`) + "," + await Deno.readTextFile(`./shared/${path2}`);
                                     }"#
                                     .to_string(),
                                     path: None,
@@ -1244,7 +1248,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
                     .into(),
                         language: ScriptLang::Deno,
                         content: r#"export async function main(path: string, loops: string[], path2: string) {
-                            return await Deno.readTextFile(`/shared/${path}`) + "," + loops + "," + await Deno.readTextFile(`/shared/${path2}`);
+                            return await Deno.readTextFile(`./shared/${path}`) + "," + loops + "," + await Deno.readTextFile(`./shared/${path2}`);
                         }"#
                         .to_string(),
                         path: None,
@@ -2329,24 +2333,143 @@ async fn test_failure_module(db: Pool<Postgres>) {
     assert_eq!(json!({ "l": [0, 1, 2] }), result);
 }
 
-// #[cfg(test)]
-// mod client_test {
-//     use windmill_common::error::to_anyhow;
+#[sqlx::test(fixtures("base"))]
+async fn test_flow_lock_all(db: Pool<Postgres>) {
+    use futures::StreamExt;
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await;
+    let port = server.addr.port();
 
-//     #[tokio::test]
-//     async fn test_rust_client() -> Result<(), Box<dyn std::error::Error>> {
-//         println!(
-//             "{:#?}",
-//             windmill_api_client::create_client(
-//                 "http://windmill.wimill.xyz",
-//                 "XXXXXXXXXXXXXXX".to_string(),
-//             )
-//             .get_variable("demo", "u/ruben/test", Some(true))
-//             .await
-//             .map_err(to_anyhow)
-//             .map(|v| { v.into_inner() })?
-//             .value
-//         );
-//         Ok(())
-//     }
-// }
+    let flow: windmill_api_client::types::OpenFlow = serde_json::from_value(serde_json::json!({
+        "summary": "",
+        "description": "",
+        "value": {
+            "modules": [
+                {
+                    "id": "a",
+                    "value": {
+                        "lock": null,
+                        "path": null,
+                        "type": "rawscript",
+                        "content": "import wmill\n\ndef main():\n  return \"Test\"\n",
+                        "language": "python3",
+                        "input_transforms": {}
+                    },
+                    "summary": null,
+                    "stop_after_if": null,
+                    "input_transforms": {}
+                },
+                {
+                    "id": "b",
+                    "value": {
+                        "lock": null,
+                        "path": null,
+                        "type": "rawscript",
+                        "content": "import * as wmill from \"https://deno.land/x/windmill@v1.50.0/mod.ts\"\n\nexport async function main() {\n  return \"Hello\"\n}\n",
+                        "language": "deno",
+                        "input_transforms": {}
+                    },
+                    "summary": null,
+                    "stop_after_if": null,
+                    "input_transforms": {}
+                },
+                {
+                    "id": "c",
+                    "value": {
+                        "lock": null,
+                        "path": null,
+                        "type": "rawscript",
+                        "content": "package inner\n\nimport (\n\t\"fmt\"\n\t\"rsc.io/quote\"\n  wmill \"github.com/windmill-labs/windmill-go-client\"\n)\n\n// the main must return (interface{}, error)\n\nfunc main() (interface{}, error) {\n\tfmt.Println(\"Hello, World\")\n  // v, _ := wmill.GetVariable(\"g/all/pretty_secret\")\n  return \"Test\"\n}\n",
+                        "language": "go",
+                        "input_transforms": {}
+                    },
+                    "summary": null,
+                    "stop_after_if": null,
+                    "input_transforms": {}
+                },
+                {
+                    "id": "d",
+                    "value": {
+                        "lock": null,
+                        "path": null,
+                        "type": "rawscript",
+                        "content": "\n# the last line of the stdout is the return value\necho \"Hello $msg\"\n",
+                        "language": "bash",
+                        "input_transforms": {}
+                    },
+                    "summary": null,
+                    "stop_after_if": null,
+                    "input_transforms": {}
+                }
+            ],
+            "failure_module": null
+        },
+        "schema": {
+            "type": "object",
+            "$schema": "https://json-schema.org/draft/2020-12/schema",
+            "required": [],
+            "properties": {}
+        }
+    }))
+    .unwrap();
+
+    let client = windmill_api_client::create_client(
+        &format!("http://localhost:{port}"),
+        "SECRET_TOKEN".to_owned(),
+    );
+    client
+        .create_flow(
+            "test-workspace",
+            &windmill_api_client::types::OpenFlowWPath {
+                open_flow: flow,
+                path: "g/all/flow_lock_all".to_owned(),
+            },
+        )
+        .await
+        .unwrap();
+    let mut str = listen_for_completed_jobs(&db).await;
+    let listen_first_job = str.next();
+    in_test_worker(&db, listen_first_job, port).await;
+
+    client
+        .get_flow_by_path("test-workspace", "g/all/flow_lock_all")
+        .await
+        .unwrap()
+        .into_inner()
+        .subtype_0
+        .value
+        .modules
+        .into_iter()
+        .for_each(|m| {
+            assert!(matches!(
+                m.value,
+                windmill_api_client::types::FlowModuleValue::Rawscript {
+                    language: windmill_api_client::types::RawScriptLanguage::Deno | windmill_api_client::types::RawScriptLanguage::Bash,
+                    lock: Some(ref lock),
+                    ..
+                } if lock == "")
+                || matches!(
+                m.value,
+                windmill_api_client::types::FlowModuleValue::Rawscript {
+                    language: windmill_api_client::types::RawScriptLanguage::Go | windmill_api_client::types::RawScriptLanguage::Python3,
+                    lock: Some(ref lock),
+                    ..
+                } if lock.len() > 0)
+            );
+        });
+}
+
+#[sqlx::test(fixtures("base"))]
+async fn test_rust_client(db: Pool<Postgres>) {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await;
+    let port = server.addr.port();
+
+    windmill_api_client::create_client(
+        &format!("http://localhost:{port}"),
+        "SECRET_TOKEN".to_string(),
+    )
+    .list_workspaces()
+    .await
+    .unwrap();
+}

backend/windmill-api/Cargo.toml

@@ -12,6 +12,9 @@ path = "src/lib.rs"
 name = "windmill_api"
 path = "src/main.rs"
 
+[features]
+enterprise = ["windmill-queue/enterprise"]
+
 [dependencies]
 windmill-queue.workspace = true
 windmill-common = { workspace = true, features = [
@@ -64,4 +67,5 @@ tokio-util.workspace = true
 tokio-tar.workspace = true
 hmac.workspace = true
 cookie.workspace = true
-sha2.workspace = true
+sha2.workspace = true
+urlencoding.workspace = true

backend/windmill-api/src/apps.rs

@@ -10,7 +10,7 @@ use std::collections::HashMap;
 use crate::{
     db::{UserDB, DB},
     jobs::script_path_to_payload,
-    users::{Authed, OptAuthed},
+    users::{require_owner_of_path, Authed, OptAuthed},
 };
 use axum::{
     extract::{Extension, Path, Query},
@@ -25,6 +25,7 @@ use sql_builder::{bind::Bind, SqlBuilder};
 use sqlx::{types::Uuid, FromRow};
 use windmill_audit::{audit_log, ActionKind};
 use windmill_common::{
+    apps::ListAppQuery,
     error::{to_anyhow, Error, JsonResult, Result},
     users::owner_to_token_owner,
     utils::{not_found_if_none, paginate, Pagination, StripPath},
@@ -36,6 +37,7 @@ pub fn workspaced_service() -> Router {
         .route("/list", get(list_apps))
         .route("/get/p/*path", get(get_app))
         .route("/get/v/*id", get(get_app_by_id))
+        .route("/exists/*path", post(exists_app))
         .route("/update/*path", post(update_app))
         .route("/delete/*path", delete(delete_app))
         .route("/create", post(create_app))
@@ -54,12 +56,13 @@ pub struct ListableApp {
     pub version: i64,
     pub extra_perms: serde_json::Value,
     pub execution_mode: String,
+    pub starred: bool,
 }
 
 #[derive(FromRow, Serialize, Deserialize)]
 pub struct AppVersion {
     pub id: i64,
-    pub flow_id: Uuid,
+    pub app_id: Uuid,
     pub value: serde_json::Value,
     pub created_by: String,
     pub created_at: chrono::DateTime<chrono::Utc>,
@@ -91,6 +94,7 @@ pub enum ExecutionMode {
 #[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct Policy {
     pub on_behalf_of: Option<String>,
+    pub on_behalf_of_email: Option<String>,
     //paths:
     // - script/<path>
     // - flow/<path>
@@ -118,26 +122,46 @@ pub struct EditApp {
 async fn list_apps(
     authed: Authed,
     Query(pagination): Query<Pagination>,
+    Query(lq): Query<ListAppQuery>,
     Extension(user_db): Extension<UserDB>,
     Path(w_id): Path<String>,
 ) -> JsonResult<Vec<ListableApp>> {
     let (per_page, offset) = paginate(pagination);
 
-    let sqlb = SqlBuilder::select_from("app")
+    let mut sqlb = SqlBuilder::select_from("app")
         .fields(&[
-            "id",
-            "workspace_id",
-            "path",
-            "summary",
-            "versions[array_upper(versions, 1)] as version",
-            "policy->>'execution_mode' as execution_mode",
-            "extra_perms",
+            "app.id",
+            "app.workspace_id",
+            "app.path",
+            "app.summary",
+            "app.versions[array_upper(app.versions, 1)] as version",
+            "app.policy->>'execution_mode' as execution_mode",
+            "app_version.created_at as edited_at",
+            "app.extra_perms",
+            "favorite.path IS NOT NULL as starred",
         ])
-        .order_by("path", true)
-        .and_where("workspace_id = ?".bind(&w_id))
+        .left()
+        .join("favorite")
+        .on(
+            "favorite.favorite_kind = 'app' AND favorite.workspace_id = app.workspace_id AND favorite.path = app.path AND favorite.usr = ?"
+                .bind(&authed.username),
+        )
+        .left()
+        .join("app_version")
+        .on(
+            "app_version.id = versions[array_upper(versions, 1)]"
+        )
+        .order_desc("favorite.path IS NOT NULL")
+        .order_by("app_version.created_at", true)
+        .and_where("app.workspace_id = ? OR app.workspace_id = 'starter'".bind(&w_id))
         .offset(offset)
         .limit(per_page)
         .clone();
+
+    if lq.starred_only.unwrap_or(false) {
+        sqlb.and_where_is_not_null("favorite.path");
+    }
+
     let sql = sqlb.sql().map_err(|e| Error::InternalErr(e.to_string()))?;
     let mut tx = user_db.begin(&authed).await?;
     let rows = sqlx::query_as::<_, ListableApp>(&sql)
@@ -186,7 +210,7 @@ async fn get_app_by_id(
         "SELECT app.id, app.path, app.summary, app.versions, app.policy,
         app.extra_perms, app_version.value, 
         app_version.created_at, app_version.created_by from app, app_version 
-        WHERE app_version.id = $1 AND app.id = app_version.flow_id AND app.workspace_id = $2",
+        WHERE app_version.id = $1 AND app.id = app_version.app_id AND app.workspace_id = $2",
         id,
         &w_id
     )
@@ -220,7 +244,7 @@ async fn create_app(
 
     let v_id = sqlx::query_scalar!(
         "INSERT INTO app_version
-            (flow_id, value, created_by)
+            (app_id, value, created_by)
             VALUES ($1, $2, $3) RETURNING id",
         id,
         app.value,
@@ -285,6 +309,7 @@ async fn delete_app(
 async fn update_app(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
+    Extension(db): Extension<DB>,
     Path((w_id, path)): Path<(String, StripPath)>,
     Json(ns): Json<EditApp>,
 ) -> Result<String> {
@@ -300,6 +325,12 @@ async fn update_app(
         sqlb.and_where_eq("workspace_id", "?".bind(&w_id));
 
         if let Some(npath) = &ns.path {
+            if npath != path {
+                if !authed.is_admin {
+                    require_owner_of_path(&w_id, &authed.username, &authed.groups, &path, &db)
+                        .await?;
+                }
+            }
             sqlb.set_str("path", npath);
         }
 
@@ -328,7 +359,7 @@ async fn update_app(
         "".to_string()
     };
     if let Some(nvalue) = &ns.value {
-        let flow_id = sqlx::query_scalar!(
+        let app_id = sqlx::query_scalar!(
             "SELECT id FROM app WHERE path = $1 AND workspace_id = $2",
             path,
             w_id
@@ -338,9 +369,9 @@ async fn update_app(
 
         let v_id = sqlx::query_scalar!(
             "INSERT INTO app_version
-                (flow_id, value, created_by)
+                (app_id, value, created_by)
                 VALUES ($1, $2, $3) RETURNING id",
-            flow_id,
+            app_id,
             nvalue,
             authed.username,
         )
@@ -434,49 +465,40 @@ async fn execute_component(
                 static_fields,
             );
         }
-        Policy { execution_mode: ExecutionMode::Viewer, triggerables: hm, on_behalf_of: None }
+        Policy {
+            execution_mode: ExecutionMode::Viewer,
+            triggerables: hm,
+            on_behalf_of: None,
+            on_behalf_of_email: None,
+        }
     } else {
         policy
     };
 
-    let (username, permissioned_as) = match policy.execution_mode {
+    let (username, permissioned_as, email) = match policy.execution_mode {
         ExecutionMode::Anonymous => {
             let username = opt_authed
                 .map(|a| a.username)
                 .unwrap_or_else(|| "anonymous".to_string());
-            let permissioned_as = policy
-                .on_behalf_of
-                .as_ref()
-                .ok_or_else(|| {
-                    Error::BadRequest(
-                "on_behalf_of is missing in the app policy and is required for anonymous execution"
-                    .to_string(),
-            )
-                })?
-                .to_string();
-            (username, permissioned_as)
+            let (permissioned_as, email) = get_on_behalf_of(&policy)?;
+            (username, permissioned_as, email)
         }
         ExecutionMode::Publisher => {
             let username = opt_authed.map(|a| a.username).ok_or_else(|| {
                 Error::BadRequest("publisher execution mode requires authentication".to_string())
             })?;
-            let permissioned_as = policy
-                .on_behalf_of
-                .as_ref()
-                .ok_or_else(|| {
-                    Error::BadRequest(
-                "on_behalf_of is missing in the app policy and is required for publisher execution"
-                    .to_string(),
-            )
-                })?
-                .to_string();
-            (username, permissioned_as)
+            let (permissioned_as, email) = get_on_behalf_of(&policy)?;
+            (username, permissioned_as, email)
         }
         ExecutionMode::Viewer => {
-            let username = opt_authed
-                .map(|a| a.username)
-                .ok_or_else(|| Error::BadRequest("".to_string()))?;
-            (username.clone(), owner_to_token_owner(&username, false))
+            let (username, email) = opt_authed.map(|a| (a.username, a.email)).ok_or_else(|| {
+                Error::BadRequest("Required to be authed in viewer mode".to_string())
+            })?;
+            (
+                username.clone(),
+                owner_to_token_owner(&username, false),
+                email,
+            )
         }
     };
 
@@ -510,14 +532,17 @@ async fn execute_component(
         tx,
         &w_id,
         job_payload,
-        Some(args),
+        args,
         &username,
+        &email,
         permissioned_as,
         None,
         None,
         None,
         false,
         false,
+        None,
+        true,
     )
     .await?;
 
@@ -525,6 +550,47 @@ async fn execute_component(
     Ok(uuid.to_string())
 }
 
+fn get_on_behalf_of(policy: &Policy) -> Result<(String, String)> {
+    let permissioned_as = policy
+        .on_behalf_of
+        .as_ref()
+        .ok_or_else(|| {
+            Error::BadRequest(
+                "on_behalf_of is missing in the app policy and is required for anonymous execution"
+                    .to_string(),
+            )
+        })?
+        .to_string();
+    let email = policy
+        .on_behalf_of_email
+        .as_ref()
+        .ok_or_else(|| {
+            Error::BadRequest(
+                "on_behalf_of is missing in the app policy and is required for anonymous execution"
+                    .to_string(),
+            )
+        })?
+        .to_string();
+    Ok((permissioned_as, email))
+}
+
+async fn exists_app(
+    Extension(db): Extension<DB>,
+    Path((w_id, path)): Path<(String, StripPath)>,
+) -> JsonResult<bool> {
+    let path = path.to_path();
+    let exists = sqlx::query_scalar!(
+        "SELECT EXISTS(SELECT 1 FROM app WHERE path = $1 AND workspace_id = $2)",
+        path,
+        w_id
+    )
+    .fetch_one(&db)
+    .await?
+    .unwrap_or(false);
+
+    Ok(Json(exists))
+}
+
 fn build_args(
     policy: Policy,
     path: String,

backend/windmill-api/src/db.rs

@@ -72,6 +72,34 @@ impl UserDB {
         )
         .fetch_optional(&mut tx)
         .await?;
+
+        let (folders_write, folders_read): &(Vec<_>, Vec<_>) =
+            &authed.folders.clone().into_iter().partition(|x| x.1);
+
+        let mut folders_read = folders_read.clone();
+        folders_read.extend(folders_write.clone());
+        sqlx::query!(
+            "SELECT set_config('session.folders_read', $1, true)",
+            folders_read
+                .iter()
+                .map(|x| x.0.clone())
+                .collect::<Vec<_>>()
+                .join(",")
+        )
+        .fetch_optional(&mut tx)
+        .await?;
+
+        sqlx::query!(
+            "SELECT set_config('session.folders_write', $1, true)",
+            folders_write
+                .iter()
+                .map(|x| x.0.clone())
+                .collect::<Vec<_>>()
+                .join(",")
+        )
+        .fetch_optional(&mut tx)
+        .await?;
+
         Ok(tx)
     }
 }

backend/windmill-api/src/favorite.rs

@@ -0,0 +1,75 @@
+/*
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
+ * This file and its contents are licensed under the AGPLv3 License.
+ * Please see the included NOTICE for copyright information and
+ * LICENSE-AGPL for a copy of the license.
+ */
+
+use crate::{db::DB, users::Authed};
+use axum::{
+    extract::{Extension, Path},
+    routing::post,
+    Json, Router,
+};
+use windmill_common::error::Result;
+
+use serde::{Deserialize, Serialize};
+
+pub fn workspaced_service() -> Router {
+    Router::new()
+        .route("/star", post(star))
+        .route("/unstar", post(unstar))
+}
+
+#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Clone)]
+#[sqlx(type_name = "FAVORITE_KIND", rename_all = "lowercase")]
+#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))]
+pub enum FavoriteKind {
+    Script,
+    Flow,
+    App,
+}
+#[derive(Deserialize)]
+pub struct Favorite {
+    pub favorite_kind: FavoriteKind,
+    pub path: String,
+}
+
+async fn star(
+    authed: Authed,
+    Extension(db): Extension<DB>,
+    Path(w_id): Path<String>,
+    Json(Favorite { favorite_kind, path }): Json<Favorite>,
+) -> Result<String> {
+    sqlx::query!(
+        "INSERT INTO favorite (workspace_id, usr, path, favorite_kind) VALUES ($1, $2, $3, $4) ON CONFLICT DO NOTHING",
+        &w_id,
+        authed.username,
+        path,
+        favorite_kind: FavoriteKind,
+    )
+    .execute(&db)
+    .await?;
+
+    Ok(format!("Starred {}", path))
+}
+
+async fn unstar(
+    authed: Authed,
+    Extension(db): Extension<DB>,
+    Path(w_id): Path<String>,
+    Json(Favorite { favorite_kind, path }): Json<Favorite>,
+) -> Result<String> {
+    sqlx::query!(
+        "DELETE FROM favorite WHERE workspace_id = $1 AND usr = $2 AND path = $3 AND favorite_kind = $4",
+        &w_id,
+        authed.username,
+        path,
+        favorite_kind: FavoriteKind,
+    )
+    .execute(&db)
+    .await?;
+
+    Ok(format!("Unstarred {}", path))
+}

backend/windmill-api/src/flows.rs

@@ -6,11 +6,12 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
+use hyper::StatusCode;
 use reqwest::Client;
 use sql_builder::prelude::*;
 
 use axum::{
-    extract::{Extension, Host, Path, Query},
+    extract::{Extension, Path, Query},
     routing::{get, post},
     Json, Router,
 };
@@ -19,16 +20,18 @@ use sqlx::{Postgres, Transaction};
 use windmill_audit::{audit_log, ActionKind};
 use windmill_common::{
     error::{self, to_anyhow, Error, JsonResult, Result},
-    flows::{Flow, ListFlowQuery, NewFlow},
+    flows::{Flow, ListFlowQuery, ListableFlow, NewFlow},
+    schedule::Schedule,
     utils::{
         http_get_from_hub, list_elems_from_hub, not_found_if_none, paginate, Pagination, StripPath,
     },
 };
-use windmill_queue::{push, JobPayload};
+use windmill_queue::{push, schedule::push_scheduled_job, JobPayload};
 
 use crate::{
     db::{UserDB, DB},
-    users::Authed,
+    schedule::clear_schedule,
+    users::{require_owner_of_path, Authed},
 };
 
 pub fn workspaced_service() -> Router {
@@ -53,24 +56,30 @@ async fn list_flows(
     Path(w_id): Path<String>,
     Query(pagination): Query<Pagination>,
     Query(lq): Query<ListFlowQuery>,
-) -> JsonResult<Vec<Flow>> {
+) -> JsonResult<Vec<ListableFlow>> {
     let (per_page, offset) = paginate(pagination);
 
     let mut sqlb = SqlBuilder::select_from("flow as o")
         .fields(&[
-            "workspace_id",
-            "path",
+            "o.workspace_id",
+            "o.path",
             "summary",
             "description",
-            "'{}'::jsonb as value",
             "edited_by",
             "edited_at",
             "archived",
-            "null schema",
             "extra_perms",
+            "favorite.path IS NOT NULL as starred",
         ])
+        .left()
+        .join("favorite")
+        .on(
+            "favorite.favorite_kind = 'flow' AND favorite.workspace_id = o.workspace_id AND favorite.path = o.path AND favorite.usr = ?"
+                .bind(&authed.username),
+        )
+        .order_desc("favorite.path IS NOT NULL")
         .order_by("edited_at", lq.order_desc.unwrap_or(true))
-        .and_where("workspace_id = ? OR workspace_id = 'starter'".bind(&w_id))
+        .and_where("o.workspace_id = ? OR o.workspace_id = 'starter'".bind(&w_id))
         .offset(offset)
         .limit(per_page)
         .clone();
@@ -87,42 +96,41 @@ async fn list_flows(
     if let Some(cb) = &lq.edited_by {
         sqlb.and_where_eq("edited_by", "?".bind(cb));
     }
+    if lq.starred_only.unwrap_or(false) {
+        sqlb.and_where_is_not_null("favorite.path");
+    }
 
     let sql = sqlb.sql().map_err(|e| Error::InternalErr(e.to_string()))?;
     let mut tx = user_db.begin(&authed).await?;
-    let rows = sqlx::query_as::<_, Flow>(&sql).fetch_all(&mut tx).await?;
+    let rows = sqlx::query_as::<_, ListableFlow>(&sql)
+        .fetch_all(&mut tx)
+        .await?;
     tx.commit().await?;
     Ok(Json(rows))
 }
 
 async fn list_hub_flows(
-    Authed { email, username, .. }: Authed,
+    Authed { email, .. }: Authed,
     Extension(http_client): Extension<Client>,
-    Host(host): Host,
 ) -> JsonResult<serde_json::Value> {
     let flows = list_elems_from_hub(
         http_client,
         "https://hub.windmill.dev/searchFlowData?approved=true",
-        email,
-        username,
-        host,
+        &email,
     )
     .await?;
     Ok(Json(flows))
 }
 
 pub async fn get_hub_flow_by_id(
-    Authed { email, username, .. }: Authed,
+    Authed { email, .. }: Authed,
     Path(id): Path<i32>,
     Extension(http_client): Extension<Client>,
-    Host(host): Host,
 ) -> JsonResult<serde_json::Value> {
     let value = http_get_from_hub(
         http_client,
         &format!("https://hub.windmill.dev/flows/{id}/json"),
-        email,
-        username,
-        host,
+        &email,
         false,
     )
     .await?
@@ -132,15 +140,35 @@ pub async fn get_hub_flow_by_id(
     Ok(Json(value))
 }
 
+async fn check_path_conflict<'c>(
+    tx: &mut Transaction<'c, Postgres>,
+    w_id: &str,
+    path: &str,
+) -> Result<()> {
+    let exists = sqlx::query_scalar!(
+        "SELECT EXISTS(SELECT 1 FROM flow WHERE path = $1 AND workspace_id = $2)",
+        path,
+        w_id
+    )
+    .fetch_one(tx)
+    .await?
+    .unwrap_or(false);
+    if exists {
+        return Err(Error::BadRequest(format!("Flow {} already exists", path)));
+    }
+    return Ok(());
+}
+
 async fn create_flow(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
     Path(w_id): Path<String>,
     Json(nf): Json<NewFlow>,
-) -> Result<String> {
+) -> Result<(StatusCode, String)> {
     // cron::Schedule::from_str(&ns.schedule).map_err(|e| error::Error::BadRequest(e.to_string()))?;
     let mut tx = user_db.clone().begin(&authed).await?;
 
+    check_path_conflict(&mut tx, &w_id, &nf.path).await?;
     check_schedule_conflict(&mut tx, &w_id, &nf.path).await?;
 
     sqlx::query!(
@@ -180,14 +208,17 @@ async fn create_flow(
         tx,
         &w_id,
         JobPayload::FlowDependencies { path: nf.path.clone() },
-        None,
+        serde_json::Map::new(),
         &authed.username,
+        &authed.email,
         windmill_common::users::owner_to_token_owner(&authed.username, false),
         None,
         None,
         None,
         false,
         false,
+        None,
+        true,
     )
     .await?;
     sqlx::query!(
@@ -200,7 +231,7 @@ async fn create_flow(
     .await?;
     tx.commit().await?;
 
-    Ok(nf.path.to_string())
+    Ok((StatusCode::CREATED, nf.path.to_string()))
 }
 
 async fn check_schedule_conflict<'c>(
@@ -229,6 +260,7 @@ async fn check_schedule_conflict<'c>(
 async fn update_flow(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
+    Extension(db): Extension<DB>,
     Path((w_id, flow_path)): Path<(String, StripPath)>,
     Json(nf): Json<NewFlow>,
 ) -> Result<String> {
@@ -240,7 +272,7 @@ async fn update_flow(
     let schema = nf.schema.map(|x| x.0);
     let old_dep_job = sqlx::query_scalar!(
         "SELECT dependency_job FROM flow WHERE path = $1 AND workspace_id = $2",
-        nf.path,
+        flow_path,
         w_id
     )
     .fetch_optional(&mut tx)
@@ -261,6 +293,45 @@ async fn update_flow(
     .execute(&mut tx)
     .await?;
 
+    if nf.path != flow_path {
+        check_schedule_conflict(&mut tx, &w_id, &nf.path).await?;
+
+        if !authed.is_admin {
+            require_owner_of_path(&w_id, &authed.username, &authed.groups, &flow_path, &db).await?;
+        }
+
+        let mut schedulables = sqlx::query_as!(
+            Schedule,
+                "UPDATE schedule SET script_path = $1 WHERE script_path = $2 AND path != $2 AND workspace_id = $3 AND is_flow IS true RETURNING *",
+                nf.path,
+                flow_path,
+                w_id,
+            )
+            .fetch_all(&mut tx)
+            .await?;
+
+        let schedule = sqlx::query_as!(Schedule,
+            "UPDATE schedule SET path = $1, script_path = $1 WHERE path = $2 AND workspace_id = $3 AND is_flow IS true RETURNING *",
+            nf.path,
+            flow_path,
+            w_id,
+        )
+        .fetch_optional(&mut tx)
+        .await?;
+
+        if let Some(schedule) = schedule {
+            schedulables.push(schedule);
+        }
+
+        for schedule in schedulables {
+            clear_schedule(&mut tx, flow_path, true).await?;
+
+            if schedule.enabled {
+                tx = push_scheduled_job(tx, schedule).await?;
+            }
+        }
+    }
+
     audit_log(
         &mut tx,
         &authed.username,
@@ -284,14 +355,17 @@ async fn update_flow(
         tx,
         &w_id,
         JobPayload::FlowDependencies { path: nf.path.clone() },
-        None,
+        serde_json::Map::new(),
         &authed.username,
+        &authed.email,
         windmill_common::users::owner_to_token_owner(&authed.username, false),
         None,
         None,
         None,
         false,
         false,
+        None,
+        true,
     )
     .await?;
     sqlx::query!(
@@ -414,6 +488,7 @@ mod tests {
                             InputTransform::Static { value: serde_json::json!("test2") },
                         )]
                         .into(),
+                        hash: None,
                     },
                     stop_after_if: None,
                     summary: None,
@@ -465,6 +540,7 @@ mod tests {
                 value: FlowModuleValue::Script {
                     path: "test".to_string(),
                     input_transforms: HashMap::new(),
+                    hash: None,
                 },
                 stop_after_if: Some(StopAfterIf {
                     expr: "previous.isEmpty()".to_string(),

backend/windmill-api/src/folders.rs

@@ -0,0 +1,544 @@
+/*
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
+ * This file and its contents are licensed under the AGPLv3 License.
+ * Please see the included NOTICE for copyright information and
+ * LICENSE-AGPL for a copy of the license.
+ */
+
+use crate::{
+    db::{UserDB, DB},
+    users::Authed,
+};
+use axum::{
+    extract::{Extension, Path, Query},
+    routing::{delete, get, post},
+    Json, Router,
+};
+use itertools::Itertools;
+use windmill_audit::{audit_log, ActionKind};
+use windmill_common::{
+    error::{self, Error, JsonResult, Result},
+    users::owner_to_token_owner,
+    utils::{not_found_if_none, paginate, Pagination},
+};
+
+use serde::{Deserialize, Serialize};
+use sqlx::{query_scalar, FromRow, Postgres, Transaction};
+
+pub fn workspaced_service() -> Router {
+    Router::new()
+        .route("/list", get(list_folders))
+        .route("/listnames", get(list_foldernames))
+        .route("/create", post(create_folder))
+        .route("/get/:name", get(get_folder))
+        .route("/update/:name", post(update_folder))
+        .route("/getusage/:name", get(get_folder_usage))
+        .route("/delete/:name", delete(delete_folder))
+        .route("/addowner/:name", post(add_owner))
+        .route("/removeowner/:name", post(remove_owner))
+        .route("/is_owner", get(is_owner))
+}
+
+#[derive(FromRow, Serialize, Deserialize, Clone)]
+pub struct Folder {
+    pub workspace_id: String,
+    pub name: String,
+    pub display_name: String,
+    pub owners: Vec<String>,
+    pub extra_perms: serde_json::Value,
+}
+
+#[derive(Deserialize)]
+pub struct NewFolder {
+    pub name: String,
+    pub display_name: Option<String>,
+    pub owners: Option<Vec<String>>,
+    pub extra_perms: Option<serde_json::Value>,
+}
+
+#[derive(Deserialize)]
+pub struct UpdateFolder {
+    pub display_name: Option<String>,
+    pub owners: Option<Vec<String>>,
+    pub extra_perms: Option<serde_json::Value>,
+}
+
+#[derive(Deserialize)]
+pub struct Owner {
+    pub owner: String,
+}
+
+async fn list_folders(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    Path(w_id): Path<String>,
+    Query(pagination): Query<Pagination>,
+) -> JsonResult<Vec<Folder>> {
+    let (per_page, offset) = paginate(pagination);
+    let mut tx = user_db.begin(&authed).await?;
+
+    let rows = sqlx::query_as!(
+        Folder,
+        "SELECT workspace_id, name, display_name, owners, extra_perms FROM folder WHERE workspace_id = $1 ORDER BY name desc LIMIT $2 OFFSET $3",
+        w_id,
+        per_page as i64,
+        offset as i64
+    )
+    .fetch_all(&mut tx)
+    .await?;
+    tx.commit().await?;
+
+    Ok(Json(rows))
+}
+async fn list_foldernames(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    Path(w_id): Path<String>,
+    Query(pagination): Query<Pagination>,
+) -> JsonResult<Vec<String>> {
+    let (per_page, offset) = paginate(pagination);
+    let mut tx = user_db.begin(&authed).await?;
+
+    let rows = sqlx::query_scalar!(
+        "SELECT name FROM folder WHERE workspace_id = $1 ORDER BY name desc LIMIT $2 OFFSET $3",
+        w_id,
+        per_page as i64,
+        offset as i64
+    )
+    .fetch_all(&mut tx)
+    .await?;
+
+    tx.commit().await?;
+
+    Ok(Json(rows))
+}
+
+async fn check_name_conflict<'c>(
+    tx: &mut Transaction<'c, Postgres>,
+    w_id: &str,
+    name: &str,
+) -> Result<()> {
+    let exists = sqlx::query_scalar!(
+        "SELECT EXISTS(SELECT 1 FROM folder WHERE name = $1 AND workspace_id = $2)",
+        name,
+        w_id
+    )
+    .fetch_one(tx)
+    .await?
+    .unwrap_or(false);
+    if exists {
+        return Err(windmill_common::error::Error::BadRequest(format!(
+            "Folder {} already exists",
+            name
+        )));
+    }
+    return Ok(());
+}
+
+async fn create_folder(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    Path(w_id): Path<String>,
+    Json(ng): Json<NewFolder>,
+) -> Result<String> {
+    let mut tx = user_db.begin(&authed).await?;
+
+    check_name_conflict(&mut tx, &w_id, &ng.name).await?;
+    let owner = owner_to_token_owner(&authed.username, false);
+    let owners = &ng.owners.unwrap_or(vec![owner.clone()]);
+
+    if let Some(extra_perms) = ng.extra_perms.clone() {
+        for o in owners {
+            if !extra_perms
+                .get(&o)
+                .and_then(|x| x.as_bool())
+                .unwrap_or(false)
+            {
+                return Err(windmill_common::error::Error::BadRequest(format!(
+                    "Owner {} would not have permission to write to folder and that is an inconsistent state",
+                    o
+                )));
+            }
+        }
+    }
+
+    let extra_perms = ng.extra_perms.unwrap_or_else(|| {
+        let mut map = serde_json::Map::new();
+        for o in owners {
+            map.insert(o.clone(), serde_json::json!(true));
+        }
+        serde_json::Value::Object(map)
+    });
+
+    sqlx::query_as!(
+        Folder,
+        "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms) VALUES ($1, $2, $3, $4, $5)",
+        w_id,
+        ng.name,
+        ng.display_name.unwrap_or(ng.name.clone()),
+        owners,
+        extra_perms,
+    )
+    .execute(&mut tx)
+    .await?;
+
+    audit_log(
+        &mut tx,
+        &authed.username,
+        "folder.create",
+        ActionKind::Create,
+        &w_id,
+        Some(&ng.name.to_string()),
+        None,
+    )
+    .await?;
+
+    tx.commit().await?;
+    Ok(format!("Created folder {}", ng.name))
+}
+
+pub async fn is_owner(
+    Authed { username, is_admin, groups, .. }: Authed,
+    Extension(db): Extension<DB>,
+    Path((w_id, name)): Path<(String, String)>,
+) -> JsonResult<bool> {
+    if is_admin {
+        Ok(Json(true))
+    } else {
+        Ok(Json(
+            require_is_owner(&name, &username, &groups, &w_id, &db)
+                .await
+                .is_ok(),
+        ))
+    }
+}
+
+pub async fn require_is_owner(
+    folder_name: &str,
+    username: &str,
+    groups: &Vec<String>,
+    w_id: &str,
+    db: &DB,
+) -> Result<()> {
+    let is_owner = query_scalar!(
+        "SELECT EXISTS(SELECT 1 FROM folder WHERE CONCAT('u/', $1::text) = ANY(owners) AND name = $2 AND workspace_id = $4) OR exists(
+            SELECT 1 FROM folder, unnest(folder.owners) as o
+            WHERE o = ANY($3::text[]) AND folder.name = $2 AND folder.workspace_id = $4)",
+        username,
+        folder_name,
+        groups,
+        w_id,
+    ).fetch_one(db)
+    .await?
+    .unwrap_or(false);
+    if !is_owner {
+        Err(Error::BadRequest(format!(
+            "{} is not an owner of {} and hence is not authorized to perform this operation",
+            username, folder_name
+        )))
+    } else {
+        Ok(())
+    }
+}
+
+async fn update_folder(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    Path((w_id, name)): Path<(String, String)>,
+    Json(ng): Json<UpdateFolder>,
+) -> Result<String> {
+    use sql_builder::prelude::*;
+
+    let mut sqlb = SqlBuilder::update_table("folder");
+    sqlb.and_where_eq("name", "?".bind(&name));
+    sqlb.and_where_eq("workspace_id", "?".bind(&w_id));
+
+    if let Some(display_name) = ng.display_name {
+        sqlb.set("display_name", display_name);
+    }
+    if let Some(owners) = ng.owners {
+        sqlb.set_str("owners", format!("{{{}}}", owners.into_iter().join(",")));
+    }
+    if let Some(extra_perms) = ng.extra_perms {
+        sqlb.set_str("extra_perms", extra_perms.to_string());
+    }
+
+    sqlb.returning("*");
+
+    let mut tx = user_db.begin(&authed).await?;
+
+    let sql = sqlb
+        .sql()
+        .map_err(|e| error::Error::InternalErr(e.to_string()))?;
+    let nfolder = sqlx::query_as::<_, Folder>(&sql).fetch_one(&mut tx).await?;
+
+    if let Some(extra_perms) = nfolder.extra_perms.as_object() {
+        for o in nfolder.owners {
+            if !extra_perms
+                .get(&o)
+                .and_then(|x| x.as_bool())
+                .unwrap_or(false)
+            {
+                return Err(windmill_common::error::Error::BadRequest(format!(
+                    "Owner {} would not have permission to write to folder and that is an invalid state",
+                    o
+                )));
+            }
+        }
+    }
+
+    audit_log(
+        &mut tx,
+        &authed.username,
+        "folder.update",
+        ActionKind::Update,
+        &w_id,
+        Some(&name.to_string()),
+        None,
+    )
+    .await?;
+
+    tx.commit().await?;
+    Ok(format!("Updated folder {}", name))
+}
+
+pub async fn get_folderopt<'c>(
+    db: &mut Transaction<'c, Postgres>,
+    w_id: &str,
+    name: &str,
+) -> Result<Option<Folder>> {
+    let folderopt = sqlx::query_as!(
+        Folder,
+        "SELECT workspace_id, name, display_name, owners, extra_perms FROM folder WHERE name = $1 AND workspace_id = $2",
+        name,
+        w_id
+    )
+    .fetch_optional(db)
+    .await?;
+    Ok(folderopt)
+}
+
+async fn get_folder(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    Path((w_id, name)): Path<(String, String)>,
+) -> JsonResult<Folder> {
+    let mut tx = user_db.begin(&authed).await?;
+
+    let folder = not_found_if_none(get_folderopt(&mut tx, &w_id, &name).await?, "Folder", &name)?;
+
+    tx.commit().await?;
+    Ok(Json(folder))
+}
+
+#[derive(Serialize)]
+struct FolderUsage {
+    pub scripts: i64,
+    pub schedules: i64,
+    pub flows: i64,
+    pub apps: i64,
+    pub resources: i64,
+    pub variables: i64,
+}
+async fn get_folder_usage(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    Path((w_id, name)): Path<(String, String)>,
+) -> JsonResult<FolderUsage> {
+    let mut tx = user_db.begin(&authed).await?;
+
+    let scripts = sqlx::query_scalar!(
+        "SELECT count(path) FROM script WHERE path LIKE 'f/' || $1 || '%' AND archived IS false AND workspace_id = $2",
+        name,
+        w_id
+    )
+    .fetch_one(&mut tx)
+    .await?
+    .unwrap_or(0);
+
+    let schedules = sqlx::query_scalar!(
+        "SELECT count(path) FROM schedule WHERE path LIKE 'f/' || $1 || '%'  AND workspace_id = $2",
+        name,
+        w_id
+    )
+    .fetch_one(&mut tx)
+    .await?
+    .unwrap_or(0);
+
+    let flows = sqlx::query_scalar!(
+        "SELECT count(path) FROM flow WHERE path LIKE 'f/' || $1 || '%' AND archived IS false AND workspace_id = $2",
+        name,
+        w_id
+    )
+    .fetch_one(&mut tx)
+    .await?
+    .unwrap_or(0);
+
+    let apps = sqlx::query_scalar!(
+        "SELECT count(path) FROM app WHERE path LIKE 'f/' || $1 || '%'  AND workspace_id = $2",
+        name,
+        w_id
+    )
+    .fetch_one(&mut tx)
+    .await?
+    .unwrap_or(0);
+
+    let resources = sqlx::query_scalar!(
+        "SELECT count(path) FROM resource WHERE path LIKE 'f/' || $1 || '%'  AND workspace_id = $2",
+        name,
+        w_id
+    )
+    .fetch_one(&mut tx)
+    .await?
+    .unwrap_or(0);
+
+    let variables = sqlx::query_scalar!(
+        "SELECT count(path) FROM variable WHERE path LIKE 'f/' || $1 || '%'  AND workspace_id = $2",
+        name,
+        w_id
+    )
+    .fetch_one(&mut tx)
+    .await?
+    .unwrap_or(0);
+
+    tx.commit().await?;
+    Ok(Json(FolderUsage {
+        scripts,
+        flows,
+        schedules,
+        apps,
+        resources,
+        variables,
+    }))
+}
+
+async fn delete_folder(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    Path((w_id, name)): Path<(String, String)>,
+) -> Result<String> {
+    let mut tx = user_db.begin(&authed).await?;
+
+    not_found_if_none(get_folderopt(&mut tx, &w_id, &name).await?, "Folder", &name)?;
+
+    sqlx::query!(
+        "DELETE FROM folder WHERE name = $1 AND workspace_id = $2",
+        name,
+        w_id
+    )
+    .execute(&mut tx)
+    .await?;
+    audit_log(
+        &mut tx,
+        &authed.username,
+        "folder.delete",
+        ActionKind::Delete,
+        &w_id,
+        Some(&name.to_string()),
+        None,
+    )
+    .await?;
+    tx.commit().await?;
+    Ok(format!("delete folder at name {}", name))
+}
+
+async fn add_owner(
+    authed: Authed,
+    Extension(db): Extension<DB>,
+    Extension(user_db): Extension<UserDB>,
+    Path((w_id, name)): Path<(String, String)>,
+    Json(Owner { owner }): Json<Owner>,
+) -> Result<String> {
+    let mut tx = user_db.begin(&authed).await?;
+
+    not_found_if_none(get_folderopt(&mut tx, &w_id, &name).await?, "Folder", &name)?;
+    if !authed.is_admin {
+        require_is_owner(&name, &authed.username, &authed.groups, &w_id, &db).await?;
+    }
+
+    sqlx::query!(
+        "UPDATE folder SET owners = array_append(owners, $1) WHERE name = $2 AND workspace_id = $3 AND NOT $1 = ANY(owners) RETURNING name",
+        owner,
+        name,
+        &w_id,
+    )
+    .fetch_optional(&mut tx)
+    .await?;
+
+    audit_log(
+        &mut tx,
+        &authed.username,
+        "folder.add_owner",
+        ActionKind::Update,
+        &w_id,
+        Some(&name.to_string()),
+        Some([("owner", owner.as_str())].into()),
+    )
+    .await?;
+    tx.commit().await?;
+    Ok(format!("Added {} to folder {}", owner, name))
+}
+
+pub async fn get_folders_for_user(
+    w_id: &str,
+    username: &str,
+    groups: &[String],
+    db: &DB,
+) -> Result<Vec<(String, bool)>> {
+    let mut perms = groups
+        .into_iter()
+        .map(|x| format!("g/{}", x))
+        .collect::<Vec<_>>();
+    perms.insert(0, format!("u/{}", username));
+    let folders = sqlx::query!(
+        "SELECT name, (EXISTS (SELECT 1 FROM (SELECT key, value FROM jsonb_each_text(extra_perms) WHERE key = ANY($1)) t  WHERE value::boolean IS true)) as write  FROM folder
+        WHERE extra_perms ?| $1  AND workspace_id = $2",
+        &perms[..],
+        w_id,
+    )
+    .fetch_all(db)
+    .await?
+    .into_iter()
+    .map(|x| (x.name, x.write.unwrap_or(false)))
+    .collect();
+
+    Ok(folders)
+}
+
+async fn remove_owner(
+    authed: Authed,
+    Extension(db): Extension<DB>,
+    Extension(user_db): Extension<UserDB>,
+    Path((w_id, name)): Path<(String, String)>,
+    Json(Owner { owner }): Json<Owner>,
+) -> Result<String> {
+    let mut tx = user_db.begin(&authed).await?;
+
+    not_found_if_none(get_folderopt(&mut tx, &w_id, &name).await?, "Folder", &name)?;
+    if !authed.is_admin {
+        require_is_owner(&name, &authed.username, &authed.groups, &w_id, &db).await?;
+    }
+
+    sqlx::query!(
+        "UPDATE folder SET owners = array_remove(owners, $1) WHERE name = $2 AND workspace_id = $3 RETURNING name",
+        owner,
+        name,
+        &w_id,
+    )
+    .fetch_optional(&mut tx)
+    .await?;
+
+    audit_log(
+        &mut tx,
+        &authed.username,
+        "folder.remove_owner",
+        ActionKind::Update,
+        &w_id,
+        Some(&name.to_string()),
+        Some([("owner", owner.as_str())].into()),
+    )
+    .await?;
+    tx.commit().await?;
+    Ok(format!("Removed {} to folder {}", owner, name))
+}

backend/windmill-api/src/granular_acls.rs

@@ -6,7 +6,10 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
-use crate::{db::UserDB, users::Authed};
+use crate::{
+    db::{UserDB, DB},
+    users::{require_owner_of_path, Authed},
+};
 use axum::{
     extract::{Extension, Path},
     routing::{get, post},
@@ -34,6 +37,7 @@ pub struct GranularAcl {
 
 async fn add_granular_acl(
     authed: Authed,
+    Extension(db): Extension<DB>,
     Extension(user_db): Extension<UserDB>,
     Path((w_id, path)): Path<(String, StripPath)>,
     Json(GranularAcl { owner, write }): Json<GranularAcl>,
@@ -44,7 +48,20 @@ async fn add_granular_acl(
         .ok_or_else(|| Error::BadRequest("Invalid path or kind".to_string()))?;
     let mut tx = user_db.begin(&authed).await?;
 
-    let identifier = if kind == "group_" { "name" } else { "path" };
+    if !authed.is_admin {
+        if kind == "folder" {
+            crate::folders::require_is_owner(&path, &authed.username, &authed.groups, &w_id, &db)
+                .await?;
+        } else if kind == "group_" {
+        } else {
+            require_owner_of_path(&w_id, &authed.username, &authed.groups, &path, &db).await?;
+        }
+    }
+    let identifier = if kind == "group_" || kind == "folder" {
+        "name"
+    } else {
+        "path"
+    };
     let obj_o = sqlx::query_scalar::<_, serde_json::Value>(&format!(
         "UPDATE {kind} SET extra_perms = jsonb_set(extra_perms, '{{\"{owner}\"}}', to_jsonb($1), \
          true) WHERE {identifier} = $2 AND workspace_id = $3 RETURNING extra_perms"
@@ -63,17 +80,25 @@ async fn add_granular_acl(
 
 async fn remove_granular_acl(
     authed: Authed,
+    Extension(db): Extension<DB>,
     Extension(user_db): Extension<UserDB>,
     Path((w_id, path)): Path<(String, StripPath)>,
     Json(GranularAcl { owner, write: _ }): Json<GranularAcl>,
 ) -> Result<String> {
     let path = path.to_path();
+    if !authed.is_admin {
+        require_owner_of_path(&w_id, &authed.username, &authed.groups, &path, &db).await?;
+    }
     let (kind, path) = path
         .split_once('/')
         .ok_or_else(|| Error::BadRequest("Invalid path or kind".to_string()))?;
     let mut tx = user_db.begin(&authed).await?;
 
-    let identifier = if kind == "group_" { "name" } else { "path" };
+    let identifier = if kind == "group_" || kind == "folder" {
+        "name"
+    } else {
+        "path"
+    };
     let obj_o = sqlx::query_scalar::<_, serde_json::Value>(&format!(
         "UPDATE {kind} SET extra_perms = extra_perms - $1 WHERE {identifier} = $2 AND \
          workspace_id = $3 RETURNING extra_perms"

backend/windmill-api/src/groups.rs

@@ -8,7 +8,7 @@
 
 use crate::{
     db::{UserDB, DB},
-    users::Authed,
+    users::{get_groups_for_user, Authed},
 };
 use axum::{
     extract::{Extension, Path, Query},
@@ -23,7 +23,8 @@ use windmill_common::{
 };
 
 use serde::{Deserialize, Serialize};
-use sqlx::{FromRow, Postgres, Transaction};
+use sqlx::{query_scalar, FromRow, Postgres, Transaction};
+use windmill_queue::CLOUD_HOSTED;
 
 pub fn workspaced_service() -> Router {
     Router::new()
@@ -35,6 +36,7 @@ pub fn workspaced_service() -> Router {
         .route("/delete/:name", delete(delete_group))
         .route("/adduser/:name", post(add_user))
         .route("/removeuser/:name", post(remove_user))
+        .route("/is_owner", get(is_owner))
 }
 
 #[derive(FromRow, Serialize, Deserialize)]
@@ -90,20 +92,97 @@ async fn list_groups(
     Ok(Json(rows))
 }
 
+#[derive(Deserialize)]
+struct QueryListGroup {
+    pub only_member_of: Option<bool>,
+}
 async fn list_group_names(
+    Authed { username, .. }: Authed,
     Extension(db): Extension<DB>,
+    Query(QueryListGroup { only_member_of }): Query<QueryListGroup>,
     Path(w_id): Path<String>,
 ) -> JsonResult<Vec<String>> {
-    let rows = sqlx::query_scalar!(
-        "SELECT name FROM group_ WHERE workspace_id = $1 ORDER BY name desc",
-        w_id
-    )
-    .fetch_all(&db)
-    .await?;
+    let rows = if !only_member_of.unwrap_or(false) {
+        sqlx::query_scalar!(
+            "SELECT name FROM group_ WHERE workspace_id = $1 ORDER BY name desc",
+            w_id
+        )
+        .fetch_all(&db)
+        .await?
+    } else {
+        get_groups_for_user(&w_id, &username, &db).await?
+    };
 
     Ok(Json(rows))
 }
 
+async fn check_name_conflict<'c>(
+    tx: &mut Transaction<'c, Postgres>,
+    w_id: &str,
+    name: &str,
+) -> Result<()> {
+    let exists = sqlx::query_scalar!(
+        "SELECT EXISTS(SELECT 1 FROM group_ WHERE name = $1 AND workspace_id = $2)",
+        name,
+        w_id
+    )
+    .fetch_one(tx)
+    .await?
+    .unwrap_or(false);
+    if exists {
+        return Err(windmill_common::error::Error::BadRequest(format!(
+            "Group {} already exists",
+            name
+        )));
+    }
+    return Ok(());
+}
+
+pub async fn is_owner(
+    Authed { username, is_admin, groups, .. }: Authed,
+    Extension(db): Extension<DB>,
+    Path((w_id, name)): Path<(String, String)>,
+) -> JsonResult<bool> {
+    if is_admin {
+        Ok(Json(true))
+    } else {
+        Ok(Json(
+            require_is_owner(&name, &username, &groups, &w_id, &db)
+                .await
+                .is_ok(),
+        ))
+    }
+}
+
+pub async fn require_is_owner(
+    group_name: &str,
+    username: &str,
+    groups: &Vec<String>,
+    w_id: &str,
+    db: &DB,
+) -> Result<()> {
+    let is_owner = query_scalar!(
+        "SELECT EXISTS(SELECT 1 FROM group_ WHERE (group_.extra_perms ->> CONCAT('u/', $1::text))::boolean AND name = $2 AND workspace_id = $4) OR exists(
+            SELECT 1 FROM group_ g, jsonb_each_text(g.extra_perms) f 
+    WHERE $2 = g.name AND $4 = g.workspace_id AND SPLIT_PART(key, '/', 1) = 'g' AND key = ANY($3::text[])
+    AND value::boolean)",
+        username,
+        group_name,
+        groups,
+        w_id,
+    ).fetch_one(db)
+    .await?
+    .unwrap_or(false);
+    if !is_owner {
+        Err(Error::BadRequest(format!(
+            "{} is not an owner of {} and hence is not authorized to perform this operation",
+            username, group_name
+        )))
+    } else {
+        Ok(())
+    }
+}
+
 async fn create_group(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
@@ -112,6 +191,8 @@ async fn create_group(
 ) -> Result<String> {
     let mut tx = user_db.begin(&authed).await?;
 
+    check_name_conflict(&mut tx, &w_id, &ng.name).await?;
+
     sqlx::query_as!(
         Group,
         "INSERT INTO group_ (workspace_id, name, summary, extra_perms) VALUES ($1, $2, $3, $4)",
@@ -159,6 +240,16 @@ async fn get_group(
     Extension(user_db): Extension<UserDB>,
     Path((w_id, name)): Path<(String, String)>,
 ) -> JsonResult<GroupInfo> {
+    if *CLOUD_HOSTED && w_id == "demo" && name == "all" && !authed.is_admin {
+        return Ok(Json(GroupInfo {
+            workspace_id: w_id,
+            name: name,
+            summary: Some("The group that contains all users".to_string()),
+            members: vec!["redacted_in_demo_workspace".to_string()],
+            extra_perms: serde_json::json!({}),
+        }));
+    }
+
     let mut tx = user_db.begin(&authed).await?;
 
     let group = not_found_if_none(get_group_opt(&mut tx, &w_id, &name).await?, "Group", &name)?;
@@ -185,11 +276,13 @@ async fn get_group(
 
 async fn delete_group(
     authed: Authed,
+    Extension(db): Extension<DB>,
     Extension(user_db): Extension<UserDB>,
     Path((w_id, name)): Path<(String, String)>,
 ) -> Result<String> {
     let mut tx = user_db.begin(&authed).await?;
 
+    require_is_owner(&name, &authed.username, &authed.groups, &w_id, &db).await?;
     not_found_if_none(get_group_opt(&mut tx, &w_id, &name).await?, "Group", &name)?;
 
     sqlx::query!(
@@ -222,12 +315,14 @@ async fn delete_group(
 
 async fn update_group(
     authed: Authed,
+    Extension(db): Extension<DB>,
     Extension(user_db): Extension<UserDB>,
     Path((w_id, name)): Path<(String, String)>,
     Json(eg): Json<EditGroup>,
 ) -> Result<String> {
     let mut tx = user_db.begin(&authed).await?;
 
+    require_is_owner(&name, &authed.username, &authed.groups, &w_id, &db).await?;
     not_found_if_none(get_group_opt(&mut tx, &w_id, &name).await?, "Group", &name)?;
 
     sqlx::query_as!(
@@ -256,17 +351,20 @@ async fn update_group(
 
 async fn add_user(
     authed: Authed,
+    Extension(db): Extension<DB>,
     Extension(user_db): Extension<UserDB>,
     Path((w_id, name)): Path<(String, String)>,
     Json(Username { username: user_username }): Json<Username>,
 ) -> Result<String> {
     let mut tx = user_db.begin(&authed).await?;
 
+    require_is_owner(&name, &authed.username, &authed.groups, &w_id, &db).await?;
+
     not_found_if_none(get_group_opt(&mut tx, &w_id, &name).await?, "Group", &name)?;
 
     sqlx::query_as!(
         Group,
-        "INSERT INTO usr_to_group (workspace_id, usr, group_) VALUES ($1, $2, $3)",
+        "INSERT INTO usr_to_group (workspace_id, usr, group_) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING",
         &w_id,
         user_username,
         name,
@@ -290,11 +388,13 @@ async fn add_user(
 
 async fn remove_user(
     authed: Authed,
+    Extension(db): Extension<DB>,
     Extension(user_db): Extension<UserDB>,
     Path((w_id, name)): Path<(String, String)>,
     Json(Username { username: user_username }): Json<Username>,
 ) -> Result<String> {
     let mut tx = user_db.begin(&authed).await?;
+    require_is_owner(&name, &authed.username, &authed.groups, &w_id, &db).await?;
 
     not_found_if_none(get_group_opt(&mut tx, &w_id, &name).await?, "Group", &name)?;
     if &name == "all" {

backend/windmill-api/src/jobs.rs

@@ -6,6 +6,8 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
+use std::sync::Arc;
+
 use anyhow::Context;
 use axum::{
     extract::{FromRequest, Path, Query},
@@ -14,10 +16,11 @@ use axum::{
     Extension, Json, Router,
 };
 use hmac::Mac;
-use hyper::StatusCode;
+use hyper::{HeaderMap, StatusCode};
 use serde::{de::DeserializeOwned, Deserialize, Serialize};
 use sql_builder::{prelude::*, quote, SqlBuilder};
 use sqlx::{query_scalar, types::Uuid, Postgres, Transaction};
+use urlencoding::encode;
 use windmill_audit::{audit_log, ActionKind};
 use windmill_common::{
     error::{self, to_anyhow, Error},
@@ -34,6 +37,7 @@ use crate::{
     db::{UserDB, DB},
     users::Authed,
     variables::get_workspace_key,
+    BaseUrl,
 };
 
 pub fn workspaced_service() -> Router {
@@ -64,6 +68,7 @@ pub fn workspaced_service() -> Router {
             "/job_signature/:job_id/:resume_id",
             get(create_job_signature),
         )
+        .route("/resume_urls/:job_id/:resume_id", get(get_resume_urls))
         .route("/result_by_id/:job_id/:node_id", get(get_result_by_id))
 }
 
@@ -128,10 +133,9 @@ async fn cancel_job_api(
         let (job_o, tx) = get_job_by_id(tx, &w_id, id).await?;
         tx.commit().await?;
         let err = match job_o {
-            Some(Job::CompletedJob(_)) => error::Error::BadRequest(format!(
-                "queued job id {} exists but is already completed and cannot be canceled",
-                id
-            )),
+            Some(Job::CompletedJob(_)) => {
+                return Ok(format!("queued job id {} is already completed", id))
+            }
             _ => error::Error::NotFound(format!("queued job id {} does not exist", id)),
         };
         Err(err)
@@ -234,18 +238,22 @@ pub struct CompletedJob {
     pub is_flow_step: bool,
     pub language: Option<ScriptLang>,
     pub is_skipped: bool,
+    pub email: String,
+    pub visible_to_owner: bool,
 }
 
-#[derive(Deserialize, Clone, Copy)]
+#[derive(Deserialize, Clone)]
 pub struct RunJobQuery {
     scheduled_for: Option<chrono::DateTime<chrono::Utc>>,
     scheduled_in_secs: Option<i64>,
     parent_job: Option<Uuid>,
+    include_header: Option<String>,
+    invisible_to_owner: Option<bool>,
 }
 
 impl RunJobQuery {
     async fn get_scheduled_for<'c>(
-        self,
+        &self,
         db: &mut Transaction<'c, Postgres>,
     ) -> error::Result<Option<chrono::DateTime<chrono::Utc>>> {
         if let Some(scheduled_for) = self.scheduled_for {
@@ -257,6 +265,27 @@ impl RunJobQuery {
             Ok(None)
         }
     }
+
+    fn add_include_headers(
+        &self,
+        headers: HeaderMap,
+        mut args: serde_json::Map<String, serde_json::Value>,
+    ) -> serde_json::Map<String, serde_json::Value> {
+        self.include_header
+            .as_ref()
+            .map(|s| s.split(",").map(|s| s.to_string()).collect::<Vec<_>>())
+            .unwrap_or_default()
+            .iter()
+            .for_each(|h| {
+                if let Some(v) = headers.get(h) {
+                    args.insert(
+                        h.to_string().to_lowercase().replace('-', "_"),
+                        serde_json::Value::String(v.to_str().unwrap().to_string()),
+                    );
+                }
+            });
+        args
+    }
 }
 
 #[derive(Deserialize)]
@@ -374,6 +403,8 @@ async fn list_jobs(
             "is_flow_step",
             "language",
             "false as is_skipped",
+            "email",
+            "visible_to_owner",
         ],
     );
     let sqlc = list_completed_jobs_query(
@@ -406,6 +437,8 @@ async fn list_jobs(
             "is_flow_step",
             "language",
             "is_skipped",
+            "email",
+            "visible_to_owner",
         ],
     );
     let sql = format!(
@@ -663,16 +696,74 @@ pub async fn create_job_signature(
     Query(approver): Query<QueryApprover>,
 ) -> error::Result<String> {
     let key = get_workspace_key(&w_id, &mut user_db.begin(&authed).await?).await?;
+    create_signature(key, job_id, resume_id, approver.approver)
+}
+
+fn create_signature(
+    key: String,
+    job_id: Uuid,
+    resume_id: u32,
+    approver: Option<String>,
+) -> Result<String, Error> {
     let mut mac = HmacSha256::new_from_slice(key.as_bytes()).map_err(to_anyhow)?;
     mac.update(job_id.as_bytes());
     mac.update(resume_id.to_be_bytes().as_ref());
-    tracing::info!("approver: {:?}", approver.approver);
-    if let Some(approver) = approver.approver {
+    if let Some(approver) = approver {
         mac.update(approver.as_bytes());
     }
     Ok(hex::encode(mac.finalize().into_bytes()))
 }
 
+#[allow(non_snake_case)]
+#[derive(Serialize)]
+pub struct ResumeUrls {
+    approvalPage: String,
+    cancel: String,
+    resume: String,
+}
+
+fn build_resume_url(
+    op: &str,
+    w_id: &str,
+    job_id: &Uuid,
+    resume_id: &u32,
+    signature: &str,
+    approver: &str,
+    base_url: &str,
+) -> String {
+    format!("{base_url}/api/w/{w_id}/jobs/{op}/{job_id}/{resume_id}/{signature}{approver}")
+}
+
+pub async fn get_resume_urls(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    Path((w_id, job_id, resume_id)): Path<(String, Uuid, u32)>,
+    Query(approver): Query<QueryApprover>,
+    Extension(base_url): Extension<Arc<BaseUrl>>,
+) -> error::JsonResult<ResumeUrls> {
+    let key = get_workspace_key(&w_id, &mut user_db.begin(&authed).await?).await?;
+    let signature = create_signature(key, job_id, resume_id, approver.approver.clone())?;
+    let base_url = base_url.0.clone();
+    let approver = approver
+        .approver
+        .as_ref()
+        .map(|x| format!("?approver={}", encode(x)))
+        .unwrap_or_else(String::new);
+    let res = ResumeUrls {
+        approvalPage: format!(
+            "{base_url}/approve/{w_id}/{job_id}/{resume_id}/{signature}{approver}"
+        ),
+        cancel: build_resume_url(
+            "cancel", &w_id, &job_id, &resume_id, &signature, &approver, &base_url,
+        ),
+        resume: build_resume_url(
+            "resume", &w_id, &job_id, &resume_id, &signature, &approver, &base_url,
+        ),
+    };
+
+    Ok(Json(res))
+}
+
 #[derive(Serialize, Debug)]
 #[serde(tag = "type")]
 pub enum Job {
@@ -723,6 +814,8 @@ struct UnifiedJob {
     is_flow_step: bool,
     language: Option<ScriptLang>,
     is_skipped: bool,
+    email: String,
+    visible_to_owner: bool,
 }
 
 impl From<UnifiedJob> for Job {
@@ -755,6 +848,8 @@ impl From<UnifiedJob> for Job {
                 is_flow_step: uj.is_flow_step,
                 language: uj.language,
                 is_skipped: uj.is_skipped,
+                email: uj.email,
+                visible_to_owner: uj.visible_to_owner,
             }),
             "QueuedJob" => Job::QueuedJob(QueuedJob {
                 workspace_id: uj.workspace_id,
@@ -783,6 +878,9 @@ impl From<UnifiedJob> for Job {
                 is_flow_step: uj.is_flow_step,
                 language: uj.language,
                 same_worker: false,
+                pre_run_error: None,
+                email: uj.email,
+                visible_to_owner: uj.visible_to_owner,
             }),
             t => panic!("job type {} not valid", t),
         }
@@ -859,22 +957,28 @@ pub async fn run_flow_by_path(
     Path((w_id, flow_path)): Path<(String, StripPath)>,
     axum::Json(args): axum::Json<Option<serde_json::Map<String, serde_json::Value>>>,
     Query(run_query): Query<RunJobQuery>,
+    headers: HeaderMap,
 ) -> error::Result<(StatusCode, String)> {
     let flow_path = flow_path.to_path();
     let mut tx = user_db.begin(&authed).await?;
     let scheduled_for = run_query.get_scheduled_for(&mut tx).await?;
+    let args = run_query.add_include_headers(headers, args.unwrap_or_default());
+
     let (uuid, tx) = push(
         tx,
         &w_id,
         JobPayload::Flow(flow_path.to_string()),
         args,
         &authed.username,
+        &authed.email,
         owner_to_token_owner(&authed.username, false),
         scheduled_for,
         None,
         run_query.parent_job,
         false,
         false,
+        None,
+        !run_query.invisible_to_owner.unwrap_or(false),
     )
     .await?;
     tx.commit().await?;
@@ -887,11 +991,13 @@ pub async fn run_job_by_path(
     Path((w_id, script_path)): Path<(String, StripPath)>,
     axum::Json(args): axum::Json<Option<serde_json::Map<String, serde_json::Value>>>,
     Query(run_query): Query<RunJobQuery>,
+    headers: HeaderMap,
 ) -> error::Result<(StatusCode, String)> {
     let script_path = script_path.to_path();
     let mut tx = user_db.begin(&authed).await?;
     let job_payload = script_path_to_payload(script_path, &mut tx, &w_id).await?;
     let scheduled_for = run_query.get_scheduled_for(&mut tx).await?;
+    let args = run_query.add_include_headers(headers, args.unwrap_or_default());
 
     let (uuid, tx) = push(
         tx,
@@ -899,12 +1005,15 @@ pub async fn run_job_by_path(
         job_payload,
         args,
         &authed.username,
+        &authed.email,
         owner_to_token_owner(&authed.username, false),
         scheduled_for,
         None,
         run_query.parent_job,
         false,
         false,
+        None,
+        !run_query.invisible_to_owner.unwrap_or(false),
     )
     .await?;
     tx.commit().await?;
@@ -949,24 +1058,30 @@ pub async fn run_wait_result_job_by_path(
     Path((w_id, script_path)): Path<(String, StripPath)>,
     axum::Json(args): axum::Json<Option<serde_json::Map<String, serde_json::Value>>>,
     Query(run_query): Query<RunJobQuery>,
+    headers: HeaderMap,
 ) -> error::JsonResult<serde_json::Value> {
     let script_path = script_path.to_path();
     let mut tx = user_db.clone().begin(&authed).await?;
     let job_payload = script_path_to_payload(script_path, &mut tx, &w_id).await?;
     let scheduled_for = run_query.get_scheduled_for(&mut tx).await?;
 
+    let args = run_query.add_include_headers(headers, args.unwrap_or_default());
+
     let (uuid, tx) = push(
         tx,
         &w_id,
         job_payload,
         args,
         &authed.username,
+        &authed.email,
         owner_to_token_owner(&authed.username, false),
         scheduled_for,
         None,
         run_query.parent_job,
         false,
         false,
+        None,
+        !run_query.invisible_to_owner.unwrap_or(false),
     )
     .await?;
     tx.commit().await?;
@@ -980,11 +1095,13 @@ pub async fn run_wait_result_job_by_hash(
     Path((w_id, script_hash)): Path<(String, ScriptHash)>,
     axum::Json(args): axum::Json<Option<serde_json::Map<String, serde_json::Value>>>,
     Query(run_query): Query<RunJobQuery>,
+    headers: HeaderMap,
 ) -> error::JsonResult<serde_json::Value> {
     let hash = script_hash.0;
     let mut tx = user_db.clone().begin(&authed).await?;
     let path = get_path_for_hash(&mut tx, &w_id, hash).await?;
     let scheduled_for = run_query.get_scheduled_for(&mut tx).await?;
+    let args = run_query.add_include_headers(headers, args.unwrap_or_default());
 
     let (uuid, tx) = push(
         tx,
@@ -992,12 +1109,15 @@ pub async fn run_wait_result_job_by_hash(
         JobPayload::ScriptHash { hash: ScriptHash(hash), path },
         args,
         &authed.username,
+        &authed.email,
         owner_to_token_owner(&authed.username, false),
         scheduled_for,
         None,
         run_query.parent_job,
         false,
         false,
+        None,
+        !run_query.invisible_to_owner.unwrap_or(false),
     )
     .await?;
     tx.commit().await?;
@@ -1025,10 +1145,12 @@ async fn run_preview_job(
     Extension(user_db): Extension<UserDB>,
     Path(w_id): Path<String>,
     Json(preview): Json<Preview>,
-    Query(sch_query): Query<RunJobQuery>,
+    Query(run_query): Query<RunJobQuery>,
+    headers: HeaderMap,
 ) -> error::Result<(StatusCode, String)> {
     let mut tx = user_db.begin(&authed).await?;
-    let scheduled_for = sch_query.get_scheduled_for(&mut tx).await?;
+    let scheduled_for = run_query.get_scheduled_for(&mut tx).await?;
+    let args = run_query.add_include_headers(headers, preview.args.unwrap_or_default());
 
     let (uuid, tx) = push(
         tx,
@@ -1039,14 +1161,17 @@ async fn run_preview_job(
             language: preview.language,
             lock: None,
         }),
-        preview.args,
+        args,
         &authed.username,
+        &authed.email,
         owner_to_token_owner(&authed.username, false),
         scheduled_for,
         None,
         None,
         false,
         false,
+        None,
+        true,
     )
     .await?;
     tx.commit().await?;
@@ -1058,22 +1183,28 @@ async fn run_preview_flow_job(
     Extension(user_db): Extension<UserDB>,
     Path(w_id): Path<String>,
     Json(raw_flow): Json<PreviewFlow>,
-    Query(sch_query): Query<RunJobQuery>,
+    Query(run_query): Query<RunJobQuery>,
+    headers: HeaderMap,
 ) -> error::Result<(StatusCode, String)> {
     let mut tx = user_db.begin(&authed).await?;
-    let scheduled_for = sch_query.get_scheduled_for(&mut tx).await?;
+    let scheduled_for = run_query.get_scheduled_for(&mut tx).await?;
+    let args = run_query.add_include_headers(headers, raw_flow.args.unwrap_or_default());
+
     let (uuid, tx) = push(
         tx,
         &w_id,
         JobPayload::RawFlow { value: raw_flow.value, path: raw_flow.path },
-        raw_flow.args,
+        args,
         &authed.username,
+        &authed.email,
         owner_to_token_owner(&authed.username, false),
         scheduled_for,
         None,
         None,
         false,
         false,
+        None,
+        true,
     )
     .await?;
     tx.commit().await?;
@@ -1086,11 +1217,13 @@ pub async fn run_job_by_hash(
     Path((w_id, script_hash)): Path<(String, ScriptHash)>,
     axum::Json(args): axum::Json<Option<serde_json::Map<String, serde_json::Value>>>,
     Query(run_query): Query<RunJobQuery>,
+    headers: HeaderMap,
 ) -> error::Result<(StatusCode, String)> {
     let hash = script_hash.0;
     let mut tx = user_db.begin(&authed).await?;
     let path = get_path_for_hash(&mut tx, &w_id, hash).await?;
     let scheduled_for = run_query.get_scheduled_for(&mut tx).await?;
+    let args = run_query.add_include_headers(headers, args.unwrap_or_default());
 
     let (uuid, tx) = push(
         tx,
@@ -1098,12 +1231,15 @@ pub async fn run_job_by_hash(
         JobPayload::ScriptHash { hash: ScriptHash(hash), path },
         args,
         &authed.username,
+        &authed.email,
         owner_to_token_owner(&authed.username, false),
         scheduled_for,
         None,
         run_query.parent_job,
         false,
         false,
+        None,
+        !run_query.invisible_to_owner.unwrap_or(false),
     )
     .await?;
     tx.commit().await?;
@@ -1275,6 +1411,8 @@ async fn list_completed_jobs(
             "is_flow_step",
             "language",
             "is_skipped",
+            "email",
+            "visible_to_owner",
         ],
     )
     .sql()?;

backend/windmill-api/src/lib.rs

@@ -27,7 +27,9 @@ mod apps;
 mod audit;
 mod capture;
 mod db;
+mod favorite;
 mod flows;
+mod folders;
 mod granular_acls;
 mod groups;
 pub mod jobs;
@@ -43,10 +45,12 @@ mod variables;
 mod worker_ping;
 mod workspaces;
 
-const GIT_VERSION: &str = git_version!(args = ["--tag", "--always"], fallback = "unknown-version");
+pub const GIT_VERSION: &str =
+    git_version!(args = ["--tag", "--always"], fallback = "unknown-version");
 
-struct BaseUrl(String);
+pub struct BaseUrl(String);
 pub struct IsSecure(bool);
+pub struct CookieDomain(Option<String>);
 pub struct CloudHosted(bool);
 pub struct ContentSecurityPolicy(String);
 
@@ -60,7 +64,10 @@ pub async fn run_server(
 ) -> anyhow::Result<()> {
     let user_db = UserDB::new(db.clone());
 
-    let auth_cache = Arc::new(users::AuthCache::new(db.clone()));
+    let auth_cache = Arc::new(users::AuthCache::new(
+        db.clone(),
+        std::env::var("SUPERADMIN_SECRET").ok(),
+    ));
     let argon2 = Arc::new(Argon2::default());
     let basic_clients = Arc::new(build_oauth_clients(&base_url).await?);
     let slack_verifier = Arc::new(
@@ -93,6 +100,9 @@ pub async fn run_server(
         .layer(Extension(Arc::new(ContentSecurityPolicy(
             std::env::var("SERVE_CSP").unwrap_or("".to_owned()),
         ))))
+        .layer(Extension(Arc::new(CookieDomain(
+            std::env::var("COOKIE_DOMAIN").ok(),
+        ))))
         .layer(Extension(http_client))
         .layer(CookieManagerLayer::new());
     // build our application with a route
@@ -119,7 +129,9 @@ pub async fn run_server(
                         .nest("/workspaces", workspaces::workspaced_service())
                         .nest("/flows", flows::workspaced_service())
                         .nest("/capture", capture::workspaced_service())
-                        .nest("/apps", apps::workspaced_service()),
+                        .nest("/apps", apps::workspaced_service())
+                        .nest("/favorites", favorite::workspaced_service())
+                        .nest("/folders", folders::workspaced_service()),
                 )
                 .nest("/workspaces", workspaces::global_service())
                 .nest(

backend/windmill-api/src/main.rs

@@ -14,7 +14,7 @@ use anyhow::Ok;
 async fn main() -> anyhow::Result<()> {
     windmill_common::tracing_init::initialize_tracing();
 
-    let db = windmill_common::connect_db().await?;
+    let db = windmill_common::connect_db(true).await?;
 
     let num_workers = std::env::var("NUM_WORKERS")
         .ok()

backend/windmill-api/src/oauth2.rs

@@ -10,6 +10,7 @@ use std::{collections::HashMap, fmt::Debug};
 
 use std::sync::Arc;
 
+use anyhow::Context;
 use axum::{
     async_trait,
     body::Bytes,
@@ -31,14 +32,15 @@ use tower_cookies::{Cookie, Cookies};
 use windmill_audit::{audit_log, ActionKind};
 use windmill_common::utils::{not_found_if_none, now_from_db};
 
-use crate::users::Authed;
-use crate::IsSecure;
+use crate::users::{truncate_token, Authed};
+use crate::workspaces::invite_user_to_all_auto_invite_worspaces;
 use crate::{
     db::{UserDB, DB},
     variables::{build_crypt, encrypt},
     workspaces::WorkspaceSettings,
     BaseUrl,
 };
+use crate::{CookieDomain, IsSecure};
 use windmill_common::error::{self, to_anyhow, Error};
 use windmill_common::oauth2::*;
 
@@ -75,7 +77,9 @@ pub struct ClientWithScopes {
     client: OClient,
     scopes: Vec<String>,
     extra_params: Option<HashMap<String, String>>,
+    extra_params_callback: Option<HashMap<String, String>>,
     allowed_domains: Option<Vec<String>>,
+    userinfo_url: Option<String>,
 }
 
 pub type BasicClientsMap = HashMap<String, ClientWithScopes>;
@@ -84,8 +88,11 @@ pub type BasicClientsMap = HashMap<String, ClientWithScopes>;
 pub struct OAuthConfig {
     auth_url: String,
     token_url: String,
+    userinfo_url: Option<String>,
     scopes: Option<Vec<String>>,
     extra_params: Option<HashMap<String, String>>,
+    extra_params_callback: Option<HashMap<String, String>>,
+    req_body_auth: Option<bool>,
 }
 
 #[derive(Clone, Debug, Serialize, Deserialize)]
@@ -93,6 +100,8 @@ pub struct OAuthClient {
     id: String,
     secret: String,
     allowed_domains: Option<Vec<String>>,
+    connect_config: Option<OAuthConfig>,
+    login_config: Option<OAuthConfig>,
 }
 pub struct AllClients {
     pub logins: BasicClientsMap,
@@ -132,20 +141,30 @@ pub async fn build_oauth_clients(base_url: &str) -> anyhow::Result<AllClients> {
 
     let logins = login_configs
         .into_iter()
-        .filter(|x| oauths.contains_key(&x.0))
-        .map(|(k, v)| {
-            let scopes = v.scopes.clone();
-            let extra_params = v.extra_params.clone();
-            let client_params = oauths.get(&k).unwrap().clone();
-            let named_client =
-                build_basic_client(k.clone(), v, client_params.clone(), true, base_url, None);
+        .filter_map(|x| oauths.get(&x.0).map(|c| (x.0, (c, x.1))))
+        .chain(oauths.iter().filter_map(|x| {
+            x.1.login_config
+                .as_ref()
+                .map(|c| (x.0.clone(), (x.1, c.clone())))
+        }))
+        .map(|(k, (client_params, config))| {
+            let named_client = build_basic_client(
+                k.clone(),
+                config.clone(),
+                client_params.clone(),
+                true,
+                base_url,
+                None,
+            );
             (
                 named_client.0,
                 ClientWithScopes {
                     client: named_client.1,
-                    scopes: scopes.unwrap_or(vec![]),
-                    extra_params,
-                    allowed_domains: client_params.allowed_domains,
+                    scopes: config.scopes.unwrap_or(vec![]),
+                    extra_params: config.extra_params,
+                    extra_params_callback: config.extra_params_callback,
+                    allowed_domains: client_params.allowed_domains.clone(),
+                    userinfo_url: config.userinfo_url,
                 },
             )
         })
@@ -153,15 +172,17 @@ pub async fn build_oauth_clients(base_url: &str) -> anyhow::Result<AllClients> {
 
     let connects = connect_configs
         .into_iter()
-        .filter(|x| oauths.contains_key(&x.0))
-        .map(|(k, v)| {
-            let scopes = v.scopes.clone();
-            let extra_params = v.extra_params.clone();
-
+        .filter_map(|x| oauths.get(&x.0).map(|c| (x.0, (c, x.1))))
+        .chain(oauths.iter().filter_map(|x| {
+            x.1.connect_config
+                .as_ref()
+                .map(|c| (x.0.clone(), (x.1, c.clone())))
+        }))
+        .map(|(k, (client_params, config))| {
             let named_client = build_basic_client(
                 k.clone(),
-                v,
-                oauths.get(&k).unwrap().clone(),
+                config.clone(),
+                client_params.clone(),
                 false,
                 base_url,
                 None,
@@ -170,9 +191,11 @@ pub async fn build_oauth_clients(base_url: &str) -> anyhow::Result<AllClients> {
                 named_client.0,
                 ClientWithScopes {
                     client: named_client.1,
-                    scopes: scopes.unwrap_or(vec![]),
-                    extra_params,
+                    scopes: config.scopes.unwrap_or(vec![]),
+                    extra_params: config.extra_params,
+                    extra_params_callback: config.extra_params_callback,
                     allowed_domains: None,
+                    userinfo_url: None,
                 },
             )
         })
@@ -184,8 +207,11 @@ pub async fn build_oauth_clients(base_url: &str) -> anyhow::Result<AllClients> {
             OAuthConfig {
                 auth_url: "https://slack.com/oauth/authorize".to_string(),
                 token_url: "https://slack.com/api/oauth.access".to_string(),
+                userinfo_url: None,
                 scopes: None,
                 extra_params: None,
+                extra_params_callback: None,
+                req_body_auth: None,
             },
             v.clone(),
             false,
@@ -218,6 +244,9 @@ pub fn build_basic_client(
     };
 
     let mut client = OClient::new(client_params.id, auth_url, token_url);
+    if config.req_body_auth.unwrap_or(false) {
+        client.set_auth_type(AuthType::RequestBody);
+    }
     client.set_client_secret(client_params.secret.clone());
     client.set_redirect_url(Url::parse(&redirect_url).expect("Invalid redirect URL"));
     // Set up the config for the Github OAuth2 process.
@@ -286,7 +315,7 @@ async fn connect(
 struct CreateAccount {
     client: String,
     owner: String,
-    refresh_token: String,
+    refresh_token: Option<String>,
     expires_in: i64,
 }
 async fn create_account(
@@ -491,18 +520,25 @@ pub async fn _refresh_token<'c>(
         .client)
         .to_owned();
 
-    let token_json = client
-        .exchange_refresh_token(&RefreshToken::from(account.refresh_token.clone()))
-        .with_client(&http_client)
-        .execute::<serde_json::Value>()
-        .await
-        .map_err(to_anyhow)?;
+    let token = _exchange_token(client, &account.refresh_token, http_client).await;
 
-    let token = serde_json::from_value::<TokenResponse>(token_json.clone()).map_err(|e| {
-        Error::BadConfig(format!(
-            "Error deserializing response as a new token: {e}\nresponse:{token_json}"
-        ))
-    })?;
+    if let Err(token_err) = token {
+        sqlx::query!(
+            "UPDATE account SET refresh_error = $1 WHERE workspace_id = $2 AND id = $3",
+            token_err.alt(),
+            w_id,
+            id,
+        )
+        .execute(&mut tx)
+        .await?;
+        tx.commit().await?;
+        return Err(error::Error::BadRequest(format!(
+            "Error refreshing token: {}",
+            token_err.alt()
+        )));
+    };
+
+    let token = token.unwrap();
 
     let expires_at = now_from_db(&mut tx).await?
         + chrono::Duration::seconds(
@@ -542,6 +578,25 @@ pub async fn _refresh_token<'c>(
     Ok(token_str)
 }
 
+async fn _exchange_token(
+    client: OClient,
+    refresh_token: &str,
+    http_client: Client,
+) -> Result<TokenResponse, Error> {
+    let token_json = client
+        .exchange_refresh_token(&RefreshToken::from(refresh_token.clone()))
+        .with_client(&http_client)
+        .execute::<serde_json::Value>()
+        .await
+        .map_err(to_anyhow)?;
+    let token = serde_json::from_value::<TokenResponse>(token_json.clone()).map_err(|e| {
+        Error::BadConfig(format!(
+            "Error deserializing response as a new token: {e}\nresponse:{token_json}"
+        ))
+    })?;
+    Ok(token)
+}
+
 #[derive(Deserialize)]
 pub struct OAuthCallback {
     code: String,
@@ -555,15 +610,16 @@ async fn connect_callback(
     Extension(clients): Extension<Arc<AllClients>>,
     Extension(http_client): Extension<Client>,
 ) -> error::JsonResult<TokenResponse> {
-    let client = (&clients
+    let client_w_scopes = &clients
         .connects
         .get(&client_name)
-        .ok_or_else(|| error::Error::BadRequest("invalid client".to_string()))?
-        .client)
-        .to_owned();
+        .ok_or_else(|| error::Error::BadRequest("invalid client".to_string()))?;
 
+    let client = client_w_scopes.client.to_owned();
+    let extra_params = client_w_scopes.extra_params_callback.clone();
     let token_response =
-        exchange_code::<TokenResponse>(callback, &cookies, client, &http_client).await?;
+        exchange_code::<TokenResponse>(callback, &cookies, client, &http_client, extra_params)
+            .await?;
 
     Ok(Json(token_response))
 }
@@ -583,18 +639,19 @@ async fn connect_slack_callback(
         .ok_or_else(|| error::Error::BadRequest("slack client not setup".to_string()))?
         .to_owned();
     let token =
-        exchange_code::<SlackTokenResponse>(callback, &cookies, client, &http_client).await?;
+        exchange_code::<SlackTokenResponse>(callback, &cookies, client, &http_client, None).await?;
 
     let mut tx = user_db.begin(&authed).await?;
 
     sqlx::query!(
         "INSERT INTO workspace_settings
-                (workspace_id, slack_team_id, slack_name)
-                VALUES ($1, $2, $3) ON CONFLICT (workspace_id) DO UPDATE SET slack_team_id = $2, \
-             slack_name = $3",
+                (workspace_id, slack_team_id, slack_name, slack_email)
+                VALUES ($1, $2, $3, $4) ON CONFLICT (workspace_id) DO UPDATE SET slack_team_id = $2, \
+             slack_name = $3, slack_email = $4",
         &w_id,
         token.team_id,
-        token.team_name
+        token.team_name,
+        authed.email
     )
     .execute(&mut tx)
     .await?;
@@ -610,6 +667,38 @@ async fn connect_slack_callback(
     )
     .execute(&mut tx)
     .await?;
+
+    let token_path = "g/slack/bot_token";
+    let mc = build_crypt(&mut tx, &w_id).await?;
+    let value = encrypt(&mc, &token.bot.bot_access_token);
+    sqlx::query!(
+        "INSERT INTO variable
+            (workspace_id, path, value, is_secret, description, account, is_oauth)
+            VALUES ($1, $2, $3, $4, $5, $6, $7)
+            ON CONFLICT (workspace_id, path) DO UPDATE SET value = $3",
+        &w_id,
+        token_path,
+        value,
+        true,
+        "The slack bot token to act on behalf of the installed app of the connected workspace",
+        None::<i32>,
+        true,
+    )
+    .execute(&mut tx)
+    .await?;
+
+    sqlx::query!(
+        "INSERT INTO resource
+            (workspace_id, path, value, description, resource_type)
+            VALUES ($1, $2, $3, $4, $5) ON CONFLICT (workspace_id, path) DO UPDATE SET value = $3",
+        w_id,
+        token_path,
+        serde_json::json!({ "token": format!("$var:{token_path}") }),
+        "The slack bot token to act on behalf of the installed app of the connected workspace",
+        "slack",
+    )
+    .execute(&mut tx)
+    .await?;
     tx.commit().await?;
     Ok("slack workspace connected".to_string())
 }
@@ -683,10 +772,19 @@ async fn slack_command(
     .await?;
 
     if let Some(settings) = settings {
-        if let Some(script) = &settings.slack_command_script {
-            let script_hash =
-                windmill_common::get_latest_hash_for_path(&mut tx, &settings.workspace_id, script)
-                    .await?;
+        if let Some(path) = &settings.slack_command_script {
+            let payload = if let Some(path) = path.strip_prefix("flow/") {
+                JobPayload::Flow(path.to_string())
+            } else {
+                let path = path.strip_prefix("script/").unwrap_or_else(|| path);
+                let script_hash = windmill_common::get_latest_hash_for_path(
+                    &mut tx,
+                    &settings.workspace_id,
+                    path,
+                )
+                .await?;
+                JobPayload::ScriptHash { hash: script_hash, path: path.to_owned() }
+            };
             let mut map = serde_json::Map::new();
             map.insert("text".to_string(), serde_json::Value::String(form.text));
             map.insert(
@@ -697,15 +795,18 @@ async fn slack_command(
             let (uuid, tx) = windmill_queue::push(
                 tx,
                 &settings.workspace_id,
-                JobPayload::ScriptHash { hash: script_hash, path: script.to_owned() },
-                Some(map),
+                payload,
+                map,
                 &form.user_name,
+                &settings.slack_email,
                 "g/slack".to_string(),
                 None,
                 None,
                 None,
                 false,
                 false,
+                None,
+                true,
             )
             .await?;
             tx.commit().await?;
@@ -723,8 +824,9 @@ async fn slack_command(
 }
 
 #[allow(non_snake_case)]
-#[derive(Deserialize)]
+#[derive(Deserialize, Debug)]
 pub struct UserInfo {
+    email: Option<String>,
     name: Option<String>,
     company: Option<String>,
     displayName: Option<String>,
@@ -738,18 +840,41 @@ async fn login_callback(
     Extension(db): Extension<DB>,
     Extension(http_client): Extension<Client>,
     Extension(is_secure): Extension<Arc<IsSecure>>,
+    Extension(cookie_domain): Extension<Arc<CookieDomain>>,
 ) -> error::Result<String> {
     let client_w_config = &clients
         .logins
         .get(&client_name)
         .ok_or_else(|| error::Error::BadRequest("invalid client".to_string()))?;
     let client = client_w_config.client.to_owned();
-    let token_res = exchange_code::<TokenResponse>(callback, &cookies, client, &http_client).await;
+    let token_res =
+        exchange_code::<TokenResponse>(callback, &cookies, client, &http_client, None).await;
 
     if let Ok(token) = token_res {
         let token = &token.access_token.to_string();
+        let userinfo_url = client_w_config.userinfo_url.as_ref().ok_or_else(|| {
+            Error::BadConfig(format!("Missing userinfo_url in client {client_name}"))
+        })?;
+        let user = http_get_user_info::<UserInfo>(&http_client, userinfo_url, token).await?;
 
-        let email = get_email(&http_client, &client_name, token).await?;
+        let email = match client_name.as_str() {
+            "github" => http_get_user_info::<Vec<GHEmailInfo>>(
+                &http_client,
+                "https://api.github.com/user/emails",
+                token,
+            )
+            .await?
+            .iter()
+            .find(|x| x.primary && x.verified)
+            .ok_or(error::Error::BadRequest(format!(
+                "user does not have any primary and verified address"
+            )))?
+            .email
+            .to_string(),
+            _ => user.email.ok_or_else(|| {
+                error::Error::BadRequest("email address not fetchable from user info".to_string())
+            })?,
+        };
 
         if let Some(domains) = &client_w_config.allowed_domains {
             if !domains.iter().any(|d| email.ends_with(d)) {
@@ -776,6 +901,7 @@ async fn login_callback(
                     &mut tx,
                     cookies,
                     is_secure.0,
+                    &cookie_domain.as_ref().0,
                 )
                 .await?;
             } else {
@@ -784,21 +910,43 @@ async fn login_callback(
                      login type {login_type}"
                 )));
             }
+            audit_log(
+                &mut tx,
+                &email,
+                "oauth.login",
+                ActionKind::Create,
+                "global",
+                Some(&truncate_token(&token)),
+                None,
+            )
+            .await?;
         } else {
-            let user = get_user_info(&http_client, &client_name, &token).await?;
-
+            let mut name = user.name;
+            if name.is_none() || name == Some(String::new()) {
+                name = user.displayName;
+            }
             sqlx::query(&format!(
                 "INSERT INTO password (email, name, company, login_type, verified) VALUES ($1, \
                  $2, $3, '{}', true)",
                 &client_name
             ))
             .bind(&email)
-            .bind(&user.name)
+            .bind(&name)
             .bind(user.company)
             .execute(&mut tx)
             .await?;
-            crate::users::create_session_token(&email, false, &mut tx, cookies, is_secure.0)
-                .await?;
+            tx.commit().await?;
+            invite_user_to_all_auto_invite_worspaces(&db, &email).await?;
+            tx = db.begin().await?;
+            crate::users::create_session_token(
+                &email,
+                false,
+                &mut tx,
+                cookies,
+                is_secure.0,
+                &cookie_domain.as_ref().0,
+            )
+            .await?;
             audit_log(
                 &mut tx,
                 &email,
@@ -817,8 +965,9 @@ async fn login_callback(
             if demo_exists {
                 if let Err(e) = sqlx::query!(
                     "INSERT INTO workspace_invite
-            (workspace_id, email, is_admin)
-            VALUES ('demo', $1, false)",
+                (workspace_id, email, is_admin)
+                VALUES ('demo', $1, false)
+                ON CONFLICT DO NOTHING",
                     &email
                 )
                 .execute(&mut tx)
@@ -843,6 +992,7 @@ async fn exchange_code<T: DeserializeOwned>(
     cookies: &Cookies,
     client: OClient,
     http_client: &Client,
+    extra_params: Option<HashMap<String, String>>,
 ) -> error::Result<T> {
     let csrf_state = cookies
         .get("csrf")
@@ -852,8 +1002,15 @@ async fn exchange_code<T: DeserializeOwned>(
         return Err(error::Error::BadRequest("csrf did not match".to_string()));
     }
 
-    client
-        .exchange_code(callback.code)
+    let mut token_url = client.exchange_code(callback.code);
+
+    if let Some(extra_params) = extra_params {
+        for (key, value) in extra_params {
+            token_url = token_url.param(key, value)
+        }
+    }
+
+    token_url
         .with_client(http_client)
         .execute::<T>()
         .await
@@ -867,95 +1024,39 @@ pub struct GHEmailInfo {
     primary: bool,
 }
 
-#[derive(Deserialize)]
-pub struct EmailInfo {
-    email: String,
-}
-
-async fn get_email(http_client: &Client, client_name: &str, token: &str) -> error::Result<String> {
-    let email = match client_name {
-        "github" => http_get_user_info::<Vec<GHEmailInfo>>(
-            http_client,
-            "https://api.github.com/user/emails",
-            token,
-        )
-        .await?
-        .iter()
-        .find(|x| x.primary && x.verified)
-        .ok_or(error::Error::BadRequest(format!(
-            "user does not have any primary and verified address"
-        )))?
-        .email
-        .to_string(),
-        "gitlab" => {
-            http_get_user_info::<EmailInfo>(http_client, "https://gitlab.com/api/v4/user", token)
-                .await?
-                .email
-                .to_string()
-        }
-        "google" => http_get_user_info::<EmailInfo>(
-            http_client,
-            "https://www.googleapis.com/oauth2/v1/userinfo?alt=json",
-            token,
-        )
-        .await?
-        .email
-        .to_string(),
-        _ => {
-            return Err(error::Error::BadRequest(
-                "client name not recognized".to_string(),
-            ))
-        }
-    };
-    Ok(email)
-}
-
-async fn get_user_info(
-    http_client: &Client,
-    client_name: &str,
-    token: &str,
-) -> error::Result<UserInfo> {
-    let email = match client_name {
-        "github" => http_get_user_info(http_client, "https://api.github.com/user", token).await?,
-        "gitlab" => {
-            http_get_user_info(http_client, "https://gitlab.com/api/v4/user", token).await?
-        }
-        "google" => {
-            let google_user_info: UserInfo = http_get_user_info(
-                http_client,
-                "https://www.googleapis.com/oauth2/v1/userinfo?alt=json",
-                token,
-            )
-            .await?;
-            UserInfo {
-                name: google_user_info.displayName.clone(),
-                company: None,
-                displayName: google_user_info.displayName,
-            }
-        }
-        _ => {
-            return Err(error::Error::BadRequest(
-                "client name not recognized".to_string(),
-            ))
-        }
-    };
-    Ok(email)
-}
-
 async fn http_get_user_info<T: DeserializeOwned>(
     http_client: &Client,
     url: &str,
     token: &str,
 ) -> error::Result<T> {
-    Ok(http_client
+    let res = http_client
         .get(url)
         .bearer_auth(token)
         .send()
         .await
-        .map_err(to_anyhow)?
+        .map_err(to_anyhow)
+        .context("failed to fetch user info")?;
+    if !res.status().is_success() {
+        tracing::debug!(
+            "The bearer token of the failed oauth user info exchange is: {}",
+            token
+        );
+        return Err(error::Error::BadConfig(format!(
+            "The user info endpoint responded with non 200: {}\n{}\n{}",
+            res.status(),
+            res.headers()
+                .iter()
+                .map(|x| format!("{}: {}", x.0.as_str(), x.1.to_str().unwrap_or_default()))
+                .collect::<Vec<_>>()
+                .join("\n"),
+            res.text().await.unwrap_or_default(),
+        )));
+    }
+    Ok(res
         .json::<T>()
         .await
-        .map_err(to_anyhow)?)
+        .map_err(to_anyhow)
+        .context("failed to decode json from user info")?)
 }
 
 fn oauth_redirect(

backend/windmill-api/src/resources.rs

@@ -8,7 +8,7 @@
 
 use crate::{
     db::{UserDB, DB},
-    users::Authed,
+    users::{require_owner_of_path, Authed},
 };
 use axum::{
     extract::{Extension, Path, Query},
@@ -18,7 +18,7 @@ use axum::{
 use hyper::StatusCode;
 use serde::{Deserialize, Serialize};
 use sql_builder::{bind::Bind, SqlBuilder};
-use sqlx::FromRow;
+use sqlx::{FromRow, Postgres, Transaction};
 use windmill_audit::{audit_log, ActionKind};
 use windmill_common::{
     error::{Error, JsonResult, Result},
@@ -72,7 +72,22 @@ pub struct Resource {
     pub description: Option<String>,
     pub resource_type: String,
     pub extra_perms: serde_json::Value,
-    pub is_oauth: bool,
+}
+
+#[derive(FromRow, Serialize, Deserialize)]
+pub struct ListableResource {
+    pub workspace_id: String,
+    pub path: String,
+    pub value: Option<serde_json::Value>,
+    pub description: Option<String>,
+    pub resource_type: String,
+    pub extra_perms: serde_json::Value,
+    pub is_linked: Option<bool>,
+    pub is_refreshed: Option<bool>,
+    pub is_oauth: Option<bool>,
+    pub is_expired: Option<bool>,
+    pub refresh_error: Option<String>,
+    pub account: Option<i32>,
 }
 
 #[derive(Deserialize)]
@@ -81,7 +96,6 @@ pub struct CreateResource {
     pub value: Option<serde_json::Value>,
     pub description: Option<String>,
     pub resource_type: String,
-    pub is_oauth: Option<bool>,
 }
 #[derive(Deserialize)]
 struct EditResource {
@@ -93,6 +107,7 @@ struct EditResource {
 #[derive(Deserialize)]
 pub struct ListResourceQuery {
     resource_type: Option<String>,
+    resource_type_exclude: Option<String>,
 }
 async fn list_resources(
     authed: Authed,
@@ -100,31 +115,50 @@ async fn list_resources(
     Query(pagination): Query<Pagination>,
     Extension(user_db): Extension<UserDB>,
     Path(w_id): Path<String>,
-) -> JsonResult<Vec<Resource>> {
+) -> JsonResult<Vec<ListableResource>> {
     let (per_page, offset) = paginate(pagination);
 
     let mut sqlb = SqlBuilder::select_from("resource")
         .fields(&[
-            "workspace_id",
-            "path",
+            "resource.workspace_id",
+            "resource.path",
             "null::JSONB as value",
-            "description",
+            "resource.description",
             "resource_type",
-            "extra_perms",
-            "is_oauth",
+            "resource.extra_perms",
+            "(now() > account.expires_at) as is_expired",
+            "variable.path IS NOT NULL as is_linked",
+            "account.refresh_token != '' as is_refreshed",
+            "variable.is_oauth",
+            "variable.account",
+            "account.refresh_error",
         ])
+        .left()
+        .join("variable")
+        .on("variable.path = resource.path AND variable.workspace_id = resource.workspace_id")
+        .left()
+        .join("account")
+        .on("variable.account = account.id AND account.workspace_id = variable.workspace_id")
         .order_by("path", true)
-        .and_where("workspace_id = ? OR workspace_id = 'starter'".bind(&w_id))
+        .and_where("resource.workspace_id = ? OR resource.workspace_id = 'starter'".bind(&w_id))
         .offset(offset)
         .limit(per_page)
         .clone();
+
     if let Some(rt) = &lq.resource_type {
-        sqlb.and_where_eq("resource_type", "?".bind(rt));
+        for rt in rt.split(',') {
+            sqlb.and_where_eq("resource_type", "?".bind(&rt));
+        }
+    }
+    if let Some(rt) = &lq.resource_type_exclude {
+        for rt in rt.split(',') {
+            sqlb.and_where_ne("resource_type", "?".bind(&rt));
+        }
     }
 
     let sql = sqlb.sql().map_err(|e| Error::InternalErr(e.to_string()))?;
     let mut tx = user_db.begin(&authed).await?;
-    let rows = sqlx::query_as::<_, Resource>(&sql)
+    let rows = sqlx::query_as::<_, ListableResource>(&sql)
         .fetch_all(&mut tx)
         .await?;
 
@@ -137,14 +171,21 @@ async fn get_resource(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
     Path((w_id, path)): Path<(String, StripPath)>,
-) -> JsonResult<Resource> {
+) -> JsonResult<ListableResource> {
     let path = path.to_path();
     let mut tx = user_db.begin(&authed).await?;
 
     let resource_o = sqlx::query_as!(
-        Resource,
-        "SELECT * from resource WHERE path = $1 AND (workspace_id = $2 OR workspace_id = \
-         'starter')",
+        ListableResource,
+        "SELECT resource.*, (now() > account.expires_at) as is_expired, account.refresh_token != '' as is_refreshed,
+        account.refresh_error,
+        variable.path IS NOT NULL as is_linked,
+        variable.is_oauth as \"is_oauth?\",
+        variable.account
+        FROM resource
+        LEFT JOIN variable ON variable.path = resource.path AND variable.workspace_id = resource.workspace_id
+        LEFT JOIN account ON variable.account = account.id AND account.workspace_id = resource.workspace_id
+        WHERE resource.path = $1 AND (resource.workspace_id = $2 OR resource.workspace_id = 'starter')",
         path.to_owned(),
         &w_id
     )
@@ -196,6 +237,28 @@ async fn get_resource_value(
     Ok(Json(value))
 }
 
+async fn check_path_conflict<'c>(
+    tx: &mut Transaction<'c, Postgres>,
+    w_id: &str,
+    path: &str,
+) -> Result<()> {
+    let exists = sqlx::query_scalar!(
+        "SELECT EXISTS(SELECT 1 FROM resource WHERE path = $1 AND workspace_id = $2)",
+        path,
+        w_id
+    )
+    .fetch_one(tx)
+    .await?
+    .unwrap_or(false);
+    if exists {
+        return Err(Error::BadRequest(format!(
+            "Resource {} already exists",
+            path
+        )));
+    }
+    return Ok(());
+}
+
 async fn create_resource(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
@@ -204,16 +267,16 @@ async fn create_resource(
 ) -> Result<(StatusCode, String)> {
     let mut tx = user_db.begin(&authed).await?;
 
+    check_path_conflict(&mut tx, &w_id, &resource.path).await?;
     sqlx::query!(
         "INSERT INTO resource
-            (workspace_id, path, value, description, resource_type, is_oauth)
-            VALUES ($1, $2, $3, $4, $5, $6)",
+            (workspace_id, path, value, description, resource_type)
+            VALUES ($1, $2, $3, $4, $5)",
         w_id,
         resource.path,
         resource.value,
         resource.description,
         resource.resource_type,
-        resource.is_oauth.unwrap_or(false)
     )
     .execute(&mut tx)
     .await?;
@@ -250,6 +313,13 @@ async fn delete_resource(
     )
     .execute(&mut tx)
     .await?;
+    sqlx::query!(
+        "DELETE FROM variable WHERE path = $1 AND workspace_id = $2",
+        path,
+        w_id
+    )
+    .execute(&mut tx)
+    .await?;
     audit_log(
         &mut tx,
         &authed.username,
@@ -268,6 +338,7 @@ async fn delete_resource(
 async fn update_resource(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
+    Extension(db): Extension<DB>,
     Path((w_id, path)): Path<(String, StripPath)>,
     Json(ns): Json<EditResource>,
 ) -> Result<String> {
@@ -298,6 +369,24 @@ async fn update_resource(
 
     let npath = not_found_if_none(npath_o, "Resource", path)?;
 
+    if let Some(npath) = ns.path {
+        if npath != path {
+            check_path_conflict(&mut tx, &w_id, &npath).await?;
+
+            if !authed.is_admin {
+                require_owner_of_path(&w_id, &authed.username, &authed.groups, &path, &db).await?;
+            }
+            sqlx::query!(
+                "UPDATE variable SET path = $1 WHERE path = $2 AND workspace_id = $3",
+                npath,
+                path,
+                w_id
+            )
+            .execute(&mut tx)
+            .await?;
+        }
+    }
+
     audit_log(
         &mut tx,
         &authed.username,
@@ -390,6 +479,8 @@ async fn create_resource_type(
 ) -> Result<(StatusCode, String)> {
     let mut tx = user_db.begin(&authed).await?;
 
+    check_rt_path_conflict(&mut tx, &w_id, &resource_type.name).await?;
+
     sqlx::query!(
         "INSERT INTO resource_type
             (workspace_id, name, schema, description)
@@ -419,6 +510,28 @@ async fn create_resource_type(
     ))
 }
 
+async fn check_rt_path_conflict<'c>(
+    tx: &mut Transaction<'c, Postgres>,
+    w_id: &str,
+    name: &str,
+) -> Result<()> {
+    let exists = sqlx::query_scalar!(
+        "SELECT EXISTS(SELECT 1 FROM resource_type WHERE name = $1 AND workspace_id = $2)",
+        name,
+        w_id
+    )
+    .fetch_one(tx)
+    .await?
+    .unwrap_or(false);
+    if exists {
+        return Err(Error::BadRequest(format!(
+            "Resource type {} already exists",
+            name
+        )));
+    }
+    return Ok(());
+}
+
 async fn delete_resource_type(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,

backend/windmill-api/src/schedule.rs

@@ -6,6 +6,8 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
+use std::str::FromStr;
+
 use crate::{
     db::{UserDB, DB},
     users::Authed,
@@ -15,15 +17,16 @@ use axum::{
     routing::{delete, get, post},
     Json, Router,
 };
-use chrono::DateTime;
+use chrono::{DateTime, FixedOffset};
+use serde::Deserialize;
+use sqlx::{Postgres, Transaction};
+use windmill_audit::{audit_log, ActionKind};
 use windmill_common::{
-    error::{JsonResult, Result},
-    utils::{not_found_if_none, Pagination, StripPath},
-};
-use windmill_queue::{
-    self,
-    schedule::{EditSchedule, NewSchedule, PreviewPayload, Schedule, SetEnabled},
+    error::{Error, JsonResult, Result},
+    schedule::Schedule,
+    utils::{not_found_if_none, paginate, Pagination, StripPath},
 };
+use windmill_queue::{self, schedule::push_scheduled_job, JobKind};
 
 pub fn workspaced_service() -> Router {
     Router::new()
@@ -40,15 +43,94 @@ pub fn global_service() -> Router {
     Router::new().route("/preview", post(preview_schedule))
 }
 
+#[derive(Deserialize)]
+pub struct NewSchedule {
+    pub path: String,
+    pub schedule: String,
+    pub offset: i32,
+    pub script_path: String,
+    pub is_flow: bool,
+    pub args: Option<serde_json::Value>,
+    pub enabled: Option<bool>,
+}
+
+async fn check_path_conflict<'c>(
+    tx: &mut Transaction<'c, Postgres>,
+    w_id: &str,
+    path: &str,
+) -> Result<()> {
+    let exists = sqlx::query_scalar!(
+        "SELECT EXISTS(SELECT 1 FROM schedule WHERE path = $1 AND workspace_id = $2)",
+        path,
+        w_id
+    )
+    .fetch_one(tx)
+    .await?
+    .unwrap_or(false);
+    if exists {
+        return Err(Error::BadRequest(format!(
+            "Schedule {} already exists",
+            path
+        )));
+    }
+    return Ok(());
+}
+
 async fn create_schedule(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
     Path(w_id): Path<String>,
     Json(ns): Json<NewSchedule>,
 ) -> Result<String> {
-    let tx = user_db.begin(&authed).await?;
-    let res = windmill_queue::schedule::create_schedule(tx, w_id, ns, &authed.username).await?;
-    Ok(res)
+    let mut tx = user_db.begin(&authed).await?;
+    cron::Schedule::from_str(&ns.schedule).map_err(|e| Error::BadRequest(e.to_string()))?;
+    check_path_conflict(&mut tx, &w_id, &ns.path).await?;
+    check_flow_conflict(&mut tx, &w_id, &ns.path, ns.is_flow, &ns.script_path).await?;
+
+    let schedule = sqlx::query_as!(
+        Schedule,
+        "INSERT INTO schedule (workspace_id, path, schedule, offset_, edited_by, script_path, \
+         is_flow, args, enabled, email) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) RETURNING *",
+        w_id,
+        ns.path,
+        ns.schedule,
+        ns.offset,
+        &authed.username,
+        ns.script_path,
+        ns.is_flow,
+        ns.args,
+        ns.enabled.unwrap_or(false),
+        &authed.email
+    )
+    .fetch_one(&mut tx)
+    .await
+    .map_err(|e| Error::InternalErr(format!("inserting schedule in {w_id}: {e}")))?;
+
+    audit_log(
+        &mut tx,
+        &authed.username,
+        "schedule.create",
+        ActionKind::Create,
+        &w_id,
+        Some(&ns.path.to_string()),
+        Some(
+            [
+                Some(("schedule", ns.schedule.as_str())),
+                Some(("script_path", ns.script_path.as_str())),
+            ]
+            .into_iter()
+            .flatten()
+            .collect(),
+        ),
+    )
+    .await?;
+
+    if ns.enabled.unwrap_or(true) {
+        tx = push_scheduled_job(tx, schedule).await?
+    }
+    tx.commit().await?;
+
+    Ok(ns.path.to_string())
 }
 
 async fn edit_schedule(
@@ -57,9 +139,55 @@ async fn edit_schedule(
     Path((w_id, path)): Path<(String, StripPath)>,
     Json(es): Json<EditSchedule>,
 ) -> Result<String> {
-    let tx = user_db.begin(&authed).await?;
-    let res = windmill_queue::schedule::edit_schedule(tx, w_id, path, es, &authed.username).await?;
-    Ok(res)
+    let mut tx = user_db.begin(&authed).await?;
+    let path = path.to_path();
+
+    cron::Schedule::from_str(&es.schedule).map_err(|e| Error::BadRequest(e.to_string()))?;
+
+    let is_flow = sqlx::query_scalar!(
+        "SELECT is_flow FROM schedule WHERE path = $1 AND workspace_id = $2",
+        path,
+        w_id
+    )
+    .fetch_one(&mut tx)
+    .await?;
+
+    clear_schedule(&mut tx, path, is_flow).await?;
+    let schedule = sqlx::query_as!(
+        Schedule,
+        "UPDATE schedule SET schedule = $1, args = $2 WHERE path \
+         = $3 AND workspace_id = $4 RETURNING *",
+        es.schedule,
+        es.args,
+        path,
+        w_id,
+    )
+    .fetch_one(&mut tx)
+    .await
+    .map_err(|e| Error::InternalErr(format!("updating schedule in {w_id}: {e}")))?;
+
+    if schedule.enabled {
+        tx = push_scheduled_job(tx, schedule).await?;
+    }
+
+    audit_log(
+        &mut tx,
+        &authed.username,
+        "schedule.edit",
+        ActionKind::Update,
+        &w_id,
+        Some(&path.to_string()),
+        Some(
+            [Some(("schedule", es.schedule.as_str()))]
+                .into_iter()
+                .flatten()
+                .collect(),
+        ),
+    )
+    .await?;
+    tx.commit().await?;
+
+    Ok(path.to_string())
 }
 
 async fn list_schedule(
@@ -68,9 +196,19 @@ async fn list_schedule(
     Path(w_id): Path<String>,
     Query(pagination): Query<Pagination>,
 ) -> JsonResult<Vec<Schedule>> {
-    let tx = user_db.begin(&authed).await?;
-    let res = windmill_queue::schedule::list_schedule(tx, w_id, pagination).await?;
-    Ok(Json(res))
+    let mut tx = user_db.begin(&authed).await?;
+    let (per_page, offset) = paginate(pagination);
+    let rows = sqlx::query_as!(
+        Schedule,
+        "SELECT * FROM schedule WHERE workspace_id = $1 ORDER BY edited_at desc LIMIT $2 OFFSET $3",
+        w_id,
+        per_page as i64,
+        offset as i64
+    )
+    .fetch_all(&mut tx)
+    .await?;
+    tx.commit().await?;
+    Ok(Json(rows))
 }
 
 async fn get_schedule(
@@ -100,7 +238,15 @@ async fn exists_schedule(
 pub async fn preview_schedule(
     Json(payload): Json<PreviewPayload>,
 ) -> JsonResult<Vec<DateTime<chrono::Utc>>> {
-    Ok(Json(windmill_queue::schedule::preview_schedule(payload)?))
+    let schedule = cron::Schedule::from_str(&payload.schedule)
+        .map_err(|e| Error::BadRequest(e.to_string()))?;
+    let upcoming: Vec<DateTime<chrono::Utc>> = schedule
+        .upcoming(get_offset(payload.offset))
+        .take(10)
+        .map(|x| x.into())
+        .collect();
+
+    Ok(Json(upcoming))
 }
 
 pub async fn set_enabled(
@@ -109,10 +255,41 @@ pub async fn set_enabled(
     Path((w_id, path)): Path<(String, StripPath)>,
     Json(payload): Json<SetEnabled>,
 ) -> Result<String> {
-    let tx = user_db.begin(&authed).await?;
-    let res =
-        windmill_queue::schedule::set_enabled(tx, w_id, path, payload, &authed.username).await?;
-    Ok(res)
+    let mut tx = user_db.begin(&authed).await?;
+    let path = path.to_path();
+    let schedule_o = sqlx::query_as!(
+        Schedule,
+        "UPDATE schedule SET enabled = $1, email = $2 WHERE path = $3 AND workspace_id = $4 RETURNING *",
+        &payload.enabled,
+        authed.email,
+        path,
+        w_id
+    )
+    .fetch_optional(&mut tx)
+    .await?;
+
+    let schedule = not_found_if_none(schedule_o, "Schedule", path)?;
+
+    clear_schedule(&mut tx, path, schedule.is_flow).await?;
+
+    if payload.enabled {
+        tx = push_scheduled_job(tx, schedule).await?;
+    }
+    audit_log(
+        &mut tx,
+        &authed.username,
+        "schedule.setenabled",
+        ActionKind::Update,
+        &w_id,
+        Some(path),
+        Some([("enabled", payload.enabled.to_string().as_ref())].into()),
+    )
+    .await?;
+    tx.commit().await?;
+    Ok(format!(
+        "succesfully updated schedule at path {} to status {}",
+        path, payload.enabled
+    ))
 }
 
 async fn delete_schedule(
@@ -120,7 +297,96 @@ async fn delete_schedule(
     Extension(user_db): Extension<UserDB>,
     Path((w_id, path)): Path<(String, StripPath)>,
 ) -> Result<String> {
-    let tx = user_db.begin(&authed).await?;
-    let res = windmill_queue::schedule::delete_schedule(tx, w_id, path, &authed.username).await?;
-    Ok(res)
+    let mut tx = user_db.begin(&authed).await?;
+    let path = path.to_path();
+
+    sqlx::query!(
+        "DELETE FROM schedule WHERE path = $1 AND workspace_id = $2",
+        path,
+        w_id
+    )
+    .execute(&mut tx)
+    .await?;
+
+    audit_log(
+        &mut tx,
+        &authed.username,
+        "schedule.delete",
+        ActionKind::Delete,
+        &w_id,
+        Some(path),
+        None,
+    )
+    .await?;
+
+    tx.commit().await?;
+
+    Ok(format!("schedule {} deleted", path))
+}
+
+async fn check_flow_conflict<'c>(
+    tx: &mut Transaction<'c, Postgres>,
+    w_id: &str,
+    path: &str,
+    is_flow: bool,
+    script_path: &str,
+) -> Result<()> {
+    if path != script_path || !is_flow {
+        let exists_flow = sqlx::query_scalar!(
+            "SELECT EXISTS (SELECT 1 FROM flow WHERE path = $1 AND workspace_id = $2)",
+            path,
+            w_id
+        )
+        .fetch_one(tx)
+        .await?
+        .unwrap_or(false);
+        if exists_flow {
+            return Err(Error::BadRequest(format!(
+                "The path is the same as a flow, it can only trigger that flow.
+            However the provided path is: {script_path} and is_flow is {is_flow}"
+            )));
+        };
+    }
+    Ok(())
+}
+
+#[derive(Deserialize)]
+pub struct EditSchedule {
+    pub schedule: String,
+    pub args: Option<serde_json::Value>,
+}
+
+pub async fn clear_schedule<'c>(
+    db: &mut Transaction<'c, Postgres>,
+    path: &str,
+    is_flow: bool,
+) -> Result<()> {
+    let job_kind = if is_flow {
+        JobKind::Flow
+    } else {
+        JobKind::Script
+    };
+    sqlx::query!(
+        "DELETE FROM queue WHERE schedule_path = $1 AND running = false AND job_kind = $2",
+        path,
+        job_kind: JobKind
+    )
+    .execute(db)
+    .await?;
+    Ok(())
+}
+
+#[derive(Deserialize)]
+pub struct PreviewPayload {
+    pub schedule: String,
+    pub offset: Option<i32>,
+}
+
+fn get_offset(offset: Option<i32>) -> FixedOffset {
+    FixedOffset::west(offset.unwrap_or(0) * 60)
+}
+
+#[derive(Deserialize)]
+pub struct SetEnabled {
+    pub enabled: bool,
 }

backend/windmill-api/src/scripts.rs

@@ -12,10 +12,11 @@ use windmill_audit::{audit_log, ActionKind};
 
 use crate::{
     db::{UserDB, DB},
-    users::{truncate_token, Authed, Tokened},
+    schedule::clear_schedule,
+    users::{require_owner_of_path, Authed},
 };
 use axum::{
-    extract::{Extension, Host, Path, Query},
+    extract::{Extension, Path, Query},
     routing::{get, post},
     Json, Router,
 };
@@ -30,15 +31,17 @@ use std::{
 };
 use windmill_common::{
     error::{Error, JsonResult, Result},
+    schedule::Schedule,
     scripts::{
-        to_i64, HubScript, ListScriptQuery, NewScript, Script, ScriptHash, ScriptKind, ScriptLang,
+        to_i64, HubScript, ListScriptQuery, ListableScript, NewScript, Script, ScriptHash,
+        ScriptKind, ScriptLang,
     },
     users::owner_to_token_owner,
     utils::{
         list_elems_from_hub, not_found_if_none, paginate, require_admin, Pagination, StripPath,
     },
 };
-use windmill_queue;
+use windmill_queue::{self, schedule::push_scheduled_job};
 
 const MAX_HASH_HISTORY_LENGTH_STORED: usize = 20;
 
@@ -76,32 +79,37 @@ async fn list_scripts(
     Path(w_id): Path<String>,
     Query(pagination): Query<Pagination>,
     Query(lq): Query<ListScriptQuery>,
-) -> JsonResult<Vec<Script>> {
+) -> JsonResult<Vec<ListableScript>> {
     let (per_page, offset) = paginate(pagination);
 
     let mut sqlb = SqlBuilder::select_from("script as o")
         .fields(&[
-            "workspace_id",
+            "o.workspace_id",
             "hash",
-            "path",
+            "o.path",
             "array_remove(array[parent_hashes[1]], NULL) as parent_hashes",
             "summary",
             "description",
-            "'' as content",
             "created_by",
             "created_at",
             "archived",
-            "null as schema",
             "deleted",
             "is_template",
             "extra_perms",
-            "null as lock",
             "CASE WHEN lock_error_logs IS NOT NULL THEN 'error' ELSE null END as lock_error_logs",
             "language",
             "kind",
+            "favorite.path IS NOT NULL as starred",
         ])
+        .left()
+        .join("favorite")
+        .on(
+            "favorite.favorite_kind = 'script' AND favorite.workspace_id = o.workspace_id AND favorite.path = o.path AND favorite.usr = ?"
+                .bind(&authed.username),
+        )
+        .order_desc("favorite.path IS NOT NULL")
         .order_by("created_at", lq.order_desc.unwrap_or(true))
-        .and_where("workspace_id = ? OR workspace_id = 'starter'".bind(&w_id))
+        .and_where("o.workspace_id = ? OR o.workspace_id = 'starter'".bind(&w_id))
         .offset(offset)
         .limit(per_page)
         .clone();
@@ -110,7 +118,8 @@ async fn list_scripts(
         sqlb.and_where_eq(
             "created_at",
             "(select max(created_at) from script where o.path = path 
-            AND (workspace_id = $1 OR workspace_id = 'starter'))",
+            AND (workspace_id = ? OR workspace_id = 'starter'))"
+                .bind(&w_id),
         );
     } else {
         sqlb.and_where_eq("archived", false);
@@ -139,25 +148,27 @@ async fn list_scripts(
     if let Some(k) = &lq.kind {
         sqlb.and_where_eq("kind", "?".bind(&k.to_lowercase()));
     }
+    if lq.starred_only.unwrap_or(false) {
+        sqlb.and_where_is_not_null("favorite.path");
+    }
 
     let sql = sqlb.sql().map_err(|e| Error::InternalErr(e.to_string()))?;
     let mut tx = user_db.begin(&authed).await?;
-    let rows = sqlx::query_as::<_, Script>(&sql).fetch_all(&mut tx).await?;
+    let rows = sqlx::query_as::<_, ListableScript>(&sql)
+        .fetch_all(&mut tx)
+        .await?;
     tx.commit().await?;
     Ok(Json(rows))
 }
 
 async fn list_hub_scripts(
-    Authed { email, username, .. }: Authed,
+    Authed { email, .. }: Authed,
     Extension(http_client): Extension<Client>,
-    Host(host): Host,
 ) -> JsonResult<serde_json::Value> {
     let asks = list_elems_from_hub(
         http_client,
         "https://hub.windmill.dev/searchData?approved=true",
-        email,
-        username,
-        host,
+        &email,
     )
     .await?;
     Ok(Json(asks))
@@ -168,10 +179,11 @@ fn hash_script(ns: &NewScript) -> i64 {
     ns.hash(&mut dh);
     dh.finish() as i64
 }
+
 async fn create_script(
     authed: Authed,
-    Tokened { token }: Tokened,
     Extension(user_db): Extension<UserDB>,
+    Extension(db): Extension<DB>,
     Path(w_id): Path<String>,
     Json(ns): Json<NewScript>,
 ) -> Result<(StatusCode, String)> {
@@ -202,87 +214,107 @@ async fn create_script(
     .fetch_optional(&mut tx)
     .await?;
 
-    let parent_hashes_and_perms: Option<(Vec<i64>, serde_json::Value)> =
-        match (&ns.parent_hash, clashing_script) {
-            (None, None) => Ok(None),
-            (None, Some(s)) => Err(Error::BadRequest(format!(
-                "Path conflict for {} with non-archived hash {}",
-                &ns.path, &s.hash
-            ))),
-            (Some(p_hash), o) => {
-                if sqlx::query_scalar!(
-                    "SELECT 1 FROM script WHERE hash = $1 AND workspace_id = $2",
-                    p_hash.0,
-                    &w_id
-                )
-                .fetch_optional(&mut tx)
-                .await?
-                .is_none()
-                {
-                    return Err(Error::BadRequest(
-                        "The parent hash does not seem to exist".to_owned(),
-                    ));
-                };
+    struct ParentInfo {
+        p_hashes: Vec<i64>,
+        perms: serde_json::Value,
+        p_path: String,
+    }
+    let parent_hashes_and_perms: Option<ParentInfo> = match (&ns.parent_hash, clashing_script) {
+        (None, None) => Ok(None),
+        (None, Some(s)) => Err(Error::BadRequest(format!(
+            "Path conflict for {} with non-archived hash {}",
+            &ns.path, &s.hash
+        ))),
+        (Some(p_hash), o) => {
+            if sqlx::query_scalar!(
+                "SELECT 1 FROM script WHERE hash = $1 AND workspace_id = $2",
+                p_hash.0,
+                &w_id
+            )
+            .fetch_optional(&mut tx)
+            .await?
+            .is_none()
+            {
+                return Err(Error::BadRequest(
+                    "The parent hash does not seem to exist".to_owned(),
+                ));
+            };
 
-                let clashing_hash_o = sqlx::query_scalar!(
-                    "SELECT hash FROM script WHERE parent_hashes[1] = $1 AND workspace_id = $2",
-                    p_hash.0,
-                    &w_id
-                )
-                .fetch_optional(&mut tx)
-                .await?;
+            let clashing_hash_o = sqlx::query_scalar!(
+                "SELECT hash FROM script WHERE parent_hashes[1] = $1 AND workspace_id = $2",
+                p_hash.0,
+                &w_id
+            )
+            .fetch_optional(&mut tx)
+            .await?;
 
-                if let Some(clashing_hash) = clashing_hash_o {
-                    return Err(Error::BadRequest(format!(
-                        "A script with hash {} with same parent_hash has been found. However, the \
+            if let Some(clashing_hash) = clashing_hash_o {
+                return Err(Error::BadRequest(format!(
+                    "A script with hash {} with same parent_hash has been found. However, the \
                          lineage must be linear: no 2 scripts can have the same parent",
-                        ScriptHash(clashing_hash)
-                    )));
-                };
+                    ScriptHash(clashing_hash)
+                )));
+            };
 
-                let ps = get_script_by_hash_internal(&mut tx, &w_id, p_hash).await?;
+            let ps = get_script_by_hash_internal(&mut tx, &w_id, p_hash).await?;
 
-                let ph = {
-                    let v = ps.parent_hashes.map(|x| x.0).unwrap_or_default();
-                    let mut v: Vec<i64> = v
-                        .into_iter()
-                        .take(MAX_HASH_HISTORY_LENGTH_STORED - 1)
-                        .collect();
-                    v.insert(0, p_hash.0);
-                    v
-                };
-                let r: Result<Option<(Vec<i64>, serde_json::Value)>> = match o {
-                    Some(clashing_script)
-                        if clashing_script.path == ns.path
-                            && clashing_script.hash.0 != p_hash.0 =>
-                    {
-                        Err(Error::BadRequest(format!(
-                            "Path conflict for {} with non-archived hash {}",
-                            &ns.path, &clashing_script.hash
-                        )))
-                    }
-                    Some(_) => Ok(Some((ph, ps.extra_perms))),
-                    None => Ok(Some((ph, ps.extra_perms))),
-                };
-                sqlx::query!(
-                    "UPDATE script SET archived = true WHERE hash = $1 AND workspace_id = $2",
-                    p_hash.0,
-                    &w_id
-                )
-                .execute(&mut tx)
-                .await?;
-                r
+            if ps.path != ns.path {
+                if !authed.is_admin {
+                    require_owner_of_path(&w_id, &authed.username, &authed.groups, &ps.path, &db)
+                        .await?;
+                }
             }
-        }?;
 
-    let p_hashes = parent_hashes_and_perms.as_ref().map(|v| &v.0[..]);
+            let ph = {
+                let v = ps.parent_hashes.map(|x| x.0).unwrap_or_default();
+                let mut v: Vec<i64> = v
+                    .into_iter()
+                    .take(MAX_HASH_HISTORY_LENGTH_STORED - 1)
+                    .collect();
+                v.insert(0, p_hash.0);
+                v
+            };
+            let r: Result<Option<ParentInfo>> = match o {
+                Some(clashing_script)
+                    if clashing_script.path == ns.path && clashing_script.hash.0 != p_hash.0 =>
+                {
+                    Err(Error::BadRequest(format!(
+                        "Path conflict for {} with non-archived hash {}",
+                        &ns.path, &clashing_script.hash
+                    )))
+                }
+                Some(_) | None => Ok(Some(ParentInfo {
+                    p_hashes: ph,
+                    perms: ps.extra_perms,
+                    p_path: ps.path,
+                })),
+            };
+            sqlx::query!(
+                "UPDATE script SET archived = true WHERE hash = $1 AND workspace_id = $2",
+                p_hash.0,
+                &w_id
+            )
+            .execute(&mut tx)
+            .await?;
+            r
+        }
+    }?;
+
+    let p_hashes = parent_hashes_and_perms.as_ref().map(|v| &v.p_hashes[..]);
     let extra_perms = parent_hashes_and_perms
         .as_ref()
-        .map(|v| v.1.clone())
+        .map(|v| v.perms.clone())
         .unwrap_or(json!({}));
 
-    let lock = ns.lock.as_ref().map(|x| x.join("\n"));
-    let lock = lock.and_then(|e| if e.is_empty() { None } else { Some(e) });
+    let lock = if ns.language == ScriptLang::Bash || ns.language == ScriptLang::Deno {
+        Some(String::new())
+    } else {
+        ns.lock
+            .as_ref()
+            .map(|x| x.join("\n"))
+            .and_then(|e| if e.is_empty() { None } else { Some(e) })
+    };
+
     let needs_lock_gen = lock.is_none();
     //::text::json is to ensure we use serde_json with preserve order
     sqlx::query!(
@@ -307,6 +339,26 @@ async fn create_script(
     .execute(&mut tx)
     .await?;
 
+    if let Some(p_path) = parent_hashes_and_perms.as_ref().map(|x| x.p_path.clone()) {
+        let schedulables = sqlx::query_as!(
+        Schedule,
+            "UPDATE schedule SET script_path = $1 WHERE script_path = $2 AND workspace_id = $3 AND is_flow IS false RETURNING *",
+            ns.path,
+            p_path,
+            w_id,
+        )
+        .fetch_all(&mut tx)
+        .await?;
+
+        for schedule in schedulables {
+            clear_schedule(&mut tx, &schedule.path, false).await?;
+
+            if schedule.enabled {
+                tx = push_scheduled_job(tx, schedule).await?;
+            }
+        }
+    }
+
     let mut tx = if needs_lock_gen {
         let dependencies = match ns.language {
             ScriptLang::Python3 => {
@@ -318,14 +370,17 @@ async fn create_script(
             tx,
             &w_id,
             windmill_queue::JobPayload::Dependencies { hash, dependencies, language: ns.language },
-            None,
+            serde_json::Map::new(),
             &authed.username,
+            &authed.email,
             owner_to_token_owner(&authed.username, false),
             None,
             None,
             None,
             false,
             false,
+            None,
+            true,
         )
         .await?;
         tx
@@ -341,13 +396,7 @@ async fn create_script(
             ActionKind::Update,
             &w_id,
             Some(&ns.path),
-            Some(
-                [
-                    ("hash", hash.to_string().as_str()),
-                    ("token", &truncate_token(&token)),
-                ]
-                .into(),
-            ),
+            Some([("hash", hash.to_string().as_str())].into()),
         )
         .await?;
     } else {
@@ -362,7 +411,6 @@ async fn create_script(
                 [
                     ("workspace", w_id.as_str()),
                     ("hash", hash.to_string().as_str()),
-                    ("token", &truncate_token(&token)),
                 ]
                 .into(),
             ),
@@ -379,33 +427,17 @@ pub async fn get_hub_script_by_path(
     authed: Authed,
     Path(path): Path<StripPath>,
     Extension(http_client): Extension<Client>,
-    Host(host): Host,
 ) -> Result<String> {
-    windmill_common::scripts::get_hub_script_by_path(
-        authed.email,
-        authed.username,
-        path,
-        http_client,
-        host,
-    )
-    .await
+    windmill_common::scripts::get_hub_script_by_path(&authed.email, path, http_client).await
 }
 
 pub async fn get_full_hub_script_by_path(
-    Authed { username, email, .. }: Authed,
+    Authed { email, .. }: Authed,
     Path(path): Path<StripPath>,
     Extension(http_client): Extension<Client>,
-    Host(host): Host,
 ) -> JsonResult<HubScript> {
     Ok(Json(
-        windmill_common::scripts::get_full_hub_script_by_path(
-            email,
-            username,
-            path,
-            http_client,
-            host,
-        )
-        .await?,
+        windmill_common::scripts::get_full_hub_script_by_path(&email, path, http_client).await?,
     ))
 }
 

backend/windmill-api/src/users.rs

@@ -10,32 +10,36 @@ use std::{sync::Arc, time::Duration};
 
 use crate::{
     db::{UserDB, DB},
+    folders::get_folders_for_user,
     utils::require_super_admin,
-    IsSecure,
+    workspaces::invite_user_to_all_auto_invite_worspaces,
+    CookieDomain, IsSecure,
 };
 use argon2::{password_hash::SaltString, Argon2, PasswordHash, PasswordHasher, PasswordVerifier};
 use axum::{
     async_trait,
     extract::{Extension, FromRequest, Path, Query, RequestParts},
     http,
+    response::{IntoResponse, Response},
     routing::{delete, get, post},
     Json, Router,
 };
-use hyper::StatusCode;
+use hyper::{header::LOCATION, StatusCode};
 use rand::rngs::OsRng;
 use retainer::Cache;
 use serde::{Deserialize, Serialize};
 use sqlx::FromRow;
 use time::OffsetDateTime;
 use tower_cookies::{Cookie, Cookies};
-use tracing::Span;
+use tracing::{Instrument, Span};
 use windmill_audit::{audit_log, ActionKind};
 use windmill_common::{
     error::{self, Error, JsonResult, Result},
-    utils::{not_found_if_none, rd_string, require_admin, Pagination},
+    utils::{not_found_if_none, rd_string, require_admin, Pagination, StripPath},
 };
+use windmill_queue::CLOUD_HOSTED;
 
-const TTL_TOKEN_CACHE_S: u64 = 60 * 5; // 5 minutes
+const TTL_TOKEN_CACHE_S: u64 = 60; // 60s
 pub const TTL_TOKEN_DB_H: u32 = 72;
 
 const COOKIE_NAME: &str = "token";
@@ -48,6 +52,7 @@ pub fn workspaced_service() -> Router {
         .route("/exists", post(exists_username))
         .route("/update/:user", post(update_workspace_user))
         .route("/delete/:user", delete(delete_workspace_user))
+        .route("/is_owner/:path", get(is_owner_of_path))
         .route("/whois/:email", get(whois))
         .route("/whoami", get(whoami))
         .route("/leave", post(leave_workspace))
@@ -68,6 +73,7 @@ pub fn global_service() -> Router {
         .route("/tokens/create", post(create_token))
         .route("/tokens/delete/:token_prefix", delete(delete_token))
         .route("/tokens/list", get(list_tokens))
+        .route("/usage", get(get_usage))
     // .route("/list_invite_codes", get(list_invite_codes))
     // .route("/create_invite_code", post(create_invite_code))
     // .route("/signup", post(signup))
@@ -79,16 +85,18 @@ pub fn make_unauthed_service() -> Router {
     Router::new()
         .route("/login", post(login))
         .route("/logout", post(logout))
+        .route("/logout", get(logout))
 }
 
 pub struct AuthCache {
     cache: Cache<(String, String), Authed>,
     db: DB,
+    superadmin_secret: Option<String>,
 }
 
 impl AuthCache {
-    pub fn new(db: DB) -> Self {
-        AuthCache { cache: Cache::new(), db }
+    pub fn new(db: DB, superadmin_secret: Option<String>) -> Self {
+        AuthCache { cache: Cache::new(), db, superadmin_secret }
     }
 
     pub async fn get_authed(&self, w_id: Option<String>, token: &str) -> Option<Authed> {
@@ -113,11 +121,70 @@ impl AuthCache {
                 if let Some(user) = user_o {
                     let authed_o = {
                         match user {
+                            (Some(owner), email, super_admin) if w_id.is_some() => {
+                                if let Some((prefix, name)) = owner.split_once('/') {
+                                    if prefix == "u" {
+                                        let is_admin = super_admin
+                                            || sqlx::query_scalar!(
+                                                "SELECT is_admin FROM usr where username = $1 AND \
+                                                 workspace_id = $2 AND disabled = false",
+                                                name,
+                                                &w_id.as_ref().unwrap()
+                                            )
+                                            .fetch_one(&self.db)
+                                            .await
+                                            .ok()
+                                            .unwrap_or(false);
+
+                                        let w_id = &w_id.unwrap();
+                                        let groups = get_groups_for_user(w_id, &name, &self.db)
+                                            .await
+                                            .ok()
+                                            .unwrap_or_default();
+
+                                        let folders =
+                                            get_folders_for_user(w_id, &name, &groups, &self.db)
+                                                .await
+                                                .ok()
+                                                .unwrap_or_default();
+
+                                        Some(Authed {
+                                            email: email
+                                                .unwrap_or_else(|| "missing@email.xyz".to_string()),
+                                            username: name.to_string(),
+                                            is_admin,
+                                            groups,
+                                            folders,
+                                        })
+                                    } else {
+                                        let groups = vec![name.to_string()];
+                                        let folders = get_folders_for_user(
+                                            &w_id.unwrap(),
+                                            "",
+                                            &groups,
+                                            &self.db,
+                                        )
+                                        .await
+                                        .ok()
+                                        .unwrap_or_default();
+                                        Some(Authed {
+                                            email: email
+                                                .unwrap_or_else(|| "missing@email.xyz".to_string()),
+                                            username: format!("group-{name}"),
+                                            is_admin: false,
+                                            groups,
+                                            folders,
+                                        })
+                                    }
+                                } else {
+                                    None
+                                }
+                            }
                             (_, Some(email), super_admin) => {
                                 if w_id.is_some() {
                                     let row_o = sqlx::query_as::<_, (String, bool)>(
                                         "SELECT username, is_admin FROM usr where email = $1 AND \
-                                         workspace_id = $2",
+                                         workspace_id = $2 AND disabled = false",
                                     )
                                     .bind(&email)
                                     .bind(&w_id.as_ref().unwrap())
@@ -136,71 +203,44 @@ impl AuthCache {
                                             .ok()
                                             .unwrap_or_default();
 
+                                            let folders = get_folders_for_user(
+                                                &w_id.unwrap(),
+                                                &username,
+                                                &groups,
+                                                &self.db,
+                                            )
+                                            .await
+                                            .ok()
+                                            .unwrap_or_default();
                                             Some(Authed {
-                                                email: Some(email),
+                                                email,
                                                 username,
                                                 is_admin: is_admin || super_admin,
                                                 groups,
+                                                folders,
                                             })
                                         }
                                         None if super_admin || w_id.unwrap() == "starter" => {
                                             Some(Authed {
-                                                email: Some(email.to_string()),
+                                                email: email.clone(),
                                                 username: email,
                                                 is_admin: super_admin,
                                                 groups: vec![],
+                                                folders: vec![],
                                             })
                                         }
                                         None => None,
                                     }
                                 } else {
                                     Some(Authed {
-                                        email: Some(email.to_string()),
+                                        email: email.to_string(),
                                         username: email,
                                         is_admin: super_admin,
                                         groups: Vec::new(),
+                                        folders: Vec::new(),
                                     })
                                 }
                             }
-                            (Some(owner), _, super_admin) if w_id.is_some() => {
-                                if let Some((prefix, name)) = owner.split_once('/') {
-                                    if prefix == "u" {
-                                        let is_admin = super_admin
-                                            || sqlx::query_scalar!(
-                                                "SELECT is_admin FROM usr where username = $1 AND \
-                                                 workspace_id = $2",
-                                                name,
-                                                &w_id.as_ref().unwrap()
-                                            )
-                                            .fetch_one(&self.db)
-                                            .await
-                                            .ok()
-                                            .unwrap_or(false);
-
-                                        let groups =
-                                            get_groups_for_user(&w_id.unwrap(), &name, &self.db)
-                                                .await
-                                                .ok()
-                                                .unwrap_or_default();
-
-                                        Some(Authed {
-                                            email: None,
-                                            username: name.to_string(),
-                                            is_admin,
-                                            groups,
-                                        })
-                                    } else {
-                                        Some(Authed {
-                                            email: None,
-                                            username: format!("group-{name}"),
-                                            is_admin: false,
-                                            groups: vec![name.to_string()],
-                                        })
-                                    }
-                                } else {
-                                    None
-                                }
-                            }
                             _ => None,
                         }
                     };
@@ -210,6 +250,19 @@ impl AuthCache {
                             .await;
                     }
                     authed_o
+                } else if self
+                    .superadmin_secret
+                    .as_ref()
+                    .map(|x| x == token)
+                    .unwrap_or(false)
+                {
+                    Some(Authed {
+                        email: "superadmin_secret@windmill.dev".to_string(),
+                        username: "superadmin_secret".to_string(),
+                        is_admin: true,
+                        groups: Vec::new(),
+                        folders: Vec::new(),
+                    })
                 } else {
                     None
                 }
@@ -281,10 +334,11 @@ where
 
 #[derive(Clone, Debug)]
 pub struct Authed {
-    pub email: Option<String>,
+    pub email: String,
     pub username: String,
     pub is_admin: bool,
     pub groups: Vec<String>,
+    pub folders: Vec<(String, bool)>,
 }
 
 #[async_trait]
@@ -316,9 +370,8 @@ where
                     if let Some(authed) = cache.get_authed(workspace_id.clone(), &token).await {
                         req.extensions_mut().insert(authed.clone());
                         Span::current().record("username", &authed.username.as_str());
-                        if let Some(email) = authed.email.clone() {
-                            Span::current().record("email", &email.as_str());
-                        }
+                        Span::current().record("email", &authed.email);
+
                         if let Some(workspace_id) = workspace_id {
                             Span::current().record("workspace_id", &workspace_id);
                         }
@@ -363,9 +416,7 @@ pub struct User {
 
 #[derive(FromRow, Serialize)]
 pub struct Usage {
-    pub duration_ms: i64,
-    pub jobs: i64,
-    pub flows: i64,
+    pub executions: i64,
 }
 
 #[derive(Serialize)]
@@ -375,7 +426,7 @@ pub struct UserWithUsage {
     pub usage: Usage,
 }
 
-#[derive(FromRow, Serialize)]
+#[derive(FromRow, Serialize, Debug)]
 pub struct GlobalUserInfo {
     email: String,
     login_type: Option<String>,
@@ -397,6 +448,7 @@ pub struct UserInfo {
     pub operator: bool,
     pub disabled: bool,
     pub role: Option<String>,
+    pub folders: Vec<String>,
 }
 
 #[derive(FromRow, Serialize)]
@@ -404,6 +456,7 @@ pub struct WorkspaceInvite {
     pub workspace_id: String,
     pub email: String,
     pub is_admin: bool,
+    pub operator: bool,
 }
 
 #[derive(FromRow, Serialize)]
@@ -460,7 +513,8 @@ pub struct EditUser {
 #[derive(Deserialize)]
 pub struct EditWorkspaceUser {
     pub is_admin: Option<bool>,
-    pub enabled: Option<bool>,
+    pub operator: Option<bool>,
+    pub disabled: Option<bool>,
 }
 
 #[derive(Deserialize)]
@@ -537,14 +591,12 @@ async fn list_users(
         SELECT usr.*, usage.*
           FROM usr
              , LATERAL (
-                SELECT COALESCE(SUM(duration_ms), 0)  duration_ms
-                     , COALESCE(SUM(job_kind     IN ('flow', 'flowpreview') ::int), 0)  flows
-                     , COALESCE(SUM(job_kind NOT IN ('flow', 'flowpreview') ::int), 0)  jobs
+                SELECT COALESCE(SUM(duration_ms + 1000)/1000 , 0) executions
                   FROM completed_job
                  WHERE workspace_id = usr.workspace_id
-                   AND created_by = usr.username
-                   AND parent_job IS NULL
-                   AND now() - '2 week'::interval < created_at 
+                   AND job_kind NOT IN ('flow', 'flowpreview')
+                   AND email = usr.email
+                   AND now() - '5 week'::interval < created_at 
                ) usage
          WHERE workspace_id = $1
          ",
@@ -566,7 +618,7 @@ async fn list_users_as_super_admin(
     Query(pagination): Query<Pagination>,
 ) -> JsonResult<Vec<GlobalUserInfo>> {
     let mut tx = db.begin().await?;
-    require_super_admin(&mut tx, authed.email).await?;
+    require_super_admin(&mut tx, &authed.email).await?;
     let per_page = pagination.per_page.unwrap_or(10000).max(1);
     let offset = (pagination.page.unwrap_or(1).max(1) - 1) * per_page;
 
@@ -604,6 +656,12 @@ async fn list_usernames(
     Extension(user_db): Extension<UserDB>,
     Path(w_id): Path<String>,
 ) -> JsonResult<Vec<String>> {
+    if *CLOUD_HOSTED && w_id == "demo" {
+        return Ok(Json(vec![
+            authed.username,
+            "other_usernames_redacted_in_demo_workspace".to_string(),
+        ]));
+    }
     let mut tx = user_db.begin(&authed).await?;
     let rows = sqlx::query_scalar!("SELECT username from usr WHERE workspace_id = $1", &w_id)
         .fetch_all(&mut tx)
@@ -628,13 +686,23 @@ async fn list_invites(
     Ok(Json(rows))
 }
 
+#[derive(Deserialize)]
+struct LogoutQuery {
+    rd: Option<String>,
+}
 async fn logout(
     Tokened { token }: Tokened,
     cookies: Cookies,
     Extension(db): Extension<DB>,
-) -> Result<String> {
+    Extension(cookie_domain): Extension<Arc<CookieDomain>>,
+    Query(LogoutQuery { rd }): Query<LogoutQuery>,
+) -> Result<Response> {
     let mut cookie = Cookie::new(COOKIE_NAME, "");
     cookie.set_path(COOKIE_PATH);
+    let domain = cookie_domain.0.clone();
+    if domain.is_some() {
+        cookie.set_domain(domain.clone().unwrap());
+    }
     cookies.remove(cookie);
     let mut tx = db.begin().await?;
     let email = sqlx::query_scalar!("DELETE FROM token WHERE token = $1 RETURNING email", token)
@@ -653,20 +721,22 @@ async fn logout(
         .await?;
     }
     tx.commit().await?;
-
-    Ok("logged out successfully".to_string())
+    if let Some(rd) = rd {
+        Ok((StatusCode::TEMPORARY_REDIRECT, [(LOCATION, rd)]).into_response())
+    } else {
+        Ok((StatusCode::OK, "logged out successfully".to_string()).into_response())
+    }
 }
 
 async fn whoami(
     Extension(db): Extension<DB>,
     Path(w_id): Path<String>,
-    Authed { username, email, is_admin, groups }: Authed,
+    Authed { username, email, is_admin, groups, folders }: Authed,
 ) -> JsonResult<UserInfo> {
     let user = get_user(&w_id, &username, &db).await?;
     if let Some(user) = user {
         Ok(Json(user))
     } else {
-        let email = email.unwrap_or_else(|| "noemail".to_string());
         Ok(Json(UserInfo {
             workspace_id: w_id,
             email: email.clone(),
@@ -678,6 +748,10 @@ async fn whoami(
             operator: false,
             disabled: false,
             role: Some("superadmin".to_string()),
+            folders: folders
+                .into_iter()
+                .filter_map(|x| if x.1 { Some(x.0) } else { None })
+                .collect(),
         }))
     }
 }
@@ -685,8 +759,9 @@ async fn whoami(
 async fn global_whoami(
     Extension(db): Extension<DB>,
     Authed { email, .. }: Authed,
+    Tokened { token }: Tokened,
 ) -> JsonResult<GlobalUserInfo> {
-    let user: GlobalUserInfo = sqlx::query_as!(
+    let user = sqlx::query_as!(
         GlobalUserInfo,
         "SELECT email, login_type::TEXT, super_admin, verified, name, company FROM password WHERE \
          email = $1",
@@ -694,18 +769,43 @@ async fn global_whoami(
     )
     .fetch_one(&db)
     .await
-    .map_err(|e| Error::InternalErr(format!("fetching global identity: {e}")))?;
+    .map_err(|e| Error::InternalErr(format!("fetching global identity: {e}")));
 
-    Ok(Json(user))
+    if let Ok(user) = user {
+        Ok(Json(user))
+    } else if std::env::var("SUPERADMIN_SECRET").ok() == Some(token) {
+        Ok(Json(GlobalUserInfo {
+            email: email.clone(),
+            login_type: Some("superadmin_secret".to_string()),
+            super_admin: true,
+            verified: true,
+            name: None,
+            company: None,
+        }))
+    } else {
+        Err(user.unwrap_err())
+    }
 }
 
 async fn get_email(Authed { email, .. }: Authed) -> Result<String> {
-    let email = email.ok_or(Error::BadRequest(
-        "current session does not correspond to an user with email".to_string(),
-    ))?;
     Ok(email)
 }
 
+async fn get_usage(Extension(db): Extension<DB>, Authed { email, .. }: Authed) -> Result<String> {
+    let usage = sqlx::query_scalar!(
+        "
+    SELECT usage.usage FROM usage 
+    WHERE is_workspace = false 
+    AND month_ = EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date)
+    AND id = $1",
+        email
+    )
+    .fetch_optional(&db)
+    .await?
+    .unwrap_or(0);
+    Ok(usage.to_string())
+}
+
 async fn get_user(w_id: &str, username: &str, db: &DB) -> Result<Option<UserInfo>> {
     let user = sqlx::query_as!(
         User,
@@ -723,6 +823,8 @@ async fn get_user(w_id: &str, username: &str, db: &DB) -> Result<Option<UserInfo
     .await?
     .unwrap_or(false);
     let groups = get_groups_for_user(&w_id, username, db).await?;
+    let folders = get_folders_for_user(&w_id, username, &groups, db).await?;
+
     Ok(user.map(|usr| UserInfo {
         groups,
         workspace_id: usr.workspace_id,
@@ -734,10 +836,14 @@ async fn get_user(w_id: &str, username: &str, db: &DB) -> Result<Option<UserInfo
         operator: usr.operator,
         disabled: usr.disabled,
         role: usr.role,
+        folders: folders
+            .into_iter()
+            .filter_map(|x| if x.1 { Some(x.0) } else { None })
+            .collect(),
     }))
 }
 
-async fn get_groups_for_user(w_id: &str, username: &str, db: &DB) -> Result<Vec<String>> {
+pub async fn get_groups_for_user(w_id: &str, username: &str, db: &DB) -> Result<Vec<String>> {
     let groups = sqlx::query_scalar!(
         "SELECT group_ FROM usr_to_group where usr = $1 AND workspace_id = $2",
         username,
@@ -747,6 +853,47 @@ async fn get_groups_for_user(w_id: &str, username: &str, db: &DB) -> Result<Vec<
     .await?;
     Ok(groups)
 }
+pub async fn is_owner_of_path(
+    Authed { username, is_admin, groups, .. }: Authed,
+    Extension(db): Extension<DB>,
+    Path((w_id, path)): Path<(String, StripPath)>,
+) -> JsonResult<bool> {
+    let path = path.to_path();
+    if is_admin {
+        Ok(Json(true))
+    } else {
+        Ok(Json(
+            require_owner_of_path(&w_id, &username, &groups, path, &db)
+                .await
+                .is_ok(),
+        ))
+    }
+}
+
+pub async fn require_owner_of_path(
+    w_id: &str,
+    username: &str,
+    groups: &Vec<String>,
+    path: &str,
+    db: &DB,
+) -> Result<()> {
+    let splitted = path.split("/").collect::<Vec<&str>>();
+    if splitted[0] == "u" {
+        if splitted[1] == username {
+            return Ok(());
+        } else {
+            return Err(Error::BadRequest(format!(
+                "only the owner {} is authorized to perform this operation",
+                splitted[1]
+            )));
+        }
+    } else if splitted[0] == "g" {
+        return crate::groups::require_is_owner(w_id, username, groups, splitted[1], db).await;
+    } else if splitted[0] == "f" {
+        return crate::folders::require_is_owner(w_id, username, groups, splitted[1], db).await;
+    }
+    Err(Error::BadRequest(format!("not recognized owner kind")))
+}
 
 async fn whois(
     Extension(db): Extension<DB>,
@@ -791,7 +938,6 @@ async fn decline_invite(
 ) -> Result<(StatusCode, String)> {
     let mut tx = db.begin().await?;
 
-    let email = email.unwrap_or("".to_string());
     let is_admin = sqlx::query_scalar!(
         "DELETE FROM workspace_invite WHERE workspace_id = $1 AND email = $2 RETURNING is_admin",
         nu.workspace_id,
@@ -835,32 +981,40 @@ async fn accept_invite(
     }
     let mut tx = db.begin().await?;
 
-    let email = email.unwrap_or("".to_string());
-    let is_admin = sqlx::query_scalar!(
-        "DELETE FROM workspace_invite WHERE workspace_id = $1 AND email = $2 RETURNING is_admin",
+    let r = sqlx::query!(
+        "DELETE FROM workspace_invite WHERE workspace_id = $1 AND email = $2 RETURNING is_admin, operator",
         nu.workspace_id,
         email,
     )
     .fetch_optional(&mut tx)
     .await?;
 
-    if let Some(is_admin) = is_admin {
-        tx = add_user_to_workspace(&nu.workspace_id, &email, &nu.username, is_admin, tx).await?;
+    let is_some = r.is_some();
+    if let Some(r) = r {
+        tx = add_user_to_workspace(
+            &nu.workspace_id,
+            &email,
+            &nu.username,
+            r.is_admin,
+            r.operator,
+            tx,
+        )
+        .await?;
+
+        audit_log(
+            &mut tx,
+            &nu.username,
+            "users.accept_invite",
+            ActionKind::Create,
+            &nu.workspace_id,
+            Some(&email),
+            None,
+        )
+        .await?;
+        tx.commit().await?;
     }
 
-    audit_log(
-        &mut tx,
-        &nu.username,
-        "users.accept_invite",
-        ActionKind::Create,
-        &nu.workspace_id,
-        Some(&email),
-        None,
-    )
-    .await?;
-    tx.commit().await?;
-
-    if is_admin.is_some() {
+    if is_some {
         Ok((
             StatusCode::CREATED,
             format!(
@@ -878,22 +1032,56 @@ async fn add_user_to_workspace<'c>(
     email: &str,
     username: &str,
     is_admin: bool,
+    operator: bool,
     mut tx: sqlx::Transaction<'c, sqlx::Postgres>,
 ) -> error::Result<sqlx::Transaction<'c, sqlx::Postgres>> {
+    let already_exists_username = sqlx::query_scalar!(
+        "SELECT EXISTS(SELECT 1 FROM usr WHERE workspace_id = $1 AND username = $2)",
+        &w_id,
+        username,
+    )
+    .fetch_one(&mut tx)
+    .await?
+    .unwrap_or(false);
+
+    if already_exists_username {
+        return Err(Error::BadRequest(format!(
+            "user with username {} already exists in workspace {}",
+            username, w_id
+        )));
+    }
+
+    let already_exists_email = sqlx::query_scalar!(
+        "SELECT EXISTS(SELECT 1 FROM usr WHERE workspace_id = $1 AND email = $2)",
+        &w_id,
+        username,
+    )
+    .fetch_one(&mut tx)
+    .await?
+    .unwrap_or(false);
+
+    if already_exists_email {
+        return Err(Error::BadRequest(format!(
+            "user with email {} already exists in workspace {}",
+            email, w_id
+        )));
+    }
+
     sqlx::query!(
         "INSERT INTO usr
-            (workspace_id, email, username, is_admin)
-            VALUES ($1, $2, $3, $4)",
+            (workspace_id, email, username, is_admin, operator)
+            VALUES ($1, $2, $3, $4, $5)",
         &w_id,
         email,
         username,
-        is_admin
+        is_admin,
+        operator
     )
     .execute(&mut tx)
     .await?;
     sqlx::query_as!(
         Group,
-        "INSERT INTO usr_to_group (workspace_id, usr, group_) VALUES ($1, $2, $3)",
+        "INSERT INTO usr_to_group (workspace_id, usr, group_) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING",
         &w_id,
         username,
         "all",
@@ -934,6 +1122,28 @@ async fn update_workspace_user(
         .await?;
     }
 
+    if let Some(a) = eu.operator {
+        sqlx::query_scalar!(
+            "UPDATE usr SET operator = $1 WHERE username = $2 AND workspace_id = $3",
+            a,
+            &username_to_update,
+            &w_id
+        )
+        .execute(&mut tx)
+        .await?;
+    }
+
+    if let Some(a) = eu.disabled {
+        sqlx::query_scalar!(
+            "UPDATE usr SET disabled = $1 WHERE username = $2 AND workspace_id = $3",
+            a,
+            &username_to_update,
+            &w_id
+        )
+        .execute(&mut tx)
+        .await?;
+    }
+
     audit_log(
         &mut tx,
         &username,
@@ -956,7 +1166,7 @@ async fn update_user(
 ) -> Result<String> {
     let mut tx = db.begin().await?;
 
-    require_super_admin(&mut tx, email.clone()).await?;
+    require_super_admin(&mut tx, &email).await?;
 
     if let Some(sa) = eu.is_super_admin {
         sqlx::query_scalar!(
@@ -970,7 +1180,7 @@ async fn update_user(
 
     audit_log(
         &mut tx,
-        &email.unwrap(),
+        &email,
         "users.update",
         ActionKind::Update,
         "global",
@@ -989,19 +1199,38 @@ async fn delete_user(
 ) -> Result<String> {
     let mut tx = db.begin().await?;
 
-    require_super_admin(&mut tx, email.clone()).await?;
-
-    sqlx::query!("DELETE FROM usr WHERE email = $1", &email_to_delete)
-        .execute(&mut tx)
-        .await?;
+    require_super_admin(&mut tx, &email).await?;
 
     sqlx::query!("DELETE FROM password WHERE email = $1", &email_to_delete)
         .execute(&mut tx)
         .await?;
 
+    let usernames = sqlx::query_scalar!(
+        "DELETE FROM usr WHERE email = $1 RETURNING username",
+        &email_to_delete
+    )
+    .fetch_all(&mut tx)
+    .await?;
+
+    for username in usernames {
+        sqlx::query!("DELETE FROM password WHERE email = $1", &email_to_delete)
+            .execute(&mut tx)
+            .await?;
+
+        sqlx::query!("DELETE FROM usr_to_group WHERE usr = $1", &username)
+            .execute(&mut tx)
+            .await?;
+
+        sqlx::query!(
+            "DELETE FROM workspace_invite WHERE email = $1",
+            &email_to_delete
+        )
+        .execute(&mut tx)
+        .await?;
+    }
     audit_log(
         &mut tx,
-        &email.unwrap(),
+        &email,
         "users.delete",
         ActionKind::Delete,
         "global",
@@ -1021,7 +1250,7 @@ async fn create_user(
 ) -> Result<(StatusCode, String)> {
     let mut tx = db.begin().await?;
 
-    require_super_admin(&mut tx, email.clone()).await?;
+    require_super_admin(&mut tx, &email).await?;
 
     sqlx::query!(
         "INSERT INTO password(email, verified, password_hash, login_type, super_admin, name, \
@@ -1039,15 +1268,17 @@ async fn create_user(
 
     audit_log(
         &mut tx,
-        &email.unwrap(),
-        "users.update",
-        ActionKind::Update,
+        &email,
+        "users.add_global",
+        ActionKind::Create,
         "global",
         Some(&nu.email),
         None,
     )
     .await?;
     tx.commit().await?;
+    invite_user_to_all_auto_invite_worspaces(&db, &nu.email).await?;
+
     Ok((StatusCode::CREATED, format!("email {} created", nu.email)))
 }
 
@@ -1070,7 +1301,14 @@ async fn delete_workspace_user(
 
     let email_to_delete = not_found_if_none(email_to_delete_o, "User", &username_to_delete)?;
 
-    sqlx::query!("DELETE FROM usr WHERE email = $1", email_to_delete)
+    let username = sqlx::query_scalar!(
+        "DELETE FROM usr WHERE email = $1 RETURNING username",
+        email_to_delete
+    )
+    .fetch_one(&mut tx)
+    .await?;
+
+    sqlx::query!("DELETE FROM usr_to_group WHERE usr = $1", &username)
         .execute(&mut tx)
         .await?;
 
@@ -1095,9 +1333,6 @@ async fn set_password(
     Json(EditPassword { password }): Json<EditPassword>,
 ) -> Result<String> {
     let mut tx = db.begin().await?;
-    let email = email
-        .ok_or("no_email")
-        .map_err(|e| Error::NotAuthorized(e.to_string()))?;
 
     let custom_type = sqlx::query_scalar!(
         "SELECT login_type::TEXT FROM password WHERE email = $1",
@@ -1287,6 +1522,7 @@ async fn login(
     Extension(db): Extension<DB>,
     Extension(argon2): Extension<Arc<Argon2<'_>>>,
     Extension(is_secure): Extension<Arc<IsSecure>>,
+    Extension(cookie_domain): Extension<Arc<CookieDomain>>,
     Json(Login { email, password }): Json<Login>,
 ) -> Result<String> {
     let mut tx = db.begin().await?;
@@ -1308,8 +1544,15 @@ async fn login(
         {
             Err(Error::BadRequest("Invalid login".to_string()))
         } else {
-            let token =
-                create_session_token(&email, super_admin, &mut tx, cookies, is_secure.0).await?;
+            let token = create_session_token(
+                &email,
+                super_admin,
+                &mut tx,
+                cookies,
+                is_secure.0,
+                &cookie_domain.as_ref().0,
+            )
+            .await?;
             tx.commit().await?;
             Ok(token)
         }
@@ -1324,6 +1567,7 @@ pub async fn create_session_token<'c>(
     tx: &mut sqlx::Transaction<'c, sqlx::Postgres>,
     cookies: Cookies,
     is_secure: bool,
+    domain: &Option<String>,
 ) -> Result<String> {
     let token = rd_string(30);
     sqlx::query!(
@@ -1343,6 +1587,9 @@ pub async fn create_session_token<'c>(
     cookie.set_same_site(cookie::SameSite::Lax);
     cookie.set_http_only(true);
     cookie.set_path(COOKIE_PATH);
+    if domain.is_some() {
+        cookie.set_domain(domain.clone().unwrap());
+    }
     let mut expire: OffsetDateTime = time::OffsetDateTime::now_utc();
     expire += time::Duration::days(3);
     cookie.set_expires(expire);
@@ -1357,9 +1604,7 @@ async fn create_token(
 ) -> Result<(StatusCode, String)> {
     let token = rd_string(30);
     let mut tx = db.begin().await?;
-    let email = email.ok_or_else(|| {
-        error::Error::BadRequest(format!("Only users with email can create tokens"))
-    })?;
+
     let is_super_admin =
         sqlx::query_scalar!("SELECT super_admin FROM password WHERE email = $1", email)
             .fetch_optional(&mut tx)
@@ -1387,6 +1632,7 @@ async fn create_token(
         Some(&token[0..10]),
         None,
     )
+    .instrument(tracing::info_span!("token", email = &email))
     .await?;
     tx.commit().await?;
     Ok((StatusCode::CREATED, token))
@@ -1414,9 +1660,7 @@ async fn delete_token(
     Path(token_prefix): Path<String>,
 ) -> Result<String> {
     let mut tx = db.begin().await?;
-    let email = email.ok_or_else(|| {
-        error::Error::BadRequest(format!("Only users with email can create tokens"))
-    })?;
+
     let tokens_deleted: Vec<String> = sqlx::query_scalar(
         "DELETE FROM token
                WHERE email = $1

backend/windmill-api/src/utils.rs

@@ -11,15 +11,12 @@ use windmill_common::error::{self, Error};
 
 pub async fn require_super_admin<'c>(
     db: &mut Transaction<'c, Postgres>,
-    email: Option<String>,
+    email: &str,
 ) -> error::Result<()> {
-    let is_admin = sqlx::query_scalar!(
-        "SELECT super_admin FROM password WHERE email = $1",
-        email.as_ref()
-    )
-    .fetch_one(db)
-    .await
-    .map_err(|e| Error::InternalErr(format!("fetching super admin: {e}")))?;
+    let is_admin = sqlx::query_scalar!("SELECT super_admin FROM password WHERE email = $1", email)
+        .fetch_one(db)
+        .await
+        .map_err(|e| Error::InternalErr(format!("fetching super admin: {e}")))?;
     if !is_admin {
         Err(Error::NotAuthorized(
             "This endpoint require caller to be a super admin".to_owned(),

backend/windmill-api/src/variables.rs

@@ -11,7 +11,7 @@ use std::sync::Arc;
 use crate::{
     db::{UserDB, DB},
     oauth2::{AllClients, _refresh_token},
-    users::Authed,
+    users::{require_owner_of_path, Authed},
     BaseUrl,
 };
 /*
@@ -60,7 +60,7 @@ async fn list_contextual_variables(
         get_reserved_variables(
             &w_id,
             "q1A0qcPuO00yxioll7iph76N9CJDqn",
-            &email.unwrap_or_else(|| "no email".to_string()),
+            &email,
             &username,
             "017e0ad5-f499-73b6-5488-92a61c5196dd",
             format!("u/{username}").as_str(),
@@ -82,11 +82,15 @@ async fn list_variables(
     let mut tx = user_db.begin(&authed).await?;
 
     let rows = sqlx::query_as::<_, ListableVariable>(
-        "SELECT workspace_id, path, CASE WHEN is_secret IS TRUE THEN null ELSE value::text END as \
-         value, is_secret, description, extra_perms, account, is_oauth, false as is_expired from \
-         variable
-         WHERE (workspace_id = $1 OR (is_secret IS NOT TRUE AND workspace_id = 'starter')) ORDER \
-         BY path",
+        "SELECT variable.workspace_id, variable.path, CASE WHEN is_secret IS TRUE THEN null ELSE variable.value::text END as value, 
+         is_secret, variable.description, variable.extra_perms, account, is_oauth, (now() > account.expires_at) as is_expired,
+         account.refresh_error,
+         resource.path IS NOT NULL as is_linked,
+         account.refresh_token != '' as is_refreshed
+         from variable
+         LEFT JOIN account ON variable.account = account.id AND account.workspace_id = variable.workspace_id
+         LEFT JOIN resource ON resource.path = variable.path AND resource.workspace_id = variable.workspace_id
+         WHERE variable.workspace_id = $1 OR (is_secret IS NOT TRUE AND variable.workspace_id = 'starter') ORDER BY path",
     )
     .bind(&w_id)
     .fetch_all(&mut tx)
@@ -113,8 +117,12 @@ async fn get_variable(
     let mut tx = user_db.begin(&authed).await?;
 
     let variable_o = sqlx::query_as::<_, ListableVariable>(
-        "SELECT variable.*, (now() > account.expires_at) as is_expired from variable
+        "SELECT variable.*, (now() > account.expires_at) as is_expired, account.refresh_error,
+        resource.path IS NOT NULL as is_linked,
+        account.refresh_token != '' as is_refreshed
+        from variable
         LEFT JOIN account ON variable.account = account.id
+        LEFT JOIN resource ON resource.path = variable.path AND resource.workspace_id = variable.workspace_id
         WHERE variable.path = $1 AND (variable.workspace_id = $2 OR (is_secret IS NOT TRUE AND \
          variable.workspace_id = 'starter')) 
         LIMIT 1",
@@ -191,6 +199,28 @@ async fn exists_variable(
     Ok(Json(exists))
 }
 
+async fn check_path_conflict<'c>(
+    tx: &mut Transaction<'c, Postgres>,
+    w_id: &str,
+    path: &str,
+) -> Result<()> {
+    let exists = sqlx::query_scalar!(
+        "SELECT EXISTS(SELECT 1 FROM variable WHERE path = $1 AND workspace_id = $2)",
+        path,
+        w_id
+    )
+    .fetch_one(tx)
+    .await?
+    .unwrap_or(false);
+    if exists {
+        return Err(Error::BadRequest(format!(
+            "Variable {} already exists",
+            path
+        )));
+    }
+    return Ok(());
+}
+
 async fn create_variable(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
@@ -199,6 +229,7 @@ async fn create_variable(
 ) -> Result<(StatusCode, String)> {
     let mut tx = user_db.begin(&authed).await?;
 
+    check_path_conflict(&mut tx, &w_id, &variable.path).await?;
     let value = if variable.is_secret {
         let mc = build_crypt(&mut tx, &w_id).await?;
         encrypt(&mc, &variable.value)
@@ -255,6 +286,13 @@ async fn delete_variable(
     )
     .execute(&mut tx)
     .await?;
+    sqlx::query!(
+        "DELETE FROM resource WHERE path = $1 AND workspace_id = $2",
+        path,
+        w_id
+    )
+    .execute(&mut tx)
+    .await?;
     audit_log(
         &mut tx,
         &authed.username,
@@ -282,6 +320,7 @@ struct EditVariable {
 async fn update_variable(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
+    Extension(db): Extension<DB>,
     Path((w_id, path)): Path<(String, StripPath)>,
     Json(ns): Json<EditVariable>,
 ) -> Result<String> {
@@ -335,6 +374,23 @@ async fn update_variable(
 
     let npath_o: Option<String> = sqlx::query_scalar(&sql).fetch_optional(&mut tx).await?;
 
+    if let Some(npath) = ns.path {
+        if npath != path {
+            check_path_conflict(&mut tx, &w_id, &npath).await?;
+            if !authed.is_admin {
+                require_owner_of_path(&w_id, &authed.username, &authed.groups, &path, &db).await?;
+            }
+            sqlx::query!(
+                "UPDATE resource SET path = $1 WHERE path = $2 AND workspace_id = $3",
+                npath,
+                path,
+                w_id
+            )
+            .execute(&mut tx)
+            .await?;
+        }
+    }
+
     let npath = not_found_if_none(npath_o, "Variable", path)?;
 
     audit_log(

backend/windmill-api/src/workspaces.rs

@@ -8,6 +8,7 @@
 
 use crate::{
     db::{UserDB, DB},
+    folders::Folder,
     resources::{Resource, ResourceType},
     users::{Authed, WorkspaceInvite},
     utils::require_super_admin,
@@ -26,12 +27,12 @@ use windmill_common::{
     flows::Flow,
     scripts::{Schema, Script, ScriptLang},
     utils::{paginate, rd_string, require_admin, Pagination},
-    variables::ListableVariable,
+    variables::ExportableListableVariable,
 };
 
 use hyper::{header, StatusCode};
 use serde::{Deserialize, Serialize};
-use sqlx::FromRow;
+use sqlx::{FromRow, Postgres, Transaction};
 use tempfile::TempDir;
 use tokio::fs::File;
 use tokio_util::io::ReaderStream;
@@ -45,7 +46,9 @@ pub fn workspaced_service() -> Router {
         .route("/delete_invite", post(delete_invite))
         .route("/get_settings", get(get_settings))
         .route("/edit_slack_command", post(edit_slack_command))
+        .route("/edit_auto_invite", post(edit_auto_invite))
         .route("/tarball", get(tarball_workspace))
+        .route("/premium_info", get(premium_info))
 }
 
 pub fn global_service() -> Router {
@@ -56,6 +59,7 @@ pub fn global_service() -> Router {
         .route("/create", post(create_workspace))
         .route("/exists", post(exists_workspace))
         .route("/exists_username", post(exists_username))
+        .route("/allowed_domain_auto_invite", get(is_allowed_auto_domain))
 }
 
 #[derive(FromRow, Serialize)]
@@ -63,7 +67,6 @@ struct Workspace {
     id: String,
     name: String,
     owner: String,
-    domain: Option<String>,
     deleted: bool,
     premium: bool,
 }
@@ -74,6 +77,18 @@ pub struct WorkspaceSettings {
     pub slack_team_id: Option<String>,
     pub slack_name: Option<String>,
     pub slack_command_script: Option<String>,
+    pub slack_email: String,
+    pub auto_invite_domain: Option<String>,
+    pub auto_invite_operator: Option<bool>,
+}
+
+#[derive(FromRow, Serialize, Debug)]
+pub struct Usage {
+    pub workspace_id: String,
+    pub slack_team_id: Option<String>,
+    pub slack_name: Option<String>,
+    pub slack_command_script: Option<String>,
+    pub slack_email: String,
 }
 
 #[derive(sqlx::Type, Serialize, Deserialize, Debug)]
@@ -86,19 +101,23 @@ pub enum WorkspaceKeyKind {
 struct EditCommandScript {
     slack_command_script: Option<String>,
 }
+
+#[derive(Deserialize)]
+struct EditAutoInvite {
+    operator: Option<bool>,
+}
+
 #[derive(Deserialize)]
 struct CreateWorkspace {
     id: String,
     name: String,
     username: String,
-    domain: Option<String>,
 }
 
 #[derive(Deserialize)]
 struct EditWorkspace {
     name: String,
     owner: String,
-    domain: Option<String>,
 }
 
 #[derive(Serialize)]
@@ -129,6 +148,7 @@ struct ValidateUsername {
 pub struct NewWorkspaceInvite {
     pub email: String,
     pub is_admin: bool,
+    pub operator: bool,
 }
 
 async fn list_pending_invites(
@@ -149,6 +169,28 @@ async fn list_pending_invites(
     Ok(Json(rows))
 }
 
+#[derive(Serialize, FromRow)]
+pub struct PremiumWorkspaceInfo {
+    pub premium: bool,
+    pub usage: Option<i32>,
+}
+async fn premium_info(
+    authed: Authed,
+    Extension(db): Extension<DB>,
+    Path(w_id): Path<String>,
+) -> JsonResult<PremiumWorkspaceInfo> {
+    require_admin(authed.is_admin, &authed.username)?;
+    let mut tx = db.begin().await?;
+    let row = sqlx::query_as::<_, PremiumWorkspaceInfo>(
+        "SELECT premium, usage.usage FROM workspace LEFT JOIN usage ON workspace.id = usage.id AND usage.is_workspace IS true WHERE workspace.id = $1",
+    )
+    .bind(w_id)
+    .fetch_one(&mut tx)
+    .await?;
+    tx.commit().await?;
+    Ok(Json(row))
+}
+
 async fn exists_workspace(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
@@ -175,7 +217,7 @@ async fn list_workspaces(
         Workspace,
         "SELECT workspace.* FROM workspace, usr WHERE usr.workspace_id = workspace.id AND \
          usr.email = $1 AND deleted = false",
-        authed.email.as_ref()
+        authed.email
     )
     .fetch_all(&mut tx)
     .await?;
@@ -225,7 +267,7 @@ async fn edit_slack_command(
         "workspaces.edit_command_script",
         ActionKind::Update,
         &w_id,
-        Some(&authed.email.unwrap()),
+        Some(&authed.email),
         Some(
             [(
                 "script",
@@ -242,6 +284,78 @@ async fn edit_slack_command(
     Ok(format!("Edit command script {}", &w_id))
 }
 
+const BANNED_DOMAINS: &str = include_str!("../banned_domains.txt");
+
+async fn is_allowed_auto_domain(Authed { email, .. }: Authed) -> JsonResult<bool> {
+    let domain = email.split('@').last().unwrap();
+    return Ok(Json(!BANNED_DOMAINS.contains(domain)));
+}
+
+async fn edit_auto_invite(
+    authed: Authed,
+    Extension(db): Extension<DB>,
+    Path(w_id): Path<String>,
+    Authed { is_admin, email, username, .. }: Authed,
+    Json(ea): Json<EditAutoInvite>,
+) -> Result<String> {
+    require_admin(is_admin, &username)?;
+    let domain = email.split('@').last().unwrap();
+
+    let mut tx = db.begin().await?;
+
+    if let Some(operator) = ea.operator {
+        if BANNED_DOMAINS.contains(domain) {
+            return Err(Error::BadRequest(format!(
+                "Domain {} is not allowed",
+                domain
+            )));
+        }
+
+        sqlx::query!(
+            "UPDATE workspace_settings SET auto_invite_domain = $1, auto_invite_operator = $2 WHERE workspace_id = $3",
+            domain,
+            operator,
+            &w_id
+        )
+        .execute(&mut tx)
+        .await?;
+
+        sqlx::query!(
+            "INSERT INTO workspace_invite
+        (workspace_id, email, is_admin, operator)
+        SELECT $1::text, email, false, $3 FROM password WHERE email LIKE CONCAT('%', $2::text) AND NOT EXISTS (
+            SELECT 1 FROM usr WHERE workspace_id = $1::text AND email = password.email
+        )
+        ON CONFLICT DO NOTHING",
+            &w_id,
+            &domain,
+            operator
+        )
+        .execute(&mut tx)
+        .await?;
+    } else {
+        sqlx::query!(
+            "UPDATE workspace_settings SET auto_invite_domain = NULL, auto_invite_operator = NULL WHERE workspace_id = $1",
+            &w_id,
+        )
+        .execute(&mut tx)
+        .await?;
+    }
+    audit_log(
+        &mut tx,
+        &authed.username,
+        "workspaces.edit_auto_invite_domain",
+        ActionKind::Update,
+        &w_id,
+        Some(&authed.email),
+        Some([("operator", &format!("{:?}", ea.operator)[..])].into()),
+    )
+    .await?;
+    tx.commit().await?;
+
+    Ok(format!("Edit command script {}", &w_id))
+}
+
 async fn list_workspaces_as_super_admin(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
@@ -249,7 +363,7 @@ async fn list_workspaces_as_super_admin(
     Authed { email, .. }: Authed,
 ) -> JsonResult<Vec<Workspace>> {
     let mut tx = user_db.begin(&authed).await?;
-    require_super_admin(&mut tx, email).await?;
+    require_super_admin(&mut tx, &email).await?;
     let (per_page, offset) = paginate(pagination);
 
     let workspaces = sqlx::query_as!(
@@ -268,9 +382,6 @@ async fn user_workspaces(
     Extension(db): Extension<DB>,
     Authed { email, .. }: Authed,
 ) -> JsonResult<WorkspaceList> {
-    let email = email
-        .ok_or("not a personal token")
-        .map_err(|x| Error::NotAuthorized(x.to_string()))?;
     let mut tx = db.begin().await?;
     let workspaces = sqlx::query_as!(
         UserWorkspace,
@@ -285,6 +396,20 @@ async fn user_workspaces(
     Ok(Json(WorkspaceList { email, workspaces }))
 }
 
+async fn check_name_conflict<'c>(tx: &mut Transaction<'c, Postgres>, w_id: &str) -> Result<()> {
+    let exists = sqlx::query_scalar!("SELECT EXISTS(SELECT 1 FROM workspace WHERE id = $1)", w_id)
+        .fetch_one(tx)
+        .await?
+        .unwrap_or(false);
+    if exists {
+        return Err(windmill_common::error::Error::BadRequest(format!(
+            "Workspace {} already exists",
+            w_id
+        )));
+    }
+    return Ok(());
+}
+
 async fn create_workspace(
     authed: Authed,
     Extension(db): Extension<DB>,
@@ -294,14 +419,14 @@ async fn create_workspace(
         return Err(Error::BadRequest("bot is a reserved username".to_string()));
     }
     let mut tx = db.begin().await?;
+    check_name_conflict(&mut tx, &nw.id).await?;
     sqlx::query!(
         "INSERT INTO workspace
-            (id, name, owner, domain)
-            VALUES ($1, $2, $3, $4)",
+            (id, name, owner)
+            VALUES ($1, $2, $3)",
         nw.id,
         nw.name,
         authed.email,
-        nw.domain
     )
     .execute(&mut tx)
     .await?;
@@ -377,7 +502,6 @@ async fn create_workspace(
     )
     .await?;
     tx.commit().await?;
-
     Ok(format!("Created workspace {}", &nw.id))
 }
 
@@ -391,10 +515,9 @@ async fn edit_workspace(
     require_admin(is_admin, &username)?;
     let mut tx = db.begin().await?;
     sqlx::query!(
-        "UPDATE workspace SET name = $1, owner = $2, domain = $3 WHERE id = $4",
+        "UPDATE workspace SET name = $1, owner = $2 WHERE id = $3",
         ew.name,
         ew.owner,
-        ew.domain,
         &w_id
     )
     .execute(&mut tx)
@@ -406,14 +529,8 @@ async fn edit_workspace(
         "workspaces.update",
         ActionKind::Update,
         &w_id,
-        Some(&authed.email.unwrap()),
-        Some(
-            [(
-                "domain",
-                ew.domain.unwrap_or("NO_DOMAIN".to_string()).as_str(),
-            )]
-            .into(),
-        ),
+        Some(&authed.email),
+        None,
     )
     .await?;
     tx.commit().await?;
@@ -426,6 +543,15 @@ async fn delete_workspace(
     Path(w_id): Path<String>,
     Authed { is_admin, username, email, .. }: Authed,
 ) -> Result<String> {
+    let w_id = match w_id.as_str() {
+        "starter" => Err(Error::BadRequest(
+            "starter workspace cannot be deleted".to_string(),
+        )),
+        "admins" => Err(Error::BadRequest(
+            "admins workspace cannot be deleted".to_string(),
+        )),
+        _ => Ok(w_id),
+    }?;
     require_admin(is_admin, &username)?;
     let mut tx = db.begin().await?;
     sqlx::query!("UPDATE workspace SET deleted = true WHERE id = $1", &w_id)
@@ -438,7 +564,7 @@ async fn delete_workspace(
         "workspaces.delete",
         ActionKind::Update,
         &w_id,
-        Some(&email.unwrap_or("noemail".to_string())),
+        Some(&email),
         None,
     )
     .await?;
@@ -447,6 +573,32 @@ async fn delete_workspace(
     Ok(format!("Deleted workspace {}", &w_id))
 }
 
+pub async fn invite_user_to_all_auto_invite_worspaces(db: &DB, email: &str) -> Result<()> {
+    let mut tx = db.begin().await?;
+    let domain = email.split('@').last().unwrap();
+    let workspaces = sqlx::query!(
+        "SELECT workspace_id, auto_invite_operator FROM workspace_settings WHERE auto_invite_domain = $1",
+        domain
+    )
+    .fetch_all(&mut tx)
+    .await?;
+    for r in workspaces {
+        sqlx::query!(
+            "INSERT INTO workspace_invite
+                (workspace_id, email, is_admin, operator)
+                VALUES ($1, $2, false, $3)
+                ON CONFLICT DO NOTHING",
+            r.workspace_id,
+            email,
+            r.auto_invite_operator
+        )
+        .execute(&mut tx)
+        .await?;
+    }
+    tx.commit().await?;
+    Ok(())
+}
+
 async fn invite_user(
     Authed { username, is_admin, .. }: Authed,
     Extension(db): Extension<DB>,
@@ -459,11 +611,12 @@ async fn invite_user(
 
     sqlx::query!(
         "INSERT INTO workspace_invite
-            (workspace_id, email, is_admin)
-            VALUES ($1, $2, $3)",
+            (workspace_id, email, is_admin, operator)
+            VALUES ($1, $2, $3, $4)",
         &w_id,
         nu.email,
-        nu.is_admin
+        nu.is_admin,
+        nu.operator
     )
     .execute(&mut tx)
     .await?;
@@ -488,10 +641,11 @@ async fn delete_invite(
 
     sqlx::query!(
         "DELETE FROM workspace_invite WHERE
-        workspace_id = $1 AND email = $2 AND is_admin = $3",
+        workspace_id = $1 AND email = $2 AND is_admin = $3 AND operator = $4",
         &w_id,
         nu.email,
-        nu.is_admin
+        nu.is_admin,
+        nu.operator
     )
     .execute(&mut tx)
     .await?;
@@ -547,6 +701,22 @@ async fn tarball_workspace(
     let file = File::create(&file_path).await?;
     let mut a = tokio_tar::Builder::new(file);
 
+    {
+        let folders = sqlx::query_as::<_, Folder>("SELECT * FROM folder WHERE workspace_id = $1")
+            .bind(&w_id)
+            .fetch_all(&db)
+            .await?;
+
+        for folder in folders {
+            write_to_archive(
+                serde_json::to_string_pretty(&folder).unwrap(),
+                format!("f/{}/folder.meta.json", folder.name),
+                &mut a,
+            )
+            .await?;
+        }
+    }
+
     {
         let scripts = sqlx::query_as::<_, Script>(
             "SELECT * FROM script as o WHERE workspace_id = $1 AND archived = false
@@ -564,12 +734,7 @@ async fn tarball_workspace(
                 ScriptLang::Go => "go",
                 ScriptLang::Bash => "sh",
             };
-            write_to_archive(
-                script.content,
-                format!("scripts/{}.{}", script.path, ext),
-                &mut a,
-            )
-            .await?;
+            write_to_archive(script.content, format!("{}.{}", script.path, ext), &mut a).await?;
 
             let lock = script
                 .lock
@@ -585,12 +750,7 @@ async fn tarball_workspace(
                 lock,
             };
             let metadata_str = serde_json::to_string_pretty(&metadata).unwrap();
-            write_to_archive(
-                metadata_str,
-                format!("scripts/{}.json", script.path),
-                &mut a,
-            )
-            .await?;
+            write_to_archive(metadata_str, format!("{}.script.json", script.path), &mut a).await?;
         }
     }
 
@@ -607,7 +767,7 @@ async fn tarball_workspace(
             let resource_str = serde_json::to_string_pretty(&resource).unwrap();
             write_to_archive(
                 resource_str,
-                format!("resources/{}.json", resource.path),
+                format!("{}.resource.json", resource.path),
                 &mut a,
             )
             .await?;
@@ -627,7 +787,7 @@ async fn tarball_workspace(
             let resource_str = serde_json::to_string_pretty(&resource_type).unwrap();
             write_to_archive(
                 resource_str,
-                format!("resource_types/{}.json", resource_type.name),
+                format!("{}.resource-type.json", resource_type.name),
                 &mut a,
             )
             .await?;
@@ -644,12 +804,12 @@ async fn tarball_workspace(
 
         for flow in flows {
             let flow_str = serde_json::to_string_pretty(&flow).unwrap();
-            write_to_archive(flow_str, format!("flows/{}.json", flow.path), &mut a).await?;
+            write_to_archive(flow_str, format!("{}.flow.json", flow.path), &mut a).await?;
         }
     }
 
     {
-        let variables = sqlx::query_as::<_, ListableVariable>(
+        let variables = sqlx::query_as::<_, ExportableListableVariable>(
             "SELECT *, false as is_expired FROM variable WHERE workspace_id = $1 AND is_secret = false",
         )
         .bind(&w_id)
@@ -658,7 +818,7 @@ async fn tarball_workspace(
 
         for var in variables {
             let flow_str = serde_json::to_string_pretty(&var).unwrap();
-            write_to_archive(flow_str, format!("variables/{}.json", var.path), &mut a).await?;
+            write_to_archive(flow_str, format!("{}.variable.json", var.path), &mut a).await?;
         }
     }
     a.into_inner().await?;

backend/windmill-common/src/apps.rs

@@ -0,0 +1,14 @@
+/*
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
+ * This file and its contents are licensed under the AGPLv3 License.
+ * Please see the included NOTICE for copyright information and
+ * LICENSE-AGPL for a copy of the license.
+ */
+
+use serde::Deserialize;
+
+#[derive(Deserialize)]
+pub struct ListAppQuery {
+    pub starred_only: Option<bool>,
+}

backend/windmill-common/src/flow_status.rs

@@ -24,7 +24,7 @@ pub const MAX_RETRY_INTERVAL: Duration = HOURS.saturating_mul(6);
 pub struct FlowStatus {
     pub step: i32,
     pub modules: Vec<FlowStatusModule>,
-    pub failure_module: FlowStatusModule,
+    pub failure_module: FlowStatusModuleWParent,
     #[serde(default)]
     #[serde(skip_serializing_if = "is_default")]
     pub retry: RetryStatus,
@@ -67,6 +67,14 @@ pub struct Approval {
     pub approver: String,
 }
 
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct FlowStatusModuleWParent {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub parent_module: Option<String>,
+    #[serde(flatten)]
+    pub module_status: FlowStatusModule,
+}
+
 #[derive(Serialize, Deserialize, Debug, Clone)]
 #[serde(tag = "type")]
 pub enum FlowStatusModule {
@@ -171,12 +179,15 @@ impl FlowStatus {
                 .iter()
                 .map(|m| FlowStatusModule::WaitingForPriorSteps { id: m.id.clone() })
                 .collect(),
-            failure_module: FlowStatusModule::WaitingForPriorSteps {
-                id: f
-                    .failure_module
-                    .as_ref()
-                    .map(|x| x.id.clone())
-                    .unwrap_or_else(|| "failure".to_string()),
+            failure_module: FlowStatusModuleWParent {
+                parent_module: None,
+                module_status: FlowStatusModule::WaitingForPriorSteps {
+                    id: f
+                        .failure_module
+                        .as_ref()
+                        .map(|x| x.id.clone())
+                        .unwrap_or_else(|| "failure".to_string()),
+                },
             },
             retry: RetryStatus { fail_count: 0, previous_result: None, failed_jobs: vec![] },
         }

backend/windmill-common/src/flows.rs

@@ -12,7 +12,7 @@ use serde::{self, Deserialize, Serialize};
 
 use crate::{
     more_serde::{default_false, default_id, default_true, is_default},
-    scripts::{Schema, ScriptLang},
+    scripts::{Schema, ScriptHash, ScriptLang},
 };
 
 #[derive(Serialize)]
@@ -30,6 +30,20 @@ pub struct Flow {
     pub extra_perms: serde_json::Value,
 }
 
+#[derive(Serialize)]
+#[cfg_attr(feature = "sqlx", derive(sqlx::FromRow))]
+pub struct ListableFlow {
+    pub workspace_id: String,
+    pub path: String,
+    pub summary: String,
+    pub description: String,
+    pub edited_by: String,
+    pub edited_at: chrono::DateTime<chrono::Utc>,
+    pub archived: bool,
+    pub extra_perms: serde_json::Value,
+    pub starred: bool,
+}
+
 #[derive(Deserialize)]
 #[cfg_attr(feature = "sqlx", derive(sqlx::FromRow))]
 pub struct NewFlow {
@@ -146,6 +160,12 @@ pub struct FlowModule {
     pub sleep: Option<InputTransform>,
 }
 
+impl FlowModule {
+    pub fn id_append(&mut self, s: &str) {
+        self.id = format!("{}-{}", self.id, s);
+    }
+}
+
 #[derive(Deserialize, Serialize, Debug, Clone, PartialEq)]
 #[serde(
     tag = "type",
@@ -186,6 +206,14 @@ pub enum FlowModuleValue {
         #[serde(alias = "input_transform")]
         input_transforms: HashMap<String, InputTransform>,
         path: String,
+        #[serde(skip_serializing_if = "Option::is_none")]
+        hash: Option<ScriptHash>,
+    },
+    Flow {
+        #[serde(default)]
+        #[serde(alias = "input_transform")]
+        input_transforms: HashMap<String, InputTransform>,
+        path: String,
     },
     ForloopFlow {
         iterator: InputTransform,
@@ -224,4 +252,5 @@ pub struct ListFlowQuery {
     pub show_archived: Option<bool>,
     pub order_by: Option<String>,
     pub order_desc: Option<bool>,
+    pub starred_only: Option<bool>,
 }

backend/windmill-common/src/lib.rs

@@ -10,12 +10,14 @@ use std::net::SocketAddr;
 
 use error::Error;
 
+pub mod apps;
 pub mod error;
 pub mod external_ip;
 pub mod flow_status;
 pub mod flows;
 pub mod more_serde;
 pub mod oauth2;
+pub mod schedule;
 pub mod scripts;
 pub mod users;
 pub mod utils;
@@ -27,7 +29,8 @@ pub mod tracing_init;
 pub const DEFAULT_NUM_WORKERS: usize = 3;
 pub const DEFAULT_TIMEOUT: i32 = 300;
 pub const DEFAULT_SLEEP_QUEUE: u64 = 50;
-pub const DEFAULT_MAX_CONNECTIONS: u32 = 100;
+pub const DEFAULT_MAX_CONNECTIONS_SERVER: u32 = 50;
+pub const DEFAULT_MAX_CONNECTIONS_WORKER: u32 = 3;
 
 #[cfg(feature = "tokio")]
 pub async fn shutdown_signal(tx: tokio::sync::broadcast::Sender<()>) -> anyhow::Result<()> {
@@ -77,7 +80,7 @@ async fn metrics() -> Result<String, Error> {
 }
 
 #[cfg(feature = "sqlx")]
-pub async fn connect_db() -> anyhow::Result<sqlx::Pool<sqlx::Postgres>> {
+pub async fn connect_db(server_mode: bool) -> anyhow::Result<sqlx::Pool<sqlx::Postgres>> {
     use anyhow::Context;
 
     let database_url = std::env::var("DATABASE_URL")
@@ -85,7 +88,13 @@ pub async fn connect_db() -> anyhow::Result<sqlx::Pool<sqlx::Postgres>> {
 
     let max_connections = match std::env::var("DATABASE_CONNECTIONS") {
         Ok(n) => n.parse::<u32>().context("invalid DATABASE_CONNECTIONS")?,
-        Err(_) => DEFAULT_MAX_CONNECTIONS,
+        Err(_) => {
+            if server_mode {
+                DEFAULT_MAX_CONNECTIONS_SERVER
+            } else {
+                DEFAULT_MAX_CONNECTIONS_WORKER
+            }
+        }
     };
 
     Ok(connect(&database_url, max_connections).await?)
@@ -99,6 +108,7 @@ pub async fn connect(
     use std::time::Duration;
 
     sqlx::postgres::PgPoolOptions::new()
+        .min_connections(3)
         .max_connections(max_connections)
         .max_lifetime(Duration::from_secs(30 * 60)) // 30 mins
         .connect(database_url)

backend/windmill-common/src/schedule.rs

@@ -0,0 +1,28 @@
+/*
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
+ * This file and its contents are licensed under the AGPLv3 License.
+ * Please see the included NOTICE for copyright information and
+ * LICENSE-AGPL for a copy of the license.
+ */
+
+use chrono::DateTime;
+use serde::{Deserialize, Serialize};
+use sqlx::FromRow;
+
+#[derive(FromRow, Serialize, Deserialize, Debug)]
+pub struct Schedule {
+    pub workspace_id: String,
+    pub path: String,
+    pub edited_by: String,
+    pub edited_at: DateTime<chrono::Utc>,
+    pub schedule: String,
+    pub offset_: i32,
+    pub enabled: bool,
+    pub script_path: String,
+    pub is_flow: bool,
+    pub args: Option<serde_json::Value>,
+    pub extra_perms: serde_json::Value,
+    pub email: String,
+    pub error: Option<String>,
+}

backend/windmill-common/src/scripts.rs

@@ -126,6 +126,27 @@ pub struct Script {
     pub kind: ScriptKind,
 }
 
+#[derive(Serialize)]
+#[cfg_attr(feature = "sqlx", derive(sqlx::FromRow))]
+pub struct ListableScript {
+    pub workspace_id: String,
+    pub hash: ScriptHash,
+    pub path: String,
+    pub parent_hashes: Option<ScriptHashes>,
+    pub summary: String,
+    pub description: String,
+    pub created_by: String,
+    pub created_at: chrono::DateTime<chrono::Utc>,
+    pub archived: bool,
+    pub deleted: bool,
+    pub is_template: bool,
+    pub extra_perms: serde_json::Value,
+    pub lock_error_logs: Option<String>,
+    pub language: ScriptLang,
+    pub kind: ScriptKind,
+    pub starred: bool,
+}
+
 #[derive(Serialize, Deserialize, Debug)]
 #[cfg_attr(feature = "sqlx", derive(sqlx::Type))]
 #[cfg_attr(feature = "sqlx", sqlx)]
@@ -168,6 +189,7 @@ pub struct ListScriptQuery {
     pub order_desc: Option<bool>,
     pub is_template: Option<bool>,
     pub kind: Option<String>,
+    pub starred_only: Option<bool>,
 }
 
 pub fn to_i64(s: &str) -> crate::error::Result<i64> {
@@ -186,11 +208,9 @@ pub fn to_hex_string(i: &i64) -> String {
 
 #[cfg(feature = "reqwest")]
 pub async fn get_hub_script_by_path(
-    email: Option<String>,
-    username: String,
+    email: &str,
     path: StripPath,
     http_client: reqwest::Client,
-    host: String,
 ) -> crate::error::Result<String> {
     use crate::{
         error::{to_anyhow, Error},
@@ -206,8 +226,6 @@ pub async fn get_hub_script_by_path(
         http_client,
         &format!("https://hub.windmill.dev/raw/{path}.ts"),
         email,
-        username,
-        host,
         true,
     )
     .await?
@@ -219,11 +237,9 @@ pub async fn get_hub_script_by_path(
 
 #[cfg(feature = "reqwest")]
 pub async fn get_full_hub_script_by_path(
-    email: Option<String>,
-    username: String,
+    email: &str,
     path: StripPath,
     http_client: reqwest::Client,
-    host: String,
 ) -> crate::error::Result<HubScript> {
     use crate::{
         error::{to_anyhow, Error},
@@ -239,8 +255,6 @@ pub async fn get_full_hub_script_by_path(
         http_client,
         &format!("https://hub.windmill.dev/raw2/{path}"),
         email,
-        username,
-        host,
         true,
     )
     .await?

backend/windmill-common/src/tracing_init.rs

@@ -31,6 +31,7 @@ fn filter_metadata(meta: &Metadata) -> bool {
 }
 
 pub fn initialize_tracing() {
+    let style = std::env::var("RUST_LOG_STYLE").unwrap_or_else(|_| "auto".into());
     let json_fmt = std::env::var("JSON_FMT")
         .map(|x| x == "true")
         .unwrap_or(false);
@@ -44,7 +45,11 @@ pub fn initialize_tracing() {
             .with(json_layer().with_filter(filter_fn(filter_metadata)))
             .init(),
         false => ts_base
-            .with(compact_layer().with_filter(filter_fn(filter_metadata)))
+            .with(
+                compact_layer()
+                    .with_ansi(style.to_lowercase() != "never")
+                    .with_filter(filter_fn(filter_metadata)),
+            )
             .init(),
     }
 }

backend/windmill-common/src/utils.rs

@@ -80,11 +80,9 @@ pub fn get_owner_from_path(path: &str) -> String {
 pub async fn list_elems_from_hub(
     http_client: reqwest::Client,
     url: &str,
-    email: Option<String>,
-    username: String,
-    host: String,
+    email: &str,
 ) -> Result<serde_json::Value> {
-    let rows = http_get_from_hub(http_client, url, email, username, host, false)
+    let rows = http_get_from_hub(http_client, url, email, false)
         .await?
         .json::<serde_json::Value>()
         .await
@@ -96,9 +94,7 @@ pub async fn list_elems_from_hub(
 pub async fn http_get_from_hub(
     http_client: reqwest::Client,
     url: &str,
-    email: Option<String>,
-    username: String,
-    host: String,
+    email: &str,
     plain: bool,
 ) -> Result<reqwest::Response> {
     let response = http_client
@@ -111,9 +107,7 @@ pub async fn http_get_from_hub(
                 "application/json"
             },
         )
-        .header("X-email", email.unwrap_or_else(|| "".to_string()))
-        .header("X-username", username)
-        .header("X-hostname", host)
+        .header("X-email", email)
         .send()
         .await
         .map_err(crate::error::to_anyhow)?;

backend/windmill-common/src/variables.rs

@@ -27,7 +27,25 @@ pub struct ListableVariable {
     pub description: String,
     pub extra_perms: serde_json::Value,
     pub account: Option<i32>,
-    pub is_oauth: bool,
+    pub is_oauth: Option<bool>,
+    pub is_expired: Option<bool>,
+    pub is_refreshed: Option<bool>,
+    pub refresh_error: Option<String>,
+    pub is_linked: Option<bool>,
+}
+
+#[derive(Serialize, Deserialize)]
+#[cfg_attr(feature = "sqlx", derive(sqlx::FromRow))]
+
+pub struct ExportableListableVariable {
+    pub workspace_id: String,
+    pub path: String,
+    pub value: Option<String>,
+    pub is_secret: bool,
+    pub description: String,
+    pub extra_perms: serde_json::Value,
+    pub account: Option<i32>,
+    pub is_oauth: Option<bool>,
     pub is_expired: Option<bool>,
 }
 
@@ -53,7 +71,24 @@ pub fn get_reserved_variables(
     flow_id: Option<String>,
     flow_path: Option<String>,
     schedule_path: Option<String>,
-) -> [ContextualVariable; 11] {
+) -> [ContextualVariable; 12] {
+    let state_path = {
+        let flow_path = flow_path
+            .clone()
+            .unwrap_or_else(|| "NO_FLOW_PATH".to_string());
+        let script_path = path.clone().unwrap_or_else(|| "NO_JOB_PATH".to_string());
+        let schedule_path = schedule_path
+            .clone()
+            .map(|x| format!("/{x}"))
+            .unwrap_or_else(String::new);
+
+        let script_path = if script_path.ends_with("/") {
+            "NO_NAME".to_string()
+        } else {
+            script_path
+        };
+        format!("{permissioned_as}/{flow_path}/{script_path}{schedule_path}")
+    };
     [
         ContextualVariable {
             name: "WM_WORKSPACE".to_string(),
@@ -114,5 +149,10 @@ pub fn get_reserved_variables(
             value: permissioned_as.to_string(),
             description: "Fully Qualified (u/g) owner name of executor of the job".to_string(),
         },
+        ContextualVariable {
+            name: "WM_STATE_PATH".to_string(),
+            value: state_path,
+            description: "State resource path unique to a script and its trigger".to_string(),
+        },
     ]
 }

backend/windmill-queue/Cargo.toml

@@ -8,6 +8,10 @@ edition.workspace = true
 name = "windmill_queue"
 path = "src/lib.rs"
 
+[features]
+default = []
+enterprise = []
+
 [dependencies]
 windmill-audit.workspace = true
 windmill-common = { workspace = true, features = ["sqlx", "reqwest"] }

backend/windmill-queue/src/jobs.rs

@@ -10,14 +10,14 @@ use std::{collections::HashMap, str::FromStr};
 
 use serde::{Deserialize, Serialize};
 use sqlx::{Pool, Postgres, Transaction};
-use tracing::instrument;
+use tracing::{instrument, Instrument};
 use ulid::Ulid;
 use uuid::Uuid;
 use windmill_audit::{audit_log, ActionKind};
 use windmill_common::{
     error::{self, to_anyhow, Error},
     flow_status::{FlowStatus, JobResult, MAX_RETRY_ATTEMPTS, MAX_RETRY_INTERVAL},
-    flows::FlowValue,
+    flows::{FlowModule, FlowModuleValue, FlowValue},
     scripts::{get_full_hub_script_by_path, HubScript, ScriptHash, ScriptLang},
     utils::StripPath,
 };
@@ -39,10 +39,11 @@ lazy_static::lazy_static! {
         "Total number of jobs pulled from the queue."
     )
     .unwrap();
+    pub static ref CLOUD_HOSTED: bool = std::env::var("CLOUD_HOSTED").is_ok();
+
 }
 
-const MAX_NB_OF_JOBS_IN_Q_PER_USER: i64 = 10;
-const MAX_DURATION_LAST_1200: std::time::Duration = std::time::Duration::from_secs(900);
+const MAX_FREE_EXECS: i32 = 1000;
 
 pub async fn cancel_job<'c>(
     username: &str,
@@ -243,175 +244,186 @@ pub async fn push<'c>(
     mut tx: Transaction<'c, Postgres>,
     workspace_id: &str,
     job_payload: JobPayload,
-    args: Option<serde_json::Map<String, serde_json::Value>>,
+    args: serde_json::Map<String, serde_json::Value>,
     user: &str,
+    email: &str,
     permissioned_as: String,
     scheduled_for_o: Option<chrono::DateTime<chrono::Utc>>,
     schedule_path: Option<String>,
     parent_job: Option<Uuid>,
     is_flow_step: bool,
     mut same_worker: bool,
+    pre_run_error: Option<&windmill_common::error::Error>,
+    visible_to_owner: bool,
 ) -> Result<(Uuid, Transaction<'c, Postgres>), Error> {
     let scheduled_for = scheduled_for_o.unwrap_or_else(chrono::Utc::now);
-    let args_json = args.map(serde_json::Value::Object);
+    let args_json = serde_json::Value::Object(args);
     let job_id: Uuid = Ulid::new().into();
 
-    let premium_workspace =
-        sqlx::query_scalar!("SELECT premium FROM workspace WHERE id = $1", workspace_id)
-            .fetch_one(&mut tx)
-            .await
-            .map_err(|e| {
-                Error::InternalErr(format!("fetching if {workspace_id} is premium: {e}"))
-            })?;
+    if cfg!(feature = "enterprise") {
+        let premium_workspace =
+            sqlx::query_scalar!("SELECT premium FROM workspace WHERE id = $1", workspace_id)
+                .fetch_one(&mut tx)
+                .await
+                .map_err(|e| {
+                    Error::InternalErr(format!("fetching if {workspace_id} is premium: {e}"))
+                })?;
 
-    if !premium_workspace && std::env::var("CLOUD_HOSTED").is_ok() {
-        let rate_limiting_queue = sqlx::query_scalar!(
-            "SELECT COUNT(id) FROM queue WHERE permissioned_as = $1 AND workspace_id = $2",
-            permissioned_as,
-            workspace_id
-        )
-        .fetch_one(&mut tx)
-        .await?;
-
-        if let Some(nb_jobs) = rate_limiting_queue {
-            if nb_jobs > MAX_NB_OF_JOBS_IN_Q_PER_USER {
-                return Err(error::Error::ExecutionErr(format!(
-                    "You have exceeded the number of authorized elements of queue at any given \
-                     time: {}",
-                    MAX_NB_OF_JOBS_IN_Q_PER_USER
-                )));
+        // we track only non flow steps
+        let usage = if !matches!(
+            job_payload,
+            JobPayload::Flow(_) | JobPayload::RawFlow { .. }
+        ) {
+            if !premium_workspace {
+                sqlx::query_scalar!(
+                    "INSERT INTO usage (id, is_workspace, month_, usage) 
+                    VALUES ($1, false, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), 0) 
+                    ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + 1 
+                    RETURNING usage.usage",
+                    email)
+                .fetch_one(&mut tx)
+                .await
+                .map_err(|e| Error::InternalErr(format!("updating usage: {e}")))?
+            } else {
+                sqlx::query_scalar!(
+                    "INSERT INTO usage (id, is_workspace, month_, usage) 
+                    VALUES ($1, true, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), 0) 
+                    ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + 1 
+                    RETURNING usage.usage",
+                    workspace_id)
+                .fetch_one(&mut tx)
+                .await
+                .map_err(|e| Error::InternalErr(format!("updating usage: {e}")))?
             }
-        }
+        } else if *CLOUD_HOSTED && !premium_workspace {
+            sqlx::query_scalar!(
+                "
+        SELECT usage.usage + 1 FROM usage 
+        WHERE is_workspace = false AND
+     month_ = EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date)
+     AND id = $1",
+                email
+            )
+            .fetch_optional(&mut tx)
+            .await?
+            .flatten()
+            .unwrap_or(0)
+        } else {
+            0
+        };
 
-        let rate_limiting_duration_ms = sqlx::query_scalar!(
-            "
-           SELECT SUM(duration_ms)
-             FROM completed_job
-            WHERE permissioned_as = $1
-              AND created_at > NOW() - INTERVAL '1200 seconds'
-              AND workspace_id = $2",
-            permissioned_as,
-            workspace_id
-        )
-        .fetch_one(&mut tx)
-        .await?;
-
-        if let Some(sum_duration_ms) = rate_limiting_duration_ms {
-            if sum_duration_ms as u128 > MAX_DURATION_LAST_1200.as_millis() {
-                return Err(error::Error::ExecutionErr(format!(
-                    "You have exceeded the scripts cumulative duration limit over the last 20m \
-                     which is: {} seconds",
-                    MAX_DURATION_LAST_1200.as_secs()
+        if *CLOUD_HOSTED && !premium_workspace {
+            let is_super_admin =
+                sqlx::query_scalar!("SELECT super_admin FROM password WHERE email = $1", email)
+                    .fetch_optional(&mut tx)
+                    .await?
+                    .unwrap_or(false);
+            if !is_super_admin && usage > MAX_FREE_EXECS {
+                return Err(error::Error::BadRequest(format!(
+                    "User {email} has exceeded the free usage limit of {MAX_FREE_EXECS} that applies outside of premium workspaces."
                 )));
             }
         }
     }
 
-    let (script_hash, script_path, raw_code_tuple, job_kind, raw_flow, language) = match job_payload
-    {
-        JobPayload::ScriptHash { hash, path } => {
-            let language = sqlx::query_scalar!(
-                "SELECT language as \"language: ScriptLang\" FROM script WHERE hash = $1 AND \
+    let (script_hash, script_path, raw_code_tuple, job_kind, mut raw_flow, language) =
+        match job_payload {
+            JobPayload::ScriptHash { hash, path } => {
+                let language = sqlx::query_scalar!(
+                    "SELECT language as \"language: ScriptLang\" FROM script WHERE hash = $1 AND \
                  (workspace_id = $2 OR workspace_id = 'starter')",
-                hash.0,
-                workspace_id
-            )
-            .fetch_one(&mut tx)
-            .await
-            .map_err(|e| {
-                Error::InternalErr(format!(
-                    "fetching language for hash {hash} in {workspace_id}: {e}"
-                ))
-            })?;
-            (
-                Some(hash.0),
-                Some(path),
+                    hash.0,
+                    workspace_id
+                )
+                .fetch_one(&mut tx)
+                .await
+                .map_err(|e| {
+                    Error::InternalErr(format!(
+                        "fetching language for hash {hash} in {workspace_id}: {e}"
+                    ))
+                })?;
+                (
+                    Some(hash.0),
+                    Some(path),
+                    None,
+                    JobKind::Script,
+                    None,
+                    Some(language),
+                )
+            }
+            JobPayload::ScriptHub { path } => {
+                let script = get_hub_script(path.clone(), email).await?;
+                (
+                    None,
+                    Some(path),
+                    Some((script.content, script.lockfile)),
+                    JobKind::Script_Hub,
+                    None,
+                    Some(script.language),
+                )
+            }
+            JobPayload::Code(RawCode { content, path, language, lock }) => (
                 None,
-                JobKind::Script,
+                path,
+                Some((content, lock)),
+                JobKind::Preview,
                 None,
                 Some(language),
-            )
-        }
-        JobPayload::ScriptHub { path } => {
-            let email = sqlx::query_scalar!(
-                "SELECT email FROM usr WHERE username = $1 AND workspace_id = $2",
-                user,
-                workspace_id
-            )
-            .fetch_optional(&mut tx)
-            .await?;
-            let script = get_hub_script(path.clone(), email, user).await?;
-            (
+            ),
+            JobPayload::Dependencies { hash, dependencies, language } => (
+                Some(hash.0),
                 None,
-                Some(path),
-                Some((script.content, script.lockfile)),
-                JobKind::Script_Hub,
+                Some((dependencies, None)),
+                JobKind::Dependencies,
                 None,
-                Some(script.language),
-            )
-        }
-        JobPayload::Code(RawCode { content, path, language, lock }) => (
-            None,
-            path,
-            Some((content, lock)),
-            JobKind::Preview,
-            None,
-            Some(language),
-        ),
-        JobPayload::Dependencies { hash, dependencies, language } => (
-            Some(hash.0),
-            None,
-            Some((dependencies, None)),
-            JobKind::Dependencies,
-            None,
-            Some(language),
-        ),
-        JobPayload::FlowDependencies { path } => {
-            let value_json = sqlx::query_scalar!(
+                Some(language),
+            ),
+            JobPayload::FlowDependencies { path } => {
+                let value_json = sqlx::query_scalar!(
                 "SELECT value FROM flow WHERE path = $1 AND (workspace_id = $2 OR workspace_id = \
                  'starter')",
                 path,
                 workspace_id
             )
-            .fetch_optional(&mut tx)
-            .await?
-            .ok_or_else(|| Error::InternalErr(format!("not found flow at path {:?}", path)))?;
-            let value = serde_json::from_value::<FlowValue>(value_json).map_err(|err| {
-                Error::InternalErr(format!(
-                    "could not convert json to flow for {path}: {err:?}"
-                ))
-            })?;
-            (
-                None,
-                Some(path),
-                None,
-                JobKind::FlowDependencies,
-                Some(value),
-                None,
-            )
-        }
-        JobPayload::RawFlow { value, path } => {
-            (None, path, None, JobKind::FlowPreview, Some(value), None)
-        }
-        JobPayload::Flow(flow) => {
-            let value_json = sqlx::query_scalar!(
+                .fetch_optional(&mut tx)
+                .await?
+                .ok_or_else(|| Error::InternalErr(format!("not found flow at path {:?}", path)))?;
+                let value = serde_json::from_value::<FlowValue>(value_json).map_err(|err| {
+                    Error::InternalErr(format!(
+                        "could not convert json to flow for {path}: {err:?}"
+                    ))
+                })?;
+                (
+                    None,
+                    Some(path),
+                    None,
+                    JobKind::FlowDependencies,
+                    Some(value),
+                    None,
+                )
+            }
+            JobPayload::RawFlow { value, path } => {
+                (None, path, None, JobKind::FlowPreview, Some(value), None)
+            }
+            JobPayload::Flow(flow) => {
+                let value_json = sqlx::query_scalar!(
                 "SELECT value FROM flow WHERE path = $1 AND (workspace_id = $2 OR workspace_id = \
                  'starter')",
                 flow,
                 workspace_id
             )
-            .fetch_optional(&mut tx)
-            .await?
-            .ok_or_else(|| Error::InternalErr(format!("not found flow at path {:?}", flow)))?;
-            let value = serde_json::from_value::<FlowValue>(value_json).map_err(|err| {
-                Error::InternalErr(format!(
-                    "could not convert json to flow for {flow}: {err:?}"
-                ))
-            })?;
-            (None, Some(flow), None, JobKind::Flow, Some(value), None)
-        }
-        JobPayload::Identity => (None, None, None, JobKind::Identity, None, None),
-    };
+                .fetch_optional(&mut tx)
+                .await?
+                .ok_or_else(|| Error::InternalErr(format!("not found flow at path {:?}", flow)))?;
+                let value = serde_json::from_value::<FlowValue>(value_json).map_err(|err| {
+                    Error::InternalErr(format!(
+                        "could not convert json to flow for {flow}: {err:?}"
+                    ))
+                })?;
+                (None, Some(flow), None, JobKind::Flow, Some(value), None)
+            }
+            JobPayload::Identity => (None, None, None, JobKind::Identity, None, None),
+        };
 
     let is_running = same_worker;
     if let Some(flow) = raw_flow.as_ref() {
@@ -433,6 +445,27 @@ pub async fn push<'c>(
                 }
             }
         }
+
+        // If last module has a sleep or suspend, we insert a virtual identity module
+        if flow.modules.len() > 0
+            && (flow.modules[flow.modules.len() - 1].sleep.is_some()
+                || flow.modules[flow.modules.len() - 1].suspend.is_some())
+        {
+            let mut modules = flow.modules.clone();
+            modules.push(FlowModule {
+                id: "".to_string(),
+                value: FlowModuleValue::Identity,
+                input_transforms: HashMap::new(),
+                stop_after_if: None,
+                summary: Some(
+                    "Virtual module needed for suspend/sleep when last module".to_string(),
+                ),
+                retry: None,
+                sleep: None,
+                suspend: None,
+            });
+            raw_flow = Some(FlowValue { modules, ..flow.clone() });
+        }
     }
 
     let (raw_code, raw_lock) = raw_code_tuple
@@ -444,8 +477,8 @@ pub async fn push<'c>(
         "INSERT INTO queue
             (workspace_id, id, running, parent_job, created_by, permissioned_as, scheduled_for, 
                 script_hash, script_path, raw_code, raw_lock, args, job_kind, schedule_path, raw_flow, \
-         flow_status, is_flow_step, language, started_at, same_worker)
-            VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, CASE WHEN $3 THEN now() END, $19) \
+         flow_status, is_flow_step, language, started_at, same_worker, pre_run_error, email, visible_to_owner)
+            VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, CASE WHEN $3 THEN now() END, $19, $20, $21, $22) \
          RETURNING id",
         workspace_id,
         job_id,
@@ -465,7 +498,10 @@ pub async fn push<'c>(
         flow_status.map(|f| serde_json::json!(f)),
         is_flow_step,
         language: ScriptLang,
-        same_worker
+        same_worker,
+        pre_run_error.map(|e| e.to_string()),
+        email,
+        visible_to_owner
     )
     .fetch_one(&mut tx)
     .await
@@ -503,6 +539,7 @@ pub async fn push<'c>(
             script_path.as_ref().map(|x| x.as_str()),
             Some(hm),
         )
+        .instrument(tracing::info_span!("job_run", email = &email))
         .await?;
     }
     Ok((uuid, tx))
@@ -517,20 +554,14 @@ pub fn canceled_job_to_result(job: &QueuedJob) -> String {
     format!("Job canceled: {reason} by {canceler}")
 }
 
-pub async fn get_hub_script(
-    path: String,
-    email: Option<String>,
-    user: &str,
-) -> error::Result<HubScript> {
+pub async fn get_hub_script(path: String, email: &str) -> error::Result<HubScript> {
     get_full_hub_script_by_path(
         email,
-        user.to_string(),
         StripPath(path),
         reqwest::ClientBuilder::new()
             .user_agent("windmill/beta")
             .build()
             .map_err(to_anyhow)?,
-        std::env::var("BASE_URL").unwrap_or_else(|_| "".to_string()),
     )
     .await
     .map(|e| e)
@@ -564,6 +595,9 @@ pub struct QueuedJob {
     pub is_flow_step: bool,
     pub language: Option<ScriptLang>,
     pub same_worker: bool,
+    pub pre_run_error: Option<String>,
+    pub email: String,
+    pub visible_to_owner: bool,
 }
 
 impl QueuedJob {

backend/windmill-queue/src/schedule.rs

@@ -8,43 +8,16 @@
 
 use std::str::FromStr;
 
-use chrono::{DateTime, Duration, FixedOffset};
-use serde::{Deserialize, Serialize};
-use sqlx::{query_scalar, FromRow, Postgres, Transaction};
-use windmill_audit::{audit_log, ActionKind};
+use chrono::Duration;
+use sqlx::{query_scalar, Postgres, Transaction};
 use windmill_common::{
-    error::{self, Error, Result},
-    utils::{get_owner_from_path, not_found_if_none, now_from_db, paginate, Pagination, StripPath},
+    error::{self, Result},
+    schedule::Schedule,
+    utils::{get_owner_from_path, now_from_db, StripPath},
 };
 
 use crate::{push, JobPayload};
 
-#[derive(FromRow, Serialize, Deserialize, Debug)]
-pub struct Schedule {
-    pub workspace_id: String,
-    pub path: String,
-    pub edited_by: String,
-    pub edited_at: DateTime<chrono::Utc>,
-    pub schedule: String,
-    pub offset_: i32,
-    pub enabled: bool,
-    pub script_path: String,
-    pub is_flow: bool,
-    pub args: Option<serde_json::Value>,
-    pub extra_perms: serde_json::Value,
-}
-
-#[derive(Deserialize)]
-pub struct NewSchedule {
-    pub path: String,
-    pub schedule: String,
-    pub offset: i32,
-    pub script_path: String,
-    pub is_flow: bool,
-    pub args: Option<serde_json::Value>,
-    pub enabled: Option<bool>,
-}
-
 pub async fn push_scheduled_job<'c>(
     mut tx: Transaction<'c, Postgres>,
     schedule: Schedule,
@@ -74,11 +47,11 @@ pub async fn push_scheduled_job<'c>(
         return Ok(tx);
     }
 
-    let mut args: Option<serde_json::Map<String, serde_json::Value>> = None;
+    let mut args: serde_json::Map<String, serde_json::Value> = serde_json::Map::new();
 
     if let Some(args_v) = schedule.args {
         if let serde_json::Value::Object(args_m) = args_v {
-            args = Some(args_m)
+            args = args_m
         } else {
             return Err(error::Error::ExecutionErr(
                 "args of scripts needs to be dict".to_string(),
@@ -100,196 +73,33 @@ pub async fn push_scheduled_job<'c>(
         }
     };
 
-    let (_, tx) = push(
+    let (_, mut tx) = push(
         tx,
         &schedule.workspace_id,
         payload,
         args,
         &schedule_to_user(&schedule.path),
+        &schedule.email,
         get_owner_from_path(&schedule.path),
         Some(next),
-        Some(schedule.path),
+        Some(schedule.path.clone()),
         None,
         false,
         false,
+        None,
+        true,
     )
     .await?;
+    sqlx::query!(
+        "UPDATE schedule SET error = NULL WHERE workspace_id = $1 AND path = $2",
+        &schedule.workspace_id,
+        &schedule.path
+    )
+    .execute(&mut tx)
+    .await?;
     Ok(tx)
 }
 
-pub async fn create_schedule(
-    mut tx: Transaction<'_, Postgres>,
-    w_id: String,
-    ns: NewSchedule,
-    username: &str,
-) -> Result<String> {
-    cron::Schedule::from_str(&ns.schedule).map_err(|e| error::Error::BadRequest(e.to_string()))?;
-    check_flow_conflict(&mut tx, &w_id, &ns.path, ns.is_flow, &ns.script_path).await?;
-
-    let schedule = sqlx::query_as!(
-        Schedule,
-        "INSERT INTO schedule (workspace_id, path, schedule, offset_, edited_by, script_path, \
-         is_flow, args, enabled) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) RETURNING *",
-        w_id,
-        ns.path,
-        ns.schedule,
-        ns.offset,
-        username,
-        ns.script_path,
-        ns.is_flow,
-        ns.args,
-        ns.enabled.unwrap_or(false),
-    )
-    .fetch_one(&mut tx)
-    .await
-    .map_err(|e| Error::InternalErr(format!("inserting schedule in {w_id}: {e}")))?;
-
-    audit_log(
-        &mut tx,
-        username,
-        "schedule.create",
-        ActionKind::Create,
-        &w_id,
-        Some(&ns.path.to_string()),
-        Some(
-            [
-                Some(("schedule", ns.schedule.as_str())),
-                Some(("script_path", ns.script_path.as_str())),
-            ]
-            .into_iter()
-            .flatten()
-            .collect(),
-        ),
-    )
-    .await?;
-
-    if ns.enabled.unwrap_or(true) {
-        tx = push_scheduled_job(tx, schedule).await?
-    }
-    tx.commit().await?;
-
-    Ok(ns.path.to_string())
-}
-
-async fn check_flow_conflict<'c>(
-    tx: &mut Transaction<'c, Postgres>,
-    w_id: &str,
-    path: &str,
-    is_flow: bool,
-    script_path: &str,
-) -> error::Result<()> {
-    if path != script_path || !is_flow {
-        let exists_flow = sqlx::query_scalar!(
-            "SELECT EXISTS (SELECT 1 FROM flow WHERE path = $1 AND workspace_id = $2)",
-            path,
-            w_id
-        )
-        .fetch_one(tx)
-        .await?
-        .unwrap_or(false);
-        if exists_flow {
-            return Err(error::Error::BadConfig(format!(
-                "The path is the same as a flow, it can only trigger that flow.
-            However the provided path is: {script_path} and is_flow is {is_flow}"
-            )));
-        };
-    }
-    Ok(())
-}
-
-#[derive(Deserialize)]
-pub struct EditSchedule {
-    pub schedule: String,
-    pub script_path: String,
-    pub is_flow: bool,
-    pub args: Option<serde_json::Value>,
-}
-
-async fn clear_schedule<'c>(db: &mut Transaction<'c, Postgres>, path: &str) -> Result<()> {
-    sqlx::query!(
-        "DELETE FROM queue WHERE schedule_path = $1 AND running = false",
-        path
-    )
-    .execute(db)
-    .await?;
-    Ok(())
-}
-
-pub async fn edit_schedule(
-    mut tx: Transaction<'_, Postgres>,
-    w_id: String,
-    path: StripPath,
-    es: EditSchedule,
-    username: &String,
-) -> Result<String> {
-    let path = path.to_path();
-
-    cron::Schedule::from_str(&es.schedule).map_err(|e| error::Error::BadRequest(e.to_string()))?;
-
-    check_flow_conflict(&mut tx, &w_id, &path, es.is_flow, &es.script_path).await?;
-
-    clear_schedule(&mut tx, path).await?;
-    let schedule = sqlx::query_as!(
-        Schedule,
-        "UPDATE schedule SET schedule = $1, script_path = $2, is_flow = $3, args = $4 WHERE path \
-         = $5 AND workspace_id = $6 RETURNING *",
-        es.schedule,
-        es.script_path,
-        es.is_flow,
-        es.args,
-        path,
-        w_id,
-    )
-    .fetch_one(&mut tx)
-    .await
-    .map_err(|e| Error::InternalErr(format!("updating schedule in {w_id}: {e}")))?;
-
-    if schedule.enabled {
-        tx = push_scheduled_job(tx, schedule).await?;
-    }
-
-    audit_log(
-        &mut tx,
-        username,
-        "schedule.edit",
-        ActionKind::Update,
-        &w_id,
-        Some(&path.to_string()),
-        Some(
-            [
-                Some(("schedule", es.schedule.as_str())),
-                Some(("script_path", es.script_path.as_str())),
-            ]
-            .into_iter()
-            .flatten()
-            .collect(),
-        ),
-    )
-    .await?;
-    tx.commit().await?;
-
-    Ok(path.to_string())
-}
-
-pub async fn list_schedule(
-    mut tx: Transaction<'_, Postgres>,
-    w_id: String,
-    pagination: Pagination,
-) -> Result<Vec<Schedule>> {
-    let (per_page, offset) = paginate(pagination);
-    let rows = sqlx::query_as!(
-        Schedule,
-        "SELECT * FROM schedule WHERE workspace_id = $1 ORDER BY edited_at desc LIMIT $2 OFFSET $3",
-        w_id,
-        per_page as i64,
-        offset as i64
-    )
-    .fetch_all(&mut tx)
-    .await?;
-    tx.commit().await?;
-    Ok(rows)
-}
-
 pub async fn get_schedule_opt<'c>(
     db: &mut Transaction<'c, Postgres>,
     w_id: &str,
@@ -325,108 +135,6 @@ pub async fn exists_schedule(
     Ok(exists)
 }
 
-#[derive(Deserialize)]
-pub struct PreviewPayload {
-    pub schedule: String,
-    pub offset: Option<i32>,
-}
-
-pub fn preview_schedule(
-    PreviewPayload { schedule, offset }: PreviewPayload,
-) -> Result<Vec<DateTime<chrono::Utc>>> {
-    let schedule =
-        cron::Schedule::from_str(&schedule).map_err(|e| error::Error::BadRequest(e.to_string()))?;
-    let upcoming: Vec<DateTime<chrono::Utc>> = schedule
-        .upcoming(get_offset(offset))
-        .take(10)
-        .map(|x| x.into())
-        .collect();
-    Ok(upcoming)
-}
-
-fn get_offset(offset: Option<i32>) -> FixedOffset {
-    FixedOffset::west(offset.unwrap_or(0) * 60)
-}
-
-#[derive(Deserialize)]
-pub struct SetEnabled {
-    pub enabled: bool,
-}
-
-pub async fn set_enabled(
-    mut tx: Transaction<'_, Postgres>,
-    w_id: String,
-    path: StripPath,
-    SetEnabled { enabled }: SetEnabled,
-    username: &str,
-) -> Result<String> {
-    let path = path.to_path();
-    let schedule_o = sqlx::query_as!(
-        Schedule,
-        "UPDATE schedule SET enabled = $1 WHERE path = $2 AND workspace_id = $3 RETURNING *",
-        enabled,
-        path,
-        w_id
-    )
-    .fetch_optional(&mut tx)
-    .await?;
-
-    let schedule = not_found_if_none(schedule_o, "Schedule", path)?;
-
-    clear_schedule(&mut tx, path).await?;
-
-    if enabled {
-        tx = push_scheduled_job(tx, schedule).await?;
-    }
-    audit_log(
-        &mut tx,
-        username,
-        "schedule.setenabled",
-        ActionKind::Update,
-        &w_id,
-        Some(path),
-        Some([("enabled", enabled.to_string().as_ref())].into()),
-    )
-    .await?;
-    tx.commit().await?;
-    Ok(format!(
-        "succesfully updated schedule at path {} to status {}",
-        path, enabled
-    ))
-}
-
-pub async fn delete_schedule(
-    mut tx: Transaction<'_, Postgres>,
-    w_id: String,
-    path: StripPath,
-    username: &str,
-) -> Result<String> {
-    let path = path.to_path();
-
-    sqlx::query!(
-        "DELETE FROM schedule WHERE path = $1 AND workspace_id = $2",
-        path,
-        w_id
-    )
-    .execute(&mut tx)
-    .await?;
-
-    audit_log(
-        &mut tx,
-        username,
-        "schedule.delete",
-        ActionKind::Delete,
-        &w_id,
-        Some(path),
-        None,
-    )
-    .await?;
-
-    tx.commit().await?;
-
-    Ok(format!("schedule {} deleted", path))
-}
-
 fn schedule_to_user(path: &str) -> String {
     format!("schedule-{}", path.replace('/', "-"))
 }

backend/windmill-worker/Cargo.toml

@@ -12,6 +12,7 @@ path = "./src/main.rs"
 [features]
 default = []
 deno-lock = []
+enterprise = ["windmill-queue/enterprise"]
 
 [dependencies]
 windmill-queue.workspace = true
@@ -44,4 +45,7 @@ lazy_static.workspace = true
 chrono.workspace = true
 dotenv.workspace = true
 rand.workspace = true # TODO: Remove. only used by token creation hack.
-deno_core.workspace = true
+deno_core.workspace = true
+const_format.workspace = true
+git-version.workspace = true
+dyn-iter.workspace = true

backend/windmill-worker/nsjail/download.py.config.proto

@@ -46,28 +46,8 @@ mount {
 }
 
 mount {
-    src: "/etc/ssl"
-    dst: "/etc/ssl"
-	is_bind: true
-}
-
-mount {
-    src: "/etc/pki"
-    dst: "/etc/pki"
-    is_bind: true
-    mandatory: false
-}
-
-mount {
-    src: "/etc/ca-certificates"
-    dst: "/etc/ca-certificates"
-	is_bind: true
-    mandatory: false
-}
-
-mount {
-    src: "/etc/resolv.conf"
-    dst: "/etc/resolv.conf"
+    src: "/etc"
+    dst: "/etc"
 	is_bind: true
 }
 
@@ -86,33 +66,17 @@ mount {
 }
 
 
-mount {
-    src: "{JOB_DIR}/requirements.txt"
-    dst: "/user/requirements.txt"
-    is_bind: true
-}
-
-mount {
-    src: "{JOB_DIR}/dependencies"
-    dst: "/out"
-    is_bind: true
-    rw: true
-}
-
-
 mount {
     src: "{WORKER_DIR}/download_deps.py.sh"
     dst: "/download_deps.sh"
     is_bind: true
 }
 
-
 mount {
     src: "{CACHE_DIR}"
-    dst: "/tmp/.cache/pip"
+    dst: "{CACHE_DIR}"
     is_bind: true
     rw: true
-    mandatory: false
 }
 
 exec_bin {

backend/windmill-worker/nsjail/download_deps.py.sh

@@ -19,9 +19,6 @@ then
       echo "\$TRUSTED_HOST is set to $TRUSTED_HOST"
 fi
 
-mkdir -p /tmp/dependencies
-touch /tmp/dependencies/_windmill
-/usr/local/bin/python3 -m pip install --cache-dir /tmp/.cache/pip -t /tmp/dependencies -r /user/requirements.txt\
-    --no-color --no-deps --isolated --no-warn-conflicts --disable-pip-version-check $INDEX_URL_ARG $EXTRA_INDEX_URL_ARG $TRUSTED_HOST_ARG
-
-mv /tmp/dependencies/* /out
+CMD="/usr/local/bin/python3 -m pip install -v $REQ -I -t $TARGET --no-cache --no-color --no-deps --isolated --no-warn-conflicts --disable-pip-version-check $INDEX_URL_ARG $EXTRA_INDEX_URL_ARG $TRUSTED_HOST_ARG"
+echo $CMD
+eval $CMD

backend/windmill-worker/nsjail/run.bash.config.proto

@@ -66,21 +66,8 @@ mount {
 }
 
 mount {
-    src: "/etc/ssl"
-    dst: "/etc/ssl"
-	is_bind: true
-}
-
-mount {
-    src: "/etc/pki"
-    dst: "/etc/pki"
-    is_bind: true
-    mandatory: false
-}
-
-mount {
-    src: "/etc/resolv.conf"
-    dst: "/etc/resolv.conf"
+    src: "/etc"
+    dst: "/etc"
 	is_bind: true
 }
 
@@ -90,16 +77,14 @@ mount {
     is_bind: true
 }
 
-iface_no_lo: true
-
 mount {
-    src: "{CACHE_DIR}"
-    dst: "/tmp/.cache/go"
+    src: "/dev/urandom"
+    dst: "/dev/urandom"
     is_bind: true
-    rw: true
-    mandatory: false
 }
 
+iface_no_lo: true
+
 {SHARED_MOUNT}
 
 

backend/windmill-worker/nsjail/run.deno.config.proto

@@ -93,21 +93,8 @@ mount {
 }
 
 mount {
-    src: "/etc/ssl"
-    dst: "/etc/ssl"
-	is_bind: true
-}
-
-mount {
-    src: "/etc/pki"
-    dst: "/etc/pki"
-    is_bind: true
-    mandatory: false
-}
-
-mount {
-    src: "/etc/resolv.conf"
-    dst: "/etc/resolv.conf"
+    src: "/etc"
+    dst: "/etc"
 	is_bind: true
 }
 
@@ -117,6 +104,12 @@ mount {
     is_bind: true
 }
 
+mount {
+    src: "/dev/urandom"
+    dst: "/dev/urandom"
+    is_bind: true
+}
+
 mount {
     src: "{CACHE_DIR}"
     dst: "/tmp/.cache/deno"

backend/windmill-worker/nsjail/run.go.config.proto

@@ -58,38 +58,14 @@ mount {
     options: "size=500000000"
 }
 
+
 mount {
-    src: "{JOB_DIR}/go.sum"
-    dst: "/tmp/go/go.sum"
+    src: "{JOB_DIR}/main"
+    dst: "/tmp/go/main"
     is_bind: true
     mandatory: false
 }
 
-mount {
-    src: "{JOB_DIR}/go.mod"
-    dst: "/tmp/go/go.mod"
-    is_bind: true
-}
-
-mount {
-    src: "{JOB_DIR}/main.go"
-    dst: "/tmp/go/main.go"
-    is_bind: true
-    mandatory: false
-}
-
-mount {
-    src: "{JOB_DIR}/inner/inner_main.go"
-    dst: "/tmp/go/inner/inner_main.go"
-    is_bind: true
-}
-
-mount {
-    src: "{JOB_DIR}/inner/runner.go"
-    dst: "/tmp/go/inner/runner.go"
-    is_bind: true
-}
-
 mount {
     src: "{JOB_DIR}/args.json"
     dst: "/tmp/go/args.json"
@@ -103,23 +79,9 @@ mount {
     is_bind: true
 }
 
-
 mount {
-    src: "/etc/ssl"
-    dst: "/etc/ssl"
-	is_bind: true
-}
-
-mount {
-    src: "/etc/pki"
-    dst: "/etc/pki"
-    is_bind: true
-    mandatory: false
-}
-
-mount {
-    src: "/etc/resolv.conf"
-    dst: "/etc/resolv.conf"
+    src: "/etc"
+    dst: "/etc"
 	is_bind: true
 }
 
@@ -129,6 +91,12 @@ mount {
     is_bind: true
 }
 
+mount {
+    src: "/dev/urandom"
+    dst: "/dev/urandom"
+    is_bind: true
+}
+
 iface_no_lo: true
 
 mount {

backend/windmill-worker/nsjail/run.python3.config.proto

@@ -91,29 +91,9 @@ mount {
     is_bind: true
 }
 
-
 mount {
-    src: "{JOB_DIR}/dependencies"
-    dst: "/tmp/dependencies"
-    is_bind: true
-}
-
-mount {
-    src: "/etc/ssl"
-    dst: "/etc/ssl"
-	is_bind: true
-}
-
-mount {
-    src: "/etc/pki"
-    dst: "/etc/pki"
-    is_bind: true
-    mandatory: false
-}
-
-mount {
-    src: "/etc/resolv.conf"
-    dst: "/etc/resolv.conf"
+    src: "/etc"
+    dst: "/etc"
 	is_bind: true
 }
 
@@ -123,6 +103,12 @@ mount {
     is_bind: true
 }
 
+mount {
+    src: "/dev/urandom"
+    dst: "/dev/urandom"
+    is_bind: true
+}
+
 {SHARED_MOUNT}
 
 {SHARED_DEPENDENCIES}
@@ -130,7 +116,7 @@ mount {
 iface_no_lo: true
 
 envar: "LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH"
-envar: "PYTHONPATH=/tmp/dependencies{ADDITIONAL_PYTHON_PATHS}"
+envar: "PYTHONPATH={ADDITIONAL_PYTHON_PATHS}"
 envar: "HOME=/tmp"
 
 

backend/windmill-worker/src/jobs.rs

@@ -7,16 +7,15 @@
  */
 
 use serde_json::{Map, Value};
-use sqlx::{Pool, Postgres, Transaction};
+use sqlx::{Pool, Postgres};
 use tracing::instrument;
 use uuid::Uuid;
-use windmill_common::{error::Error, flow_status::FlowStatusModule};
-use windmill_queue::{delete_job, JobKind, QueuedJob};
+use windmill_common::{error::Error, flow_status::FlowStatusModule, schedule::Schedule};
+use windmill_queue::{delete_job, schedule::get_schedule_opt, JobKind, QueuedJob};
 
 #[instrument(level = "trace", skip_all)]
 pub async fn add_completed_job_error<E: ToString + std::fmt::Debug>(
     db: &Pool<Postgres>,
-    client: &windmill_api_client::Client,
     queued_job: &QueuedJob,
     logs: String,
     e: E,
@@ -27,7 +26,6 @@ pub async fn add_completed_job_error<E: ToString + std::fmt::Debug>(
     error_to_result(&mut output_map, &e);
     let a = add_completed_job(
         db,
-        client,
         &queued_job,
         false,
         false,
@@ -48,10 +46,27 @@ pub fn error_to_result<E: ToString + std::fmt::Debug>(
     );
 }
 
+fn flatten_jobs(modules: Vec<FlowStatusModule>) -> Vec<Uuid> {
+    modules
+        .into_iter()
+        .filter_map(|m| match m {
+            FlowStatusModule::Success { job, flow_jobs, .. }
+            | FlowStatusModule::Failure { job, flow_jobs, .. } => {
+                if let Some(flow_jobs) = flow_jobs {
+                    Some(flow_jobs)
+                } else {
+                    Some(vec![job])
+                }
+            }
+            _ => None,
+        })
+        .flatten()
+        .collect::<Vec<_>>()
+}
+
 #[instrument(level = "trace", skip_all)]
 pub async fn add_completed_job(
     db: &Pool<Postgres>,
-    client: &windmill_api_client::Client,
     queued_job: &QueuedJob,
     success: bool,
     skipped: bool,
@@ -62,15 +77,8 @@ pub async fn add_completed_job(
         if queued_job.job_kind == JobKind::Flow || queued_job.job_kind == JobKind::FlowPreview {
             let jobs = queued_job.parse_flow_status().map(|s| {
                 let mut modules = s.modules;
-                modules.extend([s.failure_module]);
-                modules
-                    .into_iter()
-                    .filter_map(|m| match m {
-                        FlowStatusModule::Success { job, .. }
-                        | FlowStatusModule::Failure { job, .. } => Some(job),
-                        _ => None,
-                    })
-                    .collect::<Vec<_>>()
+                modules.extend([s.failure_module.module_status]);
+                flatten_jobs(modules)
             });
             if let Some(jobs) = jobs {
                 sqlx::query_scalar!(
@@ -88,6 +96,7 @@ pub async fn add_completed_job(
         } else {
             None
         };
+
     let mut tx = db.begin().await?;
     let job_id = queued_job.id.clone();
     sqlx::query!(
@@ -117,9 +126,12 @@ pub async fn add_completed_job(
                    , raw_flow
                    , is_flow_step
                    , is_skipped
-                   , language )
+                   , language
+                   , email
+                   , visible_to_owner
+                )
             VALUES ($1, $2, $3, $4, $5, $6, COALESCE($26, EXTRACT(milliseconds FROM (now() - $6))), $7, $8, $9,\
-                    $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25)
+                    $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $27, $28)
          ON CONFLICT (id) DO UPDATE SET success = $7, result = $11, logs = concat(cj.logs, $12)",
         queued_job.workspace_id,
         queued_job.id,
@@ -146,60 +158,93 @@ pub async fn add_completed_job(
         queued_job.is_flow_step,
         skipped,
         queued_job.language: ScriptLang,
-        duration: Option<i64>
+        duration: Option<i64>,
+        queued_job.email,
+        queued_job.visible_to_owner
     )
     .execute(&mut tx)
     .await
     .map_err(|e| Error::InternalErr(format!("Could not add completed job {job_id}: {e}")))?;
     let _ = delete_job(db, &queued_job.workspace_id, job_id).await?;
+    tx.commit().await?;
+
+    if cfg!(enterprise) && duration.unwrap_or(0) > 1000 {
+        let additional_usage = duration.unwrap() as i32 / 1000;
+
+        let w_id = &queued_job.workspace_id;
+        let premium_workspace =
+            sqlx::query_scalar!("SELECT premium FROM workspace WHERE id = $1", w_id)
+                .fetch_one(db)
+                .await
+                .map_err(|e| Error::InternalErr(format!("fetching if {w_id} is premium: {e}")))?;
+        if premium_workspace {
+            let _ = sqlx::query!(
+                "INSERT INTO usage (id, is_workspace, month_, usage) 
+                VALUES ($1, true, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), 0) 
+                ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + $2",
+                w_id,
+                additional_usage)
+                .execute(db)
+                .await
+                .map_err(|e| Error::InternalErr(format!("updating usage: {e}")));
+        } else {
+            sqlx::query!(
+                    "INSERT INTO usage (id, is_workspace, month_, usage) 
+                    VALUES ($1, false, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), 0) 
+                    ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + $2",
+                    queued_job.email,
+                    additional_usage)
+                        .execute(db)
+                        .await
+                        .map_err(|e| Error::InternalErr(format!("updating usage: {e}")))?;
+        }
+    }
+
     if !queued_job.is_flow_step
         && queued_job.job_kind != JobKind::Flow
         && queued_job.job_kind != JobKind::FlowPreview
         && queued_job.schedule_path.is_some()
         && queued_job.script_path.is_some()
     {
-        tx = schedule_again_if_scheduled(
-            tx,
-            client,
+        schedule_again_if_scheduled(
+            db,
             queued_job.schedule_path.as_ref().unwrap(),
             queued_job.script_path.as_ref().unwrap(),
             &queued_job.workspace_id,
         )
         .await?;
     }
-    tx.commit().await?;
     tracing::debug!("Added completed job {}", queued_job.id);
     Ok(queued_job.id)
 }
 
 #[instrument(level = "trace", skip_all)]
-pub async fn schedule_again_if_scheduled<'c>(
-    mut tx: Transaction<'c, Postgres>,
-    client: &windmill_api_client::Client,
+pub async fn schedule_again_if_scheduled(
+    db: &Pool<Postgres>,
     schedule_path: &str,
     script_path: &str,
     w_id: &str,
-) -> windmill_common::error::Result<Transaction<'c, Postgres>> {
-    let schedule = client
-        .get_schedule(w_id, schedule_path)
-        .await
-        .map_err(|_| {
+) -> windmill_common::error::Result<()> {
+    let mut tx = db.begin().await?;
+
+    let schedule = get_schedule_opt(&mut tx, w_id, schedule_path)
+        .await?
+        .ok_or_else(|| {
             Error::InternalErr(format!(
                 "Could not find schedule {:?} for workspace {}",
                 schedule_path, w_id
             ))
-        })?
-        .into_inner();
+        })?;
     if schedule.enabled && script_path == schedule.script_path {
-        tx = windmill_queue::schedule::push_scheduled_job(
+        let res = windmill_queue::schedule::push_scheduled_job(
             tx,
-            windmill_queue::schedule::Schedule {
+            Schedule {
                 workspace_id: w_id.to_owned(),
-                path: schedule.path,
+                path: schedule.path.clone(),
                 edited_by: schedule.edited_by,
                 edited_at: schedule.edited_at,
                 schedule: schedule.schedule,
-                offset_: schedule.offset as _,
+                offset_: schedule.offset_,
                 enabled: schedule.enabled,
                 script_path: schedule.script_path,
                 is_flow: schedule.is_flow,
@@ -207,10 +252,25 @@ pub async fn schedule_again_if_scheduled<'c>(
                     .args
                     .and_then(|e| serde_json::to_value(e).map_or(None, |v| Some(v))),
                 extra_perms: serde_json::to_value(schedule.extra_perms).expect("hashmap -> json"),
+                email: schedule.email,
+                error: None,
             },
         )
-        .await?;
+        .await;
+        match res {
+            Ok(tx) => tx.commit().await?,
+            Err(e) => {
+                sqlx::query!(
+                    "UPDATE schedule SET enabled = false, error = $1 WHERE workspace_id = $2 AND path = $3",
+                    e.to_string(),
+                    &schedule.workspace_id,
+                    &schedule.path
+                )
+                .execute(db)
+                .await?;
+                tracing::warn!("Could not schedule job for {}: {}", schedule_path, e);
+            }
+        }
     }
-
-    Ok(tx)
+    Ok(())
 }

backend/windmill-worker/src/js_eval.rs

@@ -316,17 +316,17 @@ async fn op_get_id(args: Vec<String>) -> Result<Option<serde_json::Value>, anyho
 }
 
 #[op]
-async fn op_resource(
-    args: Vec<String>,
-) -> Result<windmill_api_client::types::Resource, anyhow::Error> {
+async fn op_resource(args: Vec<String>) -> Result<serde_json::Value, anyhow::Error> {
     let workspace = &args[0];
     let path = &args[1];
     let token = &args[2];
     let base_url = &args[3];
     let client = windmill_api_client::create_client(base_url, token.clone());
     let result = client.get_resource(workspace, path).await?;
-    // TODO: verify this works. Previously this returned Option<serde_jons::Value>, now it's statically typed.
-    Ok(result.into_inner())
+    Ok(result
+        .into_inner()
+        .value
+        .unwrap_or_else(|| serde_json::json!({})))
 }
 
 #[cfg(test)]