reformat package.json

* chore(main): release 1.62.0

* Apply automatic changes

---------

Co-authored-by: rubenfiszel <rubenfiszel@users.noreply.github.com>

.github/uffizzi/docker-compose.uffizzi.yml

@@ -14,13 +14,13 @@ services:
 
   windmill:
     image: '${WINDMILL_IMAGE}'
-    privileged: false
-    restart: unless-stopped
     ports:
       - 8000:8000
+    entrypoint: ['/bin/sh', '-c']
+    command: 'echo ${OAUTH_JSON_BASE64} | base64 --decode > /usr/src/app/oauth.json && ./windmill'
     environment:
       - DATABASE_URL=postgres://postgres:changeme@localhost/windmill?sslmode=disable
-      - BASE_URL=http://localhost
+      - BASE_URL=${EXPECTED_URL}
       - BASE_INTERNAL_URL=http://localhost:8000
       - RUST_LOG=info
       - NUM_WORKERS=3
@@ -28,12 +28,16 @@ services:
       - DENO_PATH=/usr/bin/deno
       - PYTHON_PATH=/usr/local/bin/python3
       - METRICS_ADDR=false
+      - OAUTH_JSON_BASE64=${OAUTH_JSON_BASE64}
     volumes:
       - worker_dependency_cache:/tmp/windmill/cache
+    deploy:
+      resources:
+        limits:
+          memory: 250M
 
   lsp:
     image: '${LSP_IMAGE}'
-    restart: unless-stopped
     ports:
       - 3001:3001
 

.github/workflows/automerge-dependabot.yml

@@ -13,7 +13,7 @@ jobs:
     steps:
       - name: Dependabot metadata
         id: metadata
-        uses: dependabot/fetch-metadata@v1.3.5
+        uses: dependabot/fetch-metadata@v1.3.6
         with:
           github-token: "${{ secrets.GITHUB_TOKEN }}"
       - name: Enable auto-merge for Dependabot PRs

.github/workflows/docker-310.yml

@@ -0,0 +1,47 @@
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+name: Build and push windmill with python 3.10 and openbb
+on: workflow_dispatch
+
+concurrency:
+  group: ${{ github.ref }}-openbb
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+  id-token: write
+  packages: write
+  
+jobs:
+  build_ee:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      # - name: Set up Docker Buildx
+      #   uses: docker/setup-buildx-action@v2
+
+      - uses: depot/setup-action@v1
+
+      - name: Login to registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push publicly ee
+        uses: depot/build-push-action@v1
+        with:
+          context: .
+          push: true
+          file: ./docker/DockerfileOpenbb
+          build-args: |
+            features=enterprise
+          tags: |
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:openbb
+          labels: |
+            org.opencontainers.image.licenses=Windmill-Enterprise-License

.github/workflows/uffizzi-build.yml

@@ -4,6 +4,8 @@ on:
     types: [opened,synchronize,reopened,closed]
     paths:
       - "backend/**"
+      - ".github/uffizzi/**"
+      - ".github/workflows/**"
   workflow_dispatch:
       
 jobs:

.github/workflows/uffizzi-preview.yml

@@ -47,22 +47,35 @@ jobs:
           echo 'EVENT_JSON<<EOF' >> $GITHUB_ENV
           cat event.json >> $GITHUB_ENV
           echo 'EOF' >> $GITHUB_ENV
+      - name: Read PR Number From Event Object
+        id: pr
+        run: echo "PR_NUMBER=${{ fromJSON(env.EVENT_JSON).number }}" >> $GITHUB_ENV
+      - name: Predict Deployment URL
+        id: url
+        # Replace dots in the repo name with the plus sign
+        run: |
+          REPO=$(echo ${{ github.repository }} | sed 's/\./+/g')
+          echo "EXPECTED_URL=https://app.uffizzi.com/github.com/$REPO/pull/$PR_NUMBER" >> $GITHUB_ENV
+
+      - name: Re-Render Compose File
+        run: |
+          OAUTH_JSON_BASE64=${{ secrets.OAUTH_JSON_BASE64 }}
+          export OAUTH_JSON_BASE64
+          envsubst '${OAUTH_JSON_BASE64} ${EXPECTED_URL}' < docker-compose.rendered.yml > docker-compose.uffizzi.yml
+          # cat docker-compose.uffizzi.yml
+
       - name: Hash Rendered Compose File
         id: hash
         # If the previous workflow was triggered by a PR close event, we will not have a compose file artifact.
         if: ${{ fromJSON(env.EVENT_JSON).action != 'closed' }}
-        run: echo "COMPOSE_FILE_HASH=$(md5sum docker-compose.rendered.yml | awk '{ print $1 }')" >> $GITHUB_ENV
+        run: echo "COMPOSE_FILE_HASH=$(md5sum docker-compose.uffizzi.yml | awk '{ print $1 }')" >> $GITHUB_ENV
       - name: Cache Rendered Compose File
         if: ${{ fromJSON(env.EVENT_JSON).action != 'closed' }}
         uses: actions/cache@v3
         with:
-          path: docker-compose.rendered.yml
+          path: docker-compose.uffizzi.yml
           key: ${{ env.COMPOSE_FILE_HASH }}
 
-      - name: Read PR Number From Event Object
-        id: pr
-        run: echo "PR_NUMBER=${{ fromJSON(env.EVENT_JSON).number }}" >> $GITHUB_ENV
-
       - name: DEBUG - Print Job Outputs
         if: ${{ runner.debug }}
         run: |
@@ -80,7 +93,7 @@ jobs:
       # If this workflow was triggered by a PR close event, cache-key will be an empty string
       # and this reusable workflow will delete the preview deployment.
       compose-file-cache-key: ${{ needs.cache-compose-file.outputs.compose-file-cache-key }}
-      compose-file-cache-path: docker-compose.rendered.yml
+      compose-file-cache-path: docker-compose.uffizzi.yml
       server: https://app.uffizzi.com
       pr-number: ${{ needs.cache-compose-file.outputs.pr-number }}
     permissions:

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+  "python.analysis.typeCheckingMode": "basic"
+}

CHANGELOG.md

@@ -1,6 +1,91 @@
 # Changelog
 
 
+## [1.62.0](https://github.com/windmill-labs/windmill/compare/v1.61.1...v1.62.0) (2023-02-03)
+
+
+### Features
+
+* add INCLUDE_HEADERS env variable to pass value from request headers ([0921ba0](https://github.com/windmill-labs/windmill/commit/0921ba008535e945f2ec3255728c2e8c1f4c36dc))
+* add WHITELIST_WORKSPACES and BLACKLIST_WORKSPACES ([99568ea](https://github.com/windmill-labs/windmill/commit/99568eaa473d57123a7dde4007f8812e0053fb3f))
+* Add workspace webhook ([#1158](https://github.com/windmill-labs/windmill/issues/1158)) ([b9ac60f](https://github.com/windmill-labs/windmill/commit/b9ac60f8bb0662e364606c4b7b8a6e3c1e7e4041))
+* adding worker_busy ([23007f7](https://github.com/windmill-labs/windmill/commit/23007f7a71630fc2040e1be39db83ba56689e3c4))
+* **cli:** 2-Way sync ([#1071](https://github.com/windmill-labs/windmill/issues/1071)) ([cdd1619](https://github.com/windmill-labs/windmill/commit/cdd16195aeaf32e1f1d0648f48e4843954d16d9c))
+* **frontend:** App initial loading animations ([#1176](https://github.com/windmill-labs/windmill/issues/1176)) ([3305481](https://github.com/windmill-labs/windmill/commit/3305481d5d4ce598ceb57256cea851869cdaf25e))
+* **python:** add ADDITIONAL_PYTHON_PATHS ([14b32be](https://github.com/windmill-labs/windmill/commit/14b32be8b229372c57a167fd74cb958a96f0e8e6))
+
+
+### Bug Fixes
+
+* **frontend:** Render popups above components in app editor ([#1171](https://github.com/windmill-labs/windmill/issues/1171)) ([bc8d1a3](https://github.com/windmill-labs/windmill/commit/bc8d1a375ec7886357ce0ef5971bb35013c94d61))
+* **frontend:** Various fixes and improvements ([#1177](https://github.com/windmill-labs/windmill/issues/1177)) ([9f5500c](https://github.com/windmill-labs/windmill/commit/9f5500c1965ea50796d3bf289c0f9e0c929427f4))
+* navigate to new script page before saving script ([f171cd8](https://github.com/windmill-labs/windmill/commit/f171cd8b7c46677173572bac256cbb489a1b8526))
+
+## [1.61.1](https://github.com/windmill-labs/windmill/compare/v1.61.0...v1.61.1) (2023-01-31)
+
+
+### Bug Fixes
+
+* **backend:** compile issue ([df8cc1f](https://github.com/windmill-labs/windmill/commit/df8cc1f2482b3d8b1530cdaef1361303ff5cadff))
+
+## [1.61.0](https://github.com/windmill-labs/windmill/compare/v1.60.0...v1.61.0) (2023-01-31)
+
+
+### Features
+
+* add openapi viewer ([#1094](https://github.com/windmill-labs/windmill/issues/1094)) ([1337811](https://github.com/windmill-labs/windmill/commit/1337811438d48e23133f68e9157bd185d5fe4a82))
+* add PIP_LOCAL_DEPENDENCIES ([b7db4c7](https://github.com/windmill-labs/windmill/commit/b7db4c78c4629f1fd2dfd7a338f783b16f07b24d))
+* add QUEUE_LIMIT_WAIT_RESULT ([51a8810](https://github.com/windmill-labs/windmill/commit/51a8810aa0a9ab7702df459dd270278d42bd3899))
+* add resource and resource type from json ([080ecb0](https://github.com/windmill-labs/windmill/commit/080ecb04d7a08d035fe07f179975b52bc0f77297))
+* add sql as a valid type in Python ([0172587](https://github.com/windmill-labs/windmill/commit/0172587b129ce54d96dc99336a1f56c66ebdbef5))
+* add sync webhook for flows ([f377c84](https://github.com/windmill-labs/windmill/commit/f377c84f5a2148a2bbb7c16e93f13e1d85ceb17e))
+* **backend:** add queue_limit + configurable timeout + fix timeout cancel ([eef3bab](https://github.com/windmill-labs/windmill/commit/eef3bab6e4d9f1af1435db868c707a692558ab74))
+* **deno:** add support for DENO_AUTH_TOKENS ([832ddab](https://github.com/windmill-labs/windmill/commit/832ddabdf2239521368e5f96df144abce0db31c2))
+* **deno:** allow overriding deno sandboxing with DENO_FLAGS' ([7f40373](https://github.com/windmill-labs/windmill/commit/7f40373fd64005d87972854a565c6cf521232982))
+* **frontend:** Add app inputs configurations ([#1142](https://github.com/windmill-labs/windmill/issues/1142)) ([3ed16b8](https://github.com/windmill-labs/windmill/commit/3ed16b88a42e4db6e12f8557c5bbaa2d832b1c17))
+* **frontend:** Add app preview lock ([#1127](https://github.com/windmill-labs/windmill/issues/1127)) ([6a88e8c](https://github.com/windmill-labs/windmill/commit/6a88e8c4f4d6fa5c393ce27b2040784a74a73b06))
+* **frontend:** Add copy button option to app text display component ([#1090](https://github.com/windmill-labs/windmill/issues/1090)) ([bdfc38d](https://github.com/windmill-labs/windmill/commit/bdfc38d954a3c5548fb7f9ee6f80f741eff8cb67))
+* **frontend:** Add default codes to app editor ([#1099](https://github.com/windmill-labs/windmill/issues/1099)) ([c50c740](https://github.com/windmill-labs/windmill/commit/c50c7406f267b480af2a01b47e3fcfa1d763db7a))
+* **frontend:** Add HTML result rendering ([#1160](https://github.com/windmill-labs/windmill/issues/1160)) ([c01bf70](https://github.com/windmill-labs/windmill/commit/c01bf70f62680a4b77812ac6eb64aca2b15d9a8d))
+* **frontend:** Add more integration icons ([#1097](https://github.com/windmill-labs/windmill/issues/1097)) ([2191e85](https://github.com/windmill-labs/windmill/commit/2191e852318f069489f77a4f1c44aadf248c7f53))
+* **frontend:** add plotly support ([a4f8f9e](https://github.com/windmill-labs/windmill/commit/a4f8f9e1cf80395d5cd1229c8dd5dda244e2ba7f))
+* **frontend:** add selectedRowIndex to the table outputs ([#1145](https://github.com/windmill-labs/windmill/issues/1145)) ([f05f9e4](https://github.com/windmill-labs/windmill/commit/f05f9e4edb928e7a8e3e66a62de9c6487684a14b))
+* **frontend:** Add Supabase resource ([#1107](https://github.com/windmill-labs/windmill/issues/1107)) ([12b00a8](https://github.com/windmill-labs/windmill/commit/12b00a808d1f12827a7bc26518cc6f972bdde917))
+* **frontend:** add support for background scripts + add FormButtonCo… ([#1124](https://github.com/windmill-labs/windmill/issues/1124)) ([e969af9](https://github.com/windmill-labs/windmill/commit/e969af9e44d1b4409064080e8662552ee3e262e8))
+* **frontend:** Add surreal db logo ([#1102](https://github.com/windmill-labs/windmill/issues/1102)) ([d811675](https://github.com/windmill-labs/windmill/commit/d81167588227f2cc433aab64551d96d21a589c5b))
+* **frontend:** Add tooltip to app recompute ([#1122](https://github.com/windmill-labs/windmill/issues/1122)) ([4dfdf37](https://github.com/windmill-labs/windmill/commit/4dfdf374af358ef46ee8057373546719c6570067))
+* **frontend:** add vega-lite component ([bd79938](https://github.com/windmill-labs/windmill/commit/bd79938bed6da3875a4a2dd72dad14dedbf25ddf))
+* **frontend:** Display error as an icon in order to avoid clutter wh… ([#1143](https://github.com/windmill-labs/windmill/issues/1143)) ([22b8fed](https://github.com/windmill-labs/windmill/commit/22b8fed9d904a37aae66f6d957f4987f6ca9955c))
+* **frontend:** Open debug runs from component ([#1155](https://github.com/windmill-labs/windmill/issues/1155)) ([73bc13b](https://github.com/windmill-labs/windmill/commit/73bc13bb7d4b1eb25a3a726ac9e6bb80120a495f))
+* **frontend:** Update app table component styles ([#1100](https://github.com/windmill-labs/windmill/issues/1100)) ([172b5db](https://github.com/windmill-labs/windmill/commit/172b5dba8f4c3aaf11569c72313ad74845c668a6))
+* **python:** add support for extra args in python ([772c768](https://github.com/windmill-labs/windmill/commit/772c768cda094f208a5efb7aab03eee3a8f38f68))
+
+
+### Bug Fixes
+
+* **frontend:** Add default value for text, number and date input + fix issues with number input + add date input in the settings panel ([#1135](https://github.com/windmill-labs/windmill/issues/1135)) ([8f90602](https://github.com/windmill-labs/windmill/commit/8f906026b3203702c3b6a30bcac9fb2aca985c29))
+* **frontend:** Add highlight to selected workspace ([#1159](https://github.com/windmill-labs/windmill/issues/1159)) ([f221a6c](https://github.com/windmill-labs/windmill/commit/f221a6c17f145d0c42f7faf785c37f4037308973))
+* **frontend:** add missing condition to properly select first row ([#1128](https://github.com/windmill-labs/windmill/issues/1128)) ([3d873ed](https://github.com/windmill-labs/windmill/commit/3d873ed51c769005981a8d8dfb95faa3ca33bb83))
+* **frontend:** App form component display ([#1096](https://github.com/windmill-labs/windmill/issues/1096)) ([339742c](https://github.com/windmill-labs/windmill/commit/339742ca77dd0fda19d5a262617e42c341ef5871))
+* **frontend:** App script list panel overflow ([#1101](https://github.com/windmill-labs/windmill/issues/1101)) ([7bc59d9](https://github.com/windmill-labs/windmill/commit/7bc59d9d2650b623a2b481a727ffc495b4216f22))
+* **frontend:** App table action button cell ([#1149](https://github.com/windmill-labs/windmill/issues/1149)) ([e989662](https://github.com/windmill-labs/windmill/commit/e98966283dd9b57cc07da34876a90d19210c2927))
+* **frontend:** App table header z-index ([#1120](https://github.com/windmill-labs/windmill/issues/1120)) ([59c4cc2](https://github.com/windmill-labs/windmill/commit/59c4cc2058f86deea793b61de59e2936e50e5577))
+* **frontend:** Check if hiddenInlineScripts are undefined before iterating over them ([#1134](https://github.com/windmill-labs/windmill/issues/1134)) ([71a443e](https://github.com/windmill-labs/windmill/commit/71a443e3c56d2b8c951de6e3701a411ad1a0ce34))
+* **frontend:** fix first row selection ([#1125](https://github.com/windmill-labs/windmill/issues/1125)) ([6c9daf7](https://github.com/windmill-labs/windmill/commit/6c9daf70021859dcd7cef717bc3acdfa88cffd02))
+* **frontend:** Fix id generation when a second action ([#1110](https://github.com/windmill-labs/windmill/issues/1110)) ([4f86981](https://github.com/windmill-labs/windmill/commit/4f869811fee73826b2b10965241d2d8dba59dc2a))
+* **frontend:** Make sure AppSelect items are an array ([#1144](https://github.com/windmill-labs/windmill/issues/1144)) ([24b1fa0](https://github.com/windmill-labs/windmill/commit/24b1fa0ae327c984841f9ed8b163b3fccc6da258))
+* **frontend:** Make sure that old apps are rendering properly ([#1132](https://github.com/windmill-labs/windmill/issues/1132)) ([a78486d](https://github.com/windmill-labs/windmill/commit/a78486d7e08f76e22406063288b35e9030974d7a))
+* **frontend:** Playwright ([#1108](https://github.com/windmill-labs/windmill/issues/1108)) ([f0435f5](https://github.com/windmill-labs/windmill/commit/f0435f5f81941c5b49500003aa27956d627daadb))
+* **frontend:** Prepare app scripts code for export ([#1123](https://github.com/windmill-labs/windmill/issues/1123)) ([173093a](https://github.com/windmill-labs/windmill/commit/173093a40321f6ad35bf766a5554b21cea388771))
+* **frontend:** Prevent modal from hijacking all keypress event ([#1136](https://github.com/windmill-labs/windmill/issues/1136)) ([aa6de3b](https://github.com/windmill-labs/windmill/commit/aa6de3bb5746b9d99c8e3a52e6a9fff10d97bc6a))
+* **frontend:** Revert component input panel change ([#1092](https://github.com/windmill-labs/windmill/issues/1092)) ([0419e7e](https://github.com/windmill-labs/windmill/commit/0419e7e1c9239fd3cbc49acf82a73e9c01938153))
+* **frontend:** Runnable table overflow ([#1119](https://github.com/windmill-labs/windmill/issues/1119)) ([462adbe](https://github.com/windmill-labs/windmill/commit/462adbe42f823646413a5003fd71f3dd473c0728))
+* **frontend:** Select the first row by default, and remove the abilit… ([#1121](https://github.com/windmill-labs/windmill/issues/1121)) ([3c483f5](https://github.com/windmill-labs/windmill/commit/3c483f533759b9b4e589055dbddb31f294bea8fa))
+* **frontend:** Show app builder header always on top ([#1118](https://github.com/windmill-labs/windmill/issues/1118)) ([631a3da](https://github.com/windmill-labs/windmill/commit/631a3da17f05a3d29defdf96a50d7e96a9f8baad))
+* **frontend:** Update app scripts pane ([#1146](https://github.com/windmill-labs/windmill/issues/1146)) ([18f30c8](https://github.com/windmill-labs/windmill/commit/18f30c8286f8240158643ade8b0ef4607a80fbb0))
+* **frontend:** Use absolute path on connect images ([#1095](https://github.com/windmill-labs/windmill/issues/1095)) ([43e069e](https://github.com/windmill-labs/windmill/commit/43e069eb96c0af7d3a1fe1db4f4b69f8e31e7438))
+* improvements for error handling as first step of flow ([b77c239](https://github.com/windmill-labs/windmill/commit/b77c239f307a37777acb083b0cdb5c0d214a9dd8))
+
 ## [1.60.0](https://github.com/windmill-labs/windmill/compare/v1.59.0...v1.60.0) (2023-01-11)
 
 

Caddyfile

@@ -1,5 +1,15 @@
-{$BASE_URL} {
+{
+        auto_https off
+}
+
+http://{$BASE_URL} {
+        bind {$ADDRESS}
         reverse_proxy /ws/* http://lsp:3001
         reverse_proxy /* http://windmill_server:8000
-        reverse_proxy /* http://windmill:8000
+}
+
+https://{$BASE_URL}  {  
+  bind {$ADDRESS}
+  reverse_proxy /ws/* http://localhost:3001
+}
 }

README.md

@@ -182,7 +182,7 @@ compiling from source or using without a postgres super user, see
 `docker compose up` with the following docker-compose is sufficient:
 <https://github.com/windmill-labs/windmill/blob/main/docker-compose.yml>
 
-Go to https://localhost et voilà :)
+Go to http://localhost et voilà :)
 
 For older kernels < 4.18, set `DISABLE_NUSER=true` as env variable, otherwise
 nsjail will not be able to launch the isolated scripts.
@@ -278,7 +278,8 @@ upcoming CLI tool.
 | ------------------------- | ---------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------- |
 | DATABASE_URL              |                        | The Postgres database url.                                                                                                                                                                         | All                   |
 | DISABLE_NSJAIL            | true                   | Disable Nsjail Sandboxing                                                                                                                                                                          |                       |
-| NUM_WORKERS               | 3                      | The number of worker per Worker instance (set to 1 on Eks to have 1 pod = 1 worker)                                                                                                                | Worker                |
+| NUM_WORKERS               | 3                      | The number of worker per Worker instance (set to 1 on Eks to have 1 pod = 1 worker, set to 0 for an API only instance)                                                                             | Worker                |
+| DISABLE_SERVER            | false                  | Binary would operate as a worker only instance                                                                                                                                                     | Worker                |
 | METRICS_ADDR              | None                   | The socket addr at which to expose Prometheus metrics at the /metrics path. Set to "true" to expose it on port 8001                                                                                | All                   |
 | JSON_FMT                  | false                  | Output the logs in json format instead of logfmt                                                                                                                                                   | All                   |
 | BASE_URL                  | http://localhost:8000  | The base url that is exposed publicly to access your instance                                                                                                                                      | Server                |
@@ -308,7 +309,11 @@ upcoming CLI tool.
 | QUEUE_LIMIT_WAIT_RESULT   | None                   | The number of max jobs in the queue before rejecting immediately the request in 'run_wait_result' endpoint. Takes precedence on the query arg. If none is specified, there are no limit.           | Worker                |
 | DENO_AUTH_TOKENS          | None                   | Custom DENO_AUTH_TOKENS to pass to worker to allow the use of private modules                                                                                                                      | Worker                |
 | DENO_FLAGS                | None                   | Override the flags passed to deno (default --allow-all) to tighten permissions. Minimum permissions needed are "--allow-read=args.json --allow-write=result.json"                                  | Worker                |
-|                           |
+| PIP_LOCAL_DEPENDENCIES    | None                   | Specify dependencies that are installed locally and do not need to be solved nor installed again                                                                                                   |
+| ADDITIONAL_PYTHON_PATHS   | None                   | Specify python paths (separated by a :) to be appended to the PYTHONPATH of the python jobs. To be used with PIP_LOCAL_DEPENDENCIES to use python codebases within Windmill                        | Worker                |
+| INCLUDE_HEADERS           | None                   | Whitelist of headers that are passed to jobs as args (separated by a comma)                                                                                                                        | Server                |
+| WHITELIST_WORKSPACES      | None                   | Whitelist of workspaces this worker takes job from                                                                                                                                                 | Worker                |
+| BLACKLIST_WORKSPACES      | None                   | Blacklist of workspaces this worker takes job from                                                                                                                                                 | Worker                |
 
 
 

backend/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "windmill"
-version = "1.60.0"
+version = "1.62.0"
 authors.workspace = true
 edition.workspace = true
 
@@ -19,7 +19,7 @@ members = [
 ]
 
 [workspace.package]
-version = "1.60.0"
+version = "1.62.0"
 authors = ["Ruben Fiszel <ruben@windmill.dev>"]
 edition = "2021"
 
@@ -83,7 +83,7 @@ chrono = { version = "^0", features = ["serde"] }
 tracing = "^0"
 tracing-subscriber = { version = "^0", features = ["env-filter", "json"] }
 prometheus = { version = "^0", default-features = false }
-cookie = { version = "^0" }
+cookie = { version = "0.16.2" }
 phf = { version = "0.11", features = ["macros"] }
 rust-embed = "^6"
 mime_guess = "^2"

backend/migrations/20230127175428_increase_token_size.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230127175428_increase_token_size.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+ALTER TABLE account ALTER COLUMN refresh_token TYPE VARCHAR(1500);

backend/migrations/20230127210757_favorite_privilege.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230127210757_favorite_privilege.up.sql

@@ -0,0 +1,4 @@
+-- Add up migration script here
+
+GRANT ALL PRIVILEGES ON TABLE favorite TO windmill_admin;
+GRANT ALL PRIVILEGES ON TABLE favorite TO windmill_user;

backend/migrations/20230131143413_more_folders_privilege.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230131143413_more_folders_privilege.up.sql

@@ -0,0 +1,25 @@
+-- Add up migration script here
+
+
+CREATE POLICY see_folder_extra_perms_user ON capture FOR ALL
+USING (SPLIT_PART(capture.path, '/', 1) = 'f' AND SPLIT_PART(capture.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_read'), ',')::text[]))
+WITH CHECK (SPLIT_PART(capture.path, '/', 1) = 'f' AND SPLIT_PART(capture.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_write'), ',')::text[]));
+
+DO
+$do$
+  DECLARE
+    i text;
+    arr text[] := array['queue', 'completed_job'];
+  BEGIN
+  FOREACH i IN ARRAY arr
+  LOOP
+    EXECUTE FORMAT(
+      $$
+        CREATE POLICY see_folder_extra_perms_user ON %1$I FOR ALL
+        USING (%1$I.visible_to_owner IS true AND SPLIT_PART(%1$I.script_path, '/', 1) = 'f' AND SPLIT_PART(%1$I.script_path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_read'), ',')::text[]));
+      $$,
+      i
+    );
+  END LOOP;
+  END
+$do$;

backend/migrations/20230131145602_more_folders_privilege_2.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230131145602_more_folders_privilege_2.up.sql

@@ -0,0 +1,5 @@
+-- Add up migration script here
+
+CREATE POLICY see_folder_extra_perms_user ON account FOR ALL
+USING (SPLIT_PART(account.owner, '/', 1) = 'f' AND SPLIT_PART(account.owner, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_read'), ',')::text[]))
+WITH CHECK (SPLIT_PART(account.owner, '/', 1) = 'f' AND SPLIT_PART(account.owner, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_write'), ',')::text[]));

backend/migrations/20230204182500_add_mem_peak.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230204182500_add_mem_peak.up.sql

@@ -0,0 +1,3 @@
+-- Add up migration script here
+ALTER TABLE queue ADD COLUMN mem_peak INTEGER;
+ALTER TABLE completed_job ADD COLUMN mem_peak INTEGER;

backend/parsers/windmill-parser-py/src/lib.rs

@@ -80,7 +80,7 @@ pub fn parse_python_signature(code: &str) -> error::Result<MainArgSignature> {
         let def_arg_start = params.args.len() - params.defaults.len();
         Ok(MainArgSignature {
             star_args: params.vararg.is_some(),
-            star_kwargs: params.vararg.is_some(),
+            star_kwargs: params.kwarg.is_some(),
             args: params
                 .args
                 .into_iter()
@@ -136,7 +136,10 @@ fn to_value(et: &ExprKind) -> Option<serde_json::Value> {
                 .into_iter()
                 .zip(values)
                 .map(|(k, v)| {
-                    let key = to_value(&k.node)
+                    let key = k
+                        .as_ref()
+                        .map(|x| x.node.clone())
+                        .and_then(|n| to_value(&n))
                         .and_then(|x| match x {
                             serde_json::Value::String(s) => Some(s),
                             _ => None,
@@ -182,6 +185,7 @@ static PYTHON_IMPORTS_REPLACEMENT: phf::Map<&'static str, &'static str> = phf_ma
     "f" => "requests",
     "shopify" => "ShopifyAPI",
     "seleniumwire" => "selenium-wire",
+    "openbb-terminal" => "openbb[all]",
 };
 
 fn replace_import(x: String) -> String {
@@ -193,13 +197,13 @@ fn replace_import(x: String) -> String {
 }
 
 lazy_static! {
-    static ref RE: Regex = Regex::new(r"^\#(\S+)$").unwrap();
+    static ref RE: Regex = Regex::new(r"^\#\s?(\S+)$").unwrap();
 }
 
 pub fn parse_python_imports(code: &str) -> error::Result<Vec<String>> {
     let find_requirements = code
         .lines()
-        .find_position(|x| x.starts_with("#requirements:"));
+        .find_position(|x| x.starts_with("#requirements:") || x.starts_with("# requirements:"));
     if let Some((pos, _)) = find_requirements {
         let lines = code
             .lines()

backend/sqlx-data.json

@@ -117,17 +117,25 @@
     },
     "query": "SELECT script_path FROM queue WHERE id = $1"
   },
-  "062859f1d0e5cfba3115f4241115753b86a4ad239708851c998ff5620ebca5b8": {
+  "0715955b7e98cc669a88eca6556cd46b7f9c07fcf32a24f85b69720b54e6e95f": {
     "describe": {
-      "columns": [],
-      "nullable": [],
+      "columns": [
+        {
+          "name": "mem_peak",
+          "ordinal": 0,
+          "type_info": "Int4"
+        }
+      ],
+      "nullable": [
+        true
+      ],
       "parameters": {
         "Left": [
           "Uuid"
         ]
       }
     },
-    "query": "UPDATE queue SET last_ping = now() WHERE id = $1"
+    "query": "SELECT mem_peak FROM queue WHERE id = $1"
   },
   "07551a32c49da8c0693dd39c6a63b5b2a596ccc0e52e8918160604a5e133dd32": {
     "describe": {
@@ -409,28 +417,6 @@
     },
     "query": "INSERT INTO password(email, verified, password_hash, login_type, super_admin, name, company)\n    VALUES ($1, $2, $3, 'password', $4, $5, $6)"
   },
-  "11eb4dd4a2c9b0b759294dde5e8b505c5a4391aa0d8cb629c665711ee0fc04a0": {
-    "describe": {
-      "columns": [
-        {
-          "name": "logs",
-          "ordinal": 0,
-          "type_info": "Text"
-        }
-      ],
-      "nullable": [
-        null
-      ],
-      "parameters": {
-        "Left": [
-          "Int4",
-          "Text",
-          "Uuid"
-        ]
-      }
-    },
-    "query": "SELECT substr(logs, $1) as logs FROM queue WHERE workspace_id = $2 AND id = $3"
-  },
   "122090a0f89e5248a0a0f199ebd24582fdb302883aebd2da187ac0084e767ea3": {
     "describe": {
       "columns": [
@@ -1495,6 +1481,74 @@
     },
     "query": "UPDATE app SET versions = array_append(versions, $1) WHERE id = $2"
   },
+  "41659beec0e8d6d95258a728e8e4b2b929045a05bd0cfb2293031c0e4a62f04e": {
+    "describe": {
+      "columns": [],
+      "nullable": [],
+      "parameters": {
+        "Left": [
+          "Varchar",
+          "Uuid",
+          "Uuid",
+          "Varchar",
+          "Timestamptz",
+          "Timestamptz",
+          "Bool",
+          "Int8",
+          "Varchar",
+          "Jsonb",
+          "Jsonb",
+          "Text",
+          "Text",
+          "Text",
+          "Bool",
+          "Varchar",
+          "Text",
+          {
+            "Custom": {
+              "kind": {
+                "Enum": [
+                  "script",
+                  "preview",
+                  "flow",
+                  "dependencies",
+                  "flowpreview",
+                  "script_hub",
+                  "identity",
+                  "flowdependencies"
+                ]
+              },
+              "name": "job_kind"
+            }
+          },
+          "Varchar",
+          "Varchar",
+          "Jsonb",
+          "Jsonb",
+          "Bool",
+          "Bool",
+          {
+            "Custom": {
+              "kind": {
+                "Enum": [
+                  "python3",
+                  "deno",
+                  "go",
+                  "bash"
+                ]
+              },
+              "name": "script_lang"
+            }
+          },
+          "Numeric",
+          "Varchar",
+          "Bool",
+          "Int4"
+        ]
+      }
+    },
+    "query": "INSERT INTO completed_job AS cj\n                   ( workspace_id\n                   , id\n                   , parent_job\n                   , created_by\n                   , created_at\n                   , started_at\n                   , duration_ms\n                   , success\n                   , script_hash\n                   , script_path\n                   , args\n                   , result\n                   , logs\n                   , raw_code\n                   , raw_lock\n                   , canceled\n                   , canceled_by\n                   , canceled_reason\n                   , job_kind\n                   , schedule_path\n                   , permissioned_as\n                   , flow_status\n                   , raw_flow\n                   , is_flow_step\n                   , is_skipped\n                   , language\n                   , email\n                   , visible_to_owner\n                   , mem_peak\n                )\n            VALUES ($1, $2, $3, $4, $5, $6, COALESCE($26, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE($6, now()))))*1000), $7, $8, $9,$10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $27, $28, $29)\n         ON CONFLICT (id) DO UPDATE SET success = $7, result = $11, logs = concat(cj.logs, $12)"
+  },
   "42e1b5634a9e51247115fa73f85a97b1467c913d012cac9c45bb6a349082dc71": {
     "describe": {
       "columns": [
@@ -3396,6 +3450,27 @@
     },
     "query": "UPDATE schedule SET path = $1, script_path = $1 WHERE path = $2 AND workspace_id = $3 AND is_flow IS true RETURNING *"
   },
+  "913c634de21d46b8841f8a7c25c408da7c572f9e685db6351848cbf6e9253efc": {
+    "describe": {
+      "columns": [
+        {
+          "name": "canceled",
+          "ordinal": 0,
+          "type_info": "Bool"
+        }
+      ],
+      "nullable": [
+        false
+      ],
+      "parameters": {
+        "Left": [
+          "Int4",
+          "Uuid"
+        ]
+      }
+    },
+    "query": "UPDATE queue SET mem_peak = GREATEST($1, mem_peak), last_ping = now() WHERE id = $2 RETURNING canceled"
+  },
   "924e1d5cbb682db0d6121dd911f38cf8948d0e59ca47428daa4f43fce686ba80": {
     "describe": {
       "columns": [
@@ -3857,73 +3932,6 @@
     },
     "query": "UPDATE flow SET dependency_job = $1 WHERE path = $2 AND workspace_id = $3"
   },
-  "a46d97b0c65bbbcc29022c2d010a4853541987a58f0098f9c2e3fd6e40b11a52": {
-    "describe": {
-      "columns": [],
-      "nullable": [],
-      "parameters": {
-        "Left": [
-          "Varchar",
-          "Uuid",
-          "Uuid",
-          "Varchar",
-          "Timestamptz",
-          "Timestamptz",
-          "Bool",
-          "Int8",
-          "Varchar",
-          "Jsonb",
-          "Jsonb",
-          "Text",
-          "Text",
-          "Text",
-          "Bool",
-          "Varchar",
-          "Text",
-          {
-            "Custom": {
-              "kind": {
-                "Enum": [
-                  "script",
-                  "preview",
-                  "flow",
-                  "dependencies",
-                  "flowpreview",
-                  "script_hub",
-                  "identity",
-                  "flowdependencies"
-                ]
-              },
-              "name": "job_kind"
-            }
-          },
-          "Varchar",
-          "Varchar",
-          "Jsonb",
-          "Jsonb",
-          "Bool",
-          "Bool",
-          {
-            "Custom": {
-              "kind": {
-                "Enum": [
-                  "python3",
-                  "deno",
-                  "go",
-                  "bash"
-                ]
-              },
-              "name": "script_lang"
-            }
-          },
-          "Numeric",
-          "Varchar",
-          "Bool"
-        ]
-      }
-    },
-    "query": "INSERT INTO completed_job AS cj\n                   ( workspace_id\n                   , id\n                   , parent_job\n                   , created_by\n                   , created_at\n                   , started_at\n                   , duration_ms\n                   , success\n                   , script_hash\n                   , script_path\n                   , args\n                   , result\n                   , logs\n                   , raw_code\n                   , raw_lock\n                   , canceled\n                   , canceled_by\n                   , canceled_reason\n                   , job_kind\n                   , schedule_path\n                   , permissioned_as\n                   , flow_status\n                   , raw_flow\n                   , is_flow_step\n                   , is_skipped\n                   , language\n                   , email\n                   , visible_to_owner\n                )\n            VALUES ($1, $2, $3, $4, $5, $6, COALESCE($26, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE($6, now()))))*1000), $7, $8, $9,$10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $27, $28)\n         ON CONFLICT (id) DO UPDATE SET success = $7, result = $11, logs = concat(cj.logs, $12)"
-  },
   "a6145b0482c9e5da245059a80b1563cad20318fd2dd8aef33f9ca97de1826b8b": {
     "describe": {
       "columns": [],
@@ -4371,6 +4379,34 @@
     },
     "query": "SELECT label, concat(substring(token for 10)) as token_prefix, expiration, created_at, last_used_at FROM token WHERE email = $1\n         ORDER BY created_at DESC"
   },
+  "b69f747eae8b13a0a8d6914a3a7ad322554fcfe62cd28c5f0a475f18dd770d61": {
+    "describe": {
+      "columns": [
+        {
+          "name": "logs",
+          "ordinal": 0,
+          "type_info": "Text"
+        },
+        {
+          "name": "mem_peak",
+          "ordinal": 1,
+          "type_info": "Int4"
+        }
+      ],
+      "nullable": [
+        null,
+        true
+      ],
+      "parameters": {
+        "Left": [
+          "Int4",
+          "Text",
+          "Uuid"
+        ]
+      }
+    },
+    "query": "SELECT substr(logs, $1) as logs, mem_peak FROM queue WHERE workspace_id = $2 AND id = $3"
+  },
   "b7dd791cd69748ef51b7520f505c0c8bb1b4014a273476eddfecf1ab658a18b4": {
     "describe": {
       "columns": [

backend/windmill-api/Cargo.toml

@@ -71,4 +71,4 @@ sha2.workspace = true
 urlencoding.workspace = true
 async-stripe.workspace = true
 lazy_static.workspace = true
-prometheus.workspace = true
+prometheus.workspace = true

backend/windmill-api/openapi.yaml

@@ -1,7 +1,7 @@
 openapi: "3.0.3"
 
 info:
-  version: 1.60.0
+  version: 1.62.0
   title: Windmill API
 
   contact:
@@ -2474,6 +2474,44 @@ paths:
             application/json:
               schema: {}
 
+  /w/{workspace}/jobs/run_wait_result/f/{path}:
+    post:
+      summary: run flow by path and wait until completion
+      operationId: runWaitResultFlowByPath
+      tags:
+        - job
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/ScriptPath"
+        - name: scheduled_for
+          description: when to schedule this job (leave empty for immediate run)
+          in: query
+          schema:
+            type: string
+            format: date-time
+        - name: scheduled_in_secs
+          description: schedule the script to execute in the number of seconds starting now
+          in: query
+          schema:
+            type: integer
+        - $ref: "#/components/parameters/IncludeHeader"
+        - $ref: "#/components/parameters/QueueLimit"
+
+      requestBody:
+        description: script args
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/ScriptArgs"
+
+      responses:
+        "200":
+          description: job result
+          content:
+            application/json:
+              schema: {}
+
   /w/{workspace}/jobs/result_by_id/{flow_job_id}/{node_id}:
     get:
       summary: get job result by id
@@ -3254,6 +3292,8 @@ paths:
                     type: boolean
                   new_logs:
                     type: string
+                  mem_peak:
+                    type: integer
 
   /w/{workspace}/jobs/completed/get/{id}:
     get:
@@ -4752,6 +4792,8 @@ components:
           type: string
         visible_to_owner:
           type: boolean
+        mem_peak:
+          type: integer
       required:
         - id
         - running
@@ -4838,6 +4880,8 @@ components:
           type: string
         visible_to_owner:
           type: boolean
+        mem_peak:
+          type: integer
       required:
         - id
         - created_by

backend/windmill-api/src/apps.rs

@@ -12,7 +12,7 @@ use crate::{
     jobs::script_path_to_payload,
     users::{require_owner_of_path, Authed, OptAuthed},
     variables::build_crypt,
-    webhook_util::{WebhookMessage, WebhookUtil},
+    webhook_util::{WebhookMessage, WebhookShared},
 };
 use axum::{
     extract::{Extension, Json, Path, Query},
@@ -311,7 +311,7 @@ async fn get_secret_id(
 async fn create_app(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
-    webhook: WebhookUtil,
+    Extension(webhook): Extension<WebhookShared>,
     Path(w_id): Path<String>,
     Json(app): Json<CreateApp>,
 ) -> Result<(StatusCode, String)> {
@@ -360,10 +360,10 @@ async fn create_app(
     .await?;
 
     tx.commit().await?;
-    webhook.send_message(WebhookMessage::CreateApp {
-        workspace: w_id.clone(),
-        path: app.path.clone(),
-    });
+    webhook.send_message(
+        w_id.clone(),
+        WebhookMessage::CreateApp { workspace: w_id, path: app.path.clone() },
+    );
 
     Ok((StatusCode::CREATED, app.path))
 }
@@ -402,7 +402,7 @@ pub async fn get_hub_app_by_id(
 async fn delete_app(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
-    webhook: WebhookUtil,
+    Extension(webhook): Extension<WebhookShared>,
     Path((w_id, path)): Path<(String, StripPath)>,
 ) -> Result<String> {
     let path = path.to_path();
@@ -426,8 +426,10 @@ async fn delete_app(
     )
     .await?;
     tx.commit().await?;
-    webhook
-        .send_message(WebhookMessage::DeleteApp { workspace: w_id.clone(), path: path.to_owned() });
+    webhook.send_message(
+        w_id.clone().clone(),
+        WebhookMessage::DeleteApp { workspace: w_id, path: path.to_owned() },
+    );
 
     Ok(format!("app {} deleted", path))
 }
@@ -435,7 +437,7 @@ async fn delete_app(
 async fn update_app(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
-    webhook: WebhookUtil,
+    Extension(webhook): Extension<WebhookShared>,
     Extension(db): Extension<DB>,
     Path((w_id, path)): Path<(String, StripPath)>,
     Json(ns): Json<EditApp>,
@@ -525,11 +527,14 @@ async fn update_app(
     )
     .await?;
     tx.commit().await?;
-    webhook.send_message(WebhookMessage::UpdateApp {
-        workspace: w_id.clone(),
-        old_path: path.to_owned(),
-        new_path: npath.clone(),
-    });
+    webhook.send_message(
+        w_id.clone(),
+        WebhookMessage::UpdateApp {
+            workspace: w_id,
+            old_path: path.to_owned(),
+            new_path: npath.clone(),
+        },
+    );
 
     Ok(format!("app {} updated (npath: {:?})", path, npath))
 }

backend/windmill-api/src/flows.rs

@@ -32,7 +32,7 @@ use crate::{
     db::{UserDB, DB},
     schedule::clear_schedule,
     users::{require_owner_of_path, Authed},
-    webhook_util::{WebhookMessage, WebhookUtil},
+    webhook_util::{WebhookMessage, WebhookShared},
 };
 
 pub fn workspaced_service() -> Router {
@@ -182,7 +182,7 @@ async fn check_path_conflict<'c>(
 async fn create_flow(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
-    webhook: WebhookUtil,
+    Extension(webhook): Extension<WebhookShared>,
     Path(w_id): Path<String>,
     Json(nf): Json<NewFlow>,
 ) -> Result<(StatusCode, String)> {
@@ -223,10 +223,10 @@ async fn create_flow(
     .await?;
 
     tx.commit().await?;
-    webhook.send_message(WebhookMessage::CreateFlow {
-        workspace: w_id.clone(),
-        path: nf.path.clone(),
-    });
+    webhook.send_message(
+        w_id.clone(),
+        WebhookMessage::CreateFlow { workspace: w_id.clone(), path: nf.path.clone() },
+    );
 
     let tx = user_db.begin(&authed).await?;
     let (dependency_job_uuid, mut tx) = push(
@@ -286,7 +286,7 @@ async fn update_flow(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
     Extension(db): Extension<DB>,
-    webhook: WebhookUtil,
+    Extension(webhook): Extension<WebhookShared>,
     Path((w_id, flow_path)): Path<(String, StripPath)>,
     Json(nf): Json<NewFlow>,
 ) -> Result<String> {
@@ -375,11 +375,14 @@ async fn update_flow(
     .await?;
 
     tx.commit().await?;
-    webhook.send_message(WebhookMessage::UpdateFlow {
-        workspace: w_id.clone(),
-        old_path: flow_path.to_owned(),
-        new_path: nf.path.clone(),
-    });
+    webhook.send_message(
+        w_id.clone(),
+        WebhookMessage::UpdateFlow {
+            workspace: w_id.clone(),
+            old_path: flow_path.to_owned(),
+            new_path: nf.path.clone(),
+        },
+    );
 
     let tx = user_db.begin(&authed).await?;
     let (dependency_job_uuid, mut tx) = push(
@@ -462,7 +465,7 @@ async fn exists_flow_by_path(
 async fn archive_flow_by_path(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
-    webhook: WebhookUtil,
+    Extension(webhook): Extension<WebhookShared>,
     Path((w_id, path)): Path<(String, StripPath)>,
 ) -> Result<String> {
     let path = path.to_path();
@@ -487,10 +490,10 @@ async fn archive_flow_by_path(
     )
     .await?;
     tx.commit().await?;
-    webhook.send_message(WebhookMessage::ArchiveFlow {
-        workspace: w_id.clone(),
-        path: path.to_owned(),
-    });
+    webhook.send_message(
+        w_id.clone(),
+        WebhookMessage::ArchiveFlow { workspace: w_id, path: path.to_owned() },
+    );
 
     Ok(format!("Flow {path} archived"))
 }

backend/windmill-api/src/folders.rs

@@ -9,7 +9,7 @@
 use crate::{
     db::{UserDB, DB},
     users::Authed,
-    webhook_util::{WebhookMessage, WebhookUtil},
+    webhook_util::{WebhookMessage, WebhookShared},
 };
 use axum::{
     extract::{Extension, Path, Query},
@@ -140,7 +140,7 @@ async fn check_name_conflict<'c>(
 async fn create_folder(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
-    webhook: WebhookUtil,
+    Extension(webhook): Extension<WebhookShared>,
     Path(w_id): Path<String>,
     Json(ng): Json<NewFolder>,
 ) -> Result<String> {
@@ -196,10 +196,10 @@ async fn create_folder(
     )
     .await?;
     tx.commit().await?;
-    webhook.send_message(WebhookMessage::CreateFolder {
-        workspace: w_id.clone(),
-        name: ng.name.clone(),
-    });
+    webhook.send_message(
+        w_id.clone(),
+        WebhookMessage::CreateFolder { workspace: w_id, name: ng.name.clone() },
+    );
 
     Ok(format!("Created folder {}", ng.name))
 }
@@ -251,7 +251,7 @@ pub async fn require_is_owner(
 async fn update_folder(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
-    webhook: WebhookUtil,
+    Extension(webhook): Extension<WebhookShared>,
     Path((w_id, name)): Path<(String, String)>,
     Json(ng): Json<UpdateFolder>,
 ) -> Result<String> {
@@ -306,10 +306,10 @@ async fn update_folder(
     )
     .await?;
     tx.commit().await?;
-    webhook.send_message(WebhookMessage::UpdateFolder {
-        workspace: w_id.clone(),
-        name: name.to_owned(),
-    });
+    webhook.send_message(
+        w_id.clone().clone(),
+        WebhookMessage::UpdateFolder { workspace: w_id, name: name.to_owned() },
+    );
 
     Ok(format!("Updated folder {}", name))
 }
@@ -427,7 +427,7 @@ async fn get_folder_usage(
 async fn delete_folder(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
-    webhook: WebhookUtil,
+    Extension(webhook): Extension<WebhookShared>,
     Path((w_id, name)): Path<(String, String)>,
 ) -> Result<String> {
     let mut tx = user_db.begin(&authed).await?;
@@ -453,8 +453,10 @@ async fn delete_folder(
     .await?;
     tx.commit().await?;
 
-    webhook
-        .send_message(WebhookMessage::DeleteFolder { workspace: w_id.clone(), name: name.clone() });
+    webhook.send_message(
+        w_id.clone(),
+        WebhookMessage::DeleteFolder { workspace: w_id, name: name.clone() },
+    );
 
     Ok(format!("delete folder at name {}", name))
 }
@@ -463,7 +465,7 @@ async fn add_owner(
     authed: Authed,
     Extension(db): Extension<DB>,
     Extension(user_db): Extension<UserDB>,
-    webhook: WebhookUtil,
+    Extension(webhook): Extension<WebhookShared>,
     Path((w_id, name)): Path<(String, String)>,
     Json(Owner { owner }): Json<Owner>,
 ) -> Result<String> {
@@ -495,8 +497,10 @@ async fn add_owner(
     .await?;
     tx.commit().await?;
 
-    webhook
-        .send_message(WebhookMessage::UpdateFolder { workspace: w_id.clone(), name: name.clone() });
+    webhook.send_message(
+        w_id.clone(),
+        WebhookMessage::UpdateFolder { workspace: w_id, name: name.clone() },
+    );
 
     Ok(format!("Added {} to folder {}", owner, name))
 }
@@ -531,7 +535,7 @@ async fn remove_owner(
     authed: Authed,
     Extension(db): Extension<DB>,
     Extension(user_db): Extension<UserDB>,
-    webhook: WebhookUtil,
+    Extension(webhook): Extension<WebhookShared>,
     Path((w_id, name)): Path<(String, String)>,
     Json(Owner { owner }): Json<Owner>,
 ) -> Result<String> {
@@ -563,8 +567,10 @@ async fn remove_owner(
     .await?;
     tx.commit().await?;
 
-    webhook
-        .send_message(WebhookMessage::UpdateFolder { workspace: w_id.clone(), name: name.clone() });
+    webhook.send_message(
+        w_id.clone(),
+        WebhookMessage::UpdateFolder { workspace: w_id, name: name.clone() },
+    );
 
     Ok(format!("Removed {} to folder {}", owner, name))
 }

backend/windmill-api/src/jobs.rs

@@ -53,6 +53,10 @@ pub fn workspaced_service() -> Router {
             "/run_wait_result/h/:hash",
             post(run_wait_result_job_by_hash),
         )
+        .route(
+            "/run_wait_result/f/*script_path",
+            post(run_wait_result_flow_by_path),
+        )
         .route("/run/h/:hash", post(run_job_by_hash))
         .route("/run/preview", post(run_preview_job))
         .route("/run/preview_flow", post(run_preview_flow_job))
@@ -103,9 +107,7 @@ async fn get_result_by_id(
     Query(ResultByIdQuery { skip_direct }): Query<ResultByIdQuery>,
     Path((w_id, flow_id, node_id)): Path<(String, String, String)>,
 ) -> windmill_common::error::JsonResult<serde_json::Value> {
-    tracing::error!("get_result_by_id_bef: {:?} {:?}", flow_id, node_id);
     let res = windmill_queue::get_result_by_id(db, skip_direct, w_id, flow_id, node_id).await?;
-    tracing::error!("get_result_by_id: {:?}", res);
     Ok(Json(res))
 }
 
@@ -256,6 +258,8 @@ pub struct CompletedJob {
     pub is_skipped: bool,
     pub email: String,
     pub visible_to_owner: bool,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub mem_peak: Option<i32>,
 }
 
 #[derive(Deserialize, Clone)]
@@ -268,6 +272,14 @@ pub struct RunJobQuery {
     queue_limit: Option<i64>,
 }
 
+lazy_static::lazy_static! {
+    static ref INCLUDE_HEADERS: Vec<String> = std::env::var("INCLUDE_HEADERS")
+        .ok().map(|x| x
+        .split(',')
+        .map(|s| s.to_string())
+        .collect()).unwrap_or_default();
+}
+
 impl RunJobQuery {
     async fn get_scheduled_for<'c>(
         &self,
@@ -288,11 +300,14 @@ impl RunJobQuery {
         headers: HeaderMap,
         mut args: serde_json::Map<String, serde_json::Value>,
     ) -> serde_json::Map<String, serde_json::Value> {
-        self.include_header
+        let whitelist = self
+            .include_header
             .as_ref()
             .map(|s| s.split(",").map(|s| s.to_string()).collect::<Vec<_>>())
-            .unwrap_or_default()
+            .unwrap_or_default();
+        whitelist
             .iter()
+            .chain(INCLUDE_HEADERS.iter())
             .for_each(|h| {
                 if let Some(v) = headers.get(h) {
                     args.insert(
@@ -459,7 +474,7 @@ async fn list_jobs(
             "running",
             "script_hash",
             "script_path",
-            "args",
+            "CASE WHEN pg_column_size(args) > 1000 THEN '\"too large args\"'::jsonb ELSE args END",
             "null as duration_ms",
             "null as success",
             "false as deleted",
@@ -474,6 +489,7 @@ async fn list_jobs(
             "email",
             "visible_to_owner",
             "suspend",
+            "mem_peak",
         ],
     );
     let sqlc = list_completed_jobs_query(
@@ -493,7 +509,7 @@ async fn list_jobs(
             "null as running",
             "script_hash",
             "script_path",
-            "args",
+            "CASE WHEN pg_column_size(args) > 1000 THEN '\"too large args\"'::jsonb ELSE args END",
             "duration_ms",
             "success",
             "deleted",
@@ -508,6 +524,7 @@ async fn list_jobs(
             "email",
             "visible_to_owner",
             "null as suspend",
+            "mem_peak",
         ],
     );
     let sql = format!(
@@ -985,6 +1002,7 @@ struct UnifiedJob {
     email: String,
     visible_to_owner: bool,
     suspend: Option<i32>,
+    mem_peak: Option<i32>,
 }
 
 impl From<UnifiedJob> for Job {
@@ -1019,6 +1037,7 @@ impl From<UnifiedJob> for Job {
                 is_skipped: uj.is_skipped,
                 email: uj.email,
                 visible_to_owner: uj.visible_to_owner,
+                mem_peak: uj.mem_peak,
             }),
             "QueuedJob" => Job::QueuedJob(QueuedJob {
                 workspace_id: uj.workspace_id,
@@ -1051,6 +1070,7 @@ impl From<UnifiedJob> for Job {
                 email: uj.email,
                 visible_to_owner: uj.visible_to_owner,
                 suspend: uj.suspend,
+                mem_peak: uj.mem_peak,
             }),
             t => panic!("job type {} not valid", t),
         }
@@ -1390,6 +1410,53 @@ pub async fn run_wait_result_job_by_hash(
     .await
 }
 
+pub async fn run_wait_result_flow_by_path(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    Extension(db): Extension<DB>,
+    Extension(timeout): Extension<Arc<TimeoutWaitResult>>,
+    Path((w_id, flow_path)): Path<(String, StripPath)>,
+    Query(run_query): Query<RunJobQuery>,
+    headers: HeaderMap,
+    Json(args): Json<Option<serde_json::Map<String, serde_json::Value>>>,
+) -> error::JsonResult<serde_json::Value> {
+    check_queue_too_long(db, run_query.queue_limit).await?;
+
+    let flow_path = flow_path.to_path();
+    let mut tx = user_db.clone().begin(&authed).await?;
+    let scheduled_for = run_query.get_scheduled_for(&mut tx).await?;
+    let args = run_query.add_include_headers(headers, args.unwrap_or_default());
+
+    let (uuid, tx) = push(
+        tx,
+        &w_id,
+        JobPayload::Flow(flow_path.to_string()),
+        args,
+        &authed.username,
+        &authed.email,
+        username_to_permissioned_as(&authed.username),
+        scheduled_for,
+        None,
+        run_query.parent_job,
+        false,
+        false,
+        None,
+        !run_query.invisible_to_owner.unwrap_or(false),
+    )
+    .await?;
+
+    tx.commit().await?;
+
+    run_wait_result(
+        authed,
+        Extension(user_db),
+        timeout.0,
+        uuid,
+        Path((w_id, flow_path)),
+    )
+    .await
+}
+
 // a similar function exists on the worker
 pub async fn script_path_to_payload<'c>(
     script_path: &str,
@@ -1522,6 +1589,7 @@ pub struct JobUpdate {
     pub running: Option<bool>,
     pub completed: Option<bool>,
     pub new_logs: Option<String>,
+    pub mem_peak: Option<i32>,
 }
 
 async fn get_job_update(
@@ -1531,8 +1599,8 @@ async fn get_job_update(
 ) -> error::JsonResult<JobUpdate> {
     let mut tx = db.begin().await?;
 
-    let logs = query_scalar!(
-        "SELECT substr(logs, $1) as logs FROM queue WHERE workspace_id = $2 AND id = $3",
+    let record = sqlx::query!(
+        "SELECT substr(logs, $1) as logs, mem_peak FROM queue WHERE workspace_id = $2 AND id = $3",
         log_offset,
         &w_id,
         &id
@@ -1540,12 +1608,13 @@ async fn get_job_update(
     .fetch_optional(&mut tx)
     .await?;
 
-    if let Some(logs) = logs {
+    if let Some(record) = record {
         tx.commit().await?;
         Ok(Json(JobUpdate {
             running: if !running { Some(true) } else { None },
             completed: None,
-            new_logs: logs,
+            new_logs: record.logs,
+            mem_peak: record.mem_peak,
         }))
     } else {
         let logs = query_scalar!(
@@ -1563,6 +1632,7 @@ async fn get_job_update(
             running: Some(false),
             completed: Some(true),
             new_logs: logs,
+            mem_peak: record.map(|r| r.mem_peak).flatten(),
         }))
     }
 }
@@ -1680,6 +1750,7 @@ async fn list_completed_jobs(
             "is_skipped",
             "email",
             "visible_to_owner",
+            "mem_peak",
         ],
     )
     .sql()?;

backend/windmill-api/src/lib.rs

@@ -21,7 +21,7 @@ use crate::{
     oauth2::{build_oauth_clients, SlackVerifier},
     tracing_init::{MyMakeSpan, MyOnResponse},
     users::{Authed, OptAuthed},
-    webhook_util::{WebhookShared, WebhookUtil},
+    webhook_util::WebhookShared,
 };
 
 mod apps;
@@ -109,7 +109,7 @@ pub async fn run_server(
         ))))
         .layer(Extension(http_client))
         .layer(CookieManagerLayer::new())
-        .layer(Extension(WebhookShared::new(rx.resubscribe())));
+        .layer(Extension(WebhookShared::new(rx.resubscribe(), db.clone())));
     // build our application with a route
     let app = Router::new()
         .nest(
@@ -150,8 +150,7 @@ pub async fn run_server(
                         .nest("/flows", flows::workspaced_service())
                         .nest("/capture", capture::workspaced_service())
                         .nest("/favorites", favorite::workspaced_service())
-                        .nest("/folders", folders::workspaced_service())
-                        .route_layer(from_extractor::<WebhookUtil>()),
+                        .nest("/folders", folders::workspaced_service()),
                 )
                 .nest("/workspaces", workspaces::global_service())
                 .nest(

backend/windmill-api/src/resources.rs

@@ -9,7 +9,7 @@
 use crate::{
     db::{UserDB, DB},
     users::{require_owner_of_path, Authed},
-    webhook_util::{WebhookMessage, WebhookUtil},
+    webhook_util::{WebhookMessage, WebhookShared},
 };
 use axum::{
     extract::{Extension, Path, Query},
@@ -264,7 +264,7 @@ async fn check_path_conflict<'c>(
 async fn create_resource(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
-    webhook: WebhookUtil,
+    Extension(webhook): Extension<WebhookShared>,
     Path(w_id): Path<String>,
     Json(resource): Json<CreateResource>,
 ) -> Result<(StatusCode, String)> {
@@ -295,10 +295,10 @@ async fn create_resource(
     .await?;
     tx.commit().await?;
 
-    webhook.send_message(WebhookMessage::CreateResource {
-        workspace: w_id.clone(),
-        path: resource.path.clone(),
-    });
+    webhook.send_message(
+        w_id.clone(),
+        WebhookMessage::CreateResource { workspace: w_id, path: resource.path.clone() },
+    );
 
     Ok((
         StatusCode::CREATED,
@@ -309,7 +309,7 @@ async fn create_resource(
 async fn delete_resource(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
-    webhook: WebhookUtil,
+    Extension(webhook): Extension<WebhookShared>,
     Path((w_id, path)): Path<(String, StripPath)>,
 ) -> Result<String> {
     let path = path.to_path();
@@ -341,10 +341,10 @@ async fn delete_resource(
     .await?;
     tx.commit().await?;
 
-    webhook.send_message(WebhookMessage::DeleteResource {
-        workspace: w_id.clone(),
-        path: path.to_owned(),
-    });
+    webhook.send_message(
+        w_id.clone(),
+        WebhookMessage::DeleteResource { workspace: w_id, path: path.to_owned() },
+    );
 
     Ok(format!("resource {} deleted", path))
 }
@@ -352,7 +352,7 @@ async fn delete_resource(
 async fn update_resource(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
-    webhook: WebhookUtil,
+    Extension(webhook): Extension<WebhookShared>,
     Extension(db): Extension<DB>,
     Path((w_id, path)): Path<(String, StripPath)>,
     Json(ns): Json<EditResource>,
@@ -414,11 +414,14 @@ async fn update_resource(
     .await?;
     tx.commit().await?;
 
-    webhook.send_message(WebhookMessage::UpdateResource {
-        workspace: w_id.clone(),
-        old_path: path.to_owned(),
-        new_path: npath.clone(),
-    });
+    webhook.send_message(
+        w_id.clone(),
+        WebhookMessage::UpdateResource {
+            workspace: w_id,
+            old_path: path.to_owned(),
+            new_path: npath.clone(),
+        },
+    );
 
     Ok(format!("resource {} updated (npath: {:?})", path, npath))
 }
@@ -431,7 +434,7 @@ struct UpdateResource {
 async fn update_resource_value(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
-    webhook: WebhookUtil,
+    Extension(webhook): Extension<WebhookShared>,
     Path((w_id, path)): Path<(String, StripPath)>,
     Json(nv): Json<UpdateResource>,
 ) -> Result<String> {
@@ -457,11 +460,14 @@ async fn update_resource_value(
     )
     .await?;
     tx.commit().await?;
-    webhook.send_message(WebhookMessage::UpdateResource {
-        workspace: w_id.clone(),
-        old_path: path.to_owned(),
-        new_path: path.to_owned(),
-    });
+    webhook.send_message(
+        w_id.clone(),
+        WebhookMessage::UpdateResource {
+            workspace: w_id,
+            old_path: path.to_owned(),
+            new_path: path.to_owned(),
+        },
+    );
 
     Ok(format!("value of resource {} updated", path))
 }
@@ -539,7 +545,7 @@ async fn exists_resource_type(
 async fn create_resource_type(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
-    webhook: WebhookUtil,
+    Extension(webhook): Extension<WebhookShared>,
     Path(w_id): Path<String>,
     Json(resource_type): Json<CreateResourceType>,
 ) -> Result<(StatusCode, String)> {
@@ -570,7 +576,10 @@ async fn create_resource_type(
     .await?;
     tx.commit().await?;
 
-    webhook.send_message(WebhookMessage::CreateResourceType { name: resource_type.name.clone() });
+    webhook.send_message(
+        w_id.clone(),
+        WebhookMessage::CreateResourceType { name: resource_type.name.clone() },
+    );
 
     Ok((
         StatusCode::CREATED,
@@ -603,7 +612,7 @@ async fn check_rt_path_conflict<'c>(
 async fn delete_resource_type(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
-    webhook: WebhookUtil,
+    Extension(webhook): Extension<WebhookShared>,
     Path((w_id, name)): Path<(String, String)>,
 ) -> Result<String> {
     require_admin(authed.is_admin, &authed.username)?;
@@ -628,7 +637,10 @@ async fn delete_resource_type(
     )
     .await?;
     tx.commit().await?;
-    webhook.send_message(WebhookMessage::DeleteResourceType { name: name.clone() });
+    webhook.send_message(
+        w_id.clone(),
+        WebhookMessage::DeleteResourceType { name: name.clone() },
+    );
 
     Ok(format!("resource_type {} deleted", name))
 }
@@ -636,7 +648,7 @@ async fn delete_resource_type(
 async fn update_resource_type(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
-    webhook: WebhookUtil,
+    Extension(webhook): Extension<WebhookShared>,
     Path((w_id, name)): Path<(String, String)>,
     Json(ns): Json<EditResourceType>,
 ) -> Result<String> {
@@ -666,7 +678,10 @@ async fn update_resource_type(
     )
     .await?;
     tx.commit().await?;
-    webhook.send_message(WebhookMessage::UpdateResourceType { name: name.clone() });
+    webhook.send_message(
+        w_id.clone(),
+        WebhookMessage::UpdateResourceType { name: name.clone() },
+    );
 
     Ok(format!("resource_type {} updated", name))
 }

backend/windmill-api/src/scripts.rs

@@ -14,7 +14,7 @@ use crate::{
     db::{UserDB, DB},
     schedule::clear_schedule,
     users::{require_owner_of_path, Authed},
-    webhook_util::{WebhookMessage, WebhookUtil},
+    webhook_util::{WebhookMessage, WebhookShared},
 };
 use axum::{
     extract::{Extension, Path, Query},
@@ -185,7 +185,7 @@ fn hash_script(ns: &NewScript) -> i64 {
 async fn create_script(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
-    webhook: WebhookUtil,
+    Extension(webhook): Extension<WebhookShared>,
     Extension(db): Extension<DB>,
     Path(w_id): Path<String>,
     Json(ns): Json<NewScript>,
@@ -402,11 +402,14 @@ async fn create_script(
             Some([("hash", hash.to_string().as_str())].into()),
         )
         .await?;
-        webhook.send_message(WebhookMessage::UpdateScript {
-            workspace: w_id.clone(),
-            path: ns.path.clone(),
-            hash: hash.to_string(),
-        });
+        webhook.send_message(
+            w_id.clone(),
+            WebhookMessage::UpdateScript {
+                workspace: w_id,
+                path: ns.path.clone(),
+                hash: hash.to_string(),
+            },
+        );
     } else {
         audit_log(
             &mut tx,
@@ -424,11 +427,14 @@ async fn create_script(
             ),
         )
         .await?;
-        webhook.send_message(WebhookMessage::CreateScript {
-            workspace: w_id.clone(),
-            path: ns.path.clone(),
-            hash: hash.to_string(),
-        });
+        webhook.send_message(
+            w_id.clone(),
+            WebhookMessage::CreateScript {
+                workspace: w_id,
+                path: ns.path.clone(),
+                hash: hash.to_string(),
+            },
+        );
     }
 
     tx.commit().await?;
@@ -612,7 +618,7 @@ async fn get_deployment_status(
 
 async fn archive_script_by_path(
     authed: Authed,
-    webhook: WebhookUtil,
+    Extension(webhook): Extension<WebhookShared>,
     Extension(user_db): Extension<UserDB>,
     Extension(db): Extension<DB>,
     Path((w_id, path)): Path<(String, StripPath)>,
@@ -639,10 +645,10 @@ async fn archive_script_by_path(
     )
     .await?;
     tx.commit().await?;
-    webhook.send_message(WebhookMessage::DeleteScript {
-        workspace: w_id.clone(),
-        hash: hash.to_string(),
-    });
+    webhook.send_message(
+        w_id.clone(),
+        WebhookMessage::DeleteScript { workspace: w_id, hash: hash.to_string() },
+    );
 
     Ok(())
 }
@@ -650,7 +656,7 @@ async fn archive_script_by_path(
 async fn archive_script_by_hash(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
-    webhook: WebhookUtil,
+    Extension(webhook): Extension<WebhookShared>,
     Path((w_id, hash)): Path<(String, ScriptHash)>,
 ) -> JsonResult<Script> {
     let mut tx = user_db.begin(&authed).await?;
@@ -675,10 +681,10 @@ async fn archive_script_by_hash(
     .await?;
     tx.commit().await?;
 
-    webhook.send_message(WebhookMessage::DeleteScript {
-        workspace: w_id.clone(),
-        hash: hash.to_string(),
-    });
+    webhook.send_message(
+        w_id.clone(),
+        WebhookMessage::DeleteScript { workspace: w_id, hash: hash.to_string() },
+    );
 
     Ok(Json(script))
 }
@@ -686,7 +692,7 @@ async fn archive_script_by_hash(
 async fn delete_script_by_hash(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
-    webhook: WebhookUtil,
+    Extension(webhook): Extension<WebhookShared>,
     Extension(db): Extension<DB>,
     Path((w_id, hash)): Path<(String, ScriptHash)>,
 ) -> JsonResult<Script> {
@@ -715,10 +721,10 @@ async fn delete_script_by_hash(
     .await?;
     tx.commit().await?;
 
-    webhook.send_message(WebhookMessage::DeleteScript {
-        workspace: w_id.clone(),
-        hash: hash.to_string(),
-    });
+    webhook.send_message(
+        w_id.clone(),
+        WebhookMessage::DeleteScript { workspace: w_id, hash: hash.to_string() },
+    );
 
     Ok(Json(script))
 }

backend/windmill-api/src/variables.rs

@@ -12,7 +12,7 @@ use crate::{
     db::{UserDB, DB},
     oauth2::{AllClients, _refresh_token},
     users::{require_owner_of_path, Authed},
-    webhook_util::{WebhookMessage, WebhookUtil},
+    webhook_util::{WebhookMessage, WebhookShared},
     BaseUrl,
 };
 /*
@@ -225,7 +225,7 @@ async fn check_path_conflict<'c>(
 async fn create_variable(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
-    webhook: WebhookUtil,
+    Extension(webhook): Extension<WebhookShared>,
     Path(w_id): Path<String>,
     Json(variable): Json<CreateVariable>,
 ) -> Result<(StatusCode, String)> {
@@ -267,10 +267,10 @@ async fn create_variable(
 
     tx.commit().await?;
 
-    webhook.send_message(WebhookMessage::CreateVariable {
-        workspace: w_id.clone(),
-        path: variable.path.clone(),
-    });
+    webhook.send_message(
+        w_id.clone(),
+        WebhookMessage::CreateVariable { workspace: w_id, path: variable.path.clone() },
+    );
 
     Ok((
         StatusCode::CREATED,
@@ -281,7 +281,7 @@ async fn create_variable(
 async fn delete_variable(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
-    webhook: WebhookUtil,
+    Extension(webhook): Extension<WebhookShared>,
     Path((w_id, path)): Path<(String, StripPath)>,
 ) -> Result<String> {
     let path = path.to_path();
@@ -314,10 +314,10 @@ async fn delete_variable(
 
     tx.commit().await?;
 
-    webhook.send_message(WebhookMessage::DeleteVariable {
-        workspace: w_id.clone(),
-        path: path.to_owned(),
-    });
+    webhook.send_message(
+        w_id.clone(),
+        WebhookMessage::DeleteVariable { workspace: w_id, path: path.to_owned() },
+    );
 
     Ok(format!("variable {} deleted", path))
 }
@@ -333,7 +333,7 @@ struct EditVariable {
 async fn update_variable(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
-    webhook: WebhookUtil,
+    Extension(webhook): Extension<WebhookShared>,
     Extension(db): Extension<DB>,
     Path((w_id, path)): Path<(String, StripPath)>,
     Json(ns): Json<EditVariable>,
@@ -419,11 +419,14 @@ async fn update_variable(
     .await?;
     tx.commit().await?;
 
-    webhook.send_message(WebhookMessage::UpdateVariable {
-        workspace: w_id.clone(),
-        old_path: path.to_owned(),
-        new_path: npath.clone(),
-    });
+    webhook.send_message(
+        w_id.clone(),
+        WebhookMessage::UpdateVariable {
+            workspace: w_id,
+            old_path: path.to_owned(),
+            new_path: npath.clone(),
+        },
+    );
 
     Ok(format!("variable {} updated (npath: {:?})", path, npath))
 }

backend/windmill-api/src/webhook_util.rs

@@ -1,14 +1,7 @@
 use std::time::Duration;
 
-use axum::{
-    async_trait,
-    extract::{FromRequestParts, OriginalUri},
-    http::request::Parts,
-    Extension,
-};
-use hyper::StatusCode;
 use serde::Serialize;
-use tokio::{select, sync::mpsc};
+use tokio::{select, sync::mpsc, time::interval};
 
 use crate::db::DB;
 
@@ -54,7 +47,7 @@ pub struct WebhookShared {
 }
 
 impl WebhookShared {
-    pub fn new(mut shutdown_rx: tokio::sync::broadcast::Receiver<()>) -> Self {
+    pub fn new(mut shutdown_rx: tokio::sync::broadcast::Receiver<()>, db: DB) -> Self {
         let (tx, mut rx) = mpsc::unbounded_channel::<(String, WebhookMessage)>();
         let _process = tokio::spawn(async move {
             let client = reqwest::Client::builder()
@@ -62,95 +55,58 @@ impl WebhookShared {
                 .timeout(Duration::from_secs(5))
                 .build()
                 .unwrap();
+            let cache = retainer::Cache::new();
+            let mut cache_purge_interval = interval(Duration::from_secs(30));
+
             loop {
                 select! {
                     biased;
                     _ = shutdown_rx.recv() => break,
                     r = rx.recv() => match r {
-                        Some((url, message)) => {
-                            let timer = WEBHOOK_REQUEST_COUNT.start_timer();
-                            let _ = client.post(url).json(&message).send().await;
-                            timer.stop_and_record();
+                        Some((workspace_id, message)) => {
+                            let url_guard = match cache.get(&workspace_id).await {
+                                Some(guard) => {
+                                    guard
+                                },
+                                None => {
+                                    let Ok(webook_opt) =
+                                        sqlx::query_scalar!(
+                                            "SELECT webhook FROM workspace_settings WHERE workspace_id = $1",
+                                            workspace_id
+                                        )
+                                        .fetch_one(
+                                            &db,
+                                        )
+                                        .await else {
+                                            tracing::error!("Webhook Message to send - but cannot get workspace settings! Workspace: {workspace_id}");
+                                            continue;
+                                        };
+                                    cache.insert(workspace_id.clone(), webook_opt, Duration::from_secs(30)).await;
+                                    cache.get(&workspace_id).await.unwrap()
+                                }
+                            };
+                            let webook_opt = url_guard.value();
+                            if let Some(url) = webook_opt {
+                                let timer = WEBHOOK_REQUEST_COUNT.start_timer();
+                                let _ = client.post(url).json(&message).send().await;
+                                timer.stop_and_record();
+                                drop(url_guard);
+                            }
                         },
                         None => break,
-                    }
+                    },
+                    _ = futures::future::poll_fn(|cx| cache_purge_interval.poll_tick(cx)) => {
+                        tracing::trace!("Purging Webhook Cache");
+                        cache.purge(10, 0.50).await;
+                    },
                 }
             }
         });
 
         Self { channel: tx }
     }
-}
 
-#[derive(Clone)]
-pub struct WebhookUtil {
-    webhook: Option<String>,
-    shared: Extension<WebhookShared>,
-}
-
-impl WebhookUtil {
-    pub fn send_message(&self, message: WebhookMessage) {
-        let Some(webhook) = &self.webhook else {
-            return;
-        };
-        let _ = self.shared.channel.send((webhook.clone(), message));
-    }
-}
-
-#[async_trait]
-impl<S> FromRequestParts<S> for WebhookUtil
-where
-    S: Send + Sync,
-{
-    type Rejection = (StatusCode, String);
-
-    async fn from_request_parts(
-        parts: &mut Parts,
-        state: &S,
-    ) -> std::result::Result<Self, Self::Rejection> {
-        let original_uri = OriginalUri::from_request_parts(parts, state)
-            .await
-            .ok()
-            .map(|x| x.0)
-            .unwrap_or_default();
-        let path_vec: Vec<&str> = original_uri.path().split("/").collect();
-        let workspace_id = if path_vec.len() >= 4 && path_vec[0] == "" && path_vec[2] == "w" {
-            Some(path_vec[3].to_owned())
-        } else {
-            None
-        };
-
-        let webhook = sqlx::query_scalar!(
-            "SELECT webhook FROM workspace_settings WHERE workspace_id = $1",
-            workspace_id
-        )
-        .fetch_one(
-            &Extension::<DB>::from_request_parts(parts, state)
-                .await
-                .map_err(|_| {
-                    (
-                        StatusCode::INTERNAL_SERVER_ERROR,
-                        "Could not aquire DB while retrieving webhook".to_owned(),
-                    )
-                })?
-                .0,
-        )
-        .await
-        .map_err(|e| {
-            (
-                StatusCode::INTERNAL_SERVER_ERROR,
-                format!("Could not execute DB query {:?}", e),
-            )
-        })?;
-        let shared = Extension::<WebhookShared>::from_request_parts(parts, state)
-            .await
-            .map_err(|_| {
-                (
-                    StatusCode::INTERNAL_SERVER_ERROR,
-                    "Could not aquire shared process while retrieving webhook".to_owned(),
-                )
-            })?;
-
-        Ok(Self { webhook, shared })
+    pub fn send_message(&self, workspace_id: String, message: WebhookMessage) {
+        let _ = self.channel.send((workspace_id.clone(), message));
     }
 }

backend/windmill-api/src/workspaces.rs

@@ -240,28 +240,38 @@ async fn stripe_checkout(
             params.line_items = match plan.plan.as_str() {
                 "team" => Some(vec![
                     stripe::CreateCheckoutSessionLineItems {
-                        quantity: None,
-                        price: Some("price_1MSdSyGU3NdFi9eLMdV6cS6F".to_string()),
+                        quantity: Some(1),
+                        price: Some("price_1MUlrWGU3NdFi9eLE9GBZhoY".to_string()),
                         ..Default::default()
                     },
                     stripe::CreateCheckoutSessionLineItems {
                         quantity: None,
-                        price: Some("price_1MShsNGU3NdFi9eLJMEZUW8b".to_string()),
-                        ..Default::default()
-                    },
-                ]),
-                "enterprise" => Some(vec![
-                    stripe::CreateCheckoutSessionLineItems {
-                        quantity: None,
-                        price: Some("price_1MSdf6GU3NdFi9eLJFRkntlx".to_string()),
-                        ..Default::default()
-                    },
-                    stripe::CreateCheckoutSessionLineItems {
-                        quantity: None,
-                        price: Some("price_1MShsNGU3NdFi9eLJMEZUW8b".to_string()),
+                        price: Some("price_1MUlreGU3NdFi9eLi6sOyvVa".to_string()),
+                        ..Default::default()
+                    },
+                    stripe::CreateCheckoutSessionLineItems {
+                        quantity: None,
+                        price: Some("price_1MUlrlGU3NdFi9eLFLggSXZV".to_string()),
+                        ..Default::default()
+                    },
+                    stripe::CreateCheckoutSessionLineItems {
+                        quantity: None,
+                        price: Some("price_1MUlr3GU3NdFi9eLbZYFjR9p".to_string()),
                         ..Default::default()
                     },
                 ]),
+                // "enterprise" => Some(vec![
+                //     stripe::CreateCheckoutSessionLineItems {
+                //         quantity: None,
+                //         price: Some("price_1MSdf6GU3NdFi9eLJFRkntlx".to_string()),
+                //         ..Default::default()
+                //     },
+                //     stripe::CreateCheckoutSessionLineItems {
+                //         quantity: None,
+                //         price: Some("price_1MShsNGU3NdFi9eLJMEZUW8b".to_string()),
+                //         ..Default::default()
+                //     },
+                // ]),
                 _ => Err(Error::BadRequest("invalid plan".to_string()))?,
             };
             params.customer_email = Some(&authed.email);

backend/windmill-queue/src/jobs.rs

@@ -45,7 +45,7 @@ lazy_static::lazy_static! {
 }
 
 const MAX_FREE_EXECS: i32 = 1000;
-const MAX_FREE_CONCURRENT_RUNS: i32 = 3;
+const MAX_FREE_CONCURRENT_RUNS: i32 = 15;
 
 pub async fn cancel_job<'c>(
     username: &str,
@@ -82,7 +82,32 @@ pub async fn cancel_job<'c>(
     Ok((tx, job_option))
 }
 
-pub async fn pull(db: &Pool<Postgres>) -> windmill_common::error::Result<Option<QueuedJob>> {
+pub async fn pull(
+    db: &Pool<Postgres>,
+    whitelist_workspaces: Option<Vec<String>>,
+    blacklist_workspaces: Option<Vec<String>>,
+) -> windmill_common::error::Result<Option<QueuedJob>> {
+    let mut workspaces_filter = String::new();
+    if let Some(whitelist) = whitelist_workspaces {
+        workspaces_filter.push_str(&format!(
+            " AND workspace_id IN ({})",
+            whitelist
+                .into_iter()
+                .map(|x| format!("'{x}'"))
+                .collect::<Vec<String>>()
+                .join(",")
+        ));
+    }
+    if let Some(blacklist) = blacklist_workspaces {
+        workspaces_filter.push_str(&format!(
+            " AND workspace_id NOT IN ({})",
+            blacklist
+                .into_iter()
+                .map(|x| format!("'{x}'"))
+                .collect::<Vec<String>>()
+                .join(",")
+        ));
+    }
     /* Jobs can be started if they:
      * - haven't been started before,
      *   running = false
@@ -90,7 +115,7 @@ pub async fn pull(db: &Pool<Postgres>) -> windmill_common::error::Result<Option<
      *   suspend_until is non-null
      *   and suspend = 0 when the resume messages are received
      *   or suspend_until <= now() if it has timed out */
-    let job: Option<QueuedJob> = sqlx::query_as::<_, QueuedJob>(
+    let job: Option<QueuedJob> = sqlx::query_as::<_, QueuedJob>(&format!(
         "UPDATE queue
             SET running = true
               , started_at = coalesce(started_at, now())
@@ -99,17 +124,17 @@ pub async fn pull(db: &Pool<Postgres>) -> windmill_common::error::Result<Option<
             WHERE id = (
                 SELECT id
                 FROM queue
-                WHERE (    running = false
+                WHERE ((running = false
                        AND scheduled_for <= now())
                    OR (suspend_until IS NOT NULL
                        AND (   suspend <= 0
-                            OR suspend_until <= now()))
+                            OR suspend_until <= now()))) {workspaces_filter}
                 ORDER BY scheduled_for
                 FOR UPDATE SKIP LOCKED
                 LIMIT 1
             )
-            RETURNING *",
-    )
+            RETURNING *"
+    ))
     .fetch_optional(db)
     .await?;
 
@@ -646,6 +671,8 @@ pub struct QueuedJob {
     pub visible_to_owner: bool,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub suspend: Option<i32>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub mem_peak: Option<i32>,
 }
 
 impl QueuedJob {

backend/windmill-worker/src/jobs.rs

@@ -77,6 +77,12 @@ pub async fn add_completed_job(
             None
         };
 
+    let mem_peak = sqlx::query_scalar!("SELECT mem_peak FROM queue WHERE id = $1", &queued_job.id)
+        .fetch_optional(db)
+        .await
+        .ok()
+        .flatten()
+        .flatten();
     let mut tx = db.begin().await?;
     let job_id = queued_job.id.clone();
     sqlx::query!(
@@ -109,9 +115,10 @@ pub async fn add_completed_job(
                    , language
                    , email
                    , visible_to_owner
+                   , mem_peak
                 )
             VALUES ($1, $2, $3, $4, $5, $6, COALESCE($26, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE($6, now()))))*1000), $7, $8, $9,\
-                    $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $27, $28)
+                    $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $27, $28, $29)
          ON CONFLICT (id) DO UPDATE SET success = $7, result = $11, logs = concat(cj.logs, $12)",
         queued_job.workspace_id,
         queued_job.id,
@@ -140,7 +147,8 @@ pub async fn add_completed_job(
         queued_job.language: ScriptLang,
         duration: Option<i64>,
         queued_job.email,
-        queued_job.visible_to_owner
+        queued_job.visible_to_owner,
+        mem_peak
     )
     .execute(&mut tx)
     .await

backend/windmill-worker/src/js_eval.rs

@@ -307,19 +307,6 @@ async fn op_get_id(args: Vec<String>) -> Result<Option<serde_json::Value>, anyho
     let node_id = &args[4];
 
     let client = windmill_api_client::create_client(base_url, token.clone());
-    let err = client
-        .result_by_id(workspace, flow_job_id, node_id, Some(true))
-        .await
-        .err()
-        .unwrap();
-    let res = match err {
-        windmill_api_client::Error::UnexpectedResponse(e) => {
-            tracing::error!("{:?}", e.text().await);
-            anyhow::anyhow!("bar")
-        }
-        _ => anyhow::anyhow!("foo"),
-    };
-    tracing::error!("{:?}", res);
     let result = client
         .result_by_id(workspace, flow_job_id, node_id, Some(true))
         .await

backend/windmill-worker/src/worker.rs

@@ -26,7 +26,7 @@ use windmill_common::{
     utils::rd_string,
     variables,
 };
-use windmill_queue::{canceled_job_to_result, get_queued_job, pull, JobKind, QueuedJob};
+use windmill_queue::{canceled_job_to_result, get_queued_job, pull, JobKind, QueuedJob, CLOUD_HOSTED};
 
 use serde_json::{json, Value};
 
@@ -35,8 +35,7 @@ use tokio::{
     io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader},
     process::{Child, Command},
     sync::{
-        mpsc::{self, Sender},
-        oneshot, watch,
+        mpsc::{self, Sender},  watch,
     },
     time::{interval, sleep, Instant, MissedTickBehavior},
 };
@@ -133,7 +132,7 @@ async fn copy_cache_to_bucket_as_tar(bucket: &str) {
         .current_dir(ROOT_CACHE_DIR)
         .arg("-c")
         .arg("-f")
-        .arg(format!("{ROOT_CACHE_DIR}/{TAR_CACHE_FILENAME}"))
+        .arg(format!("{ROOT_CACHE_DIR}{TAR_CACHE_FILENAME}"))
         .args(&["pip", "go", "deno"])
         .stdin(Stdio::null())
         .stdout(Stdio::null())
@@ -181,7 +180,7 @@ async fn copy_cache_from_bucket_as_tar(bucket: &str) -> bool {
     match Command::new("rclone")
         .arg("copyto")
         .arg(format!(":s3,env_auth=true:{bucket}/{TAR_CACHE_FILENAME}"))
-        .arg(format!("{ROOT_CACHE_DIR}/{TAR_CACHE_FILENAME}"))
+        .arg(format!("{ROOT_CACHE_DIR}{TAR_CACHE_FILENAME}"))
         .arg("--size-only")
         .arg("--fast-list")
         .stdin(Stdio::null())
@@ -317,6 +316,11 @@ lazy_static::lazy_static! {
     );
 }
 
+//only matter if CLOUD_HOSTED
+const MAX_RESULT_SIZE: usize = 1024 * 1024 * 2; // 2MB
+
+
+
 #[tracing::instrument(level = "trace")]
 pub async fn run_worker(
     db: &Pool<Postgres>,
@@ -404,6 +408,13 @@ pub async fn run_worker(
     )
     .expect("register prometheus metric");
 
+    let worker_busy: prometheus::IntGauge = prometheus::register_int_gauge!(prometheus::Opts::new(
+        "worker_busy",
+        "Is the worker busy executing a job?",
+    )
+    .const_label("name", &worker_name))
+    .unwrap();
+
     let mut jobs_executed = 0;
 
     let deno_path = std::env::var("DENO_PATH").unwrap_or_else(|_| "/usr/bin/deno".to_string());
@@ -420,13 +431,28 @@ pub async fn run_worker(
         .ok()
         .map(|x| format!(";{x}"))
         .unwrap_or_else(|| String::new());
-    let max_log_size = std::env::var("MAX_LOG_SIZE")
-        .ok()
-        .and_then(|x| x.parse::<i64>().ok())
-        .unwrap_or(500000);
+
+    
+
+
     let deno_flags = std::env::var("DENO_FLAGS")
         .ok()
         .map(|x| x.split(' ').map(|x| x.to_string()).collect());
+    let pip_local_dependencies = std::env::var("PIP_LOCAL_DEPENDENCIES")
+        .ok()
+        .map(|x| x.split(',').map(|x| x.to_string()).collect());
+    let whitelist_workspaces = std::env::var("WHITELIST_WORKSPACES")
+        .ok()
+        .map(|x| x.split(',').map(|x| x.to_string()).collect());
+    let blacklist_workspaces = std::env::var("BLACKLIST_WORKSPACES")
+        .ok()
+        .map(|x| x.split(',').map(|x| x.to_string()).collect());
+
+    let pip_local_dependencies = if pip_local_dependencies == Some(vec!["".to_string()]) {
+        None
+    } else {
+        pip_local_dependencies
+    };
 
     #[cfg(feature = "enterprise")]
     let tar_cache_rate = std::env::var("TAR_CACHE_RATE")
@@ -434,6 +460,10 @@ pub async fn run_worker(
         .and_then(|x| x.parse::<i32>().ok())
         .unwrap_or(100);
 
+    let additional_python_paths = std::env::var("ADDITIONAL_PYTHON_PATHS")
+        .ok()
+        .map(|x| x.split(':').map(|x| x.to_string()).collect());
+
     let envs = Envs {
         deno_path,
         go_path,
@@ -444,9 +474,10 @@ pub async fn run_worker(
         pip_index_url,
         pip_extra_index_url,
         pip_trusted_host,
-        max_log_size,
         deno_flags,
         deno_auth_tokens,
+        pip_local_dependencies,
+        additional_python_paths,
     };
     WORKER_STARTED.inc();
 
@@ -466,11 +497,14 @@ pub async fn run_worker(
     let (same_worker_tx, mut same_worker_rx) = mpsc::channel::<Uuid>(5);
 
     loop {
+        worker_busy.set(0);
+
         uptime_metric.inc_by(
             ((Instant::now() - start_time).as_millis() - uptime_metric.get() as u128)
                 .try_into()
                 .unwrap(),
         );
+
         let do_break = async {
             if last_ping.elapsed().as_secs() > NUM_SECS_ENV_CHECK {
                 sqlx::query!(
@@ -511,7 +545,9 @@ pub async fn run_worker(
                         .await
                         .map_err(|_| Error::InternalErr("Impossible to fetch same_worker job".to_string())))
                     },
-                    (job, timer) = {let timer = worker_pull_duration.start_timer(); pull(&db).map(|x| (x, timer)) } => {
+                    (job, timer) = {
+                        let timer = worker_pull_duration.start_timer(); 
+                        pull(&db, whitelist_workspaces.clone(), blacklist_workspaces.clone()).map(|x| (x, timer)) } => {
                         drop(timer);
                         (false, job)
                     },
@@ -521,6 +557,7 @@ pub async fn run_worker(
                 return true;
             }
 
+            worker_busy.set(1);
             match next_job {
                 Ok(Some(job)) => {
                     let label_values = [
@@ -756,7 +793,8 @@ struct Envs {
     pip_trusted_host: Option<String>,
     deno_auth_tokens: String,
     deno_flags: Option<Vec<String>>,
-    max_log_size: i64,
+    pip_local_dependencies: Option<Vec<String>>,
+    additional_python_paths: Option<Vec<String>>,
 }
 
 fn extract_error_value(log_lines: &str) -> serde_json::Value {
@@ -1150,7 +1188,7 @@ mount {{
 #[tracing::instrument(level = "trace", skip_all)]
 async fn handle_go_job(
     WorkerConfig { base_internal_url, disable_nuser, disable_nsjail, base_url, .. }: &WorkerConfig,
-    Envs { nsjail_path, go_path, path_env, home_env, max_log_size, .. }: &Envs,
+    Envs { nsjail_path, go_path, path_env, home_env, .. }: &Envs,
     logs: &mut String,
     job: &QueuedJob,
     db: &sqlx::Pool<sqlx::Postgres>,
@@ -1188,7 +1226,6 @@ async fn handle_go_job(
         job_dir,
         db,
         timeout,
-        *max_log_size,
         go_path,
         true,
         skip_go_mod,
@@ -1309,7 +1346,7 @@ func Run(req Req) (interface{{}}, error){{
             .stdout(Stdio::piped())
             .stderr(Stdio::piped())
             .spawn()?;
-        handle_child(&job.id, db, logs, timeout, *max_log_size, build_go).await?;
+        handle_child(&job.id, db, logs, timeout,  build_go, false).await?;
 
         Command::new(nsjail_path)
             .current_dir(job_dir)
@@ -1335,14 +1372,14 @@ func Run(req Req) (interface{{}}, error){{
             .stderr(Stdio::piped())
             .spawn()?
     };
-    handle_child(&job.id, db, logs, timeout, *max_log_size, child).await?;
+    handle_child(&job.id, db, logs, timeout, child, !disable_nsjail).await?;
     read_result(job_dir).await
 }
 
 #[tracing::instrument(level = "trace", skip_all)]
 async fn handle_bash_job(
     WorkerConfig { base_internal_url, disable_nuser, disable_nsjail, base_url, .. }: &WorkerConfig,
-    Envs { nsjail_path, path_env, home_env, max_log_size, .. }: &Envs,
+    Envs { nsjail_path, path_env, home_env,  .. }: &Envs,
     logs: &mut String,
     job: &QueuedJob,
     db: &sqlx::Pool<sqlx::Postgres>,
@@ -1414,7 +1451,7 @@ async fn handle_bash_job(
             .stderr(Stdio::piped())
             .spawn()?
     };
-    handle_child(&job.id, db, logs, timeout, *max_log_size, child).await?;
+    handle_child(&job.id, db, logs, timeout, child, !disable_nsjail).await?;
     //for now bash jobs have an empty result object
     Ok(serde_json::json!(logs
         .lines()
@@ -1434,7 +1471,7 @@ fn capitalize(s: &str) -> String {
 #[tracing::instrument(level = "trace", skip_all)]
 async fn handle_deno_job(
     WorkerConfig { base_internal_url, base_url, disable_nuser, disable_nsjail, .. }: &WorkerConfig,
-    Envs { nsjail_path, deno_path, path_env, max_log_size, deno_auth_tokens, deno_flags, .. }: &Envs,
+    Envs { nsjail_path, deno_path, path_env,  deno_auth_tokens, deno_flags, .. }: &Envs,
     logs: &mut String,
     job: &QueuedJob,
     db: &sqlx::Pool<sqlx::Postgres>,
@@ -1578,7 +1615,7 @@ run().catch(async (e) => {{
     }
     .instrument(trace_span!("create_deno_jail"))
     .await?;
-    handle_child(&job.id, db, logs, timeout, *max_log_size, child).await?;
+    handle_child(&job.id, db, logs, timeout,  child, !disable_nsjail).await?;
     read_result(job_dir).await
 }
 
@@ -1606,7 +1643,9 @@ lazy_static! {
 #[tracing::instrument(level = "trace", skip_all)]
 async fn handle_python_job(
     WorkerConfig { base_internal_url, base_url, disable_nuser, disable_nsjail, .. }: &WorkerConfig,
-    envs @ Envs { nsjail_path, python_path, path_env, max_log_size, .. }: &Envs,
+    envs @ Envs {
+        nsjail_path, python_path, path_env,  additional_python_paths, ..
+    }: &Envs,
     requirements_o: Option<String>,
     job_dir: &str,
     worker_dir: &str,
@@ -1622,7 +1661,8 @@ async fn handle_python_job(
 ) -> error::Result<serde_json::Value> {
     create_dependencies_dir(job_dir).await;
 
-    let mut additional_python_paths: Vec<String> = vec![];
+    let mut additional_python_paths: Vec<String> =
+        additional_python_paths.to_owned().unwrap_or_else(|| vec![]);
 
     let requirements = match requirements_o {
         Some(r) => r,
@@ -1729,6 +1769,15 @@ async fn handle_python_job(
     } else {
         ""
     };
+    let spread = if sig.star_kwargs {
+        "args = kwargs".to_string()
+    } else {
+        sig.args
+            .into_iter()
+            .map(|x| format!("args[\"{}\"] = kwargs[\"{}\"]", x.name, x.name))
+            .join("\n")
+    };
+
     let wrapper_content: String = format!(
         r#"
 import json
@@ -1742,12 +1791,15 @@ inner_script = __import__("inner")
 
 with open("args.json") as f:
     kwargs = json.load(f, strict=False)
-for k, v in list(kwargs.items()):
-    if v == '<function call>':
-        del kwargs[k]
+args = {{}}
+{spread}
 {transforms}
+for k, v in list(args.items()):
+    if v == '<function call>':
+        del args[k]
+
 try:
-    res = inner_script.main(**kwargs)
+    res = inner_script.main(**args)
     res_json = json.dumps(res, separators=(',', ':'), default=str).replace('\n', '')
     with open("result.json", 'w') as f:
         f.write(res_json)
@@ -1837,7 +1889,7 @@ mount {{
             .spawn()?
     };
 
-    handle_child(&job.id, db, logs, timeout, *max_log_size, child).await?;
+    handle_child(&job.id, db, logs, timeout, child, !disable_nsjail).await?;
     read_result(job_dir).await
 }
 
@@ -1853,6 +1905,10 @@ async fn read_result(job_dir: &str) -> error::Result<serde_json::Value> {
     let mut file = File::open(format!("{job_dir}/result.json")).await?;
     let mut content = "".to_string();
     file.read_to_string(&mut content).await?;
+    if *CLOUD_HOSTED && content.len() > MAX_RESULT_SIZE {
+        return Err(Error::ExecutionErr("Result is too large for the cloud app (limit 2MB). 
+        If using this script as part of the flow, use the shared folder to pass heavy data between steps.".to_owned()));
+    }
     serde_json::from_str(&content)
         .map_err(|e| Error::ExecutionErr(format!("Error parsing result: {e}")))
 }
@@ -2033,7 +2089,7 @@ async fn generate_deno_lock(
     job_dir: &str,
     db: &sqlx::Pool<sqlx::Postgres>,
     timeout: i32,
-    Envs { deno_path, max_log_size, .. }: &Envs,
+    Envs { deno_path,  .. }: &Envs,
 ) -> error::Result<String> {
     let _ = write_file(job_dir, "main.ts", code).await?;
 
@@ -2051,7 +2107,7 @@ async fn generate_deno_lock(
         .stderr(Stdio::piped())
         .spawn()?;
 
-    handle_child(job_id, db, logs, timeout, *max_log_size, child).await?;
+    handle_child(job_id, db, logs, timeout, * child).await?;
 
     let path_lock = format!("{job_dir}/lock.json");
     let mut file = File::open(path_lock).await?;
@@ -2082,7 +2138,6 @@ async fn capture_dependency_job(
                 job_dir,
                 db,
                 timeout,
-                envs.max_log_size,
                 &envs.go_path,
                 false,
                 false,
@@ -2101,7 +2156,14 @@ async fn pip_compile(
     requirements: &str,
     logs: &mut String,
     job_dir: &str,
-    Envs { pip_extra_index_url, pip_index_url, pip_trusted_host, max_log_size, .. }: &Envs,
+    Envs {
+        pip_extra_index_url,
+        pip_index_url,
+        pip_trusted_host,
+        
+        pip_local_dependencies,
+        ..
+    }: &Envs,
     db: &Pool<Postgres>,
     timeout: i32,
 ) -> error::Result<String> {
@@ -2109,6 +2171,15 @@ async fn pip_compile(
     set_logs(logs, job_id, db).await;
     logs.push_str(&format!("\ncontent of requirements:\n{}", requirements));
     let file = "requirements.in";
+    let requirements = if let Some(pip_local_dependencies) = pip_local_dependencies {
+        let deps = pip_local_dependencies.clone();
+        requirements
+            .lines()
+            .filter(|s| !deps.contains(&s.to_string()))
+            .join("\n")
+    } else {
+        requirements.to_string()
+    };
     write_file(job_dir, file, &requirements).await?;
 
     let mut args = vec!["-q", "--no-header", file, "--resolver=backtracking"];
@@ -2127,7 +2198,7 @@ async fn pip_compile(
         .stdout(Stdio::piped())
         .stderr(Stdio::piped())
         .spawn()?;
-    handle_child(job_id, db, logs, timeout, *max_log_size, child)
+    handle_child(job_id, db, logs, timeout, child, false)
         .await
         .map_err(|e| Error::ExecutionErr(format!("Lock file generation failed: {e:?}")))?;
     let path_lock = format!("{job_dir}/requirements.txt");
@@ -2149,7 +2220,6 @@ async fn install_go_dependencies(
     job_dir: &str,
     db: &sqlx::Pool<sqlx::Postgres>,
     timeout: i32,
-    max_log_size: i64,
     go_path: &str,
     preview: bool,
     skip_go_mod: bool,
@@ -2163,7 +2233,7 @@ async fn install_go_dependencies(
             .stderr(Stdio::piped())
             .spawn()?;
 
-        handle_child(job_id, db, logs, timeout, max_log_size, child).await?;
+        handle_child(job_id, db, logs, timeout,  child, false).await?;
     }
     let child = Command::new(go_path)
         .current_dir(job_dir)
@@ -2171,7 +2241,7 @@ async fn install_go_dependencies(
         .stdout(Stdio::piped())
         .stderr(Stdio::piped())
         .spawn()?;
-    handle_child(job_id, db, logs, timeout, max_log_size, child)
+    handle_child(job_id, db, logs, timeout,  child, false)
         .await
         .map_err(|e| Error::ExecutionErr(format!("Lock file generation failed: {e:?}")))?;
 
@@ -2241,6 +2311,30 @@ async fn get_reserved_variables(
         .collect())
 }
 
+async fn get_mem_peak(pid: Option<u32>, nsjail: bool) -> i32 {
+    if pid.is_none() {
+        return -1
+    }
+    let pid = if nsjail {
+        // This is a bit hacky, but the process id of the nsjail process is the pid of nsjail + 1. 
+        // Ideally, we would get the number from fork() itself. This works in MOST cases.
+        pid.unwrap() + 1
+    } else {
+        pid.unwrap()
+    };
+    
+    if let Ok(file) = File::open(format!("/proc/{}/status", pid)).await {
+        let mut lines = BufReader::new(file).lines();
+        while let Some(line) = lines.next_line().await.unwrap_or(None) {
+            if line.starts_with("VmHWM:") {
+                return line.split_whitespace().nth(1).and_then(|s| s.parse::<i32>().ok()).unwrap_or(-1);
+            };
+        }
+        -2
+    } else {
+        -3
+    }
+}
 /// - wait until child exits and return with exit status
 /// - read lines from stdout and stderr and append them to the "queue"."logs"
 ///   quitting early if output exceedes MAX_LOG_SIZE characters (not bytes)
@@ -2253,14 +2347,21 @@ async fn handle_child(
     db: &Pool<Postgres>,
     logs: &mut String,
     timeout: i32,
-    max_log_size: i64,
     mut child: Child,
+    nsjail: bool,
 ) -> error::Result<()> {
     let timeout = Duration::from_secs(u64::try_from(timeout).expect("invalid timeout"));
-    let ping_interval = Duration::from_secs(5);
-    let cancel_check_interval = Duration::from_millis(500);
+    let update_job_interval = Duration::from_millis(500);
     let write_logs_delay = Duration::from_millis(500);
 
+    let pid = child.id();
+    if let Some(pid) = pid {
+        //set the highest oom priority
+        let mut file = File::create(format!("/proc/{pid}/oom_score_adj")).await?;
+        let _ = file.write_all(b"1000").await;
+    } else {
+        tracing::info!("could not get child pid");
+    }
     let (set_too_many_logs, mut too_many_logs) = watch::channel::<bool>(false);
 
     let output = child_joined_output_stream(&mut child);
@@ -2270,17 +2371,20 @@ async fn handle_child(
 
     /* the cancellation future is polled on by `wait_on_child` while
      * waiting for the child to exit normally */
-    let cancel_check = async {
+    let update_job = async {
         let db = db.clone();
 
-        let mut interval = interval(cancel_check_interval);
+        let mut interval = interval(update_job_interval);
         interval.set_missed_tick_behavior(MissedTickBehavior::Skip);
 
         loop {
             tokio::select!(
                 _ = rx.recv() => break,
                 _ = interval.tick() => {
-                    if sqlx::query_scalar!("SELECT canceled FROM queue WHERE id = $1", job_id)
+                    let mem_peak = get_mem_peak(pid, nsjail).await;
+                    tracing::info!("{job_id} still running. mem peak: {}kB", mem_peak);
+                    let mem_peak = if mem_peak > 0 { Some(mem_peak) } else { None };
+                    if sqlx::query_scalar!("UPDATE queue SET mem_peak = GREATEST($1, mem_peak), last_ping = now() WHERE id = $2 RETURNING canceled", mem_peak, job_id)
                         .fetch_optional(&db)
                         .await
                         .map(|v| Some(true) == v)
@@ -2310,7 +2414,7 @@ async fn handle_child(
             biased;
             result = child.wait() => return result.map(Ok),
             Ok(()) = too_many_logs.changed() => KillReason::TooManyLogs,
-            _ = cancel_check => KillReason::Cancelled,
+            _ = update_job => KillReason::Cancelled,
             _ = sleep(timeout) => KillReason::Timeout,
         };
         tx.send(()).await.expect("rx should never be dropped");
@@ -2344,8 +2448,13 @@ async fn handle_child(
 
     /* a future that reads output from the child and appends to the database */
     let lines = async move {
+        let max_log_size = if *CLOUD_HOSTED {
+            MAX_RESULT_SIZE
+        } else {
+            usize::MAX
+        };
         /* log_remaining is zero when output limit was reached */
-        let mut log_remaining = (max_log_size as usize).saturating_sub(logs.chars().count());
+        let mut log_remaining = max_log_size.saturating_sub(logs.chars().count());
         let mut result = io::Result::Ok(());
         let mut output = output;
         /* `do_write` resolves the task, but does not contain the Result.
@@ -2376,7 +2485,7 @@ async fn handle_child(
                             tracing::info!(%job_id, "Too many logs lines for job {job_id}");
                             let _ = set_too_many_logs.send(true);
                             joined.push_str(&format!(
-                                "Job logs or result reached character limit of {max_log_size}; killing job."
+                                "Job logs or result reached character limit of {MAX_RESULT_SIZE}; killing job."
                             ));
                             /* stop reading and drop our streams fairly quickly */
                             break;
@@ -2433,35 +2542,12 @@ async fn handle_child(
         }
     }.instrument(trace_span!("child_lines"));
 
-    /* a stream updating "queue"."last_ping" at an interval */
 
-    let (kill_tx, mut kill_rx) = oneshot::channel::<()>();
-
-    let mut interval = interval(ping_interval);
-    interval.set_missed_tick_behavior(MissedTickBehavior::Skip);
-
-    let db1 = db.clone();
-    tokio::spawn(async move {
-        loop {
-            tokio::select! {
-                _ = interval.tick() => {
-                    if let Err(err) = sqlx::query!("UPDATE queue SET last_ping = now() WHERE id = $1", job_id)
-                        .execute(&db1)
-                    .await
-            {
-                tracing::error!(%job_id, %err, "error setting last ping for job {job_id}: {err}");
-                    };
-                },
-                _ = (&mut kill_rx) => return,
-            }
-        }
-    });
     let (wait_result, _) = tokio::join!(wait_on_child, lines);
-    kill_tx.send(()).expect("send should always work");
 
     match wait_result {
         _ if *too_many_logs.borrow() => Err(Error::ExecutionErr(format!(
-            "logs or result reached limit. Set MAX_LOG_SIZE higher (current: {max_log_size})"
+            "logs or result reached limit. (current max size: {MAX_RESULT_SIZE} characters)"
         ))),
         Ok(Ok(status)) => {
             if status.success() {
@@ -2674,7 +2760,7 @@ async fn handle_python_reqs(
         pip_extra_index_url,
         pip_trusted_host,
         nsjail_path,
-        max_log_size,
+        
         ..
     }: &Envs,
     job: &QueuedJob,
@@ -2769,7 +2855,7 @@ async fn handle_python_reqs(
                 .spawn()?
         };
 
-        let child = handle_child(&job.id, db, logs, timeout, *max_log_size, child).await;
+        let child = handle_child(&job.id, db, logs, timeout, child, false).await;
         tracing::info!(
             worker_name = %worker_name,
             job_id = %job.id,

cli/context.ts

@@ -1,10 +1,12 @@
 // deno-lint-ignore-file no-explicit-any
-import { colors, setClient } from "./deps.ts";
-import { tryGetLoginInfo } from "./login.ts";
+import { colors, setClient, UserService } from "./deps.ts";
+import { loginInteractive, tryGetLoginInfo } from "./login.ts";
 import { GlobalOptions } from "./types.ts";
 import {
+  addWorkspace,
   getActiveWorkspace,
   getWorkspaceByName,
+  removeWorkspace,
   Workspace,
 } from "./workspace.ts";
 
@@ -67,6 +69,27 @@ export async function requireLogin(opts: GlobalOptions) {
   }
 
   setClient(token, workspace.remote.substring(0, workspace.remote.length - 1));
+
+  try {
+    await UserService.globalWhoami();
+  } catch {
+    console.log(
+      "! Could not reach API given existing credentials. Attempting to reauth...",
+    );
+    const newToken = await loginInteractive(workspace.remote);
+    if (!newToken) {
+      throw new Error("Could not reauth");
+    }
+    removeWorkspace(workspace.name);
+    workspace.token = newToken;
+    addWorkspace(workspace);
+
+    setClient(
+      token,
+      workspace.remote.substring(0, workspace.remote.length - 1),
+    );
+    await UserService.globalWhoami();
+  }
 }
 
 export async function tryResolveVersion(

cli/decoverto.ts

@@ -0,0 +1,8 @@
+// globally shared decoverto instance
+import { Decoverto } from "npm:decoverto";
+const decoverto = new Decoverto();
+
+// TODO: Properly type FlowModule
+
+export { Any, array, map, MapShape, model, property } from "npm:decoverto";
+export { decoverto };

cli/deps.ts

@@ -1,32 +1,40 @@
 // windmill
-export { setClient } from "https://deno.land/x/windmill@v1.56.0/mod.ts";
-export * from "https://deno.land/x/windmill@v1.56.0/windmill-api/index.ts";
+export { setClient } from "https://deno.land/x/windmill@v1.61.1/mod.ts";
+export * from "https://deno.land/x/windmill@v1.61.1/windmill-api/index.ts";
 
 // cliffy
-export { Command } from "https://deno.land/x/cliffy@v0.25.6/command/command.ts";
-export { Table } from "https://deno.land/x/cliffy@v0.25.6/table/table.ts";
-export { colors } from "https://deno.land/x/cliffy@v0.25.6/ansi/colors.ts";
-export { Secret } from "https://deno.land/x/cliffy@v0.25.6/prompt/secret.ts";
-export { Select } from "https://deno.land/x/cliffy@v0.25.6/prompt/select.ts";
-export { Confirm } from "https://deno.land/x/cliffy@v0.25.6/prompt/confirm.ts";
-export { Input } from "https://deno.land/x/cliffy@v0.25.6/prompt/input.ts";
+export { Command } from "https://deno.land/x/cliffy@v0.25.7/command/command.ts";
+export { Table } from "https://deno.land/x/cliffy@v0.25.7/table/table.ts";
+export { colors } from "https://deno.land/x/cliffy@v0.25.7/ansi/colors.ts";
+export { Secret } from "https://deno.land/x/cliffy@v0.25.7/prompt/secret.ts";
+export { Select } from "https://deno.land/x/cliffy@v0.25.7/prompt/select.ts";
+export { Confirm } from "https://deno.land/x/cliffy@v0.25.7/prompt/confirm.ts";
+export { Input } from "https://deno.land/x/cliffy@v0.25.7/prompt/input.ts";
 export {
   DenoLandProvider,
   UpgradeCommand,
-} from "https://deno.land/x/cliffy@v0.25.6/command/upgrade/mod.ts";
+} from "https://deno.land/x/cliffy@v0.25.7/command/upgrade/mod.ts";
 
 // std
-export { Untar } from "https://deno.land/std@0.170.0/archive/untar.ts";
-export * as path from "https://deno.land/std@0.170.0/path/mod.ts";
-export { ensureDir } from "https://deno.land/std@0.170.0/fs/ensure_dir.ts";
+export { Untar } from "https://deno.land/std@0.176.0/archive/untar.ts";
+export * as path from "https://deno.land/std@0.176.0/path/mod.ts";
+export { ensureDir } from "https://deno.land/std@0.176.0/fs/ensure_dir.ts";
 export {
   copy,
   readAll,
   readerFromStreamReader,
-} from "https://deno.land/std@0.170.0/streams/mod.ts";
-export { DelimiterStream } from "https://deno.land/std@0.170.0/streams/mod.ts";
+} from "https://deno.land/std@0.176.0/streams/mod.ts";
+export { DelimiterStream } from "https://deno.land/std@0.176.0/streams/mod.ts";
+export { iterateReader } from "https://deno.land/std@0.176.0/streams/iterate_reader.ts";
 
 // other
 export { getAvailablePort } from "https://deno.land/x/port@1.0.0/mod.ts";
 export { default as dir } from "https://deno.land/x/dir@1.5.1/mod.ts";
 export { passwordGenerator } from "https://deno.land/x/password_generator@latest/mod.ts"; // TODO: I think the version is called latest, but it's still pinned.
+export { nanoid } from "https://deno.land/x/nanoid@v3.0.0/mod.ts";
+export * as cbor from "https://deno.land/x/cbor@v1.4.1/index.js";
+export { default as Murmurhash3 } from "https://deno.land/x/murmurhash@v1.0.0/mod.ts";
+export {
+  default as microdiff,
+} from "https://deno.land/x/microdiff@v1.3.1/index.ts";
+export { default as objectHash } from "https://deno.land/x/object_hash@2.0.3.1/mod.ts";

cli/flow.ts

@@ -1,16 +1,145 @@
 // deno-lint-ignore-file no-explicit-any
-import { GlobalOptions } from "./types.ts";
+import {
+  Difference,
+  GlobalOptions,
+  PushDiffs,
+  Resource,
+  setValueByPath,
+} from "./types.ts";
 import {
   colors,
   Command,
   Flow,
+  FlowModule,
   FlowService,
   JobService,
-  OpenFlow,
+  microdiff,
+  OpenFlowWPath,
   Table,
 } from "./deps.ts";
 import { requireLogin, resolveWorkspace, validatePath } from "./context.ts";
 import { resolve, track_job } from "./script.ts";
+import { Any, array, decoverto, model, property } from "./decoverto.ts";
+
+@model()
+export class FlowValueFilePart {
+  @property(array(Any))
+  modules: Array<FlowModule>;
+  @property(Any)
+  failure_module?: FlowModule;
+  @property(() => Boolean)
+  same_worker?: boolean;
+
+  constructor(modules: Array<FlowModule>) {
+    this.modules = modules;
+  }
+}
+
+// this is effectively "OpenFlow" but a copy as it is accepted by the CLI
+@model()
+export class FlowFile implements Resource, PushDiffs {
+  @property(() => String)
+  summary: string;
+  @property(() => String)
+  description?: string;
+  @property(() => FlowValueFilePart)
+  value: FlowValueFilePart;
+  @property(Any)
+  schema?: any;
+
+  constructor(summary: string, value: FlowValueFilePart) {
+    this.summary = summary;
+    this.value = value;
+  }
+  async pushDiffs(
+    workspace: string,
+    remotePath: string,
+    diffs: Difference[],
+  ): Promise<void> {
+    if (
+      await FlowService.existsFlowByPath({
+        workspace: workspace,
+        path: remotePath,
+      })
+    ) {
+      console.log(
+        colors.bold.yellow(
+          `Applying ${diffs.length} diffs to existing flow...`,
+        ),
+      );
+
+      // TODO: Make these optional in backend (not path ofc)
+      const changeset: OpenFlowWPath = {
+        path: remotePath,
+        summary: this.summary,
+        value: this.value,
+        description: this.description, // This is OpenAPIed as optional, but isn't
+        schema: this.schema, // Same
+      };
+      for (const diff of diffs) {
+        if (
+          diff.type !== "REMOVE" &&
+          (
+            diff.path[0] !== "value" && (
+              diff.path.length !== 1 ||
+              !["summary", "description", "schema"].includes(
+                diff.path[0] as string,
+              )
+            )
+          )
+        ) {
+          throw new Error("Invalid flow diff with path " + diff.path);
+        }
+        if (diff.type === "CREATE" || diff.type === "CHANGE") {
+          setValueByPath(changeset, diff.path, diff.value);
+        } else if (diff.type === "REMOVE") {
+          setValueByPath(changeset, diff.path, null);
+        }
+      }
+      const hasChanges = Object.values(changeset).some((v) =>
+        v !== null && typeof v !== "undefined"
+      );
+      if (!hasChanges) {
+        console.log(colors.yellow("! Skipping empty changeset"));
+        return;
+      }
+
+      await FlowService.updateFlow({
+        workspace: workspace,
+        path: remotePath,
+        requestBody: changeset,
+      });
+    } else {
+      console.log(colors.bold.yellow("Creating new flow..."));
+      await FlowService.createFlow({
+        workspace: workspace,
+        requestBody: {
+          path: remotePath,
+          summary: this.summary,
+          value: this.value,
+          schema: this.schema,
+          description: this.description,
+        },
+      });
+    }
+  }
+  async push(workspace: string, remotePath: string): Promise<void> {
+    let remote: Flow | undefined;
+    try {
+      remote = await FlowService.getFlowByPath({
+        workspace,
+        path: remotePath,
+      });
+    } catch {
+      remote = undefined;
+    }
+    await this.pushDiffs(
+      workspace,
+      remotePath,
+      microdiff(remote ?? {}, this, { cyclesFix: false }),
+    );
+  }
+}
 
 type Options = GlobalOptions;
 
@@ -30,38 +159,10 @@ export async function pushFlow(
   workspace: string,
   remotePath: string,
 ) {
-  const data: OpenFlow = JSON.parse(await Deno.readTextFile(filePath));
-  if (
-    await FlowService.existsFlowByPath({
-      workspace: workspace,
-      path: remotePath,
-    })
-  ) {
-    console.log(colors.bold.yellow("Updating existing flow..."));
-    await FlowService.updateFlow({
-      workspace: workspace,
-      path: remotePath,
-      requestBody: {
-        path: remotePath,
-        summary: data.summary,
-        value: data.value,
-        schema: data.schema,
-        description: data.description,
-      },
-    });
-  } else {
-    console.log(colors.bold.yellow("Creating new flow..."));
-    await FlowService.createFlow({
-      workspace: workspace,
-      requestBody: {
-        path: remotePath,
-        summary: data.summary,
-        value: data.value,
-        schema: data.schema,
-        description: data.description,
-      },
-    });
-  }
+  const data = decoverto.type(FlowFile).rawToInstance(
+    await Deno.readTextFile(filePath),
+  );
+  await data.push(workspace, remotePath);
 }
 
 async function list(opts: GlobalOptions & { showArchived?: boolean }) {

cli/folder.ts

@@ -1,6 +1,122 @@
-import { colors, Command, Folder, FolderService } from "./deps.ts";
+import { colors, Command, Folder, FolderService, microdiff } from "./deps.ts";
 import { requireLogin, resolveWorkspace, validatePath } from "./context.ts";
-import { GlobalOptions } from "./types.ts";
+import {
+  Difference,
+  GlobalOptions,
+  PushDiffs,
+  Resource,
+  setValueByPath,
+} from "./types.ts";
+import {
+  array,
+  decoverto,
+  map,
+  MapShape,
+  model,
+  property,
+} from "./decoverto.ts";
+
+@model()
+export class FolderFile implements Resource, PushDiffs {
+  @property(array(() => String))
+  owners: Array<string> | undefined;
+  @property(map(() => String, () => Boolean, { shape: MapShape.Object }))
+  extra_perms: Map<string, boolean> | undefined;
+
+  async push(workspace: string, remotePath: string): Promise<void> {
+    if (remotePath.startsWith("/")) {
+      remotePath = remotePath.substring(1);
+    }
+    if (remotePath.startsWith("f/")) {
+      remotePath = remotePath.substring(2);
+    }
+
+    let existing: Folder | undefined;
+    try {
+      existing = await FolderService.getFolder({ workspace, name: remotePath });
+    } catch {
+      existing = undefined;
+    }
+    await this.pushDiffs(
+      workspace,
+      remotePath,
+      microdiff(existing ?? {}, this, { cyclesFix: false }),
+    );
+  }
+
+  async pushDiffs(
+    workspace: string,
+    remotePath: string,
+    diffs: Difference[],
+  ): Promise<void> {
+    if (remotePath.startsWith("/")) {
+      remotePath = remotePath.substring(1);
+    }
+    if (remotePath.startsWith("f/")) {
+      remotePath = remotePath.substring(2);
+    }
+
+    // TODO: Support this in backend
+    let exists: boolean;
+    try {
+      exists = !!await FolderService.getFolder({ workspace, name: remotePath });
+    } catch {
+      exists = false;
+    }
+    if (exists) {
+      console.log(
+        colors.bold.yellow(
+          `Applying ${diffs.length} diffs to existing folder...`,
+        ),
+      );
+
+      const changeset: {
+        owners?: string[] | undefined;
+        extra_perms?: any;
+      } = {};
+      for (const diff of diffs) {
+        if (
+          diff.type !== "REMOVE" &&
+          (
+            diff.path.length !== 1 ||
+            !["owners", "extra_perms"].includes(diff.path[0] as string)
+          )
+        ) {
+          throw new Error("Invalid folder diff with path " + diff.path);
+        }
+        if (diff.type === "CREATE" || diff.type === "CHANGE") {
+          setValueByPath(changeset, diff.path, diff.value);
+        } else if (diff.type === "REMOVE") {
+          setValueByPath(changeset, diff.path, null);
+        }
+      }
+
+      const hasChanges = Object.values(changeset).some((v) =>
+        v !== null && typeof v !== "undefined"
+      );
+      if (!hasChanges) {
+        console.log(colors.yellow("! Skipping empty changeset"));
+        return;
+      }
+
+      await FolderService.updateFolder({
+        workspace: workspace,
+        name: remotePath,
+        requestBody: changeset,
+      });
+    } else {
+      console.log(colors.bold.yellow("Creating new folder..."));
+      await FolderService.createFolder({
+        workspace: workspace,
+        requestBody: {
+          name: remotePath,
+          extra_perms: this.extra_perms,
+          owners: this.owners,
+        },
+      });
+    }
+  }
+}
 
 async function push(opts: GlobalOptions, filePath: string, remotePath: string) {
   const workspace = await resolveWorkspace(opts);
@@ -21,61 +137,15 @@ async function push(opts: GlobalOptions, filePath: string, remotePath: string) {
   console.log(colors.bold.underline.green("Resource successfully pushed"));
 }
 
-type FolderFile = {
-  owners: Array<string> | undefined;
-  extra_perms: Record<string, boolean> | undefined;
-};
-
 export async function pushFolder(
   workspace: string,
   filePath: string,
   remotePath: string,
 ) {
-  if (remotePath.startsWith("/")) {
-    remotePath = remotePath.substring(1);
-  }
-  if (remotePath.startsWith("f/")) {
-    remotePath = remotePath.substring(2);
-  }
-  const data: FolderFile = JSON.parse(await Deno.readTextFile(filePath));
-  let optFolder: Folder | undefined;
-  try {
-    optFolder = await FolderService.getFolder({ workspace, name: remotePath });
-  } catch {
-    optFolder = undefined;
-  }
-
-  if (optFolder) {
-    // for (const [k, v] of Object.entries(optFolder.extra_perms)) {
-    //   if (!data.extra_perms || data.extra_perms[k] !== v) {
-    //     console.log(colors.red.underline.bold(`Extra Perms missmatch on ${k}`));
-    //     return;
-    //   }
-    // }
-
-    console.log(colors.yellow("Updating existing folder..."));
-    await FolderService.updateFolder({
-      workspace,
-      name: remotePath,
-      requestBody: {
-        extra_perms: data.extra_perms,
-        owners: data.owners,
-      },
-    });
-  } else {
-    console.log(colors.yellow("Creating new folder..."));
-    await FolderService.createFolder({
-      workspace,
-      requestBody: {
-        name: remotePath,
-        extra_perms: data.extra_perms,
-        owners: data.owners,
-      },
-    });
-
-    // HACK: Workaround backend automatically adding current user to folder.
-    await pushFolder(workspace, filePath, remotePath);
-  }
+  const data = decoverto.type(FolderFile).rawToInstance(
+    await Deno.readTextFile(filePath),
+  );
+  data.push(workspace, remotePath);
 }
 
 const command = new Command()

cli/hub.ts

@@ -1,6 +1,6 @@
 import { Command } from "./deps.ts";
 import { requireLogin, resolveWorkspace } from "./context.ts";
-import { pushResourceTypeDef } from "./resource-type.ts";
+import { ResourceTypeFile } from "./resource-type.ts";
 import { GlobalOptions } from "./types.ts";
 
 async function pull(opts: GlobalOptions) {
@@ -56,14 +56,10 @@ async function pull(opts: GlobalOptions) {
     const x of list
   ) {
     console.log("syncing " + x.name);
-    await pushResourceTypeDef(
-      workspace.workspaceId,
-      x.name,
-      {
-        description: x.description,
-        schema: JSON.parse(x.schema),
-      },
-    );
+    const f = new ResourceTypeFile();
+    f.description = x.description;
+    f.schema = JSON.parse(x.schema);
+    await f.push(workspace.workspaceId, x.name);
   }
 }
 

cli/main.ts

@@ -8,13 +8,14 @@ import variable from "./variable.ts";
 import push from "./push.ts";
 import pull from "./pull.ts";
 import hub from "./hub.ts";
-// import folder from "./folder.ts";
+import folder from "./folder.ts";
+import sync from "./sync.ts";
 import { tryResolveVersion } from "./context.ts";
 import { GlobalOptions } from "./types.ts";
 
-const VERSION = "v1.60.0";
+const VERSION = "v1.62.0";
 
-const command = new Command()
+let command: any = new Command()
   .name("wmill")
   .description("A simple CLI tool for windmill.")
   .globalOption(
@@ -32,10 +33,9 @@ const command = new Command()
   .command("resource", resource)
   .command("user", user)
   .command("variable", variable)
-  .command("push", push)
-  .command("pull", pull)
   .command("hub", hub)
-  // .command("folder", folder)
+  .command("folder", folder)
+  .command("sync", sync)
   .command("version", "Show version information")
   .action(async (opts) => {
     console.log("CLI build against " + VERSION);
@@ -61,6 +61,12 @@ const command = new Command()
     }),
   );
 
+if (Number.parseInt(VERSION.replace("v", "").replace(".", "")) > 1700) {
+  command = command
+    .command("push", push)
+    .command("pull", pull);
+}
+
 try {
   await command.parse(Deno.args);
 } catch (e) {

cli/pull.ts

@@ -1,20 +1,11 @@
 // deno-lint-ignore-file no-explicit-any
-import { resolveWorkspace } from "./context.ts";
 import { GlobalOptions } from "./types.ts";
-import {
-  colors,
-  Command,
-  Confirm,
-  copy,
-  ensureDir,
-  path,
-  readerFromStreamReader,
-  Untar,
-} from "./deps.ts";
-
-async function pull(opts: GlobalOptions & { override: boolean }, dir: string) {
-  const workspace = await resolveWorkspace(opts);
+import { colors, Command, readerFromStreamReader, Untar } from "./deps.ts";
+import { Workspace } from "./workspace.ts";
 
+export async function downloadTar(
+  workspace: Workspace,
+): Promise<Untar | undefined> {
   const requestHeaders: HeadersInit = new Headers();
   requestHeaders.set("Authorization", "Bearer " + workspace.token);
   requestHeaders.set("Content-Type", "application/octet-stream");
@@ -35,51 +26,29 @@ async function pull(opts: GlobalOptions & { override: boolean }, dir: string) {
       ),
     );
     console.log(await tarResponse.text());
-    return;
+    return undefined;
   }
 
   const streamReader = tarResponse.body?.getReader();
   if (!streamReader) {
     console.log(colors.red("Failed to read tar request body"));
-    return;
+    return undefined;
   }
   console.log(colors.yellow("Streaming tarball to disk..."));
   const denoReader = readerFromStreamReader(streamReader);
   const untar = new Untar(denoReader);
-  for await (const entry of untar) {
-    console.log(entry.fileName);
-    const filePath = path.resolve(dir, entry.fileName);
-    if (entry.type === "directory") {
-      await ensureDir(filePath);
-      continue;
-    }
-    await ensureDir(path.dirname(filePath));
-    if (!opts.override) {
-      let exists = false;
-      try {
-        const _stat = await Deno.stat(filePath);
-        exists = true;
-      } catch {
-        exists = false;
-      }
-      if (exists) {
-        if (
-          !(await Confirm.prompt(
-            "Conflict at " +
-              filePath +
-              " do you want to override the local version?",
-          ))
-        ) {
-          continue;
-        }
-      }
-    }
-    const file = await Deno.open(filePath, { write: true, create: true });
-    const len = await copy(entry, file);
-    await file.truncate(len);
-    file.close();
-  }
-  console.log(colors.green("Done. Wrote all files to disk."));
+  return untar;
+}
+
+async function stub(
+  _opts: GlobalOptions & { override: boolean },
+  _dir: string,
+) {
+  console.log(
+    colors.red.underline(
+      'Pull is deprecated. Use "sync pull --raw" instead. See <TODO_LINK_HERE> for more information.',
+    ),
+  );
 }
 
 const command = new Command()
@@ -87,6 +56,6 @@ const command = new Command()
     "Pull all definitions in the current workspace from the API and write them to disk.",
   )
   .arguments("<dir:string>")
-  .action(pull as any);
+  .action(stub as any);
 
 export default command;

cli/push.ts

@@ -1,266 +1,21 @@
 // deno-lint-ignore-file no-explicit-any
-import { colors, Command, path } from "./deps.ts";
-import { requireLogin, resolveWorkspace } from "./context.ts";
-import { pushFlow } from "./flow.ts";
-import { pushResource } from "./resource.ts";
-import { findContentFile, pushScript } from "./script.ts";
+import { colors, Command } from "./deps.ts";
 import { GlobalOptions } from "./types.ts";
-import { pushVariable } from "./variable.ts";
-import { pushResourceType } from "./resource-type.ts";
-import { pushFolder } from "./folder.ts";
 
-type Candidate = {
-  path: string;
-  namespaceKind: "user" | "group" | "folder";
-  namespaceName: string;
-};
-
-type ResourceTypeCandidate = {
-  path: string;
-};
-
-type FolderCandidate = {
-  path: string;
-  namespaceName: string;
-};
-
-async function findCandidateFiles(
-  dir: string,
-): Promise<
-  {
-    normal: Candidate[];
-    resourceTypes: ResourceTypeCandidate[];
-    folders: FolderCandidate[];
-  }
-> {
-  dir = path.resolve(dir);
-  if (path.dirname(dir).startsWith(".")) {
-    return { normal: [], resourceTypes: [], folders: [] };
-  }
-  const normalCandidates: Candidate[] = [];
-  const resourceTypeCandidates: ResourceTypeCandidate[] = [];
-  const folderCandidates: FolderCandidate[] = [];
-  for await (const e of Deno.readDir(dir)) {
-    if (e.isDirectory) {
-      if (e.name == "u" || e.name == "g" || e.name == "f") { // TODO: Check version for f
-        const newDir = dir + (dir.endsWith("/") ? "" : "/") + e.name;
-        for await (const e2 of Deno.readDir(newDir)) {
-          if (e2.isDirectory) {
-            if (e2.name.startsWith(".")) continue;
-            const namespaceName = e2.name;
-            const stack: string[] = [];
-            {
-              const path = newDir + "/" + namespaceName + "/";
-              stack.push(path);
-              try {
-                await Deno.stat(path + "folder.meta.json");
-                folderCandidates.push({
-                  namespaceName,
-                  path: path + "folder.meta.json",
-                });
-              } catch {}
-            }
-
-            while (stack.length > 0) {
-              const dir2 = stack.pop()!;
-              for await (const e3 of Deno.readDir(dir2)) {
-                if (e3.isFile) {
-                  if (e3.name === "folder.meta.json") continue;
-                  normalCandidates.push({
-                    path: dir2 + e3.name,
-                    namespaceKind: e.name == "g"
-                      ? "group"
-                      : e.name == "u"
-                      ? "user"
-                      : "folder",
-                    namespaceName: namespaceName,
-                  });
-                } else {
-                  stack.push(dir2 + e3.name + "/");
-                }
-              }
-            }
-          }
-        }
-      } else {
-        console.log(
-          colors.yellow(
-            "Including organizational folder " + e.name + " in push!",
-          ),
-        );
-        const { normal, resourceTypes, folders } = await findCandidateFiles(
-          path.join(dir, e.name),
-        );
-        normalCandidates.push(...normal);
-        resourceTypeCandidates.push(...resourceTypes);
-        folderCandidates.push(...folders);
-      }
-    } else {
-      // handle root files
-      if (e.name.endsWith(".resource-type.json")) {
-        resourceTypeCandidates.push({
-          path: dir + (dir.endsWith("/") ? "" : "/") + e.name,
-        });
-      }
-    }
-  }
-  return {
-    normal: normalCandidates,
-    folders: folderCandidates,
-    resourceTypes: resourceTypeCandidates,
-  };
-}
-
-async function push(opts: GlobalOptions, dir?: string) {
-  dir = dir ?? Deno.cwd();
-  const workspace = await resolveWorkspace(opts);
-  await requireLogin(opts);
-
-  console.log(colors.blue("Searching Directory..."));
-  const { normal, resourceTypes, folders } = await findCandidateFiles(dir);
+async function stub(
+  _opts: GlobalOptions,
+  _dir?: string,
+) {
   console.log(
-    colors.blue(
-      "Found " + (normal.length + resourceTypes.length + folders.length) +
-        " candidates",
+    colors.red.underline(
+      'Push is deprecated. Use "sync push --raw" instead. See <TODO_LINK_HERE> for more information.',
     ),
   );
-  for (const resourceType of resourceTypes) {
-    const fileName = resourceType.path.substring(
-      resourceType.path.lastIndexOf("/") + 1,
-    );
-    const fileNameParts = fileName.split(".");
-    // invalid file names, like my.cool.script.script.json. Not valid.
-    if (fileNameParts.length != 3) {
-      console.log(
-        colors.yellow("invalid file name found at " + resourceType.path),
-      );
-      continue;
-    }
-
-    // filter out non-json files. Note that we filter out script contents above, so this is really an error.
-    if (fileNameParts.at(-1) != "json") {
-      console.log(colors.yellow("non-JSON file found at " + resourceType.path));
-      continue;
-    }
-
-    console.log("pushing resource type " + fileNameParts.at(-3)!);
-    await pushResourceType(
-      workspace.workspaceId,
-      resourceType.path,
-      fileNameParts.at(-3)!,
-    );
-  }
-  for (const folder of folders) {
-    await pushFolder(
-      workspace.workspaceId,
-      folder.path,
-      "f/" + folder.namespaceName,
-    );
-  }
-  for (const candidate of normal) {
-    // full file name. No leading /. includes .type.json
-    const fileName = candidate.path.substring(
-      candidate.path.lastIndexOf("/") + 1,
-    );
-    // figure out just the path after ...../u|g/username|group/ (in extra dir)
-    const dirParts = candidate.path.split("/").filter((x) => x.length > 0);
-    // TODO: check version for folder
-    const gIndex = dirParts.findIndex((x) => x == "u" || x == "g" || x == "f");
-    const extraDir = dirParts.slice(gIndex + 2, -1).join("/");
-
-    // file name parts has .json (hopefully) at -1, type at -2, and the actual name at -3. Dots in names are not allowed.
-    const fileNameParts = fileName.split(".");
-
-    // filter out script content files
-    if (
-      fileNameParts.at(-1) == "ts" ||
-      fileNameParts.at(-1) == "py" ||
-      fileNameParts.at(-1) == "go"
-    ) {
-      // probably part of a script. Silent ignore.
-      continue;
-    }
-
-    // invalid file names, like my.cool.script.script.json. Not valid.
-    if (fileNameParts.length != 3) {
-      console.log(
-        colors.yellow("invalid file name found at " + candidate.path),
-      );
-      continue;
-    }
-
-    // filter out non-json files. Note that we filter out script contents above, so this is really an error.
-    if (fileNameParts.at(-1) != "json") {
-      console.log(colors.yellow("non-JSON file found at " + candidate.path));
-      continue;
-    }
-
-    // get the type & filter it for valid ones.
-    const type = fileNameParts.at(-2);
-    if (type == "resource-type") {
-      console.log(
-        colors.yellow(
-          "Found resource type file at " +
-            candidate.path +
-            " this appears to be inside a path folder. Resource types are not addressed by path. Place them at the root or inside only an organizational folder. Ignoring this file!",
-        ),
-      );
-      continue;
-    }
-
-    if (
-      type != "flow" &&
-      type != "resource" &&
-      type != "script" &&
-      type != "variable"
-    ) {
-      console.log(
-        colors.yellow(
-          "file with invalid type " + type + " found at " + candidate.path,
-        ),
-      );
-      continue;
-    }
-
-    // create the remotePath for the API
-    const remotePath = (candidate.namespaceKind === "group"
-      ? "g/"
-      : (candidate.namespaceKind === "user" ? "u/" : "f/")) +
-      candidate.namespaceName +
-      "/" +
-      (extraDir.length > 0 ? extraDir + "/" : "") +
-      fileNameParts.at(-3);
-
-    console.log("pushing " + type + " to " + remotePath);
-
-    if (type == "flow") {
-      await pushFlow(candidate.path, workspace.workspaceId, remotePath);
-    } else if (type == "resource") {
-      await pushResource(workspace.workspaceId, candidate.path, remotePath);
-    } else if (type == "script") {
-      let contentPath: string;
-      try {
-        contentPath = await findContentFile(candidate.path);
-      } catch (e) {
-        console.log(colors.red(e.toString()));
-        continue;
-      }
-      await pushScript(
-        candidate.path,
-        contentPath,
-        workspace.workspaceId,
-        remotePath,
-      );
-    } else if (type == "variable") {
-      await pushVariable(workspace.workspaceId, candidate.path, remotePath);
-    }
-  }
-  console.log(colors.underline.bold.green("Successfully Pushed all files."));
 }
 
 const command = new Command()
   .description("Push all files from a folder")
   .arguments("[dir:string]")
-  .action(push as any);
+  .action(stub as any);
 
 export default command;

cli/resource-type.ts

@@ -1,65 +1,129 @@
 // deno-lint-ignore-file no-explicit-any
-import { GlobalOptions } from "./types.ts";
+import {
+  Difference,
+  GlobalOptions,
+  PushDiffs,
+  Resource as ResourceI,
+  setValueByPath,
+} from "./types.ts";
 import { requireLogin, resolveWorkspace } from "./context.ts";
-import { colors, Command, ResourceService, Table } from "./deps.ts";
+import {
+  colors,
+  Command,
+  EditResourceType,
+  microdiff,
+  ResourceService,
+  ResourceType,
+  Table,
+} from "./deps.ts";
+import { Any, decoverto, model, property } from "./decoverto.ts";
 
-type ResourceTypeFile = {
+@model()
+export class ResourceTypeFile implements ResourceI, PushDiffs {
+  @property(Any)
   schema?: any;
+  @property(() => String)
   description?: string;
-};
+
+  async push(workspace: string, remotePath: string): Promise<void> {
+    let existing: ResourceType | undefined;
+    try {
+      existing = await ResourceService.getResourceType({
+        workspace,
+        path: remotePath,
+      });
+    } catch {
+      existing = undefined;
+    }
+    this.pushDiffs(
+      workspace,
+      remotePath,
+      microdiff(existing ?? {}, this, { cyclesFix: false }),
+    );
+  }
+
+  async pushDiffs(
+    workspace: string,
+    remotePath: string,
+    diffs: Difference[],
+  ): Promise<void> {
+    if (
+      await ResourceService.existsResourceType({
+        workspace: workspace,
+        path: remotePath,
+      })
+    ) {
+      if (
+        (await ResourceService.listResourceType({ workspace })).findIndex((x) =>
+          x.name === remotePath
+        ) === -1
+      ) {
+        console.log(
+          "Resource type " + remotePath +
+            " is already taken for the current workspace, but cannot be updated. Is this a conflict with starter?",
+        );
+        return;
+      }
+      console.log(
+        colors.yellow(
+          `Applying ${diffs.length} diffs to existing resource type...`,
+        ),
+      );
+      const changeset: EditResourceType = {};
+      for (const diff of diffs) {
+        if (
+          diff.type !== "REMOVE" &&
+          (
+            diff.path.length !== 1 ||
+            !["schema", "description"].includes(diff.path[0] as string)
+          )
+        ) {
+          throw new Error("Invalid resource type diff with path " + diff.path);
+        }
+        if (diff.type === "CREATE" || diff.type === "CHANGE") {
+          setValueByPath(changeset, diff.path, diff.value);
+        } else if (diff.type === "REMOVE") {
+          setValueByPath(changeset, diff.path, null);
+        }
+      }
+
+      const hasChanges = Object.values(changeset).some((v) =>
+        v !== null && typeof v !== "undefined"
+      );
+      if (!hasChanges) {
+        console.log(colors.yellow("! Skipping empty changeset"));
+        return;
+      }
+
+      await ResourceService.updateResourceType({
+        workspace: workspace,
+        path: remotePath,
+        requestBody: changeset,
+      });
+    } else {
+      console.log(colors.yellow("Creating new resource type..."));
+      await ResourceService.createResourceType({
+        workspace: workspace,
+        requestBody: {
+          name: remotePath,
+          description: this.description,
+          schema: this.schema,
+          workspace_id: workspace,
+        },
+      });
+    }
+  }
+}
 
 export async function pushResourceType(
   workspace: string,
   filePath: string,
   name: string,
 ) {
-  const data: ResourceTypeFile = JSON.parse(await Deno.readTextFile(filePath));
-  await pushResourceTypeDef(workspace, name, data);
-}
-
-export async function pushResourceTypeDef(
-  workspace: string,
-  name: string,
-  data: ResourceTypeFile,
-) {
-  if (
-    await ResourceService.existsResourceType({
-      workspace: workspace,
-      path: name,
-    })
-  ) {
-    console.log(colors.yellow("Updating existing resource type..."));
-    if (
-      (await ResourceService.listResourceType({ workspace })).findIndex((x) =>
-        x.name === name
-      ) === -1
-    ) {
-      console.log(
-        "Resource type " + name +
-          " is already taken for the current workspace, but cannot be updated. Is this a conflict with starter?",
-      );
-      return;
-    }
-    await ResourceService.updateResourceType({
-      workspace: workspace,
-      path: name,
-      requestBody: {
-        description: data.description,
-        schema: data.schema,
-      },
-    });
-  } else {
-    console.log(colors.yellow("Creating new resource type..."));
-    await ResourceService.createResourceType({
-      workspace: workspace,
-      requestBody: {
-        name: name,
-        description: data.description,
-        schema: data.schema,
-        workspace_id: workspace,
-      },
-    });
-  }
+  const data: ResourceTypeFile = decoverto.type(ResourceTypeFile).rawToInstance(
+    await Deno.readTextFile(filePath),
+  );
+  await data.push(workspace, name);
 }
 
 type PushOptions = GlobalOptions;

cli/resource.ts

@@ -1,80 +1,142 @@
-import { GlobalOptions } from "./types.ts";
+import {
+  Difference,
+  GlobalOptions,
+  PushDiffs,
+  Resource as Resource2,
+  setValueByPath,
+} from "./types.ts";
 import { requireLogin, resolveWorkspace, validatePath } from "./context.ts";
-import { colors, Command, Resource, ResourceService, Table } from "./deps.ts";
+import {
+  colors,
+  Command,
+  EditResource,
+  microdiff,
+  Resource,
+  ResourceService,
+  Table,
+} from "./deps.ts";
+import { Any, decoverto, model, property } from "./decoverto.ts";
 
-type ResourceFile = {
-  value: any;
+@model()
+export class ResourceFile implements Resource2, PushDiffs {
+  @property(Any)
+  value?: any;
+  @property(() => String)
   description?: string;
+  @property(() => String)
   resource_type: string;
+  @property(() => Boolean)
   is_oauth?: boolean; // deprecated
-};
+
+  constructor(resource_type: string) {
+    this.resource_type = resource_type;
+  }
+  async pushDiffs(
+    workspace: string,
+    remotePath: string,
+    diffs: Difference[],
+  ): Promise<void> {
+    if (
+      await ResourceService.existsResource({
+        workspace: workspace,
+        path: remotePath,
+      })
+    ) {
+      console.log(
+        colors.yellow(`Applying ${diffs.length} diffs to existing resource...`),
+      );
+
+      const changeset: EditResource = {
+        path: remotePath, // TODO: Remove this in backend
+      };
+      for (const diff of diffs) {
+        if (diff.path[0] === "is_oauth") {
+          console.log(
+            colors.yellow(
+              "! is_oauth has been removed in newer versions. Ignoring.",
+            ),
+          );
+          continue;
+        }
+        if (
+          diff.type !== "REMOVE" &&
+          (
+            diff.path[0] !== "value" && (
+              diff.path.length !== 1 ||
+              diff.path[0] !== "description"
+            )
+          )
+        ) {
+          throw new Error("Invalid folder diff with path " + diff.path);
+        }
+        if (diff.type === "CREATE" || diff.type === "CHANGE") {
+          setValueByPath(changeset, diff.path, diff.value);
+        } else if (diff.type === "REMOVE") {
+          setValueByPath(changeset, diff.path, null);
+        }
+      }
+
+      const hasChanges = Object.values(changeset).some((v) =>
+        v !== null && typeof v !== "undefined"
+      );
+      if (!hasChanges) {
+        console.log(colors.yellow("! Skipping empty changeset"));
+        return;
+      }
+
+      await ResourceService.updateResource({
+        workspace: workspace,
+        path: remotePath,
+        requestBody: changeset,
+      });
+    } else {
+      if (typeof this.is_oauth !== "undefined") {
+        console.log(
+          colors.yellow(
+            "! is_oauth has been removed in newer versions. Ignoring.",
+          ),
+        );
+      }
+
+      console.log(colors.yellow("Creating new resource..."));
+      await ResourceService.createResource({
+        workspace: workspace,
+        requestBody: {
+          path: remotePath,
+          resource_type: this.resource_type,
+          value: this.value,
+          description: this.description,
+        },
+      });
+    }
+  }
+  async push(workspace: string, remotePath: string): Promise<void> {
+    let existing: Resource | undefined;
+    try {
+      existing = await ResourceService.getResource({
+        workspace,
+        path: remotePath,
+      });
+    } catch {
+      existing = undefined;
+    }
+    await this.pushDiffs(
+      workspace,
+      remotePath,
+      microdiff(existing ?? {}, this, { cyclesFix: false }),
+    );
+  }
+}
 
 export async function pushResource(
   workspace: string,
   filePath: string,
   remotePath: string,
 ) {
-  const data: ResourceFile = JSON.parse(await Deno.readTextFile(filePath));
-  if (
-    await ResourceService.existsResource({
-      workspace: workspace,
-      path: remotePath,
-    })
-  ) {
-    console.log(colors.yellow("Updating existing resource..."));
-    const existing = await ResourceService.getResource({
-      workspace: workspace,
-      path: remotePath,
-    });
-
-    if (existing.resource_type != data.resource_type) {
-      console.log(
-        colors.red.underline.bold(
-          "Remote resource at " +
-            remotePath +
-            " exists & has a different resource type. This cannot be updated. If you wish to do this anyways, consider deleting the remote resource.",
-        ),
-      );
-      return;
-    }
-
-    if (typeof data.is_oauth !== "undefined") {
-      console.log(
-        colors.yellow(
-          "! is_oauth has been removed in newer versions. Ignoring.",
-        ),
-      );
-    }
-
-    await ResourceService.updateResource({
-      workspace: workspace,
-      path: remotePath,
-      requestBody: {
-        path: remotePath,
-        value: data.value,
-        description: data.description,
-      },
-    });
-  } else {
-    if (typeof data.is_oauth !== "undefined") {
-      console.log(
-        colors.yellow(
-          "! is_oauth has been removed in newer versions. Ignoring.",
-        ),
-      );
-    }
-
-    console.log(colors.yellow("Creating new resource..."));
-    await ResourceService.createResource({
-      workspace: workspace,
-      requestBody: {
-        path: remotePath,
-        resource_type: data.resource_type,
-        value: data.value,
-        description: data.description,
-      },
-    });
-  }
+  const data = decoverto.type(ResourceFile).rawToInstance(
+    await Deno.readTextFile(filePath),
+  );
+  await data.push(workspace, remotePath);
 }
 
 type PushOptions = GlobalOptions;

cli/script.ts

@@ -2,20 +2,52 @@
 import { GlobalOptions } from "./types.ts";
 import { requireLogin, resolveWorkspace, validatePath } from "./context.ts";
 import {
+  colors,
+  Command,
   JobService,
+  readAll,
   Script,
-} from "https://deno.land/x/windmill@v1.50.0/windmill-api/index.ts";
-import { colors, Command, readAll, ScriptService, Table } from "./deps.ts";
+  ScriptService,
+  Table,
+} from "./deps.ts";
+import { Any, array, decoverto, model, property } from "./decoverto.ts";
 
-type ScriptFile = {
+@model()
+export class ScriptFile {
+  @property(() => String)
   parent_hash?: string;
+  @property(() => String)
   summary: string;
+  @property(() => String)
   description: string;
+  @property(Any)
   schema?: any;
+  @property(() => Boolean)
   is_template?: boolean;
+  @property(array(() => String))
   lock?: Array<string>;
+  @property({
+    toInstance: (data) => {
+      if (data == null) return data;
+
+      if (
+        data === "script" || data === "failure" || data === "trigger" ||
+        data === "command" || data === "approvial"
+      ) {
+        return data;
+      }
+
+      throw new Error("Invalid kind " + data);
+    },
+    toPlain: (data) => data,
+  })
   kind?: "script" | "failure" | "trigger" | "command" | "approval";
-};
+
+  constructor(summary: string, description: string) {
+    this.summary = summary;
+    this.description = description;
+  }
+}
 
 type PushOptions = GlobalOptions;
 async function push(
@@ -48,10 +80,12 @@ async function push(
 }
 
 export async function findContentFile(filePath: string) {
+  console.log("Searching " + filePath);
   const candidates = [
     filePath.replace(".script.json", ".ts"),
     filePath.replace(".script.json", ".py"),
     filePath.replace(".script.json", ".go"),
+    filePath.replace(".script.json", ".sh"),
   ];
   const validCandidates = (
     await Promise.all(
@@ -79,23 +113,35 @@ export async function findContentFile(filePath: string) {
   return validCandidates[0];
 }
 
+export function inferContentTypeFromFilePath(
+  contentPath: string,
+): "python3" | "deno" | "go" | "bash" {
+  let language = contentPath.substring(contentPath.lastIndexOf("."));
+  if (language == ".ts") language = "deno";
+  if (language == ".py") language = "python3";
+  if (language == ".sh") language = "bash";
+  if (language == ".go") language = "go";
+  if (
+    language != "python3" && language != "deno" && language != "go" &&
+    language != "bash"
+  ) {
+    throw new Error("Invalid language: " + language);
+  }
+  return language;
+}
+
 export async function pushScript(
   filePath: string,
   contentPath: string,
   workspace: string,
   remotePath: string,
 ) {
-  const data: ScriptFile = JSON.parse(await Deno.readTextFile(filePath));
+  const data = decoverto.type(ScriptFile).rawToInstance(
+    await Deno.readTextFile(filePath),
+  );
   const content = await Deno.readTextFile(contentPath);
 
-  let language = contentPath.substring(contentPath.lastIndexOf("."));
-  if (language == ".ts") language = "deno";
-  if (language == ".py") language = "python3";
-  if (language == ".go") language = "go";
-  if (language != "python3" && language != "deno" && language != "go") {
-    throw new Error("Invalid language: " + language);
-  }
-
+  const language = inferContentTypeFromFilePath(contentPath);
   let parent_hash = data.parent_hash;
   if (!parent_hash) {
     try {

cli/sync.ts

@@ -0,0 +1,969 @@
+import { requireLogin, resolveWorkspace } from "./context.ts";
+import { getWorkspaceStream, Workspace } from "./workspace.ts";
+import { decoverto, map, MapShape, model, property } from "./decoverto.ts";
+import {
+  cbor,
+  colors,
+  Command,
+  Confirm,
+  copy,
+  ensureDir,
+  iterateReader,
+  microdiff,
+  nanoid,
+  objectHash,
+  path,
+  ScriptService,
+} from "./deps.ts";
+import {
+  Difference,
+  getTypeStrFromPath,
+  GlobalOptions,
+  inferTypeFromPath,
+  setValueByPath,
+} from "./types.ts";
+import { downloadTar } from "./pull.ts";
+import { FolderFile } from "./folder.ts";
+import { ResourceTypeFile } from "./resource-type.ts";
+import {
+  findContentFile,
+  inferContentTypeFromFilePath,
+  pushScript,
+  ScriptFile,
+} from "./script.ts";
+import { ResourceFile } from "./resource.ts";
+import { FlowFile } from "./flow.ts";
+import { VariableFile } from "./variable.ts";
+
+type TrackedId = string;
+const TrackedId = String;
+
+const CONTENT_ENCODER: cbor.Encoder = new cbor.Encoder({ pack: true });
+
+export class Tracked {
+  #id: TrackedId;
+  #parent: State;
+
+  path: string;
+
+  #content_cache?: string;
+
+  constructor(id: TrackedId, parent: State, path: string) {
+    this.#id = id;
+    this.#parent = parent;
+    this.path = path;
+  }
+
+  async getContent(): Promise<unknown | undefined> {
+    if (this.#content_cache) {
+      return this.#content_cache;
+    }
+
+    if (!this.#parent.stateRoot) {
+      throw new Error("Parent uninitialized");
+    }
+
+    const f = this.#parent.contentFile(this.#id);
+    if (!f) {
+      return undefined;
+    }
+
+    const data = await Deno.readFile(
+      path.join(this.#parent.stateRoot, ".wmill", f),
+    );
+    const content = CONTENT_ENCODER.decode(data);
+    this.#content_cache = content;
+    return content;
+  }
+
+  getHash(): string {
+    return this.#parent.hashes.get(this.#id)!;
+  }
+
+  getId(): TrackedId {
+    return this.#id;
+  }
+}
+
+@model()
+export class State {
+  // TODO: Handle script contents, as they are separate files & need to be tracked separately.
+  // I think the way this makes most sense is actually reating them as a separate kind of file everywhere
+  // And then move to using a diff-based push system everywhere. This should greatly simplify code & enable this easily
+
+  @property(map(() => TrackedId, () => String, { shape: MapShape.Object }))
+  hashes: Map<TrackedId, string>;
+
+  @property(map(() => TrackedId, () => String, { shape: MapShape.Object }))
+  contentFiles: Map<TrackedId, string>;
+
+  @property(map(() => String, () => TrackedId, { shape: MapShape.Object }))
+  tracked: Map<string, TrackedId>;
+
+  @property(() => String)
+  workspaceId: string;
+
+  @property(() => String)
+  remoteUrl: string;
+
+  stateRoot?: string;
+
+  constructor(
+    hashes: Map<TrackedId, string>,
+    tracked: Map<string, TrackedId>,
+    contentFiles: Map<TrackedId, string>,
+    workspaceId: string,
+    remoteUrl: string,
+  ) {
+    this.hashes = hashes;
+    this.tracked = tracked;
+    this.contentFiles = contentFiles;
+    this.workspaceId = workspaceId;
+    this.remoteUrl = remoteUrl;
+  }
+
+  add(path: string) {
+    if (this.tracked.get(path)) {
+      throw new Error("Cannot newly track already tracked paths");
+    } else {
+      this.tracked.set(path, nanoid());
+    }
+  }
+
+  public forget(path: string) {
+    const id = this.tracked.get(path);
+    if (id) {
+      this.tracked.delete(path);
+      this.hashes.delete(id);
+    }
+  }
+
+  public contentFile(id: TrackedId): string | undefined {
+    return this.contentFiles.get(id);
+  }
+
+  public get(path: string): Tracked {
+    const id = this.tracked.get(path);
+    if (!id) {
+      throw new Error("Could not resolve path " + path);
+    }
+    return new Tracked(id, this, path);
+  }
+
+  public async save(): Promise<void> {
+    if (!this.stateRoot) {
+      throw new Error("Uninitialized state root");
+    }
+    const encoder = new cbor.Encoder({});
+    const plain = decoverto.type(State).instanceToPlain(this);
+    const result: Uint8Array = encoder.encode(plain, {});
+    await Deno.writeFile(path.join(this.stateRoot, ".wmill", "main"), result, {
+      create: true,
+    });
+  }
+
+  public static async loadState(dir: string): Promise<State> {
+    const source = await Deno.readFile(path.join(dir, ".wmill", "main"));
+    const encoder = new cbor.Encoder({});
+    const raw = encoder.decode(source);
+    const state = decoverto.type(State).plainToInstance(
+      raw,
+    );
+    state.stateRoot = dir;
+    return Object.freeze(state);
+  }
+}
+
+async function getState(opts: GlobalOptions) {
+  const existingState = await State.loadState(Deno.cwd());
+
+  const workspaceStream = await getWorkspaceStream();
+  const reader = workspaceStream.getReader();
+  while (true) {
+    const res = await reader.read();
+    if (res.value) {
+      if (
+        new URL(res.value.remote) == new URL(existingState.remoteUrl) &&
+        res.value.workspaceId == existingState.workspaceId
+      ) {
+        opts.workspace = res.value.name;
+        (opts as any).__secret_workspace = res.value;
+        break;
+      }
+    }
+    if (res.done) {
+      break;
+    }
+  }
+  return existingState;
+}
+
+async function updateStateFromRemote(
+  workspace: Workspace,
+  state: State,
+  callback: (filename: string) => PromiseLike<boolean> | boolean,
+) {
+  const untar = await downloadTar(workspace);
+  if (!untar) throw new Error("Failed to pull Tar");
+
+  const decoder = new TextDecoder();
+  for await (const entry of untar) {
+    const id = state.tracked.get(entry.fileName);
+    if (id) {
+      let val = "";
+      for await (const e of iterateReader(entry)) {
+        const tmp = decoder.decode(e);
+        val += tmp;
+      }
+
+      if (entry.fileName.endsWith(".json")) {
+        const parsed = JSON.parse(val);
+        const typed = inferTypeFromPath(entry.fileName, parsed);
+
+        const oldHash = state.hashes.get(id);
+        const newHash = objectHash(typed);
+
+        if (!oldHash || oldHash !== newHash) {
+          if (!await callback(entry.fileName)) {
+            return; // notice that we are not saving
+          }
+          state.hashes.set(id, newHash);
+
+          const encoded = CONTENT_ENCODER.encode(typed);
+
+          const fileName = nanoid();
+          await Deno.writeFile(
+            path.join(state.stateRoot!, ".wmill", fileName),
+            encoded,
+            { create: true },
+          );
+          state.contentFiles.set(id, fileName);
+        }
+      } else {
+        const fileName = nanoid();
+        await Deno.writeTextFile(
+          path.join(state.stateRoot!, ".wmill", fileName),
+          val,
+          { create: true },
+        );
+        state.contentFiles.set(id, fileName);
+      }
+    }
+  }
+
+  await state.save();
+}
+
+async function pull(
+  opts: GlobalOptions & { raw: boolean; rawOverride: boolean },
+) {
+  if (opts.raw) {
+    const opts2 = opts as any;
+    opts2.override = opts2.rawOverride;
+    opts2.raw = undefined;
+    opts2.rawOverride = undefined;
+    await pullRaw(opts2, Deno.cwd());
+    return;
+  }
+
+  const state = await getState(opts);
+
+  const workspace = await resolveWorkspace(opts);
+  await requireLogin(opts);
+
+  console.log("Pulling remote changes");
+  await updateStateFromRemote(workspace, state, (_) => true);
+
+  const diffs = diffState(state);
+
+  await copyNonJsonFiles(state);
+
+  console.log(`Applying changes to files`);
+  for await (const diff of diffs) {
+    await applyDiff(
+      diff.diff,
+      path.join(state.stateRoot!, diff.localPath),
+    );
+  }
+  console.log(colors.green.underline("Done! All changes applied."));
+
+  async function applyDiff(diffs: Difference[], file: string) {
+    ensureDir(path.dirname(file));
+    let json;
+    try {
+      json = JSON.parse(await Deno.readTextFile(file));
+    } catch {
+      json = {};
+    }
+    // TODO: Delegate the below to the object itself
+    // This would work by infering the type of `JSON` (which includes then statically typing it using decoverto) and then
+    // delegating the applying of the diffs to the object via an interface
+    for (const diff of diffs) {
+      if (diff.type === "CREATE") {
+        setValueByPath(json, diff.path, diff.value);
+      } else if (diff.type === "REMOVE") {
+        setValueByPath(json, diff.path, undefined);
+      } else if (diff.type === "CHANGE") {
+        setValueByPath(json, diff.path, diff.value);
+      }
+    }
+
+    // TODO: Use decoverto instanceToPlain below.
+    await Deno.writeTextFile(file, JSON.stringify(json, undefined, "  "), {
+      create: true,
+    });
+  }
+
+  async function copyNonJsonFiles(state: State): Promise<void> {
+    for (const t of state.tracked.keys()) {
+      if (t.endsWith(".json")) {
+        continue;
+      }
+      const entry = state.get(t);
+
+      const target = await Deno.open(entry.path, { create: true, write: true });
+      const source = await Deno.open(
+        path.join(
+          state.stateRoot!,
+          ".wmill",
+          state.contentFile(entry.getId())!,
+        ),
+        {
+          read: true,
+          write: false,
+        },
+      );
+      await source.readable.pipeTo(target.writable);
+    }
+  }
+}
+
+async function* diffState(state: State): AsyncGenerator<StateDiff, void, void> {
+  for (const t of state.tracked.keys()) {
+    if (!t.endsWith(".json")) {
+      continue;
+    }
+    const entry = state.get(t);
+    let fileText;
+    try {
+      fileText = await Deno.readTextFile(
+        path.join(state.stateRoot!, entry.path),
+      );
+    } catch {
+      fileText = "{}";
+    }
+    const old = JSON.parse(fileText);
+    const fileHash = objectHash(old);
+
+    if (fileHash !== entry.getHash()) {
+      const stateContent = await entry.getContent() as any;
+
+      const diff = microdiff(
+        old,
+        stateContent ?? {},
+        { cyclesFix: false },
+      );
+
+      yield new StateDiff(entry.getId(), entry.path, diff);
+    }
+  }
+}
+
+function prettyDiff(diffs: Difference[]) {
+  for (const diff of diffs) {
+    let pathString = "";
+    for (const pathSegment of diff.path) {
+      if (typeof pathSegment === "string") {
+        pathString += ".";
+        pathString += pathSegment;
+      } else {
+        pathString += "[";
+        pathString += pathSegment;
+        pathString += "]";
+      }
+    }
+    if (diff.type === "REMOVE" || diff.type === "CHANGE") {
+      console.log(colors.red("- " + pathString + " = " + diff.oldValue));
+    }
+    if (diff.type === "CREATE" || diff.type === "CHANGE") {
+      console.log(colors.green("+ " + pathString + " = " + diff.value));
+    }
+  }
+}
+
+async function push(opts: GlobalOptions & { raw: boolean }) {
+  if (opts.raw) {
+    const opts2 = opts as any;
+    opts2.raw = undefined;
+    await pushRaw(opts2, undefined);
+    return;
+  }
+
+  const state = await getState(opts);
+
+  const workspace = await resolveWorkspace(opts);
+  await requireLogin(opts);
+
+  let error = false;
+  await updateStateFromRemote(workspace, state, async (filename) => {
+    const e = state.get(filename);
+    if (!e) {
+      throw new Error("!? State change on untracked file ?!");
+    }
+
+    let fileJSON;
+    try {
+      fileJSON = JSON.parse(
+        await Deno.readTextFile(path.join(state.stateRoot!, e.path)),
+      );
+    } catch {
+      fileJSON = {};
+    }
+    const file = inferTypeFromPath(e.path, fileJSON);
+    const eContent = (inferTypeFromPath(e.path, await e.getContent())) ?? {};
+
+    const fileHash = objectHash(file);
+    const eHash = objectHash(eContent);
+
+    if (fileHash !== eHash) {
+      console.log(
+        colors.red("!! Local and Remote change present. Local diff:"),
+      );
+      prettyDiff(microdiff(eContent as any, file, { cyclesFix: false }));
+      console.log(
+        colors.red(
+          "Consider comitting or otherwise saving your work and pulling to load any remote changes",
+        ),
+      );
+      error = true;
+      return false;
+    }
+    return true;
+  });
+  if (error) {
+    return;
+  }
+
+  for (const p of state.tracked.keys()) {
+    if (!p.endsWith(".json")) continue;
+    const entry = state.get(p);
+    let fileJSON;
+    try {
+      fileJSON = JSON.parse(
+        await Deno.readTextFile(path.join(state.stateRoot!, entry.path)),
+      );
+    } catch {
+      fileJSON = {};
+    }
+    const file = inferTypeFromPath(entry.path, fileJSON);
+    const eContent =
+      (inferTypeFromPath(entry.path, await entry.getContent())) ?? {};
+
+    const fileHash = objectHash(file);
+    const eHash = objectHash(eContent);
+
+    if (fileHash !== eHash) {
+      const remotePath = entry.path.split(".")[0];
+      const type = getTypeStrFromPath(entry.path);
+      if (type === "script") {
+        // Diffing makes no sense for scripts - instead fetch parent hash & check hash again.
+        // If hash is still missmatched - create new script as child.
+        const typed = decoverto.type(ScriptFile).plainToInstance(file);
+        const contentPath = await findContentFile(entry.path);
+        const language = inferContentTypeFromFilePath(contentPath);
+        const content = await Deno.readTextFile(contentPath);
+        try {
+          const remote = await ScriptService.getScriptByPath({
+            workspace: workspace.workspaceId,
+            path: remotePath,
+          });
+          if (objectHash(remote) !== fileHash) {
+            await ScriptService.createScript({
+              workspace: workspace.workspaceId,
+              requestBody: {
+                content,
+                description: typed.description,
+                language,
+                path: remotePath,
+                summary: typed.summary,
+                is_template: typed.is_template,
+                kind: typed.kind,
+                lock: undefined,
+                parent_hash: remote.hash,
+                schema: typed.schema,
+              },
+            });
+          }
+        } catch {
+          // no parent hash
+          await ScriptService.createScript({
+            workspace: workspace.workspaceId,
+            requestBody: {
+              content,
+              description: typed.description,
+              language,
+              path: remotePath,
+              summary: typed.summary,
+              is_template: typed.is_template,
+              kind: typed.kind,
+              lock: undefined,
+              parent_hash: undefined,
+              schema: typed.schema,
+            },
+          });
+        }
+      } else {
+        const diff = microdiff(eContent as any, file, { cyclesFix: false });
+        await applyDiff(
+          workspace.workspaceId,
+          remotePath,
+          file,
+          diff,
+        );
+      }
+    }
+  }
+
+  let anyRemoteChanges = false;
+  await updateStateFromRemote(workspace, state, (_) => {
+    anyRemoteChanges = true;
+    return true;
+  });
+  if (anyRemoteChanges) {
+    console.log("New remote changes - consider pulling");
+  }
+
+  function applyDiff(
+    workspace: string,
+    remotePath: string,
+    file:
+      | ScriptFile
+      | VariableFile
+      | FlowFile
+      | ResourceFile
+      | ResourceTypeFile
+      | FolderFile,
+    diffs: Difference[],
+  ) {
+    if (file instanceof ScriptFile) {
+      throw new Error(
+        "This code path should be unreachable - we should never generate diffs for scripts",
+      );
+    } else if (file instanceof FolderFile) {
+      const parts = remotePath.split("/");
+      if (parts[0] === "f") {
+        remotePath = parts[1];
+      } else {
+        remotePath = parts[0];
+      }
+    }
+    return file.pushDiffs(workspace, remotePath, diffs);
+  }
+}
+
+class StateDiff {
+  trackedId: TrackedId;
+  localPath: string;
+  diff: Difference[];
+
+  constructor(
+    trackedId: TrackedId,
+    localPath: string,
+    diff: Difference[],
+  ) {
+    this.trackedId = trackedId;
+    this.localPath = localPath;
+    this.diff = diff;
+  }
+}
+
+async function add(opts: GlobalOptions, path: string) {
+  const state = await getState(opts);
+
+  // TODO: Automatically check whether this path exists either locally or on the remote
+  state.add(path);
+
+  if (path.endsWith(".script.json")) {
+    try {
+      const f = await findContentFile(path);
+      state.add(f);
+    } catch {
+      const workspace = await resolveWorkspace(opts);
+      await requireLogin(opts);
+      try {
+        const old = await ScriptService.getScriptByPath({
+          workspace: workspace.workspaceId,
+          path: path.split(".")[0],
+        });
+        if (old.language === "python3") {
+          state.add(path.replace(".script.json", ".py"));
+        } else if (old.language === "bash") {
+          state.add(path.replace(".script.json", ".sh"));
+        } else if (old.language === "deno") {
+          state.add(path.replace(".script.json", ".ts"));
+        } else if (old.language === "go") {
+          state.add(path.replace(".script.json", ".go"));
+        } else {
+          throw new Error("Remote returned invalid language?! " + old.language);
+        }
+      } catch {
+        throw new Error(
+          "Could not infer script language from local or remote. Exiting.",
+        );
+      }
+    }
+  }
+
+  await state.save();
+}
+
+async function init(opts: GlobalOptions) {
+  const root = Deno.cwd();
+  try {
+    await Deno.mkdir(path.join(root, ".wmill"));
+  } catch {
+    console.log(
+      colors.red(
+        "! Looks like this folder is already initialized or we are missing permissions to do so. Exiting.",
+      ),
+    );
+    return;
+  }
+
+  const workspace = await resolveWorkspace(opts);
+
+  const newState = new State(
+    new Map(),
+    new Map(),
+    new Map(),
+    workspace.workspaceId,
+    workspace.remote,
+  );
+  newState.stateRoot = root;
+
+  await newState.save();
+}
+
+async function pullRaw(
+  opts: GlobalOptions & { override: boolean },
+  dir: string,
+) {
+  const workspace = await resolveWorkspace(opts);
+
+  const untar = await downloadTar(workspace);
+  if (!untar) return;
+
+  for await (const entry of untar) {
+    console.log(entry.fileName);
+    const filePath = path.resolve(dir, entry.fileName);
+    if (entry.type === "directory") {
+      await ensureDir(filePath);
+      continue;
+    }
+    await ensureDir(path.dirname(filePath));
+    if (!opts.override) {
+      let exists = false;
+      try {
+        const _stat = await Deno.stat(filePath);
+        exists = true;
+      } catch {
+        exists = false;
+      }
+      if (exists) {
+        if (
+          !(await Confirm.prompt(
+            "Conflict at " +
+              filePath +
+              " do you want to override the local version?",
+          ))
+        ) {
+          continue;
+        }
+      }
+    }
+    const file = await Deno.open(filePath, { write: true, create: true });
+    const len = await copy(entry, file);
+    await file.truncate(len);
+    file.close();
+  }
+  console.log(colors.green("Done. Wrote all files to disk."));
+}
+
+type PushRawCandidate = {
+  path: string;
+  namespaceKind: "user" | "group" | "folder";
+  namespaceName: string;
+};
+
+type PushRawResourceTypeCandidate = {
+  path: string;
+};
+
+type PushRawFolderCandidate = {
+  path: string;
+  namespaceName: string;
+};
+
+async function pushRawFindCandidateFiles(
+  dir: string,
+): Promise<
+  {
+    normal: PushRawCandidate[];
+    resourceTypes: PushRawResourceTypeCandidate[];
+    folders: PushRawFolderCandidate[];
+  }
+> {
+  dir = path.resolve(dir);
+  if (path.dirname(dir).startsWith(".")) {
+    return { normal: [], resourceTypes: [], folders: [] };
+  }
+  const normalCandidates: PushRawCandidate[] = [];
+  const resourceTypeCandidates: PushRawResourceTypeCandidate[] = [];
+  const folderCandidates: PushRawFolderCandidate[] = [];
+  for await (const e of Deno.readDir(dir)) {
+    if (e.isDirectory) {
+      if (e.name == "u" || e.name == "g" || e.name == "f") { // TODO: Check version for f
+        const newDir = dir + (dir.endsWith("/") ? "" : "/") + e.name;
+        for await (const e2 of Deno.readDir(newDir)) {
+          if (e2.isDirectory) {
+            if (e2.name.startsWith(".")) continue;
+            const namespaceName = e2.name;
+            const stack: string[] = [];
+            {
+              const path = newDir + "/" + namespaceName + "/";
+              stack.push(path);
+              try {
+                await Deno.stat(path + "folder.meta.json");
+                folderCandidates.push({
+                  namespaceName,
+                  path: path + "folder.meta.json",
+                });
+              } catch {}
+            }
+
+            while (stack.length > 0) {
+              const dir2 = stack.pop()!;
+              for await (const e3 of Deno.readDir(dir2)) {
+                if (e3.isFile) {
+                  if (e3.name === "folder.meta.json") continue;
+                  normalCandidates.push({
+                    path: dir2 + e3.name,
+                    namespaceKind: e.name == "g"
+                      ? "group"
+                      : e.name == "u"
+                      ? "user"
+                      : "folder",
+                    namespaceName: namespaceName,
+                  });
+                } else {
+                  stack.push(dir2 + e3.name + "/");
+                }
+              }
+            }
+          }
+        }
+      } else {
+        console.log(
+          colors.yellow(
+            "Including organizational folder " + e.name + " in push!",
+          ),
+        );
+        const { normal, resourceTypes, folders } =
+          await pushRawFindCandidateFiles(
+            path.join(dir, e.name),
+          );
+        normalCandidates.push(...normal);
+        resourceTypeCandidates.push(...resourceTypes);
+        folderCandidates.push(...folders);
+      }
+    } else {
+      // handle root files
+      if (e.name.endsWith(".resource-type.json")) {
+        resourceTypeCandidates.push({
+          path: dir + (dir.endsWith("/") ? "" : "/") + e.name,
+        });
+      }
+    }
+  }
+  return {
+    normal: normalCandidates,
+    folders: folderCandidates,
+    resourceTypes: resourceTypeCandidates,
+  };
+}
+
+async function pushRaw(opts: GlobalOptions, dir?: string) {
+  dir = dir ?? Deno.cwd();
+  const workspace = await resolveWorkspace(opts);
+  await requireLogin(opts);
+
+  console.log(colors.blue("Searching Directory..."));
+  const { normal, resourceTypes, folders } = await pushRawFindCandidateFiles(
+    dir,
+  );
+  console.log(
+    colors.blue(
+      "Found " + (normal.length + resourceTypes.length + folders.length) +
+        " candidates",
+    ),
+  );
+  for (const resourceType of resourceTypes) {
+    const fileName = resourceType.path.substring(
+      resourceType.path.lastIndexOf("/") + 1,
+    );
+    const fileNameParts = fileName.split(".");
+    // invalid file names, like my.cool.script.script.json. Not valid.
+    if (fileNameParts.length != 3) {
+      console.log(
+        colors.yellow("invalid file name found at " + resourceType.path),
+      );
+      continue;
+    }
+
+    // filter out non-json files. Note that we filter out script contents above, so this is really an error.
+    if (fileNameParts.at(-1) != "json") {
+      console.log(colors.yellow("non-JSON file found at " + resourceType.path));
+      continue;
+    }
+
+    console.log("pushing resource type " + fileNameParts.at(-3)!);
+    await decoverto.type(ResourceTypeFile).rawToInstance(
+      await Deno.readTextFile(resourceType.path),
+    ).push(workspace.workspaceId, fileNameParts.at(-3)!);
+  }
+  for (const folder of folders) {
+    await decoverto.type(FolderFile).plainToInstance(
+      JSON.parse(await Deno.readTextFile(folder.path)),
+    ).push(
+      workspace.workspaceId,
+      "f/" + folder.namespaceName,
+    );
+  }
+  for (const candidate of normal) {
+    // full file name. No leading /. includes .type.json
+    const fileName = candidate.path.substring(
+      candidate.path.lastIndexOf("/") + 1,
+    );
+    // figure out just the path after ...../u|g/username|group/ (in extra dir)
+    const dirParts = candidate.path.split("/").filter((x) => x.length > 0);
+    // TODO: check version for folder
+    const gIndex = dirParts.findIndex((x) => x == "u" || x == "g" || x == "f");
+    const extraDir = dirParts.slice(gIndex + 2, -1).join("/");
+
+    // file name parts has .json (hopefully) at -1, type at -2, and the actual name at -3. Dots in names are not allowed.
+    const fileNameParts = fileName.split(".");
+
+    // filter out script content files
+    if (
+      fileNameParts.at(-1) == "ts" ||
+      fileNameParts.at(-1) == "py" ||
+      fileNameParts.at(-1) == "go"
+    ) {
+      // probably part of a script. Silent ignore.
+      continue;
+    }
+
+    // invalid file names, like my.cool.script.script.json. Not valid.
+    if (fileNameParts.length != 3) {
+      console.log(
+        colors.yellow("invalid file name found at " + candidate.path),
+      );
+      continue;
+    }
+
+    // filter out non-json files. Note that we filter out script contents above, so this is really an error.
+    if (fileNameParts.at(-1) != "json") {
+      console.log(colors.yellow("non-JSON file found at " + candidate.path));
+      continue;
+    }
+
+    // get the type & filter it for valid ones.
+    const type = fileNameParts.at(-2);
+    if (type == "resource-type") {
+      console.log(
+        colors.yellow(
+          "Found resource type file at " +
+            candidate.path +
+            " this appears to be inside a path folder. Resource types are not addressed by path. Place them at the root or inside only an organizational folder. Ignoring this file!",
+        ),
+      );
+      continue;
+    }
+
+    if (
+      type != "flow" &&
+      type != "resource" &&
+      type != "script" &&
+      type != "variable"
+    ) {
+      console.log(
+        colors.yellow(
+          "file with invalid type " + type + " found at " + candidate.path,
+        ),
+      );
+      continue;
+    }
+
+    // create the remotePath for the API
+    const remotePath = (candidate.namespaceKind === "group"
+      ? "g/"
+      : (candidate.namespaceKind === "user" ? "u/" : "f/")) +
+      candidate.namespaceName +
+      "/" +
+      (extraDir.length > 0 ? extraDir + "/" : "") +
+      fileNameParts.at(-3);
+
+    console.log("pushing " + type + " to " + remotePath);
+
+    const typed = inferTypeFromPath(
+      candidate.path,
+      JSON.parse(await Deno.readTextFile(candidate.path)),
+    );
+    if (typed instanceof ResourceTypeFile || typed instanceof FolderFile) {
+      throw new Error(
+        "Resource Types and Folders should  be filtered out at this point!",
+      );
+    } else if (typed instanceof ScriptFile) {
+      let contentPath: string;
+      try {
+        contentPath = await findContentFile(candidate.path);
+      } catch (e) {
+        console.log(colors.red(e.toString()));
+        continue;
+      }
+      await pushScript(
+        candidate.path,
+        contentPath,
+        workspace.workspaceId,
+        remotePath,
+      );
+    } else {
+      typed.push(workspace.workspaceId, remotePath);
+    }
+  }
+  console.log(colors.underline.bold.green("Successfully Pushed all files."));
+}
+
+const command = new Command()
+  .command("init")
+  .description(
+    "Initialize this folder as sync root for the currently selected workspace & remote.",
+  )
+  .action(init as any)
+  .command("add")
+  .description("Add a local file for tracking")
+  .arguments("<path:string>")
+  .action(add as any)
+  .command("pull")
+  .description("Pull any remote changes and apply them locally")
+  .option("--raw", "Pull without using state.")
+  .option("--raw-override", "Always override local files with remote.", {
+    depends: ["raw"],
+  })
+  .action(pull as any)
+  .command("push")
+  .description("Push any local changes and apply them remotely")
+  .option("--raw", "Push without using state.")
+  .action(push as any);
+
+export default command;

cli/tmp/f/my_folder/folder.meta.json

@@ -1,10 +0,0 @@
-{
-  "workspace_id": "admins",
-  "name": "my_folder",
-  "display_name": "my_folder",
-  "owners": [],
-  "extra_perms": {
-    "u/test": true,
-    "u/admin@windmill.dev": false
-  }
-}

cli/tmp/u/admin/hub_sync.script.json

@@ -1,12 +0,0 @@
-{
-  "summary": "Syncronize Hub Resource types with starter workspace",
-  "description": "Basic administrative script to sync latest resource types from hub. Recommended to run at least once. On a schedule by default.",
-  "schema": {
-    "$schema": "https://json-schema.org/draft/2020-12/schema",
-    "properties": {},
-    "required": [],
-    "type": "object"
-  },
-  "is_template": false,
-  "lock": []
-}

cli/tmp/u/admin/hub_sync.ts

@@ -1,13 +0,0 @@
-import wmill from "https://deno.land/x/wmill@v1.55.0/main.ts";
-
-export async function main() {
-  await run(
-    "workspace", "add", "__automation", "starter", Deno.env.get("WM_BASE_URL") + "/", "--token", Deno.env.get("WM_TOKEN"));
-
-  await run("hub", "pull");
-}
-
-async function run(...cmd: string[]) {
-  console.log("Running \"" + cmd.join(' ') + "\"");
-  await wmill.parse(cmd);
-}

cli/types.ts

@@ -1,4 +1,137 @@
+import { decoverto } from "./decoverto.ts";
+import { FlowFile } from "./flow.ts";
+import { ResourceTypeFile } from "./resource-type.ts";
+import { ResourceFile } from "./resource.ts";
+import { ScriptFile } from "./script.ts";
+import { VariableFile } from "./variable.ts";
+import { path } from "./deps.ts";
+import { FolderFile } from "./folder.ts";
+
+// TODO: Remove this & replace with a "pull" that lets the object either pull the remote version or return undefined.
+// Then combine those with diffing, which then gives the new push impl
+export interface Resource {
+  push(workspace: string, remotePath: string): Promise<void>;
+}
+
+export interface PushDiffs {
+  pushDiffs(
+    workspace: string,
+    remotePath: string,
+    diffs: Difference[],
+  ): Promise<void>;
+}
+
+export interface DifferenceCreate {
+  type: "CREATE";
+  path: (string | number)[];
+  value: any;
+}
+
+export interface DifferenceRemove {
+  type: "REMOVE";
+  path: (string | number)[];
+  oldValue: any;
+}
+
+export interface DifferenceChange {
+  type: "CHANGE";
+  path: (string | number)[];
+  value: any;
+  oldValue: any;
+}
+
+export type Difference = DifferenceCreate | DifferenceRemove | DifferenceChange;
+
+export function setValueByPath(
+  obj: any,
+  path: (string | number)[],
+  value: any,
+) {
+  let i;
+  let lastObj = undefined;
+  for (i = 0; i < path.length - 1; i++) {
+    if (!obj) {
+      let oldNewObj;
+      if (typeof path[i] === "number") {
+        oldNewObj = [];
+      } else {
+        oldNewObj = {};
+      }
+      lastObj[path[i - 1]] = oldNewObj;
+      obj = oldNewObj;
+    }
+    lastObj = obj;
+    obj = obj[path[i]];
+  }
+  if (!obj) {
+    let oldNewObj;
+    if (typeof path[i] === "number") {
+      oldNewObj = [];
+    } else {
+      oldNewObj = {};
+    }
+    lastObj[path[i - 1]] = oldNewObj;
+    obj = oldNewObj;
+  }
+  obj[path[i]] = value;
+}
+
 export type GlobalOptions = {
   workspace: string | undefined;
   token: string | undefined;
 };
+
+export function inferTypeFromPath(
+  p: string,
+  obj: any,
+):
+  | ScriptFile
+  | VariableFile
+  | FlowFile
+  | ResourceFile
+  | ResourceTypeFile
+  | FolderFile {
+  const typeEnding = getTypeStrFromPath(p);
+
+  if (typeEnding === "folder") {
+    return decoverto.type(FolderFile).plainToInstance(obj);
+  } else if (typeEnding === "script") {
+    return decoverto.type(ScriptFile).plainToInstance(obj);
+  } else if (typeEnding === "variable") {
+    return decoverto.type(VariableFile).plainToInstance(obj);
+  } else if (typeEnding === "flow") {
+    return decoverto.type(FlowFile).plainToInstance(obj);
+  } else if (typeEnding === "resource") {
+    return decoverto.type(ResourceFile).plainToInstance(obj);
+  } else if (typeEnding === "resource-type") {
+    return decoverto.type(ResourceTypeFile).plainToInstance(obj);
+  } else {
+    throw new Error("infer type unreachable");
+  }
+}
+
+export function getTypeStrFromPath(
+  p: string,
+): "script" | "variable" | "flow" | "resource" | "resource-type" | "folder" {
+  const parsed = path.parse(p);
+  if (parsed.ext !== ".json") {
+    throw new Error(
+      "Cannot infer type of non-json file " + JSON.stringify(parsed),
+    );
+  }
+
+  if (parsed.name === "folder.meta") {
+    return "folder";
+  }
+
+  const typeEnding = parsed.name.split(".").at(-1);
+  if (
+    typeEnding === "script" || typeEnding === "variable" ||
+    typeEnding === "flow" || typeEnding === "resource" ||
+    typeEnding === "resource-type"
+  ) {
+    return typeEnding;
+  } else {
+    throw new Error("Could not infer type of path " + JSON.stringify(parsed));
+  }
+}

cli/variable.ts

@@ -1,7 +1,22 @@
 // deno-lint-ignore-file no-explicit-any
 import { requireLogin, resolveWorkspace, validatePath } from "./context.ts";
-import { GlobalOptions } from "./types.ts";
-import { colors, Command, Table, VariableService } from "./deps.ts";
+import {
+  Difference,
+  GlobalOptions,
+  PushDiffs,
+  Resource,
+  setValueByPath,
+} from "./types.ts";
+import {
+  colors,
+  Command,
+  EditVariable,
+  ListableVariable,
+  microdiff,
+  Table,
+  VariableService,
+} from "./deps.ts";
+import { decoverto, model, property } from "./decoverto.ts";
 
 async function list(opts: GlobalOptions) {
   const workspace = await resolveWorkspace(opts);
@@ -26,13 +41,100 @@ async function list(opts: GlobalOptions) {
     .render();
 }
 
-type VariableFile = {
+@model()
+export class VariableFile implements Resource, PushDiffs {
+  @property(() => String)
   value: string;
+  @property(() => Boolean)
   is_secret: boolean;
+  @property(() => String)
   description: string;
+  @property(() => Number)
   account?: number;
+  @property(() => Boolean)
   is_oauth?: boolean;
-};
+
+  constructor(value: string, is_secret: boolean, description: string) {
+    this.value = value;
+    this.is_secret = is_secret;
+    this.description = description;
+  }
+  async pushDiffs(
+    workspace: string,
+    remotePath: string,
+    diffs: Difference[],
+  ): Promise<void> {
+    if (await VariableService.existsVariable({ workspace, path: remotePath })) {
+      console.log(
+        colors.bold.yellow(
+          `Applying ${diffs.length} diffs to existing variable...`,
+        ),
+      );
+      const changeset: EditVariable = {};
+      for (const diff of diffs) {
+        if (
+          diff.type !== "REMOVE" &&
+          (
+            diff.path.length !== 1 ||
+            !["path", "value", "is_secret", "description"].includes(
+              diff.path[0] as string,
+            )
+          )
+        ) {
+          throw new Error("Invalid variable diff with path " + diff.path);
+        }
+        if (diff.type === "CREATE" || diff.type === "CHANGE") {
+          setValueByPath(changeset, diff.path, diff.value);
+        } else if (diff.type === "REMOVE") {
+          setValueByPath(changeset, diff.path, null);
+        }
+      }
+
+      const hasChanges = Object.values(changeset).some((v) =>
+        v !== null && typeof v !== "undefined"
+      );
+      if (!hasChanges) {
+        console.log(colors.yellow("! Skipping empty changeset"));
+        return;
+      }
+
+      await VariableService.updateVariable({
+        workspace,
+        path: remotePath,
+        requestBody: changeset,
+      });
+    } else {
+      console.log(colors.yellow("Creating new variable..."));
+      await VariableService.createVariable({
+        workspace,
+        requestBody: {
+          path: remotePath,
+          description: this.description,
+          is_secret: this.is_secret,
+          value: this.value,
+          account: this.account,
+          is_oauth: this.is_oauth,
+        },
+      });
+    }
+  }
+  async push(workspace: string, remotePath: string): Promise<void> {
+    let existing: ListableVariable | undefined;
+    try {
+      existing = await VariableService.getVariable({
+        workspace: workspace,
+        path: remotePath,
+      });
+    } catch {
+      existing = undefined;
+    }
+    await this.pushDiffs(
+      workspace,
+      remotePath,
+      microdiff(existing ?? {}, this, { cyclesFix: false }),
+    );
+  }
+}
 
 async function push(opts: GlobalOptions, filePath: string, remotePath: string) {
   const workspace = await resolveWorkspace(opts);
@@ -58,72 +160,10 @@ export async function pushVariable(
   filePath: string,
   remotePath: string,
 ) {
-  const data: VariableFile = JSON.parse(await Deno.readTextFile(filePath));
-  if (await VariableService.existsVariable({ workspace, path: remotePath })) {
-    const existing = await VariableService.getVariable({
-      workspace: workspace,
-      path: remotePath,
-    });
-    if (existing.is_oauth != data.is_oauth) {
-      console.log(
-        colors.red.underline.bold(
-          "Remote variable at " +
-            remotePath +
-            " exists & has a different oauth state. This cannot be updated. If you wish to do this anyways, consider deleting the remote resource.",
-        ),
-      );
-      return;
-    }
-
-    if (existing.account != data.account) {
-      console.log(
-        colors.red.underline.bold(
-          "Remote variable at " +
-            remotePath +
-            " exists & has a different account state. This cannot be updated. If you wish to do this anyways, consider deleting the remote resource.",
-        ),
-      );
-      return;
-    }
-
-    if (existing.is_secret && !data.is_secret) {
-      console.log(
-        colors.red.underline.bold(
-          "Remote variable at " +
-            remotePath +
-            " exists & is secret. Variables cannot be updated to be no longer secret. If you wish to do this anyways, consider deleting the remote resource.",
-        ),
-      );
-      return;
-    }
-
-    const actual_secret = data.is_secret ? true : undefined;
-
-    console.log(colors.yellow("Updating existing variable..."));
-    await VariableService.updateVariable({
-      workspace,
-      path: remotePath,
-      requestBody: {
-        description: data.description,
-        is_secret: actual_secret,
-        path: remotePath,
-        value: data.value,
-      },
-    });
-  } else {
-    console.log(colors.yellow("Creating new variable..."));
-    await VariableService.createVariable({
-      workspace,
-      requestBody: {
-        path: remotePath,
-        description: data.description,
-        is_secret: data.is_secret,
-        value: data.value,
-        account: data.account,
-        is_oauth: data.is_oauth,
-      },
-    });
-  }
+  const data = decoverto.type(VariableFile).rawToInstance(
+    await Deno.readTextFile(filePath),
+  );
+  await data.push(workspace, remotePath);
 }
 
 const command = new Command()

cli/workspace.ts

@@ -11,14 +11,31 @@ import {
   Table,
   WorkspaceService,
 } from "./deps.ts";
-import { requireLogin } from "./context.ts";
+import { decoverto, model, property } from "./decoverto.ts";
 
-export type Workspace = {
+@model()
+export class Workspace {
+  @property(() => String)
   remote: string;
+  @property(() => String)
   workspaceId: string;
+  @property(() => String)
   name: string;
+  @property(() => String)
   token: string;
-};
+
+  constructor(
+    remote: string,
+    workspaceId: string,
+    name: string,
+    token: string,
+  ) {
+    this.remote = remote;
+    this.workspaceId = workspaceId;
+    this.name = name;
+    this.token = token;
+  }
+}
 
 function makeWorkspaceStream(
   readable: ReadableStream<Uint8Array>,
@@ -33,7 +50,9 @@ function makeWorkspaceStream(
             if (line.length <= 2) {
               return;
             }
-            controller.enqueue(JSON.parse(line) as Workspace);
+            const workspace = decoverto.type(Workspace).rawToInstance(line);
+            workspace.remote = new URL(workspace.remote).toString(); // add trailing slash in all cases!
+            controller.enqueue(workspace);
           } catch {
             /* ignore */
           }
@@ -42,14 +61,17 @@ function makeWorkspaceStream(
     );
 }
 
-async function allWorkspaces(): Promise<Workspace[]> {
-  try {
-    const file = await Deno.open((await getRootStore()) + "remotes.ndjson", {
-      write: false,
-      read: true,
-    });
-    const workspaceStream = makeWorkspaceStream(file.readable);
+export async function getWorkspaceStream() {
+  const file = await Deno.open((await getRootStore()) + "remotes.ndjson", {
+    write: false,
+    read: true,
+  });
+  return makeWorkspaceStream(file.readable);
+}
 
+export async function allWorkspaces(): Promise<Workspace[]> {
+  try {
+    const workspaceStream = await getWorkspaceStream();
     const workspaces: Workspace[] = [];
     for await (const workspace of workspaceStream) {
       workspaces.push(workspace);
@@ -87,8 +109,7 @@ export async function getActiveWorkspace(
 export async function getWorkspaceByName(
   workspaceName: string,
 ): Promise<Workspace | undefined> {
-  const file = await Deno.open((await getRootStore()) + "remotes.ndjson");
-  const workspaceStream = makeWorkspaceStream(file.readable);
+  const workspaceStream = await getWorkspaceStream();
   for await (const workspace of workspaceStream) {
     if (workspace.name === workspaceName) {
       return workspace;
@@ -185,6 +206,7 @@ export async function add(
       remote = new URL(await Input.prompt("Enter the Remote URL")).toString();
     }
   }
+  remote = new URL(remote).toString(); // add trailing slash in all cases!
 
   let token = await tryGetLoginInfo(opts);
   while (!token) {
@@ -192,7 +214,10 @@ export async function add(
   }
 
   if (opts.create) {
-    setClient(token, remote.endsWith('/') ? remote.substring(0, remote.length - 1) : remote);
+    setClient(
+      token,
+      remote.endsWith("/") ? remote.substring(0, remote.length - 1) : remote,
+    );
 
     if (
       !await WorkspaceService.existsWorkspace({
@@ -224,6 +249,7 @@ export async function add(
 }
 
 export async function addWorkspace(workspace: Workspace) {
+  workspace.remote = new URL(workspace.remote).toString(); // add trailing slash in all cases!
   const file = await Deno.open((await getRootStore()) + "remotes.ndjson", {
     append: true,
     write: true,
@@ -234,7 +260,7 @@ export async function addWorkspace(workspace: Workspace) {
   file.close();
 }
 
-async function remove(_opts: GlobalOptions, name: string) {
+export async function removeWorkspace(name: string) {
   const orgWorkspaces = await allWorkspaces();
   await Deno.writeTextFile(
     (await getRootStore()) + "remotes.ndjson",
@@ -243,6 +269,10 @@ async function remove(_opts: GlobalOptions, name: string) {
       .map((x) => JSON.stringify(x))
       .join("\n"),
   );
+}
+
+async function remove(_opts: GlobalOptions, name: string) {
+  await removeWorkspace(name);
   console.log(colors.green.underline("Succesfully removed workspace!"));
 }
 

docker-compose.yml

@@ -20,29 +20,41 @@ services:
       timeout: 5s
       retries: 5
 
-  windmill:
+  windmill_server:
     image: ghcr.io/windmill-labs/windmill:main
-    # Set privileged to true if enabling nsjail
-    privileged: false
+    deploy:
+      replicas: 1
     restart: unless-stopped
     ports:
       - 8000:8000
+    environment:
+      - DATABASE_URL=postgres://postgres:${DB_PASSWORD}@db/windmill?sslmode=disable
+      - BASE_URL=http://${WM_BASE_URL}
+      - RUST_LOG=info
+      ## You can set the number of workers to > 0 and not need any separate worker service
+      - NUM_WORKERS=0
+      - DISABLE_SERVER=false
+      - METRICS_ADDR=false
+    depends_on:
+      db:
+        condition: service_healthy
+
+  windmill_worker:
+    image: ghcr.io/windmill-labs/windmill:main
+    deploy:
+      replicas: 3
+    restart: unless-stopped
     environment:
       - DATABASE_URL=postgres://postgres:${DB_PASSWORD}@db/windmill?sslmode=disable
       - BASE_URL=http://${WM_BASE_URL}
       - BASE_INTERNAL_URL=http://localhost:8000
       - RUST_LOG=info
-      - NUM_WORKERS=3
+      - NUM_WORKERS=1
+      - DISABLE_SERVER=true
       - KEEP_JOB_DIR=false
       - DENO_PATH=/usr/bin/deno
       - PYTHON_PATH=/usr/local/bin/python3
       - METRICS_ADDR=false
-      # for ease of use, nsjail which provide isolation in untrusted environment is disabled by default. 
-      # To enable it, uncomment the following line, set the container as privileged and
-      # rebuild the image with nsjail=true or use the enterprise edition
-      # - DISABLE_NSJAIL=false
-      # - DISABLE_NUSER=false
-      # - NSJAIL_PATH=nsjail
     depends_on:
       db:
         condition: service_healthy

docker/DockerfileOpenbb

@@ -0,0 +1,140 @@
+FROM debian:buster-slim as nsjail
+
+WORKDIR /nsjail
+
+ARG nsjail=""
+
+RUN  if [ "$nsjail" = "true" ]; then apt-get -y update \
+    && apt-get install -y \
+    bison=2:3.3.* \
+    flex=2.6.* \
+    g++=4:8.3.* \
+    gcc=4:8.3.* \
+    git=1:2.20.* \
+    libprotobuf-dev=3.6.* \
+    libnl-route-3-dev=3.4.* \
+    make=4.2.* \
+    pkg-config=0.29-6 \
+    protobuf-compiler=3.6.*; fi
+
+
+RUN if [ "$nsjail" = "true" ]; then git clone -b master --single-branch https://github.com/google/nsjail.git . \
+    && git checkout dccf911fd2659e7b08ce9507c25b2b38ec2c5800; fi
+RUN if [ "$nsjail" = "true" ]; then make; else touch nsjail; fi
+
+FROM rust:slim-buster AS rust_base
+
+RUN apt-get update && apt-get install -y git libssl-dev pkg-config npm
+
+RUN apt-get -y update \
+    && apt-get install -y \
+    curl lld nodejs npm
+
+RUN rustup component add rustfmt
+
+RUN CARGO_NET_GIT_FETCH_WITH_CLI=true cargo install cargo-chef 
+
+WORKDIR /windmill
+
+ENV SQLX_OFFLINE=true
+ENV CARGO_INCREMENTAL=1
+
+FROM node:19-alpine as frontend
+
+# install dependencies
+WORKDIR /frontend
+COPY ./frontend/package.json ./frontend/package-lock.json ./
+RUN npm ci
+
+# Copy all local files into the image.
+COPY frontend .
+RUN mkdir /backend
+COPY /backend/windmill-api/openapi.yaml /backend/windmill-api/openapi.yaml
+COPY /openflow.openapi.yaml /openflow.openapi.yaml
+COPY /backend/windmill-api/build_openapi.sh /backend/windmill-api/build_openapi.sh
+RUN cd /backend/windmill-api && . ./build_openapi.sh
+
+RUN npm run generate-backend-client
+ENV NODE_OPTIONS "--max-old-space-size=8192"
+RUN npm run build
+RUN npm run check
+
+
+
+FROM rust_base AS planner
+
+COPY ./openflow.openapi.yaml /openflow.openapi.yaml
+COPY ./backend ./
+
+RUN CARGO_NET_GIT_FETCH_WITH_CLI=true cargo chef prepare --recipe-path recipe.json
+
+FROM rust_base AS builder
+ARG features=""
+
+COPY --from=planner /windmill/recipe.json recipe.json
+
+RUN CARGO_NET_GIT_FETCH_WITH_CLI=true cargo chef cook --release --features "$features" --recipe-path recipe.json
+
+COPY ./openflow.openapi.yaml /openflow.openapi.yaml
+COPY ./backend ./
+
+COPY --from=frontend /frontend /frontend
+COPY --from=frontend /backend/windmill-api/openapi-deref.yaml ./windmill-api/openapi-deref.yaml
+COPY .git/ .git/
+
+RUN CARGO_NET_GIT_FETCH_WITH_CLI=true cargo build --release --features "$features"
+
+
+FROM python:3.10.8-slim-buster
+
+ARG APP=/usr/src/app
+
+RUN apt-get update \
+    && apt-get install -y ca-certificates wget curl git jq libprotobuf-dev libnl-route-3-dev unzip \
+    && apt-get install -y ca-certificates wget curl git jq libprotobuf-dev libnl-route-3-dev unzip build-essential \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN arch="$(dpkg --print-architecture)"; arch="${arch##*-}"; \
+    curl -o rclone.zip "https://downloads.rclone.org/v1.60.1/rclone-v1.60.1-linux-$arch.zip"; \
+    unzip -p rclone.zip rclone-v1.60.1-linux-$arch/rclone > /usr/bin/rclone; rm rclone.zip; \
+    chown root:root /usr/bin/rclone; chmod 755 /usr/bin/rclone
+
+RUN set -eux; \
+    arch="$(dpkg --print-architecture)"; arch="${arch##*-}"; \
+    url=; \
+    case "$arch" in \
+    'amd64') \
+    targz='go1.19.3.linux-amd64.tar.gz'; \
+    ;; \
+    'arm64') \
+    targz='go1.19.3.linux-arm64.tar.gz'; \
+    ;; \
+    'armhf') \
+    targz='go1.19.3.linux-armv6l.tar.gz'; \
+    ;; \
+    *) echo >&2 "error: unsupported architecture '$arch' (likely packaging update needed)"; exit 1 ;; \
+    esac; \
+    wget "https://golang.org/dl/$targz" -nv && tar -C /usr/local -xzf "$targz" && rm "$targz";
+
+ENV PATH="${PATH}:/usr/local/go/bin"
+ENV GO_PATH=/usr/local/go/bin/go
+
+ENV TZ=Etc/UTC
+
+RUN /usr/local/bin/python3 -m pip install pip-tools
+
+COPY --from=builder /windmill/target/release/windmill ${APP}/windmill
+
+COPY --from=nsjail /nsjail/nsjail /bin/nsjail
+
+COPY --from=denoland/deno:latest /usr/bin/deno /usr/bin/deno
+
+RUN /usr/local/bin/python3 -m pip install openbb[all]
+
+RUN mkdir -p ${APP}
+
+WORKDIR ${APP}
+
+EXPOSE 8000
+
+CMD ["./windmill"]

frontend/.gitignore

@@ -1,6 +1,7 @@
 .DS_Store
 node_modules
 /build
+/dist
 /.svelte-kit
 /package
 /src/lib/gen

frontend/embed/README.md

@@ -0,0 +1,5 @@
+# Build the app preview embed code
+
+./embed/build.sh 
+
+serve dist

frontend/embed/build.sh

@@ -0,0 +1,6 @@
+mkdir -p dist
+npm run build-embed
+cp static/Inter-Variable.ttf dist
+cp static/tailwind.js dist
+npx tailwindcss -o src/lib/assets/app.css --minify -o dist/tailwind.css
+cp embed/index.html dist

frontend/embed/index.html

@@ -0,0 +1,840 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Embed demo</title>
+    <link href="/style.css" rel="stylesheet">
+    <link href="/tailwind.css" rel="stylesheet">
+  </head>
+  <body>
+    <div id="app"></div>
+    <script src="/appviewer.iife.js">
+    </script>    
+    </script>
+    <script>
+            loadApp(document.getElementById("app"), {
+    "grid": [
+        {
+            "3": {
+                "h": 2,
+                "w": 1,
+                "x": 0,
+                "y": 0,
+                "max": {
+                    "h": 100,
+                    "w": 3
+                },
+                "min": {
+                    "h": 1,
+                    "w": 1
+                },
+                "fixed": false,
+                "draggable": true,
+                "resizable": true,
+                "customDragger": false,
+                "customResizer": false
+            },
+            "12": {
+                "h": 5,
+                "w": 3,
+                "x": 0,
+                "y": 0,
+                "id": "a",
+                "max": {
+                    "h": 100,
+                    "w": 12
+                },
+                "min": {
+                    "h": 1,
+                    "w": 1
+                },
+                "fixed": false,
+                "draggable": true,
+                "resizable": true,
+                "customDragger": false,
+                "customResizer": false
+            },
+            "id": "a",
+            "data": {
+                "id": "a",
+                "card": false,
+                "type": "htmlcomponent",
+                "softWrap": false,
+                "configuration": {},
+                "componentInput": {
+                    "eval": "<img\n\tsrc=\"https://images.unsplash.com/photo-1554629947-334ff61d85dc?ixid=MnwxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8&amp;ixlib=rb-1.2.1&amp;auto=format&amp;fit=crop&amp;w=1024&amp;h=1280&amp;q=80\"\n>\n<div class=\"absolute top-4 left-2  text-white\">\n\tHello ${ctx.username}\n</div>\t\t\t\n",
+                    "type": "template",
+                    "value": "<img\n\tsrc=\"https://images.unsplash.com/photo-1554629947-334ff61d85dc?ixid=MnwxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8&amp;ixlib=rb-1.2.1&amp;auto=format&amp;fit=crop&amp;w=1024&amp;h=1280&amp;q=80\"\n>\n<div class=\"absolute top-4 left-2  text-white\">\n\tHello ${ctx.username}\n</div>\t\t\t\n",
+                    "fieldType": "template"
+                }
+            }
+        },
+        {
+            "3": {
+                "h": 10,
+                "w": 3,
+                "x": 0,
+                "y": 2,
+                "max": {
+                    "h": 100,
+                    "w": 3
+                },
+                "min": {
+                    "h": 1,
+                    "w": 1
+                },
+                "fixed": false,
+                "draggable": true,
+                "resizable": true,
+                "customDragger": false,
+                "customResizer": false
+            },
+            "12": {
+                "h": 10,
+                "w": 6,
+                "x": 3,
+                "y": 0,
+                "id": "b",
+                "max": {
+                    "h": 100,
+                    "w": 12
+                },
+                "min": {
+                    "h": 1,
+                    "w": 1
+                },
+                "fixed": false,
+                "draggable": true,
+                "resizable": true,
+                "customDragger": false,
+                "customResizer": false
+            },
+            "id": "b",
+            "data": {
+                "id": "b",
+                "card": true,
+                "type": "tablecomponent",
+                "actionButtons": [],
+                "configuration": {
+                    "search": {
+                        "type": "static",
+                        "value": "Disabled",
+                        "fieldType": "select",
+                        "onlyStatic": true,
+                        "optionValuesKey": "tableSearchOptions"
+                    }
+                },
+                "componentInput": {
+                    "type": "static",
+                    "value": [
+                        {
+                            "id": 1,
+                            "age": 42,
+                            "name": "A cell with a long name"
+                        },
+                        {
+                            "id": 2,
+                            "age": 84,
+                            "name": "A briefer cell"
+                        }
+                    ],
+                    "fieldType": "array",
+                    "subFieldType": "object"
+                }
+            }
+        },
+        {
+            "3": {
+                "h": 8,
+                "w": 2,
+                "x": 0,
+                "y": 12,
+                "max": {
+                    "h": 100,
+                    "w": 3
+                },
+                "min": {
+                    "h": 1,
+                    "w": 1
+                },
+                "fixed": false,
+                "draggable": true,
+                "resizable": true,
+                "customDragger": false,
+                "customResizer": false
+            },
+            "12": {
+                "h": 8,
+                "w": 6,
+                "x": 0,
+                "y": 10,
+                "max": {
+                    "h": 100,
+                    "w": 12
+                },
+                "min": {
+                    "h": 1,
+                    "w": 1
+                },
+                "fixed": false,
+                "draggable": true,
+                "resizable": true,
+                "customDragger": false,
+                "customResizer": false
+            },
+            "id": "c",
+            "data": {
+                "id": "c",
+                "card": true,
+                "type": "scatterchartcomponent",
+                "configuration": {
+                    "pannable": {
+                        "type": "static",
+                        "value": false,
+                        "fieldType": "boolean",
+                        "onlyStatic": true
+                    },
+                    "zoomable": {
+                        "type": "static",
+                        "value": false,
+                        "fieldType": "boolean",
+                        "onlyStatic": true
+                    }
+                },
+                "componentInput": {
+                    "type": "static",
+                    "value": [
+                        {
+                            "data": [
+                                {
+                                    "x": 25,
+                                    "y": 50
+                                },
+                                {
+                                    "x": 23,
+                                    "y": 23
+                                },
+                                {
+                                    "x": 12,
+                                    "y": 37
+                                }
+                            ],
+                            "label": "foo",
+                            "backgroundColor": "rgb(255, 12, 137)"
+                        },
+                        {
+                            "data": [
+                                {
+                                    "x": 32,
+                                    "y": 32
+                                },
+                                {
+                                    "x": 25,
+                                    "y": 42
+                                },
+                                {
+                                    "x": 3,
+                                    "y": 27
+                                }
+                            ],
+                            "label": "foobar",
+                            "backgroundColor": "orange"
+                        }
+                    ],
+                    "fieldType": "array",
+                    "subFieldType": "object"
+                }
+            }
+        },
+        {
+            "3": {
+                "h": 8,
+                "w": 2,
+                "x": 0,
+                "y": 20,
+                "max": {
+                    "h": 100,
+                    "w": 3
+                },
+                "min": {
+                    "h": 1,
+                    "w": 1
+                },
+                "fixed": false,
+                "draggable": true,
+                "resizable": true,
+                "customDragger": false,
+                "customResizer": false
+            },
+            "12": {
+                "h": 8,
+                "w": 6,
+                "x": 6,
+                "y": 10,
+                "id": "d",
+                "max": {
+                    "h": 100,
+                    "w": 12
+                },
+                "min": {
+                    "h": 1,
+                    "w": 1
+                },
+                "fixed": false,
+                "draggable": true,
+                "resizable": true,
+                "customDragger": false,
+                "customResizer": false
+            },
+            "id": "d",
+            "data": {
+                "id": "d",
+                "card": false,
+                "type": "displaycomponent",
+                "configuration": {},
+                "componentInput": {
+                    "type": "connected",
+                    "value": {
+                        "foo": 42
+                    },
+                    "fields": {
+                        "x": {
+                            "type": "connected",
+                            "value": "gooo",
+                            "visible": true,
+                            "fieldType": "string",
+                            "connection": {
+                                "path": "result",
+                                "componentId": "e"
+                            }
+                        }
+                    },
+                    "runnable": {
+                        "name": "Inline Script 1",
+                        "type": "runnableByName",
+                        "inlineScript": {
+                            "path": "/inline-script/Inline Script 1",
+                            "schema": {
+                                "type": "object",
+                                "$schema": "https://json-schema.org/draft/2020-12/schema",
+                                "required": [
+                                    "x"
+                                ],
+                                "properties": {
+                                    "x": {
+                                        "type": "string",
+                                        "default": null,
+                                        "description": ""
+                                    }
+                                }
+                            },
+                            "content": "// import * as wmill from \"https://deno.land/x/windmill@v1.57.1/mod.ts\"\n\nexport async function main(x: string) {\n  return x\n}\n",
+                            "language": "deno"
+                        }
+                    },
+                    "fieldType": "object",
+                    "connection": {
+                        "path": "result",
+                        "componentId": "e"
+                    }
+                }
+            }
+        },
+        {
+            "3": {
+                "h": 1,
+                "w": 2,
+                "x": 1,
+                "y": 0,
+                "max": {
+                    "h": 100,
+                    "w": 3
+                },
+                "min": {
+                    "h": 1,
+                    "w": 1
+                },
+                "fixed": false,
+                "draggable": true,
+                "resizable": true,
+                "customDragger": false,
+                "customResizer": false
+            },
+            "12": {
+                "h": 1,
+                "w": 3,
+                "x": 9,
+                "y": 0,
+                "id": "e",
+                "max": {
+                    "h": 100,
+                    "w": 12
+                },
+                "min": {
+                    "h": 1,
+                    "w": 1
+                },
+                "fixed": false,
+                "draggable": true,
+                "resizable": true,
+                "customDragger": false,
+                "customResizer": false
+            },
+            "id": "e",
+            "data": {
+                "id": "e",
+                "card": false,
+                "type": "textinputcomponent",
+                "softWrap": true,
+                "configuration": {
+                    "placeholder": {
+                        "type": "static",
+                        "value": "Type...",
+                        "fieldType": "text",
+                        "onlyStatic": true
+                    }
+                },
+                "verticalAlignment": "center"
+            }
+        },
+        {
+            "3": {
+                "h": 1,
+                "w": 2,
+                "x": 1,
+                "y": 1,
+                "max": {
+                    "h": 100,
+                    "w": 3
+                },
+                "min": {
+                    "h": 1,
+                    "w": 1
+                },
+                "fixed": false,
+                "draggable": true,
+                "resizable": true,
+                "customDragger": false,
+                "customResizer": false
+            },
+            "12": {
+                "h": 1,
+                "w": 3,
+                "x": 9,
+                "y": 1,
+                "max": {
+                    "h": 100,
+                    "w": 12
+                },
+                "min": {
+                    "h": 1,
+                    "w": 1
+                },
+                "fixed": false,
+                "draggable": true,
+                "resizable": true,
+                "customDragger": false,
+                "customResizer": false
+            },
+            "id": "f",
+            "data": {
+                "id": "f",
+                "card": false,
+                "type": "passwordinputcomponent",
+                "softWrap": true,
+                "configuration": {
+                    "placeholder": {
+                        "type": "static",
+                        "value": "Password",
+                        "fieldType": "text",
+                        "onlyStatic": true
+                    }
+                },
+                "verticalAlignment": "center"
+            }
+        },
+        {
+            "3": {
+                "h": 1,
+                "w": 2,
+                "x": 0,
+                "y": 28,
+                "max": {
+                    "h": 100,
+                    "w": 3
+                },
+                "min": {
+                    "h": 1,
+                    "w": 1
+                },
+                "fixed": false,
+                "draggable": true,
+                "resizable": true,
+                "customDragger": false,
+                "customResizer": false
+            },
+            "12": {
+                "h": 1,
+                "w": 3,
+                "x": 9,
+                "y": 2,
+                "max": {
+                    "h": 100,
+                    "w": 12
+                },
+                "min": {
+                    "h": 1,
+                    "w": 1
+                },
+                "fixed": false,
+                "draggable": true,
+                "resizable": true,
+                "customDragger": false,
+                "customResizer": false
+            },
+            "id": "g",
+            "data": {
+                "id": "g",
+                "card": false,
+                "type": "dateinputcomponent",
+                "softWrap": true,
+                "configuration": {
+                    "maxDate": {
+                        "type": "static",
+                        "value": "",
+                        "fieldType": "date"
+                    },
+                    "minDate": {
+                        "type": "static",
+                        "value": "",
+                        "fieldType": "date"
+                    }
+                },
+                "verticalAlignment": "center"
+            }
+        },
+        {
+            "3": {
+                "h": 1,
+                "w": 1,
+                "x": 2,
+                "y": 12,
+                "max": {
+                    "h": 100,
+                    "w": 3
+                },
+                "min": {
+                    "h": 1,
+                    "w": 1
+                },
+                "fixed": false,
+                "draggable": true,
+                "resizable": true,
+                "customDragger": false,
+                "customResizer": false
+            },
+            "12": {
+                "h": 1,
+                "w": 2,
+                "x": 9,
+                "y": 3,
+                "max": {
+                    "h": 100,
+                    "w": 12
+                },
+                "min": {
+                    "h": 1,
+                    "w": 1
+                },
+                "fixed": false,
+                "draggable": true,
+                "resizable": true,
+                "customDragger": false,
+                "customResizer": false
+            },
+            "id": "h",
+            "data": {
+                "id": "h",
+                "card": false,
+                "type": "checkboxcomponent",
+                "softWrap": true,
+                "configuration": {
+                    "label": {
+                        "type": "static",
+                        "value": "Label",
+                        "fieldType": "text"
+                    }
+                },
+                "verticalAlignment": "center",
+                "horizontalAlignment": "center"
+            }
+        },
+        {
+            "3": {
+                "h": 1,
+                "w": 2,
+                "x": 0,
+                "y": 29,
+                "max": {
+                    "h": 100,
+                    "w": 3
+                },
+                "min": {
+                    "h": 1,
+                    "w": 1
+                },
+                "fixed": false,
+                "draggable": true,
+                "resizable": true,
+                "customDragger": false,
+                "customResizer": false
+            },
+            "12": {
+                "h": 1,
+                "w": 3,
+                "x": 9,
+                "y": 4,
+                "max": {
+                    "h": 100,
+                    "w": 12
+                },
+                "min": {
+                    "h": 1,
+                    "w": 1
+                },
+                "fixed": false,
+                "draggable": true,
+                "resizable": true,
+                "customDragger": false,
+                "customResizer": false
+            },
+            "id": "i",
+            "data": {
+                "id": "i",
+                "card": false,
+                "type": "selectcomponent",
+                "softWrap": true,
+                "configuration": {
+                    "items": {
+                        "type": "static",
+                        "value": [
+                            {
+                                "label": "Foo",
+                                "value": "foo"
+                            },
+                            {
+                                "label": "Bar",
+                                "value": "bar"
+                            }
+                        ],
+                        "fieldType": "array",
+                        "subFieldType": "object"
+                    },
+                    "itemKey": {
+                        "type": "static",
+                        "value": "value",
+                        "fieldType": "text"
+                    }
+                },
+                "verticalAlignment": "center"
+            }
+        },
+        {
+            "3": {
+                "h": 1,
+                "w": 1,
+                "x": 2,
+                "y": 13,
+                "max": {
+                    "h": 100,
+                    "w": 3
+                },
+                "min": {
+                    "h": 1,
+                    "w": 1
+                },
+                "fixed": false,
+                "draggable": true,
+                "resizable": true,
+                "customDragger": false,
+                "customResizer": false
+            },
+            "12": {
+                "h": 1,
+                "w": 2,
+                "x": 0,
+                "y": 5,
+                "max": {
+                    "h": 100,
+                    "w": 12
+                },
+                "min": {
+                    "h": 1,
+                    "w": 1
+                },
+                "fixed": false,
+                "draggable": true,
+                "resizable": true,
+                "customDragger": false,
+                "customResizer": false
+            },
+            "id": "j",
+            "data": {
+                "id": "j",
+                "card": false,
+                "type": "buttoncomponent",
+                "softWrap": true,
+                "configuration": {
+                    "size": {
+                        "type": "static",
+                        "value": "xs",
+                        "fieldType": "select",
+                        "onlyStatic": true,
+                        "optionValuesKey": "buttonSizeOptions"
+                    },
+                    "color": {
+                        "type": "static",
+                        "value": "blue",
+                        "fieldType": "select",
+                        "onlyStatic": true,
+                        "optionValuesKey": "buttonColorOptions"
+                    },
+                    "label": {
+                        "type": "static",
+                        "value": "Press me",
+                        "fieldType": "text"
+                    }
+                },
+                "componentInput": {
+                    "type": "runnable",
+                    "fields": {},
+                    "fieldType": "any"
+                },
+                "verticalAlignment": "center",
+                "horizontalAlignment": "center"
+            }
+        },
+        {
+            "3": {
+                "h": 5,
+                "w": 3,
+                "x": 0,
+                "y": 30,
+                "max": {
+                    "h": 100,
+                    "w": 3
+                },
+                "min": {
+                    "h": 1,
+                    "w": 1
+                },
+                "fixed": false,
+                "draggable": true,
+                "resizable": true,
+                "customDragger": false,
+                "customResizer": false
+            },
+            "12": {
+                "h": 5,
+                "w": 6,
+                "x": 0,
+                "y": 18,
+                "id": "k",
+                "max": {
+                    "h": 100,
+                    "w": 12
+                },
+                "min": {
+                    "h": 1,
+                    "w": 1
+                },
+                "fixed": false,
+                "draggable": true,
+                "resizable": true,
+                "customDragger": false,
+                "customResizer": false
+            },
+            "id": "k",
+            "data": {
+                "id": "k",
+                "card": true,
+                "type": "formcomponent",
+                "configuration": {
+                    "size": {
+                        "type": "static",
+                        "value": "xs",
+                        "fieldType": "select",
+                        "onlyStatic": true,
+                        "optionValuesKey": "buttonSizeOptions"
+                    },
+                    "color": {
+                        "type": "static",
+                        "value": "dark",
+                        "fieldType": "select",
+                        "onlyStatic": true,
+                        "optionValuesKey": "buttonColorOptions"
+                    },
+                    "label": {
+                        "type": "static",
+                        "value": "Submit",
+                        "fieldType": "text"
+                    }
+                },
+                "componentInput": {
+                    "type": "runnable",
+                    "fields": {
+                        "x": {
+                            "type": "user",
+                            "value": null,
+                            "visible": true,
+                            "fieldType": "string"
+                        },
+                        "y": {
+                            "type": "user",
+                            "value": null,
+                            "visible": true,
+                            "fieldType": "string"
+                        }
+                    },
+                    "runnable": {
+                        "name": "Inline Script 0",
+                        "type": "runnableByName",
+                        "inlineScript": {
+                            "path": "/inline-script/Inline Script 0",
+                            "schema": {
+                                "type": "object",
+                                "$schema": "https://json-schema.org/draft/2020-12/schema",
+                                "required": [
+                                    "x",
+                                    "y"
+                                ],
+                                "properties": {
+                                    "x": {
+                                        "type": "string",
+                                        "default": null,
+                                        "description": ""
+                                    },
+                                    "y": {
+                                        "type": "string",
+                                        "default": null,
+                                        "description": ""
+                                    }
+                                }
+                            },
+                            "content": "// import * as wmill from \"https://deno.land/x/windmill@v1.57.1/mod.ts\"\n\nexport async function main(x: string, y: string) {\n  return x\n}\n",
+                            "language": "deno"
+                        }
+                    },
+                    "fieldType": "any"
+                },
+                "horizontalAlignment": "center"
+            }
+        }
+    ],
+    "fullscreen": false,
+    "unusedInlineScripts": []
+})
+    </script>
+  </body>
+</html>

frontend/package.json

@@ -1,9 +1,10 @@
 {
 	"name": "windmill",
-	"version": "1.60.0",
+	"version": "1.62.0",
 	"scripts": {
 		"dev": "vite dev",
 		"build": "vite build",
+		"build-embed": "vite build -c vite.embed.config.js",
 		"preview": "vite preview",
 		"check": "svelte-check --tsconfig ./tsconfig.json --threshold warning",
 		"check:watch": "svelte-check --tsconfig ./tsconfig.json --watch",
@@ -16,9 +17,14 @@
 	},
 	"devDependencies": {
 		"@playwright/test": "^1.29.2",
+		"@rollup/plugin-commonjs": "^24.0.1",
+		"@rollup/plugin-html": "^1.0.2",
+		"@rollup/plugin-node-resolve": "^15.0.1",
+		"@rollup/plugin-terser": "^0.4.0",
+		"@rollup/plugin-typescript": "^11.0.0",
 		"@sveltejs/adapter-static": "^1.0.0",
 		"@sveltejs/kit": "^1.0.0-next.589",
-		"@sveltejs/package": "^1.0.1",
+		"@sveltejs/package": "^1.0.2",
 		"@tailwindcss/forms": "^0.5.3",
 		"@tailwindcss/typography": "^0.5.8",
 		"@types/node": "^18.11.18",
@@ -34,15 +40,17 @@
 		"eslint-config-prettier": "^8.6.0",
 		"eslint-plugin-svelte3": "^4.0.0",
 		"lucide-svelte": "^0.104.0",
+		"monaco-editor-workers": "0.33.0",
 		"openapi-typescript-codegen": "^0.23.0",
 		"path-browserify": "^1.0.1",
 		"postcss": "^8.4.18",
 		"postcss-load-config": "^4.0.1",
 		"prettier": "^2.8.3",
 		"prettier-plugin-svelte": "^2.9.0",
+		"rollup-plugin-svelte": "^7.1.0",
 		"simple-svelte-autocomplete": "^2.5.1",
 		"stylelint-config-recommended": "^9.0.0",
-		"svelte": "^3.53.1",
+		"svelte": "^3.55.1",
 		"svelte-awesome": "^3.0.0",
 		"svelte-check": "^3.0.2",
 		"svelte-dnd-action": "^0.9.22",
@@ -51,13 +59,13 @@
 		"svelte-highlight": "^6.2.1",
 		"svelte-overlay": "^1.4.1",
 		"svelte-popperjs": "^1.3.2",
-		"svelte-preprocess": "^5.0.0",
+		"svelte-preprocess": "^5.0.1",
 		"svelte-splitpanes": "^0.7.3",
 		"svelte2tsx": "^0.5.22",
 		"tailwindcss": "^3.2.2",
 		"tslib": "^2.4.1",
 		"typescript": "^4.9.3",
-		"vite": "^4.0.3"
+		"vite": "^4.0.4"
 	},
 	"type": "module",
 	"dependencies": {
@@ -67,6 +75,7 @@
 		"@redocly/json-to-json-schema": "^0.0.1",
 		"@tanstack/svelte-table": "^8.7.6",
 		"async-mutex": "^0.4.0",
+		"chart.js": "3.8.2",
 		"chartjs-adapter-date-fns": "^3.0.0",
 		"chartjs-plugin-zoom": "^2.0.0",
 		"date-fns": "^2.29.3",
@@ -83,5 +92,6 @@
 	},
 	"peerDependencies": {
 		"svelte": "^3.53.1"
-	}
+	},
+	"main": "index.js"
 }

frontend/src/embed.js

@@ -0,0 +1,9 @@
+import AppPreview from './lib/components/apps/editor/AppPreview.svelte'
+
+globalThis.process = { env: { NODE_ENV: 'production' } }
+globalThis.loadApp = (target, value) => {
+	new AppPreview({
+		target: target,
+		props: { app: value }
+	})
+}

frontend/src/lib/common.ts

@@ -19,7 +19,7 @@ export interface SchemaProperty {
 }
 
 export type Schema = {
-	$schema: string
+	$schema: string | undefined
 	type: string
 	properties: { [name: string]: SchemaProperty }
 	required: string[]

frontend/src/lib/components/ApiConnectForm.svelte

@@ -4,6 +4,7 @@
 	import { emptySchema, emptyString } from '$lib/utils'
 	import SchemaForm from './SchemaForm.svelte'
 	import SimpleEditor from './SimpleEditor.svelte'
+	import Toggle from './Toggle.svelte'
 
 	export let resource_type: string
 	export let args: Record<string, any> | any = {}
@@ -12,7 +13,10 @@
 
 	let schema = emptySchema()
 	let notFound = false
+
 	async function loadSchema() {
+		rawCode = '{}'
+		viewJsonSchema = false
 		try {
 			const rt = await ResourceService.getResourceType({
 				workspace: $workspaceStore!,
@@ -21,8 +25,6 @@
 			schema = rt.schema
 			notFound = false
 		} catch (e) {
-			rawCode = '{}'
-
 			notFound = true
 		}
 	}
@@ -43,17 +45,40 @@
 	}
 	let error = ''
 	let rawCode = ''
+	let viewJsonSchema = false
+
+	$: rawCode && parseJson()
+
+	function switchTab(asJson: boolean) {
+		viewJsonSchema = asJson
+		if (asJson) {
+			rawCode = JSON.stringify(args, null, 2)
+		} else {
+			parseJson()
+		}
+	}
 </script>
 
-{#if notFound}
+{#if !notFound}
+	<div class="w-full flex flex-row-reverse">
+		<Toggle
+			on:change={(e) => switchTab(e.detail)}
+			options={{
+				right: 'As JSON'
+			}}
+		/>
+	</div>
+{:else}
 	<p class="italic text-gray-500 text-xs mb-4"
 		>No corresponding resource type found in your workspace for {resource_type}. Define the value in
 		JSON directly</p
 	>
+{/if}
+{#if notFound || viewJsonSchema}
 	{#if !emptyString(error)}<span class="text-red-400 text-xs mb-1 flex flex-row-reverse"
 			>{error}</span
 		>{:else}<div class="py-2" />{/if}
 	<SimpleEditor autoHeight lang="json" bind:code={rawCode} fixedOverflowWidgets={false} />
 {:else}
-	<SchemaForm {password} isValid {schema} bind:args />
+	<SchemaForm noDelete {password} isValid {schema} bind:args />
 {/if}

frontend/src/lib/components/AppConnect.svelte

@@ -155,6 +155,7 @@
 	async function next() {
 		if (step == 1 && manual) {
 			step += 1
+			args = {}
 		} else if (step == 1 && !manual) {
 			const url = new URL(`/api/oauth/connect/${resource_type}`, $page.url.origin)
 			url.searchParams.append('scopes', scopes.join('+'))
@@ -410,15 +411,17 @@
 					</ol>
 				</div>
 				{#if apiTokenApps[resource_type].img}
-					<div class="mt-4 w-full">
-						<img class="max-h-96 m-auto" alt="connect" src={apiTokenApps[resource_type].img} />
+					<div class="mt-4 w-full overflow-hidden">
+						<img class="m-auto  max-h-60" alt="connect" src={apiTokenApps[resource_type].img} />
 					</div>
 				{/if}
 			{/if}
 
 			<h2 class="mt-4">Value</h2>
 			<div class="mt-4">
-				<ApiConnectForm password={key ?? ''} {resource_type} bind:args bind:isValid />
+				{#key resource_type}
+					<ApiConnectForm password={key ?? ''} {resource_type} bind:args bind:isValid />
+				{/key}
 			</div>
 
 			<h2 class="mt-4 mb-2">Description</h2>

frontend/src/lib/components/ArgInfo.svelte

@@ -1,8 +1,6 @@
 <script lang="ts">
 	import { truncate } from '$lib/utils'
 	import Tooltip from './Tooltip.svelte'
-	import json from 'svelte-highlight/languages/json'
-	import { Highlight } from 'svelte-highlight'
 	import { ResourceService } from '$lib/gen'
 	import { workspaceStore } from '$lib/stores'
 	import Drawer from './common/drawer/Drawer.svelte'

frontend/src/lib/components/ArgInput.svelte

@@ -166,7 +166,7 @@
 			<FieldHeader {label} {required} {type} {contentEncoding} {format} {itemsType} />
 		{/if}
 		{#if editableSchema}
-			<div class="my-1 text-xs border-solid border border-gray-400 rounded p-2">
+			<div class="p-2 my-1 text-xs border-solid border border-gray-400">
 				<span
 					class="underline"
 					on:click={() => {

frontend/src/lib/components/CliHelpBox.svelte

@@ -0,0 +1,51 @@
+<script lang="ts">
+	import { page } from '$app/stores'
+	import { userWorkspaces, workspaceStore } from '$lib/stores'
+	import { faChevronDown, faChevronUp } from '@fortawesome/free-solid-svg-icons'
+	import Icon from 'svelte-awesome'
+	import { slide } from 'svelte/transition'
+	import InlineCodeCopy from './InlineCodeCopy.svelte'
+
+	$: opened = false
+	$: workspace = $userWorkspaces.find((e) => e.id === $workspaceStore)
+	$: workspaceName = workspace?.name
+	$: workspaceId = workspace?.id
+	$: url = `${$page.url.protocol}//${$page.url.hostname}/`
+</script>
+
+<div class="text-sm mt-2 flex">
+	<!-- svelte-ignore a11y-click-events-have-key-events -->
+	<span
+		class="underline mr-4"
+		on:click={() => {
+			opened = !opened
+		}}
+	>
+		CLI quick setup
+		<Icon class="ml-2" data={opened ? faChevronUp : faChevronDown} scale={0.7} />
+	</span>
+</div>
+
+{#if opened}
+	<div
+		transition:slide|local
+		class="bg-gray-100 border-l-4 border-gray-600 text-gray-700 p-4 m-4"
+		role="alert"
+		id="dynamic-input-help-box"
+	>
+		<ul class="pl-0 list-decimal list-inside">
+			<li
+				>Install the latest wmill CLI from deno.land: <InlineCodeCopy
+					content={'deno install --unstable -A https://deno.land/x/wmill/main.ts'}
+				/></li
+			>
+			<li
+				>Setup the wmill cli for this workspace & remote: <InlineCodeCopy
+					content={`wmill workspace add ${workspaceName} ${workspaceId} ${url}`}
+				/></li
+			>
+			<li>Follow the prompts in your terminal</li>
+			<li>Use the run command above!</li>
+		</ul>
+	</div>
+{/if}

frontend/src/lib/components/DisplayResult.svelte

@@ -7,21 +7,26 @@
 	import autosize from 'svelte-autosize'
 
 	export let result: any
+	export let requireHtmlApproval = false
 
 	let resultKind:
 		| 'json'
 		| 'table-col'
 		| 'table-row'
+		| 'html'
 		| 'png'
 		| 'file'
 		| 'jpeg'
 		| 'gif'
 		| 'error'
 		| 'approval'
+		| 'svg'
 		| undefined
+
 	$: resultKind = inferResultKind(result)
 
 	let forceJson = false
+	let enableHtml = false
 
 	function isRectangularArray(obj: any) {
 		if (!Array.isArray(obj) || obj.length == 0) {
@@ -53,8 +58,12 @@
 					return 'table-row'
 				} else if (keys.map((k) => Array.isArray(result[k])).reduce((a, b) => a && b)) {
 					return 'table-col'
+				} else if (keys.length == 1 && keys[0] == 'html') {
+					return 'html'
 				} else if (keys.length == 1 && keys[0] == 'png') {
 					return 'png'
+				} else if (keys.length == 1 && keys[0] == 'svg') {
+					return 'svg'
 				} else if (keys.length == 1 && keys[0] == 'jpeg') {
 					return 'jpeg'
 				} else if (keys.length == 1 && keys[0] == 'file') {
@@ -119,6 +128,34 @@
 					</tbody>
 				</TableCustom>
 			</div>
+		{:else if !forceJson && resultKind == 'html'}
+			<div class="h-full">
+				{#if !requireHtmlApproval || enableHtml}
+					{@html result.html}
+				{:else}
+					<div class="font-main text-sm">
+						<div class="flex flex-col">
+							<div class="bg-red-400 py-1 rounded-t text-white font-bold text-center">
+								Warning
+							</div>
+							<p
+								class="text-gray-600 mb-2 text-left border-2 !border-t-0 rounded-b border-red-400 overflow-auto p-1"
+								>Rendering HTML can expose you to <a
+									href="https://owasp.org/www-community/attacks/xss/"
+									target="_blank"
+									rel="noreferrer"
+									class="hover:underline">XSS attacks</a
+								>. Only enable it if you trust the author of the script.
+							</p>
+						</div>
+						<div class="center-center">
+							<Button size="sm" color="dark" on:click={() => (enableHtml = true)}>
+								Enable HTML rendering
+							</Button>
+						</div>
+					</div>
+				{/if}
+			</div>
 		{:else if !forceJson && resultKind == 'png'}
 			<div class="h-full"
 				><img alt="png rendered" class="w-auto h-full" src="data:image/png;base64,{result.png}" />
@@ -131,6 +168,11 @@
 					src="data:image/jpeg;base64,{result.jpeg}"
 				/>
 			</div>
+		{:else if !forceJson && resultKind == 'svg'}
+			<div
+				><a download="windmill.svg" href="data:text/plain;base64,{btoa(result.svg)}">Download</a>
+			</div>
+			<div class="h-full overflow-auto">{@html result.svg} </div>
 		{:else if !forceJson && resultKind == 'gif'}
 			<div class="h-full"
 				><img alt="gif rendered" class="w-auto h-full" src="data:image/gif;base64,{result.gif}" />

frontend/src/lib/components/Editor.svelte

@@ -19,7 +19,6 @@
 	import editorWorker from 'monaco-editor/esm/vs/editor/editor.worker?worker'
 	import tsWorker from 'monaco-editor/esm/vs/language/typescript/ts.worker?worker'
 
-	import { buildWorkerDefinition } from 'monaco-editor-workers'
 	import type {
 		Disposable,
 		DocumentUri,
@@ -90,13 +89,7 @@
 	const uri = `file:///tmp/monaco/${hash}.${langToExt(lang)}`
 
 	if (browser) {
-		if (dev) {
-			buildWorkerDefinition(
-				'../../../node_modules/monaco-editor-workers/dist/workers',
-				import.meta.url,
-				false
-			)
-		} else {
+		if (!dev) {
 			// @ts-ignore
 			self.MonacoEnvironment = {
 				getWorker: function (_moduleId: any, label: string) {
@@ -298,8 +291,9 @@
 			}
 		}
 
+		const wsProtocol = $page.url.protocol == 'https:' ? 'wss' : 'ws'
 		if (lang == 'typescript') {
-			await connectToLanguageServer(`wss://${$page.url.host}/ws/deno`, 'deno', {
+			await connectToLanguageServer(`${wsProtocol}://${$page.url.host}/ws/deno`, 'deno', {
 				certificateStores: null,
 				enablePaths: [],
 				config: null,
@@ -330,7 +324,7 @@
 				}
 			})
 		} else if (lang === 'python') {
-			await connectToLanguageServer(`wss://${$page.url.host}/ws/pyright`, 'pyright', {
+			await connectToLanguageServer(`${wsProtocol}://${$page.url.host}/ws/pyright`, 'pyright', {
 				executionEnvironments: [
 					{
 						root: '/tmp/pyright',
@@ -341,7 +335,7 @@
 				]
 			})
 
-			connectToLanguageServer(`wss://${$page.url.host}/ws/black`, 'black', {
+			connectToLanguageServer(`${wsProtocol}://${$page.url.host}/ws/black`, 'black', {
 				formatters: {
 					black: {
 						command: 'black',
@@ -353,7 +347,7 @@
 				}
 			})
 		} else if (lang === 'go') {
-			connectToLanguageServer(`wss://${$page.url.host}/ws/go`, 'go', {
+			connectToLanguageServer(`${wsProtocol}://${$page.url.host}/ws/go`, 'go', {
 				'build.allowImplicitNetworkAccess': true
 			})
 		}
@@ -407,6 +401,14 @@
 	}
 
 	async function loadMonaco() {
+		if (dev) {
+			let { buildWorkerDefinition } = await import('monaco-editor-workers')
+			buildWorkerDefinition(
+				'../../../node_modules/monaco-editor-workers/dist/workers',
+				import.meta.url,
+				false
+			)
+		}
 		const model = meditor.createModel(code, lang, mUri.parse(uri))
 
 		model.updateOptions(updateOptions)

frontend/src/lib/components/EditorBar.svelte

@@ -29,6 +29,8 @@
 	import PickHubScript from './flows/pickers/PickHubScript.svelte'
 	import ToggleHubWorkspace from './ToggleHubWorkspace.svelte'
 	import Skeleton from './common/skeleton/Skeleton.svelte'
+	import Popover from './Popover.svelte'
+	import Kbd from './common/kbd/Kbd.svelte'
 
 	export let lang: 'python3' | 'deno' | 'go' | 'bash'
 	export let editor: Editor
@@ -225,121 +227,137 @@
 <ResourceEditor bind:this={resourceEditor} on:refresh={resourcePicker.openDrawer} />
 <VariableEditor bind:this={variableEditor} on:create={variablePicker.openDrawer} />
 
-<div class="flex flex-row justify-between items-center overflow-hidden w-full px-1">
-	<div class="flex flex-row divide-x items-center">
-		<div class="mx-2">
-			{#if validCode}
-				<Badge color="green">Inputs</Badge>
-			{:else}
-				<Badge color="red">Inputs</Badge>
-			{/if}
-		</div>
-		<div>
-			<Button
-				color="light"
-				btnClasses="mr-1 !font-medium"
-				on:click={contextualVariablePicker.openDrawer}
-				size="xs"
-				spacingSize="md"
-				startIcon={{ icon: faDollarSign }}
-				{iconOnly}
-			>
-				+Context Var
-			</Button>
-		</div>
-		<div>
-			<Button
-				color="light"
-				btnClasses="mx-1 !font-medium"
-				on:click={variablePicker.openDrawer}
-				size="xs"
-				spacingSize="md"
-				startIcon={{ icon: faDollarSign }}
-				{iconOnly}
-			>
-				+Variable
-			</Button>
-		</div>
-		<div>
-			<Button
-				btnClasses="mx-1 !font-medium"
-				size="xs"
-				spacingSize="md"
-				color="light"
-				on:click={resourcePicker.openDrawer}
-				{iconOnly}
-				startIcon={{ icon: faCube }}
-			>
-				+Resource
-			</Button>
-		</div>
-		<div>
-			<Button
-				btnClasses="mx-1 !font-medium"
-				size="xs"
-				spacingSize="md"
-				color="light"
-				on:click={editor.clearContent}
-				{iconOnly}
-				startIcon={{ icon: faRotateLeft }}
-			>
-				Reset
-			</Button>
-		</div>
-		<div>
-			<Button
-				btnClasses="!font-medium"
-				size="xs"
-				spacingSize="md"
-				color="light"
-				on:click={editor.reloadWebsocket}
-				startIcon={{ icon: faRotate }}
-			>
-				{#if !iconOnly}
-					Assistant
-				{/if}
-				<span class="ml-1 -my-1">
-					{#if lang == 'deno'}
-						(<span class={websocketAlive.deno ? 'green' : 'text-red-700'}>Deno</span>)
-					{:else if lang == 'go'}
-						(<span class={websocketAlive.go ? 'green' : 'text-red-700'}>Go</span>)
-					{:else if lang == 'python3'}
-						(<span class={websocketAlive.pyright ? 'green' : 'text-red-700'}>Pyright</span>
-						<span class={websocketAlive.black ? 'green' : 'text-red-700'}>Black</span>)
+<div class="flex justify-between items-center overflow-y-auto w-full p-1">
+	<div class="flex items-center">
+		<Badge color={validCode ? 'green' : 'red'} class="min-w-[60px] mr-3">
+			{validCode ? 'Valid' : 'Invalid'}
+		</Badge>
+		<div class="flex items-center divide-x">
+			<Popover notClickable placement="bottom" disapperTimoout={0} class="pr-1" disablePopup={!iconOnly}>
+				<Button
+					color="light"
+					btnClasses="!font-medium !h-full"
+					on:click={contextualVariablePicker.openDrawer}
+					size="xs"
+					spacingSize="md"
+					startIcon={{ icon: faDollarSign }}
+					{iconOnly}
+				>
+					+Context Var
+				</Button>
+				<svelte:fragment slot="text">
+					Add context variable
+				</svelte:fragment>
+			</Popover>
+			<Popover notClickable placement="bottom" disapperTimoout={0} class="px-1" disablePopup={!iconOnly}>
+				<Button
+					color="light"
+					btnClasses="!font-medium !h-full"
+					on:click={variablePicker.openDrawer}
+					size="xs"
+					spacingSize="md"
+					startIcon={{ icon: faDollarSign }}
+					{iconOnly}
+				>
+					+Variable
+				</Button>
+				<svelte:fragment slot="text">
+					Add variable
+				</svelte:fragment>
+			</Popover>
+			<Popover notClickable placement="bottom" disapperTimoout={0} class="px-1" disablePopup={!iconOnly}>
+				<Button
+					btnClasses="!font-medium !h-full"
+					size="xs"
+					spacingSize="md"
+					color="light"
+					on:click={resourcePicker.openDrawer}
+					{iconOnly}
+					startIcon={{ icon: faCube }}
+				>
+					+Resource
+				</Button>
+				<svelte:fragment slot="text">
+					Add resource
+				</svelte:fragment>
+			</Popover>
+			<Popover notClickable placement="bottom" disapperTimoout={0} class="px-1" disablePopup={!iconOnly}>
+				<Button
+					btnClasses="!font-medium !h-full"
+					size="xs"
+					spacingSize="md"
+					color="light"
+					on:click={editor.clearContent}
+					{iconOnly}
+					startIcon={{ icon: faRotateLeft }}
+				>
+					Reset
+				</Button>
+				<svelte:fragment slot="text">
+					Reset
+				</svelte:fragment>
+			</Popover>
+			<Popover notClickable placement="bottom" disapperTimoout={0} class="px-1" disablePopup={!iconOnly}>
+				<Button
+					btnClasses="!font-medium !h-full"
+					size="xs"
+					spacingSize="md"
+					color="light"
+					on:click={editor.reloadWebsocket}
+					startIcon={{ icon: faRotate }}
+				>
+					{#if !iconOnly}
+						Assistant
 					{/if}
-				</span>
-			</Button></div
-		>
-		<div>
-			<Button
-				btnClasses="!font-medium"
-				size="xs"
-				spacingSize="md"
-				color="light"
-				on:click={editor.format}
-				startIcon={{ icon: faBroom }}
-			>
-				{#if !iconOnly}
+					<span class="ml-1 -my-1">
+						{#if lang == 'deno'}
+							(<span class={websocketAlive.deno ? 'green' : 'text-red-700'}>Deno</span>)
+						{:else if lang == 'go'}
+							(<span class={websocketAlive.go ? 'green' : 'text-red-700'}>Go</span>)
+						{:else if lang == 'python3'}
+							(<span class={websocketAlive.pyright ? 'green' : 'text-red-700'}>Pyright</span>
+							<span class={websocketAlive.black ? 'green' : 'text-red-700'}>Black</span>)
+						{/if}
+					</span>
+				</Button>
+				<svelte:fragment slot="text">
+					Reload assistant
+				</svelte:fragment>
+			</Popover>
+			<Popover notClickable placement="bottom" disapperTimoout={0} class="px-1" disablePopup={!iconOnly}>
+				<Button
+					btnClasses="!font-medium"
+					size="xs"
+					spacingSize="md"
+					color="light"
+					on:click={editor.format}
+					{iconOnly}
+					startIcon={{ icon: faBroom }}
+				>
 					Format (Ctrl+S)
-				{/if}
-			</Button></div
-		>
-	</div>
-	<div class="py-1">
-		<div>
-			<Button
-				btnClasses="mx-1 !font-medium"
-				size="xs"
-				spacingSize="md"
-				color="light"
-				on:click={scriptPicker.openDrawer}
-				{iconOnly}
-				startIcon={{ icon: faEye }}
-			>
-				Script
-			</Button>
+				</Button>
+				<svelte:fragment slot="text">
+					Format <Kbd class="!text-gray-800">Ctrl</Kbd> + <Kbd class="!text-gray-800">S</Kbd>
+				</svelte:fragment>
+			</Popover>
 		</div>
 	</div>
+	<Popover notClickable placement="bottom" disapperTimoout={0} class="px-1" disablePopup={!iconOnly}>
+		<Button
+			btnClasses="!font-medium"
+			size="xs"
+			spacingSize="md"
+			color="light"
+			on:click={scriptPicker.openDrawer}
+			{iconOnly}
+			startIcon={{ icon: faEye }}
+		>
+			Script
+		</Button>
+		<svelte:fragment slot="text">
+			Script
+		</svelte:fragment>
+	</Popover>
 </div>
 
 <style>

frontend/src/lib/components/FlowBuilder.svelte

@@ -112,8 +112,8 @@
 				await createSchedule(flow.path)
 			}
 		}
-		sendUserToast(`Success! flow saved at ${$flowStore.path}`)
-		goto(`/flows/get/${$flowStore.path}`)
+		sendUserToast(`Flow saved at ${$flowStore.path}`)
+		goto(`/flows/get/${$flowStore.path}?workspace_id=${$workspaceStore}`)
 	}
 
 	let timeout: NodeJS.Timeout | undefined = undefined

frontend/src/lib/components/FlowMetadata.svelte

@@ -2,14 +2,15 @@
 	import type { Job } from '$lib/gen'
 	import JobStatus from '$lib/components/JobStatus.svelte'
 	import Icon from 'svelte-awesome'
-	import { displayDaysAgo } from '$lib/utils'
+	import { displayDate, displayDaysAgo } from '$lib/utils'
 	import {
 		faCalendar,
 		faClock,
 		faRobot,
 		faScroll,
 		faUser,
-		faBarsStaggered
+		faBarsStaggered,
+		faMemory
 	} from '@fortawesome/free-solid-svg-icons'
 	import ScheduleEditor from './ScheduleEditor.svelte'
 	import { onDestroy, onMount } from 'svelte'
@@ -39,16 +40,31 @@
 		<Icon class="text-gray-700" data={faClock} scale={SMALL_ICON_SCALE} /><span
 			class="mx-2 text-2xs text-gray-600"
 		>
-			{#key time}
-				Received job {displayDaysAgo(job.created_at ?? '')}
-			{/key}</span
+			{#if job['success'] != undefined}
+				Received job: {displayDate(job.created_at ?? '')}
+			{:else}
+				{#key time}
+					Received job {displayDaysAgo(job.created_at ?? '')}
+				{/key}
+			{/if}</span
 		>
 	</div>
 	{#if job && 'started_at' in job && job.started_at}
 		<div>
 			<Icon class="text-gray-700" data={faClock} scale={SMALL_ICON_SCALE} /><span class="mx-2">
-				{#key time}
-					Started {displayDaysAgo(job.started_at ?? '')}{/key}</span
+				{#if job['success'] != undefined}
+					Started: {displayDate(job.started_at ?? '')}
+				{:else}
+					{#key time}
+						Started {displayDaysAgo(job.started_at ?? '')}{/key}
+				{/if}
+			</span>
+		</div>
+	{/if}
+	{#if job && job['mem_peak']}
+		<div>
+			<Icon class="text-gray-700" data={faMemory} scale={SMALL_ICON_SCALE} /><span class="mx-2">
+				Mem peak: {(job['mem_peak'] / 1024).toPrecision(5)}MB</span
 			>
 		</div>
 	{/if}

frontend/src/lib/components/HighlightCode.svelte

@@ -26,4 +26,10 @@
 	$: lang = getLang(language)
 </script>
 
-<Highlight className="nowrap {$$props.class}" language={lang} {code} />
+{#if code?.length < 5000}
+	<Highlight className="nowrap {$$props.class}" language={lang} {code} />
+{:else}
+	<pre class="overflow-auto max-h-screen {$$props.class}"
+		><code class="language-{language}">{code}</code></pre
+	>
+{/if}

frontend/src/lib/components/InlineCodeCopy.svelte

@@ -0,0 +1,13 @@
+<script lang="ts">
+	import { copyToClipboard } from '$lib/utils'
+	import { faClipboard } from '@fortawesome/free-solid-svg-icons'
+	import { Icon } from 'svelte-awesome'
+
+	export let content: string
+</script>
+
+<pre class="bg-gray-700 text-gray-100 p-2  font-mono text-sm whitespace-pre-wrap"
+	>{content} <span on:click={() => copyToClipboard(content)} class="cursor-pointer ml-2"
+		><Icon data={faClipboard} /></span
+	></pre
+>

frontend/src/lib/components/InputTransformForm.svelte

@@ -8,7 +8,6 @@
 	import DynamicInputHelpBox from './flows/content/DynamicInputHelpBox.svelte'
 	import type { PropPickerWrapperContext } from './flows/propPicker/PropPickerWrapper.svelte'
 	import { codeToStaticTemplate, getDefaultExpr, isCodeInjection } from './flows/utils'
-	import SimpleEditor from './SimpleEditor.svelte'
 	import { Button, ToggleButton, ToggleButtonGroup } from './common'
 	import { faCode } from '@fortawesome/free-solid-svg-icons'
 	import type VariableEditor from './VariableEditor.svelte'
@@ -17,6 +16,7 @@
 	import TemplateEditor from './TemplateEditor.svelte'
 	import Tooltip from './Tooltip.svelte'
 	import { setInputCat as computeInputCat } from '$lib/utils'
+	import JavascriptEditor from './JavascriptEditor.svelte'
 
 	export let schema: Schema
 	export let arg: InputTransform | any
@@ -29,7 +29,7 @@
 	export let itemPicker: ItemPicker | undefined = undefined
 	export let noDynamicToggle = false
 
-	let monaco: SimpleEditor | undefined = undefined
+	let monaco: JavascriptEditor | undefined = undefined
 	let monacoTemplate: TemplateEditor | undefined = undefined
 	let argInput: ArgInput | undefined = undefined
 
@@ -60,7 +60,11 @@
 		}
 
 		if (isCodeInjection(rawValue)) {
-			arg.expr = getDefaultExpr(argName, previousModuleId, `\`${rawValue}\``)
+			arg.expr = getDefaultExpr(
+				argName,
+				previousModuleId,
+				`\`${rawValue.toString().replaceAll('`', '\\`')}\``
+			)
 			arg.type = 'javascript'
 			propertyType = 'static'
 		} else {
@@ -117,9 +121,13 @@
 	const openBracket = '${'
 	const closeBracket = '}'
 
-	$: schema.properties[argName].default &&
-		!arg?.value &&
-		monacoTemplate?.setCode(schema.properties[argName].default)
+	function setDefaultCode() {
+		if (!arg?.value) {
+			monacoTemplate?.setCode(schema.properties[argName].default)
+		}
+	}
+
+	$: schema.properties[argName].default && setDefaultCode()
 </script>
 
 {#if arg != undefined}
@@ -157,7 +165,7 @@
 									argName,
 									previousModuleId,
 									staticTemplate
-										? `\`${arg?.value ?? ''}\``
+										? `\`${arg?.value.toString().replaceAll('`', '\\`') ?? ''}\``
 										: arg.value
 										? JSON.stringify(arg?.value, null, 4)
 										: ''
@@ -172,6 +180,7 @@
 							if (staticTemplate) {
 								if (arg) {
 									arg.value = codeToStaticTemplate(arg.expr)
+									arg.expr = undefined
 								}
 								setPropertyType(arg?.value)
 							} else {
@@ -275,7 +284,7 @@
 			/>
 		{:else if arg.expr != undefined}
 			<div class="border rounded mt-2 border-gray-300">
-				<SimpleEditor
+				<JavascriptEditor
 					bind:this={monaco}
 					bind:code={arg.expr}
 					{extraLib}

frontend/src/lib/components/JavascriptEditor.svelte

@@ -0,0 +1,212 @@
+<script lang="ts">
+	import { browser, dev } from '$app/environment'
+
+	import editorWorker from 'monaco-editor/esm/vs/editor/editor.worker?worker'
+
+	import tsWorker from 'monaco-editor/esm/vs/language/typescript/ts.worker?worker'
+
+	import { createEventDispatcher, onDestroy, onMount } from 'svelte'
+	import { createHash, editorConfig, langToExt, updateOptions } from '$lib/editorUtils'
+	import { languages, editor as meditor, KeyCode, KeyMod, Uri as mUri } from 'monaco-editor'
+
+	import libStdContent from '$lib/es5.d.ts.txt?raw'
+
+	meditor.defineTheme('myTheme', {
+		base: 'vs',
+		inherit: true,
+		rules: [],
+		colors: {
+			'editorLineNumber.foreground': '#999',
+			'editorGutter.background': '#F9FAFB'
+		}
+	})
+	meditor.setTheme('myTheme')
+
+	languages.typescript.javascriptDefaults.setCompilerOptions({
+		target: languages.typescript.ScriptTarget.Latest,
+		allowNonTsExtensions: true,
+		noLib: true
+	})
+
+	let divEl: HTMLDivElement | null = null
+	let editor: meditor.IStandaloneCodeEditor
+	let model: meditor.ITextModel
+
+	export let lang: string
+	export let code: string = ''
+	export let hash: string = createHash()
+	export let cmdEnterAction: (() => void) | undefined = undefined
+	export let formatAction: (() => void) | undefined = undefined
+	export let automaticLayout = true
+	export let extraLib: string = ''
+	export let shouldBindKey: boolean = true
+	export let autoHeight = false
+	export let fixedOverflowWidgets = true
+
+	const dispatch = createEventDispatcher()
+
+	const uri = `file:///${hash}.${langToExt(lang)}`
+
+	if (browser) {
+		if (!dev) {
+			// @ts-ignore
+			self.MonacoEnvironment = {
+				getWorker: function (_moduleId: any, label: string) {
+					if (label === 'typescript' || label === 'javascript') {
+						return new tsWorker()
+					} else {
+						return new editorWorker()
+					}
+				}
+			}
+		}
+	}
+
+	export function getCode(): string {
+		return editor?.getValue() ?? ''
+	}
+
+	export function insertAtCursor(code: string): void {
+		if (editor) {
+			editor.trigger('keyboard', 'type', { text: code })
+		}
+	}
+
+	export function setCode(ncode: string): void {
+		code = ncode
+		if (editor) {
+			editor.setValue(ncode)
+		}
+	}
+
+	function format() {
+		if (editor) {
+			code = getCode()
+			editor.getAction('editor.action.formatDocument').run()
+			if (formatAction) {
+				formatAction()
+				code = getCode()
+			}
+		}
+	}
+
+	let width = 0
+	async function loadMonaco() {
+		if (dev) {
+			let { buildWorkerDefinition } = await import('monaco-editor-workers')
+			buildWorkerDefinition(
+				'../../../node_modules/monaco-editor-workers/dist/workers',
+				import.meta.url,
+				false
+			)
+		}
+		model = meditor.createModel(code, lang, mUri.parse(uri))
+
+		model.updateOptions(updateOptions)
+
+		editor = meditor.create(
+			divEl as HTMLDivElement,
+			editorConfig(model, code, lang, automaticLayout, fixedOverflowWidgets)
+		)
+
+		let timeoutModel: NodeJS.Timeout | undefined = undefined
+		editor.onDidChangeModelContent((event) => {
+			timeoutModel && clearTimeout(timeoutModel)
+			timeoutModel = setTimeout(() => {
+				code = getCode()
+				dispatch('change', { code })
+			}, 200)
+		})
+
+		editor.onDidFocusEditorText(() => {
+			dispatch('focus')
+
+			editor.addCommand(KeyMod.CtrlCmd | KeyCode.KeyS, function () {
+				code = getCode()
+				shouldBindKey && format && format()
+			})
+		})
+
+		if (autoHeight) {
+			let ignoreEvent = false
+			const updateHeight = () => {
+				const contentHeight = Math.min(1000, editor.getContentHeight())
+				if (divEl) {
+					divEl.style.height = `${contentHeight}px`
+				}
+				try {
+					ignoreEvent = true
+					editor.layout({ width, height: contentHeight })
+				} finally {
+					ignoreEvent = false
+				}
+			}
+			editor.onDidContentSizeChange(updateHeight)
+			updateHeight()
+		}
+
+		editor.onDidFocusEditorText(() => {
+			editor.addCommand(KeyMod.CtrlCmd | KeyCode.KeyS, function () {
+				code = getCode()
+				shouldBindKey && format && format()
+			})
+
+			editor.addCommand(KeyMod.CtrlCmd | KeyCode.Enter, function () {
+				code = getCode()
+				shouldBindKey && cmdEnterAction && cmdEnterAction()
+			})
+			dispatch('focus')
+		})
+
+		editor.onDidBlurEditorText(() => {
+			code = getCode()
+			dispatch('blur')
+		})
+
+		if (lang == 'javascript') {
+			const stdLib = { content: libStdContent, filePath: 'es5.d.ts' }
+			if (extraLib != '') {
+				languages.typescript.javascriptDefaults.setExtraLibs([
+					{
+						content: extraLib,
+						filePath: 'windmill.d.ts'
+					},
+					stdLib
+				])
+			} else {
+				languages.typescript.javascriptDefaults.setExtraLibs([stdLib])
+			}
+		}
+	}
+
+	onMount(() => {
+		if (browser) {
+			loadMonaco()
+		}
+	})
+
+	onDestroy(() => {
+		try {
+			model && model.dispose()
+			editor && editor.dispose()
+		} catch (err) {}
+	})
+</script>
+
+<div bind:this={divEl} class="{$$props.class} editor" bind:clientWidth={width} />
+
+<style>
+	.editor {
+		@apply rounded-lg p-0;
+	}
+
+	.small-editor {
+		/* stylelint-disable-next-line unit-allowed-list */
+		height: 26vh;
+	}
+
+	.few-lines-editor {
+		/* stylelint-disable-next-line unit-allowed-list */
+		height: 100px;
+	}
+</style>

frontend/src/lib/components/LogViewer.svelte

@@ -5,6 +5,7 @@
 	export let content: string | undefined
 	export let isLoading: boolean
 	export let duration: number | undefined = undefined
+	export let mem: number | undefined = undefined
 
 	let scroll = true
 	let div: HTMLElement | null = null
@@ -50,6 +51,11 @@
 		{:else if duration}
 			<span class="absolute text-xs text-gray-500 top-2 left-2">took {duration}ms</span>
 		{/if}
+		{#if mem}
+			<span class="absolute text-xs text-gray-500 top-2 left-36"
+				>mem peak: {(mem / 1024).toPrecision(4)}MB</span
+			>
+		{/if}
 		<pre class="whitespace-pre-wrap break-words bg-gray-50 text-xs w-full p-2"
 			>{#if content}<span>{content}</span>{:else if !isLoading}<span class="text-gray-600"
 					>No logs are available yet</span

frontend/src/lib/components/ObjectResourceInput.svelte

@@ -34,7 +34,11 @@
 	}
 
 	function resourceToValue() {
-		value = `$res:${path}`
+		if (path) {
+			value = `$res:${path}`
+		} else {
+			value = undefined
+		}
 	}
 
 	function isResource() {

frontend/src/lib/components/Popover.svelte

@@ -5,6 +5,7 @@
 	export let placement: PopoverPlacement = 'auto'
 	export let notClickable = false
 	export let popupClass = ''
+	export let disablePopup = false
 	export let disapperTimoout = 100
 
 	const [popperRef, popperContent] = createPopperActions({ placement })
@@ -20,7 +21,12 @@
 	const extraOpts = {
 		modifiers: [
 			betterPreventOverflow({ padding: 10 }),
-			{ name: 'offset', options: { offset: [8, 8] } },
+			{
+				name: 'offset',
+				options: {
+					offset: [8, 8]
+				}
+			},
 			{
 				name: 'arrow',
 				options: {
@@ -62,7 +68,7 @@
 		<slot />
 	</button>
 {/if}
-{#if showTooltip}
+{#if showTooltip && !disablePopup}
 	<div
 		use:popperContent={extraOpts}
 		on:mouseenter={open}

frontend/src/lib/components/ResourceEditor.svelte

@@ -13,6 +13,7 @@
 	import SimpleEditor from './SimpleEditor.svelte'
 	import { faSave } from '@fortawesome/free-solid-svg-icons'
 	import SchemaForm from './SchemaForm.svelte'
+	import Toggle from './Toggle.svelte'
 
 	let path = ''
 	let initialPath = ''
@@ -38,6 +39,7 @@
 		path = p
 		resourceToEdit = undefined
 		resourceSchema = undefined
+		viewJsonSchema = false
 		loadingSchema = true
 		drawer.openDrawer?.()
 		resourceToEdit = await ResourceService.getResource({ workspace: $workspaceStore!, path: p })
@@ -111,6 +113,16 @@
 			args[k] = `$var:${path}`
 		})
 	}
+	let viewJsonSchema = false
+
+	function switchTab(asJson: boolean) {
+		viewJsonSchema = asJson
+		if (asJson) {
+			rawCode = JSON.stringify(args, null, 2)
+		} else {
+			parseJson()
+		}
+	}
 </script>
 
 <Drawer bind:this={drawer} size="800px">
@@ -136,12 +148,21 @@
 						kind="resource"
 					/>
 				</div>
-				<h3 class="mt-4">Value</h3>
+				<div class="flex w-full justify-between max-w-lg items-center mt-4">
+					<h3>Value</h3>
+					<Toggle
+						on:change={(e) => switchTab(e.detail)}
+						options={{
+							right: 'As JSON'
+						}}
+					/>
+				</div>
 				<div class="text-sm">
 					{#if loadingSchema}
 						<Skeleton layout={[[4]]} />
-					{:else if resourceSchema && resourceSchema?.properties}
+					{:else if !viewJsonSchema && resourceSchema && resourceSchema?.properties}
 						<SchemaForm
+							noDelete
 							disabled={!can_write}
 							compact
 							schema={resourceSchema}
@@ -151,10 +172,13 @@
 					{:else if !can_write}
 						<input type="text" disabled value={rawCode} />
 					{:else}
-						<p class="italic text-gray-500 text-xs mb-4"
-							>No corresponding resource type found in your workspace for {selectedResourceType}.
-							Define the value in JSON directly</p
-						>
+						{#if !viewJsonSchema}
+							<p class="italic text-gray-500 text-xs mb-4"
+								>No corresponding resource type found in your workspace for {selectedResourceType}.
+								Define the value in JSON directly</p
+							>
+						{/if}
+
 						{#if !emptyString(jsonError)}<span
 								class="text-red-400 text-xs mb-1 flex flex-row-reverse">{jsonError}</span
 							>{:else}<div class="py-2" />{/if}

frontend/src/lib/components/RunForm.svelte

@@ -18,6 +18,8 @@
 	import Toggle from './Toggle.svelte'
 	import { userStore } from '$lib/stores'
 	import Tooltip from './Tooltip.svelte'
+	import CliHelpBox from './CliHelpBox.svelte'
+	import InlineCodeCopy from './InlineCodeCopy.svelte'
 
 	export let runnable:
 		| {
@@ -55,10 +57,15 @@
 
 	export let isValid = true
 
-	// Run later
 	let viewOptions = false
+	let viewCliOptions = false
+
 	let scheduledForStr: string | undefined
 	let invisible_to_owner: false
+
+	$: cliCommand = `wmill ${runnable?.kind} run ${runnable?.path} ${Object.entries(args)
+		.map(([k, v]) => `-i ${k}=${JSON.stringify(v)}`)
+		.join(' ')}`
 </script>
 
 <div class="max-w-6xl">
@@ -171,7 +178,7 @@
 				<div class="flex items-center gap-1">
 					<Toggle
 						options={{
-							right: `run only visible to you`
+							right: `make run invisible to others`
 						}}
 						bind:checked={invisible_to_owner}
 					/>
@@ -200,4 +207,20 @@
 			{buttonText}
 		</Button>
 	{/if}
+
+	<div class="my-10" />
+	<Button
+		color="light"
+		size="sm"
+		endIcon={{ icon: viewCliOptions ? faChevronUp : faChevronDown }}
+		on:click={() => (viewCliOptions = !viewCliOptions)}
+	>
+		Run it from the CLI
+	</Button>
+	{#if viewCliOptions}
+		<div transition:slide class="mt-2 px-4 pt-2">
+			<InlineCodeCopy content={cliCommand} />
+			<CliHelpBox />
+		</div>
+	{/if}
 </div>

frontend/src/lib/components/SchemaForm.svelte

@@ -7,11 +7,9 @@
 	import { slide } from 'svelte/transition'
 	import ArgInput from './ArgInput.svelte'
 	import { Button } from './common'
-	import InputTransformForm from './InputTransformForm.svelte'
 	import ItemPicker from './ItemPicker.svelte'
 	import VariableEditor from './VariableEditor.svelte'
 
-	export let inputTransform = false
 	export let schema: Schema
 	export let args: Record<string, InputTransform | any> = {}
 	export let disabledArgs: string[] = []
@@ -19,17 +17,15 @@
 
 	export let editableSchema = false
 	export let isValid: boolean = true
-	export let extraLib: string = 'missing extraLib'
 	export let autofocus = false
-	export let previousModuleId: string | undefined = undefined
 
 	export let shouldHideNoInputs: boolean = false
 	export let compact = false
 	export let password: string | undefined = undefined
 	export let noVariablePicker = false
 	export let filter: string[] | undefined = undefined
-	export let noDynamicToggle = false
 	export let flexWrap = false
+	export let noDelete = false
 
 	let clazz: string = ''
 	export { clazz as class }
@@ -46,7 +42,6 @@
 			if (!keys.includes(key)) {
 				delete args[key]
 				delete inputCheck[key]
-				console.log('DELETED', key)
 			}
 		})
 	}
@@ -59,7 +54,9 @@
 		let lkeys = Object.keys(schema?.properties ?? {})
 		if (schema?.properties && JSON.stringify(lkeys) != JSON.stringify(keys)) {
 			keys = lkeys
-			removeExtraKey()
+			if (!noDelete) {
+				removeExtraKey()
+			}
 		}
 	}
 </script>
@@ -69,20 +66,7 @@
 		{#each keys as argName, i (argName)}
 			{#if !filter || filter.includes(argName)}
 				<div transition:slide|local>
-					{#if inputTransform}
-						<InputTransformForm
-							{previousModuleId}
-							bind:arg={args[argName]}
-							bind:schema
-							bind:argName
-							bind:inputCheck={inputCheck[argName]}
-							bind:extraLib
-							{variableEditor}
-							{itemPicker}
-							bind:pickForField
-							{noDynamicToggle}
-						/>
-					{:else if typeof args == 'object'}
+					{#if typeof args == 'object'}
 						<ArgInput
 							autofocus={i == 0 && autofocus}
 							label={argName}
@@ -123,11 +107,7 @@
 		bind:this={itemPicker}
 		pickCallback={(path, _) => {
 			if (pickForField) {
-				if (inputTransform) {
-					args[pickForField].value = '$var:' + path
-				} else {
-					args[pickForField] = '$var:' + path
-				}
+				args[pickForField] = '$var:' + path
 			}
 		}}
 		itemName="Variable"

frontend/src/lib/components/SchemaFormTransform.svelte

@@ -0,0 +1,120 @@
+<script lang="ts">
+	import type { Schema } from '$lib/common'
+	import { VariableService, type InputTransform } from '$lib/gen'
+	import { workspaceStore } from '$lib/stores'
+	import { allTrue } from '$lib/utils'
+	import { faPlus } from '@fortawesome/free-solid-svg-icons'
+	import { slide } from 'svelte/transition'
+	import { Button } from './common'
+	import InputTransformForm from './InputTransformForm.svelte'
+	import ItemPicker from './ItemPicker.svelte'
+	import VariableEditor from './VariableEditor.svelte'
+
+	export let schema: Schema
+	export let args: Record<string, InputTransform | any> = {}
+	export let isValid: boolean = true
+	export let extraLib: string = 'missing extraLib'
+	export let previousModuleId: string | undefined = undefined
+
+	export let shouldHideNoInputs: boolean = false
+	export let noVariablePicker = false
+	export let filter: string[] | undefined = undefined
+	export let noDynamicToggle = false
+	export let flexWrap = false
+	export let noDelete = false
+
+	let clazz: string = ''
+	export { clazz as class }
+
+	let inputCheck: { [id: string]: boolean } = {}
+	$: isValid = allTrue(inputCheck) ?? false
+
+	$: if (args == undefined || typeof args !== 'object') {
+		args = {}
+	}
+
+	function removeExtraKey() {
+		Object.keys(args ?? {}).forEach((key) => {
+			if (!keys.includes(key)) {
+				delete args[key]
+				delete inputCheck[key]
+			}
+		})
+	}
+
+	let pickForField: string | undefined
+	let itemPicker: ItemPicker | undefined = undefined
+	let variableEditor: VariableEditor | undefined = undefined
+	let keys: string[] = []
+	$: {
+		let lkeys = Object.keys(schema?.properties ?? {})
+		if (schema?.properties && JSON.stringify(lkeys) != JSON.stringify(keys)) {
+			keys = lkeys
+			if (!noDelete) {
+				removeExtraKey()
+			}
+		}
+	}
+</script>
+
+<div class="w-full {clazz} {flexWrap ? 'flex flex-row flex-wrap gap-x-6 gap-y-2' : ''}">
+	{#if keys.length > 0}
+		{#each keys as argName, i (argName)}
+			{#if !filter || filter.includes(argName)}
+				<div transition:slide|local>
+					<InputTransformForm
+						{previousModuleId}
+						bind:arg={args[argName]}
+						bind:schema
+						bind:argName
+						bind:inputCheck={inputCheck[argName]}
+						bind:extraLib
+						{variableEditor}
+						{itemPicker}
+						bind:pickForField
+						{noDynamicToggle}
+					/>
+				</div>
+			{/if}
+		{/each}
+	{:else if !shouldHideNoInputs}
+		<div class="text-gray-500 text-sm">No inputs</div>
+	{/if}
+</div>
+
+{#if !noVariablePicker}
+	<ItemPicker
+		bind:this={itemPicker}
+		pickCallback={(path, _) => {
+			if (pickForField) {
+				args[pickForField].value = '$var:' + path
+			}
+		}}
+		itemName="Variable"
+		extraField="path"
+		loadItems={async () =>
+			(await VariableService.listVariable({ workspace: $workspaceStore ?? '' })).map((x) => ({
+				name: x.path,
+				...x
+			}))}
+	>
+		<div
+			slot="submission"
+			class="flex flex-row-reverse w-full bg-white border-t border-gray-200 rounded-bl-lg rounded-br-lg"
+		>
+			<Button
+				variant="border"
+				color="blue"
+				size="sm"
+				startIcon={{ icon: faPlus }}
+				on:click={() => {
+					variableEditor?.initNew?.()
+				}}
+			>
+				New variable
+			</Button>
+		</div>
+	</ItemPicker>
+
+	<VariableEditor bind:this={variableEditor} />
+{/if}

frontend/src/lib/components/ScriptBuilder.svelte

@@ -14,7 +14,7 @@
 	import CenteredPage from './CenteredPage.svelte'
 	import UnsavedConfirmationModal from './common/confirmationModal/UnsavedConfirmationModal.svelte'
 	import { dirtyStore } from './common/confirmationModal/dirtyStore'
-	import { Button } from './common'
+	import { Button, Kbd } from './common'
 	import { faChevronDown, faChevronUp, faPen, faSave } from '@fortawesome/free-solid-svg-icons'
 	import Breadcrumb from './common/breadcrumb/Breadcrumb.svelte'
 	import Toggle from './Toggle.svelte'
@@ -83,8 +83,9 @@
 					kind: script.kind
 				}
 			})
-			sendUserToast(`Success! New script version created with hash ${newHash}`)
-			goto(`/scripts/get/${newHash}`)
+			sendUserToast(`New script created at hash ${newHash}`)
+			await goto(`/scripts/edit/${newHash}?step=2`)
+			goto(`/scripts/get/${newHash}?workspace_id=${$workspaceStore}`)
 		} catch (error) {
 			sendUserToast(`Impossible to save the script: ${error.body}`, true)
 		}
@@ -103,9 +104,18 @@
 		}
 		goto(`?step=${step}`)
 	}
+
 	$: kind = script.kind as 'script' | 'trigger' | 'approval' | undefined
+
+	function onKeyDown(event: KeyboardEvent) {
+		if (event.key == 'Enter' && step == 1) {
+			changeStep(2)
+		}
+	}
 </script>
 
+<svelte:window on:keydown={onKeyDown} />
+
 {#if !$userStore?.operator}
 	<UnsavedConfirmationModal />
 	<div class="flex flex-col h-screen">
@@ -167,11 +177,11 @@
 					<Button
 						size="sm"
 						variant={step == 1 ? 'contained' : 'border'}
-						btnClasses={step == 3 ? 'invisible' : ''}
+						btnClasses={step == 3 ? 'invisible' : 'inline-flex gap-2'}
 						disabled={step === 1 && pathError !== ''}
 						on:click={() => changeStep(step + 1)}
 					>
-						Next
+						Next {#if step == 1}<Kbd>Enter</Kbd>{/if}
 					</Button>
 					<Button
 						size="sm"
@@ -188,7 +198,7 @@
 		<!-- metadata -->
 		{#if step === 1}
 			<CenteredPage>
-				<h2 class="border-b pb-1 mt-4 mb-2">Path</h2>
+				<h2 class="border-b pb-1 mt-8 mb-2">Path</h2>
 				<Path
 					bind:this={pathC}
 					bind:error={pathError}
@@ -198,14 +208,14 @@
 					namePlaceholder="my_script"
 					kind="script"
 				/>
-				<h2 class="border-b pb-1 mt-8 mb-4">Summary</h2>
+				<h2 class="border-b pb-1 mt-12 mb-4">Summary</h2>
 				<input
 					type="text"
 					bind:this={summaryC}
 					bind:value={script.summary}
 					placeholder="A short summary of the script displayed when the script is listed"
 				/>
-				<h2 class="border-b pb-1 mt-8 mb-6">Language</h2>
+				<h2 class="border-b pb-1 mt-12 mb-6">Language</h2>
 				<div class="flex flex-row gap-2 flex-wrap">
 					{#each langs as [label, lang]}
 						{@const isPicked = script.language == lang && template == 'script'}
@@ -220,7 +230,7 @@
 								initContent(lang, script.kind, template)
 							}}
 						>
-							<LanguageIcon {lang} /><span class="ml-2">{label}</span>
+							<LanguageIcon {lang} /><span class="ml-2 py-4">{label}</span>
 						</Button>
 					{/each}
 					<Button
@@ -234,7 +244,7 @@
 							initContent(script.language, script.kind, template)
 						}}
 					>
-						<LanguageIcon lang="pgsql" /><span class="ml-2">PostgreSQL</span>
+						<LanguageIcon lang="pgsql" /><span class="ml-2 py-4">PostgreSQL</span>
 					</Button>
 					<Button
 						size="sm"
@@ -247,11 +257,10 @@
 							initContent(script.language, script.kind, template)
 						}}
 					>
-						<LanguageIcon lang="mysql" /><span class="ml-2">MySQL</span>
+						<LanguageIcon lang="mysql" /><span class="ml-2 py-4">MySQL</span>
 					</Button>
 				</div>
-				<h2 class="border-b pb-1 mt-8 mb-4">Advanced</h2>
-				<div class="mb-4">
+				<div class="mt-16 mb-4">
 					<Button
 						color="light"
 						size="sm"
@@ -301,12 +310,12 @@
 						/>
 					</div>
 				{/if}
-				<div class="ml-3">
+				<!-- <div class="ml-3">
 					<Toggle
 						bind:checked={script.is_template}
 						options={{ right: 'Save as a workspace template' }}
 					/>
-				</div>
+				</div> -->
 			</CenteredPage>
 		{:else if step === 2}
 			<ScriptEditor

frontend/src/lib/components/SimpleEditor.svelte

@@ -4,15 +4,16 @@
 	import editorWorker from 'monaco-editor/esm/vs/editor/editor.worker?worker'
 	import jsonWorker from 'monaco-editor/esm/vs/language/json/json.worker?worker'
 	import yamlWorker from 'monaco-yaml/yaml.worker?worker'
-	import tsWorker from 'monaco-editor/esm/vs/language/typescript/ts.worker?worker'
-
-	import { buildWorkerDefinition } from 'monaco-editor-workers'
 
 	import { createEventDispatcher, onDestroy, onMount } from 'svelte'
 	import { createHash, editorConfig, langToExt, updateOptions } from '$lib/editorUtils'
-	import { languages, editor as meditor, KeyCode, KeyMod, Uri as mUri } from 'monaco-editor'
-
-	import libStdContent from '$lib/es5.d.ts.txt?raw'
+	import {
+		languages,
+		editor as meditor,
+		KeyCode,
+		KeyMod,
+		Uri as mUri
+	} from 'monaco-editor/esm/vs/editor/editor.main'
 
 	meditor.defineTheme('myTheme', {
 		base: 'vs',
@@ -25,12 +26,6 @@
 	})
 	meditor.setTheme('myTheme')
 
-	languages.typescript.javascriptDefaults.setCompilerOptions({
-		target: languages.typescript.ScriptTarget.Latest,
-		allowNonTsExtensions: true,
-		noLib: true
-	})
-
 	languages.json.jsonDefaults.setDiagnosticsOptions({
 		validate: true,
 		allowComments: false,
@@ -48,7 +43,6 @@
 	export let cmdEnterAction: (() => void) | undefined = undefined
 	export let formatAction: (() => void) | undefined = undefined
 	export let automaticLayout = true
-	export let extraLib: string = ''
 	export let shouldBindKey: boolean = true
 	export let autoHeight = false
 	export let fixedOverflowWidgets = true
@@ -58,13 +52,7 @@
 	const uri = `file:///${hash}.${langToExt(lang)}`
 
 	if (browser) {
-		if (dev) {
-			buildWorkerDefinition(
-				'../../../node_modules/monaco-editor-workers/dist/workers',
-				import.meta.url,
-				false
-			)
-		} else {
+		if (!dev) {
 			// @ts-ignore
 			self.MonacoEnvironment = {
 				getWorker: function (_moduleId: any, label: string) {
@@ -72,8 +60,6 @@
 						return new jsonWorker()
 					} else if (label === 'yaml') {
 						return new yamlWorker()
-					} else if (label === 'typescript' || label === 'javascript') {
-						return new tsWorker()
 					} else {
 						return new editorWorker()
 					}
@@ -174,21 +160,6 @@
 			code = getCode()
 			dispatch('blur')
 		})
-
-		if (lang == 'javascript') {
-			const stdLib = { content: libStdContent, filePath: 'es5.d.ts' }
-			if (extraLib != '') {
-				languages.typescript.javascriptDefaults.setExtraLibs([
-					{
-						content: extraLib,
-						filePath: 'windmill.d.ts'
-					},
-					stdLib
-				])
-			} else {
-				languages.typescript.javascriptDefaults.setExtraLibs([stdLib])
-			}
-		}
 	}
 
 	onMount(() => {

frontend/src/lib/components/StringTypeNarrowing.svelte

@@ -70,7 +70,6 @@
 		['None', 'none'],
 		['File (base64)', 'base64'],
 		['Enum', 'enum'],
-		['Resource Path', 'resource'],
 		['Format', 'format'],
 		['Pattern', 'pattern']
 	]}

frontend/src/lib/components/TestJobLoader.svelte

@@ -109,12 +109,10 @@
 				console.error(err)
 			}
 		}
-		console.debug('cancelled')
 	}
 
 	export async function clearCurrentJob() {
 		if (currentId) {
-			console.debug('clear')
 			job = undefined
 			await cancelJob()
 		}
@@ -148,6 +146,9 @@
 					if (previewJobUpdates.new_logs) {
 						job.logs = (job?.logs ?? '').concat(previewJobUpdates.new_logs)
 					}
+					if (previewJobUpdates.mem_peak) {
+						job.mem_peak = previewJobUpdates.mem_peak
+					}
 					if ((previewJobUpdates.running ?? false) || (previewJobUpdates.completed ?? false)) {
 						job = await JobService.getJob({ workspace: workspace!, id })
 					}
@@ -180,7 +181,6 @@
 
 	async function syncer(id: string): Promise<void> {
 		if (currentId != id) {
-			console.debug('stop')
 			return
 		}
 		syncIteration++

frontend/src/lib/components/Toggle.svelte

@@ -1,5 +1,4 @@
 <script lang="ts">
-	import { classNames } from '$lib/utils'
 	import { createEventDispatcher } from 'svelte'
 
 	export let options: {

frontend/src/lib/components/UserSettings.svelte

@@ -162,7 +162,7 @@
 					>
 						<div>
 							Added token: <button on:click={() => copyToClipboard(newToken ?? '')} class="inline"
-								>{truncate(newToken ?? '', 10, '****')} <Icon data={faClipboard} />
+								>{newToken} <Icon data={faClipboard} />
 							</button>
 						</div>
 						<div class="pt-1 text-xs ml-2">

frontend/src/lib/components/apps/components/AppDisplayComponent.svelte

@@ -1,21 +1,25 @@
 <script lang="ts">
 	import DisplayResult from '$lib/components/DisplayResult.svelte'
+	import { getContext } from 'svelte'
 	import type { AppInput } from '../inputType'
+	import { IS_APP_PUBLIC_CONTEXT_KEY } from '../types'
 	import RunnableWrapper from './helpers/RunnableWrapper.svelte'
 
 	export let id: string
 	export let componentInput: AppInput | undefined
+	export let initializing: boolean | undefined = undefined
 
+	const requireHtmlApproval = getContext<boolean | undefined>(IS_APP_PUBLIC_CONTEXT_KEY)
 	let result: any = undefined
 
 	export const staticOutputs: string[] = ['result', 'loading']
 </script>
 
-<RunnableWrapper flexWrap bind:result bind:componentInput {id}>
+<RunnableWrapper flexWrap bind:componentInput {id} bind:initializing bind:result>
 	<div class="w-full border-b px-2 text-xs p-1 font-semibold bg-gray-500 text-white rounded-t-sm">
 		Results
 	</div>
 	<div class="p-2">
-		<DisplayResult {result} />
+		<DisplayResult {result} {requireHtmlApproval} />
 	</div>
 </RunnableWrapper>

frontend/src/lib/components/apps/components/buttons/AppButton.svelte

@@ -18,6 +18,7 @@
 	export let horizontalAlignment: 'left' | 'center' | 'right' | undefined = undefined
 	export let verticalAlignment: 'top' | 'center' | 'bottom' | undefined = undefined
 	export let noWFull = false
+	export let preclickAction: (() => Promise<void>) | undefined = undefined
 
 	export const staticOutputs: string[] = ['loading', 'result']
 
@@ -28,6 +29,7 @@
 	let size: ButtonType.Size
 	let runnableComponent: RunnableComponent
 	let disabled: boolean | undefined = undefined
+	let fillContainer: boolean | undefined = undefined
 
 	let isLoading: boolean = false
 	let ownClick: boolean = false
@@ -67,6 +69,7 @@
 	bind:value={disabled}
 	bind:error={errors.disabled}
 />
+<InputValue {id} input={configuration.fillContainer} bind:value={fillContainer} />
 
 <RunnableWrapper
 	flexWrap
@@ -81,12 +84,16 @@
 			<div class="text-red-500 text-xs">{errorsMessage}</div>
 		{/if}
 		<Button
+			btnClasses={fillContainer ? 'w-full h-full' : ''}
 			{disabled}
 			on:pointerdown={(e) => {
 				e?.stopPropagation()
 				window.dispatchEvent(new Event('pointerup'))
 			}}
 			on:click={async (e) => {
+				if (preclickAction) {
+					await preclickAction()
+				}
 				e?.stopPropagation()
 				e?.preventDefault()
 				ownClick = true
@@ -102,7 +109,9 @@
 			{color}
 			{loading}
 		>
-			{labelValue}
+			<span class="truncate">
+				{labelValue}
+			</span>
 		</Button>
 	</AlignWrapper>
 </RunnableWrapper>

frontend/src/lib/components/apps/components/dataDisplay/AppBarChart.svelte

@@ -1,6 +1,5 @@
 <script lang="ts">
 	import { Bar, Line } from 'svelte-chartjs'
-
 	import {
 		Chart as ChartJS,
 		Title,
@@ -12,7 +11,6 @@
 		CategoryScale,
 		BarElement
 	} from 'chart.js'
-
 	import RunnableWrapper from '../helpers/RunnableWrapper.svelte'
 	import type { AppInput } from '../../inputType'
 	import InputValue from '../helpers/InputValue.svelte'
@@ -20,6 +18,7 @@
 	export let id: string
 	export let componentInput: AppInput | undefined
 	export let configuration: Record<string, AppInput>
+	export let initializing: boolean | undefined = undefined
 
 	export const staticOutputs: string[] = ['loading', 'result']
 
@@ -71,7 +70,7 @@
 <InputValue {id} input={configuration.theme} bind:value={theme} />
 <InputValue {id} input={configuration.line} bind:value={lineChart} />
 
-<RunnableWrapper flexWrap autoRefresh bind:componentInput {id} bind:result>
+<RunnableWrapper flexWrap autoRefresh bind:componentInput {id} bind:initializing bind:result>
 	{#if result}
 		{#if lineChart}
 			<Line {data} {options} />