chore(main): release 1.104.1 (#1643)

* chore(main): release 1.104.1

* Apply automatic changes

---------

Co-authored-by: rubenfiszel <rubenfiszel@users.noreply.github.com>

.env

@@ -3,5 +3,4 @@ DB_PASSWORD=changeme
 # this is the url that your instance is publicly exposed to
 WM_BASE_URL=http://localhost
 
-# this is the addres caddy will listen on
-CADDY_REVERSE_PROXY=":80"
+# To use another port than :80, setup the Caddyfile and the caddy section of the docker-compose to your needs: https://caddyserver.com/docs/getting-started

.github/uffizzi/caddy/Caddyfile

@@ -1,5 +0,0 @@
-localhost {
-        bind 0.0.0.0
-        reverse_proxy /ws/* http://0.0.0.0:3001
-        reverse_proxy /* http://0.0.0.0:8000
-}

.github/uffizzi/docker-compose.uffizzi.yml

@@ -1,53 +0,0 @@
-version: '3.7'
-
-x-uffizzi:
-  ingress:
-    service: windmill
-    port: 8000
-
-services:
-  db:
-    image: postgres:14
-    environment:
-      POSTGRES_PASSWORD: changeme
-      POSTGRES_DB: windmill
-
-  windmill:
-    image: '${WINDMILL_IMAGE}'
-    ports:
-      - 8000:8000
-    entrypoint: ['/bin/sh', '-c']
-    command: 'echo ${OAUTH_JSON_BASE64} | base64 --decode > /usr/src/app/oauth.json && ./windmill'
-    environment:
-      - DATABASE_URL=postgres://postgres:changeme@localhost/windmill?sslmode=disable
-      - BASE_URL=${EXPECTED_URL}
-      - BASE_INTERNAL_URL=http://localhost:8000
-      - RUST_LOG=info
-      - NUM_WORKERS=3
-      - KEEP_JOB_DIR=false
-      - DENO_PATH=/usr/bin/deno
-      - PYTHON_PATH=/usr/local/bin/python3
-      - METRICS_ADDR=false
-      - OAUTH_JSON_BASE64=${OAUTH_JSON_BASE64}
-    volumes:
-      - worker_dependency_cache:/tmp/windmill/cache
-    deploy:
-      resources:
-        limits:
-          memory: 250M
-
-  lsp:
-    image: '${LSP_IMAGE}'
-    ports:
-      - 3001:3001
-
-  # caddy:
-  #   image: caddy:2.5.2-alpine
-  #   restart: unless-stopped
-  #   volumes:
-  #     - ./.github/uffizzi/caddy:/etc/caddy
-  #   environment:
-  #     - BASE_URL=localhost
-
-volumes:
-  worker_dependency_cache:

.github/workflows/docker-image.yml

@@ -21,7 +21,7 @@ permissions:
 
 jobs:
   build:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
         with:
@@ -64,7 +64,7 @@ jobs:
             org.opencontainers.image.licenses=AGPLv3
 
   build_ee:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
         with:

.github/workflows/pypi_on_release.yml

@@ -10,9 +10,15 @@ on:
       - "v*"
   workflow_dispatch:
 
+permissions:
+  contents: read
+  id-token: write
+  packages: write
+
 jobs:
   publish_pypi:
     runs-on: ubuntu-latest
+    if: startsWith(github.ref, 'refs/tags/v')
     container:
       image: ghcr.io/windmill-labs/python-client-builder
     steps:
@@ -24,26 +30,25 @@ jobs:
           cd python-client
           ./publish.sh
 
+
   publish_lsp:
-    needs: [publish_pypi]
-    runs-on: [self-hosted, new]
+    runs-on: ubuntu-latest
     steps:
-      - name: Sleep for 30 seconds waiting for pypi to update index
-        run: sleep 30s
+      - name: Sleep for 300 seconds waiting for pypi to update index
+        if: startsWith(github.ref, 'refs/tags/v')
+        run: sleep 300
         shell: bash
+        
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-
+      - uses: depot/setup-action@v1
       - name: Docker meta
         id: meta
         uses: docker/metadata-action@v4
         with:
           images: |
             ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
-            ${{ env.ECR_REGISTRY }}/${{ env.IMAGE_NAME }}
           tags: |
             type=ref,event=branch
             type=ref,event=pr
@@ -57,6 +62,30 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Build and push publicly
+        uses: depot/build-push-action@v1
+        with:
+          context: "{{defaultContext}}:lsp"
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: |
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
+            ${{ steps.meta.outputs.tags }}
+          labels: |
+            ${{ steps.meta.outputs.labels }}
+            org.opencontainers.image.licenses=AGPLv3
+
+
+  publish_lsp_private:
+    needs: [publish_lsp]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
       - name: Login to ECR
         uses: docker/login-action@v2
         with:
@@ -64,17 +93,9 @@ jobs:
           username: ${{ secrets.AWS_ACCESS_KEY_ID }}
           password: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
 
-      - name: Build and push publicly
-        uses: docker/build-push-action@v4
-        with:
-          context: "{{defaultContext}}:lsp"
-          push: true
-          tags: |
-            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
-            ${{ env.ECR_REGISTRY }}/${{ env.IMAGE_NAME }}:latest
-            ${{ steps.metalocal.outputs.tags }}
-            ${{ steps.meta.outputs.tags }}
-            registry.uffizzi.com/windmill-lsp:60d
-          labels: ${{ steps.metalocal.outputs.labels }}
-          cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache
-          cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache,mode=max
+
+      - name: Push image to GHCR
+        run: |
+          docker buildx imagetools create \
+            --tag ${{ env.ECR_REGISTRY }}/${{ env.IMAGE_NAME }}:latest \
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest

.github/workflows/uffizzi-build.yml.archived

@@ -1,93 +0,0 @@
-name: Build PR Image
-on:
-  pull_request:
-    types: [opened,synchronize,reopened,closed]
-    paths:
-      - "backend/**"
-      - ".github/uffizzi/**"
-      - ".github/workflows/**"
-  workflow_dispatch:
-      
-jobs:
-  build-windmill:
-    name: Build and Push `windmill`
-    runs-on: ubuntu-latest
-    if: ${{ (github.event_name != 'pull_request' || github.event.action != 'closed')}}
-    outputs:
-      tags: ${{ steps.meta.outputs.tags }}
-    steps:
-      - name: Checkout git repo
-        uses: actions/checkout@v3
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-      - name: Generate UUID image name
-        id: uuid
-        run: echo "UUID_TAG_APP=$(uuidgen)" >> $GITHUB_ENV
-      - name: Docker metadata
-        id: meta
-        uses: docker/metadata-action@v4
-        with:
-          images: registry.uffizzi.com/${{ env.UUID_TAG_APP }}
-          tags: type=raw,value=60d
-      - name: Build and Push Image to registry.uffizzi.com ephemeral registry
-        uses: docker/build-push-action@v4
-        with:
-          push: true
-          context: ./
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-
-
-  render-compose-file:
-    name: Render Docker Compose File
-    # Pass output of this workflow to another triggered by `workflow_run` event.
-    runs-on: ubuntu-latest
-    needs:
-      - build-windmill
-    outputs:
-      compose-file-cache-key: ${{ steps.hash.outputs.hash }}
-    steps:
-      - name: Checkout git repo
-        uses: actions/checkout@v3
-      - name: Render Compose File
-        run: |
-          WINDMILL_IMAGE=${{ needs.build-windmill.outputs.tags }}
-          export WINDMILL_IMAGE
-          LSP_IMAGE=registry.uffizzi.com/windmill-lsp:60d
-          export LSP_IMAGE
-          envsubst '${WINDMILL_IMAGE} ${LSP_IMAGE}' < ./.github/uffizzi/docker-compose.uffizzi.yml > docker-compose.rendered.yml
-          cat docker-compose.rendered.yml
-      - name: Upload Rendered Compose File as Artifact
-        uses: actions/upload-artifact@v3
-        with:
-          name: preview-spec
-          path: docker-compose.rendered.yml
-          retention-days: 2
-      - name: Serialize PR Event to File
-        run:  |
-          cat << EOF > event.json
-          ${{ toJSON(github.event) }}
-          EOF
-      - name: Upload PR Event as Artifact
-        uses: actions/upload-artifact@v3
-        with:
-          name: preview-spec
-          path: event.json
-          retention-days: 2
-
-  delete-preview:
-    name: Call for Preview Deletion
-    runs-on: ubuntu-latest
-    if: ${{ github.event.action == 'closed' }}
-    steps:
-      # If this PR is closing, we will not render a compose file nor pass it to the next workflow.
-      - name: Serialize PR Event to File
-        run: echo '${{ toJSON(github.event) }}' > event.json
-      - name: Upload PR Event as Artifact
-        uses: actions/upload-artifact@v3
-        with:
-          name: preview-spec
-          path: event.json
-          retention-days: 2

.github/workflows/uffizzi-preview.yml.archived

@@ -1,115 +0,0 @@
-name: Deploy Uffizzi Preview
-
-on:
-  workflow_run:
-    workflows:
-      - "Build PR Image"
-    types:
-      - completed
-
-jobs:
-  cache-compose-file:
-    name: Cache Compose File
-    runs-on: ubuntu-latest
-    if: ${{ github.event.workflow_run.conclusion == 'success' }}
-    outputs:
-      compose-file-cache-key: ${{ env.COMPOSE_FILE_HASH }}
-      pr-number: ${{ env.PR_NUMBER }}
-    steps:
-      - name: 'Download artifacts'
-        # Fetch output (zip archive) from the workflow run that triggered this workflow.
-        uses: actions/github-script@v6
-        with:
-          script: |
-            let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({
-               owner: context.repo.owner,
-               repo: context.repo.repo,
-               run_id: context.payload.workflow_run.id,
-            });
-            let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => {
-              return artifact.name == "preview-spec"
-            })[0];
-            if (matchArtifact === undefined) {
-              throw TypeError('Build Artifact not found!');
-            }
-            let download = await github.rest.actions.downloadArtifact({
-               owner: context.repo.owner,
-               repo: context.repo.repo,
-               artifact_id: matchArtifact.id,
-               archive_format: 'zip',
-            });
-            let fs = require('fs');
-            fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/preview-spec.zip`, Buffer.from(download.data));
-      - name: 'Unzip artifact'
-        run: unzip preview-spec.zip
-      - name: Read Event into ENV
-        run: |
-          echo 'EVENT_JSON<<EOF' >> $GITHUB_ENV
-          cat event.json >> $GITHUB_ENV
-          echo 'EOF' >> $GITHUB_ENV
-      - name: Read PR Number From Event Object
-        id: pr
-        run: echo "PR_NUMBER=${{ fromJSON(env.EVENT_JSON).number }}" >> $GITHUB_ENV
-      - name: Predict Deployment URL
-        id: url
-        # Replace dots in the repo name with the plus sign
-        run: |
-          REPO=$(echo ${{ github.repository }} | sed 's/\./+/g')
-          echo "EXPECTED_URL=https://app.uffizzi.com/github.com/$REPO/pull/$PR_NUMBER" >> $GITHUB_ENV
-
-      - name: Re-Render Compose File
-        run: |
-          OAUTH_JSON_BASE64=${{ secrets.OAUTH_JSON_BASE64 }}
-          export OAUTH_JSON_BASE64
-          envsubst '${OAUTH_JSON_BASE64} ${EXPECTED_URL}' < docker-compose.rendered.yml > docker-compose.uffizzi.yml
-          # cat docker-compose.uffizzi.yml
-
-      - name: Hash Rendered Compose File
-        id: hash
-        # If the previous workflow was triggered by a PR close event, we will not have a compose file artifact.
-        if: ${{ fromJSON(env.EVENT_JSON).action != 'closed' }}
-        run: echo "COMPOSE_FILE_HASH=$(md5sum docker-compose.uffizzi.yml | awk '{ print $1 }')" >> $GITHUB_ENV
-      - name: Cache Rendered Compose File
-        if: ${{ fromJSON(env.EVENT_JSON).action != 'closed' }}
-        uses: actions/cache@v3
-        with:
-          path: docker-compose.uffizzi.yml
-          key: ${{ env.COMPOSE_FILE_HASH }}
-
-      - name: DEBUG - Print Job Outputs
-        if: ${{ runner.debug }}
-        run: |
-          echo "PR number: ${{ env.PR_NUMBER }}"
-          echo "Compose file hash: ${{ env.COMPOSE_FILE_HASH }}"
-          cat event.json
-
-  deploy-uffizzi-preview:
-    name: Use Remote Workflow to Preview on Uffizzi
-    needs:
-      - cache-compose-file
-    if: ${{ github.event.workflow_run.conclusion == 'success' }}
-    uses: UffizziCloud/preview-action/.github/workflows/reusable.yaml@v2
-    with:
-      # If this workflow was triggered by a PR close event, cache-key will be an empty string
-      # and this reusable workflow will delete the preview deployment.
-      compose-file-cache-key: ${{ needs.cache-compose-file.outputs.compose-file-cache-key }}
-      compose-file-cache-path: docker-compose.uffizzi.yml
-      server: https://app.uffizzi.com
-      pr-number: ${{ needs.cache-compose-file.outputs.pr-number }}
-    permissions:
-      contents: read
-      pull-requests: write
-      id-token: write
-
-  playwright:
-    runs-on: ubuntu-latest
-    needs:
-      - deploy-uffizzi-preview
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-node@v3
-        with:
-          node-version: 16
-      - name: "Playwright run"
-        timeout-minutes: 2
-        run: cd frontend && npm ci @playwright/test && npx playwright install && export BASE_URL=${{ needs.deploy-uffizzi-preview.outputs.url }} && npm run test

CHANGELOG.md

@@ -1,6 +1,218 @@
 # Changelog
 
 
+## [1.104.1](https://github.com/windmill-labs/windmill/compare/v1.104.0...v1.104.1) (2023-05-24)
+
+
+### Bug Fixes
+
+* **cli:** avoid looping infinitely and avoid prompt if interactive ([97b4403](https://github.com/windmill-labs/windmill/commit/97b4403b7aaae80e4801487d7edfce62ccf116da))
+* **cli:** fix hub pull ([d892ca5](https://github.com/windmill-labs/windmill/commit/d892ca56b7d9fd4f006dfb9f666995d710036422))
+* **cli:** parse schema when pulling from hub ([6851b86](https://github.com/windmill-labs/windmill/commit/6851b86eb5781cc7c652458503be9374f123f53e))
+* **frontend:** Fix app toolbar z-index ([#1641](https://github.com/windmill-labs/windmill/issues/1641)) ([42af285](https://github.com/windmill-labs/windmill/commit/42af2854b28c4149c1def8f7e60c9cb4360a7182))
+
+## [1.104.0](https://github.com/windmill-labs/windmill/compare/v1.103.0...v1.104.0) (2023-05-24)
+
+
+### Features
+
+* schedule error handler ([#1636](https://github.com/windmill-labs/windmill/issues/1636)) ([34048f9](https://github.com/windmill-labs/windmill/commit/34048f9ea655a0afb1983a169b69b454023ec6a8))
+
+
+### Bug Fixes
+
+* **cli:** do not rely on x.nest.land ([ad66bfa](https://github.com/windmill-labs/windmill/commit/ad66bfadaf0c3153975f7452779ac664c0d0dd41))
+* **python:** handle nan ([de4042e](https://github.com/windmill-labs/windmill/commit/de4042e9dcc813d88ef872f694cf6568b087bd1f))
+
+## [1.103.0](https://github.com/windmill-labs/windmill/compare/v1.102.1...v1.103.0) (2023-05-22)
+
+
+### Features
+
+* docker as a new supported language ([b8da43d](https://github.com/windmill-labs/windmill/commit/b8da43db2c31225b0ade8cd9995aeacf2c0eae86))
+* **frontend:** add flowstatus and log component for apps ([11a52f2](https://github.com/windmill-labs/windmill/commit/11a52f2d593a9b233fd138c7af52fc34fa1e6173))
+* **frontend:** add plain chartjs component ([#1621](https://github.com/windmill-labs/windmill/issues/1621)) ([eb99b73](https://github.com/windmill-labs/windmill/commit/eb99b73346a02993fcaeb6df906fcaf663db259d))
+* **frontend:** disable tabs ([#1623](https://github.com/windmill-labs/windmill/issues/1623)) ([5905d3b](https://github.com/windmill-labs/windmill/commit/5905d3b103b0d1466c4d11b248aec9adbe3bfaad))
+
+## [1.102.1](https://github.com/windmill-labs/windmill/compare/v1.102.0...v1.102.1) (2023-05-21)
+
+
+### Bug Fixes
+
+* add setVariable to deno-client ([501bb11](https://github.com/windmill-labs/windmill/commit/501bb11d9676439062be7a96c9f6655c2b609ee1))
+
+## [1.102.0](https://github.com/windmill-labs/windmill/compare/v1.101.1...v1.102.0) (2023-05-19)
+
+
+### Features
+
+* add  ability to pass the full raw body ([#1611](https://github.com/windmill-labs/windmill/issues/1611)) ([b91f7d5](https://github.com/windmill-labs/windmill/commit/b91f7d501390358b01b6656297f56a9f24ef4683))
+* add GOPROXY + fix on saved inputs ([cdc4f29](https://github.com/windmill-labs/windmill/commit/cdc4f29ec1231820cfb2e0882d167b7dae3ae06e))
+* **backend:** add REQUEST_SIZE_LIMIT env variable ([1cbd704](https://github.com/windmill-labs/windmill/commit/1cbd704a257bcf2bd7b344958104e5c626c52a79))
+* **backend:** non mapped values are passed as arg 'body' ([a13d283](https://github.com/windmill-labs/windmill/commit/a13d2832d47d262f0b3ac222a8eb889fb17c75ad))
+* expose a react sdk to integrate windmill into react apps ([#1605](https://github.com/windmill-labs/windmill/issues/1605)) ([632be3b](https://github.com/windmill-labs/windmill/commit/632be3b8fb547ca4a2d976f868ee931218b653b3))
+* **frontend:** add presets components ([#1589](https://github.com/windmill-labs/windmill/issues/1589)) ([f7338c9](https://github.com/windmill-labs/windmill/commit/f7338c9c9a4cfa10d9c22d32a5ae70c4e3504ef3))
+* **lsp:** add  black + ruff + shellcheck ([#1597](https://github.com/windmill-labs/windmill/issues/1597)) ([89e55e0](https://github.com/windmill-labs/windmill/commit/89e55e0226d50951c5c99ce789af80ccaa5c1e25))
+
+
+### Bug Fixes
+
+* **frontend:** Add missing tooltips + multilpe small fix in the app builder ([#1590](https://github.com/windmill-labs/windmill/issues/1590)) ([fff2b5a](https://github.com/windmill-labs/windmill/commit/fff2b5a24abdd70371e2b8a49ff810c217c01bb1))
+* **frontend:** Fix inputValue connection to ensure done event is always sent ([#1607](https://github.com/windmill-labs/windmill/issues/1607)) ([f27abec](https://github.com/windmill-labs/windmill/commit/f27abecbaca4be46715ef15216492cb0984fe32b))
+* **frontend:** fix pdf header + icon picker ([#1586](https://github.com/windmill-labs/windmill/issues/1586)) ([a1cdf13](https://github.com/windmill-labs/windmill/commit/a1cdf13cb33494457e9f3cba24d5c7398565881f))
+* **frontend:** Fix rx ([#1609](https://github.com/windmill-labs/windmill/issues/1609)) ([c687a77](https://github.com/windmill-labs/windmill/commit/c687a775eb8efdb792c495bec72e7e547b82c068))
+* **frontend:** Fix the spinning logo position ([#1595](https://github.com/windmill-labs/windmill/issues/1595)) ([94b8bc4](https://github.com/windmill-labs/windmill/commit/94b8bc47380ea537512042ae412a3ace1ef709e7))
+* rework multiselect as app component ([#1599](https://github.com/windmill-labs/windmill/issues/1599)) ([85576b0](https://github.com/windmill-labs/windmill/commit/85576b00836225656b88c6751fdc619034b1ebca))
+
+## [1.101.1](https://github.com/windmill-labs/windmill/compare/v1.101.0...v1.101.1) (2023-05-16)
+
+
+### Bug Fixes
+
+* **backend:** make result job endpoints public ([41f2d35](https://github.com/windmill-labs/windmill/commit/41f2d35c971c42b9a4842b1411dd21603cabf084))
+* **frontend:** add temp hidden span to compute the text max length ([#1573](https://github.com/windmill-labs/windmill/issues/1573)) ([2a17d60](https://github.com/windmill-labs/windmill/commit/2a17d60caaef11f4b6cce464e1905a52095fe228))
+* **frontend:** fix app multi select ([#1574](https://github.com/windmill-labs/windmill/issues/1574)) ([45acdc8](https://github.com/windmill-labs/windmill/commit/45acdc895b6b5047a17e59dfcd9ca8cba2dd234a))
+
+## [1.101.0](https://github.com/windmill-labs/windmill/compare/v1.100.2...v1.101.0) (2023-05-15)
+
+
+### Features
+
+* **backend:** add job_id as a query arg to force set the new job_id ([b6c0018](https://github.com/windmill-labs/windmill/commit/b6c0018e2acaaed324832dfc715853ea58a4a268))
+* **frontend:** stepper standalone ([#1558](https://github.com/windmill-labs/windmill/issues/1558)) ([ad6e967](https://github.com/windmill-labs/windmill/commit/ad6e967205550b86cc8744f1ce08bb86215ce3e6))
+
+
+### Bug Fixes
+
+* **frontend:** Handle empty required in SchemaForm ([#1571](https://github.com/windmill-labs/windmill/issues/1571)) ([efc4e9c](https://github.com/windmill-labs/windmill/commit/efc4e9ce8a988aacb8e8dda264702dc08d25f7e0))
+
+## [1.100.2](https://github.com/windmill-labs/windmill/compare/v1.100.1...v1.100.2) (2023-05-14)
+
+
+### Bug Fixes
+
+* **cli:** update wmill script push ([678b574](https://github.com/windmill-labs/windmill/commit/678b574efcae66801a115d576db9d00aa9e4145d))
+* discriminate execute apps by component ([908358e](https://github.com/windmill-labs/windmill/commit/908358eb08614d07b5e846630743242b68b9e149))
+* transform_inputs now only support single line expressions ([c252b76](https://github.com/windmill-labs/windmill/commit/c252b765f1b1fd38f07cbe06548ca5cbe4047ea1))
+
+## [1.100.1](https://github.com/windmill-labs/windmill/compare/v1.100.0...v1.100.1) (2023-05-12)
+
+
+### Bug Fixes
+
+* update setup step ([178ed6f](https://github.com/windmill-labs/windmill/commit/178ed6f426020c9966380392088562e27aa77cf3))
+
+## [1.100.0](https://github.com/windmill-labs/windmill/compare/v1.99.0...v1.100.0) (2023-05-12)
+
+
+### Features
+
+* **frontend:** add download button ([9b9730d](https://github.com/windmill-labs/windmill/commit/9b9730d2b7239827fd8dfe8f46b6bd98d535e8d0))
+
+
+### Bug Fixes
+
+* **backend:** handle Date type ([5e7e46e](https://github.com/windmill-labs/windmill/commit/5e7e46e0259bfc11e92f2446858ddbe9f1b4b08e))
+* **frontend:** apps rendering should not depend on local time ([8e785d8](https://github.com/windmill-labs/windmill/commit/8e785d8ba6da16d06816d0379cadfb899be99b06))
+* **frontend:** only download result for apps ([6bbd937](https://github.com/windmill-labs/windmill/commit/6bbd9374cbd2c516dd3b56551103fcfeba01f80f))
+
+## [1.99.0](https://github.com/windmill-labs/windmill/compare/v1.98.0...v1.99.0) (2023-05-10)
+
+
+### Features
+
+* **backend:** run endpoints also support support x-www-form-urlencoded encoded payloads ([2b57418](https://github.com/windmill-labs/windmill/commit/2b57418427e9417599f9f969cb78088c5166958a))
+* **frontend:** add hide refresh button ([ef089ab](https://github.com/windmill-labs/windmill/commit/ef089ab56c5ef493118574076d9512ae3b6a42bf))
+* **frontend:** add input library to flow builder ([957fd81](https://github.com/windmill-labs/windmill/commit/957fd81576dfe65326cad2ed8487121e157e0953))
+* **frontend:** allow copy pasting nested containers ([742ee3a](https://github.com/windmill-labs/windmill/commit/742ee3a5181fdcfba1f59889a8d99347fd0c4610))
+* **frontend:** app on error ([#1556](https://github.com/windmill-labs/windmill/issues/1556)) ([6c2ba05](https://github.com/windmill-labs/windmill/commit/6c2ba053a1a023e454296a5ebf2842abf90362a8))
+* **frontend:** App select tabs ([#1557](https://github.com/windmill-labs/windmill/issues/1557)) ([4ad530f](https://github.com/windmill-labs/windmill/commit/4ad530f2f004cb33cbc95d5c3b1591a44f93bdee))
+* **frontend:** conditional rendering ([#1555](https://github.com/windmill-labs/windmill/issues/1555)) ([3d371d5](https://github.com/windmill-labs/windmill/commit/3d371d5b6524a4ec0152b15d001d8758900de457))
+* increase timeout to 900 ([018b504](https://github.com/windmill-labs/windmill/commit/018b504986a6c36c1e5ecbc5e92a763a6b6e613b))
+
+
+### Bug Fixes
+
+* **backend:** run endpoints also support support x-www-form-urlencoded encoded payloads ([5601d04](https://github.com/windmill-labs/windmill/commit/5601d047fe4736a996d064dc8ff34af5d70706a5))
+
+## [1.98.0](https://github.com/windmill-labs/windmill/compare/v1.97.0...v1.98.0) (2023-05-09)
+
+
+### Features
+
+* **frontend:** if member of a single workspace, autoset at login ([2dfb74e](https://github.com/windmill-labs/windmill/commit/2dfb74e7e45b279f5169ac89b483ed336e0bd109))
+
+
+### Bug Fixes
+
+* **backend:** grant all on raw_app ([c62670f](https://github.com/windmill-labs/windmill/commit/c62670f735da8378a896b538f3c3afeef100f7ab))
+
+## [1.97.0](https://github.com/windmill-labs/windmill/compare/v1.96.3...v1.97.0) (2023-05-09)
+
+
+### Features
+
+* **backend:** add windmill_status_code to run_wait_result ([38ec7d3](https://github.com/windmill-labs/windmill/commit/38ec7d3a857a19e474f0a1b07b73b85aa5f10f41))
+* **backend:** cache hub scripts in the worker cache ([7537f1a](https://github.com/windmill-labs/windmill/commit/7537f1a1d7162610f78a7e84a53d57f8478a5965))
+* **backend:** in python, if a value is bytes, it is encoded to base64 automaticaly ([6b5ceed](https://github.com/windmill-labs/windmill/commit/6b5ceed6525d4251517627351a15a4fe604629fc))
+
+
+### Bug Fixes
+
+* **lsp:** handle write_message errors ([9392890](https://github.com/windmill-labs/windmill/commit/939289030ba667afc9b517dfdc90f26378fa44a6))
+
+## [1.96.3](https://github.com/windmill-labs/windmill/compare/v1.96.2...v1.96.3) (2023-05-08)
+
+
+### Bug Fixes
+
+* **cli:** add folder listing ([c598083](https://github.com/windmill-labs/windmill/commit/c5980839251cdc759c5afc688e7084f5d58ad57f))
+* **cli:** show diffs only with --show-diffs ([d254088](https://github.com/windmill-labs/windmill/commit/d254088fce00091352ad95888deeaf88bc6c9d6f))
+* **cli:** show diffs only with --show-diffs ([37f08e9](https://github.com/windmill-labs/windmill/commit/37f08e9357c7a74a1c6cdc18bad2d6dc4de5d33d))
+* **cli:** variable updating ([2639250](https://github.com/windmill-labs/windmill/commit/2639250b43c3c55f5b9a43f4020fc2f0747e792b))
+
+## [1.96.2](https://github.com/windmill-labs/windmill/compare/v1.96.1...v1.96.2) (2023-05-08)
+
+
+### Bug Fixes
+
+* **cli:** add debug mode to CLI + improve error output ([8f1cdf1](https://github.com/windmill-labs/windmill/commit/8f1cdf1d61adf80bf0d7c4a5160fd3085d3814ac))
+
+## [1.96.1](https://github.com/windmill-labs/windmill/compare/v1.96.0...v1.96.1) (2023-05-08)
+
+
+### Bug Fixes
+
+* **cli:** fix cli folder sync ([239f401](https://github.com/windmill-labs/windmill/commit/239f40199955d47e4943be4c72c3d150a58f2dd9))
+* **cli:** fix cli folder sync ([a90514b](https://github.com/windmill-labs/windmill/commit/a90514b8e99a419c26512eb370895322088b6aa9))
+
+## [1.96.0](https://github.com/windmill-labs/windmill/compare/v1.95.1...v1.96.0) (2023-05-08)
+
+
+### Features
+
+* add support for full fleged apps (react, svelte, vue) ([#1536](https://github.com/windmill-labs/windmill/issues/1536)) ([13242ab](https://github.com/windmill-labs/windmill/commit/13242abff153b021cac1ecaa3cbf65ae9d87fb69))
+* **frontend:** Add a custom deepEqualWithOrderedArray to handle orde… ([#1537](https://github.com/windmill-labs/windmill/issues/1537)) ([3a291f7](https://github.com/windmill-labs/windmill/commit/3a291f7108623b5c7194f0a7f6a3774499669313))
+* **frontend:** Add label, description, input style + add displayType… ([#1540](https://github.com/windmill-labs/windmill/issues/1540)) ([bef829d](https://github.com/windmill-labs/windmill/commit/bef829d4805bb6c5330b13dc17c9a89a84ad48ca))
+* **frontend:** app modal ([#1518](https://github.com/windmill-labs/windmill/issues/1518)) ([686f5bb](https://github.com/windmill-labs/windmill/commit/686f5bbe1847cb2a92678e7cfdbb51ecf6bbe2b6))
+
+## [1.95.1](https://github.com/windmill-labs/windmill/compare/v1.95.0...v1.95.1) (2023-05-06)
+
+
+### Bug Fixes
+
+* **cli:** cli flow sync improvements ([e585e3a](https://github.com/windmill-labs/windmill/commit/e585e3aea2b18b6dc0c9fa7ffa1e6c1dfb2a3ce2))
+
+## [1.95.0](https://github.com/windmill-labs/windmill/compare/v1.94.0...v1.95.0) (2023-05-05)
+
+
+### Features
+
+* **backend:** default parameters are used in python if missing from args ([8791a86](https://github.com/windmill-labs/windmill/commit/8791a86a936301d44ae05ea09d26c9815abf8929))
+* **frontend:** App Schema Form component ([#1533](https://github.com/windmill-labs/windmill/issues/1533)) ([85c0d93](https://github.com/windmill-labs/windmill/commit/85c0d939f59411d023cd4b173ce11224d3cbc9db))
+* **frontend:** App stepper ([#1529](https://github.com/windmill-labs/windmill/issues/1529)) ([15f1c94](https://github.com/windmill-labs/windmill/commit/15f1c947bb233147f7da261fd32054a51a9c6efa))
+* **frontend:** Merge run configuration + triggers ([#1530](https://github.com/windmill-labs/windmill/issues/1530)) ([1be4658](https://github.com/windmill-labs/windmill/commit/1be4658150ef20a9f1f0fe57b5f30ba3c2d4d94e))
+
 ## [1.94.0](https://github.com/windmill-labs/windmill/compare/v1.93.1...v1.94.0) (2023-05-04)
 
 

Dockerfile

@@ -129,16 +129,21 @@ COPY --from=builder /windmill/target/release/windmill ${APP}/windmill
 
 COPY --from=nsjail /nsjail/nsjail /bin/nsjail
 
-COPY --from=denoland/deno:latest /usr/bin/deno /usr/bin/deno
+COPY --from=denoland/deno:1.33.3 /usr/bin/deno /usr/bin/deno
 
 # docker does not support conditional COPY and we want to use the same Dockerfile for both amd64 and arm64 and privilege the official image
-COPY --from=lukechannings/deno:latest /usr/bin/deno /usr/bin/deno-arm
+COPY --from=lukechannings/deno:v1.33.3 /usr/bin/deno /usr/bin/deno-arm
 RUN if [ "$TARGETPLATFORM" = "linux/amd64" ]; then rm /usr/bin/deno-arm; elif [ "$TARGETPLATFORM" = "linux/arm64" ]; then mv /usr/bin/deno-arm /usr/bin/deno; fi
 
+# add the docker client to call docker from a worker if enabled
+COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/
+
 RUN mkdir -p ${APP}
 
+RUN ln -s ${APP}/windmill /usr/local/bin/windmill
+
 WORKDIR ${APP}
 
 EXPOSE 8000
 
-CMD ["./windmill"]
+CMD ["windmill"]

LICENSE

@@ -1,13 +1,13 @@
 
 Source code in this repository is variously licensed under the Apache License
-Version 2.0 (see file ./LICENSE-APACHE),or the AGPLv3 License (see file ./LICENSE-AGPL)
+Version 2.0 (see file ./LICENSE-APACHE), or the AGPLv3 License (see file ./LICENSE-AGPL)
 
 Every file is under copyright (c) Windmill Labs, Inc 2022 unless otherwise specified.
 Every file is under License AGPL unless otherwise specified 
 or belonging to one of the below cases:
 
-The files under backend/ are AGPL Licensed.
-The files under frontend/ are AGPL Licensed.
+The files under backend/ are AGPLv3 Licensed.
+The files under frontend/ are AGPLv3 Licensed.
 The files under python-client/ deno-client/ go-client/ are Apache 2.0 Licensed.
 
 The openapi files, including the OpenFlow spec is Apache 2.0 Licensed.

README.md

@@ -1,20 +1,32 @@
 <p align="center">
-  <a href="https://app.windmill.dev"><img src="./imgs/windmill-banner.png" alt="windmill.dev"></a>
+  <a href="https://www.windmill.dev/"><img src="./imgs/windmill-banner.png" alt="windmill.dev"></a>
 </p>
 <p align="center">
     <em>.</em>
 </p>
 <p align=center>
-Open-source developer infrastructure for internal tools. Self-hostable alternative to Airplane, Pipedream, Superblocks and a simplified Temporal with autogenerated UIsm and custom UIs to trigger workflows and scripts as internal apps. Scripts are turned into UIs and no-code modules, no-code modules can be composed into very rich flows, and script and flows can be triggered from internal UIs made with a low-code builder. The script languages supported are: Python, Typescript, Go, Bash, SQL.
+Open-source developer infrastructure for internal tools. Self-hostable alternative to Airplane, Pipedream, Superblocks and a simplified Temporal with autogenerated UIsm and custom UIs to trigger workflows and scripts as internal apps.
+
+<p align=center>
+Scripts are turned into UIs and no-code modules, no-code modules can be composed into very rich flows, and script and flows can be triggered from internal UIs made with a low-code builder. The script languages supported are: Python, Typescript, Go, Bash, SQL.
 </p>
 
 <p align="center">
+<a href="https://github.com/windmill-labs/windmill/blob/main/LICENSE-AGPL" target="_blank">
+    <img src="https://img.shields.io/badge/License-AGPLv3-blue.svg" alt="Package version">
+</a>
 <a href="https://github.com/windmill-labs/windmill/actions/workflows/docker-image.yml" target="_blank">
     <img src="https://github.com/windmill-labs/windmill/actions/workflows/docker-image.yml/badge.svg" alt="Docker Image CI">
 </a>
 <a href="https://pypi.org/project/wmill" target="_blank">
     <img src="https://img.shields.io/pypi/v/wmill?color=%2334D058&label=pypi%20package" alt="Package version">
 </a>
+</p>
+
+<p align="center">
+<a href="https://img.shields.io/github/commit-activity/m/windmill-labs/windmill" target="_blank">
+    <img src="https://img.shields.io/github/commit-activity/m/windmill-labs/windmill" alt="Commit activity">
+</a>
 <a href="https://discord.gg/V7PM2YHsPB" target="_blank">
   <img src="https://discordapp.com/api/guilds/930051556043276338/widget.png" alt="Discord Shield"/>
 </a>
@@ -29,9 +41,9 @@ Open-source developer infrastructure for internal tools. Self-hostable alternati
 Windmill is <b>fully open-sourced (AGPLv3)</b> and Windmill Labs offers
 dedicated instance and commercial support and licenses.
 
-![Windmill Diagram](/imgs/stacks.svg)
+![Windmill Diagram](./imgs/stacks.svg)
 
-https://user-images.githubusercontent.com/275584/218350457-bc2fdc3b-e667-4da5-a2bd-3bacc1f0ec79.mp4
+https://github.com/windmill-labs/windmill/assets/122811744/0b132cd1-ee67-4505-822f-0c7ee7104252
 
 - [Windmill - Turn scripts into workflows and UIs that you can share and run at scale](#windmill---turn-scripts-into-workflows-and-uis-that-you-can-share-and-run-at-scale)
   - [Main Concepts](#main-concepts)
@@ -61,8 +73,8 @@ https://user-images.githubusercontent.com/275584/218350457-bc2fdc3b-e667-4da5-a2
 ## Main Concepts
 
 1. Define a minimal and generic script in Python, Typescript, Go or Bash that
-   solves a specific task. Here sending an email with SMTP. The code can be
-   defined in the provided Web IDE or synchronized with your own github repo:
+   solves a specific task. Here sending a POST request. The code can be defined
+   in the provided Web IDE or synchronized with your own github repo:
    ![Step 1](./imgs/windmill-editor.png)
 
 2. Your scripts parameters are automatically parsed and generate a frontend.
@@ -116,7 +128,7 @@ export async function main(
 We have a powerful CLI to interact with the windmill platform and sync your
 scripts from local files, github repos and to run scripts and flows on the
 instance from local commands. See
-[more details](https://github.com/windmill-labs/windmill/tree/main/cli)
+[more details](https://github.com/windmill-labs/windmill/tree/main/cli).
 
 ![CLI Screencast](./cli/vhs/output/setup.gif)
 
@@ -125,7 +137,7 @@ instance from local commands. See
 You can run your script locally easily, you simply need to pass the right
 environment variables for the `wmill` client library to fetch resources and
 variables from your instance if necessary. See more:
-<https://docs.windmill.dev/docs/advanced/local_development/>
+<https://docs.windmill.dev/docs/advanced/local_development/>.
 
 ## Stack
 
@@ -152,7 +164,7 @@ variables from your instance if necessary. See more:
 
 Windmill uses [nsjail](https://github.com/google/nsjail) on top of the deno
 sandboxing. It is production multi-tenant grade secure. Do not take our word for
-it, take [fly.io's one](https://fly.io/blog/sandboxing-and-workload-isolation/)
+it, take [fly.io's one](https://fly.io/blog/sandboxing-and-workload-isolation/).
 
 ### Secrets, credentials and sensitive values
 
@@ -182,7 +194,7 @@ back to the database is ~50ms. A typical lightweight deno job will take around
 
 We only provide docker-compose setup here. For more advanced setups, like
 compiling from source or using without a postgres super user, see
-[documentation](https://docs.windmill.dev/docs/advanced/self_host)
+[documentation](https://docs.windmill.dev/docs/advanced/self_host).
 
 ### Docker compose
 
@@ -196,20 +208,20 @@ docker compose up -d --pull always
 
 Go to http://localhost et voilà :)
 
-The default super-admin user is: admin@windmill.dev / changeme
+The default super-admin user is: admin@windmill.dev / changeme.
 
-From there, you can follow the setup app and creat other users.
+From there, you can follow the setup app and create other users.
 
 ### Kubernetes (k8s) and Helm charts
 
 We publish helm charts at:
-<https://github.com/windmill-labs/windmill-helm-charts>
+<https://github.com/windmill-labs/windmill-helm-charts>.
 
 ### Postgres without superuser
 
 If you do not want, or cannot (for instance, in AWS Aurora or Cloud sql) use a
 postgres superuser, you can run `./init-db-as-superuser.sql` to init the
-required users for windmill.
+required users for Windmill.
 
 ### Commercial license
 
@@ -224,9 +236,9 @@ be AGPLv3 or you must get a commercial license. Contact us at
 <ruben@windmill.dev> if you have any doubts.
 
 In addition, a commercial license grants you a dedicated engineer to transition
-your current infrastructure to Windmill, support with tight SLA, audit logs
-export features, SSO, unlimited users creation, advanced permission managing
-features such as groups and the ability to create more than one workspace.
+your current infrastructure to Windmill, support with tight SLA, and our global
+cache sync for high-performance/no dependency cache miss of cluster from 10+
+nodes to 200+ nodes.
 
 ### OAuth for self-hosting
 
@@ -248,6 +260,10 @@ and mount it at `/usr/src/app/oauth.json`.
 The redirect url for the oauth clients is:
 `<instance_url>/user/login_callback/<client>`
 
+Even if you setup oauth, you will still want to **login as admin@windmill.dev /
+changeme** to setup your instance as a super-admin and give yourself admin
+rights.
+
 [The list of all possible "connect an app" oauth clients](https://github.com/windmill-labs/windmill/blob/main/backend/oauth_connect.json)
 
 To add more "connect an app" OAuth clients to the Windmill project, read the
@@ -316,6 +332,7 @@ it being synced automatically everyday.
 | PYTHON_PATH                         | /usr/local/bin/python3                     | The path to the python binary.                                                                                                                                                                     | Worker                |
 | GO_PATH                             | /usr/bin/go                                | The path to the go binary.                                                                                                                                                                         | Worker                |
 | GOPRIVATE                           |                                            | The GOPRIVATE env variable to use private go modules                                                                                                                                               | Worker                |
+| GOPROXY                             |                                            | The GOPROXY env variable to use                                                                                                                                                                    | Worker                |
 | NETRC                               |                                            | The netrc content to use a private go registry                                                                                                                                                     | Worker                |
 | PIP_INDEX_URL                       | None                                       | The index url to pass for pip.                                                                                                                                                                     | Worker                |
 | PIP_EXTRA_INDEX_URL                 | None                                       | The extra index url to pass to pip.                                                                                                                                                                | Worker                |
@@ -343,6 +360,8 @@ it being synced automatically everyday.
 | WAIT_RESULT_SLOW_POLL_INTERVAL_MS   | 200                                        | The time in between polling for the run_wait_result endpoints in fast poll mode                                                                                                                    | Server                |
 | WAIT_RESULT_FAST_POLL_DURATION_SECS | 2                                          | The duration of fast poll mode before switching to slow poll                                                                                                                                       | Server                |
 | EXIT_AFTER_NO_JOB_FOR_SECS          | None                                       | Exit worker if no job is received after duration in secs if defined                                                                                                                                | Worker                |
+| OAUTH_JSON_AS_BASE64                | None                                       | Base64 encoded JSON of the OAuth configuration. e.g `OAUTH_JSON_AS_BASE64=$(base64 oauth.json \| tr -d '\n')` to encode it                                                                         | Server                |
+| REQUEST_SIZE_LIMIT                  | 2097152 (2MB)                              | Max request size which impact the maximum size of resources and payload size of job args                                                                                                           | Server                |
 
 ## Run a local dev setup
 

backend/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "windmill"
-version = "1.94.0"
+version = "1.104.1"
 authors.workspace = true
 edition.workspace = true
 
@@ -19,7 +19,7 @@ members = [
 ]
 
 [workspace.package]
-version = "1.94.0"
+version = "1.104.1"
 authors = ["Ruben Fiszel <ruben@windmill.dev>"]
 edition = "2021"
 
@@ -104,7 +104,7 @@ rand = "0.8.5"
 rand_core = { version = "^0", features = ["std"] }
 magic-crypt = "^3"
 git-version = "^0"
-rustpython-parser = { git = "https://github.com/RustPython/RustPython" }
+rustpython-parser = "0.2.0"
 cron = "^0"
 lettre = { version = "^0", features = [
     "rustls-tls",
@@ -162,5 +162,5 @@ async_zip = { version = "0.0.11", features = ["full"] }
 once_cell = "1.17.1"
 rsmq_async = { version = "5.1.5" }
 gosyn = "0.2.2"
+bytes = "1.4.0"
 
-[patch.crates-io]

backend/migrations/20230507170410_raw_app.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230507170410_raw_app.up.sql

@@ -0,0 +1,33 @@
+-- Add up migration script here
+CREATE TABLE raw_app (
+    path varchar(255) PRIMARY KEY,
+    version INTEGER NOT NULL DEFAULT 0,
+    workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id),
+    summary VARCHAR(1000) NOT NULL DEFAULT '',
+    edited_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
+    data TEXT NOT NULL,
+    extra_perms JSONB NOT NULL DEFAULT '{}'
+);
+
+CREATE POLICY see_own ON raw_app FOR ALL
+USING (SPLIT_PART(raw_app.path, '/', 1) = 'u' AND SPLIT_PART(raw_app.path, '/', 2) = current_setting('session.user'));
+
+CREATE POLICY see_member ON raw_app FOR ALL
+USING (SPLIT_PART(raw_app.path, '/', 1) = 'g' AND SPLIT_PART(raw_app.path, '/', 2) = any(regexp_split_to_array(current_setting('session.groups'), ',')::text[]));
+
+CREATE POLICY see_extra_perms_user ON raw_app FOR ALL
+USING (extra_perms ? CONCAT('u/', current_setting('session.user')))
+WITH CHECK ((extra_perms ->> CONCAT('u/', current_setting('session.user')))::boolean);
+
+CREATE POLICY see_extra_perms_groups ON raw_app FOR ALL
+USING (extra_perms ?| regexp_split_to_array(current_setting('session.pgroups'), ',')::text[])
+WITH CHECK (exists(
+    SELECT key, value FROM jsonb_each_text(extra_perms) 
+    WHERE SPLIT_PART(key, '/', 1) = 'g' AND key = ANY(regexp_split_to_array(current_setting('session.pgroups'), ',')::text[])
+    AND value::boolean));
+
+CREATE POLICY see_folder_extra_perms_user ON raw_app FOR ALL
+USING (SPLIT_PART(raw_app.path, '/', 1) = 'f' AND SPLIT_PART(raw_app.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_read'), ',')::text[]))
+WITH CHECK (SPLIT_PART(raw_app.path, '/', 1) = 'f' AND SPLIT_PART(raw_app.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_write'), ',')::text[]));
+
+ALTER TYPE FAVORITE_KIND ADD VALUE 'raw_app';

backend/migrations/20230509130016_raw_app_grant.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230509130016_raw_app_grant.up.sql

@@ -0,0 +1,3 @@
+-- Add up migration script here
+GRANT ALL ON raw_app TO windmill_admin;
+GRANT ALL ON raw_app TO windmill_user;

backend/migrations/20230510101952_jsonb_to_json.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230510101952_jsonb_to_json.up.sql

@@ -0,0 +1,3 @@
+-- Add up migration script here
+ALTER TABLE app_version
+ALTER COLUMN value TYPE json;

backend/migrations/20230510104144_jsonb_to_json_draft.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230510104144_jsonb_to_json_draft.up.sql

@@ -0,0 +1,3 @@
+-- Add up migration script here
+ALTER TABLE draft
+ALTER COLUMN value TYPE json;

backend/migrations/20230512191209_update_app_n.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230515131104_rls_raw_app.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230515131104_rls_raw_app.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+ALTER TABLE raw_app ENABLE ROW LEVEL SECURITY;

backend/migrations/20230516140626_capture_check_increase.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230516140626_capture_check_increase.up.sql

@@ -0,0 +1,3 @@
+-- Add up migration script here
+alter table capture DROP constraint capture_payload_check;
+alter table capture add constraint caputre_payload_too_big check  (length(payload::text) < 512 * 1024);

backend/migrations/20230516141630_capture_check_increase_2.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230516141630_capture_check_increase_2.up.sql

@@ -0,0 +1,3 @@
+-- Add up migration script here
+alter table capture DROP constraint caputre_payload_too_big;
+alter table capture add constraint capture_payload_too_big check (length(payload::text) < 512 * 1024);

backend/migrations/20230522170237_schedule_on_failure.down.sql

@@ -0,0 +1,2 @@
+-- Add down migration script here
+ALTER TABLE schedule DROP COLUMN on_failure;

backend/migrations/20230522170237_schedule_on_failure.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+ALTER TABLE schedule ADD COLUMN on_failure VARCHAR(1000) DEFAULT NULL;

backend/migrations/20230524065920_remove_demo_starter.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230524065920_remove_demo_starter.up.sql

@@ -0,0 +1,43 @@
+-- Add up migration script here
+do
+ $$
+BEGIN
+
+IF (NOT EXISTS (SELECT workspace_id FROM script  WHERE workspace_id = 'demo' UNION ALL SELECT workspace_id FROM flow  WHERE workspace_id = 'demo' UNION ALL SELECT workspace_id FROM app WHERE workspace_id = 'demo'))
+THEN
+    DELETE FROM usr WHERE workspace_id = 'demo';
+    DELETE FROM usr_to_group WHERE workspace_id = 'demo';
+    DELETE FROM queue WHERE workspace_id = 'demo';
+    DELETE FROM completed_job WHERE workspace_id = 'demo';
+    DELETE FROM raw_app WHERE workspace_id = 'demo';
+    DELETE FROM variable WHERE workspace_id = 'demo';
+    DELETE FROM schedule WHERE workspace_id = 'demo';
+    DELETE FROM resource WHERE workspace_id = 'demo';
+    DELETE FROM resource_type WHERE workspace_id = 'demo';
+    DELETE FROM workspace_key WHERE workspace_id = 'demo';
+    DELETE FROM group_ WHERE workspace_id = 'demo';
+    DELETE FROM workspace_settings WHERE workspace_id = 'demo';
+    DELETE FROM workspace WHERE id = 'demo';
+END IF;
+
+IF (NOT EXISTS (SELECT workspace_id FROM script  WHERE workspace_id = 'starter' UNION ALL SELECT workspace_id FROM flow  WHERE workspace_id = 'starter' UNION ALL SELECT workspace_id FROM app WHERE workspace_id = 'starter'))
+THEN
+    DELETE FROM usr WHERE workspace_id = 'starter';
+    DELETE FROM usr_to_group WHERE workspace_id = 'starter';
+    DELETE FROM queue WHERE workspace_id = 'starter';
+    DELETE FROM completed_job WHERE workspace_id = 'starter';
+    DELETE FROM raw_app WHERE workspace_id = 'starter';
+    DELETE FROM variable WHERE workspace_id = 'starter';
+    DELETE FROM schedule WHERE workspace_id = 'starter';
+    DELETE FROM resource WHERE workspace_id = 'starter';
+    DELETE FROM resource_type WHERE workspace_id = 'starter';
+    DELETE FROM workspace_key WHERE workspace_id = 'starter';
+    DELETE FROM group_ WHERE workspace_id = 'starter';
+    DELETE FROM workspace_settings WHERE workspace_id = 'starter';
+    DELETE FROM workspace WHERE id = 'starter';
+END IF;
+
+
+END
+$$
+

backend/migrations/20230524133926_update_cli.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230524133926_update_cli.up.sql

@@ -0,0 +1,15 @@
+-- Add up migration script here
+UPDATE script SET content = 'import wmill from "https://deno.land/x/wmill@v1.104.0/main.ts";
+export async function main() {
+  await run(
+    "workspace", "add", "__automation", "admins", Deno.env.get("BASE_INTERNAL_URL") + "/", "--token", Deno.env.get("WM_TOKEN"));
+
+  await run("hub", "pull");
+}
+
+async function run(...cmd: string[]) {
+  console.log("Running \"" + cmd.join('' '') + "\"");
+  await wmill.parse(cmd);
+}', summary = 'Synchronize Hub Resource types with instance',
+description = 'Basic administrative script to sync latest resource types from hub to share to every workspace. Recommended to run at least once. On a schedule by default.'
+WHERE hash = -28028598712388162 AND workspace_id = 'admins';

backend/migrations/20230524163022_update_cli_2.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230524163022_update_cli_2.up.sql

@@ -0,0 +1,15 @@
+-- Add up migration script here
+UPDATE script SET content = 'import wmill from "https://deno.land/x/wmill@v1.104.1/main.ts";
+export async function main() {
+  await run(
+    "workspace", "add", "__automation", "admins", Deno.env.get("BASE_INTERNAL_URL") + "/", "--token", Deno.env.get("WM_TOKEN"));
+
+  await run("hub", "pull");
+}
+
+async function run(...cmd: string[]) {
+  console.log("Running \"" + cmd.join('' '') + "\"");
+  await wmill.parse(cmd);
+}', summary = 'Synchronize Hub Resource types with instance',
+description = 'Basic administrative script to sync latest resource types from hub to share to every workspace. Recommended to run at least once. On a schedule by default.'
+WHERE hash = -28028598712388162 AND workspace_id = 'admins';

backend/parsers/windmill-parser-py/src/lib.rs

@@ -17,8 +17,8 @@ use serde_json::json;
 use windmill_common::error;
 use windmill_parser::{json_to_typ, Arg, MainArgSignature, Typ};
 
-use rustpython_parser as parser;
 use rustpython_parser::ast::{Constant, ExprKind, Located, StmtKind};
+use rustpython_parser::parser::parse_program;
 
 const DEF_MAIN: &str = "def main(";
 const FUNCTION_CALL: &str = "<function call>";
@@ -64,7 +64,7 @@ pub fn parse_python_signature(code: &str) -> error::Result<MainArgSignature> {
             "No main function found".to_string(),
         ));
     }
-    let ast = parser::parse_program(&filtered_code, "main.py").map_err(|e| {
+    let ast = parse_program(&filtered_code, "main.py").map_err(|e| {
         error::Error::ExecutionErr(format!("Error parsing code: {}", e.to_string()))
     })?;
     let param = ast.into_iter().find_map(|x| match x {
@@ -134,10 +134,7 @@ fn to_value(et: &ExprKind) -> Option<serde_json::Value> {
                 .into_iter()
                 .zip(values)
                 .map(|(k, v)| {
-                    let key = k
-                        .as_ref()
-                        .map(|x| x.node.clone())
-                        .and_then(|n| to_value(&n))
+                    let key = to_value(&k.node)
                         .and_then(|x| match x {
                             serde_json::Value::String(s) => Some(s),
                             _ => None,
@@ -185,6 +182,8 @@ static PYTHON_IMPORTS_REPLACEMENT: phf::Map<&'static str, &'static str> = phf_ma
     "shopify" => "ShopifyAPI",
     "seleniumwire" => "selenium-wire",
     "openbb-terminal" => "openbb[all]",
+    "riskfolio" => "riskfolio-lib",
+    "smb" => "pysmb",
 };
 
 fn replace_import(x: String) -> String {
@@ -215,7 +214,7 @@ pub fn parse_python_imports(code: &str) -> error::Result<Vec<String>> {
         Ok(lines)
     } else {
         let code = code.split(DEF_MAIN).next().unwrap_or("");
-        let ast = parser::parse_program(code, "main.py").map_err(|e| {
+        let ast = parse_program(code, "main.py").map_err(|e| {
             error::Error::ExecutionErr(format!("Error parsing code: {}", e.to_string()))
         })?;
         let mut imports: Vec<String> = ast

backend/parsers/windmill-parser-ts/src/lib.rs

@@ -255,6 +255,7 @@ fn tstype_to_typ(ts_type: &TsType) -> (Typ, bool) {
                     ),
                     false,
                 ),
+                "Date" => (Typ::Datetime, false),
                 "Base64" => (Typ::Bytes, false),
                 "Email" => (Typ::Email, false),
                 "Sql" => (Typ::Sql, false),

backend/sqlx-data.json

@@ -85,7 +85,7 @@
         {
           "name": "value",
           "ordinal": 6,
-          "type_info": "Jsonb"
+          "type_info": "Json"
         },
         {
           "name": "created_at",
@@ -692,32 +692,6 @@
     },
     "query": "SELECT * FROM workspace_settings WHERE workspace_id = $1"
   },
-  "17c2a1abcff52cfb39ae55cfafb6b5945bb87eebfdb2e030e7de3db938d0c6c9": {
-    "describe": {
-      "columns": [],
-      "nullable": [],
-      "parameters": {
-        "Left": [
-          "Varchar",
-          "Varchar",
-          "Jsonb",
-          {
-            "Custom": {
-              "kind": {
-                "Enum": [
-                  "script",
-                  "flow",
-                  "app"
-                ]
-              },
-              "name": "draft_type"
-            }
-          }
-        ]
-      }
-    },
-    "query": "INSERT INTO draft\n            (workspace_id, path, value, typ)\n            VALUES ($1, $2, $3, $4)\n            ON CONFLICT (workspace_id, path, typ) DO UPDATE SET value = $3"
-  },
   "187a27d18f78f068319a0ec684f71a76c49dd09c5c74c2c777e35233c4f5a281": {
     "describe": {
       "columns": [
@@ -947,6 +921,109 @@
     },
     "query": "\n        SELECT id, flow_status, suspend, script_path\n        FROM queue\n        WHERE id = $1\n        "
   },
+  "1e0769c59ada78848e034250c11be0ca2e8c49b96a0066b9ab67f5a205ec456d": {
+    "describe": {
+      "columns": [
+        {
+          "name": "workspace_id",
+          "ordinal": 0,
+          "type_info": "Varchar"
+        },
+        {
+          "name": "path",
+          "ordinal": 1,
+          "type_info": "Varchar"
+        },
+        {
+          "name": "edited_by",
+          "ordinal": 2,
+          "type_info": "Varchar"
+        },
+        {
+          "name": "edited_at",
+          "ordinal": 3,
+          "type_info": "Timestamptz"
+        },
+        {
+          "name": "schedule",
+          "ordinal": 4,
+          "type_info": "Varchar"
+        },
+        {
+          "name": "enabled",
+          "ordinal": 5,
+          "type_info": "Bool"
+        },
+        {
+          "name": "script_path",
+          "ordinal": 6,
+          "type_info": "Varchar"
+        },
+        {
+          "name": "args",
+          "ordinal": 7,
+          "type_info": "Jsonb"
+        },
+        {
+          "name": "extra_perms",
+          "ordinal": 8,
+          "type_info": "Jsonb"
+        },
+        {
+          "name": "is_flow",
+          "ordinal": 9,
+          "type_info": "Bool"
+        },
+        {
+          "name": "email",
+          "ordinal": 10,
+          "type_info": "Varchar"
+        },
+        {
+          "name": "error",
+          "ordinal": 11,
+          "type_info": "Text"
+        },
+        {
+          "name": "timezone",
+          "ordinal": 12,
+          "type_info": "Varchar"
+        },
+        {
+          "name": "on_failure",
+          "ordinal": 13,
+          "type_info": "Varchar"
+        }
+      ],
+      "nullable": [
+        false,
+        false,
+        false,
+        false,
+        false,
+        false,
+        false,
+        true,
+        false,
+        false,
+        false,
+        true,
+        false,
+        true
+      ],
+      "parameters": {
+        "Left": [
+          "Varchar",
+          "Varchar",
+          "Jsonb",
+          "Varchar",
+          "Text",
+          "Text"
+        ]
+      }
+    },
+    "query": "UPDATE schedule SET schedule = $1, timezone = $2, args = $3, on_failure = $4 WHERE path = $5 AND workspace_id = $6 RETURNING *"
+  },
   "1eaf8d677d520c7f2f303a731de6b6d939918e41ad0d1c748d80db3fd33cb9d3": {
     "describe": {
       "columns": [],
@@ -1101,7 +1178,8 @@
                 "Enum": [
                   "app",
                   "script",
-                  "flow"
+                  "flow",
+                  "raw_app"
                 ]
               },
               "name": "favorite_kind"
@@ -1228,6 +1306,28 @@
     },
     "query": "SELECT set_config('session.folders_read', $1, true)"
   },
+  "2a3992b5e9abcfbb032d10e142d98efa969dae26a7242eb7ac12593ed5421ef3": {
+    "describe": {
+      "columns": [
+        {
+          "name": "id",
+          "ordinal": 0,
+          "type_info": "Int8"
+        }
+      ],
+      "nullable": [
+        false
+      ],
+      "parameters": {
+        "Left": [
+          "Int8",
+          "Text",
+          "Varchar"
+        ]
+      }
+    },
+    "query": "INSERT INTO app_version\n                (app_id, value, created_by)\n                VALUES ($1, $2::text::json, $3) RETURNING id"
+  },
   "2a3ebe1b0eae5b2164894321e138cc4dc0293788aeb98d05d95d18dfc708d6a6": {
     "describe": {
       "columns": [],
@@ -1385,6 +1485,32 @@
     },
     "query": "UPDATE folder SET owners = array_append(owners, $1) WHERE name = $2 AND workspace_id = $3 AND NOT $1 = ANY(owners) RETURNING name"
   },
+  "366609f7e7fbd73ea807128b931eff2f1ab763fa630c8531f590fed2110c03d9": {
+    "describe": {
+      "columns": [],
+      "nullable": [],
+      "parameters": {
+        "Left": [
+          "Varchar",
+          "Varchar",
+          "Text",
+          {
+            "Custom": {
+              "kind": {
+                "Enum": [
+                  "script",
+                  "flow",
+                  "app"
+                ]
+              },
+              "name": "draft_type"
+            }
+          }
+        ]
+      }
+    },
+    "query": "INSERT INTO draft\n            (workspace_id, path, value, typ)\n            VALUES ($1, $2, $3::text::json, $4)\n            ON CONFLICT (workspace_id, path, typ) DO UPDATE SET value = $3::text::json"
+  },
   "372b0a81d37b6874727c1aa4dc1bec17f9475a5f446e8987ab7359f55b96436e": {
     "describe": {
       "columns": [],
@@ -1728,7 +1854,7 @@
         {
           "name": "value",
           "ordinal": 6,
-          "type_info": "Jsonb"
+          "type_info": "Json"
         },
         {
           "name": "created_at",
@@ -2308,6 +2434,27 @@
     },
     "query": "SELECT EXISTS(SELECT 1 FROM app WHERE path = $1 AND workspace_id = $2)"
   },
+  "5a0333e29280c0814d62473b5e5e7b187c4d8cbf2caeb645bfef82f6880467a7": {
+    "describe": {
+      "columns": [
+        {
+          "name": "data",
+          "ordinal": 0,
+          "type_info": "Text"
+        }
+      ],
+      "nullable": [
+        false
+      ],
+      "parameters": {
+        "Left": [
+          "Text",
+          "Text"
+        ]
+      }
+    },
+    "query": "SELECT data FROM raw_app\n        WHERE path = $1 AND workspace_id = $2"
+  },
   "5b9b58612ca0f703a5d154a76fab82ac2329aef965fa937bfab2810b6e1336a4": {
     "describe": {
       "columns": [],
@@ -2489,7 +2636,8 @@
                 "Enum": [
                   "app",
                   "script",
-                  "flow"
+                  "flow",
+                  "raw_app"
                 ]
               },
               "name": "favorite_kind"
@@ -2712,102 +2860,6 @@
     },
     "query": "SELECT EXISTS(SELECT 1 FROM usr WHERE workspace_id = $1 AND username = $2)"
   },
-  "7226d470c04e58fcab92f3edac21956d2a10e5ed6b594747b22682f05a60a341": {
-    "describe": {
-      "columns": [
-        {
-          "name": "workspace_id",
-          "ordinal": 0,
-          "type_info": "Varchar"
-        },
-        {
-          "name": "path",
-          "ordinal": 1,
-          "type_info": "Varchar"
-        },
-        {
-          "name": "edited_by",
-          "ordinal": 2,
-          "type_info": "Varchar"
-        },
-        {
-          "name": "edited_at",
-          "ordinal": 3,
-          "type_info": "Timestamptz"
-        },
-        {
-          "name": "schedule",
-          "ordinal": 4,
-          "type_info": "Varchar"
-        },
-        {
-          "name": "enabled",
-          "ordinal": 5,
-          "type_info": "Bool"
-        },
-        {
-          "name": "script_path",
-          "ordinal": 6,
-          "type_info": "Varchar"
-        },
-        {
-          "name": "args",
-          "ordinal": 7,
-          "type_info": "Jsonb"
-        },
-        {
-          "name": "extra_perms",
-          "ordinal": 8,
-          "type_info": "Jsonb"
-        },
-        {
-          "name": "is_flow",
-          "ordinal": 9,
-          "type_info": "Bool"
-        },
-        {
-          "name": "email",
-          "ordinal": 10,
-          "type_info": "Varchar"
-        },
-        {
-          "name": "error",
-          "ordinal": 11,
-          "type_info": "Text"
-        },
-        {
-          "name": "timezone",
-          "ordinal": 12,
-          "type_info": "Varchar"
-        }
-      ],
-      "nullable": [
-        false,
-        false,
-        false,
-        false,
-        false,
-        false,
-        false,
-        true,
-        false,
-        false,
-        false,
-        true,
-        false
-      ],
-      "parameters": {
-        "Left": [
-          "Varchar",
-          "Varchar",
-          "Jsonb",
-          "Text",
-          "Text"
-        ]
-      }
-    },
-    "query": "UPDATE schedule SET schedule = $1, timezone = $2, args = $3 WHERE path = $4 AND workspace_id = $5 RETURNING *"
-  },
   "73d3ed17fd0723ba75722f394904f6ee306b59aaf8ecfcf56484d38541343f06": {
     "describe": {
       "columns": [],
@@ -2957,6 +3009,27 @@
     },
     "query": "DELETE FROM usr WHERE email = $1 RETURNING username"
   },
+  "7a819bbf0522315952fd94d9b320fdd58e5d71dbd7e5f79eaa83c14572159805": {
+    "describe": {
+      "columns": [
+        {
+          "name": "count",
+          "ordinal": 0,
+          "type_info": "Int8"
+        }
+      ],
+      "nullable": [
+        null
+      ],
+      "parameters": {
+        "Left": [
+          "Text",
+          "Text"
+        ]
+      }
+    },
+    "query": "SELECT count(path) FROM raw_app WHERE path LIKE 'f/' || $1 || '%'  AND workspace_id = $2"
+  },
   "7aef087f9e10dd32417477109b97c99b85d68ce1be2bafdc145aed0aa8e5d989": {
     "describe": {
       "columns": [],
@@ -3164,6 +3237,114 @@
     },
     "query": "INSERT INTO group_ (workspace_id, name, summary, extra_perms) VALUES ($1, $2, $3, $4)"
   },
+  "80e632cb6618cde2bac2c12d0afc796a4a1768f902d0540af33fe6f442cf583a": {
+    "describe": {
+      "columns": [
+        {
+          "name": "workspace_id",
+          "ordinal": 0,
+          "type_info": "Varchar"
+        },
+        {
+          "name": "path",
+          "ordinal": 1,
+          "type_info": "Varchar"
+        },
+        {
+          "name": "edited_by",
+          "ordinal": 2,
+          "type_info": "Varchar"
+        },
+        {
+          "name": "edited_at",
+          "ordinal": 3,
+          "type_info": "Timestamptz"
+        },
+        {
+          "name": "schedule",
+          "ordinal": 4,
+          "type_info": "Varchar"
+        },
+        {
+          "name": "enabled",
+          "ordinal": 5,
+          "type_info": "Bool"
+        },
+        {
+          "name": "script_path",
+          "ordinal": 6,
+          "type_info": "Varchar"
+        },
+        {
+          "name": "args",
+          "ordinal": 7,
+          "type_info": "Jsonb"
+        },
+        {
+          "name": "extra_perms",
+          "ordinal": 8,
+          "type_info": "Jsonb"
+        },
+        {
+          "name": "is_flow",
+          "ordinal": 9,
+          "type_info": "Bool"
+        },
+        {
+          "name": "email",
+          "ordinal": 10,
+          "type_info": "Varchar"
+        },
+        {
+          "name": "error",
+          "ordinal": 11,
+          "type_info": "Text"
+        },
+        {
+          "name": "timezone",
+          "ordinal": 12,
+          "type_info": "Varchar"
+        },
+        {
+          "name": "on_failure",
+          "ordinal": 13,
+          "type_info": "Varchar"
+        }
+      ],
+      "nullable": [
+        false,
+        false,
+        false,
+        false,
+        false,
+        false,
+        false,
+        true,
+        false,
+        false,
+        false,
+        true,
+        false,
+        true
+      ],
+      "parameters": {
+        "Left": [
+          "Varchar",
+          "Varchar",
+          "Varchar",
+          "Varchar",
+          "Varchar",
+          "Varchar",
+          "Bool",
+          "Jsonb",
+          "Bool",
+          "Varchar",
+          "Varchar"
+        ]
+      }
+    },
+    "query": "INSERT INTO schedule (workspace_id, path, schedule, timezone, edited_by, script_path, is_flow, args, enabled, email, on_failure) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) RETURNING *"
+  },
   "826c6a36020f402169cab2ebe097e9f38f416f6f080a2fe8a9b83e97c4e15150": {
     "describe": {
       "columns": [
@@ -3212,6 +3393,43 @@
     },
     "query": "UPDATE group_ SET summary = $1 WHERE name = $2 AND workspace_id = $3"
   },
+  "83d18f4e1cda2c1867168551d855d5626e766b5469c6976c986bdafc8b9407c5": {
+    "describe": {
+      "columns": [
+        {
+          "name": "id",
+          "ordinal": 0,
+          "type_info": "Int8"
+        }
+      ],
+      "nullable": [
+        false
+      ],
+      "parameters": {
+        "Left": [
+          "Int8",
+          "Text",
+          "Varchar"
+        ]
+      }
+    },
+    "query": "INSERT INTO app_version\n            (app_id, value, created_by)\n            VALUES ($1, $2::text::json, $3) RETURNING id"
+  },
+  "84bbd11f7bb0c65dcfb2e12038b8a9cb51d303480ac3c5f8a5e833a53c40ed9b": {
+    "describe": {
+      "columns": [],
+      "nullable": [],
+      "parameters": {
+        "Left": [
+          "Varchar",
+          "Varchar",
+          "Varchar",
+          "Text"
+        ]
+      }
+    },
+    "query": "INSERT INTO raw_app\n            (workspace_id, path, summary, extra_perms, data)\n            VALUES ($1, $2, $3, '{}', $4)"
+  },
   "8543f029d9784234e4c6a6dcd7b03e62d544b98be261334ee210594e0bb839f2": {
     "describe": {
       "columns": [
@@ -3518,6 +3736,11 @@
           "name": "timezone",
           "ordinal": 12,
           "type_info": "Varchar"
+        },
+        {
+          "name": "on_failure",
+          "ordinal": 13,
+          "type_info": "Varchar"
         }
       ],
       "nullable": [
@@ -3533,7 +3756,8 @@
         false,
         false,
         true,
-        false
+        false,
+        true
       ],
       "parameters": {
         "Left": [
@@ -3829,6 +4053,11 @@
           "name": "timezone",
           "ordinal": 12,
           "type_info": "Varchar"
+        },
+        {
+          "name": "on_failure",
+          "ordinal": 13,
+          "type_info": "Varchar"
         }
       ],
       "nullable": [
@@ -3844,7 +4073,8 @@
         false,
         false,
         true,
-        false
+        false,
+        true
       ],
       "parameters": {
         "Left": [
@@ -4016,6 +4246,11 @@
           "name": "timezone",
           "ordinal": 12,
           "type_info": "Varchar"
+        },
+        {
+          "name": "on_failure",
+          "ordinal": 13,
+          "type_info": "Varchar"
         }
       ],
       "nullable": [
@@ -4031,7 +4266,8 @@
         false,
         false,
         true,
-        false
+        false,
+        true
       ],
       "parameters": {
         "Left": [
@@ -4141,7 +4377,7 @@
         {
           "name": "value",
           "ordinal": 6,
-          "type_info": "Jsonb"
+          "type_info": "Json"
         },
         {
           "name": "created_at",
@@ -4541,6 +4777,11 @@
           "name": "timezone",
           "ordinal": 12,
           "type_info": "Varchar"
+        },
+        {
+          "name": "on_failure",
+          "ordinal": 13,
+          "type_info": "Varchar"
         }
       ],
       "nullable": [
@@ -4556,7 +4797,8 @@
         false,
         false,
         true,
-        false
+        false,
+        true
       ],
       "parameters": {
         "Left": [
@@ -4722,6 +4964,19 @@
     },
     "query": "UPDATE usr SET is_admin = $1 WHERE username = $2 AND workspace_id = $3"
   },
+  "b2b2b7251be2b80207f47b10eeff78d61c84161caaf16b8fd25d82b97aac1186": {
+    "describe": {
+      "columns": [],
+      "nullable": [],
+      "parameters": {
+        "Left": [
+          "Text",
+          "Text"
+        ]
+      }
+    },
+    "query": "DELETE FROM raw_app WHERE path = $1 AND workspace_id = $2"
+  },
   "b3b80de52d0931a2fdb5d38b7603a2d69cc25ab1cda413228c363a5ffd777113": {
     "describe": {
       "columns": [
@@ -5116,6 +5371,11 @@
           "name": "timezone",
           "ordinal": 12,
           "type_info": "Varchar"
+        },
+        {
+          "name": "on_failure",
+          "ordinal": 13,
+          "type_info": "Varchar"
         }
       ],
       "nullable": [
@@ -5131,7 +5391,8 @@
         false,
         false,
         true,
-        false
+        false,
+        true
       ],
       "parameters": {
         "Left": [
@@ -5232,6 +5493,27 @@
     },
     "query": "\n                UPDATE queue\n                   SET flow_status = JSONB_SET(\n                                     JSONB_SET(flow_status, ARRAY['failure_module'], $1),\n                                                            ARRAY['step'], $2)\n                 WHERE id = $3\n                  "
   },
+  "c4b3ab3d87f158b20f0557f78c6307c50620425aeb43636e19931c142b5ce0d7": {
+    "describe": {
+      "columns": [
+        {
+          "name": "exists",
+          "ordinal": 0,
+          "type_info": "Bool"
+        }
+      ],
+      "nullable": [
+        null
+      ],
+      "parameters": {
+        "Left": [
+          "Text",
+          "Text"
+        ]
+      }
+    },
+    "query": "SELECT EXISTS(SELECT 1 FROM raw_app WHERE path = $1 AND workspace_id = $2)"
+  },
   "c4f1c14c3aae145b52ff39bdecd779cd3eba27a869fda68516dc68dc2abefd38": {
     "describe": {
       "columns": [],
@@ -5399,6 +5681,26 @@
     },
     "query": "UPDATE account SET refresh_token = $1, expires_at = $2 WHERE workspace_id = $3 AND id = $4"
   },
+  "cb0eecb1130617f2132b8ec74a401ac3fb932bbeafe3aecc0b14465cce7e192d": {
+    "describe": {
+      "columns": [
+        {
+          "name": "?column?",
+          "ordinal": 0,
+          "type_info": "Int4"
+        }
+      ],
+      "nullable": [
+        null
+      ],
+      "parameters": {
+        "Left": [
+          "Uuid"
+        ]
+      }
+    },
+    "query": "SELECT 1 FROM queue WHERE id = $1 UNION ALL select 1 FROM completed_job WHERE id = $1"
+  },
   "cb12aee5f8e04cb196d4b8fad81699fcbb1ae7b0c84090d5705b14eac76074ff": {
     "describe": {
       "columns": [],
@@ -5488,7 +5790,7 @@
         {
           "name": "value",
           "ordinal": 6,
-          "type_info": "Jsonb"
+          "type_info": "Json"
         },
         {
           "name": "created_at",
@@ -5587,28 +5889,6 @@
     },
     "query": "SELECT name, (EXISTS (SELECT 1 FROM (SELECT key, value FROM jsonb_each_text(extra_perms) WHERE key = ANY($1)) t  WHERE value::boolean IS true)) as write, $1 && owners::text[] as owner  FROM folder\n        WHERE extra_perms ?| $1  AND workspace_id = $2"
   },
-  "d47bff7d6b54cd6da8bb330f7321c37af5dcbd76f9acad73b5ba1b8a4afb5091": {
-    "describe": {
-      "columns": [
-        {
-          "name": "id",
-          "ordinal": 0,
-          "type_info": "Int8"
-        }
-      ],
-      "nullable": [
-        false
-      ],
-      "parameters": {
-        "Left": [
-          "Int8",
-          "Jsonb",
-          "Varchar"
-        ]
-      }
-    },
-    "query": "INSERT INTO app_version\n                (app_id, value, created_by)\n                VALUES ($1, $2, $3) RETURNING id"
-  },
   "d4bf68b4c2bfd11db84787e41e92c13612fda7352e841f3cdef45ea762588f04": {
     "describe": {
       "columns": [],
@@ -5654,28 +5934,6 @@
     },
     "query": "DELETE FROM resource_type WHERE name = $1 AND workspace_id = $2"
   },
-  "d68eaf1533bbe9052f6b1737e2f212c969a18c9d7840fe419c20995404ed0be8": {
-    "describe": {
-      "columns": [
-        {
-          "name": "id",
-          "ordinal": 0,
-          "type_info": "Int8"
-        }
-      ],
-      "nullable": [
-        false
-      ],
-      "parameters": {
-        "Left": [
-          "Int8",
-          "Jsonb",
-          "Varchar"
-        ]
-      }
-    },
-    "query": "INSERT INTO app_version\n            (app_id, value, created_by)\n            VALUES ($1, $2, $3) RETURNING id"
-  },
   "d697b7311430e7bd5375ec5494179f4071e3bfe123d9600249cfa80c1103edd8": {
     "describe": {
       "columns": [],
@@ -6108,107 +6366,6 @@
     },
     "query": "INSERT INTO workspace_settings\n                (workspace_id, slack_team_id, slack_name, slack_email)\n                VALUES ($1, $2, $3, $4) ON CONFLICT (workspace_id) DO UPDATE SET slack_team_id = $2, slack_name = $3, slack_email = $4"
   },
-  "ebbabe9b71804ee983235db7d7844052139c23ae7a27cae925c19e1c01037143": {
-    "describe": {
-      "columns": [
-        {
-          "name": "workspace_id",
-          "ordinal": 0,
-          "type_info": "Varchar"
-        },
-        {
-          "name": "path",
-          "ordinal": 1,
-          "type_info": "Varchar"
-        },
-        {
-          "name": "edited_by",
-          "ordinal": 2,
-          "type_info": "Varchar"
-        },
-        {
-          "name": "edited_at",
-          "ordinal": 3,
-          "type_info": "Timestamptz"
-        },
-        {
-          "name": "schedule",
-          "ordinal": 4,
-          "type_info": "Varchar"
-        },
-        {
-          "name": "enabled",
-          "ordinal": 5,
-          "type_info": "Bool"
-        },
-        {
-          "name": "script_path",
-          "ordinal": 6,
-          "type_info": "Varchar"
-        },
-        {
-          "name": "args",
-          "ordinal": 7,
-          "type_info": "Jsonb"
-        },
-        {
-          "name": "extra_perms",
-          "ordinal": 8,
-          "type_info": "Jsonb"
-        },
-        {
-          "name": "is_flow",
-          "ordinal": 9,
-          "type_info": "Bool"
-        },
-        {
-          "name": "email",
-          "ordinal": 10,
-          "type_info": "Varchar"
-        },
-        {
-          "name": "error",
-          "ordinal": 11,
-          "type_info": "Text"
-        },
-        {
-          "name": "timezone",
-          "ordinal": 12,
-          "type_info": "Varchar"
-        }
-      ],
-      "nullable": [
-        false,
-        false,
-        false,
-        false,
-        false,
-        false,
-        false,
-        true,
-        false,
-        false,
-        false,
-        true,
-        false
-      ],
-      "parameters": {
-        "Left": [
-          "Varchar",
-          "Varchar",
-          "Varchar",
-          "Varchar",
-          "Varchar",
-          "Varchar",
-          "Bool",
-          "Jsonb",
-          "Bool",
-          "Varchar"
-        ]
-      }
-    },
-    "query": "INSERT INTO schedule (workspace_id, path, schedule, timezone, edited_by, script_path, is_flow, args, enabled, email) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) RETURNING *"
-  },
   "ec85a425f88044c6ed4f8fcea223c28eb9fb8c16c89a52d4c4552bd149badafa": {
     "describe": {
       "columns": [
@@ -6303,6 +6460,11 @@
           "name": "timezone",
           "ordinal": 12,
           "type_info": "Varchar"
+        },
+        {
+          "name": "on_failure",
+          "ordinal": 13,
+          "type_info": "Varchar"
         }
       ],
       "nullable": [
@@ -6318,7 +6480,8 @@
         false,
         false,
         true,
-        false
+        false,
+        true
       ],
       "parameters": {
         "Left": [
@@ -6428,7 +6591,7 @@
         {
           "name": "value",
           "ordinal": 6,
-          "type_info": "Jsonb"
+          "type_info": "Json"
         },
         {
           "name": "created_at",
@@ -6448,7 +6611,7 @@
         {
           "name": "draft?",
           "ordinal": 10,
-          "type_info": "Jsonb"
+          "type_info": "Json"
         }
       ],
       "nullable": [

backend/src/ee.rs

@@ -8,16 +8,31 @@ use sha2::Sha256;
 #[cfg(feature = "enterprise")]
 pub fn verify_license_key(license_key: Option<String>) -> anyhow::Result<()> {
     if let Some(license_key) = license_key {
-        let splitted_lk = license_key
-            .split_once(".")
-            .expect("license_key can be splitted with a .");
+        let mut splitted_lk = license_key.split(".");
+        if splitted_lk.clone().count() != 3 {
+            panic!("license_key can be splitted with 2 . (<client id>.<expiry>.<signature>)");
+        }
+        let id = splitted_lk.next().unwrap();
+        let expiry = splitted_lk.next().unwrap();
+        let signature_b64 = splitted_lk.next().unwrap();
 
+        let expiry_nb = expiry.parse::<u64>()?;
+        if expiry_nb < chrono::Utc::now().timestamp() as u64 {
+            panic!(
+                "License key is expired (timestamp expiry: {expiry_nb}. Now: {}",
+                chrono::Utc::now().timestamp()
+            );
+        }
+        const PUBLIC_KEY: &str = "MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDgVShzcLSPiOi+8ET8fggob1kmi47/cE12JaidPkwfGnScZItghkqtiLsct0U4kJhlp5gO89DYTBmIKadvxwY7kMsLlZzmi2emVH7c27cByGASY8QmWDNdG4Ggy/NDflGGBdAtN6gHawZAg4zHv3qpbPQGHH1/6sXIohcXhOnouwIDAQAB";
         let pub_key = rsa::RsaPublicKey::from_public_key_der(
-    &base64::engine::general_purpose::STANDARD.decode("MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDgVShzcLSPiOi+8ET8fggob1kmi47/cE12JaidPkwfGnScZItghkqtiLsct0U4kJhlp5gO89DYTBmIKadvxwY7kMsLlZzmi2emVH7c27cByGASY8QmWDNdG4Ggy/NDflGGBdAtN6gHawZAg4zHv3qpbPQGHH1/6sXIohcXhOnouwIDAQAB")?)?;
-        let msg = base64::engine::general_purpose::STANDARD.decode(splitted_lk.0)?;
-        let signature = base64::engine::general_purpose::STANDARD.decode(splitted_lk.1)?;
+            &base64::engine::general_purpose::STANDARD.decode(PUBLIC_KEY)?,
+        )?;
+        let signature = base64::engine::general_purpose::STANDARD.decode(signature_b64)?;
         rsa::pss::VerifyingKey::<Sha256>::new(pub_key)
-            .verify(&msg, &rsa::pss::Signature::from(signature))
+            .verify(
+                &format!("{id}{expiry}").as_bytes(),
+                &rsa::pss::Signature::from(signature),
+            )
             .map_err(|_| anyhow::anyhow!("Invalid license key".to_string()))?;
     } else {
         panic!("License key is required for the enterprise edition");

backend/src/main.rs

@@ -19,10 +19,11 @@ use tokio::{
     join,
     sync::RwLock,
 };
+use windmill_api::LICENSE_KEY;
 use windmill_common::{utils::rd_string, METRICS_ADDR};
 use windmill_worker::{
-    DENO_CACHE_DIR, DENO_TMP_CACHE_DIR, GO_CACHE_DIR, GO_TMP_CACHE_DIR, PIP_CACHE_DIR,
-    ROOT_TMP_CACHE_DIR, TAR_PIP_TMP_CACHE_DIR,
+    DENO_CACHE_DIR, DENO_TMP_CACHE_DIR, GO_CACHE_DIR, GO_TMP_CACHE_DIR, HUB_CACHE_DIR,
+    HUB_TMP_CACHE_DIR, PIP_CACHE_DIR, ROOT_TMP_CACHE_DIR, TAR_PIP_TMP_CACHE_DIR,
 };
 
 const GIT_VERSION: &str = git_version!(args = ["--tag", "--always"], fallback = "unknown-version");
@@ -143,6 +144,7 @@ Windmill Community Edition {GIT_VERSION}
         "DENO_PATH",
         "GO_PATH",
         "GOPRIVATE",
+        "GOPROXY",
         "NETRC",
         "PIP_INDEX_URL",
         "PIP_EXTRA_INDEX_URL",
@@ -170,6 +172,7 @@ Windmill Community Edition {GIT_VERSION}
         "WAIT_RESULT_SLOW_POLL_INTERVAL_MS",
         "WAIT_RESULT_FAST_POLL_INTERVAL_MS",
         "EXIT_AFTER_NO_JOB_FOR_SECS",
+        "REQUEST_SIZE_LIMIT",
     ]);
 
     if server_mode || num_workers > 0 {
@@ -268,12 +271,11 @@ pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + '
     base_internal_url: String,
     rsmq: Option<R>,
 ) -> anyhow::Result<()> {
-    let license_key = std::env::var("LICENSE_KEY").ok();
     #[cfg(feature = "enterprise")]
-    ee::verify_license_key(license_key)?;
+    ee::verify_license_key(LICENSE_KEY.clone())?;
 
     #[cfg(not(feature = "enterprise"))]
-    if license_key.is_some() {
+    if LICENSE_KEY.is_some() {
         panic!("License key is required ONLY for the enterprise edition");
     }
 
@@ -299,9 +301,11 @@ pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + '
         PIP_CACHE_DIR,
         DENO_CACHE_DIR,
         GO_CACHE_DIR,
+        HUB_CACHE_DIR,
         TAR_PIP_TMP_CACHE_DIR,
         DENO_TMP_CACHE_DIR,
         GO_TMP_CACHE_DIR,
+        HUB_TMP_CACHE_DIR,
     ] {
         DirBuilder::new()
             .recursive(true)

backend/tests/worker.rs

@@ -842,6 +842,7 @@ impl RunJob {
             /* schedule_path */ None,
             /* parent_job */ None,
             /* root job  */ None,
+            /* job_id */ None,
             /* is_flow_step */ false,
             /* running */ false,
             None,

backend/windmill-api-client/Cargo.toml

@@ -11,7 +11,7 @@ path = "./src/lib.rs"
 
 
 [dependencies]
-progenitor-client = { git = "https://github.com/oxidecomputer/progenitor" }
+progenitor-client = { git = "https://github.com/oxidecomputer/progenitor", rev = "3d96016ae8d422e90513b2d34fb5b63eeab30b01" }
 reqwest = { version = "0.11", features = ["json", "stream"] }
 serde = { version = "1.0", features = ["derive"] }
 chrono.workspace = true
@@ -22,6 +22,6 @@ base64.workspace = true
 
 [build-dependencies]
 prettyplease = "0.1.25"
-progenitor = { git = "https://github.com/oxidecomputer/progenitor" }
+progenitor = { git = "https://github.com/oxidecomputer/progenitor", rev = "3d96016ae8d422e90513b2d34fb5b63eeab30b01" }
 serde_json = "1.0"
 syn = "1.0"

backend/windmill-api/Cargo.toml

@@ -72,3 +72,4 @@ prometheus.workspace = true
 async_zip.workspace = true
 rsmq_async.workspace = true
 regex.workspace = true
+bytes.workspace = true

backend/windmill-api/openapi.yaml

@@ -1,7 +1,7 @@
 openapi: "3.0.3"
 
 info:
-  version: 1.94.0
+  version: 1.104.1
   title: Windmill API
 
   contact:
@@ -41,6 +41,20 @@ paths:
               schema:
                 type: string
 
+  /ee_license:
+    get:
+      summary: get license id
+      operationId: getLicenseId
+      tags:
+        - settings
+      responses:
+        "200":
+          description: get license id (empty if not ee)
+          content:
+            text/plain:
+              schema:
+                type: string
+
   /openapi.yaml:
     get:
       summary: get openapi yaml spec
@@ -2517,6 +2531,7 @@ paths:
           schema:
             type: integer
         - $ref: "#/components/parameters/ParentJob"
+        - $ref: "#/components/parameters/NewJobId"
         - name: invisible_to_owner
           description: make the run invisible to the the script owner (default false)
           in: query
@@ -2549,6 +2564,7 @@ paths:
         - $ref: "#/components/parameters/WorkspaceId"
         - $ref: "#/components/parameters/ScriptPath"
         - $ref: "#/components/parameters/ParentJob"
+        - $ref: "#/components/parameters/NewJobId"
         - $ref: "#/components/parameters/IncludeHeader"
         - $ref: "#/components/parameters/QueueLimit"
 
@@ -2576,6 +2592,7 @@ paths:
         - $ref: "#/components/parameters/WorkspaceId"
         - $ref: "#/components/parameters/ScriptPath"
         - $ref: "#/components/parameters/ParentJob"
+        - $ref: "#/components/parameters/NewJobId"
         - $ref: "#/components/parameters/IncludeHeader"
         - $ref: "#/components/parameters/QueueLimit"
         - $ref: "#/components/parameters/Payload"
@@ -2598,6 +2615,7 @@ paths:
         - $ref: "#/components/parameters/ScriptPath"
         - $ref: "#/components/parameters/IncludeHeader"
         - $ref: "#/components/parameters/QueueLimit"
+        - $ref: "#/components/parameters/NewJobId"
 
       requestBody:
         description: script args
@@ -2888,6 +2906,80 @@ paths:
                 items:
                   $ref: "#/components/schemas/Input"
 
+  /w/{workspace}/raw_apps/list:
+    get:
+      summary: list all available raw apps
+      operationId: listRawApps
+      tags:
+        - raw_app
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/Page"
+        - $ref: "#/components/parameters/PerPage"
+        - $ref: "#/components/parameters/OrderDesc"
+        - $ref: "#/components/parameters/CreatedBy"
+        - name: path_start
+          description: mask to filter matching starting path
+          in: query
+          schema:
+            type: string
+        - name: path_exact
+          description: mask to filter exact matching path
+          in: query
+          schema:
+            type: string
+        - name: starred_only
+          description: |
+            (default false)
+            show only the starred items
+          in: query
+          schema:
+            type: boolean
+      responses:
+        "200":
+          description: All available raw apps
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: "#/components/schemas/ListableRawApp"
+
+  /w/{workspace}/raw_apps/exists/{path}:
+    get:
+      summary: does an app exisst at path
+      operationId: existsRawApp
+      tags:
+        - raw_app
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/Path"
+      responses:
+        "200":
+          description: app exists
+          content:
+            application/json:
+              schema:
+                type: boolean
+
+  /w/{workspace}/apps/get_data/{version}/{path}:
+    get:
+      summary: get app by path
+      operationId: getRawAppData
+      tags:
+        - raw_app
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/VersionId"
+        - $ref: "#/components/parameters/ScriptPath"
+      responses:
+        "200":
+          description: app details
+          content:
+            text/javascript:
+              schema:
+                type: string
+
   /w/{workspace}/apps/list:
     get:
       summary: list all available apps
@@ -2927,6 +3019,44 @@ paths:
                 items:
                   $ref: "#/components/schemas/ListableApp"
 
+  /w/{workspace}/apps/create:
+    post:
+      summary: create app
+      operationId: createApp
+      tags:
+        - app
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+      requestBody:
+        description: new app
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                path:
+                  type: string
+                value: {}
+                summary:
+                  type: string
+                policy:
+                  $ref: "#/components/schemas/Policy"
+                draft_only:
+                  type: boolean
+              required:
+                - path
+                - value
+                - summary
+                - policy
+      responses:
+        "201":
+          description: app created
+          content:
+            text/plain:
+              schema:
+                type: string
+
   /w/{workspace}/apps/exists/{path}:
     get:
       summary: does an app exisst at path
@@ -3030,16 +3160,16 @@ paths:
               schema:
                 $ref: "#/components/schemas/AppWithLastVersion"
 
-  /w/{workspace}/apps/create:
+  /w/{workspace}/raw_apps/create:
     post:
-      summary: create app
-      operationId: createApp
+      summary: create raw app
+      operationId: createRawApp
       tags:
-        - app
+        - raw_app
       parameters:
         - $ref: "#/components/parameters/WorkspaceId"
       requestBody:
-        description: new app
+        description: new raw app
         required: true
         content:
           application/json:
@@ -3048,21 +3178,65 @@ paths:
               properties:
                 path:
                   type: string
-                value: {}
+                value:
+                  type: string
                 summary:
                   type: string
-                policy:
-                  $ref: "#/components/schemas/Policy"
-                draft_only:
-                  type: boolean
               required:
                 - path
                 - value
                 - summary
-                - policy
       responses:
         "201":
-          description: app created
+          description: raw app created
+          content:
+            text/plain:
+              schema:
+                type: string
+
+  /w/{workspace}/raw_apps/update/{path}:
+    post:
+      summary: update app
+      operationId: updateRawApp
+      tags:
+        - raw_app
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/ScriptPath"
+      requestBody:
+        description: updateraw  app
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                path:
+                  type: string
+                summary:
+                  type: string
+                value:
+                  type: string
+      responses:
+        "200":
+          description: app updated
+          content:
+            text/plain:
+              schema:
+                type: string
+
+  /w/{workspace}/raw_apps/delete/{path}:
+    delete:
+      summary: delete raw app
+      operationId: deleteRawApp
+      tags:
+        - raw_app
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/Path"
+      responses:
+        "200":
+          description: app deleted
           content:
             text/plain:
               schema:
@@ -3134,6 +3308,8 @@ paths:
             schema:
               type: object
               properties:
+                component:
+                  type: string
                 #script: script/<path>
                 #flow: flow/<path>
                 path:
@@ -3155,6 +3331,7 @@ paths:
                   type: object
               required:
                 - args
+                - component
 
       responses:
         "200":
@@ -3185,6 +3362,7 @@ paths:
           schema:
             type: integer
         - $ref: "#/components/parameters/ParentJob"
+        - $ref: "#/components/parameters/NewJobId"
         - $ref: "#/components/parameters/IncludeHeader"
         - name: invisible_to_owner
           description: make the run invisible to the the flow owner (default false)
@@ -3230,6 +3408,7 @@ paths:
           schema:
             type: integer
         - $ref: "#/components/parameters/ParentJob"
+        - $ref: "#/components/parameters/NewJobId"
         - $ref: "#/components/parameters/IncludeHeader"
         - name: invisible_to_owner
           description: make the run invisible to the the script owner (default false)
@@ -3267,6 +3446,8 @@ paths:
           in: query
           schema:
             type: boolean
+        - $ref: "#/components/parameters/NewJobId"
+
       requestBody:
         description: preview
         required: true
@@ -3298,6 +3479,7 @@ paths:
           in: query
           schema:
             type: boolean
+        - $ref: "#/components/parameters/NewJobId"
 
       requestBody:
         description: preview
@@ -3486,7 +3668,7 @@ paths:
                   mem_peak:
                     type: integer
 
-  /w/{workspace}/jobs/completed/get/{id}:
+  /w/{workspace}/jobs_u/completed/get/{id}:
     get:
       summary: get completed job
       operationId: getCompletedJob
@@ -3503,7 +3685,7 @@ paths:
               schema:
                 $ref: "#/components/schemas/CompletedJob"
 
-  /w/{workspace}/jobs/completed/get_result/{id}:
+  /w/{workspace}/jobs_u/completed/get_result/{id}:
     get:
       summary: get completed job result
       operationId: getCompletedJobResult
@@ -3519,6 +3701,30 @@ paths:
             application/json:
               schema: {}
 
+  /w/{workspace}/jobs_u/completed/get_result_maybe/{id}:
+    get:
+      summary: get completed job result if job is completed
+      operationId: getCompletedJobResultMaybe
+      tags:
+        - job
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/JobId"
+      responses:
+        "200":
+          description: result
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  completed:
+                    type: boolean
+                  result: {}
+                required:
+                  - completed
+                  - result
+
   /w/{workspace}/jobs/completed/delete/{id}:
     post:
       summary: delete completed job (erase content but keep run id)
@@ -4499,7 +4705,7 @@ paths:
           schema:
             type: string
             enum:
-              [script, group_, resource, schedule, variable, flow, folder, app]
+              [script, group_, resource, schedule, variable, flow, folder, app, raw_app]
       responses:
         "200":
           description: acls
@@ -4525,7 +4731,7 @@ paths:
           schema:
             type: string
             enum:
-              [script, group_, resource, schedule, variable, flow, folder, app]
+              [script, group_, resource, schedule, variable, flow, folder, app, raw_app]
       requestBody:
         description: acl to add
         required: true
@@ -4562,7 +4768,7 @@ paths:
           schema:
             type: string
             enum:
-              [script, group_, resource, schedule, variable, flow, folder, app]
+              [script, group_, resource, schedule, variable, flow, folder, app, raw_app]
       requestBody:
         description: acl to add
         required: true
@@ -4642,7 +4848,7 @@ paths:
                   type: string
                 favorite_kind:
                   type: string
-                  enum: [flow, app, script]
+                  enum: [flow, app, script, raw_app]
       responses:
         "200":
           description: star item
@@ -4665,7 +4871,7 @@ paths:
                   type: string
                 favorite_kind:
                   type: string
-                  enum: [flow, app, script]
+                  enum: [flow, app, script, raw_app]
       responses:
         "200":
           description: unstar item
@@ -4799,6 +5005,12 @@ components:
       required: true
       schema:
         type: string
+    VersionId:
+      name: version
+      in: path
+      required: true
+      schema:
+        type: number
     Token:
       name: token
       in: path
@@ -4887,6 +5099,15 @@ components:
       schema:
         type: string
         format: uuid
+    NewJobId:
+      name: job_id
+      description:
+        The job id to assign to the created job. if missing, job is chosen randomly using the ULID scheme.
+        If a job id already exists in the queue or as a completed job, the request to create one will fail (Bad Request)
+      in: query
+      schema:
+        type: string
+        format: uuid
     IncludeHeader:
       name: include_header
       description: |
@@ -5953,6 +6174,9 @@ components:
           type: string
         error:
           type: string
+        on_failure:
+          # a reference to a script path, flow path, or webhook (script/<path>, flow/<path>)
+          type: string
       required:
         - path
         - edited_by
@@ -5982,6 +6206,9 @@ components:
           $ref: "#/components/schemas/ScriptArgs"
         enabled:
           type: boolean
+        on_failure:
+          # a reference to a script path, flow path, or webhook (script/<path>, flow/<path>)
+          type: string
       required:
         - path
         - schedule
@@ -5999,6 +6226,9 @@ components:
           type: string
         args:
           $ref: "#/components/schemas/ScriptArgs"
+        on_failure:
+          # a reference to a script path, flow path, or webhook (script/<path>, flow/<path>)
+          type: string
       required:
         - schedule
         - timezone
@@ -6269,6 +6499,34 @@ components:
         - edited_at
         - execution_mode
 
+    ListableRawApp:
+      type: object
+      properties:
+        workspace_id:
+          type: string
+        path:
+          type: string
+        summary:
+          type: string
+        extra_perms:
+          type: object
+          additionalProperties:
+            type: boolean
+        starred:
+          type: boolean
+        version:
+          type: number
+        edited_at:
+          type: string
+          format: date-time
+      required:
+        - workspace_id
+        - path
+        - summary
+        - extra_perms
+        - version
+        - edited_at
+
     AppWithLastVersion:
       type: object
       properties:

backend/windmill-api/src/apps.rs

@@ -31,7 +31,7 @@ use windmill_audit::{audit_log, ActionKind};
 use windmill_common::{
     apps::ListAppQuery,
     error::{to_anyhow, Error, JsonResult, Result},
-    jobs::{script_path_to_payload, JobPayload, RawCode},
+    jobs::{get_payload_tag_from_prefixed_path, JobPayload, RawCode},
     users::username_to_permissioned_as,
     utils::{
         http_get_from_hub, list_elems_from_hub, not_found_if_none, paginate, Pagination, StripPath,
@@ -384,6 +384,22 @@ async fn create_app(
         return Err(Error::BadRequest("App path cannot be empty".to_string()));
     }
 
+    let exists = sqlx::query_scalar!(
+        "SELECT EXISTS(SELECT 1 FROM raw_app WHERE path = $1 AND workspace_id = $2)",
+        &app.path,
+        w_id
+    )
+    .fetch_one(&mut tx)
+    .await?
+    .unwrap_or(false);
+
+    if exists {
+        return Err(Error::BadRequest(format!(
+            "App with path {} already exists",
+            &app.path
+        )));
+    }
+
     sqlx::query!(
         "DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'app'",
         &app.path,
@@ -408,9 +424,10 @@ async fn create_app(
     let v_id = sqlx::query_scalar!(
         "INSERT INTO app_version
             (app_id, value, created_by)
-            VALUES ($1, $2, $3) RETURNING id",
+            VALUES ($1, $2::text::json, $3) RETURNING id",
         id,
-        app.value,
+        //to preserve key orders
+        serde_json::to_string(&app.value).unwrap(),
         authed.username,
     )
     .fetch_one(&mut tx)
@@ -536,6 +553,22 @@ async fn update_app(
         if let Some(npath) = &ns.path {
             if npath != path {
                 require_owner_of_path(&authed, path)?;
+
+                let exists = sqlx::query_scalar!(
+                    "SELECT EXISTS(SELECT 1 FROM raw_app WHERE path = $1 AND workspace_id = $2)",
+                    npath,
+                    w_id
+                )
+                .fetch_one(&mut tx)
+                .await?
+                .unwrap_or(false);
+
+                if exists {
+                    return Err(Error::BadRequest(format!(
+                        "App with path {} already exists",
+                        npath
+                    )));
+                }
             }
             sqlb.set_str("path", npath);
         }
@@ -578,9 +611,10 @@ async fn update_app(
         let v_id = sqlx::query_scalar!(
             "INSERT INTO app_version
                 (app_id, value, created_by)
-                VALUES ($1, $2, $3) RETURNING id",
+                VALUES ($1, $2::text::json, $3) RETURNING id",
             app_id,
-            nvalue,
+            //to preserve key orders
+            serde_json::to_string(&nvalue).unwrap(),
             authed.username,
         )
         .fetch_one(&mut tx)
@@ -633,6 +667,7 @@ pub struct ExecuteApp {
     // - script: script/<path>
     // - flow: flow/<path>
     pub path: Option<String>,
+    pub component: String,
     pub raw_code: Option<RawCode>,
     // if set, the app is executed as viewer with the given static fields
     pub force_viewer_static_fields: Option<StaticFields>,
@@ -671,11 +706,16 @@ async fn execute_component(
 
     let policy = if let Some(static_fields) = payload.clone().force_viewer_static_fields {
         let mut hm = HashMap::new();
+
         if let Some(path) = payload.path.clone() {
-            hm.insert(path, static_fields);
+            hm.insert(format!("{}:{path}", payload.component), static_fields);
         } else {
             hm.insert(
-                digest(payload.raw_code.clone().unwrap().content.as_str()),
+                format!(
+                    "{}:{}",
+                    payload.component,
+                    digest(payload.raw_code.clone().unwrap().content.as_str())
+                ),
                 static_fields,
             );
         }
@@ -727,33 +767,17 @@ async fn execute_component(
     };
 
     let (job_payload, args, tag) = match &payload {
-        ExecuteApp { args, raw_code: Some(raw_code), path: None, .. } => {
+        ExecuteApp { args, component, raw_code: Some(raw_code), path: None, .. } => {
             let content = &raw_code.content;
             let payload = JobPayload::Code(raw_code.clone());
             let path = digest(content);
-            let args = build_args(policy, path, args)?;
+            let args = build_args(policy, component, path, args)?;
             (payload, args, None)
         }
-        ExecuteApp { args, raw_code: None, path: Some(path), .. } => {
-            let (payload, tag) = if path.starts_with("script/") {
-                script_path_to_payload(
-                    path.strip_prefix("script/").unwrap(),
-                    tx.transaction_mut(),
-                    &w_id,
-                )
-                .await?
-            } else if path.starts_with("flow/") {
-                (
-                    JobPayload::Flow(path.strip_prefix("flow/").unwrap().to_string()),
-                    None,
-                )
-            } else {
-                return Err(Error::BadRequest(format!(
-                    "path must start with script/ or flow/ (got {})",
-                    path
-                )));
-            };
-            let args = build_args(policy, path.to_string(), args)?;
+        ExecuteApp { args, component, raw_code: None, path: Some(path), .. } => {
+            let (payload, tag) =
+                get_payload_tag_from_prefixed_path(path, tx.transaction_mut(), &w_id).await?;
+            let args = build_args(policy, component, path.to_string(), args)?;
             (payload, args, tag)
         }
         _ => unreachable!(),
@@ -771,6 +795,7 @@ async fn execute_component(
         None,
         None,
         None,
+        None,
         false,
         false,
         None,
@@ -826,15 +851,18 @@ async fn exists_app(
 
 fn build_args(
     policy: Policy,
+    component: &str,
     path: String,
     args: &Map<String, Value>,
 ) -> Result<Map<String, Value>> {
     // disallow var and res access in args coming from the user for security reasons
     args.into_iter()
         .try_for_each(|x| disallow_var_res_access(x.1))?;
+    let key = format!("{}:{}", component, &path);
     let static_args = policy
         .triggerables
-        .get(&path)
+        .get(&key)
+        .or_else(|| policy.triggerables.get(&path))
         .map(|x| x.clone())
         .or_else(|| {
             if matches!(policy.execution_mode, ExecutionMode::Viewer) {

backend/windmill-api/src/drafts.rs

@@ -56,11 +56,12 @@ async fn create_draft(
     sqlx::query!(
         "INSERT INTO draft
             (workspace_id, path, value, typ)
-            VALUES ($1, $2, $3, $4)
-            ON CONFLICT (workspace_id, path, typ) DO UPDATE SET value = $3",
+            VALUES ($1, $2, $3::text::json, $4)
+            ON CONFLICT (workspace_id, path, typ) DO UPDATE SET value = $3::text::json",
         &w_id,
         draft.path,
-        draft.value,
+        //to preserve key orders
+        serde_json::to_string(&draft.value).unwrap(),
         draft.typ: DraftType,
     )
     .execute(&mut tx)

backend/windmill-api/src/favorite.rs

@@ -29,6 +29,8 @@ pub enum FavoriteKind {
     Script,
     Flow,
     App,
+    #[allow(non_camel_case_types)]
+    Raw_App,
 }
 #[derive(Deserialize)]
 pub struct Favorite {

backend/windmill-api/src/flows.rs

@@ -257,6 +257,7 @@ async fn create_flow(
         None,
         None,
         None,
+        None,
         false,
         false,
         None,
@@ -425,6 +426,7 @@ async fn update_flow(
         None,
         None,
         None,
+        None,
         false,
         false,
         None,

backend/windmill-api/src/folders.rs

@@ -405,6 +405,15 @@ async fn get_folder_usage(
     .await?
     .unwrap_or(0);
 
+    let raw_apps = sqlx::query_scalar!(
+        "SELECT count(path) FROM raw_app WHERE path LIKE 'f/' || $1 || '%'  AND workspace_id = $2",
+        name,
+        w_id
+    )
+    .fetch_one(&mut tx)
+    .await?
+    .unwrap_or(0);
+
     let resources = sqlx::query_scalar!(
         "SELECT count(path) FROM resource WHERE path LIKE 'f/' || $1 || '%'  AND workspace_id = $2",
         name,
@@ -428,7 +437,7 @@ async fn get_folder_usage(
         scripts,
         flows,
         schedules,
-        apps,
+        apps: apps + raw_apps,
         resources,
         variables,
     }))

backend/windmill-api/src/inputs.rs

@@ -169,7 +169,7 @@ async fn list_saved_inputs(
     let rows = sqlx::query_as::<_, InputRow>(
         "select * from input \
          where runnable_id = $1 and runnable_type = $2 and workspace_id = $3 \
-         and is_public IS true OR created_by = $4 \
+         and (is_public IS true OR created_by = $4) \
          order by created_at desc limit $5 offset $6",
     )
     .bind(&r.runnable_id)

backend/windmill-api/src/jobs.rs

@@ -17,11 +17,12 @@ use axum::{
     extract::{FromRequest, Json, Path, Query},
     response::{IntoResponse, Response},
     routing::{get, post},
-    Extension, Router,
+    Extension, Form, RequestExt, Router,
 };
 use base64::Engine;
+use bytes::Bytes;
 use hmac::Mac;
-use hyper::{HeaderMap, Request, StatusCode};
+use hyper::{header::CONTENT_TYPE, http, HeaderMap, Request, StatusCode};
 use serde::{de::DeserializeOwned, Deserialize, Serialize};
 use sql_builder::{prelude::*, quote, SqlBuilder};
 use sqlx::{query_scalar, types::Uuid, FromRow, Postgres, Transaction};
@@ -41,11 +42,17 @@ use windmill_queue::{get_queued_job, push, QueueTransaction};
 
 pub fn workspaced_service() -> Router {
     Router::new()
-        .route("/run/f/*script_path", post(run_flow_by_path))
-        .route("/run/p/*script_path", post(run_job_by_path))
+        .route(
+            "/run/f/*script_path",
+            post(run_flow_by_path).head(|| async { "" }),
+        )
+        .route(
+            "/run/p/*script_path",
+            post(run_job_by_path).head(|| async { "" }),
+        )
         .route(
             "/run_wait_result/p/*script_path",
-            post(run_wait_result_job_by_path),
+            post(run_wait_result_job_by_path).head(|| async { "" }),
         )
         .route(
             "/run_wait_result/p/*script_path",
@@ -53,13 +60,13 @@ pub fn workspaced_service() -> Router {
         )
         .route(
             "/run_wait_result/h/:hash",
-            post(run_wait_result_job_by_hash),
+            post(run_wait_result_job_by_hash).head(|| async { "" }),
         )
         .route(
             "/run_wait_result/f/*script_path",
-            post(run_wait_result_flow_by_path),
+            post(run_wait_result_flow_by_path).head(|| async { "" }),
         )
-        .route("/run/h/:hash", post(run_job_by_hash))
+        .route("/run/h/:hash", post(run_job_by_hash).head(|| async { "" }))
         .route("/run/preview", post(run_preview_job))
         .route("/run/preview_flow", post(run_preview_flow_job))
         .route("/list", get(list_jobs))
@@ -68,6 +75,10 @@ pub fn workspaced_service() -> Router {
         .route("/completed/list", get(list_completed_jobs))
         .route("/completed/get/:id", get(get_completed_job))
         .route("/completed/get_result/:id", get(get_completed_job_result))
+        .route(
+            "/completed/get_result_maybe/:id",
+            get(get_completed_job_result_maybe),
+        )
         .route("/completed/delete/:id", post(delete_completed_job))
         .route("/flow/resume/:id", post(resume_suspended_flow_as_owner))
         .route(
@@ -101,6 +112,12 @@ pub fn global_service() -> Router {
             get(get_suspended_job_flow),
         )
         .route("/get/:id", get(get_job))
+        .route("/completed/get/:id", get(get_completed_job))
+        .route("/completed/get_result/:id", get(get_completed_job_result))
+        .route(
+            "/completed/get_result_maybe/:id",
+            get(get_completed_job_result_maybe),
+        )
         .route("/getupdate/:id", get(get_job_update))
         .route("/queue/cancel/:id", post(cancel_job_api))
         .route("/queue/force_cancel/:id", post(force_cancel))
@@ -338,6 +355,7 @@ pub struct RunJobQuery {
     invisible_to_owner: Option<bool>,
     queue_limit: Option<i64>,
     payload: Option<String>,
+    job_id: Option<Uuid>,
 }
 
 lazy_static::lazy_static! {
@@ -693,7 +711,7 @@ pub async fn resume_suspended_job(
     }
     mac.verify_slice(hex::decode(secret)?.as_ref())
         .map_err(|_| anyhow::anyhow!("Invalid signature"))?;
-    let flow = get_suspended_prent_flow_info(job_id, &mut tx).await?;
+    let flow = get_suspended_parent_flow_info(job_id, &mut tx).await?;
 
     let exists = sqlx::query_scalar!(
         r#"
@@ -784,7 +802,7 @@ struct FlowInfo {
     script_path: Option<String>,
 }
 
-async fn get_suspended_prent_flow_info<'c>(
+async fn get_suspended_parent_flow_info<'c>(
     job_id: Uuid,
     tx: &mut Transaction<'c, Postgres>,
 ) -> error::Result<FlowInfo> {
@@ -855,8 +873,9 @@ pub async fn cancel_suspended_job(
         .map_err(|_| anyhow::anyhow!("Invalid signature"))?;
 
     let whom = approver.approver.unwrap_or_else(|| "unknown".to_string());
-    let parent_flow = get_root_job(db, &w_id, job).await?;
-    let (mut tx, job) = windmill_queue::cancel_job(
+    let parent_flow = get_suspended_parent_flow_info(job, &mut tx).await?.id;
+
+    let (mut tx, cjob) = windmill_queue::cancel_job(
         &whom,
         Some("approval request disapproved".to_string()),
         parent_flow,
@@ -866,7 +885,7 @@ pub async fn cancel_suspended_job(
         false,
     )
     .await?;
-    if job.is_some() {
+    if cjob.is_some() {
         audit_log(
             &mut tx,
             &whom,
@@ -877,26 +896,13 @@ pub async fn cancel_suspended_job(
             None,
         )
         .await?;
-    }
-    tx.commit().await?;
-    Ok("Flow of job cancelled".to_string())
-}
+        tx.commit().await?;
 
-pub async fn get_root_job(db: DB, w_id: &str, job: Uuid) -> error::Result<Uuid> {
-    let mut tx = db.begin().await?;
-    let mut job_id = job;
-    loop {
-        let (job, ntx) = get_job_by_id(tx, w_id, job_id).await?;
-        tx = ntx;
-        let p_job = job.and_then(|x| match x {
-            Job::QueuedJob(job) => job.parent_job,
-            Job::CompletedJob(job) => job.parent_job,
-        });
-        if let Some(p_job) = p_job {
-            job_id = p_job;
-        } else {
-            return Ok(job_id);
-        }
+        Ok(format!("Flow {parent_flow} of job {job} cancelled"))
+    } else {
+        Ok(format!(
+            "Flow {parent_flow} of job {job} was not cancellable"
+        ))
     }
 }
 
@@ -1211,6 +1217,75 @@ struct PreviewFlow {
     args: Option<serde_json::Map<String, serde_json::Value>>,
 }
 
+pub struct JsonOrForm(
+    Option<serde_json::Map<String, serde_json::Value>>,
+    Option<String>,
+);
+
+#[axum::async_trait]
+impl<S> FromRequest<S, axum::body::Body> for JsonOrForm
+where
+    S: Send + Sync,
+{
+    type Rejection = Response;
+
+    async fn from_request(
+        req: Request<axum::body::Body>,
+        _state: &S,
+    ) -> Result<Self, Self::Rejection> {
+        let content_type_header = req.headers().get(CONTENT_TYPE);
+        let content_type = content_type_header.and_then(|value| value.to_str().ok());
+        if let Some(content_type) = content_type {
+            if content_type.starts_with("application/json") {
+                if req
+                    .uri()
+                    .query()
+                    .map(|x| x.contains("raw=true"))
+                    .unwrap_or(false)
+                {
+                    let bytes = Bytes::from_request(req, _state)
+                        .await
+                        .map_err(IntoResponse::into_response)?;
+                    let str = String::from_utf8(bytes.to_vec()).map_err(|e| {
+                        Error::BadRequest(format!("invalid utf8: {}", e)).into_response()
+                    })?;
+                    let payload =
+                        serde_json::from_str::<Option<serde_json::Value>>(&str).map_err(|e| {
+                            Error::BadRequest(format!("invalid json: {}", e)).into_response()
+                        })?;
+                    return match payload {
+                        Some(serde_json::Value::Object(map)) => Ok(Self(Some(map), Some(str))),
+                        None => Ok(Self(None, Some(str))),
+                        Some(x) => {
+                            let mut map = serde_json::Map::new();
+                            map.insert("body".to_string(), x);
+                            Ok(Self(Some(map), Some(str)))
+                        }
+                    };
+                } else {
+                    let Json(payload): Json<Option<serde_json::Value>> =
+                        req.extract().await.map_err(IntoResponse::into_response)?;
+                    return match payload {
+                        Some(serde_json::Value::Object(map)) => Ok(Self(Some(map), None)),
+                        None => Ok(Self(None, None)),
+                        Some(x) => {
+                            let mut map = serde_json::Map::new();
+                            map.insert("body".to_string(), x);
+                            Ok(Self(Some(map), None))
+                        }
+                    };
+                }
+            }
+
+            if content_type.starts_with("application/x-www-form-urlencoded") {
+                let Form(payload) = req.extract().await.map_err(IntoResponse::into_response)?;
+                return Ok(Self(Some(payload), None));
+            }
+        }
+
+        Err(StatusCode::UNSUPPORTED_MEDIA_TYPE.into_response())
+    }
+}
 pub struct QueryOrBody<D>(pub Option<D>);
 
 #[axum::async_trait]
@@ -1257,6 +1332,19 @@ fn decode_payload<D: DeserializeOwned>(t: String) -> anyhow::Result<D> {
         .context("invalid base64")?;
     serde_json::from_slice(vec.as_slice()).context("invalid json")
 }
+
+fn add_raw_string(
+    raw_string: Option<String>,
+    mut args: serde_json::Map<String, serde_json::Value>,
+) -> serde_json::Map<String, serde_json::Value> {
+    if let Some(raw_string) = raw_string {
+        args.insert(
+            "raw_string".to_string(),
+            serde_json::Value::String(raw_string),
+        );
+    }
+    return args;
+}
 pub async fn run_flow_by_path(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
@@ -1264,13 +1352,13 @@ pub async fn run_flow_by_path(
     Path((w_id, flow_path)): Path<(String, StripPath)>,
     Query(run_query): Query<RunJobQuery>,
     headers: HeaderMap,
-    Json(args): Json<Option<serde_json::Map<String, serde_json::Value>>>,
+    JsonOrForm(args, raw_string): JsonOrForm,
 ) -> error::Result<(StatusCode, String)> {
     let flow_path = flow_path.to_path();
     let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
     let scheduled_for = run_query.get_scheduled_for(tx.transaction_mut()).await?;
     let args = run_query.add_include_headers(headers, args.unwrap_or_default());
-
+    let args = add_raw_string(raw_string, args);
     let (uuid, tx) = push(
         tx,
         &w_id,
@@ -1283,6 +1371,7 @@ pub async fn run_flow_by_path(
         None,
         run_query.parent_job,
         run_query.parent_job,
+        run_query.job_id,
         false,
         false,
         None,
@@ -1301,7 +1390,7 @@ pub async fn run_job_by_path(
     Path((w_id, script_path)): Path<(String, StripPath)>,
     Query(run_query): Query<RunJobQuery>,
     headers: HeaderMap,
-    Json(args): Json<Option<serde_json::Map<String, serde_json::Value>>>,
+    JsonOrForm(args, raw_string): JsonOrForm,
 ) -> error::Result<(StatusCode, String)> {
     let script_path = script_path.to_path();
     let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
@@ -1309,6 +1398,7 @@ pub async fn run_job_by_path(
         script_path_to_payload(script_path, tx.transaction_mut(), &w_id).await?;
     let scheduled_for = run_query.get_scheduled_for(tx.transaction_mut()).await?;
     let args = run_query.add_include_headers(headers, args.unwrap_or_default());
+    let args = add_raw_string(raw_string, args);
 
     let (uuid, tx) = push(
         tx,
@@ -1321,6 +1411,7 @@ pub async fn run_job_by_path(
         scheduled_for,
         None,
         run_query.parent_job,
+        run_query.job_id,
         run_query.parent_job,
         false,
         false,
@@ -1422,6 +1513,13 @@ async fn run_wait_result<T>(
     }
     if let Some(result) = result {
         g.done = true;
+        let status_code = result
+            .get("windmill_status_code")
+            .and_then(|x| x.as_i64())
+            .and_then(|x| StatusCode::from_u16(x as u16).ok());
+        if let Some(status_code) = status_code {
+            return Err(Error::CustomStatusCode(status_code, result));
+        }
         Ok(Json(result))
     } else {
         Err(Error::ExecutionErr(format!("timeout after {}s", timeout)))
@@ -1472,6 +1570,7 @@ lazy_static::lazy_static! {
 }
 
 pub async fn run_wait_result_job_by_path_get(
+    method: hyper::http::Method,
     authed: Authed,
     Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
     Extension(user_db): Extension<UserDB>,
@@ -1479,6 +1578,9 @@ pub async fn run_wait_result_job_by_path_get(
     Path((w_id, script_path)): Path<(String, StripPath)>,
     Query(run_query): Query<RunJobQuery>,
 ) -> error::JsonResult<serde_json::Value> {
+    if method == http::Method::HEAD {
+        return Ok(Json(serde_json::json!("")));
+    }
     let payload_r = run_query
         .payload
         .map(decode_payload)
@@ -1508,6 +1610,7 @@ pub async fn run_wait_result_job_by_path_get(
         None,
         run_query.parent_job,
         run_query.parent_job,
+        run_query.job_id,
         false,
         false,
         None,
@@ -1535,7 +1638,7 @@ pub async fn run_wait_result_job_by_path(
     Path((w_id, script_path)): Path<(String, StripPath)>,
     Query(run_query): Query<RunJobQuery>,
     headers: HeaderMap,
-    Json(args): Json<Option<serde_json::Map<String, serde_json::Value>>>,
+    JsonOrForm(args, raw_string): JsonOrForm,
 ) -> error::JsonResult<serde_json::Value> {
     check_queue_too_long(db, QUEUE_LIMIT_WAIT_RESULT.or(run_query.queue_limit)).await?;
     let script_path = script_path.to_path();
@@ -1544,6 +1647,7 @@ pub async fn run_wait_result_job_by_path(
         script_path_to_payload(script_path, tx.transaction_mut(), &w_id).await?;
 
     let args = run_query.add_include_headers(headers, args.unwrap_or_default());
+    let args = add_raw_string(raw_string, args);
 
     let (uuid, tx) = push(
         tx,
@@ -1557,6 +1661,7 @@ pub async fn run_wait_result_job_by_path(
         None,
         run_query.parent_job,
         run_query.parent_job,
+        run_query.job_id,
         false,
         false,
         None,
@@ -1584,7 +1689,7 @@ pub async fn run_wait_result_job_by_hash(
     Path((w_id, script_hash)): Path<(String, ScriptHash)>,
     Query(run_query): Query<RunJobQuery>,
     headers: HeaderMap,
-    Json(args): Json<Option<serde_json::Map<String, serde_json::Value>>>,
+    JsonOrForm(args, raw_string): JsonOrForm,
 ) -> error::JsonResult<serde_json::Value> {
     check_queue_too_long(db, run_query.queue_limit).await?;
 
@@ -1593,6 +1698,7 @@ pub async fn run_wait_result_job_by_hash(
     let (path, tag) = get_path_and_tag_for_hash(tx.transaction_mut(), &w_id, hash).await?;
 
     let args = run_query.add_include_headers(headers, args.unwrap_or_default());
+    let args = add_raw_string(raw_string, args);
 
     let (uuid, tx) = push(
         tx,
@@ -1606,6 +1712,7 @@ pub async fn run_wait_result_job_by_hash(
         None,
         run_query.parent_job,
         run_query.parent_job,
+        run_query.job_id,
         false,
         false,
         None,
@@ -1633,7 +1740,7 @@ pub async fn run_wait_result_flow_by_path(
     Path((w_id, flow_path)): Path<(String, StripPath)>,
     Query(run_query): Query<RunJobQuery>,
     headers: HeaderMap,
-    Json(args): Json<Option<serde_json::Map<String, serde_json::Value>>>,
+    JsonOrForm(args, raw_string): JsonOrForm,
 ) -> error::JsonResult<serde_json::Value> {
     check_queue_too_long(db, run_query.queue_limit).await?;
 
@@ -1641,6 +1748,7 @@ pub async fn run_wait_result_flow_by_path(
     let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.clone().begin(&authed).await?).into();
     let scheduled_for = run_query.get_scheduled_for(tx.transaction_mut()).await?;
     let args = run_query.add_include_headers(headers, args.unwrap_or_default());
+    let args = add_raw_string(raw_string, args);
 
     let (uuid, tx) = push(
         tx,
@@ -1654,6 +1762,7 @@ pub async fn run_wait_result_flow_by_path(
         None,
         run_query.parent_job,
         run_query.parent_job,
+        run_query.job_id,
         false,
         false,
         None,
@@ -1703,6 +1812,7 @@ async fn run_preview_job(
         None,
         None,
         None,
+        run_query.job_id,
         false,
         false,
         None,
@@ -1740,6 +1850,7 @@ async fn run_preview_flow_job(
         None,
         None,
         None,
+        run_query.job_id,
         false,
         false,
         None,
@@ -1759,13 +1870,14 @@ pub async fn run_job_by_hash(
     Path((w_id, script_hash)): Path<(String, ScriptHash)>,
     Query(run_query): Query<RunJobQuery>,
     headers: HeaderMap,
-    Json(args): Json<Option<serde_json::Map<String, serde_json::Value>>>,
+    JsonOrForm(args, raw_string): JsonOrForm,
 ) -> error::Result<(StatusCode, String)> {
     let hash = script_hash.0;
     let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
     let (path, tag) = get_path_and_tag_for_hash(tx.transaction_mut(), &w_id, hash).await?;
     let scheduled_for = run_query.get_scheduled_for(tx.transaction_mut()).await?;
     let args = run_query.add_include_headers(headers, args.unwrap_or_default());
+    let args = add_raw_string(raw_string, args);
 
     let (uuid, tx) = push(
         tx,
@@ -1779,6 +1891,7 @@ pub async fn run_job_by_hash(
         None,
         run_query.parent_job,
         run_query.parent_job,
+        run_query.job_id,
         false,
         false,
         None,
@@ -2023,6 +2136,31 @@ async fn get_completed_job_result(
     Ok(Json(result))
 }
 
+#[derive(Serialize)]
+struct CompletedJobResult {
+    completed: bool,
+    result: Option<serde_json::Value>,
+}
+
+async fn get_completed_job_result_maybe(
+    Extension(db): Extension<DB>,
+    Path((w_id, id)): Path<(String, Uuid)>,
+) -> error::JsonResult<CompletedJobResult> {
+    let result_o = sqlx::query_scalar!(
+        "SELECT result FROM completed_job WHERE id = $1 AND workspace_id = $2",
+        id,
+        w_id,
+    )
+    .fetch_optional(&db)
+    .await?;
+
+    if let Some(result) = result_o {
+        Ok(Json(CompletedJobResult { completed: true, result }))
+    } else {
+        Ok(Json(CompletedJobResult { completed: false, result: None }))
+    }
+}
+
 async fn delete_completed_job(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,

backend/windmill-api/src/lib.rs

@@ -15,6 +15,7 @@ use crate::{
     webhook_util::WebhookShared,
 };
 use argon2::Argon2;
+use axum::extract::DefaultBodyLimit;
 use axum::{middleware::from_extractor, routing::get, Extension, Router};
 use db::DB;
 use git_version::git_version;
@@ -42,6 +43,7 @@ mod groups;
 mod inputs;
 pub mod jobs;
 mod oauth2;
+mod raw_apps;
 mod resources;
 mod schedule;
 mod scripts;
@@ -59,9 +61,15 @@ pub const GIT_VERSION: &str =
 
 pub use users::delete_expired_items_perdiodically;
 
+pub const DEFAULT_BODY_LIMIT: usize = 2097152; // 2MB
 lazy_static::lazy_static! {
     pub static ref BASE_URL: String = std::env::var("BASE_URL").unwrap_or_else(|_| "http://localhost".to_string());
 
+    pub static ref REQUEST_SIZE_LIMIT: usize = std::env::var("REQUEST_SIZE_LIMIT")
+    .ok()
+    .and_then(|x| x.parse::<usize>().ok())
+    .unwrap_or(DEFAULT_BODY_LIMIT);
+
 
     pub static ref COOKIE_DOMAIN: Option<String> = std::env::var("COOKIE_DOMAIN").ok();
 
@@ -78,6 +86,8 @@ lazy_static::lazy_static! {
     pub static ref OAUTH_CLIENTS: AllClients = build_oauth_clients(&BASE_URL)
         .map_err(|e| tracing::error!("Error building oauth clients: {}", e))
         .unwrap();
+
+    pub static ref LICENSE_KEY: Option<String> = std::env::var("LICENSE_KEY").ok();
 }
 
 pub async fn run_server(
@@ -106,7 +116,8 @@ pub async fn run_server(
         .layer(Extension(user_db))
         .layer(Extension(auth_cache.clone()))
         .layer(CookieManagerLayer::new())
-        .layer(Extension(WebhookShared::new(rx.resubscribe(), db.clone())));
+        .layer(Extension(WebhookShared::new(rx.resubscribe(), db.clone())))
+        .layer(DefaultBodyLimit::max(*REQUEST_SIZE_LIMIT));
 
     let cors = CorsLayer::new()
         .allow_methods([Method::GET, Method::POST])
@@ -123,6 +134,7 @@ pub async fn run_server(
                         // Reordered alphabetically
                         .nest("/acls", granular_acls::workspaced_service())
                         .nest("/apps", apps::workspaced_service())
+                        .nest("/raw_apps", raw_apps::workspaced_service())
                         .nest("/audit", audit::workspaced_service())
                         .nest("/capture", capture::workspaced_service())
                         .nest("/favorites", favorite::workspaced_service())
@@ -176,6 +188,7 @@ pub async fn run_server(
                 )
                 .nest("/oauth", oauth2::global_service())
                 .route("/version", get(git_v))
+                .route("/ee_license", get(ee_license))
                 .route("/openapi.yaml", get(openapi)),
         )
         .fallback(static_assets::static_handler)
@@ -197,8 +210,30 @@ pub async fn run_server(
     Ok(())
 }
 
-async fn git_v() -> &'static str {
-    GIT_VERSION
+#[cfg(feature = "enterprise")]
+async fn git_v() -> String {
+    format!("EE {GIT_VERSION}")
+}
+
+#[cfg(not(feature = "enterprise"))]
+async fn git_v() -> String {
+    format!("CE {GIT_VERSION}")
+}
+
+#[cfg(feature = "enterprise")]
+async fn ee_license() -> &'static str {
+    ""
+}
+
+#[cfg(not(feature = "enterprise"))]
+async fn ee_license() -> String {
+    LICENSE_KEY
+        .as_ref()
+        .unwrap()
+        .split(".")
+        .next()
+        .unwrap()
+        .to_string()
 }
 
 async fn openapi() -> &'static str {

backend/windmill-api/src/oauth2.rs

@@ -19,6 +19,7 @@ use axum::{
     routing::{get, post},
     Json, Router,
 };
+use base64::Engine;
 use hmac::Mac;
 use hyper::StatusCode;
 use itertools::Itertools;
@@ -119,7 +120,14 @@ pub fn build_oauth_clients(base_url: &str) -> anyhow::Result<AllClients> {
     ))?;
 
     let path = "./oauth.json";
-    let content = if std::path::Path::new(path).exists() {
+    let content: String = if let Ok(e) = std::env::var("OAUTH_JSON_AS_BASE64") {
+        str::from_utf8(
+            &base64::engine::general_purpose::STANDARD
+                .decode(e)
+                .map_err(to_anyhow)?,
+        )?
+        .to_string()
+    } else if std::path::Path::new(path).exists() {
         fs::read_to_string(path).map_err(to_anyhow)?
     } else {
         "{}".to_string()
@@ -788,6 +796,7 @@ async fn slack_command(
                 None,
                 None,
                 None,
+                None,
                 false,
                 false,
                 None,

backend/windmill-api/src/raw_apps.rs

@@ -0,0 +1,333 @@
+/*
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
+ * This file and its contents are licensed under the AGPLv3 License.
+ * Please see the included NOTICE for copyright information and
+ * LICENSE-AGPL for a copy of the license.
+ */
+use crate::{
+    db::{UserDB, DB},
+    users::{require_owner_of_path, Authed},
+    webhook_util::{WebhookMessage, WebhookShared},
+};
+use axum::{
+    body,
+    extract::{Extension, Json, Path, Query},
+    response::Response,
+    routing::{delete, get, post},
+    Router,
+};
+use bytes::Bytes;
+use hyper::{header, StatusCode};
+use serde::{Deserialize, Serialize};
+use sql_builder::{bind::Bind, SqlBuilder};
+use sqlx::FromRow;
+use std::str;
+use windmill_audit::{audit_log, ActionKind};
+use windmill_common::{
+    apps::ListAppQuery,
+    error::{Error, JsonResult, Result},
+    utils::{not_found_if_none, paginate, Pagination, StripPath},
+};
+
+pub fn workspaced_service() -> Router {
+    Router::new()
+        .route("/list", get(list_apps))
+        .route("/get_data/:version/*path", get(get_data))
+        .route("/exists/*path", get(exists_app))
+        .route("/update/*path", post(update_app))
+        .route("/delete/*path", delete(delete_app))
+        .route("/create", post(create_app))
+}
+
+#[derive(FromRow, Deserialize, Serialize)]
+pub struct ListableApp {
+    pub path: String,
+    pub workspace_id: String,
+    pub summary: String,
+    pub edited_at: chrono::DateTime<chrono::Utc>,
+    pub extra_perms: serde_json::Value,
+    pub starred: bool,
+    pub version: i32,
+}
+
+#[derive(Deserialize)]
+pub struct CreateApp {
+    pub path: String,
+    pub summary: String,
+    pub value: String,
+}
+
+#[derive(Deserialize)]
+pub struct EditApp {
+    pub path: Option<String>,
+    pub summary: Option<String>,
+    pub value: Option<String>,
+}
+
+async fn list_apps(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    Path(w_id): Path<String>,
+    Query(pagination): Query<Pagination>,
+    Query(lq): Query<ListAppQuery>,
+) -> JsonResult<Vec<ListableApp>> {
+    let (per_page, offset) = paginate(pagination);
+
+    let mut sqlb = SqlBuilder::select_from("raw_app as app")
+        .fields(&[
+            "app.workspace_id",
+            "app.path",
+            "app.edited_at",
+            "app.summary",
+            "app.extra_perms",
+            "app.version",
+            "favorite.path IS NOT NULL as starred",
+        ])
+        .left()
+        .join("favorite")
+        .on(
+            "favorite.favorite_kind = 'raw_app' AND favorite.workspace_id = app.workspace_id AND favorite.path = app.path AND favorite.usr = ?"
+                .bind(&authed.username),
+        )
+        .order_desc("favorite.path IS NOT NULL")
+        .order_by("app.edited_at", true)
+        .and_where("app.workspace_id = ?".bind(&w_id))
+        .offset(offset)
+        .limit(per_page)
+        .clone();
+
+    if lq.starred_only.unwrap_or(false) {
+        sqlb.and_where_is_not_null("favorite.path");
+    }
+
+    let sql = sqlb.sql().map_err(|e| Error::InternalErr(e.to_string()))?;
+    let mut tx = user_db.begin(&authed).await?;
+    let rows = sqlx::query_as::<_, ListableApp>(&sql)
+        .fetch_all(&mut tx)
+        .await?;
+
+    tx.commit().await?;
+
+    Ok(Json(rows))
+}
+
+async fn get_data(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    Path((w_id, _version, path)): Path<(String, u16, StripPath)>,
+) -> Result<Response> {
+    let path = path.to_path();
+    let mut tx = user_db.begin(&authed).await?;
+
+    let app_o = sqlx::query_scalar!(
+        "SELECT data FROM raw_app
+        WHERE path = $1 AND workspace_id = $2",
+        path.to_owned(),
+        &w_id
+    )
+    .fetch_optional(&mut tx)
+    .await?;
+    tx.commit().await?;
+
+    let app = not_found_if_none(app_o, "App", path)?;
+    let res = Response::builder().header(header::CONTENT_TYPE, "text/javascript");
+
+    Ok(res
+        .body(body::boxed(body::Full::from(Bytes::from(app))))
+        .unwrap())
+}
+
+async fn create_app(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    Extension(webhook): Extension<WebhookShared>,
+    Path(w_id): Path<String>,
+    Json(app): Json<CreateApp>,
+) -> Result<(StatusCode, String)> {
+    let mut tx = user_db.begin(&authed).await?;
+    if &app.path == "" {
+        return Err(Error::BadRequest("App path cannot be empty".to_string()));
+    }
+
+    let exists = sqlx::query_scalar!(
+        "SELECT EXISTS(SELECT 1 FROM app WHERE path = $1 AND workspace_id = $2)",
+        app.path,
+        w_id
+    )
+    .fetch_one(&mut tx)
+    .await?
+    .unwrap_or(false);
+
+    if exists {
+        return Err(Error::BadRequest(format!(
+            "App with path {} already exists",
+            &app.path
+        )));
+    }
+
+    sqlx::query!(
+        "INSERT INTO raw_app
+            (workspace_id, path, summary, extra_perms, data)
+            VALUES ($1, $2, $3, '{}', $4)",
+        w_id,
+        app.path,
+        app.summary,
+        app.value,
+    )
+    .execute(&mut tx)
+    .await?;
+
+    audit_log(
+        &mut tx,
+        &authed.username,
+        "apps.create",
+        ActionKind::Create,
+        &w_id,
+        Some(&app.path),
+        None,
+    )
+    .await?;
+
+    tx.commit().await?;
+    webhook.send_message(
+        w_id.clone(),
+        WebhookMessage::CreateApp { workspace: w_id, path: app.path.clone() },
+    );
+
+    Ok((StatusCode::CREATED, app.path))
+}
+
+async fn delete_app(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    Extension(webhook): Extension<WebhookShared>,
+    Path((w_id, path)): Path<(String, StripPath)>,
+) -> Result<String> {
+    let path = path.to_path();
+    let mut tx = user_db.begin(&authed).await?;
+
+    sqlx::query!(
+        "DELETE FROM raw_app WHERE path = $1 AND workspace_id = $2",
+        path,
+        w_id
+    )
+    .execute(&mut tx)
+    .await?;
+    audit_log(
+        &mut tx,
+        &authed.username,
+        "apps.delete",
+        ActionKind::Delete,
+        &w_id,
+        Some(path),
+        None,
+    )
+    .await?;
+    tx.commit().await?;
+    webhook.send_message(
+        w_id.clone().clone(),
+        WebhookMessage::DeleteApp { workspace: w_id, path: path.to_owned() },
+    );
+
+    Ok(format!("app {} deleted", path))
+}
+
+async fn update_app(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    Extension(webhook): Extension<WebhookShared>,
+    Path((w_id, path)): Path<(String, StripPath)>,
+    Json(app): Json<EditApp>,
+) -> Result<String> {
+    use sql_builder::prelude::*;
+
+    let path = path.to_path();
+
+    let mut tx = user_db.begin(&authed).await?;
+    let npath = &app.path;
+    let npath = if npath.is_some() || app.summary.is_some() {
+        let mut sqlb = SqlBuilder::update_table("raw_app");
+        sqlb.and_where_eq("path", "?".bind(&path));
+        sqlb.and_where_eq("workspace_id", "?".bind(&w_id));
+
+        if let Some(npath) = &app.path {
+            if npath != path {
+                require_owner_of_path(&authed, path)?;
+
+                let exists = sqlx::query_scalar!(
+                    "SELECT EXISTS(SELECT 1 FROM app WHERE path = $1 AND workspace_id = $2)",
+                    npath,
+                    w_id
+                )
+                .fetch_one(&mut tx)
+                .await?
+                .unwrap_or(false);
+
+                if exists {
+                    return Err(Error::BadRequest(format!(
+                        "App with path {} already exists",
+                        npath
+                    )));
+                }
+            }
+            sqlb.set_str("path", npath);
+        }
+
+        if let Some(nsummary) = &app.summary {
+            sqlb.set_str("summary", nsummary);
+        }
+
+        if let Some(value) = &app.value {
+            sqlb.set_str("data", value);
+            sqlb.set("version", "version + 1");
+        }
+
+        sqlb.returning("path");
+
+        let sql = sqlb.sql().map_err(|e| Error::InternalErr(e.to_string()))?;
+        let npath_o: Option<String> = sqlx::query_scalar(&sql).fetch_optional(&mut tx).await?;
+        not_found_if_none(npath_o, "Raw App", path)?
+    } else {
+        "".to_string()
+    };
+
+    audit_log(
+        &mut tx,
+        &authed.username,
+        "apps.update",
+        ActionKind::Update,
+        &w_id,
+        Some(&npath),
+        None,
+    )
+    .await?;
+    tx.commit().await?;
+    webhook.send_message(
+        w_id.clone(),
+        WebhookMessage::UpdateApp {
+            workspace: w_id,
+            old_path: path.to_owned(),
+            new_path: npath.clone(),
+        },
+    );
+
+    Ok(format!("app {} updated (npath: {:?})", path, npath))
+}
+
+async fn exists_app(
+    Extension(db): Extension<DB>,
+    Path((w_id, path)): Path<(String, StripPath)>,
+) -> JsonResult<bool> {
+    let path = path.to_path();
+    let exists = sqlx::query_scalar!(
+        "SELECT EXISTS(SELECT 1 FROM raw_app WHERE path = $1 AND workspace_id = $2)",
+        path,
+        w_id
+    )
+    .fetch_one(&db)
+    .await?
+    .unwrap_or(false);
+
+    Ok(Json(exists))
+}

backend/windmill-api/src/schedule.rs

@@ -22,8 +22,9 @@ use std::str::FromStr;
 use windmill_audit::{audit_log, ActionKind};
 use windmill_common::{
     error::{Error, JsonResult, Result},
+    jobs::JobKind,
     schedule::Schedule,
-    utils::{not_found_if_none, paginate, Pagination, StripPath}, jobs::JobKind,
+    utils::{not_found_if_none, paginate, Pagination, StripPath},
 };
 use windmill_queue::{self, schedule::push_scheduled_job, QueueTransaction};
 
@@ -51,6 +52,7 @@ pub struct NewSchedule {
     pub is_flow: bool,
     pub args: Option<serde_json::Value>,
     pub enabled: Option<bool>,
+    pub on_failure: Option<String>,
 }
 
 async fn check_path_conflict<'c>(
@@ -100,7 +102,7 @@ async fn create_schedule(
     let schedule = sqlx::query_as!(
         Schedule,
         "INSERT INTO schedule (workspace_id, path, schedule, timezone, edited_by, script_path, \
-         is_flow, args, enabled, email) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) RETURNING *",
+         is_flow, args, enabled, email, on_failure) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) RETURNING *",
         w_id,
         ns.path,
         ns.schedule,
@@ -110,7 +112,8 @@ async fn create_schedule(
         ns.is_flow,
         ns.args,
         ns.enabled.unwrap_or(false),
-        &authed.email
+        &authed.email,
+        ns.on_failure
     )
     .fetch_one(&mut tx)
     .await
@@ -154,7 +157,7 @@ async fn edit_schedule(
     let path = path.to_path();
 
     let authed = maybe_refresh_folders(&path, &w_id, authed, &db).await;
-      let mut tx: QueueTransaction<'_, rsmq_async::MultiplexedRsmq> =
+    let mut tx: QueueTransaction<'_, rsmq_async::MultiplexedRsmq> =
         (rsmq, user_db.begin(&authed).await?).into();
 
     cron::Schedule::from_str(&es.schedule).map_err(|e| Error::BadRequest(e.to_string()))?;
@@ -170,11 +173,12 @@ async fn edit_schedule(
     clear_schedule(tx.transaction_mut(), path, is_flow).await?;
     let schedule = sqlx::query_as!(
         Schedule,
-        "UPDATE schedule SET schedule = $1, timezone = $2, args = $3 WHERE path \
-         = $4 AND workspace_id = $5 RETURNING *",
+        "UPDATE schedule SET schedule = $1, timezone = $2, args = $3, on_failure = $4 WHERE path \
+         = $5 AND workspace_id = $6 RETURNING *",
         es.schedule,
         es.timezone,
         es.args,
+        es.on_failure,
         path,
         w_id,
     )
@@ -386,6 +390,7 @@ pub struct EditSchedule {
     pub schedule: String,
     pub timezone: String,
     pub args: Option<serde_json::Value>,
+    pub on_failure: Option<String>,
 }
 
 pub async fn clear_schedule<'c>(

backend/windmill-api/src/scripts.rs

@@ -489,6 +489,7 @@ async fn create_script(
             None,
             None,
             None,
+            None,
             false,
             false,
             None,
@@ -600,7 +601,22 @@ async fn raw_script_by_path(
     Extension(user_db): Extension<UserDB>,
     Path((w_id, path)): Path<(String, StripPath)>,
 ) -> Result<String> {
-    let path = path.to_path().split(".").next().unwrap_or_default();
+    let path = path.to_path();
+    if !path.ends_with(".py")
+        && !path.ends_with(".ts")
+        && !path.ends_with(".go")
+        && !path.ends_with(".sh")
+    {
+        return Err(Error::BadRequest(format!(
+            "Path must ends with a .py, .ts, .go. or .sh extension: {}",
+            path
+        )));
+    }
+    let path = path
+        .trim_end_matches(".py")
+        .trim_end_matches(".ts")
+        .trim_end_matches(".go")
+        .trim_end_matches(".sh");
     let mut tx = user_db.begin(&authed).await?;
 
     let content_o = sqlx::query_scalar!(

backend/windmill-api/src/variables.rs

@@ -337,15 +337,20 @@ async fn update_variable(
     if let Some(npath) = &ns.path {
         sqlb.set_str("path", npath);
     }
+    let ns_value_is_none = ns.value.is_none();
     if let Some(nvalue) = ns.value {
-        let is_secret = sqlx::query_scalar!(
-            "SELECT is_secret from variable WHERE path = $1 AND workspace_id = $2",
-            &path,
-            &w_id
-        )
-        .fetch_optional(&mut tx)
-        .await?
-        .unwrap_or(false);
+        let is_secret = if ns.is_secret.is_some() {
+            ns.is_secret.unwrap()
+        } else {
+            sqlx::query_scalar!(
+                "SELECT is_secret from variable WHERE path = $1 AND workspace_id = $2",
+                &path,
+                &w_id
+            )
+            .fetch_optional(&mut tx)
+            .await?
+            .unwrap_or(false)
+        };
 
         let value = if is_secret && !already_encrypted.unwrap_or(false) {
             let mc = build_crypt(&mut tx, &w_id).await?;
@@ -361,9 +366,17 @@ async fn update_variable(
     }
 
     if let Some(nbool) = ns.is_secret {
-        if !nbool {
+        let old_secret = sqlx::query_scalar!(
+            "SELECT is_secret from variable WHERE path = $1 AND workspace_id = $2",
+            &path,
+            &w_id
+        )
+        .fetch_optional(&mut tx)
+        .await?
+        .unwrap_or(false);
+        if old_secret != nbool && ns_value_is_none {
             return Err(Error::BadRequest(
-                "A variable can not be updated to be non secret".to_owned(),
+                "cannot change is_secret without updating value too".to_string(),
             ));
         }
         sqlb.set_str("is_secret", nbool);

backend/windmill-common/src/error.rs

@@ -13,6 +13,7 @@ use axum::{
     response::Json,
 };
 
+use hyper::StatusCode;
 #[cfg(feature = "sqlx")]
 use sqlx::migrate::MigrateError;
 use thiserror::Error;
@@ -60,6 +61,8 @@ pub enum Error {
     Anyhow(#[from] anyhow::Error),
     #[error("Error: {0:#?}")]
     JsonErr(serde_json::Value),
+    #[error("Custom Status Code: {0:#?}")]
+    CustomStatusCode(StatusCode, serde_json::Value),
 }
 
 impl Error {
@@ -82,6 +85,7 @@ impl IntoResponse for Error {
             Self::NotFound(_) => axum::http::StatusCode::NOT_FOUND,
             Self::NotAuthorized(_) => axum::http::StatusCode::UNAUTHORIZED,
             Self::RequireAdmin(_) => axum::http::StatusCode::FORBIDDEN,
+            Self::CustomStatusCode(code, _) => code,
             Self::SqlErr(_) | Self::BadRequest(_) => axum::http::StatusCode::BAD_REQUEST,
             _ => axum::http::StatusCode::INTERNAL_SERVER_ERROR,
         };

backend/windmill-common/src/jobs.rs

@@ -3,7 +3,7 @@ use sqlx::{Postgres, Transaction};
 use uuid::Uuid;
 
 use crate::{
-    error,
+    error::{self, Error},
     flow_status::FlowStatus,
     flows::FlowValue,
     get_latest_deployed_hash_for_path,
@@ -171,7 +171,7 @@ type Tag = String;
 pub async fn script_path_to_payload<'c>(
     script_path: &str,
     db: &mut Transaction<'c, Postgres>,
-    w_id: &String,
+    w_id: &str,
 ) -> error::Result<(JobPayload, Option<Tag>)> {
     let (job_payload, tag) = if script_path.starts_with("hub/") {
         (JobPayload::ScriptHub { path: script_path.to_owned() }, None)
@@ -199,3 +199,24 @@ pub async fn script_hash_to_tag<'c>(
     .await?
     .flatten())
 }
+
+pub async fn get_payload_tag_from_prefixed_path<'c>(
+    path: &str,
+    db: &mut Transaction<'c, Postgres>,
+    w_id: &str,
+) -> Result<(JobPayload, Option<String>), Error> {
+    let (payload, tag) = if path.starts_with("script/") {
+        script_path_to_payload(path.strip_prefix("script/").unwrap(), db, w_id).await?
+    } else if path.starts_with("flow/") {
+        (
+            JobPayload::Flow(path.strip_prefix("flow/").unwrap().to_string()),
+            None,
+        )
+    } else {
+        return Err(Error::BadRequest(format!(
+            "path must start with script/ or flow/ (got {})",
+            path
+        )));
+    };
+    Ok((payload, tag))
+}

backend/windmill-common/src/schedule.rs

@@ -25,4 +25,9 @@ pub struct Schedule {
     pub extra_perms: serde_json::Value,
     pub email: String,
     pub error: Option<String>,
+    pub on_failure: Option<String>,
+}
+
+pub fn schedule_to_user(path: &str) -> String {
+    format!("schedule-{}", path.replace('/', "-"))
 }

backend/windmill-queue/src/jobs.rs

@@ -8,7 +8,6 @@
 
 use std::collections::HashMap;
 
-use anyhow::Context;
 use itertools::Itertools;
 use reqwest::Client;
 use sqlx::{Pool, Postgres, Transaction};
@@ -21,8 +20,7 @@ use windmill_common::{
     flow_status::{FlowStatus, JobResult, MAX_RETRY_ATTEMPTS, MAX_RETRY_INTERVAL},
     flows::{FlowModule, FlowModuleValue, FlowValue},
     jobs::{JobKind, JobPayload, QueuedJob, RawCode},
-    scripts::{get_full_hub_script_by_path, HubScript, ScriptHash, ScriptLang},
-    utils::StripPath,
+    scripts::{ScriptHash, ScriptLang},
     METRICS_ENABLED,
 };
 
@@ -326,6 +324,7 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
     schedule_path: Option<String>,
     parent_job: Option<Uuid>,
     root_job: Option<Uuid>,
+    job_id: Option<Uuid>,
     is_flow_step: bool,
     mut same_worker: bool,
     pre_run_error: Option<&windmill_common::error::Error>,
@@ -333,7 +332,24 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
     mut tag: Option<String>,
 ) -> Result<(Uuid, QueueTransaction<'c, R>), Error> {
     let args_json = serde_json::Value::Object(args);
-    let job_id: Uuid = Ulid::new().into();
+    let job_id: Uuid = if let Some(job_id) = job_id {
+        let conflicting_id = sqlx::query_scalar!(
+            "SELECT 1 FROM queue WHERE id = $1 UNION ALL select 1 FROM completed_job WHERE id = $1",
+            job_id
+        )
+        .fetch_optional(&mut tx)
+        .await?;
+
+        if conflicting_id.is_some() {
+            return Err(Error::BadRequest(format!(
+                "Job with id {job_id} already exists"
+            )));
+        }
+
+        job_id
+    } else {
+        Ulid::new().into()
+    };
 
     if cfg!(feature = "enterprise") {
         let premium_workspace = *CLOUD_HOSTED
@@ -457,16 +473,14 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
                 )
             }
             JobPayload::ScriptHub { path } => {
-                let script = get_hub_script(&HTTP_CLIENT, path.clone(), email)
-                    .await
-                    .context("error fetching hub script")?;
                 (
                     None,
                     Some(path),
-                    Some((script.content, script.lockfile)),
+                    None,
+                    // Some((script.content, script.lockfile)),
                     JobKind::Script_Hub,
                     None,
-                    Some(script.language),
+                    None,
                 )
             }
             JobPayload::Code(RawCode { content, path, language, lock }) => (
@@ -688,13 +702,3 @@ pub fn canceled_job_to_result(job: &QueuedJob) -> serde_json::Value {
     let canceler = job.canceled_by.as_deref().unwrap_or_else(|| "unknown");
     serde_json::json!({"message": format!("Job canceled: {reason} by {canceler}"), "name": "Canceled", "reason": reason, "canceler": canceler})
 }
-
-pub async fn get_hub_script(
-    client: &reqwest::Client,
-    path: String,
-    email: &str,
-) -> error::Result<HubScript> {
-    get_full_hub_script_by_path(email, StripPath(path), client)
-        .await
-        .map(|e| e)
-}

backend/windmill-queue/src/schedule.rs

@@ -11,6 +11,7 @@ use crate::QueueTransaction;
 use sqlx::{query_scalar, Postgres, Transaction};
 use std::str::FromStr;
 use windmill_common::jobs::JobPayload;
+use windmill_common::schedule::schedule_to_user;
 use windmill_common::{
     error::{self, Result},
     schedule::Schedule,
@@ -102,6 +103,7 @@ pub async fn push_scheduled_job<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
         Some(schedule.path.clone()),
         None,
         None,
+        None,
         false,
         false,
         None,
@@ -146,7 +148,3 @@ pub async fn exists_schedule(
 
     Ok(exists)
 }
-
-fn schedule_to_user(path: &str) -> String {
-    format!("schedule-{}", path.replace('/', "-"))
-}

backend/windmill-worker/loader.py

@@ -30,7 +30,7 @@ class WindmillFinder(MetaPathFinder):
             script_path = "/".join(splitted)
             import requests
 
-            url = f"{os.environ.get('BASE_INTERNAL_URL')}/api/w/{os.environ.get('WM_WORKSPACE')}/scripts/raw/p/{script_path}"
+            url = f"{os.environ.get('BASE_INTERNAL_URL')}/api/w/{os.environ.get('WM_WORKSPACE')}/scripts/raw/p/{script_path}.py"
 
             r = requests.get(
                 url, headers={"Authorization": f"Bearer {os.environ.get('WM_TOKEN')}"}

backend/windmill-worker/nsjail/download.py.config.proto

@@ -3,7 +3,7 @@ name: "python download pip"
 mode: ONCE
 hostname: "python"
 log_level: ERROR
-time_limit: 300
+time_limit: 900
 
 rlimit_as: 2048
 rlimit_cpu: 1000

backend/windmill-worker/src/go_executor.rs

@@ -17,8 +17,8 @@ use windmill_parser_go::parse_go_imports;
 use crate::{
     common::{capitalize, read_result, set_logs},
     create_args_and_out_file, get_reserved_variables, handle_child, write_file,
-    AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, GOPRIVATE, GO_CACHE_DIR, HOME_ENV,
-    NETRC, NSJAIL_PATH, PATH_ENV,
+    AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, GOPRIVATE, GOPROXY, GO_CACHE_DIR,
+    HOME_ENV, NETRC, NSJAIL_PATH, PATH_ENV,
 };
 
 const GO_REQ_SPLITTER: &str = "//go.sum\n";
@@ -207,16 +207,23 @@ func Run(req Req) (interface{{}}, error){{
         if let Some(ref netrc) = *NETRC {
             write_file(&HOME_ENV, ".netrc", netrc).await?;
         }
-        Command::new(GO_PATH.as_str())
-            .current_dir(job_dir)
+        let mut cmd = Command::new(GO_PATH.as_str());
+        cmd.current_dir(job_dir)
             .env_clear()
             .envs(reserved_variables)
             .env("PATH", PATH_ENV.as_str())
             .env("BASE_INTERNAL_URL", base_internal_url)
             .env("GOPATH", GO_CACHE_DIR)
-            .env("GOPRIVATE", GOPRIVATE.as_ref().unwrap_or(&String::new()))
-            .env("HOME", HOME_ENV.as_str())
-            .args(vec!["run", "main.go"])
+            .env("HOME", HOME_ENV.as_str());
+
+        if let Some(ref goprivate) = *GOPRIVATE {
+            cmd.env("GOPRIVATE", goprivate);
+        }
+        if let Some(ref goproxy) = *GOPROXY {
+            cmd.env("GOPROXY", goproxy);
+        }
+
+        cmd.args(vec!["run", "main.go"])
             .stdout(Stdio::piped())
             .stderr(Stdio::piped())
             .spawn()?

backend/windmill-worker/src/jobs.rs

@@ -6,17 +6,21 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
+use serde_json::json;
 use sqlx::{Pool, Postgres};
 use tracing::instrument;
 use uuid::Uuid;
 use windmill_common::{
     error::Error,
     flow_status::FlowStatusModule,
-    jobs::{JobKind, QueuedJob},
-    schedule::Schedule,
+    jobs::{get_payload_tag_from_prefixed_path, JobKind, QueuedJob},
+    schedule::{schedule_to_user, Schedule},
+    users::username_to_permissioned_as,
     METRICS_ENABLED,
 };
-use windmill_queue::{delete_job, schedule::get_schedule_opt, QueueTransaction, CLOUD_HOSTED};
+use windmill_queue::{
+    delete_job, push, schedule::get_schedule_opt, QueueTransaction, CLOUD_HOSTED,
+};
 
 #[instrument(level = "trace", skip_all)]
 pub async fn add_completed_job_error<R: rsmq_async::RsmqConnection + Clone + Send>(
@@ -165,6 +169,7 @@ pub async fn add_completed_job<R: rsmq_async::RsmqConnection + Clone + Send>(
     .execute(&mut tx)
     .await
     .map_err(|e| Error::InternalErr(format!("Could not add completed job {job_id}: {e}")))?;
+
     tx = delete_job(tx, &queued_job.workspace_id, job_id).await?;
     if !queued_job.is_flow_step
         && queued_job.job_kind != JobKind::Flow
@@ -172,12 +177,14 @@ pub async fn add_completed_job<R: rsmq_async::RsmqConnection + Clone + Send>(
         && queued_job.schedule_path.is_some()
         && queued_job.script_path.is_some()
     {
-        tx = schedule_again_if_scheduled(
+        tx = handle_maybe_scheduled_job(
             tx,
             db,
             queued_job.schedule_path.as_ref().unwrap(),
             queued_job.script_path.as_ref().unwrap(),
             &queued_job.workspace_id,
+            success,
+            if success { None } else { Some(result) },
         )
         .await?;
     }
@@ -220,12 +227,14 @@ pub async fn add_completed_job<R: rsmq_async::RsmqConnection + Clone + Send>(
 }
 
 #[instrument(level = "trace", skip_all)]
-pub async fn schedule_again_if_scheduled<'c, R: rsmq_async::RsmqConnection + Clone + Send + 'c>(
+pub async fn handle_maybe_scheduled_job<'c, R: rsmq_async::RsmqConnection + Clone + Send + 'c>(
     mut tx: QueueTransaction<'c, R>,
     db: &Pool<Postgres>,
     schedule_path: &str,
     script_path: &str,
     w_id: &str,
+    success: bool,
+    result: Option<serde_json::Value>,
 ) -> windmill_common::error::Result<QueueTransaction<'c, R>> {
     let schedule = get_schedule_opt(tx.transaction_mut(), w_id, schedule_path).await?;
 
@@ -239,6 +248,45 @@ pub async fn schedule_again_if_scheduled<'c, R: rsmq_async::RsmqConnection + Clo
     let schedule = schedule.unwrap();
 
     if schedule.enabled && script_path == schedule.script_path {
+        if !success {
+            if let Some(on_failure_path) = schedule.on_failure.clone() {
+                let on_failure_result = handle_on_failure(
+                    tx,
+                    schedule_path,
+                    script_path,
+                    w_id,
+                    &on_failure_path,
+                    result,
+                    &schedule.email,
+                    &schedule_to_user(&schedule.path),
+                    username_to_permissioned_as(&schedule.edited_by),
+                )
+                .await;
+
+                match on_failure_result {
+                    Ok(ntx) => {
+                        tx = ntx;
+                    }
+                    Err(err) => {
+                        sqlx::query!(
+                        "UPDATE schedule SET enabled = false, error = $1 WHERE workspace_id = $2 AND path = $3",
+                        format!("Could not trigger error handler: {err}"),
+                        &schedule.workspace_id,
+                        &schedule.path
+                    )
+                    .execute(db)
+                    .await?;
+                        tracing::warn!(
+                            "Could not trigger error handler for {}: {}",
+                            schedule_path,
+                            err
+                        );
+                        return Err(err);
+                    }
+                }
+            }
+        }
+
         let res = windmill_queue::schedule::push_scheduled_job(
             tx,
             Schedule {
@@ -257,6 +305,7 @@ pub async fn schedule_again_if_scheduled<'c, R: rsmq_async::RsmqConnection + Clo
                 extra_perms: serde_json::to_value(schedule.extra_perms).expect("hashmap -> json"),
                 email: schedule.email,
                 error: None,
+                on_failure: schedule.on_failure,
             },
         )
         .await;
@@ -279,3 +328,52 @@ pub async fn schedule_again_if_scheduled<'c, R: rsmq_async::RsmqConnection + Clo
         Ok(tx)
     }
 }
+
+async fn handle_on_failure<'c, R: rsmq_async::RsmqConnection + Clone + Send + 'c>(
+    mut tx: QueueTransaction<'c, R>,
+    schedule_path: &str,
+    script_path: &str,
+    w_id: &str,
+    on_failure_path: &str,
+    result: Option<serde_json::Value>,
+    username: &str,
+    email: &str,
+    permissioned_as: String,
+) -> windmill_common::error::Result<QueueTransaction<'c, R>> {
+    let (payload, tag) =
+        get_payload_tag_from_prefixed_path(on_failure_path, tx.transaction_mut(), w_id).await?;
+
+    let mut args = result
+        .unwrap_or_else(|| json!({}))
+        .as_object()
+        .unwrap()
+        .clone();
+    args.insert("schedule_path".to_string(), json!(schedule_path));
+    args.insert("path".to_string(), json!(script_path));
+    let (uuid, tx) = push(
+        tx,
+        w_id,
+        payload,
+        args,
+        username,
+        email,
+        permissioned_as,
+        None,
+        None,
+        None,
+        None,
+        None,
+        false,
+        false,
+        None,
+        true,
+        tag,
+    )
+    .await?;
+    tracing::info!(
+        "Pushed on_failure job {} for {} to queue",
+        uuid,
+        schedule_path
+    );
+    return Ok(tx);
+}

backend/windmill-worker/src/js_eval.rs

@@ -143,7 +143,6 @@ fn add_closing_bracket(s: &str) -> String {
     s
 }
 
-const SPLIT_PAT: &str = ";";
 async fn eval(
     context: &mut JsRuntime,
     expr: &str,
@@ -151,21 +150,6 @@ async fn eval(
     by_id: Option<IdContext>,
     has_client: bool,
 ) -> anyhow::Result<serde_json::Value> {
-    let exprs = expr
-        .trim()
-        .split(SPLIT_PAT)
-        .map(|x| x.trim())
-        .filter(|x| !x.is_empty())
-        .collect::<Vec<&str>>();
-    let expr = if exprs.is_empty() {
-        "return undefined;".to_string()
-    } else {
-        format!(
-            "{};\n    return {};",
-            exprs.iter().take(exprs.len() - 1).join(";\n"),
-            exprs.last().unwrap()
-        )
-    };
     let (api_code, by_id_code) = if has_client {
         let by_id_code = if let Some(by_id) = by_id {
             format!(
@@ -232,13 +216,18 @@ async function resource(path) {{
         (String::new(), String::new())
     };
 
+    let f = if expr.contains("return ") {
+        expr.to_string()
+    } else {
+        format!("return {expr}")
+    };
     let code = format!(
         r#"
 {api_code}
 {}
 {by_id_code}
 (async () => {{ 
-    {expr} 
+    {f};
 }})()
         "#,
         env.into_iter()
@@ -251,7 +240,6 @@ async function resource(path) {{
             })
             .join(""),
     );
-    tracing::debug!("{}", code);
     let global = context.execute_script("<anon>", code.into())?;
     let global = context.resolve_value(global).await?;
 
@@ -379,7 +367,7 @@ mod tests {
     async fn test_eval_multiline() -> anyhow::Result<()> {
         let env = vec![];
         let code = "let x = 5;
-`my ${x}
+return `my ${x}
 multiline template`";
 
         let mut runtime = JsRuntime::new(RuntimeOptions::default());

backend/windmill-worker/src/python_executor.rs

@@ -74,7 +74,23 @@ pub async fn pip_compile(
 ) -> error::Result<String> {
     logs.push_str(&format!("\nresolving dependencies..."));
     set_logs(logs, job_id, db).await;
-    logs.push_str(&format!("\ncontent of requirements:\n{}", requirements));
+    logs.push_str(&format!("\ncontent of requirements:\n{}\n", requirements));
+    let requirements = if let Some(pip_local_dependencies) = PIP_LOCAL_DEPENDENCIES.as_ref() {
+        let deps = pip_local_dependencies.clone();
+        requirements
+            .lines()
+            .filter(|s| {
+                if !deps.contains(&s.to_string()) {
+                    return true;
+                } else {
+                    logs.push_str(&format!("\nignoring local dependency: {}", s));
+                    return false;
+                }
+            })
+            .join("\n")
+    } else {
+        requirements.to_string()
+    };
     let req_hash = calculate_hash(&requirements);
     if let Some(cached) = sqlx::query_scalar!(
         "SELECT lockfile FROM pip_resolution_cache WHERE hash = $1",
@@ -87,15 +103,7 @@ pub async fn pip_compile(
         return Ok(cached);
     }
     let file = "requirements.in";
-    let requirements = if let Some(pip_local_dependencies) = PIP_LOCAL_DEPENDENCIES.as_ref() {
-        let deps = pip_local_dependencies.clone();
-        requirements
-            .lines()
-            .filter(|s| !deps.contains(&s.to_string()))
-            .join("\n")
-    } else {
-        requirements.to_string()
-    };
+
     write_file(job_dir, file, &requirements).await?;
 
     let mut args = vec!["-q", "--no-header", file, "--resolver=backtracking"];
@@ -289,7 +297,18 @@ pub async fn handle_python_job(
     } else {
         sig.args
             .into_iter()
-            .map(|x| format!("args[\"{}\"] = kwargs.get(\"{}\")", x.name, x.name))
+            .map(|x| {
+                let name = &x.name;
+                if x.default.is_none() {
+                    format!("args[\"{name}\"] = kwargs.get(\"{name}\")")
+                } else {
+                    format!(
+                        r#"args["{name}"] = kwargs.get("{name}")
+if args["{name}"] is None:
+    del args["{name}"]"#
+                    )
+                }
+            })
             .join("\n")
     };
 
@@ -303,7 +322,7 @@ import json
 import traceback
 import sys
 from {module_dir_dot} import {last} as inner_script
-
+import regex as re
 
 with open("args.json") as f:
     kwargs = json.load(f, strict=False)
@@ -314,6 +333,12 @@ for k, v in list(args.items()):
     if v == '<function call>':
         del args[k]
 
+def to_b_64(v: bytes):
+    import base64
+    b64 = base64.b64encode(v)
+    return b64.decode('ascii')
+
+replace_nan = re.compile(r'\bnan\b', re.IGNORECASE)
 try:
     res = inner_script.main(**args)
     typ = type(res)
@@ -322,7 +347,13 @@ try:
             res = res.values.tolist()
         elif typ.__module__ == 'polars.dataframe.frame':
             res = res.rows()
-    res_json = json.dumps(res, separators=(',', ':'), default=str).replace('\n', '')
+    elif typ.__name__ == 'bytes':
+        res = to_b_64(res)
+    elif typ.__name__ == 'dict':
+        for k, v in res.items():
+            if type(v).__name__ == 'bytes':
+                res[k] = to_b_64(v)
+    res_json = re.sub(replace_nan, ' null ', json.dumps(res, separators=(',', ':'), default=str).replace('\n', ''))
     with open("result.json", 'w') as f:
         f.write(res_json)
 except Exception as e:

backend/windmill-worker/src/worker.rs

@@ -6,12 +6,13 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
-use anyhow::Result;
+use anyhow::{Result};
 use const_format::concatcp;
 use itertools::Itertools;
 use once_cell::sync::OnceCell;
 use sqlx::{Pool, Postgres};
 use windmill_api_client::Client;
+use windmill_parser::Typ;
 use std::{
     borrow::Borrow, collections::HashMap, io, os::unix::process::ExitStatusExt, panic,
     process::Stdio, time::{Duration},
@@ -23,11 +24,11 @@ use uuid::Uuid;
 use windmill_common::{
     error::{self, to_anyhow, Error},
     flows::{FlowModuleValue, FlowValue},
-    scripts::{ScriptHash, ScriptLang},
-    utils::{rd_string},
+    scripts::{ScriptHash, ScriptLang, get_full_hub_script_by_path},
+    utils::{rd_string, StripPath},
     variables, BASE_URL, users::SUPERADMIN_SECRET_EMAIL, METRICS_ENABLED, jobs::{JobKind, QueuedJob}, IS_READY,
 };
-use windmill_queue::{canceled_job_to_result, get_queued_job, pull, CLOUD_HOSTED};
+use windmill_queue::{canceled_job_to_result, get_queued_job, pull, CLOUD_HOSTED, HTTP_CLIENT};
 
 use serde_json::{json, Value};
 
@@ -127,9 +128,12 @@ pub const ROOT_TMP_CACHE_DIR: &str = "/tmp/windmill/tmpcache/";
 pub const PIP_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "pip");
 pub const DENO_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "deno");
 pub const GO_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "go");
+pub const HUB_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "hub");
+
 pub const TAR_PIP_TMP_CACHE_DIR: &str = concatcp!(ROOT_TMP_CACHE_DIR, "tar/pip");
 pub const DENO_TMP_CACHE_DIR: &str = concatcp!(ROOT_TMP_CACHE_DIR, "deno");
 pub const GO_TMP_CACHE_DIR: &str = concatcp!(ROOT_TMP_CACHE_DIR, "go");
+pub const HUB_TMP_CACHE_DIR: &str = concatcp!(ROOT_TMP_CACHE_DIR, "hub");
 
 const NUM_SECS_PING: u64 = 5;
 
@@ -143,8 +147,7 @@ pub struct Metrics {
     pub worker_execution_failed: prometheus::IntCounter,
 }
 
-
-pub const DEFAULT_TIMEOUT: u64 = 300;
+pub const DEFAULT_TIMEOUT: u64 = 900;
 pub const DEFAULT_SLEEP_QUEUE: u64 = 50;
 
 lazy_static::lazy_static! {
@@ -175,6 +178,7 @@ lazy_static::lazy_static! {
     pub static ref PATH_ENV: String = std::env::var("PATH").unwrap_or_else(|_| String::new());
     pub static ref HOME_ENV: String = std::env::var("HOME").unwrap_or_else(|_| String::new());
     pub static ref GOPRIVATE: Option<String> = std::env::var("GOPRIVATE").ok();
+    pub static ref GOPROXY: Option<String> = std::env::var("GOPROXY").ok();
     pub static ref NETRC: Option<String> = std::env::var("NETRC").ok();
 
     static ref DENO_AUTH_TOKENS: String = std::env::var("DENO_AUTH_TOKENS")
@@ -630,7 +634,7 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
                     if job.id == Uuid::nil() {
                         tracing::info!(worker = %worker_name, "running warmup job");
                     } else {
-                        tracing::info!(worker = %worker_name, id = %job.id, root_id = %job_root, "fetched job {}, root job: {}", job.id, job_root);
+                        tracing::info!(worker = %worker_name, workspace_id = %job.workspace_id, id = %job.id, root_id = %job_root, "fetched job {}, root job: {}", job.id, job_root);
                     }
 
                     let job_dir = format!("{worker_dir}/{}", job.id);
@@ -1130,13 +1134,34 @@ async fn handle_code_execution_job(
 
 ) -> error::Result<serde_json::Value> {
     let (inner_content, requirements_o, language) = match job.job_kind {
-        JobKind::Preview | JobKind::Script_Hub => (
+        JobKind::Preview  => (
             job.raw_code
                 .clone()
                 .unwrap_or_else(|| "no raw code".to_owned()),
             job.raw_lock.clone(),
             job.language.to_owned(),
         ),
+        JobKind::Script_Hub => {
+            let script_path = job.script_path.clone().ok_or_else(|| Error::InternalErr(format!("expected script path for hub script")))?;
+            let mut script_path_iterator = script_path.split("/");
+            script_path_iterator.next();
+            let version = script_path_iterator.next().ok_or_else(|| Error::InternalErr(format!("expected hub path to have version number")))?;
+            let cache_path = format!("{HUB_CACHE_DIR}/{version}");
+            let script;
+            if tokio::fs::metadata(&cache_path).await.is_err() {
+                script =  get_full_hub_script_by_path(&job.email, StripPath(script_path.clone()), &HTTP_CLIENT).await?;
+                write_file(HUB_CACHE_DIR, &version, &serde_json::to_string(&script).map_err(to_anyhow)?).await?;
+                tracing::info!("wrote hub script {script_path} to cache");
+            } else {
+                let cache_content = tokio::fs::read_to_string(cache_path).await?;
+                script = serde_json::from_str(&cache_content).unwrap();
+                tracing::info!("read hub script {script_path} from cache");
+            }
+            (
+            script.content,
+            script.lockfile,
+            Some(script.language)
+        )},
         JobKind::Script => sqlx::query_as::<_, (String, Option<String>, Option<ScriptLang>)>(
             "SELECT content, lock, language FROM script WHERE hash = $1 AND workspace_id = $2",
         )
@@ -1275,7 +1300,7 @@ async fn handle_bash_job(
 ) -> Result<serde_json::Value, Error> {
     logs.push_str("\n\n--- BASH CODE EXECUTION ---\n");
     set_logs(logs, &job.id, db).await;
-    write_file(job_dir, "main.sh", &format!("{content}\nsleep 0.02")).await?;
+    write_file(job_dir, "main.sh", &format!("set -e\n{content}\nsleep 0.02")).await?;
     let token = client.get_token().await;
     let mut reserved_variables = get_reserved_variables(job, &token, db).await?;
     reserved_variables.insert("RUST_LOG".to_string(), "info".to_string());
@@ -1392,26 +1417,36 @@ async fn handle_deno_job(
 
     let write_wrapper_f = async {
         // let mut start = Instant::now();
-        let spread =  windmill_parser_ts::parse_deno_signature(inner_content, true)?.args.into_iter().map(|x| x.name).join(",");
+        let args = windmill_parser_ts::parse_deno_signature(inner_content, true)?.args;
+        let dates = args.iter().enumerate().filter_map(|(i, x)| if matches!(x.typ, Typ::Datetime) {
+            Some(i) 
+        } else {
+            None
+        }).map(|x| {
+            return format!("args[{x}] = args[{x}] ? new Date(args[{x}]) : undefined")
+        }).join("\n");
+
+        let spread = args.into_iter().map(|x| x.name).join(",");
         // logs.push_str(format!("infer args: {:?}\n", start.elapsed().as_micros()).as_str());
         let wrapper_content: String = format!(
             r#"
-    import {{ main }} from "./main.ts";
+import {{ main }} from "./main.ts";
 
-    const args = await Deno.readTextFile("args.json")
-        .then(JSON.parse)
-        .then(({{ {spread} }}) => [ {spread} ])
+const args = await Deno.readTextFile("args.json")
+    .then(JSON.parse)
+    .then(({{ {spread} }}) => [ {spread} ])
 
-    async function run() {{
-        let res: any = await main(...args);
-        const res_json = JSON.stringify(res ?? null, (key, value) => typeof value === 'undefined' ? null : value);
-        await Deno.writeTextFile("result.json", res_json);
-        Deno.exit(0);
-    }}
-    run().catch(async (e) => {{
-        await Deno.writeTextFile("result.json", JSON.stringify({{ message: e.message, name: e.name, stack: e.stack }}));
-        Deno.exit(1);
-    }});
+{dates}
+async function run() {{
+    let res: any = await main(...args);
+    const res_json = JSON.stringify(res ?? null, (key, value) => typeof value === 'undefined' ? null : value);
+    await Deno.writeTextFile("result.json", res_json);
+    Deno.exit(0);
+}}
+run().catch(async (e) => {{
+    await Deno.writeTextFile("result.json", JSON.stringify({{ message: e.message, name: e.name, stack: e.stack }}));
+    Deno.exit(1);
+}});
     "#,
         );
         write_file(job_dir, "wrapper.ts", &wrapper_content).await?;

backend/windmill-worker/src/worker_flow.rs

@@ -9,7 +9,7 @@
 use std::collections::HashMap;
 use std::time::Duration;
 
-use crate::jobs::{add_completed_job, add_completed_job_error, schedule_again_if_scheduled};
+use crate::jobs::{add_completed_job, add_completed_job_error, handle_maybe_scheduled_job};
 use crate::js_eval::{eval_timeout, IdContext};
 use crate::{worker, AuthedClient, KEEP_JOB_DIR};
 use anyhow::Context;
@@ -525,12 +525,14 @@ pub async fn update_flow_status_after_job_completion_internal<
             && flow_job.schedule_path.is_some()
             && flow_job.script_path.is_some()
         {
-            tx = schedule_again_if_scheduled(
+            tx = handle_maybe_scheduled_job(
                 tx,
                 db,
                 flow_job.schedule_path.as_ref().unwrap(),
                 flow_job.script_path.as_ref().unwrap(),
                 &w_id,
+                success,
+                if success { None } else { Some(nresult.clone()) },
             )
             .await?;
         }
@@ -1387,6 +1389,7 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
             flow_job.schedule_path.clone(),
             Some(flow_job.id),
             root_job,
+            None,
             true,
             continue_on_same_worker,
             err,

cli/README.md

@@ -3,6 +3,8 @@
 A simple CLI allowing interactions with windmill from the command line.
 ![](./vhs/output/setup.gif)
 
+You can find more information in [Windmill Docs](https://docs.windmill.dev/docs/advanced/cli)
+
 ## Installation
 
 Install the `wmill` CLI tool using
@@ -18,11 +20,14 @@ workspace settings.
 ## Running Flows & Scripts
 
 Run a script or flow using `wmill flow/script run u/username/path/to/script` and
-pass any inputs using `--input/-i <name>=<value>` curl-style syntax using
-`-i @-` for stdin or `-i @<filename>` is also supported
+pass any inputs using `--data` + Inputs specified as a JSON string or a file using `@ <filename>` or stdin using @-.
+
+Curl-style syntax using `-d @-` for stdin or `-d @<filename>` is also supported.
 
 Flow Steps and Logs will be streamed during execution automatically.
 
+![CLI input example](./vhs/output/cli_inputs_example.png)
+
 ## Pushing Resources, Scripts & More
 
 The CLI can push specifications to a windmill instance. See the

cli/apps.ts

@@ -26,9 +26,19 @@ export async function pushApp(
   workspace: string,
   remotePath: string,
   app: AppFile | AppWithLastVersion | undefined,
-  newApp: AppFile
+  newApp: AppFile,
+  raw: boolean
 ): Promise<void> {
   remotePath = removeType(remotePath, "app");
+  if (raw) {
+    // deleting old app if it exists in raw mode
+    try {
+      app = await AppService.getAppByPath({ workspace, path: remotePath });
+    } catch {
+      //ignore
+    }
+  }
+
   if (app) {
     if (isSuperset(newApp, app)) {
       return;
@@ -103,7 +113,8 @@ async function push(opts: GlobalOptions, filePath: string) {
     workspace.workspaceId,
     remotePath,
     app,
-    parseFromFile(filePath)
+    parseFromFile(filePath),
+    false
   );
   console.log(colors.bold.underline.green("App pushed"));
 }

cli/context.ts

@@ -1,5 +1,5 @@
 // deno-lint-ignore-file no-explicit-any
-import { colors, GlobalUserInfo, setClient, UserService } from "./deps.ts";
+import { colors, GlobalUserInfo, log, setClient, UserService } from "./deps.ts";
 import { loginInteractive, tryGetLoginInfo } from "./login.ts";
 import { GlobalOptions } from "./types.ts";
 import {
@@ -53,7 +53,7 @@ export async function resolveWorkspace(
 ): Promise<Workspace> {
   const res = await tryResolveWorkspace(opts);
   if (res.isError) {
-    console.log(colors.red.bold(res.error));
+    log.info(colors.red.bold(res.error));
     return Deno.exit(-1);
   } else {
     return res.value;
@@ -75,7 +75,7 @@ export async function requireLogin(
   try {
     return await UserService.globalWhoami();
   } catch {
-    console.log(
+    log.info(
       "! Could not reach API given existing credentials. Attempting to reauth..."
     );
     const newToken = await loginInteractive(workspace.remote);
@@ -119,7 +119,7 @@ export async function tryResolveVersion(
 
 export function validatePath(path: string): boolean {
   if (!(path.startsWith("g") || path.startsWith("u") || path.startsWith("f"))) {
-    console.log(
+    log.info(
       colors.red(
         "Given remote path looks invalid. Remote paths are typically of the form <u|g|f>/<username|group|folder>/..."
       )

cli/deps.ts

@@ -1,6 +1,6 @@
 // windmill
-export { setClient } from "https://deno.land/x/windmill@v1.69.3/mod.ts";
-export * from "https://deno.land/x/windmill@v1.69.3/windmill-api/index.ts";
+export { setClient } from "https://deno.land/x/windmill@v1.95.1/mod.ts";
+export * from "https://deno.land/x/windmill@v1.95.1/windmill-api/index.ts";
 
 // cliffy
 export { Command } from "https://deno.land/x/cliffy@v0.25.7/command/command.ts";
@@ -25,6 +25,7 @@ export {
 } from "https://deno.land/std@0.176.0/streams/mod.ts";
 export { DelimiterStream } from "https://deno.land/std@0.176.0/streams/mod.ts";
 export { iterateReader } from "https://deno.land/std@0.176.0/streams/iterate_reader.ts";
+export { writeAllSync } from "https://deno.land/std@0.176.0/streams/mod.ts";
 
 // other
 export { getAvailablePort } from "https://deno.land/x/port@1.0.0/mod.ts";
@@ -37,3 +38,9 @@ export { default as microdiff } from "https://deno.land/x/microdiff@v1.3.1/index
 export { default as objectHash } from "https://deno.land/x/object_hash@2.0.3.1/mod.ts";
 export { default as gitignore_parser } from "npm:gitignore-parser";
 export { default as JSZip } from "npm:jszip@3.7.1";
+export * as log from "https://deno.land/std@0.186.0/log/mod.ts";
+export {
+  stringify as yamlStringify,
+  parse as yamlParse,
+} from "https://deno.land/std@0.184.0/yaml/mod.ts";
+export { open } from "https://deno.land/x/open@v0.0.5/index.ts";

cli/flow.ts

@@ -1,17 +1,15 @@
 // deno-lint-ignore-file no-explicit-any
-import {
-  GlobalOptions,
-  isSuperset,
-  parseFromFile,
-  removeType,
-} from "./types.ts";
+import { GlobalOptions, isSuperset } from "./types.ts";
+
 import {
   colors,
   Command,
   Flow,
+  FlowModule,
   FlowService,
   JobService,
   Table,
+  yamlParse,
 } from "./deps.ts";
 import { requireLogin, resolveWorkspace, validatePath } from "./context.ts";
 import { resolve, track_job } from "./script.ts";
@@ -23,18 +21,57 @@ export interface FlowFile {
   schema?: any;
 }
 
+const alreadySynced: string[] = [];
+
 export async function pushFlow(
   workspace: string,
   remotePath: string,
-  flow: Flow | FlowFile | undefined,
-  localFlow: FlowFile
+  localFlowPath: string
 ): Promise<void> {
-  remotePath = removeType(remotePath, "flow");
+  if (alreadySynced.includes(localFlowPath)) {
+    return;
+  }
+  alreadySynced.push(localFlowPath);
+  let flow: Flow | undefined = undefined;
+  try {
+    flow = await FlowService.getFlowByPath({
+      workspace: workspace,
+      path: remotePath,
+    });
+  } catch {
+    // flow doesn't exist
+  }
+
+  if (!localFlowPath.endsWith("/")) {
+    localFlowPath += "/";
+  }
+  const localFlowRaw = await Deno.readTextFile(localFlowPath + "flow.yaml");
+  const localFlow = yamlParse(localFlowRaw) as FlowFile;
+
+  function replaceInlineScripts(modules: FlowModule[]) {
+    modules.forEach((m) => {
+      if (m.value.type == "rawscript") {
+        const path = m.value.content.split(" ")[1];
+        m.value.content = Deno.readTextFileSync(localFlowPath + path);
+      } else if (m.value.type == "forloopflow") {
+        replaceInlineScripts(m.value.modules);
+      } else if (m.value.type == "branchall") {
+        m.value.branches.forEach((b) => replaceInlineScripts(b.modules));
+      } else if (m.value.type == "branchone") {
+        m.value.branches.forEach((b) => replaceInlineScripts(b.modules));
+        replaceInlineScripts(m.value.default);
+      }
+    });
+  }
+
+  replaceInlineScripts(localFlow.value.modules);
 
   if (flow) {
     if (isSuperset(localFlow, flow)) {
+      console.log(colors.bold.green("Flow is up to date"));
       return;
     }
+    console.log(colors.bold.yellow(`Updating flow ${remotePath}...`));
     await FlowService.updateFlow({
       workspace: workspace,
       path: remotePath,
@@ -63,21 +100,8 @@ async function push(opts: Options, filePath: string, remotePath: string) {
   }
   const workspace = await resolveWorkspace(opts);
   await requireLogin(opts);
-  let flow: Flow | undefined = undefined;
-  try {
-    flow = await FlowService.getFlowByPath({
-      workspace: workspace.workspaceId,
-      path: remotePath,
-    });
-  } catch {
-    // flow doesn't exist
-  }
-  await pushFlow(
-    workspace.workspaceId,
-    remotePath,
-    flow,
-    parseFromFile(filePath)
-  );
+
+  await pushFlow(workspace.workspaceId, remotePath, filePath);
   console.log(colors.bold.underline.green("Flow pushed"));
 }
 

cli/folder.ts

@@ -1,5 +1,5 @@
 // deno-lint-ignore-file no-explicit-any
-import { colors, Command, Folder, FolderService } from "./deps.ts";
+import { colors, Command, Folder, FolderService, log, Table } from "./deps.ts";
 import { requireLogin, resolveWorkspace, validatePath } from "./context.ts";
 import { GlobalOptions, isSuperset, parseFromFile } from "./types.ts";
 
@@ -9,11 +9,34 @@ export interface FolderFile {
   display_name: string | undefined;
 }
 
+async function list(opts: GlobalOptions) {
+  const workspace = await resolveWorkspace(opts);
+  await requireLogin(opts);
+
+  const folders = await FolderService.listFolders({
+    workspace: workspace.workspaceId,
+  });
+
+  new Table()
+    .header(["Name", "Owners", "Extra Perms"])
+    .padding(2)
+    .border(true)
+    .body(
+      folders.map((x) => [
+        x.name,
+        x.owners?.join(",") ?? "-",
+        JSON.stringify(x.extra_perms ?? {}),
+      ])
+    )
+    .render();
+}
+
 export async function pushFolder(
   workspace: string,
   name: string,
   folder: Folder | FolderFile | undefined,
-  localFolder: FolderFile
+  localFolder: FolderFile,
+  raw: boolean
 ): Promise<void> {
   if (name.startsWith("/")) {
     name = name.substring(1);
@@ -21,23 +44,33 @@ export async function pushFolder(
   if (name.startsWith("f/")) {
     name = name.substring(2);
   }
+  name = name.split("/")[0];
+  log.debug(`Processing local folder ${name}`);
+
+  if (raw) {
+    // deleting old app if it exists in raw mode
+    try {
+      folder = await FolderService.getFolder({ workspace, name });
+      log.debug(`Folder ${name} exists on remote`);
+    } catch {
+      log.debug(`Folder ${name} does not exist on remote`);
+      //ignore
+    }
+  }
 
   if (folder) {
     if (isSuperset(localFolder, folder)) {
+      log.debug(`Folder ${name} is up to date`);
       return;
     }
-    try {
-      await FolderService.updateFolder({
-        workspace: workspace,
-        name: name,
-        requestBody: {
-          ...localFolder,
-        },
-      });
-    } catch (e) {
-      console.error(colors.red.bold(e.body));
-      throw e;
-    }
+    log.debug(`Folder ${name} is not up-to-date, updating...`);
+    await FolderService.updateFolder({
+      workspace: workspace,
+      name: name,
+      requestBody: {
+        ...localFolder,
+      },
+    });
   } else {
     console.log(colors.bold.yellow("Creating new folder: " + name));
     await FolderService.createFolder({
@@ -64,27 +97,20 @@ async function push(opts: GlobalOptions, filePath: string, remotePath: string) {
   }
 
   console.log(colors.bold.yellow("Pushing folder..."));
-  let folder: Folder | undefined = undefined;
-  try {
-    folder = await FolderService.getFolder({
-      workspace: workspace.workspaceId,
-      name: remotePath,
-    });
-  } catch {
-    // folder doesn't exist
-  }
 
   await pushFolder(
     workspace.workspaceId,
     remotePath,
-    folder,
-    parseFromFile(filePath)
+    undefined,
+    parseFromFile(filePath),
+    false
   );
   console.log(colors.bold.underline.green("Folder pushed"));
 }
 
 const command = new Command()
-  .description("resource related commands")
+  .description("folder related commands")
+  .action(list as any)
   .command(
     "push",
     "push a local folder spec. This overrides any remote versions."

cli/hub.ts

@@ -1,5 +1,5 @@
 // deno-lint-ignore-file no-explicit-any
-import { Command, ResourceService } from "./deps.ts";
+import { Command, ResourceService, log } from "./deps.ts";
 import { requireLogin, resolveWorkspace } from "./context.ts";
 import { pushResourceType } from "./resource-type.ts";
 import { GlobalOptions } from "./types.ts";
@@ -8,7 +8,7 @@ async function pull(opts: GlobalOptions) {
   const workspace = await resolveWorkspace(opts);
 
   if (workspace.workspaceId !== "admins") {
-    console.log(
+    log.info(
       "Should only sync to admins workspace, but current is not admins."
     );
     return;
@@ -48,7 +48,7 @@ async function pull(opts: GlobalOptions) {
     .then((x) =>
       x.map((x) =>
         x.json().catch((e) => {
-          console.log(e);
+          log.info(e);
           return undefined;
         })
       )
@@ -61,16 +61,27 @@ async function pull(opts: GlobalOptions) {
   });
 
   for (const x of list) {
-    if (resourceTypes.find((y) => y.name === x.name)) {
-      console.log("skipping " + x.name);
+    if (
+      resourceTypes.find(
+        (y) => y.name === x.name && typeof y.schema !== "string"
+      )
+    ) {
+      log.info("skipping " + x.name);
+      continue;
+    }
+    log.info("syncing " + x.name);
+    try {
+      x.schema = JSON.parse(x.schema);
+    } catch (e) {
+      log.info("failed to parse schema for " + x.name);
       continue;
     }
-    console.log("syncing " + x.name);
     await pushResourceType(
       workspace.workspaceId,
       x.name + ".resource-type.json",
       undefined,
-      x
+      x,
+      false
     );
   }
 }

cli/login.ts

@@ -1,11 +1,14 @@
 import { GlobalOptions } from "./types.ts";
-import { colors, getAvailablePort, Secret, Select } from "./deps.ts";
-import { open } from 'https://deno.land/x/open/index.ts';
+import { colors, getAvailablePort, log, open, Secret, Select } from "./deps.ts";
 
 export async function loginInteractive(remote: string) {
   let token: string | undefined;
+  if (!Deno.isatty(Deno.stdin.rid)) {
+    log.info("Not a TTY, can't login interactively.");
+    return undefined;
+  }
   if (
-    await Select.prompt({
+    (await Select.prompt({
       message: "How do you want to login",
       options: [
         {
@@ -17,7 +20,7 @@ export async function loginInteractive(remote: string) {
           value: "t",
         },
       ],
-    }) === "b"
+    })) === "b"
   ) {
     token = await browserLogin(remote);
   } else {
@@ -29,7 +32,7 @@ export async function loginInteractive(remote: string) {
 
 // deno-lint-ignore require-await
 export async function tryGetLoginInfo(
-  opts: GlobalOptions,
+  opts: GlobalOptions
 ): Promise<string | undefined> {
   if (opts.token) {
     return opts.token;
@@ -39,22 +42,22 @@ export async function tryGetLoginInfo(
 }
 
 export async function browserLogin(
-  baseUrl: string,
+  baseUrl: string
 ): Promise<string | undefined> {
   const port = await getAvailablePort();
   if (port == undefined) {
-    console.log(colors.red.underline("failed to aquire port"));
+    log.info(colors.red.underline("failed to aquire port"));
     return undefined;
   }
 
   const server = Deno.listen({ transport: "tcp", port });
-  const url = `${baseUrl}user/cli?port=${port}`
-  console.log(`Login by going to ${url}`);
+  const url = `${baseUrl}user/cli?port=${port}`;
+  log.info(`Login by going to ${url}`);
   try {
-    await open(url)
-    console.log("Opened browser for you");
+    await open(url);
+    log.info("Opened browser for you");
   } catch {
-    console.error(`Failed to open browser, please navigate to ${url}`)
+    console.error(`Failed to open browser, please navigate to ${url}`);
   }
   const firstConnection = await server.accept();
   const httpFirstConnection = Deno.serveHttp(firstConnection);
@@ -63,7 +66,7 @@ export async function browserLogin(
   const token = params.get("token");
   // const _workspace = params.get("workspace");
   await firstRequest?.respondWith(
-    Response.redirect(baseUrl + "user/cli-success", 302),
+    Response.redirect(baseUrl + "user/cli-success", 302)
   );
 
   setTimeout(() => {

cli/main.ts

@@ -3,6 +3,7 @@ import {
   CompletionsCommand,
   DenoLandProvider,
   UpgradeCommand,
+  log,
 } from "./deps.ts";
 import flow from "./flow.ts";
 import app from "./apps.ts";
@@ -19,7 +20,14 @@ import sync from "./sync.ts";
 import { tryResolveVersion } from "./context.ts";
 import { GlobalOptions } from "./types.ts";
 
-export const VERSION = "v1.94.0";
+addEventListener("error", (event) => {
+  if (event.error) {
+    console.error("Error details of: " + event.error.message);
+    console.error(JSON.stringify(event.error, null, 4));
+  }
+});
+
+export const VERSION = "v1.104.1";
 
 let command: any = new Command()
   .name("wmill")
@@ -28,6 +36,11 @@ let command: any = new Command()
     "--workspace <workspace:string>",
     "Specify the target workspace. This overrides the default workspace."
   )
+  .globalOption("--debug", "Show debug logs")
+  .globalOption(
+    "--show-diffs",
+    "Show diff informations when syncing (may show sensitive informations)"
+  )
   .globalOption(
     "--token <token:string>",
     "Specify an API token. This will override any stored token."
@@ -73,7 +86,26 @@ if (Number.parseInt(VERSION.replace("v", "").replace(".", "")) > 1700) {
   command = command.command("push", push).command("pull", pull);
 }
 
+export let showDiffs = false;
 try {
+  const LOG_LEVEL = Deno.args.includes("--debug") ? "DEBUG" : "INFO";
+  // const NO_COLORS = Deno.args.includes("--no-colors");
+  showDiffs = Deno.args.includes("--show-diffs");
+
+  log.setup({
+    handlers: {
+      console: new log.handlers.ConsoleHandler(LOG_LEVEL, {
+        formatter: "{msg}",
+      }),
+    },
+    loggers: {
+      default: {
+        level: LOG_LEVEL,
+        handlers: ["console"],
+      },
+    },
+  });
+  log.debug("Debug logging enabled. CLI build against " + VERSION);
   await command.parse(Deno.args);
 } catch (e) {
   if (e.name === "ApiError") {

cli/pull.ts

@@ -30,7 +30,7 @@ export async function downloadZip(
     throw new Error(await zipResponse.text());
   }
   const blob = await zipResponse.blob();
-  return await JSZip.loadAsync(blob);
+  return await JSZip.loadAsync(blob as any);
 }
 
 async function stub(

cli/push.ts

@@ -2,14 +2,11 @@
 import { colors, Command } from "./deps.ts";
 import { GlobalOptions } from "./types.ts";
 
-async function stub(
-  _opts: GlobalOptions,
-  _dir?: string,
-) {
-  console.log(
+async function stub(_opts: GlobalOptions, _dir?: string) {
+  log.info(
     colors.red.underline(
-      'Push is deprecated. Use "sync push --raw" instead. See <TODO_LINK_HERE> for more information.',
-    ),
+      'Push is deprecated. Use "sync push --raw" instead. See <TODO_LINK_HERE> for more information.'
+    )
   );
 }
 

cli/resource-type.ts

@@ -9,6 +9,7 @@ import { requireLogin, resolveWorkspace } from "./context.ts";
 import {
   colors,
   Command,
+  log,
   ResourceService,
   ResourceType,
   Table,
@@ -23,9 +24,20 @@ export async function pushResourceType(
   workspace: string,
   remotePath: string,
   resource: ResourceTypeFile | ResourceType | undefined,
-  localResource: ResourceTypeFile
+  localResource: ResourceTypeFile,
+  raw: boolean
 ): Promise<void> {
   remotePath = removeType(remotePath, "resource-type");
+  if (raw) {
+    try {
+      resource = await ResourceService.getResourceType({
+        workspace: workspace,
+        path: remotePath,
+      });
+    } catch {
+      // resource type doesn't exist
+    }
+  }
   if (resource) {
     if (isSuperset(localResource, resource)) {
       return;
@@ -39,7 +51,7 @@ export async function pushResourceType(
       },
     });
   } else {
-    console.log(colors.yellow.bold("Creating new resource type..."));
+    log.info(colors.yellow.bold("Creating new resource type..."));
     await ResourceService.createResourceType({
       workspace: workspace,
       requestBody: {
@@ -59,25 +71,16 @@ async function push(opts: PushOptions, filePath: string, name: string) {
   const workspace = await resolveWorkspace(opts);
   await requireLogin(opts);
 
-  console.log(colors.bold.yellow("Pushing resource..."));
-
-  let resourceType: ResourceType | undefined = undefined;
-  try {
-    resourceType = await ResourceService.getResourceType({
-      workspace: workspace.workspaceId,
-      path: name,
-    });
-  } catch {
-    // resource type doesn't exist
-  }
+  log.info(colors.bold.yellow("Pushing resource..."));
 
   await pushResourceType(
     workspace.workspaceId,
     name,
-    resourceType,
-    parseFromFile(filePath)
+    undefined,
+    parseFromFile(filePath),
+    true
   );
-  console.log(colors.bold.underline.green("Resource pushed"));
+  log.info(colors.bold.underline.green("Resource pushed"));
 }
 
 async function list(opts: GlobalOptions) {

cli/resource.ts

@@ -6,7 +6,14 @@ import {
   removeType,
 } from "./types.ts";
 import { requireLogin, resolveWorkspace, validatePath } from "./context.ts";
-import { colors, Command, Resource, ResourceService, Table } from "./deps.ts";
+import {
+  colors,
+  Command,
+  log,
+  Resource,
+  ResourceService,
+  Table,
+} from "./deps.ts";
 
 export interface ResourceFile {
   value: any;
@@ -19,10 +26,20 @@ export async function pushResource(
   workspace: string,
   remotePath: string,
   resource: ResourceFile | Resource | undefined,
-  localResource: ResourceFile
+  localResource: ResourceFile,
+  raw: boolean
 ): Promise<void> {
   remotePath = removeType(remotePath, "resource");
-
+  if (raw) {
+    try {
+      resource = await ResourceService.getResource({
+        workspace: workspace,
+        path: remotePath,
+      });
+    } catch {
+      // flow doesn't exist
+    }
+  }
   if (resource) {
     if (isSuperset(localResource, resource)) {
       return;
@@ -35,14 +52,14 @@ export async function pushResource(
     });
   } else {
     if (localResource.is_oauth) {
-      console.log(
+      log.info(
         colors.yellow(
           "! is_oauth has been removed in newer versions. Ignoring."
         )
       );
     }
 
-    console.log(colors.yellow.bold("Creating new resource..."));
+    log.info(colors.yellow.bold("Creating new resource..."));
     await ResourceService.createResource({
       workspace: workspace,
       requestBody: {
@@ -67,24 +84,16 @@ async function push(opts: PushOptions, filePath: string, remotePath: string) {
     throw new Error("file path must refer to a file.");
   }
 
-  console.log(colors.bold.yellow("Pushing resource..."));
-  let resource: Resource | undefined = undefined;
-  try {
-    resource = await ResourceService.getResource({
-      workspace: workspace.workspaceId,
-      path: remotePath,
-    });
-  } catch {
-    // flow doesn't exist
-  }
+  log.info(colors.bold.yellow("Pushing resource..."));
 
   await pushResource(
     workspace.workspaceId,
     remotePath,
-    resource,
-    parseFromFile(filePath)
+    undefined,
+    parseFromFile(filePath),
+    true
   );
-  console.log(colors.bold.underline.green(`Resource ${remotePath} pushed`));
+  log.info(colors.bold.underline.green(`Resource ${remotePath} pushed`));
 }
 
 async function list(opts: GlobalOptions) {

cli/script.ts

@@ -1,17 +1,19 @@
 // deno-lint-ignore-file no-explicit-any
-import { GlobalOptions, parseFromFile } from "./types.ts";
+import { GlobalOptions } from "./types.ts";
 import { requireLogin, resolveWorkspace, validatePath } from "./context.ts";
 import {
   colors,
   Command,
   JobService,
+  log,
+  NewScript,
   readAll,
   Script,
   ScriptService,
   Table,
+  writeAllSync,
+  yamlParse,
 } from "./deps.ts";
-import { writeAllSync } from "https://deno.land/std@0.176.0/streams/mod.ts";
-import { parse as yamlParse } from "https://deno.land/std@0.184.0/yaml/mod.ts";
 
 export interface ScriptFile {
   parent_hash?: string;
@@ -26,9 +28,8 @@ export interface ScriptFile {
 type PushOptions = GlobalOptions;
 async function push(opts: PushOptions, filePath: string) {
   const workspace = await resolveWorkspace(opts);
-  const remotePath = filePath.split(".")[0];
 
-  if (!validatePath(remotePath)) {
+  if (!validatePath(filePath)) {
     return;
   }
 
@@ -36,19 +37,16 @@ async function push(opts: PushOptions, filePath: string) {
   if (!fstat.isFile) {
     throw new Error("file path must refer to a file.");
   }
-  let contentPath: string;
-  let metaPath: string | undefined;
+
   if (filePath.endsWith(".script.json") || filePath.endsWith(".script.yaml")) {
-    metaPath = filePath;
-    contentPath = await findContentFile(filePath);
-  } else {
-    contentPath = filePath;
-    metaPath = undefined;
+    throw Error(
+      "Cannot push a script metadata file, point to the script content file instead (.py, .ts, .go|.sh)"
+    );
   }
 
   await requireLogin(opts);
-  await pushScript(metaPath, contentPath, workspace.workspaceId, remotePath);
-  console.log(colors.bold.underline.green(`Script ${remotePath} pushed`));
+  await handleFile(filePath, workspace.workspaceId, []);
+  log.info(colors.bold.underline.green(`Script ${filePath} pushed`));
 }
 
 export async function handleScriptMetadata(
@@ -58,12 +56,7 @@ export async function handleScriptMetadata(
 ): Promise<boolean> {
   if (path.endsWith(".script.json") || path.endsWith(".script.yaml")) {
     const contentPath = await findContentFile(path);
-    return handleFile(
-      contentPath,
-      await Deno.readTextFile(contentPath),
-      workspace,
-      alreadySynced
-    );
+    return handleFile(contentPath, workspace, alreadySynced);
   } else {
     return false;
   }
@@ -71,19 +64,21 @@ export async function handleScriptMetadata(
 
 export async function handleFile(
   path: string,
-  content: string,
   workspace: string,
   alreadySynced: string[]
 ): Promise<boolean> {
   if (
-    path.endsWith(".ts") ||
-    path.endsWith(".py") ||
-    path.endsWith(".go") ||
-    path.endsWith(".sh")
+    !path.includes(".inline_script.") &&
+    (path.endsWith(".ts") ||
+      path.endsWith(".py") ||
+      path.endsWith(".go") ||
+      path.endsWith(".sh"))
   ) {
     if (alreadySynced.includes(path)) {
       return true;
     }
+    log.debug(`Processing local script ${path}`);
+
     alreadySynced.push(path);
     const remotePath = path.substring(0, path.length - 3);
     const metaPath = remotePath + ".script.json";
@@ -109,9 +104,11 @@ export async function handleFile(
         workspace,
         path: remotePath,
       });
+      log.debug(`Script ${remotePath} exists on remote`);
     } catch {
-      // no remote script
+      log.debug(`Script ${remotePath} does not exist on remote`);
     }
+    const content = await Deno.readTextFile(path);
 
     if (remote) {
       if (content === remote.content) {
@@ -125,7 +122,7 @@ export async function handleFile(
             remote?.lock == typed.lock?.join("\n") &&
             JSON.stringify(typed.schema) == JSON.stringify(remote.schema))
         ) {
-          console.log(
+          log.info(
             colors.yellow(
               `No change to push for script ${remotePath}, skipping`
             )
@@ -133,13 +130,15 @@ export async function handleFile(
           return true;
         }
       }
-      console.log(typed);
+      log.info(
+        colors.yellow.bold(`Creating script with a parent ${remotePath}`)
+      );
       await ScriptService.createScript({
         workspace,
         requestBody: {
           content,
           description: typed?.description ?? "",
-          language,
+          language: language as NewScript.language,
           path: remotePath,
           summary: typed?.summary ?? "",
           is_template: typed?.is_template,
@@ -149,18 +148,17 @@ export async function handleFile(
           schema: typed?.schema,
         },
       });
-
-      console.log(
-        colors.yellow.bold(`Creating script with a parent ${remotePath}`)
-      );
     } else {
+      log.info(
+        colors.yellow.bold(`Creating script without parent ${remotePath}`)
+      );
       // no parent hash
       await ScriptService.createScript({
         workspace: workspace,
         requestBody: {
           content,
           description: typed?.description ?? "",
-          language,
+          language: language as NewScript.language,
           path: remotePath,
           summary: typed?.summary ?? "",
           is_template: typed?.is_template,
@@ -170,9 +168,6 @@ export async function handleFile(
           schema: typed?.schema,
         },
       });
-      console.log(
-        colors.yellow.bold(`Creating script without parent ${remotePath}`)
-      );
     }
     return true;
   }
@@ -238,50 +233,6 @@ export function inferContentTypeFromFilePath(
   return language;
 }
 
-export async function pushScript(
-  filePath: string | undefined,
-  contentPath: string,
-  workspace: string,
-  remotePath: string
-) {
-  const data: ScriptFile | undefined = filePath
-    ? parseFromFile(filePath)
-    : undefined;
-  const content = await Deno.readTextFile(contentPath);
-
-  const language = inferContentTypeFromFilePath(contentPath);
-  let parent_hash = data?.parent_hash;
-  if (!parent_hash) {
-    try {
-      parent_hash = (
-        await ScriptService.getScriptByPath({
-          workspace: workspace,
-          path: remotePath,
-        })
-      ).hash;
-    } catch {
-      /* no parent. New Script. */
-    }
-  }
-
-  console.log(colors.bold.yellow("Pushing script..."));
-  await ScriptService.createScript({
-    workspace: workspace,
-    requestBody: {
-      path: remotePath,
-      summary: data?.summary ?? "",
-      content: content,
-      description: data?.description ?? "",
-      language: language,
-      is_template: data?.is_template,
-      kind: data?.kind,
-      lock: data?.lock,
-      parent_hash: parent_hash,
-      schema: data?.schema,
-    },
-  });
-}
-
 async function list(opts: GlobalOptions & { showArchived?: boolean }) {
   const workspace = await resolveWorkspace(opts);
   await requireLogin(opts);
@@ -360,7 +311,7 @@ async function run(
             id,
           })
         ).result ?? {};
-      console.log(result);
+      log.info(result);
 
       break;
     } catch {
@@ -373,16 +324,16 @@ export async function track_job(workspace: string, id: string) {
   try {
     const result = await JobService.getCompletedJob({ workspace, id });
 
-    console.log(result.logs);
-    console.log();
-    console.log(colors.bold.underline.green("Job Completed"));
-    console.log();
+    log.info(result.logs);
+    log.info("\n");
+    log.info(colors.bold.underline.green("Job Completed"));
+    log.info("\n");
     return;
   } catch {
     /* ignore */
   }
 
-  console.log(colors.yellow("Waiting for Job " + id + " to start..."));
+  log.info(colors.yellow("Waiting for Job " + id + " to start..."));
 
   let logOffset = 0;
   let running = false;
@@ -403,7 +354,7 @@ export async function track_job(workspace: string, id: string) {
     } catch {
       retry++;
       if (retry > 3) {
-        console.log("failed to get job updated. skipping log streaming.");
+        log.info("failed to get job updated. skipping log streaming.");
         break;
       }
       continue;
@@ -411,7 +362,7 @@ export async function track_job(workspace: string, id: string) {
 
     if (!running && updates.running === true) {
       running = true;
-      console.log(colors.green("Job running. Streaming logs..."));
+      log.info(colors.green("Job running. Streaming logs..."));
     }
 
     if (updates.new_logs) {
@@ -426,9 +377,7 @@ export async function track_job(workspace: string, id: string) {
 
     if (running && updates.running === false) {
       running = false;
-      console.log(
-        colors.yellow("Job suspended. Waiting for it to continue...")
-      );
+      log.info(colors.yellow("Job suspended. Waiting for it to continue..."));
     }
   }
   await new Promise((resolve, _) => setTimeout(() => resolve(undefined), 1000));
@@ -436,17 +385,17 @@ export async function track_job(workspace: string, id: string) {
   try {
     const final_job = await JobService.getCompletedJob({ workspace, id });
     if ((final_job.logs?.length ?? -1) > logOffset) {
-      console.log(final_job.logs!.substring(logOffset));
+      log.info(final_job.logs!.substring(logOffset));
     }
-    console.log("\n");
+    log.info("\n");
     if (final_job.success) {
-      console.log(colors.bold.underline.green("Job Completed"));
+      log.info(colors.bold.underline.green("Job Completed"));
     } else {
-      console.log(colors.bold.underline.red("Job Completed"));
+      log.info(colors.bold.underline.red("Job Completed"));
     }
-    console.log();
+    log.info("\n");
   } catch {
-    console.log("Job appears to have completed, but no data can be retrieved");
+    log.info("Job appears to have completed, but no data can be retrieved");
   }
 }
 
@@ -457,10 +406,10 @@ async function show(opts: GlobalOptions, path: string) {
     workspace: workspace.workspaceId,
     path,
   });
-  console.log(colors.underline(s.path));
-  if (s.description) console.log(s.description);
-  console.log("");
-  console.log(s.content);
+  log.info(colors.underline(s.path));
+  if (s.description) log.info(s.description);
+  log.info("");
+  log.info(s.content);
 }
 
 const command = new Command()
@@ -469,9 +418,9 @@ const command = new Command()
   .action(list as any)
   .command(
     "push",
-    "push a local script spec. This overrides any remote versions. Can use a script file (.ts, .js, .py, .sh) or a script spec file (.json). "
+    "push a local script spec. This overrides any remote versions. Use the script file (.ts, .js, .py, .sh)"
   )
-  .arguments("<file_path:file>")
+  .arguments("<path:file>")
   .action(push as any)
   .command("show", "show a scripts content")
   .arguments("<path:file>")

cli/sync.ts

@@ -13,6 +13,12 @@ import {
   VariableService,
   AppService,
   FlowService,
+  OpenFlow,
+  FlowModule,
+  RawScript,
+  log,
+  yamlStringify,
+  yamlParse,
 } from "./deps.ts";
 import {
   getTypeStrFromPath,
@@ -27,11 +33,7 @@ import { downloadZip } from "./pull.ts";
 import { handleScriptMetadata } from "./script.ts";
 
 import { handleFile } from "./script.ts";
-import { equal } from "https://deno.land/x/equal@v1.5.0/mod.ts";
-import {
-  stringify as yamlStringify,
-  parse as yamlParse,
-} from "https://deno.land/std@0.184.0/yaml/mod.ts";
+import { deepEqual } from "./utils.ts";
 
 type DynFSElement = {
   isDirectory: boolean;
@@ -47,6 +49,7 @@ async function FSFSElement(p: string): Promise<DynFSElement> {
       isDirectory: isDir,
       path: localP.substring(p.length + 1),
       async *getChildren(): AsyncIterable<DynFSElement> {
+        if (!isDir) return [];
         for await (const e of Deno.readDir(localP)) {
           yield _internal_element(path.join(localP, e.name), e.isDirectory);
         }
@@ -56,9 +59,7 @@ async function FSFSElement(p: string): Promise<DynFSElement> {
       // },
       async getContentText(): Promise<string> {
         const content = await Deno.readTextFile(localP);
-        return p.endsWith(".yaml")
-          ? JSON.stringify(yamlParse(content))
-          : content;
+        return content;
       },
     };
   }
@@ -66,22 +67,125 @@ async function FSFSElement(p: string): Promise<DynFSElement> {
 }
 
 function prioritizeName(name: string): string {
-  if (name == "summary") return "aa";
-  if (name == "name") return "aaaa";
-  if (name == "display_name") return "aaa";
-  if (name == "description") return "ab";
-  if (name == "value") return "ac";
+  if (name == "id") return "aa";
+  if (name == "type") return "ab";
+  if (name == "summary") return "ad";
+  if (name == "name") return "ae";
+  if (name == "display_name") return "af";
+  if (name == "description") return "ag";
+  if (name == "value") return "ah";
+  if (name == "content") return "ai";
+  if (name == "modules") return "aj";
+  if (name == "failure_module") return "ak";
+  if (name == "input_transforms") return "al";
+  if (name == "lock") return "az";
+
   return name;
 }
 
+export const yamlOptions = {
+  sortKeys: (a: any, b: any) => {
+    return prioritizeName(a).localeCompare(prioritizeName(b));
+  },
+  noCompatMode: true,
+  noRefs: true,
+};
+
 function ZipFSElement(zip: JSZip, useYaml: boolean): DynFSElement {
   function _internal_file(p: string, f: JSZip.JSZipObject): DynFSElement {
+    const isFlow = p.endsWith("flow.json");
+    function transformPath() {
+      if (isFlow) {
+        return p.replace("flow.json", "flow");
+      } else {
+        return useYaml && p.endsWith(".json")
+          ? p.replaceAll(".json", ".yaml")
+          : p;
+      }
+    }
+
+    interface InlineScript {
+      path: string;
+      content: string;
+    }
+    let counter = 0;
+    const seen_names = new Set<string>();
+    function assignPath(
+      summary: string | undefined,
+      language: RawScript.language
+    ): string {
+      let name;
+      if (summary && summary != "" && !seen_names.has(summary)) {
+        name = summary.toLowerCase().replaceAll(" ", "_");
+        seen_names.add(name);
+      } else {
+        name = `inline_script_${counter}`;
+        while (seen_names.has(name)) {
+          counter++;
+          name = `inline_script_${counter}`;
+        }
+        seen_names.add(name);
+      }
+      let ext;
+      if (language == "python3") ext = "py";
+      else if (language == "deno") ext = "ts";
+      else if (language == "go") ext = "go";
+      else if (language == "bash") ext = "sh";
+      return `${name}.inline_script.${ext}`;
+    }
+    function extractInlineScripts(modules: FlowModule[]): InlineScript[] {
+      return modules.flatMap((m) => {
+        if (m.value.type == "rawscript") {
+          const path = assignPath(m.summary, m.value.language);
+          const content = m.value.content;
+          m.value.content = "!inline " + path;
+          return [{ path: path, content: content }];
+        } else if (m.value.type == "forloopflow") {
+          return extractInlineScripts(m.value.modules);
+        } else if (m.value.type == "branchall") {
+          return m.value.branches.flatMap((b) =>
+            extractInlineScripts(b.modules)
+          );
+        } else if (m.value.type == "branchone") {
+          return [
+            ...m.value.branches.flatMap((b) => extractInlineScripts(b.modules)),
+            ...extractInlineScripts(m.value.default),
+          ];
+        } else {
+          return [];
+        }
+      });
+    }
+    const flowPath = transformPath();
     return {
-      isDirectory: false,
-      path: useYaml && p.endsWith(".json") ? p.replaceAll(".json", ".yaml") : p,
-      // deno-lint-ignore require-yield
+      isDirectory: isFlow,
+      path: flowPath,
       async *getChildren(): AsyncIterable<DynFSElement> {
-        throw new Error("Cannot get children of file");
+        if (isFlow) {
+          const flow: OpenFlow = JSON.parse(await f.async("text"));
+          const inlineScripts = extractInlineScripts(flow.value.modules);
+          for (const s of inlineScripts) {
+            yield {
+              isDirectory: false,
+              path: path.join(flowPath, s.path),
+              async *getChildren() {},
+              // deno-lint-ignore require-await
+              async getContentText() {
+                return s.content;
+              },
+            };
+          }
+
+          yield {
+            isDirectory: false,
+            path: path.join(flowPath, "flow.yaml"),
+            async *getChildren() {},
+            // deno-lint-ignore require-await
+            async getContentText() {
+              return yamlStringify(flow, yamlOptions);
+            },
+          };
+        }
       },
       // async getContentBytes(): Promise<Uint8Array> {
       //   return await f.async("uint8array");
@@ -89,13 +193,7 @@ function ZipFSElement(zip: JSZip, useYaml: boolean): DynFSElement {
       async getContentText(): Promise<string> {
         const content = await f.async("text");
         return useYaml && p.endsWith(".json")
-          ? yamlStringify(JSON.parse(content), {
-              sortKeys: (a, b) => {
-                return prioritizeName(a).localeCompare(prioritizeName(b));
-              },
-              noCompatMode: true,
-              noRefs: true,
-            })
+          ? yamlStringify(JSON.parse(content), yamlOptions)
           : content;
       },
     };
@@ -164,7 +262,6 @@ async function* readDirRecursiveWithIgnore(
   while (stack.length > 0) {
     const e = stack.pop()!;
     yield e;
-    if (!e.isDirectory) continue;
     for await (const e2 of e.c()) {
       stack.push({
         path: e2.path,
@@ -186,11 +283,20 @@ type Change = Added | Deleted | Edit;
 
 async function elementsToMap(
   els: DynFSElement,
-  ignore: (path: string, isDirectory: boolean) => boolean
+  ignore: (path: string, isDirectory: boolean) => boolean,
+  json: boolean
 ): Promise<{ [key: string]: string }> {
   const map: { [key: string]: string } = {};
   for await (const entry of readDirRecursiveWithIgnore(ignore, els)) {
     if (entry.isDirectory || entry.ignored) continue;
+    if (json && entry.path.endsWith(".yaml")) continue;
+    if (!json && entry.path.endsWith(".json")) continue;
+    if (
+      !["json", "yaml", "go", "sh", "ts", "py"].includes(
+        entry.path.split(".").pop() ?? ""
+      )
+    )
+      continue;
     const content = await entry.getContentText();
     map[entry.path] = content;
   }
@@ -200,14 +306,15 @@ async function elementsToMap(
 async function compareDynFSElement(
   els1: DynFSElement,
   els2: DynFSElement | undefined,
-  ignore: (path: string, isDirectory: boolean) => boolean
+  ignore: (path: string, isDirectory: boolean) => boolean,
+  json: boolean
 ): Promise<Change[]> {
   const [m1, m2] = els2
     ? await Promise.all([
-        elementsToMap(els1, ignore),
-        elementsToMap(els2, ignore),
+        elementsToMap(els1, ignore, json),
+        elementsToMap(els2, ignore, json),
       ])
-    : [await elementsToMap(els1, ignore), {}];
+    : [await elementsToMap(els1, ignore, json), {}];
 
   const changes: Change[] = [];
 
@@ -216,8 +323,8 @@ async function compareDynFSElement(
       changes.push({ name: "added", path: k, content: v });
     } else if (
       m2[k] != v &&
-      (!k.endsWith(".json") || !equal(JSON.parse(v), JSON.parse(m2[k]))) &&
-      (!k.endsWith(".yaml") || !equal(yamlParse(v), yamlParse(m2[k])))
+      (!k.endsWith(".json") || !deepEqual(JSON.parse(v), JSON.parse(m2[k]))) &&
+      (!k.endsWith(".yaml") || !deepEqual(yamlParse(v), yamlParse(m2[k])))
     ) {
       changes.push({ name: "edited", path: k, after: v, before: m2[k] });
     }
@@ -280,7 +387,7 @@ async function pull(
     yes: boolean;
     failConflicts: boolean;
     plainSecrets?: boolean;
-    yaml?: boolean;
+    json?: boolean;
   }
 ) {
   if (!opts.raw) {
@@ -290,21 +397,26 @@ async function pull(
   const workspace = await resolveWorkspace(opts);
   await requireLogin(opts);
 
-  console.log(
+  log.info(
     colors.gray(
       "Computing the files to update locally to match remote (taking .wmillignore into account)"
     )
   );
   const remote = ZipFSElement(
     (await downloadZip(workspace, opts.plainSecrets))!,
-    opts.yaml ?? false
+    !opts.json
   );
   const local = opts.raw
     ? undefined
-    : await FSFSElement(path.join(Deno.cwd(), opts.raw ? "" : ".wmill"));
-  const changes = await compareDynFSElement(remote, local, await ignoreF());
+    : await FSFSElement(path.join(Deno.cwd(), ".wmill"));
+  const changes = await compareDynFSElement(
+    remote,
+    local,
+    await ignoreF(),
+    opts.json ?? false
+  );
 
-  console.log(
+  log.info(
     `remote (${workspace.name}) -> local: ${changes.length} changes to apply`
   );
   if (changes.length > 0) {
@@ -321,7 +433,7 @@ async function pull(
     }
 
     const conflicts = [];
-    console.log(colors.gray(`Applying changes to files ...`));
+    log.info(colors.gray(`Applying changes to files ...`));
     for await (const change of changes) {
       const target = path.join(Deno.cwd(), change.path);
       const stateTarget = path.join(Deno.cwd(), ".wmill", change.path);
@@ -329,7 +441,7 @@ async function pull(
         try {
           const currentLocal = await Deno.readTextFile(target);
           if (currentLocal !== change.before && currentLocal !== change.after) {
-            console.log(
+            log.info(
               colors.red(
                 `Conflict detected on ${change.path}\nBoth local and remote have been modified.`
               )
@@ -342,7 +454,7 @@ async function pull(
               });
               continue;
             } else if (opts.yes) {
-              console.log(
+              log.info(
                 colors.red(
                   `Override local version with remote since --yes was passed and no --fail-conflicts.`
                 )
@@ -362,11 +474,9 @@ async function pull(
           // ignore
         }
         if (!change.path.endsWith(".json") && !change.path.endsWith(".yaml")) {
-          console.log(`Editing script content of ${change.path}`);
+          log.info(`Editing script content of ${change.path}`);
         } else {
-          console.log(
-            `Editing ${getTypeStrFromPath(change.path)} ${change.path}`
-          );
+          log.info(`Editing ${getTypeStrFromPath(change.path)} ${change.path}`);
         }
         await Deno.writeTextFile(target, change.after);
 
@@ -378,9 +488,7 @@ async function pull(
         await ensureDir(path.dirname(target));
         if (!opts.raw) {
           await ensureDir(path.dirname(stateTarget));
-          console.log(
-            `Adding ${getTypeStrFromPath(change.path)} ${change.path}`
-          );
+          log.info(`Adding ${getTypeStrFromPath(change.path)} ${change.path}`);
         }
         await Deno.writeTextFile(target, change.content);
         if (!opts.raw) {
@@ -388,7 +496,7 @@ async function pull(
         }
       } else if (change.name === "deleted") {
         try {
-          console.log(
+          log.info(
             `Deleting ${getTypeStrFromPath(change.path)} ${change.path}`
           );
           await Deno.remove(target);
@@ -405,11 +513,11 @@ async function pull(
     if (opts.failConflicts) {
       if (conflicts.length > 0) {
         console.error(colors.red(`Conflicts were found`));
-        console.log("Conflicts:");
+        log.info("Conflicts:");
         for (const conflict of conflicts) {
           showConflict(conflict.path, conflict.local, conflict.change.after);
         }
-        console.log(
+        log.info(
           colors.red(`Please resolve theses conflicts manually by either:
   - reverting the content back to its remote (\`wmill pull\` and refuse to preserve local when prompted)
   - pushing the changes with \`wmill push --skip-pull\` to override wmill with all your local changes
@@ -418,7 +526,7 @@ async function pull(
         Deno.exit(1);
       }
     }
-    console.log(
+    log.info(
       colors.green.underline(
         `Done! All ${changes.length} changes applied locally.`
       )
@@ -429,15 +537,15 @@ async function pull(
 function prettyChanges(changes: Change[]) {
   for (const change of changes) {
     if (change.name === "added") {
-      console.log(
+      log.info(
         colors.green(`+ ${getTypeStrFromPath(change.path)} ` + change.path)
       );
     } else if (change.name === "deleted") {
-      console.log(
+      log.info(
         colors.red(`- ${getTypeStrFromPath(change.path)} ` + change.path)
       );
     } else if (change.name === "edited") {
-      console.log(
+      log.info(
         colors.yellow(`~ ${getTypeStrFromPath(change.path)} ` + change.path)
       );
       showDiff(change.before, change.after);
@@ -459,10 +567,10 @@ function prettyChanges(changes: Change[]) {
 //       }
 //     }
 //     if (diff.type === "REMOVE" || diff.type === "CHANGE") {
-//       console.log(colors.red("- " + pathString + " = " + diff.oldValue));
+//      log.info(colors.red("- " + pathString + " = " + diff.oldValue));
 //     }
 //     if (diff.type === "CREATE" || diff.type === "CHANGE") {
-//       console.log(colors.green("+ " + pathString + " = " + diff.value));
+//      log.info(colors.green("+ " + pathString + " = " + diff.value));
 //     }
 //   }
 // }
@@ -478,24 +586,24 @@ async function push(
     skipPull: boolean;
     failConflicts: boolean;
     plainSecrets?: boolean;
-    yaml?: boolean;
+    json?: boolean;
   }
 ) {
   if (!opts.raw) {
     if (!opts.skipPull) {
-      console.log(
+      log.info(
         colors.gray("You need to be up-to-date before pushing, pulling first.")
       );
       await pull(opts);
-      console.log(colors.green("Pull done, now pushing."));
-      console.log();
+      log.info(colors.green("Pull done, now pushing."));
+      log.info("\n");
     }
   }
 
   const workspace = await resolveWorkspace(opts);
   await requireLogin(opts);
 
-  console.log(
+  log.info(
     colors.gray(
       "Computing the files to update on the remote to match local (taking .wmillignore into account)"
     )
@@ -504,12 +612,17 @@ async function push(
     ? undefined
     : ZipFSElement(
         (await downloadZip(workspace, opts.plainSecrets))!,
-        opts.yaml ?? false
+        !opts.json
       );
   const local = await FSFSElement(path.join(Deno.cwd(), ""));
-  const changes = await compareDynFSElement(local, remote, await ignoreF());
+  const changes = await compareDynFSElement(
+    local,
+    remote,
+    await ignoreF(),
+    opts.json ?? false
+  );
 
-  console.log(
+  log.info(
     `remote (${workspace.name}) <- local: ${changes.length} changes to apply`
   );
   if (changes.length > 0) {
@@ -523,7 +636,7 @@ async function push(
     ) {
       return;
     }
-    console.log(colors.gray(`Applying changes to files ...`));
+    log.info(colors.gray(`Applying changes to files ...`));
     const alreadySynced: string[] = [];
     for await (const change of changes) {
       const stateTarget = path.join(Deno.cwd(), ".wmill", change.path);
@@ -547,12 +660,7 @@ async function push(
           }
           continue;
         } else if (
-          await handleFile(
-            change.path,
-            change.after,
-            workspace.workspaceId,
-            alreadySynced
-          )
+          await handleFile(change.path, workspace.workspaceId, alreadySynced)
         ) {
           if (!opts.raw && stateExists) {
             await Deno.writeTextFile(stateTarget, change.after);
@@ -561,9 +669,7 @@ async function push(
         }
         if (!opts.raw) {
           await ensureDir(path.dirname(stateTarget));
-          console.log(
-            `Editing ${getTypeStrFromPath(change.path)} ${change.path}`
-          );
+          log.info(`Editing ${getTypeStrFromPath(change.path)} ${change.path}`);
         }
         const oldObj = parseFromPath(change.path, change.before);
         const newObj = parseFromPath(change.path, change.after);
@@ -573,7 +679,8 @@ async function push(
           change.path,
           oldObj,
           newObj,
-          opts.plainSecrets ?? false
+          opts.plainSecrets ?? false,
+          opts.raw
         );
 
         if (!opts.raw && stateExists) {
@@ -586,20 +693,13 @@ async function push(
         ) {
           continue;
         } else if (
-          await handleFile(
-            change.path,
-            change.content,
-            workspace.workspaceId,
-            alreadySynced
-          )
+          await handleFile(change.path, workspace.workspaceId, alreadySynced)
         ) {
           continue;
         }
         if (!opts.raw && stateExists) {
           await ensureDir(path.dirname(stateTarget));
-          console.log(
-            `Adding ${getTypeStrFromPath(change.path)} ${change.path}`
-          );
+          log.info(`Adding ${getTypeStrFromPath(change.path)} ${change.path}`);
         }
         const obj = parseFromPath(change.path, change.content);
         pushObj(
@@ -607,7 +707,8 @@ async function push(
           change.path,
           undefined,
           obj,
-          opts.plainSecrets ?? false
+          opts.plainSecrets ?? false,
+          opts.raw
         );
 
         if (!opts.raw && stateExists) {
@@ -617,9 +718,7 @@ async function push(
         if (!change.path.includes(".json") && !change.path.includes(".yaml")) {
           continue;
         }
-        console.log(
-          `Deleting ${getTypeStrFromPath(change.path)} ${change.path}`
-        );
+        log.info(`Deleting ${getTypeStrFromPath(change.path)} ${change.path}`);
         const typ = getTypeStrFromPath(change.path);
         const workspaceId = workspace.workspaceId;
         switch (typ) {
@@ -680,7 +779,7 @@ async function push(
         }
       }
     }
-    console.log(
+    log.info(
       colors.green.underline(
         `Done! All ${changes.length} changes pushed to the remote workspace ${workspace.workspaceId} named ${workspace.name}.`
       )
@@ -700,7 +799,7 @@ const command = new Command()
   .option("--yes", "Pull without needing confirmation")
   .option("--raw", "Pull without using state, just overwrite.")
   .option("--plain-secrets", "Pull secrets as plain text")
-  .option("--yaml", "Use YAML instead of JSON")
+  .option("--json", "Use JSON instead of YAML")
   // deno-lint-ignore no-explicit-any
   .action(pull as any)
   .command("push")
@@ -715,7 +814,7 @@ const command = new Command()
   .option("--yes", "Push without needing confirmation")
   .option("--raw", "Push without using state, just overwrite.")
   .option("--plain-secrets", "Push secrets as plain text")
-  .option("--yaml", "Use YAML instead of JSON")
+  .option("--json", "Use JSON instead of YAML")
   // deno-lint-ignore no-explicit-any
   .action(push as any);
 

cli/types.ts

@@ -1,19 +1,16 @@
 // deno-lint-ignore-file no-explicit-any
 
-import { colors, path } from "./deps.ts";
+import { colors, log, path, yamlParse, yamlStringify } from "./deps.ts";
 import { pushApp } from "./apps.ts";
-import {
-  parse as yamlParse,
-  stringify as yamlStringify,
-} from "https://deno.land/std@0.184.0/yaml/mod.ts";
-import { equal } from "https://deno.land/x/equal@v1.5.0/equal.ts";
 import { pushFolder } from "./folder.ts";
-import { pushScript } from "./script.ts";
 import { pushFlow } from "./flow.ts";
 import { pushResource } from "./resource.ts";
 import { pushResourceType } from "./resource-type.ts";
 import { pushVariable } from "./variable.ts";
 import * as Diff from "npm:diff";
+import { yamlOptions } from "./sync.ts";
+import { showDiffs } from "./main.ts";
+import { deepEqual } from "./utils.ts";
 
 export interface DifferenceCreate {
   type: "CREATE";
@@ -46,12 +43,19 @@ export function isSuperset(
   superset: Record<string, any>
 ): boolean {
   return Object.keys(subset).every((key) => {
-    const eq = equal(subset[key], superset[key]);
-    if (!eq) {
-      console.log(
-        `Found diff for ${key}:`,
-        showDiff(yamlStringify(subset[key]), yamlStringify(superset[key]))
-      );
+    const eq = deepEqual(subset[key], superset[key]);
+    if (!eq && showDiffs) {
+      const sub = subset[key];
+      const supers = superset[key];
+      if (!supers) {
+        log.info(`Key ${key} not found in remote`);
+      } else {
+        log.info(`Found diff for ${key}:`);
+        showDiff(
+          yamlStringify(sub, yamlOptions),
+          yamlStringify(supers, yamlOptions)
+        );
+      }
     }
     return eq;
   });
@@ -67,18 +71,25 @@ export function showDiff(local: string, remote: string) {
       // print green if added
       finalString += `\x1b[32m${part.value}\x1b[0m`;
     } else {
+      let lines = part.value.split("\n");
+
+      if (lines.length > 4) {
+        lines = lines.slice(0, 2);
+        lines.push("...");
+        lines = lines.concat(part.value.split("\n").slice(-2));
+      }
       // print white if unchanged
-      finalString += `\x1b[37m${part.value}\x1b[0m`;
+      finalString += `\x1b[37m${lines.join("\n")}\x1b[0m`;
     }
   }
-  console.log(finalString);
+  log.info(finalString);
 }
 
 export function showConflict(path: string, local: string, remote: string) {
-  console.log(colors.yellow(`- ${path}`));
+  log.info(colors.yellow(`- ${path}`));
   showDiff(local, remote);
-  console.log("\x1b[31mlocal\x1b[31m - \x1b[32mremote\x1b[32m");
-  console.log();
+  log.info("\x1b[31mlocal\x1b[31m - \x1b[32mremote\x1b[32m");
+  log.info("\n");
 }
 
 export function pushObj(
@@ -86,31 +97,35 @@ export function pushObj(
   p: string,
   befObj: any,
   newObj: any,
-  plainSecrets: boolean
+  plainSecrets: boolean,
+  checkForCreate: boolean
 ) {
   const typeEnding = getTypeStrFromPath(p);
 
   if (typeEnding === "app") {
-    pushApp(workspace, p, befObj, newObj);
+    pushApp(workspace, p, befObj, newObj, checkForCreate);
   } else if (typeEnding === "folder") {
-    pushFolder(workspace, p, befObj, newObj);
-  } else if (typeEnding === "script") {
-    pushScript(workspace, p, befObj, newObj);
+    pushFolder(workspace, p, befObj, newObj, checkForCreate);
   } else if (typeEnding === "variable") {
-    pushVariable(workspace, p, befObj, newObj, plainSecrets);
+    pushVariable(workspace, p, befObj, newObj, plainSecrets, checkForCreate);
   } else if (typeEnding === "flow") {
-    pushFlow(workspace, p, befObj, newObj);
+    const flowName = p.split(".flow/")[0];
+    pushFlow(workspace, flowName, flowName + ".flow");
   } else if (typeEnding === "resource") {
-    pushResource(workspace, p, befObj, newObj);
+    pushResource(workspace, p, befObj, newObj, checkForCreate);
   } else if (typeEnding === "resource-type") {
-    pushResourceType(workspace, p, befObj, newObj);
+    pushResourceType(workspace, p, befObj, newObj, checkForCreate);
   } else {
     throw new Error("infer type unreachable");
   }
 }
 
 export function parseFromPath(p: string, content: string): any {
-  return p.endsWith(".yaml") ? yamlParse(content) : JSON.parse(content);
+  return p.endsWith(".yaml")
+    ? yamlParse(content)
+    : p.endsWith(".json")
+    ? JSON.parse(content)
+    : content;
 }
 export function parseFromFile(p: string): any {
   if (p.endsWith(".json")) {
@@ -131,6 +146,9 @@ export function getTypeStrFromPath(
   | "resource-type"
   | "folder"
   | "app" {
+  if (p.includes(".flow/")) {
+    return "flow";
+  }
   const parsed = path.parse(p);
   if (
     parsed.ext == ".go" ||
@@ -149,7 +167,6 @@ export function getTypeStrFromPath(
   if (
     typeEnding === "script" ||
     typeEnding === "variable" ||
-    typeEnding === "flow" ||
     typeEnding === "resource" ||
     typeEnding === "resource-type" ||
     typeEnding === "app"

cli/user.ts

@@ -5,6 +5,7 @@ import {
   colors,
   Command,
   GlobalUserInfo,
+  log,
   passwordGenerator,
   Table,
   UserService,
@@ -36,7 +37,7 @@ async function list(opts: GlobalOptions) {
         x.company ?? "-",
         x.verified ? "true" : "false",
         x.super_admin ? "true" : "false",
-      ]),
+      ])
     )
     .render();
 }
@@ -48,7 +49,7 @@ async function add(
     name?: string;
   },
   email: string,
-  password?: string,
+  password?: string
 ) {
   await requireLogin(opts);
   const password_final = password ?? passwordGenerator("*", 15);
@@ -62,33 +63,34 @@ async function add(
     },
   });
   if (!password) {
-    console.log(colors.red("New Password: " + password_final));
+    log.info(colors.red("New Password: " + password_final));
   }
-  console.log(colors.underline.green("New User Created."));
+  log.info(colors.underline.green("New User Created."));
 }
 async function remove(opts: GlobalOptions, email: string) {
   await requireLogin(opts);
 
   await UserService.globalUserDelete({ email });
-  console.log(colors.green("Deleted User " + email));
+  log.info(colors.green("Deleted User " + email));
 }
 
 async function createToken(
-  opts: GlobalOptions & { email: string; password: string },
+  opts: GlobalOptions & { email: string; password: string }
 ) {
   if (opts.email && opts.password) {
-    console.log(
-      "Token: " + await UserService.login({
-        requestBody: {
-          email: opts.email,
-          password: opts.password,
-        },
-      }),
+    log.info(
+      "Token: " +
+        (await UserService.login({
+          requestBody: {
+            email: opts.email,
+            password: opts.password,
+          },
+        }))
     );
   }
 
   await requireLogin(opts);
-  console.log("Token: " + await UserService.createToken({ requestBody: {} }));
+  log.info("Token: " + (await UserService.createToken({ requestBody: {} })));
 }
 
 const command = new Command()
@@ -99,7 +101,7 @@ const command = new Command()
   .option("--superadmin", "Specify to make the new user superadmin.")
   .option(
     "--company <company:string>",
-    "Specify to set the company of the new user.",
+    "Specify to set the company of the new user."
   )
   .option("--name <name:string>", "Specify to set the name of the new user.")
   .action(add as any)
@@ -112,14 +114,14 @@ const command = new Command()
     "Specify credentials to use for authentication. This will not be stored. It will only be used to exchange for a token with the API server, which will not be stored either.",
     {
       depends: ["password"],
-    },
+    }
   )
   .option(
     "--password <password:string>",
     "Specify credentials to use for authentication. This will not be stored. It will only be used to exchange for a token with the API server, which will not be stored either.",
     {
       depends: ["email"],
-    },
+    }
   )
   .action(createToken as any);
 

cli/utils.ts

@@ -0,0 +1,76 @@
+// Modified from: https://raw.githubusercontent.com/epoberezkin/fast-deep-equal/master/src/index.jst
+// eslint-disable-next-line @typescript-eslint/ban-ts-comment
+// @ts-nocheck This file is copied from a JS project, so it's not type-safe.
+export function deepEqual<T>(a: T, b: T): boolean {
+  if (a === b) return true;
+
+  if (a && b && typeof a === "object" && typeof b === "object") {
+    if (a.constructor !== b.constructor) return false;
+
+    let length, i;
+    if (Array.isArray(a)) {
+      length = a.length;
+      if (length != b.length) return false;
+      for (i = length; i-- !== 0; ) {
+        if (!deepEqual(a[i], b[i])) return false;
+      }
+      return true;
+    }
+
+    if (a instanceof Map && b instanceof Map) {
+      if (a.size !== b.size) return false;
+      for (i of a.entries()) {
+        if (!b.has(i[0])) return false;
+      }
+      for (i of a.entries()) {
+        if (!deepEqual(i[1], b.get(i[0]))) return false;
+      }
+      return true;
+    }
+
+    if (a instanceof Set && b instanceof Set) {
+      if (a.size !== b.size) return false;
+      for (i of a.entries()) {
+        if (!b.has(i[0])) return false;
+      }
+      return true;
+    }
+
+    if (ArrayBuffer.isView(a) && ArrayBuffer.isView(b)) {
+      length = a.length;
+      if (length != b.length) return false;
+      for (i = length; i-- !== 0; ) {
+        if (a[i] !== b[i]) return false;
+      }
+      return true;
+    }
+
+    if (a.constructor === RegExp) {
+      return a.source === b.source && a.flags === b.flags;
+    }
+    if (a.valueOf !== Object.prototype.valueOf) {
+      return a.valueOf() === b.valueOf();
+    }
+    if (a.toString !== Object.prototype.toString) {
+      return a.toString() === b.toString();
+    }
+
+    const keys = Object.keys(a);
+    length = keys.length;
+    if (length !== Object.keys(b).length) return false;
+
+    for (i = length; i-- !== 0; ) {
+      if (!Object.prototype.hasOwnProperty.call(b, keys[i])) return false;
+    }
+
+    for (i = length; i-- !== 0; ) {
+      const key = keys[i];
+      if (!deepEqual(a[key], b[key])) return false;
+    }
+
+    return true;
+  }
+
+  // true if both NaN, false otherwise
+  return a !== a && b !== b;
+}

cli/variable.ts

@@ -1,10 +1,16 @@
 // deno-lint-ignore-file no-explicit-any
 import { requireLogin, resolveWorkspace, validatePath } from "./context.ts";
-import { GlobalOptions, isSuperset, parseFromFile } from "./types.ts";
+import {
+  GlobalOptions,
+  isSuperset,
+  parseFromFile,
+  removeType,
+} from "./types.ts";
 import {
   colors,
   Command,
   ListableVariable,
+  log,
   Table,
   VariableService,
 } from "./deps.ts";
@@ -45,12 +51,31 @@ export async function pushVariable(
   remotePath: string,
   variable: VariableFile | ListableVariable | undefined,
   localVariable: VariableFile,
-  plainSecrets: boolean
+  plainSecrets: boolean,
+  raw: boolean
 ): Promise<void> {
+  remotePath = removeType(remotePath, "variable");
+  log.debug(`Processing local variable ${remotePath}`);
+
+  if (raw) {
+    try {
+      variable = await VariableService.getVariable({
+        workspace: workspace,
+        path: remotePath,
+        decryptSecret: plainSecrets,
+      });
+      log.debug(`Variable ${remotePath} exists on remote`);
+    } catch {
+      log.debug(`Variable ${remotePath} does not exist on remote`);
+    }
+  }
+
   if (variable) {
     if (isSuperset(localVariable, variable)) {
+      log.debug(`Variable ${remotePath} is up-to-date`);
       return;
     }
+    log.debug(`Variable ${remotePath} is not up-to-date, updating`);
 
     await VariableService.updateVariable({
       workspace,
@@ -58,10 +83,12 @@ export async function pushVariable(
       alreadyEncrypted: !plainSecrets,
       requestBody: {
         ...localVariable,
+        is_secret:
+          localVariable.is_secret && !variable.is_secret ? true : undefined,
       },
     });
   } else {
-    console.log(colors.yellow.bold("Creating new variable..."));
+    log.info(colors.yellow.bold(`Creating new variable ${remotePath}...`));
     await VariableService.createVariable({
       workspace,
       alreadyEncrypted: !plainSecrets,
@@ -90,26 +117,17 @@ async function push(
     throw new Error("file path must refer to a file.");
   }
 
-  console.log(colors.bold.yellow("Pushing variable..."));
-
-  let variable: ListableVariable | undefined = undefined;
-  try {
-    variable = await VariableService.getVariable({
-      workspace: workspace.workspaceId,
-      path: remotePath,
-    });
-  } catch {
-    // resource type doesn't exist
-  }
+  log.info(colors.bold.yellow("Pushing variable..."));
 
   await pushVariable(
     workspace.workspaceId,
     remotePath,
-    variable,
+    undefined,
     parseFromFile(filePath),
-    opts.plainSecrets
+    opts.plainSecrets,
+    true
   );
-  console.log(colors.bold.underline.green(`Variable ${remotePath} pushed`));
+  log.info(colors.bold.underline.green(`Variable ${remotePath} pushed`));
 }
 
 const command = new Command()

cli/vhs/example.flow.json

@@ -9,7 +9,7 @@
           "lock": "",
           "path": null,
           "type": "rawscript",
-          "content": "// import * as wmill from \"https://deno.land/x/windmill@v1.69.3/mod.ts\"\n\nexport async function main(x: string) {\n  console.log(\"Hello from Deno! The argument x is \" + x);\n  return x;\n}\n",
+          "content": "// import * as wmill from \"https://deno.land/x/windmill@v1.69.3/mod.ts\"\n\nexport async function main(x: string) {\n log.info(\"Hello from Deno! The argument x is \" + x);\n  return x;\n}\n",
           "language": "deno",
           "input_transforms": {
             "x": {

cli/workspace.ts

@@ -7,6 +7,7 @@ import {
   Command,
   DelimiterStream,
   Input,
+  log,
   setClient,
   Table,
   UserService,
@@ -122,12 +123,12 @@ async function list(opts: GlobalOptions) {
     )
     .render();
 
-  console.log("Active: " + colors.green.bold(activeName || "none"));
+  log.info("Active: " + colors.green.bold(activeName || "none"));
 }
 
 async function switchC(opts: GlobalOptions, workspaceName: string) {
   if (opts.workspace) {
-    console.log(
+    log.info(
       colors.red.bold(
         "! Workspace needs to be specified as positional argument, not as option."
       )
@@ -137,14 +138,14 @@ async function switchC(opts: GlobalOptions, workspaceName: string) {
 
   const all = await allWorkspaces();
   if (all.findIndex((x) => x.name === workspaceName) === -1) {
-    console.log(
+    log.info(
       colors.red.bold(
         `! This workspace profile ${workspaceName} does not exist locally.`
       )
     );
-    console.log("available workspace profiles:");
+    log.info("available workspace profiles:");
     for (const w of all) {
-      console.log("  - " + w.name);
+      log.info("  - " + w.name);
     }
     return;
   }
@@ -166,7 +167,7 @@ export async function add(
   remote: string | undefined
 ) {
   if (opts.workspace) {
-    console.log(
+    log.info(
       colors.red.bold(
         "! Workspace needs to be specified as positional argument, not as option."
       )
@@ -207,6 +208,10 @@ export async function add(
   remote = new URL(remote).toString(); // add trailing slash in all cases!
 
   let token = await tryGetLoginInfo(opts);
+  if (!token && !Deno.isatty(Deno.stdin.rid)) {
+    log.info("Not a TTY, can't login interactively. Pass the token in --token");
+    return;
+  }
   while (!token) {
     token = await loginInteractive(remote);
   }
@@ -221,14 +226,14 @@ export async function add(
       requestBody: { id: workspaceId },
     });
   } catch (e) {
-    console.log(
+    log.info(
       colors.red.bold("! Credentials or instance is invalid. Aborting.")
     );
     throw e;
   }
   if (opts.create) {
     if (!alreadyExists) {
-      console.log(
+      log.info(
         colors.yellow(
           `Workspace at id ${workspaceId} on ${remote} does not exist. Creating...`
         )
@@ -242,17 +247,17 @@ export async function add(
       });
     }
   } else if (!alreadyExists) {
-    console.log(
+    log.info(
       colors.red.bold(
         `! Workspace at id ${workspaceId} on ${remote} does not exist. Re-run with --create to create it. Aborting.`
       )
     );
-    console.log(
+    log.info(
       "On that instance and with those credentials, the workspaces that you can access are:"
     );
     const workspaces = await WorkspaceService.listWorkspaces();
     for (const workspace of workspaces) {
-      console.log(`- ${workspace.id} (name: ${workspace.name})`);
+      log.info(`- ${workspace.id} (name: ${workspace.name})`);
     }
     Deno.exit(1);
   }
@@ -270,7 +275,7 @@ export async function add(
     (await getRootStore()) + "/activeWorkspace",
     workspaceName
   );
-  console.log(
+  log.info(
     colors.green.underline(
       `Added workspace ${workspaceName} for ${workspaceId} on ${remote}!`
     )
@@ -299,16 +304,16 @@ export async function removeWorkspace(
   const orgWorkspaces = await allWorkspaces();
   if (orgWorkspaces.findIndex((x) => x.name === name) === -1) {
     if (!silent) {
-      console.log(
+      log.info(
         colors.red.bold(`! Workspace profile ${name} does not exist locally`)
       );
-      console.log("available workspace profiles:");
+      log.info("available workspace profiles:");
       await list(opts);
     }
     return;
   }
   if (silent) {
-    console.log(colors.yellow(`Replacing existing workspace ${name}`));
+    log.info(colors.yellow(`Replacing existing workspace ${name}`));
   }
 
   await Deno.writeTextFile(
@@ -320,9 +325,7 @@ export async function removeWorkspace(
   );
 
   if (!silent) {
-    console.log(
-      colors.green.underline(`Succesfully removed workspace ${name}!`)
-    );
+    log.info(colors.green.underline(`Succesfully removed workspace ${name}!`));
   }
 }
 
@@ -332,9 +335,9 @@ async function remove(_opts: GlobalOptions, name: string) {
 
 async function whoami(_opts: GlobalOptions) {
   await requireLogin(_opts);
-  console.log(await UserService.globalWhoami());
+  log.info(await UserService.globalWhoami());
   const activeName = await getActiveWorkspaceName(_opts);
-  console.log("Active: " + colors.green.bold(activeName || "none"));
+  log.info("Active: " + colors.green.bold(activeName || "none"));
 }
 
 const command = new Command()

deno-client/mod.ts

@@ -210,6 +210,39 @@ export async function getVariable(path: string): Promise<string | undefined> {
   }
 }
 
+/**
+ * Set a variable by path, create if not exist
+ * @param path path of the variable
+ * @param value value of the variable
+ * @param isSecretIfNotExist if the variable does not exist, create it as secret or not (default: false)
+ * @param descriptionIfNotExist if the variable does not exist, create it with this description (default: "")
+ */
+export async function setVariable(
+  path: string,
+  value: string,
+  isSecretIfNotExist?: boolean,
+  descriptionIfNotExist?: string
+): Promise<void> {
+  const workspace = getWorkspace();
+  if (await VariableService.existsVariable({ workspace, path })) {
+    await VariableService.updateVariable({
+      workspace,
+      path,
+      requestBody: { value },
+    });
+  } else {
+    await VariableService.createVariable({
+      workspace,
+      requestBody: {
+        path,
+        value,
+        is_secret: isSecretIfNotExist ?? false,
+        description: descriptionIfNotExist ?? "",
+      },
+    });
+  }
+}
+
 async function transformLeaves(d: {
   [key: string]: any;
 }): Promise<{ [key: string]: any }> {

docker-compose.yml

@@ -60,6 +60,8 @@ services:
     # to mount the worker folder to debug,, KEEP_JOB_DIR=true and mount /tmp/windmill
     volumes:
       - worker_dependency_cache:/tmp/windmill/cache
+      # mount the docker socket to allow to run docker containers from within the workers
+      # - /var/run/docker.sock:/var/run/docker.sock
 
   lsp:
     image: ghcr.io/windmill-labs/windmill-lsp:latest
@@ -70,13 +72,14 @@ services:
   caddy:
     image: caddy:2.5.2-alpine
     restart: unless-stopped
+
+    # Configure the mounted Caddyfile and the exposed ports or use another reverse proxy if needed
     volumes:
       - ./Caddyfile:/etc/caddy/Caddyfile
     ports:
       - 80:80
-      - 443:443
     environment:
-      - BASE_URL=${CADDY_REVERSE_PROXY}
+      - BASE_URL=":80"
 
 volumes:
   db_data: null

docker/DockerfileHelm

@@ -12,4 +12,4 @@ RUN apt-get update && apt install unzip && curl "https://awscli.amazonaws.com/aw
     unzip awscliv2.zip && \
     ./aws/install 
 
-CMD ["./windmill"]
+CMD ["windmill"]

docker/DockerfileOpenbb

@@ -90,8 +90,7 @@ FROM python:3.10.8-slim-buster
 ARG APP=/usr/src/app
 
 RUN apt-get update \
-    && apt-get install -y ca-certificates wget curl git jq libprotobuf-dev libnl-route-3-dev unzip \
-    && apt-get install -y ca-certificates wget curl git jq libprotobuf-dev libnl-route-3-dev unzip build-essential \
+    && apt-get install -y ca-certificates wget curl git jq libprotobuf-dev libnl-route-3-dev unzip build-essential pkg-config libcairo2-dev libwebkit2gtk-4.0-37 \
     && rm -rf /var/lib/apt/lists/*
 
 RUN arch="$(dpkg --print-architecture)"; arch="${arch##*-}"; \
@@ -133,8 +132,10 @@ RUN /usr/local/bin/python3 -m pip install openbb[all]
 
 RUN mkdir -p ${APP}
 
+RUN ln -s ${APP}/windmill /usr/local/bin/windmill
+
 WORKDIR ${APP}
 
 EXPOSE 8000
 
-CMD ["./windmill"]
+CMD ["windmill"]

frontend/.vscode/settings.json

@@ -1,3 +1,5 @@
 {
-	"cSpell.words": ["semibold", "stylelint"]
+	"cSpell.words": ["semibold", "stylelint"],
+	"deno.enable": true,
+	"deno.unstable": true
 }

frontend/Caddyfile

@@ -1,12 +0,0 @@
-:80 {  
-  bind {$ADDRESS}
-  reverse_proxy /api/* http://localhost:8000
-  reverse_proxy /* http://localhost:3000
-  reverse_proxy /ws/* http://localhost:3001
-}
-
-:443 {  
-  bind {$ADDRESS}
-  reverse_proxy /api/* http://localhost:8000
-  reverse_proxy /ws/* http://localhost:3001
-}