noopener noreferrer

.github/DockerfileBackendTests

@@ -43,9 +43,6 @@ RUN wget https://golang.org/dl/go1.21.5.linux-amd64.tar.gz && tar -C /usr/local
 ENV PATH="${PATH}:/usr/local/go/bin"
 ENV GO_PATH=/usr/local/go/bin/go
 
-# Install UV
-RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.4.18/uv-installer.sh | sh && mv /usr/local/cargo/bin/uv /usr/local/bin/uv
-
 ENV TZ=Etc/UTC
 
 ENV PYTHON_VERSION 3.11.4

.github/workflows/backend-test.yml

@@ -44,5 +44,5 @@ jobs:
           mkdir frontend/build && cd backend && touch
           windmill-api/openapi-deref.yaml &&
           DATABASE_URL=postgres://postgres:changeme@postgres:5432/windmill
-          DISABLE_EMBEDDING=true RUST_LOG=info cargo test --features
-          enterprise,deno_core --all -- --nocapture
+          DISABLE_EMBEDDING=true RUST_LOG=info cargo test --features enterprise
+          --all -- --nocapture

.github/workflows/build-staging-image.yml

@@ -62,7 +62,7 @@ jobs:
           platforms: linux/amd64,linux/arm64
           push: true
           build-args: |
-            features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core
+            features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc
           tags: |
             ${{ steps.meta-ee-public.outputs.tags }}
           labels: |

.github/workflows/build_windows_worker.yml

@@ -1,56 +0,0 @@
-name: Build and Publish Windows Worker
-
-on:
-  push:
-    tags:
-      - "v*"
-
-env:
-  CARGO_INCREMENTAL: 0
-  SQLX_OFFLINE: true
-  DISABLE_EMBEDDING: true
-  RUST_LOG: info
-
-jobs:
-  cargo_build_windows:
-    runs-on: windows-latest
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Read EE repo commit hash
-        shell: pwsh
-        run: |
-          $ee_repo_ref = Get-Content .\backend\ee-repo-ref.txt
-          echo "ee_repo_ref=$ee_repo_ref" | Out-File -FilePath $env:GITHUB_ENV -Append
-
-      - name: Checkout windmill-ee-private repository
-        uses: actions/checkout@v4
-        with:
-          repository: windmill-labs/windmill-ee-private
-          path: ./windmill-ee-private
-          ref: ${{ env.ee_repo_ref }}
-          token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }}
-          fetch-depth: 0
-
-      - name: Substitute EE code
-        shell: bash
-        run: |
-          ./backend/substitute_ee_code.sh --copy --dir ./windmill-ee-private
-
-      - name: Cargo build windows
-        timeout-minutes: 60
-        run: |
-          $env:OPENSSL_DIR = "${Env:ProgramFiles}\OpenSSL"
-          mkdir frontend/build && cd backend
-          New-Item -Path . -Name "windmill-api/openapi-deref.yaml" -ItemType "File" -Force
-          cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy
-
-      - name: Rename binary with corresponding architecture
-        run: |
-          Rename-Item -Path ".\backend\target\release\windmill.exe" -NewName "windmill-ee.exe"
-
-      - name: Attach binary to release
-        uses: softprops/action-gh-release@v2
-        with:
-          files: |
-            ./backend/target/release/windmill-ee.exe

.github/workflows/docker-image-rpi4.yml

@@ -67,7 +67,7 @@ jobs:
           platforms: linux/amd64,linux/arm64
           push: true
           build-args: |
-            features=embedding,parquet,openidconnect,deno_core
+            features=embedding,parquet,openidconnect
           tags: |
             ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev
             ${{ steps.meta-public.outputs.tags }}

.github/workflows/docker-image.yml

@@ -1,10 +1,8 @@
 env:
   REGISTRY: ghcr.io
-  IMAGE_NAME:
-    ${{ github.event_name != 'pull_request' && github.repository ||
+  IMAGE_NAME: ${{ github.event_name != 'pull_request' && github.repository ||
     'windmill-labs/windmill-test' }}
-  DEV_SHA:
-    ${{ github.event_name != 'pull_request' && 'dev' || format('pr-{0}',
+  DEV_SHA: ${{ github.event_name != 'pull_request' && 'dev' || format('pr-{0}',
     github.event.number) }}
 
 name: Build windmill:main
@@ -77,7 +75,7 @@ jobs:
           platforms: linux/amd64,linux/arm64
           push: true
           build-args: |
-            features=embedding,parquet,openidconnect,jemalloc,deno_core
+            features=embedding,parquet,openidconnect,jemalloc
           tags: |
             ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }}
             ${{ steps.meta-public.outputs.tags }}
@@ -138,7 +136,7 @@ jobs:
           platforms: linux/amd64,linux/arm64
           push: true
           build-args: |
-            features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core
+            features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy
           tags: |
             ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${{ env.DEV_SHA }}
             ${{ steps.meta-ee-public.outputs.tags }}
@@ -200,7 +198,7 @@ jobs:
           platforms: linux/amd64
           push: true
           build-args: |
-            features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core
+            features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy
             PYTHON_IMAGE=python:3.12.2-slim-bookworm
           tags: |
             ${{ steps.meta-ee-public-py312.outputs.tags }}

CHANGELOG.md

@@ -1,142 +1,5 @@
 # Changelog
 
-## [1.405.4](https://github.com/windmill-labs/windmill/compare/v1.405.3...v1.405.4) (2024-10-04)
-
-
-### Bug Fixes
-
-* **frontend:** correctly initialize step inputs on new inline script ([289ad51](https://github.com/windmill-labs/windmill/commit/289ad51374f0344582372572fc521f3b2bf12b33))
-
-## [1.405.3](https://github.com/windmill-labs/windmill/compare/v1.405.2...v1.405.3) (2024-10-04)
-
-
-### Bug Fixes
-
-* fix id save on apps ([b034b07](https://github.com/windmill-labs/windmill/commit/b034b070c075a0fab74678bec1fd8b829d55b204))
-
-## [1.405.2](https://github.com/windmill-labs/windmill/compare/v1.405.1...v1.405.2) (2024-10-03)
-
-
-### Bug Fixes
-
-* **cli:** fix opts.yes for instance sync ([26659ce](https://github.com/windmill-labs/windmill/commit/26659ce37d2887d5b98dbdbdbba27bab85d4fe3f))
-* fix uv path ([19c62ba](https://github.com/windmill-labs/windmill/commit/19c62ba195b1df85c38c748dab7d9f137696a5c3))
-
-## [1.405.1](https://github.com/windmill-labs/windmill/compare/v1.405.0...v1.405.1) (2024-10-03)
-
-
-### Bug Fixes
-
-* flow picker of flows + precache hub scripts as bundles ([c84e6fd](https://github.com/windmill-labs/windmill/commit/c84e6fd05de2bea426cae61fa25db0323b8770f5))
-
-## [1.405.0](https://github.com/windmill-labs/windmill/compare/v1.404.1...v1.405.0) (2024-10-03)
-
-
-### Features
-
-* Replace `pip-compile` with `uv` ([#4460](https://github.com/windmill-labs/windmill/issues/4460)) ([b54c9ee](https://github.com/windmill-labs/windmill/commit/b54c9ee657cc88fabe694cae39dc0d3c1918fcbb))
-* **worker:** support workers to run natively on windows ([#4446](https://github.com/windmill-labs/windmill/issues/4446)) ([f5c4727](https://github.com/windmill-labs/windmill/commit/f5c472727465dd95f5378bc08ee9bbb983f4d259))
-
-
-### Bug Fixes
-
-* **cli:** fix set client of instance when passing token and base url ([794c4cd](https://github.com/windmill-labs/windmill/commit/794c4cde3cd47042472dccdf4b60a012014dd26d))
-
-## [1.404.1](https://github.com/windmill-labs/windmill/compare/v1.404.0...v1.404.1) (2024-10-03)
-
-
-### Bug Fixes
-
-* flow picker of flows ([92f61f0](https://github.com/windmill-labs/windmill/commit/92f61f07ed6d354407d26843e3a270b95bae90bc))
-
-## [1.404.0](https://github.com/windmill-labs/windmill/compare/v1.403.1...v1.404.0) (2024-10-03)
-
-
-### Features
-
-* **frontend:** add quick access menu in flow editor ([#4415](https://github.com/windmill-labs/windmill/issues/4415)) ([45ccd45](https://github.com/windmill-labs/windmill/commit/45ccd45e306c66931880a9b8fd48bfe684c774ac))
-
-
-### Bug Fixes
-
-* **cli:** improve schedule path handling on windows ([9ac3b6b](https://github.com/windmill-labs/windmill/commit/9ac3b6b1d5d64d7467dd80506f8a8d772c4630bd))
-* fix id editor for app ([8e58e43](https://github.com/windmill-labs/windmill/commit/8e58e4320a31d71c40a5ed352416a4c2dd3adb26))
-* **frontend:** disable runnable field on route editor from detail panel ([#4469](https://github.com/windmill-labs/windmill/issues/4469)) ([3134f79](https://github.com/windmill-labs/windmill/commit/3134f79ced80aab86912643ab7a60dcf909ab104))
-
-## [1.403.1](https://github.com/windmill-labs/windmill/compare/v1.403.0...v1.403.1) (2024-10-01)
-
-
-### Bug Fixes
-
-* fix new instance db setup ([73ab8e1](https://github.com/windmill-labs/windmill/commit/73ab8e1653d6e0c0c69fa7dcd96583f25d13ef86))
-
-## [1.403.0](https://github.com/windmill-labs/windmill/compare/v1.402.3...v1.403.0) (2024-10-01)
-
-
-### Features
-
-* flow step skipping ([#4461](https://github.com/windmill-labs/windmill/issues/4461)) ([0df169e](https://github.com/windmill-labs/windmill/commit/0df169e3f996ed54b91569b13cce15d7d019a213))
-
-
-### Bug Fixes
-
-* skip one migration to avoid using md5 for azure support ([630ae5d](https://github.com/windmill-labs/windmill/commit/630ae5d425cd9957d674befd2df96e2befec52a3))
-
-## [1.402.3](https://github.com/windmill-labs/windmill/compare/v1.402.2...v1.402.3) (2024-09-30)
-
-
-### Bug Fixes
-
-* improve allowed domains setting for sso ([24f4a7c](https://github.com/windmill-labs/windmill/commit/24f4a7caaafa93f51669dcf44a3dca09d5b228bb))
-
-## [1.402.2](https://github.com/windmill-labs/windmill/compare/v1.402.1...v1.402.2) (2024-09-28)
-
-
-### Bug Fixes
-
-* make form properties disablable ([0779d47](https://github.com/windmill-labs/windmill/commit/0779d47c1d39626d11bd3769cd787cb036df0a94))
-
-## [1.402.1](https://github.com/windmill-labs/windmill/compare/v1.402.0...v1.402.1) (2024-09-28)
-
-
-### Bug Fixes
-
-* allow preprocessor to write to args.json on nsjail ([#4455](https://github.com/windmill-labs/windmill/issues/4455)) ([0b9ec83](https://github.com/windmill-labs/windmill/commit/0b9ec83036e2a1d0773b4ec5856f907b383e9323))
-* **frontend:** Fix flow graph bg in dark mode on chrome ([#4454](https://github.com/windmill-labs/windmill/issues/4454)) ([6956a3a](https://github.com/windmill-labs/windmill/commit/6956a3a2ba6d189528cb34ab05f7137fdf4f840b))
-* improve suspend_first behavior and frequency ([b5e226b](https://github.com/windmill-labs/windmill/commit/b5e226b977e6d24ebd28bc1e7c867cb4888f77b2))
-
-## [1.402.0](https://github.com/windmill-labs/windmill/compare/v1.401.0...v1.402.0) (2024-09-26)
-
-
-### Features
-
-* **cli:** add queues, workers and worker-groups commands ([#4439](https://github.com/windmill-labs/windmill/issues/4439)) ([9f91b19](https://github.com/windmill-labs/windmill/commit/9f91b1995a98c9e096c6e599c4d5a8d5ea499ada))
-
-## [1.401.0](https://github.com/windmill-labs/windmill/compare/v1.400.0...v1.401.0) (2024-09-25)
-
-
-### Features
-
-* add return_last_result annotation to sql ([#4443](https://github.com/windmill-labs/windmill/issues/4443)) ([3ce5587](https://github.com/windmill-labs/windmill/commit/3ce5587faae3912ceedae4644732fa9704eb6d76))
-
-
-### Bug Fixes
-
-* fix flow rendering ([fd58e7e](https://github.com/windmill-labs/windmill/commit/fd58e7eb48c4fb66d199c33d0f8aaf2535485a2f))
-
-## [1.400.0](https://github.com/windmill-labs/windmill/compare/v1.399.0...v1.400.0) (2024-09-25)
-
-
-### Features
-
-* add static variable and resources support to ansible ([#4435](https://github.com/windmill-labs/windmill/issues/4435)) ([398a09b](https://github.com/windmill-labs/windmill/commit/398a09b7419c7adafa76e5f9f5a981bec6f0611d))
-* **frontend:** Catch flow errors in the UI ([#4429](https://github.com/windmill-labs/windmill/issues/4429)) ([84eefad](https://github.com/windmill-labs/windmill/commit/84eefadfcf06b4f38117b6c4c534f47fb2ef7cc7))
-
-
-### Bug Fixes
-
-* remove autocomplete for searchbar ([#4440](https://github.com/windmill-labs/windmill/issues/4440)) ([a1ac583](https://github.com/windmill-labs/windmill/commit/a1ac583f05c34a534c4ae78ca7d2a6850723ce85))
-
 ## [1.399.0](https://github.com/windmill-labs/windmill/compare/v1.398.1...v1.399.0) (2024-09-25)
 
 

Dockerfile

@@ -158,9 +158,6 @@ RUN set -eux; \
 ENV PATH="${PATH}:/usr/local/go/bin"
 ENV GO_PATH=/usr/local/go/bin/go
 
-# Install UV
-RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.4.18/uv-installer.sh | sh && mv /root/.cargo/bin/uv /usr/local/bin/uv
-
 RUN curl -sL https://deb.nodesource.com/setup_20.x | bash - 
 RUN apt-get -y update && apt-get install -y curl nodejs awscli && apt-get clean \
     && rm -rf /var/lib/apt/lists/*

backend/.sqlx/query-19f0abd79372698f378cb6deea3ee6d098a2758d16ede000809bd9a09660b604.json

@@ -1,20 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT created_at FROM metrics WHERE id = 'telemetry' ORDER BY created_at DESC LIMIT 1",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "created_at",
-        "type_info": "Timestamptz"
-      }
-    ],
-    "parameters": {
-      "Left": []
-    },
-    "nullable": [
-      false
-    ]
-  },
-  "hash": "19f0abd79372698f378cb6deea3ee6d098a2758d16ede000809bd9a09660b604"
-}

backend/.sqlx/query-1e188d8e427cab25dbe18aa900260e26e644a9d939e74a8317c4a09335f110fe.json

@@ -1,25 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "UPDATE queue\n                                SET flow_status = JSONB_SET(\n                                JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4),\n                                ARRAY['modules', $1::TEXT, 'branchall', 'branch'], ((flow_status->'modules'->$1::int->'branchall'->>'branch')::int + 1)::text::jsonb),\n                                last_ping = NULL\n                                WHERE id = $2\n                                RETURNING (flow_status->'modules'->$1::int->'branchall'->>'branch')::int",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "int4",
-        "type_info": "Int4"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Int4",
-        "Uuid",
-        "Text",
-        "Jsonb"
-      ]
-    },
-    "nullable": [
-      null
-    ]
-  },
-  "hash": "1e188d8e427cab25dbe18aa900260e26e644a9d939e74a8317c4a09335f110fe"
-}

backend/.sqlx/query-3e539fef054ad31bc1736e27276087775a721a6ee7ae35b03fd4ce3563ea3838.json

@@ -1,23 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT raw_flow->'modules'->($1)->'value'->>'type' = 'flow' FROM queue WHERE id = $2",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "?column?",
-        "type_info": "Bool"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Text",
-        "Uuid"
-      ]
-    },
-    "nullable": [
-      null
-    ]
-  },
-  "hash": "3e539fef054ad31bc1736e27276087775a721a6ee7ae35b03fd4ce3563ea3838"
-}

backend/.sqlx/query-6776dc50f184188756ad7fe263b0304333536768527525a43bdd45aedffa3c4f.json

@@ -1,22 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT pg_try_advisory_xact_lock($1)",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "pg_try_advisory_xact_lock",
-        "type_info": "Bool"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Int8"
-      ]
-    },
-    "nullable": [
-      null
-    ]
-  },
-  "hash": "6776dc50f184188756ad7fe263b0304333536768527525a43bdd45aedffa3c4f"
-}

backend/.sqlx/query-6e7f234267fbb4720b29f288fba82c1df21ba601ac0989e175f834c569962d46.json

@@ -1,23 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "UPDATE queue\n                                SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'iterator', 'index'], ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb),\n                                last_ping = NULL\n                                WHERE id = $2\n                                RETURNING (flow_status->'modules'->$1::int->'iterator'->>'index')::int",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "int4",
-        "type_info": "Int4"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Int4",
-        "Uuid"
-      ]
-    },
-    "nullable": [
-      null
-    ]
-  },
-  "hash": "6e7f234267fbb4720b29f288fba82c1df21ba601ac0989e175f834c569962d46"
-}

backend/.sqlx/query-6f4817fad2739a11d89b6704edf62c3c267ca336a8b6bec5b29d4409030ed561.json

@@ -1,73 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "INSERT INTO queue (id, script_hash, script_path, job_kind, language, tag, created_by, permissioned_as, email, scheduled_for, workspace_id, raw_flow, flow_status) (SELECT gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12 FROM generate_series(1, 1))",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Int8",
-        "Varchar",
-        {
-          "Custom": {
-            "name": "job_kind",
-            "kind": {
-              "Enum": [
-                "script",
-                "preview",
-                "flow",
-                "dependencies",
-                "flowpreview",
-                "script_hub",
-                "identity",
-                "flowdependencies",
-                "http",
-                "graphql",
-                "postgresql",
-                "noop",
-                "appdependencies",
-                "deploymentcallback",
-                "singlescriptflow"
-              ]
-            }
-          }
-        },
-        {
-          "Custom": {
-            "name": "script_lang",
-            "kind": {
-              "Enum": [
-                "python3",
-                "deno",
-                "go",
-                "bash",
-                "postgresql",
-                "nativets",
-                "bun",
-                "mysql",
-                "bigquery",
-                "snowflake",
-                "graphql",
-                "powershell",
-                "mssql",
-                "php",
-                "bunnative",
-                "rust",
-                "ansible"
-              ]
-            }
-          }
-        },
-        "Varchar",
-        "Varchar",
-        "Varchar",
-        "Varchar",
-        "Timestamptz",
-        "Varchar",
-        "Jsonb",
-        "Jsonb"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "6f4817fad2739a11d89b6704edf62c3c267ca336a8b6bec5b29d4409030ed561"
-}

backend/.sqlx/query-804fc11e35f4afc0db194b6fe2594f91df7e588d4d2431bc85f4d8734920c8bf.json

@@ -1,45 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "INSERT INTO script (summary, description, dedicated_worker, content, workspace_id, path, hash, language, tag, created_by, lock) VALUES ('', '', true, $1, $2, $3, $4, $5, $6, $7, '') ON CONFLICT (workspace_id, hash) DO NOTHING",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Text",
-        "Varchar",
-        "Varchar",
-        "Int8",
-        {
-          "Custom": {
-            "name": "script_lang",
-            "kind": {
-              "Enum": [
-                "python3",
-                "deno",
-                "go",
-                "bash",
-                "postgresql",
-                "nativets",
-                "bun",
-                "mysql",
-                "bigquery",
-                "snowflake",
-                "graphql",
-                "powershell",
-                "mssql",
-                "php",
-                "bunnative",
-                "rust",
-                "ansible"
-              ]
-            }
-          }
-        },
-        "Varchar",
-        "Varchar"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "804fc11e35f4afc0db194b6fe2594f91df7e588d4d2431bc85f4d8734920c8bf"
-}

backend/.sqlx/query-8375c1efeb1e2a2d2803052a2899bf70f4a6434eb91b4b05b9fb8420beae26af.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT worker, worker_instance,  EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed,\n        CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as last_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as last_job_workspace_id, \n        custom_tags, worker_group, wm_version, occupancy_rate, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, memory, vcpus, memory_usage, wm_memory_usage\n        FROM worker_ping\n        WHERE ($1::integer IS NULL AND ping_at > now() - interval '5 minute') OR (ping_at > now() - ($1 || ' seconds')::interval)\n        ORDER BY ping_at desc LIMIT $2 OFFSET $3",
+  "query": "SELECT worker, worker_instance,  EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed, CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as last_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as last_job_workspace_id, custom_tags, worker_group, wm_version, occupancy_rate, memory, vcpus, memory_usage, wm_memory_usage\n        FROM worker_ping\n        WHERE ($1::integer IS NULL AND ping_at > now() - interval '5 minute') OR (ping_at > now() - ($1 || ' seconds')::interval)\n        ORDER BY ping_at desc LIMIT $2 OFFSET $3",
   "describe": {
     "columns": [
       {
@@ -65,36 +65,21 @@
       },
       {
         "ordinal": 12,
-        "name": "occupancy_rate_15s",
-        "type_info": "Float4"
-      },
-      {
-        "ordinal": 13,
-        "name": "occupancy_rate_5m",
-        "type_info": "Float4"
-      },
-      {
-        "ordinal": 14,
-        "name": "occupancy_rate_30m",
-        "type_info": "Float4"
-      },
-      {
-        "ordinal": 15,
         "name": "memory",
         "type_info": "Int8"
       },
       {
-        "ordinal": 16,
+        "ordinal": 13,
         "name": "vcpus",
         "type_info": "Int8"
       },
       {
-        "ordinal": 17,
+        "ordinal": 14,
         "name": "memory_usage",
         "type_info": "Int8"
       },
       {
-        "ordinal": 18,
+        "ordinal": 15,
         "name": "wm_memory_usage",
         "type_info": "Int8"
       }
@@ -123,11 +108,8 @@
       true,
       true,
       true,
-      true,
-      true,
-      true,
       true
     ]
   },
-  "hash": "6a497334c98bfaf70be44fced572a1cc0dde4141aa4c5002765a95432d0101ab"
+  "hash": "8375c1efeb1e2a2d2803052a2899bf70f4a6434eb91b4b05b9fb8420beae26af"
 }

backend/.sqlx/query-95cb1fe8658f98fb736d899fa21cd7378b0c9d3b5f3d6bd6cafcba273f8277d4.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "UPDATE worker_ping SET ping_at = now(), current_job_id = $1, current_job_workspace_id = $2, memory_usage = $3, wm_memory_usage = $4,\n                            occupancy_rate = $6, occupancy_rate_15s = $7, occupancy_rate_5m = $8, occupancy_rate_30m = $9 WHERE worker = $5",
+  "query": "UPDATE worker_ping SET ping_at = now(), current_job_id = $1, current_job_workspace_id = $2, memory_usage = $3, wm_memory_usage = $4 WHERE worker = $5",
   "describe": {
     "columns": [],
     "parameters": {
@@ -9,14 +9,10 @@
         "Varchar",
         "Int8",
         "Int8",
-        "Text",
-        "Float4",
-        "Float4",
-        "Float4",
-        "Float4"
+        "Text"
       ]
     },
     "nullable": []
   },
-  "hash": "e968e879d3c52f7dd502c3cd15fc8fbd983a4a3ab25648c562497a27c74b5c8c"
+  "hash": "95cb1fe8658f98fb736d899fa21cd7378b0c9d3b5f3d6bd6cafcba273f8277d4"
 }

backend/.sqlx/query-9bf41c3161a02b7d0731c4e1d79519cef5255f5df1b759af3aa4985bb64313e5.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "INSERT INTO metrics (id, value)\n                            VALUES ($1, to_jsonb((SELECT EXTRACT(EPOCH FROM now() - scheduled_for)\n                            FROM queue WHERE tag = $2 AND running = false AND scheduled_for <= now() - ('3 seconds')::interval\n                            ORDER BY priority DESC NULLS LAST, scheduled_for LIMIT 1)))",
+  "query": "INSERT INTO metrics (id, value)\n                            VALUES ($1, to_jsonb((SELECT EXTRACT(EPOCH FROM now() - scheduled_for)\n                            FROM queue WHERE tag = $2 AND running = false AND scheduled_for <= now() - ('3 seconds')::interval\n                            ORDER BY priority DESC NULLS LAST, scheduled_for, created_at LIMIT 1)))",
   "describe": {
     "columns": [],
     "parameters": {
@@ -11,5 +11,5 @@
     },
     "nullable": []
   },
-  "hash": "41f68f4ce5bed783cf69e42da115e9ad2c9fcbd75f55817b2114c04207f66e4a"
+  "hash": "9bf41c3161a02b7d0731c4e1d79519cef5255f5df1b759af3aa4985bb64313e5"
 }

backend/.sqlx/query-9cf96fa6364b7f34dc83719b4a0e97e8494393c29f7c8d915aa54da7ab7eed51.json

@@ -0,0 +1,21 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1, custom_tags = $2, occupancy_rate = $3, memory_usage = $4, wm_memory_usage = $5, vcpus = COALESCE($7, vcpus), memory = COALESCE($8, memory) WHERE worker = $6",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Int4",
+        "TextArray",
+        "Float4",
+        "Int8",
+        "Int8",
+        "Text",
+        "Int8",
+        "Int8"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "9cf96fa6364b7f34dc83719b4a0e97e8494393c29f7c8d915aa54da7ab7eed51"
+}

backend/.sqlx/query-a439552f74ed0ba305e3d9cb99ae9e5d24834082ebf2fe9fd3964fdd80b69ccb.json

@@ -1,24 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1, custom_tags = $2,\n                 occupancy_rate = $3, memory_usage = $4, wm_memory_usage = $5, vcpus = COALESCE($7, vcpus),\n                 memory = COALESCE($8, memory), occupancy_rate_15s = $9, occupancy_rate_5m = $10, occupancy_rate_30m = $11 WHERE worker = $6",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Int4",
-        "TextArray",
-        "Float4",
-        "Int8",
-        "Int8",
-        "Text",
-        "Int8",
-        "Int8",
-        "Float4",
-        "Float4",
-        "Float4"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "a439552f74ed0ba305e3d9cb99ae9e5d24834082ebf2fe9fd3964fdd80b69ccb"
-}

backend/.sqlx/query-a94dbd1f7aab20682548471c5cc06c7a496edac36e3de537b00b76ad93c7556c.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "UPDATE queue\n                            SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'branchall', 'branch'], ((flow_status->'modules'->$1::int->'branchall'->>'branch')::int + 1)::text::jsonb),\n                            last_ping = NULL\n                            WHERE id = $2\n                            RETURNING (flow_status->'modules'->$1::int->'branchall'->>'branch')::int",
+  "query": "UPDATE queue\n                            SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'branchall', 'branch'], ((flow_status->'modules'->$1::int->'branchall'->>'branch')::int + 1)::text::jsonb)\n                            WHERE id = $2\n                            RETURNING (flow_status->'modules'->$1::int->'branchall'->>'branch')::int",
   "describe": {
     "columns": [
       {
@@ -19,5 +19,5 @@
       null
     ]
   },
-  "hash": "777190559e27c8c8fb6718b0a0c1d7db9b956abd88b94db3948f2c579c3826d0"
+  "hash": "a94dbd1f7aab20682548471c5cc06c7a496edac36e3de537b00b76ad93c7556c"
 }

backend/.sqlx/query-ae2f005af8ab4b035a907e0c8fc9a9d035f3eb1d9d833041969fce967daa91a4.json

@@ -1,22 +1,23 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT job_kind = 'identity' FROM completed_job WHERE id = $1",
+  "query": "SELECT raw_flow->'modules'->$2::int->'retry' FROM queue WHERE id = $1",
   "describe": {
     "columns": [
       {
         "ordinal": 0,
         "name": "?column?",
-        "type_info": "Bool"
+        "type_info": "Jsonb"
       }
     ],
     "parameters": {
       "Left": [
-        "Uuid"
+        "Uuid",
+        "Int4"
       ]
     },
     "nullable": [
       null
     ]
   },
-  "hash": "829130d74c107e4e1a86f3e772657ae030463c139c39072777e453bfb7e9c0c3"
+  "hash": "ae2f005af8ab4b035a907e0c8fc9a9d035f3eb1d9d833041969fce967daa91a4"
 }

backend/.sqlx/query-c3b5abbf2c9079d597a55f7c63bc83b8b4da98bda204a40f045a62172cfb4ebb.json

@@ -1,29 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\n        SELECT tag as \"tag!\", COUNT(*) as \"count!\"\n        FROM completed_job\n        WHERE started_at > NOW() - make_interval(secs => $1) AND ($2::text IS NULL OR workspace_id = $2)\n        GROUP BY tag\n        ORDER BY \"count!\" DESC\n        ",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "tag!",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 1,
-        "name": "count!",
-        "type_info": "Int8"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Float8",
-        "Text"
-      ]
-    },
-    "nullable": [
-      false,
-      null
-    ]
-  },
-  "hash": "c3b5abbf2c9079d597a55f7c63bc83b8b4da98bda204a40f045a62172cfb4ebb"
-}

backend/.sqlx/query-d0df57fc4cd0be7b541dee081ffbe86b869b01ca1ab10aa17634bb6dae879f12.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE queue\n                        SET last_ping = null\n                        WHERE id = $1",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Uuid"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "d0df57fc4cd0be7b541dee081ffbe86b869b01ca1ab10aa17634bb6dae879f12"
+}

backend/.sqlx/query-d6c25421bb6513908697ebe74c159da0ef78b5252b6ac669a3c9e545d05c0d43.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE queue\n                            SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'iterator', 'index'], ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb)\n                            WHERE id = $2\n                            RETURNING (flow_status->'modules'->$1::int->'iterator'->>'index')::int",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "int4",
+        "type_info": "Int4"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int4",
+        "Uuid"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "d6c25421bb6513908697ebe74c159da0ef78b5252b6ac669a3c9e545d05c0d43"
+}

backend/.sqlx/query-f916ec232837ece9323675e5f5142e7285f4266a1471e5ffdefadf421a67e44b.json

@@ -1,25 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "UPDATE queue\n                            SET flow_status = JSONB_SET(\n                                JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4),\n                                ARRAY['modules', $1::TEXT, 'iterator', 'index'],\n                                ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb\n                            ),\n                            last_ping = NULL\n                            WHERE id = $2\n                            RETURNING (flow_status->'modules'->$1::int->'iterator'->>'index')::int",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "int4",
-        "type_info": "Int4"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Int4",
-        "Uuid",
-        "Text",
-        "Jsonb"
-      ]
-    },
-    "nullable": [
-      null
-    ]
-  },
-  "hash": "f916ec232837ece9323675e5f5142e7285f4266a1471e5ffdefadf421a67e44b"
-}

backend/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "windmill"
-version = "1.405.4"
+version = "1.399.0"
 authors.workspace = true
 edition.workspace = true
 
@@ -27,7 +27,7 @@ members = [
 ]
 
 [workspace.package]
-version = "1.405.4"
+version = "1.399.0"
 authors = ["Ruben Fiszel <ruben@windmill.dev>"]
 edition = "2021"
 
@@ -44,9 +44,10 @@ default = []
 enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmill-api/enterprise", "windmill-git-sync/enterprise", "windmill-common/prometheus", "windmill-common/enterprise", "windmill-indexer/enterprise"]
 enterprise_saml = ["windmill-api/enterprise_saml"]
 stripe = ["windmill-api/stripe"]
-benchmark = ["windmill-api/benchmark", "windmill-worker/benchmark", "windmill-queue/benchmark", "windmill-common/benchmark"]
+benchmark = ["windmill-api/benchmark", "windmill-worker/benchmark", "windmill-queue/benchmark"]
 flamegraph = ["windmill-common/flamegraph", "windmill-worker/flamegraph"]
 loki = ["windmill-common/loki"]
+pg_embed = ["dep:pg-embed"]
 embedding = ["windmill-api/embedding"]
 parquet = ["windmill-api/parquet", "windmill-common/parquet", "windmill-worker/parquet", "windmill-indexer/parquet", "dep:object_store"]
 prometheus = ["windmill-common/prometheus", "windmill-api/prometheus", "windmill-worker/prometheus", "windmill-queue/prometheus"]
@@ -56,7 +57,6 @@ cloud = ["windmill-queue/cloud", "windmill-worker/cloud"]
 jemalloc = ["windmill-common/jemalloc", "dep:tikv-jemallocator", "dep:tikv-jemalloc-sys", "dep:tikv-jemalloc-ctl"]
 tantivy = ["dep:windmill-indexer", "windmill-api/tantivy"]
 sqlx = ["windmill-worker/sqlx"]
-deno_core = ["windmill-worker/deno_core", "dep:deno_core"]
 
 [dependencies]
 anyhow.workspace = true
@@ -85,8 +85,9 @@ uuid.workspace = true
 gethostname.workspace = true
 serde_json.workspace = true
 serde.workspace = true
-deno_core = { workspace = true, optional = true }
+deno_core.workspace = true
 object_store = { workspace = true, optional = true }
+pg-embed = {git = "https://github.com/faokunega/pg-embed", optional = true, default-features = false, features = ['rt_tokio']}
 quote.workspace = true
 
 
@@ -104,7 +105,6 @@ serde.workspace = true
 windmill-api-client.workspace = true
 deno_core = { workspace = true, features = ["include_js_files_for_snapshotting", "unsafe_use_unprotected_platform"] }
 
-
 [workspace.dependencies]
 windmill-api = { path = "./windmill-api", default-features = false  }
 windmill-queue = { path = "./windmill-queue" }

backend/custom_migrations/create_workspace_without_md5.sql

@@ -1,16 +0,0 @@
-INSERT INTO workspace(id, name, owner) VALUES
-	('admins', 'Admins', 'admin@windmill.dev') ON CONFLICT DO NOTHING;
-
-INSERT INTO workspace_settings (workspace_id) VALUES
-	('admins') ON CONFLICT DO NOTHING;
-
-INSERT INTO workspace_key
-	(workspace_id, kind, key)
-	VALUES ('admins', 'cloud', array_to_string(
-    array(
-        SELECT chr( (trunc(65 + random() * 25)::int) + 
-                    CASE WHEN random() > 0.5 THEN 32 ELSE 0 END )  -- generates random uppercase/lowercase letters
-        FROM generate_series(1, 32) -- generates 32 characters
-    ),
-    ''
-)) ON CONFLICT DO NOTHING;

backend/ee-repo-ref.txt

@@ -1 +1 @@
-0f5f42d2f8f5f1af05c8086f3dc7ad38d83750df
+3d37b6c31155265d8d026ae9d6ced0b433078f87

backend/migrations/20240924213010_add_last_occupancy_rates.down.sql

@@ -1,5 +0,0 @@
--- Add down migration script here
-ALTER TABLE worker_ping
-DROP COLUMN occupancy_rate_15s,
-DROP COLUMN occupancy_rate_5m,
-DROP COLUMN occupancy_rate_30m;

backend/migrations/20240924213010_add_last_occupancy_rates.up.sql

@@ -1,5 +0,0 @@
--- Add up migration script here
-ALTER TABLE worker_ping
-ADD COLUMN occupancy_rate_15s REAL,
-ADD COLUMN occupancy_rate_5m REAL,
-ADD COLUMN occupancy_rate_30m REAL;

backend/migrations/20240930183601_add_preprocessor_kind.down.sql

@@ -1 +0,0 @@
--- Add down migration script here

backend/migrations/20240930183601_add_preprocessor_kind.up.sql

@@ -1,2 +0,0 @@
--- Add up migration script here
-ALTER TYPE SCRIPT_KIND ADD VALUE IF NOT EXISTS 'preprocessor';

backend/parsers/windmill-parser-yaml/src/lib.rs

@@ -399,11 +399,15 @@ fn parse_ansible_options(opts: &Vec<Yaml>) -> AnsiblePlaybookOptions {
                             if c > 0 && c <= 6 {
                                 ret.verbosity = Some("v".repeat(c.min(6)));
                             }
+
                         }
                     }
-                    _ => (),
+                    _ => ()
+
                 }
             }
+
+
         }
     }
 
@@ -418,10 +422,10 @@ fn count_consecutive_vs(s: &str) -> usize {
         if c == 'v' {
             current_count += 1;
             if current_count == 6 {
-                return 6; // Stop early if we reach 6
+                return 6;  // Stop early if we reach 6
             }
         } else {
-            current_count = 0; // Reset count if the character is not 'v'
+            current_count = 0;  // Reset count if the character is not 'v'
         }
         max_count = max_count.max(current_count);
     }

backend/plot.py

@@ -1,89 +0,0 @@
-import json
-import matplotlib.pyplot as plt
-
-# Function to load JSON data from a file
-def load_json_data(filepath):
-    with open(filepath, 'r') as file:
-        data = json.load(file)
-    return data
-
-# Function to plot two arrays of subarrays with tuples (step_name, duration)
-def plot_two_arrays_of_subarrays(arrays1, arrays2):
-    # Function to calculate sum of durations for each step
-    def calculate_sums(arrays):
-        steps = [step for step, _ in arrays[0]['timings']]  # Extract steps from the first iteration
-        sums = {step: 0.0 for step in steps}  # Initialize sums dictionary with step names
-
-        # Sum up the durations for each step across all subarrays
-        for subarray in arrays:
-            for step_name, duration in subarray['timings']:
-                if step_name not in sums:
-                    sums[step_name] = 0
-                sums[step_name] += duration
-
-        for step_name, duration in sums.items():
-            sums[step_name] = duration / 1000000000
-
-        # Convert the sums dictionary to two lists (for plotting)
-        step_names = list(sums.keys())
-        durations = list(sums.values())
-        return step_names, durations
-
-    # Calculate sums for both arrays of subarrays
-    step_names1, sums1 = calculate_sums(arrays1)
-    step_names2, sums2 = calculate_sums(arrays2)
-
-    # Create two subplots, one on top of the other
-    fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(10, 12))
-
-    # First plot (top) for the first array of subarrays
-    ax1.bar(step_names1, sums1, color='b')
-    ax1.set_title('Total Duration per Step - Main Loop')
-    ax1.set_xlabel('Step Name')
-    ax1.set_ylabel('Total Duration (s)')
-    ax1.grid(True, axis='y')
-    ax1.tick_params(axis='x', rotation=45)
-
-    # Second plot (bottom) for the second array of subarrays
-    ax2.bar(step_names2, sums2, color='r')
-    ax2.set_title('Total Duration per Step - Result Processor')
-    ax2.set_xlabel('Step Name')
-    ax2.set_ylabel('Total Duration (s)')
-    ax2.grid(True, axis='y')
-    ax2.tick_params(axis='x', rotation=45)
-
-    # Adjust layout so the plots don't overlap
-    plt.tight_layout()
-
-    # Display the plot
-    plt.show()
-
-# Load arrays from the JSON files
-main = load_json_data('/tmp/windmill/profiling_main.json')
-result_processor = load_json_data('/tmp/windmill/profiling_result_processor.json')
-
-arrays1 = main['timings']
-arrays2 = result_processor['timings']
-
-total_duration1 = main['total_duration']/1000
-total_duration2 = result_processor['total_duration']/1000
-
-print(f"Total duration for main: {total_duration1}s")
-print(f"Total duration for result processor: {total_duration2}s")
-
-iterations_total = sum(main['iter_durations']) / 1000000000
-iterations_total2 = sum(result_processor['iter_durations']) / 1000000000
-
-print(f"Number of iterations: {len(main['iter_durations'])}")
-print(f"Total iterations for main: {iterations_total}s")
-print(f"Total iterations for result processor: {iterations_total2}s")
-
-# Calculate RPS
-rps1 = len(main['iter_durations']) / total_duration1
-rps2 = len(result_processor['iter_durations']) / total_duration2
-
-print(f"RPS for main: {rps1}")
-print(f"RPS for result processor: {rps2}")
-
-# Plot the data
-plot_two_arrays_of_subarrays(arrays1, arrays2)

backend/src/main.rs

@@ -67,7 +67,7 @@ use windmill_worker::{
     get_hub_script_content_and_requirements, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR,
     BUN_DEPSTAR_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM,
     GO_BIN_CACHE_DIR, GO_CACHE_DIR, LOCK_CACHE_DIR, PIP_CACHE_DIR, POWERSHELL_CACHE_DIR,
-    RUST_CACHE_DIR, TAR_PIP_CACHE_DIR, TMP_LOGS_DIR, UV_CACHE_DIR,
+    RUST_CACHE_DIR, TAR_PIP_CACHE_DIR, TMP_LOGS_DIR,
 };
 
 use crate::monitor::{
@@ -92,6 +92,9 @@ const DEFAULT_SERVER_BIND_ADDR: Ipv4Addr = Ipv4Addr::new(0, 0, 0, 0);
 mod ee;
 mod monitor;
 
+#[cfg(feature = "pg_embed")]
+mod pg_embed;
+
 #[inline(always)]
 fn create_and_run_current_thread_inner<F, R>(future: F) -> R
 where
@@ -115,7 +118,6 @@ where
 }
 
 pub fn main() -> anyhow::Result<()> {
-    #[cfg(feature = "deno_core")]
     deno_core::JsRuntime::init_platform(None);
     create_and_run_current_thread_inner(windmill_main())
 }
@@ -135,7 +137,6 @@ async fn cache_hub_scripts(file_path: Option<String>) -> anyhow::Result<()> {
     })?;
 
     create_dir_all(HUB_CACHE_DIR).await?;
-    create_dir_all(BUN_BUNDLE_CACHE_DIR).await?;
 
     for path in paths.values() {
         tracing::info!("Caching hub script at {path}");
@@ -157,7 +158,6 @@ async fn cache_hub_scripts(file_path: Option<String>) -> anyhow::Result<()> {
                 "global",
                 "global",
                 "",
-                &mut None,
             )
             .await?;
             tokio::fs::remove_dir_all(job_dir).await?;
@@ -167,7 +167,7 @@ async fn cache_hub_scripts(file_path: Option<String>) -> anyhow::Result<()> {
             create_dir_all(&job_dir).await?;
             if let Some(lockfile) = res.lockfile {
                 let _ = windmill_worker::prepare_job_dir(&lockfile, &job_dir).await?;
-                let envs = windmill_worker::get_common_bun_proc_envs(None).await;
+
                 let _ = windmill_worker::install_bun_lockfile(
                     &mut 0,
                     &mut None,
@@ -176,31 +176,10 @@ async fn cache_hub_scripts(file_path: Option<String>) -> anyhow::Result<()> {
                     None,
                     &job_dir,
                     "cache_init",
-                    envs.clone(),
+                    windmill_worker::get_common_bun_proc_envs(None).await,
                     false,
-                    &mut None,
                 )
                 .await?;
-
-                let _ = windmill_common::worker::write_file(&job_dir, "main.js", &res.content)?;
-
-                if let Err(e) = windmill_worker::prebundle_bun_script(
-                    &res.content,
-                    Some(lockfile),
-                    &path,
-                    &job_id,
-                    "admins",
-                    None,
-                    &job_dir,
-                    "",
-                    "cache_init",
-                    "",
-                    &mut None,
-                )
-                .await
-                {
-                    panic!("Error prebundling bun script: {e:#}");
-                }
             } else {
                 tracing::warn!("No lockfile found for bun script {path}, skipping...");
             }
@@ -360,6 +339,14 @@ async fn windmill_main() -> anyhow::Result<()> {
         config
     });
 
+    #[cfg(feature = "pg_embed")]
+    let _pg = {
+        let (db_url, pg) = pg_embed::start().await.expect("pg embed");
+        tracing::info!("Use embedded pg: {db_url}");
+        std::env::set_var("DATABASE_URL", db_url);
+        pg
+    };
+
     tracing::info!("Connecting to database...");
     let db = windmill_common::connect_db(server_mode, indexer_mode).await?;
     tracing::info!("Database connected");
@@ -384,16 +371,8 @@ async fn windmill_main() -> anyhow::Result<()> {
     let is_agent = mode == Mode::Agent;
 
     if !is_agent {
-        let skip_migration = std::env::var("SKIP_MIGRATION")
-            .map(|val| val == "true")
-            .unwrap_or(false);
-
-        if !skip_migration {
-            // migration code to avoid break
-            windmill_api::migrate_db(&db).await?;
-        } else {
-            tracing::info!("SKIP_MIGRATION set, skipping db migration...")
-        }
+        // migration code to avoid break
+        windmill_api::migrate_db(&db).await?;
     }
 
     let (killpill_tx, mut killpill_rx) = tokio::sync::broadcast::channel::<()>(2);
@@ -764,8 +743,9 @@ Windmill Community Edition {GIT_VERSION}
             Ok(()) as anyhow::Result<()>
         };
 
+        let instance_name = rd_string(8);
         if mode == Mode::Server || mode == Mode::Standalone {
-            schedule_stats(&db, &HTTP_CLIENT).await;
+            schedule_stats(instance_name, &db, &HTTP_CLIENT).await;
         }
 
         #[cfg(feature = "enterprise")]
@@ -893,7 +873,6 @@ pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + '
         LOCK_CACHE_DIR,
         TMP_LOGS_DIR,
         PIP_CACHE_DIR,
-        UV_CACHE_DIR,
         TAR_PIP_CACHE_DIR,
         DENO_CACHE_DIR,
         DENO_CACHE_DIR_DEPS,

backend/src/monitor.rs

@@ -525,7 +525,7 @@ fn read_log_counters(ts_str: String) -> (usize, usize) {
             ok_lines = counter.non_error_count;
             err_lines = counter.error_count;
         } else {
-            // println!("no counter found for {ts_str}");
+            println!("no counter found for {ts_str}");
         }
     } else {
         println!("Error reading log counters 2");
@@ -1071,68 +1071,73 @@ pub async fn monitor_db(
 }
 
 pub async fn expose_queue_metrics(db: &Pool<Postgres>) {
-    let last_check = sqlx::query_scalar!(
+    let tx = db.begin().await;
+    if let Ok(mut tx) = tx {
+        let last_check = sqlx::query_scalar!(
             "SELECT created_at FROM metrics WHERE id LIKE 'queue_count_%' ORDER BY created_at DESC LIMIT 1"
         )
         .fetch_optional(db)
         .await
         .unwrap_or(Some(chrono::Utc::now()));
 
-    let metrics_enabled = METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed);
-    let save_metrics = last_check
-        .map(|last_check| chrono::Utc::now() - last_check > chrono::Duration::seconds(25))
-        .unwrap_or(true);
+        let metrics_enabled = METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed);
+        let save_metrics = last_check
+            .map(|last_check| chrono::Utc::now() - last_check > chrono::Duration::seconds(25))
+            .unwrap_or(true);
 
-    if metrics_enabled || save_metrics {
-        let queue_counts = sqlx::query!(
-            "SELECT tag, count(*) as count FROM queue WHERE
+        if metrics_enabled || save_metrics {
+            let queue_counts = sqlx::query!(
+                "SELECT tag, count(*) as count FROM queue WHERE
                 scheduled_for <= now() - ('3 seconds')::interval AND running = false
                 GROUP BY tag"
-        )
-        .fetch_all(db)
-        .await
-        .ok()
-        .unwrap_or_else(|| vec![]);
+            )
+            .fetch_all(&mut *tx)
+            .await
+            .ok()
+            .unwrap_or_else(|| vec![]);
 
-        for q in queue_counts {
-            let count = q.count.unwrap_or(0);
-            let tag = q.tag;
-            if metrics_enabled {
-                let metric = (*QUEUE_COUNT).with_label_values(&[&tag]);
-                metric.set(count as i64);
-            }
+            for q in queue_counts {
+                let count = q.count.unwrap_or(0);
+                let tag = q.tag;
+                if metrics_enabled {
+                    let metric = (*QUEUE_COUNT).with_label_values(&[&tag]);
+                    metric.set(count as i64);
+                }
 
-            // save queue_count and delay metrics per tag
-            if save_metrics {
-                sqlx::query!(
-                    "INSERT INTO metrics (id, value) VALUES ($1, $2)",
-                    format!("queue_count_{}", tag),
-                    serde_json::json!(count)
-                )
-                .execute(db)
-                .await
-                .ok();
-                if count > 0 {
+                // save queue_count and delay metrics per tag
+                if save_metrics {
                     sqlx::query!(
+                        "INSERT INTO metrics (id, value) VALUES ($1, $2)",
+                        format!("queue_count_{}", tag),
+                        serde_json::json!(count)
+                    )
+                    .execute(&mut *tx)
+                    .await
+                    .ok();
+                    if count > 0 {
+                        sqlx::query!(
                             "INSERT INTO metrics (id, value)
                             VALUES ($1, to_jsonb((SELECT EXTRACT(EPOCH FROM now() - scheduled_for)
                             FROM queue WHERE tag = $2 AND running = false AND scheduled_for <= now() - ('3 seconds')::interval
-                            ORDER BY priority DESC NULLS LAST, scheduled_for LIMIT 1)))",
+                            ORDER BY priority DESC NULLS LAST, scheduled_for, created_at LIMIT 1)))",
                             format!("queue_delay_{}", tag),
                             tag
-                        ).execute(db).await.ok();
+                        ).execute(&mut *tx).await.ok();
+                    }
                 }
             }
         }
-    }
 
-    // clean queue metrics older than 14 days
-    sqlx::query!(
-        "DELETE FROM metrics WHERE id LIKE 'queue_%' AND created_at < NOW() - INTERVAL '14 day'"
-    )
-    .execute(db)
-    .await
-    .ok();
+        // clean queue metrics older than 14 days
+        sqlx::query!(
+            "DELETE FROM metrics WHERE id LIKE 'queue_%' AND created_at < NOW() - INTERVAL '14 day'"
+        )
+        .execute(&mut *tx)
+        .await
+        .ok();
+
+        tx.commit().await.ok();
+    }
 }
 
 pub async fn reload_smtp_config(db: &Pool<Postgres>) {
@@ -1368,8 +1373,6 @@ async fn handle_zombie_jobs<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
             rsmq.clone(),
             worker_name,
             send_result_never_used,
-            #[cfg(feature = "benchmark")]
-            &mut windmill_common::bench::BenchmarkIter::new(),
         )
         .await;
     }

backend/src/pg_embed.rs

@@ -0,0 +1,46 @@
+use pg_embed::pg_enums::PgAuthMethod;
+use pg_embed::pg_fetch::PgFetchSettings;
+use pg_embed::postgres::{PgEmbed, PgSettings};
+use std::path::PathBuf;
+use std::time::Duration;
+
+pub async fn start() -> anyhow::Result<(String, PgEmbed)> {
+    let pg_settings = PgSettings {
+        database_dir: PathBuf::from("/tmp/db"),
+        port: 6543,
+        user: "postgres".to_string(),
+        password: "password".to_string(),
+        auth_method: PgAuthMethod::Plain,
+        persistent: false,
+        timeout: Some(Duration::from_secs(15)),
+        migration_dir: None,
+    };
+
+    let fetch_settings = PgFetchSettings {
+        version: pg_embed::pg_fetch::PostgresVersion("15.3.0"),
+
+        ..Default::default()
+    };
+
+    tracing::info!(
+        "Fetch settings: {:?} {:?}",
+        fetch_settings.operating_system,
+        fetch_settings.architecture
+    );
+
+    let mut pg = PgEmbed::new(pg_settings, fetch_settings).await?;
+
+    pg.setup().await.expect("pg setup");
+
+    pg.start_db().await.expect("pg start db");
+
+    //TODO: re-enable this to make it work
+    // if !pg.database_exists("windmill").await.expect("db exists") {
+    //     pg.create_database("windmill")
+    //         .await
+    //         .expect("pg create database");
+    // }
+
+    let uri = pg.full_db_uri("windmill");
+    Ok((uri, pg))
+}

backend/tests/worker.rs

@@ -1126,7 +1126,6 @@ async fn test_deno_flow(db: Pool<Postgres>) {
                     priority: None,
                     delete_after_use: None,
                     continue_on_error: None,
-                    skip_if: None,
                 },
                 FlowModule {
                     id: "b".to_string(),
@@ -1167,7 +1166,6 @@ async fn test_deno_flow(db: Pool<Postgres>) {
                             priority: None,
                             delete_after_use: None,
                             continue_on_error: None,
-                            skip_if: None,
                         }],
                     }
                     .into(),
@@ -1183,7 +1181,6 @@ async fn test_deno_flow(db: Pool<Postgres>) {
                     priority: None,
                     delete_after_use: None,
                     continue_on_error: None,
-                    skip_if: None,
                 },
             ],
             same_worker: false,
@@ -1289,7 +1286,6 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
                     priority: None,
                     delete_after_use: None,
                     continue_on_error: None,
-                    skip_if: None,
                 },
                 FlowModule {
                     id: "b".to_string(),
@@ -1340,7 +1336,6 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
                                 priority: None,
                                 delete_after_use: None,
                                 continue_on_error: None,
-                                skip_if: None,
                             },
                             FlowModule {
                                 id: "e".to_string(),
@@ -1377,7 +1372,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
                                 priority: None,
                                 delete_after_use: None,
                                 continue_on_error: None,
-                                skip_if: None,
+
                             },
                         ],
                     }.into(),
@@ -1393,7 +1388,6 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
                     priority: None,
                     delete_after_use: None,
                     continue_on_error: None,
-                    skip_if: None,
                 },
                 FlowModule {
                     id: "c".to_string(),
@@ -1437,7 +1431,6 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
                     priority: None,
                     delete_after_use: None,
                     continue_on_error: None,
-                    skip_if: None,
                 },
             ],
             same_worker: true,

backend/windmill-api/Cargo.toml

@@ -97,6 +97,7 @@ jsonwebtoken = { workspace = true }
 matchit.workspace = true
 
 pin-project.workspace = true
+crc.workspace = true
 http.workspace = true
 async-stream.workspace = true
 ulid.workspace = true

backend/windmill-api/openapi.yaml

@@ -1,7 +1,7 @@
 openapi: "3.0.3"
 
 info:
-  version: 1.405.4
+  version: 1.399.0
   title: Windmill API
 
   contact:
@@ -5964,43 +5964,6 @@ paths:
               schema:
                 type: integer
 
-  /jobs/completed/count_by_tag:
-    get:
-      summary: Count jobs by tag
-      operationId: countJobsByTag
-      tags:
-        - job
-      parameters:
-        - name: horizon_secs
-          in: query
-          description: Past Time horizon in seconds (when to start the count = now - horizon) (default is 3600)
-          required: false
-          schema:
-            type: integer
-        - name: workspace_id
-          in: query
-          description: Specific workspace ID to filter results (optional)
-          required: false
-          schema:
-            type: string
-      responses:
-        "200":
-          description: Job counts by tag
-          content:
-            application/json:
-              schema:
-                type: array
-                items:
-                  type: object
-                  properties:
-                    tag:
-                      type: string
-                    count:
-                      type: integer
-                  required:
-                    - tag
-                    - count
-
   /w/{workspace}/jobs_u/get/{id}:
     get:
       summary: get job
@@ -6535,7 +6498,6 @@ paths:
               schema:
                 type: string
 
-
   /w/{workspace}/jobs_u/cancel/{id}/{resume_id}/{signature}:
     get:
       summary: cancel a job for a suspended flow
@@ -9987,8 +9949,6 @@ components:
           type: number
         aggregate_wait_time_ms:
           type: number
-        suspend:
-          type: number
       required:
         - id
         - running
@@ -11198,12 +11158,6 @@ components:
           type: string
         occupancy_rate:
           type: number
-        occupancy_rate_15s:
-          type: number
-        occupancy_rate_5m:
-          type: number
-        occupancy_rate_30m:
-          type: number
         memory:
           type: number
         vcpus:

backend/windmill-api/src/configs.rs

@@ -33,7 +33,7 @@ pub fn global_service() -> Router {
 
 #[derive(Serialize, Deserialize, FromRow)]
 struct Config {
-    name: Option<String>,
+    name: String,
     config: serde_json::Value,
 }
 
@@ -41,18 +41,9 @@ async fn list_worker_groups(
     authed: ApiAuthed,
     Extension(db): Extension<DB>,
 ) -> error::JsonResult<Vec<Config>> {
-    let mut configs_raw =
-        sqlx::query_as!(Config, "SELECT * FROM config WHERE name LIKE 'worker__%'")
-            .fetch_all(&db)
-            .await?;
-    // Remove the 'worker__' prefix from all config names
-    for config in configs_raw.iter_mut() {
-        if let Some(name) = &config.name {
-            if name.starts_with("worker__") {
-                config.name = Some(name.strip_prefix("worker__").unwrap().to_string());
-            }
-        }
-    }
+    let configs_raw = sqlx::query_as!(Config, "SELECT * FROM config WHERE name LIKE 'worker__%'")
+        .fetch_all(&db)
+        .await?;
     let configs = if !authed.is_admin {
         let mut obfuscated_configs: Vec<Config> = vec![];
         for config in configs_raw {

backend/windmill-api/src/db.rs

@@ -15,7 +15,6 @@ use sqlx::{
     PgConnection, Pool, Postgres,
 };
 use windmill_audit::audit_ee::{AuditAuthor, AuditAuthorable};
-use windmill_common::utils::generate_lock_id;
 use windmill_common::{
     db::{Authable, Authed},
     error::Error,
@@ -30,6 +29,13 @@ async fn current_database(conn: &mut PgConnection) -> Result<String, MigrateErro
         .await?)
 }
 
+// inspired from rails: https://github.com/rails/rails/blob/6e49cc77ab3d16c06e12f93158eaf3e507d4120e/activerecord/lib/active_record/migration.rb#L1308
+fn generate_lock_id(database_name: &str) -> i64 {
+    const CRC_IEEE: crc::Crc<u32> = crc::Crc::<u32>::new(&crc::CRC_32_ISO_HDLC);
+    // 0x3d32ad9e chosen by fair dice roll
+    0x3d32ad9e * (CRC_IEEE.checksum(database_name.as_bytes()) as i64)
+}
+
 struct CustomMigrator {
     inner: PoolConnection<Postgres>,
 }
@@ -130,30 +136,9 @@ impl Migrate for CustomMigrator {
                 migration.version,
                 migration.description
             );
-            if migration.version == 20221207103910 {
-                tracing::info!("Skipping migration 20221207103910 to avoid using md5");
-                self.inner
-                    .execute(include_str!(
-                        "../../custom_migrations/create_workspace_without_md5.sql"
-                    ))
-                    .await?;
-                let _ = sqlx::query(
-                    r#"
-                INSERT INTO _sqlx_migrations ( version, description, success, checksum, execution_time )
-                VALUES ( $1, $2, TRUE, $3, -1 ) ON CONFLICT DO NOTHING
-                            "#,
-                )
-                .bind(migration.version)
-                .bind(&*migration.description)
-                .bind(&*migration.checksum)
-                .execute(&mut *self.inner)
-                .await?;
-                return Ok(std::time::Duration::from_secs(0));
-            } else {
-                let r = self.inner.apply(migration).await;
-                tracing::info!("Finished applying migration {}", migration.version);
-                return r;
-            }
+            let r = self.inner.apply(migration).await;
+            tracing::info!("Finished applying migration {}", migration.version);
+            r
         }
         .boxed()
     }

backend/windmill-api/src/flows.rs

@@ -242,7 +242,6 @@ pub async fn get_hub_flow_by_id(
 
 #[derive(Deserialize)]
 pub struct ToggleWorkspaceErrorHandler {
-    #[cfg(feature = "enterprise")]
     pub muted: Option<bool>,
 }
 
@@ -1176,7 +1175,6 @@ mod tests {
                     priority: None,
                     delete_after_use: None,
                     continue_on_error: None,
-                    skip_if: None,
                 },
                 FlowModule {
                     id: "b".to_string(),
@@ -1206,7 +1204,6 @@ mod tests {
                     priority: None,
                     delete_after_use: None,
                     continue_on_error: None,
-                    skip_if: None,
                 },
                 FlowModule {
                     id: "c".to_string(),
@@ -1234,7 +1231,6 @@ mod tests {
                     priority: None,
                     delete_after_use: None,
                     continue_on_error: None,
-                    skip_if: None,
                 },
             ],
             failure_module: Some(Box::new(FlowModule {
@@ -1261,7 +1257,6 @@ mod tests {
                 priority: None,
                 delete_after_use: None,
                 continue_on_error: None,
-                skip_if: None,
             })),
             preprocessor_module: None,
             same_worker: false,

backend/windmill-api/src/jobs.rs

@@ -18,7 +18,6 @@ use tokio::io::AsyncReadExt;
 #[cfg(feature = "prometheus")]
 use tokio::time::Instant;
 use tower::ServiceBuilder;
-use windmill_common::error::JsonResult;
 use windmill_common::flow_status::{JobResult, RestartedFrom};
 use windmill_common::jobs::{
     format_completed_job_result, format_result, CompletedJobWithFormattedResult, FormattedResult,
@@ -70,7 +69,7 @@ use windmill_common::{
     oauth2::HmacSha256,
     scripts::{ScriptHash, ScriptLang},
     users::username_to_permissioned_as,
-    utils::{not_found_if_none, now_from_db, paginate, paginate_without_limits, require_admin, Pagination, StripPath},
+    utils::{not_found_if_none, now_from_db, paginate, require_admin, Pagination, StripPath},
 };
 
 #[cfg(all(feature = "enterprise", feature = "parquet"))]
@@ -248,7 +247,7 @@ pub fn workspaced_service() -> Router {
         .route("/run/flow_dependencies", post(run_flow_dependencies_job))
 }
 
-pub fn workspace_unauthed_service() -> Router {
+pub fn global_service() -> Router {
     Router::new()
         .route(
             "/resume/:job_id/:resume_id/:secret",
@@ -292,9 +291,7 @@ pub fn workspace_unauthed_service() -> Router {
 }
 
 pub fn global_root_service() -> Router {
-    Router::new()
-        .route("/db_clock", get(get_db_clock))
-        .route("/completed/count_by_tag", get(count_by_tag))
+    Router::new().route("/db_clock", get(get_db_clock))
 }
 
 #[derive(Deserialize)]
@@ -1251,16 +1248,13 @@ pub fn list_queue_jobs_query(
     w_id: &str,
     lq: &ListQueueQuery,
     fields: &[&str],
-    pagination: Pagination,
     join_outstanding_wait_times: bool,
     tags: Option<Vec<&str>>,
 ) -> SqlBuilder {
-    let (limit, offset) = paginate_without_limits(pagination);
     let mut sqlb = SqlBuilder::select_from("queue")
         .fields(fields)
         .order_by("created_at", lq.order_desc.unwrap_or(true))
-        .limit(limit)
-        .offset(offset)
+        .limit(1000)
         .clone();
 
     if let Some(tags) = tags {
@@ -1273,7 +1267,6 @@ pub fn list_queue_jobs_query(
 #[derive(Serialize, FromRow)]
 struct ListableQueuedJob {
     pub id: Uuid,
-    pub running: bool,
     pub created_by: String,
     pub created_at: chrono::DateTime<chrono::Utc>,
     pub started_at: Option<chrono::DateTime<chrono::Utc>>,
@@ -1296,7 +1289,6 @@ async fn list_queue_jobs(
     authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
     Path(w_id): Path<String>,
-    Query(pagination): Query<Pagination>,
     Query(lq): Query<ListQueueQuery>,
 ) -> error::JsonResult<Vec<ListableQueuedJob>> {
     let sql = list_queue_jobs_query(
@@ -1304,7 +1296,6 @@ async fn list_queue_jobs(
         &lq,
         &[
             "id",
-            "running",
             "created_by",
             "created_at",
             "started_at",
@@ -1324,7 +1315,6 @@ async fn list_queue_jobs(
             "priority",
             "workspace_id",
         ],
-        pagination,
         false,
         get_scope_tags(&authed),
     )
@@ -1580,7 +1570,6 @@ async fn list_jobs(
 ) -> error::JsonResult<Vec<Job>> {
     check_scopes(&authed, || format!("jobs:listjobs"))?;
 
-    let limit = pagination.per_page.unwrap_or(1000);
     let (per_page, offset) = paginate(pagination);
     let lqc = lq.clone();
 
@@ -1612,7 +1601,6 @@ async fn list_jobs(
             &w_id,
             &ListQueueQuery { order_desc: Some(true), ..lq.into() },
             UnifiedJob::queued_job_fields(),
-            Pagination { per_page: Some(limit), page: None },
             true,
             get_scope_tags(&authed),
         );
@@ -4127,7 +4115,7 @@ async fn run_dependencies_job(
             JsonRawValue::from_string("true".to_string()).unwrap(),
         );
         if language == ScriptLang::Bun {
-            let annotation = windmill_common::worker::get_annotation_ts(&raw_code);
+            let annotation = windmill_common::worker::get_annotation(&raw_code);
             hm.insert(
                 "npm_mode".to_string(),
                 JsonRawValue::from_string(annotation.npm_mode.to_string()).unwrap(),
@@ -4680,8 +4668,8 @@ async fn get_job_update(
     .fetch_optional(&db)
     .await?;
 
-    let progress: Option<i32> = if get_progress == Some(true) {
-        sqlx::query_scalar!(
+    let progress: Option<i32> = if get_progress == Some(true){
+         sqlx::query_scalar!(
                 "SELECT scalar_int FROM job_stats WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3",
                 &w_id,
                  job_id,
@@ -5112,44 +5100,6 @@ async fn get_completed_job_result(
     Ok(Json(result).into_response())
 }
 
-#[derive(Deserialize)]
-struct CountByTagQuery {
-    horizon_secs: Option<i64>,
-    workspace_id: Option<String>,
-}
-
-#[derive(Serialize)]
-struct TagCount {
-    tag: String,
-    count: i64,
-}
-
-async fn count_by_tag(
-    ApiAuthed { email, .. }: ApiAuthed,
-    Extension(db): Extension<DB>,
-    Query(query): Query<CountByTagQuery>,
-) -> JsonResult<Vec<TagCount>> {
-    require_super_admin(&db, &email).await?;
-    let horizon = query.horizon_secs.unwrap_or(3600); // Default to 1 hour if not specified
-
-    let counts = sqlx::query_as!(
-        TagCount,
-        r#"
-        SELECT tag as "tag!", COUNT(*) as "count!"
-        FROM completed_job
-        WHERE started_at > NOW() - make_interval(secs => $1) AND ($2::text IS NULL OR workspace_id = $2)
-        GROUP BY tag
-        ORDER BY "count!" DESC
-        "#,
-        horizon as f64,
-        query.workspace_id
-    )
-    .fetch_all(&db)
-    .await?;
-
-    Ok(Json(counts))
-}
-
 #[derive(Serialize)]
 struct CompletedJobResult {
     started: Option<bool>,

backend/windmill-api/src/lib.rs

@@ -39,8 +39,9 @@ use tower_http::{
     trace::TraceLayer,
 };
 use windmill_common::db::UserDB;
+use windmill_common::utils::rd_string;
 use windmill_common::worker::ALL_TAGS;
-use windmill_common::{BASE_URL, INSTANCE_NAME};
+use windmill_common::BASE_URL;
 
 use crate::scim_ee::has_scim_token;
 use windmill_common::error::AppError;
@@ -329,7 +330,7 @@ pub async fn run_server(
                 )
                 .nest(
                     "/w/:workspace_id/jobs_u",
-                    jobs::workspace_unauthed_service().layer(cors.clone()),
+                    jobs::global_service().layer(cors.clone()),
                 )
                 .nest(
                     "/w/:workspace_id/resources_u",
@@ -372,6 +373,8 @@ pub async fn run_server(
         )
     };
 
+    let instance_name = rd_string(5);
+
     let listener = tokio::net::TcpListener::bind(addr).await.unwrap();
     let port = listener.local_addr().map(|x| x.port()).unwrap_or(8000);
     let ip = listener
@@ -382,7 +385,7 @@ pub async fn run_server(
     let server = axum::serve(listener, app.into_make_service());
 
     tracing::info!(
-        instance = %*INSTANCE_NAME,
+        instance = %instance_name,
         "server started on port={} and addr={}",
         port,
         ip

backend/windmill-api/src/resources.rs

@@ -58,10 +58,7 @@ pub fn workspaced_service() -> Router {
         .route("/type/exists/:name", get(exists_resource_type))
         .route("/type/update/:name", post(update_resource_type))
         .route("/type/delete/:name", delete(delete_resource_type))
-        .route(
-            "/file_resource_type_to_file_ext_map",
-            get(file_resource_ext_to_resource_type),
-        )
+        .route("/file_resource_type_to_file_ext_map", get(file_resource_ext_to_resource_type))
         .route("/type/create", post(create_resource_type))
 }
 

backend/windmill-api/src/scripts.rs

@@ -53,7 +53,7 @@ use windmill_common::{
     utils::{
         not_found_if_none, paginate, query_elems_from_hub, require_admin, Pagination, StripPath,
     },
-    worker::{get_annotation_ts, to_raw_value},
+    worker::{get_annotation, to_raw_value},
     HUB_BASE_URL,
 };
 use windmill_git_sync::{handle_deployment_metadata, DeployedObject};
@@ -601,7 +601,7 @@ async fn create_script_internal<'c>(
     };
 
     let lang = if &ns.language == &ScriptLang::Bun || &ns.language == &ScriptLang::Bunnative {
-        let anns = get_annotation_ts(&ns.content);
+        let anns = get_annotation(&ns.content);
         if anns.native_mode {
             ScriptLang::Bunnative
         } else {
@@ -967,7 +967,6 @@ async fn list_paths(
 
 #[derive(Deserialize)]
 pub struct ToggleWorkspaceErrorHandler {
-    #[cfg(feature = "enterprise")]
     pub muted: Option<bool>,
 }
 

backend/windmill-api/src/settings.rs

@@ -24,9 +24,9 @@ use axum::{
 #[cfg(feature = "enterprise")]
 use axum::extract::Query;
 
-use serde::Deserialize;
 #[cfg(feature = "enterprise")]
 use windmill_common::ee::{send_critical_alert, CriticalAlertKind, CriticalErrorChannel};
+use serde::Deserialize;
 use windmill_common::{
     error::{self, JsonResult, Result},
     global_settings::{
@@ -298,13 +298,7 @@ async fn list_global_settings() -> JsonResult<String> {
 
 pub async fn send_stats(Extension(db): Extension<DB>, authed: ApiAuthed) -> Result<String> {
     require_super_admin(&db, &authed.email).await?;
-    windmill_common::stats_ee::send_stats(
-        &HTTP_CLIENT,
-        &db,
-        true,
-        windmill_common::stats_ee::SendStatsReason::Manual,
-    )
-    .await?;
+    windmill_common::stats_ee::send_stats(&"manual".to_string(), &HTTP_CLIENT, &db).await?;
 
     Ok("Sent stats".to_string())
 }
@@ -363,13 +357,8 @@ pub async fn renew_license_key(
     authed: ApiAuthed,
 ) -> Result<String> {
     require_super_admin(&db, &authed.email).await?;
-    let result = windmill_common::ee::renew_license_key(
-        &HTTP_CLIENT,
-        &db,
-        license_key,
-        windmill_common::ee::RenewReason::Manual,
-    )
-    .await;
+    windmill_common::stats_ee::send_stats(&"manual".to_string(), &HTTP_CLIENT, &db).await?;
+    let result = windmill_common::ee::renew_license_key(&HTTP_CLIENT, &db, license_key, true).await;
 
     if result != "success" {
         return Err(error::Error::BadRequest(format!(

backend/windmill-api/src/workers.rs

@@ -54,12 +54,6 @@ struct WorkerPing {
     #[serde(skip_serializing_if = "Option::is_none")]
     occupancy_rate: Option<f32>,
     #[serde(skip_serializing_if = "Option::is_none")]
-    occupancy_rate_15s: Option<f32>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    occupancy_rate_5m: Option<f32>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    occupancy_rate_30m: Option<f32>,
-    #[serde(skip_serializing_if = "Option::is_none")]
     memory: Option<i64>,
     #[serde(skip_serializing_if = "Option::is_none")]
     vcpus: Option<i64>,
@@ -94,9 +88,7 @@ async fn list_worker_pings(
 
     let rows = sqlx::query_as!(
         WorkerPing,
-        "SELECT worker, worker_instance,  EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed,
-        CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as last_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as last_job_workspace_id, 
-        custom_tags, worker_group, wm_version, occupancy_rate, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, memory, vcpus, memory_usage, wm_memory_usage
+        "SELECT worker, worker_instance,  EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed, CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as last_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as last_job_workspace_id, custom_tags, worker_group, wm_version, occupancy_rate, memory, vcpus, memory_usage, wm_memory_usage
         FROM worker_ping
         WHERE ($1::integer IS NULL AND ping_at > now() - interval '5 minute') OR (ping_at > now() - ($1 || ' seconds')::interval)
         ORDER BY ping_at desc LIMIT $2 OFFSET $3",

backend/windmill-api/src/workspaces.rs

@@ -42,10 +42,7 @@ use windmill_common::schedule::Schedule;
 use windmill_common::users::username_to_permissioned_as;
 use windmill_common::variables::build_crypt;
 use windmill_common::worker::{to_raw_value, CLOUD_HOSTED};
-#[cfg(feature = "enterprise")]
-use windmill_common::workspaces::WorkspaceDeploymentUISettings;
-#[cfg(feature = "enterprise")]
-use windmill_common::workspaces::WorkspaceGitSyncSettings;
+use windmill_common::workspaces::{WorkspaceDeploymentUISettings, WorkspaceGitSyncSettings};
 use windmill_common::{
     error::{to_anyhow, Error, JsonResult, Result},
     flows::Flow,
@@ -994,7 +991,6 @@ async fn edit_large_file_storage_config(
 
 #[derive(Deserialize)]
 pub struct EditGitSyncConfig {
-    #[cfg(feature = "enterprise")]
     pub git_sync_settings: Option<WorkspaceGitSyncSettings>,
 }
 
@@ -1060,7 +1056,6 @@ async fn edit_git_sync_config(
 
 #[derive(Deserialize)]
 struct EditDeployUIConfig {
-    #[cfg(feature = "enterprise")]
     deploy_ui_settings: Option<WorkspaceDeploymentUISettings>,
 }
 
@@ -1069,6 +1064,7 @@ async fn edit_deploy_ui_config(
     _authed: ApiAuthed,
     Extension(_db): Extension<DB>,
     Path(_w_id): Path<String>,
+    Json(_new_config): Json<EditDeployUIConfig>,
 ) -> Result<String> {
     return Err(Error::BadRequest(
         "Deployment UI is only available on Windmill Enterprise Edition".to_string(),
@@ -1126,7 +1122,6 @@ async fn edit_deploy_ui_config(
 
 #[derive(Deserialize)]
 pub struct EditDefaultApp {
-    #[cfg(feature = "enterprise")]
     pub default_app_path: Option<String>,
 }
 

backend/windmill-common/Cargo.toml

@@ -11,7 +11,6 @@ jemalloc = ["dep:tikv-jemalloc-ctl"]
 prometheus = ["dep:prometheus"]
 flamegraph = ["dep:tracing-flame"]
 loki = ["dep:tracing-loki"]
-benchmark = []
 parquet = ["dep:object_store", "dep:aws-config", "dep:aws-sdk-sts"]
 
 [lib]
@@ -56,7 +55,6 @@ mail-send.workspace = true
 futures-core.workspace = true
 async-stream.workspace = true
 const_format.workspace = true
-crc.workspace = true
 
 [target.'cfg(not(target_env = "msvc"))'.dependencies]
 tikv-jemalloc-ctl = { optional = true, workspace = true }

backend/windmill-common/src/bench.rs

@@ -1,213 +0,0 @@
-use crate::{
-    worker::{write_file, TMP_DIR},
-    DB,
-};
-use serde::Serialize;
-use tokio::time::Instant;
-
-#[derive(Serialize)]
-pub struct BenchmarkInfo {
-    #[serde(skip)]
-    pub start: Instant,
-    #[serde(skip)]
-    pub iters: u64,
-    timings: Vec<BenchmarkIter>,
-    pub iter_durations: Vec<u64>,
-    pub total_duration: Option<u64>,
-}
-
-impl BenchmarkInfo {
-    pub fn new() -> Self {
-        BenchmarkInfo {
-            iters: 0,
-            timings: vec![],
-            start: Instant::now(),
-            iter_durations: vec![],
-            total_duration: None,
-        }
-    }
-
-    pub fn add_iter(&mut self, bench: BenchmarkIter, inc_iters: bool) {
-        if inc_iters {
-            self.iters += 1;
-        }
-        let elapsed_total = bench.start.elapsed().as_nanos() as u64;
-        self.timings.push(bench);
-        self.iter_durations.push(elapsed_total);
-    }
-
-    pub fn write_to_file(&mut self, path: &str) -> anyhow::Result<()> {
-        let total_duration = self.start.elapsed().as_millis() as u64;
-        self.total_duration = Some(total_duration as u64);
-
-        println!(
-            "Writing benchmark {path}, duration of benchmark: {total_duration}s and RPS: {}",
-            self.iters as f64 / total_duration as f64
-        );
-        write_file(TMP_DIR, path, &serde_json::to_string(&self).unwrap()).expect("write profiling");
-        Ok(())
-    }
-}
-
-#[derive(Serialize)]
-pub struct BenchmarkIter {
-    #[serde(skip)]
-    pub start: Instant,
-    #[serde(skip)]
-    last_instant: Instant,
-    last_step: String,
-    timings: Vec<(String, u32)>,
-}
-
-impl BenchmarkIter {
-    pub fn new() -> Self {
-        BenchmarkIter {
-            last_instant: Instant::now(),
-            timings: vec![],
-            start: Instant::now(),
-            last_step: String::new(),
-        }
-    }
-
-    pub fn add_timing(&mut self, name: &str) {
-        let elapsed = self.last_instant.elapsed().as_nanos() as u32;
-        self.timings
-            .push((format!("{}->{}", self.last_step, name), elapsed));
-        self.last_instant = Instant::now();
-        self.last_step = name.to_string();
-    }
-}
-
-pub async fn benchmark_init(benchmark_jobs: i32, db: &DB) {
-    use crate::{jobs::JobKind, scripts::ScriptLang};
-
-    let benchmark_kind = std::env::var("BENCHMARK_KIND").unwrap_or("noop".to_string());
-
-    if benchmark_jobs > 0 {
-        match benchmark_kind.as_str() {
-            "dedicated" => {
-                // you need to create the script first, check https://github.com/windmill-labs/windmill/blob/b76a92cfe454c686f005c65f534e29e039f3c706/benchmarks/lib.ts#L47
-                let hash = sqlx::query_scalar!(
-                    "SELECT hash FROM script WHERE path = $1 AND workspace_id = $2",
-                    "f/benchmarks/dedicated",
-                    "admins"
-                )
-                .fetch_one(db)
-                .await
-                .unwrap_or_else(|_e| panic!("failed to insert dedicated jobs"));
-                sqlx::query!("INSERT INTO queue (id, script_hash, script_path, job_kind, language, tag, created_by, permissioned_as, email, scheduled_for, workspace_id) (SELECT gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8, $9, $10 FROM generate_series(1, $11))",
-                    hash,
-                    "f/benchmarks/dedicated",
-                    JobKind::Script as JobKind,
-                    ScriptLang::Bun as ScriptLang,
-                    "admins:f/benchmarks/dedicated",
-                    "admin",
-                    "u/admin",
-                    "admin@windmill.dev",
-                    chrono::Utc::now(),
-                    "admins",
-                    benchmark_jobs
-                )
-                .execute(db)
-                .await.unwrap_or_else(|_e| panic!("failed to insert dedicated jobs"));
-            }
-            "parallelflow" => {
-                //create dedicated script
-                sqlx::query!("INSERT INTO script (summary, description, dedicated_worker, content, workspace_id, path, hash, language, tag, created_by, lock) VALUES ('', '', true, $1, $2, $3, $4, $5, $6, $7, '') ON CONFLICT (workspace_id, hash) DO NOTHING",
-                "export async function main() {
-                    console.log('hello world');
-                }",
-                "admins",
-                "u/admin/parallelflow",
-                1234567890,
-                ScriptLang::Deno as ScriptLang,
-                "flow",
-                "admin",
-                )
-                .execute(db)
-                .await.unwrap_or_else(|_e| panic!("failed to insert parallelflow jobs {_e:#}"));
-                sqlx::query!("INSERT INTO queue (id, script_hash, script_path, job_kind, language, tag, created_by, permissioned_as, email, scheduled_for, workspace_id, raw_flow, flow_status) (SELECT gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12 FROM generate_series(1, 1))",
-                    None::<i64>,
-                    None::<String>,
-                    JobKind::FlowPreview as JobKind,
-                    ScriptLang::Deno as ScriptLang,
-                    "flow",
-                    "admin",
-                    "u/admin",
-                    "admin@windmill.dev",
-                    chrono::Utc::now(),
-                    "admins",
-                    serde_json::from_str::<serde_json::Value>(r#"
-{
-  "modules": [
-    {
-      "id": "a",
-      "value": {
-        "type": "forloopflow",
-        "modules": [
-          {
-            "id": "b",
-            "value": {
-              "path": "u/admin/parallelflow",
-              "type": "script",
-              "tag_override": "",
-              "input_transforms": {}
-            },
-            "summary": "calctest"
-          }
-        ],
-        "iterator": {
-          "expr": "[...new Array(300)]",
-          "type": "javascript"
-        },
-        "parallel": true,
-        "parallelism": 10,
-        "skip_failures": true
-      }
-    }
-  ],
-  "preprocessor_module": null
-}
-                    "#).unwrap(),
-                    serde_json::from_str::<serde_json::Value>(r#"
-{
-		"step": 0,
-		"modules": [
-			{
-				"id": "a",
-				"type": "WaitingForPriorSteps"
-			}
-		],
-		"cleanup_module": {},
-		"failure_module": {
-			"id": "failure",
-			"type": "WaitingForPriorSteps"
-		},
-		"preprocessor_module": null
-	}
-
-                "#).unwrap()
-                )
-                .execute(db)
-                .await.unwrap_or_else(|_e| panic!("failed to insert parallelflow jobs"));
-            }
-            _ => {
-                sqlx::query!("INSERT INTO queue (id, script_hash, script_path, job_kind, language, tag, created_by, permissioned_as, email, scheduled_for, workspace_id) (SELECT gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8, $9, $10 FROM generate_series(1, $11))",
-                    None::<i64>,
-                    None::<String>,
-                    JobKind::Noop as JobKind,
-                    ScriptLang::Deno as ScriptLang,
-                    "deno",
-                    "admin",
-                    "u/admin",
-                    "admin@windmill.dev",
-                    chrono::Utc::now(),
-                    "admins",
-                    benchmark_jobs
-                )
-                .execute(db)
-                .await.unwrap_or_else(|_e| panic!("failed to insert noop jobs"));
-            }
-        }
-    }
-}

backend/windmill-common/src/ee.rs

@@ -52,19 +52,12 @@ pub async fn schedule_key_renewal(_http_client: &reqwest::Client, _db: &crate::d
     // Implementation is not open source
 }
 
-#[cfg(feature = "enterprise")]
-pub enum RenewReason {
-    Manual,
-    Schedule,
-    OnStart,
-}
-
 #[cfg(feature = "enterprise")]
 pub async fn renew_license_key(
     _http_client: &reqwest::Client,
     _db: &crate::db::DB,
     _key: Option<String>,
-    _reason: RenewReason,
+    _manual: bool,
 ) -> String {
     // Implementation is not open source
     "".to_string()

backend/windmill-common/src/ee.rs~main

@@ -1,73 +0,0 @@
-#[cfg(feature = "enterprise")]
-use crate::db::DB;
-use crate::ee::LicensePlan::Community;
-#[cfg(feature = "enterprise")]
-use crate::error;
-use serde::Deserialize;
-use std::sync::Arc;
-use tokio::sync::RwLock;
-
-lazy_static::lazy_static! {
-  pub static ref LICENSE_KEY_VALID: Arc<RwLock<bool>> = Arc::new(RwLock::new(true));
-  pub static ref LICENSE_KEY_ID: Arc<RwLock<String>> = Arc::new(RwLock::new("".to_string()));
-  pub static ref LICENSE_KEY: Arc<RwLock<String>> = Arc::new(RwLock::new("".to_string()));
-}
-
-pub enum LicensePlan {
-    Community,
-    Pro,
-    Enterprise,
-}
-
-pub async fn get_license_plan() -> LicensePlan {
-    // Implementation is not open source
-    return Community;
-}
-
-#[derive(Deserialize)]
-#[serde(untagged)]
-pub enum CriticalErrorChannel {}
-
-pub enum CriticalAlertKind {
-    #[cfg(feature = "enterprise")]
-    CriticalError,
-    #[cfg(feature = "enterprise")]
-    RecoveredCriticalError,
-}
-
-#[cfg(feature = "enterprise")]
-pub async fn send_critical_alert(
-    _error_message: String,
-    _db: &DB,
-    _kind: CriticalAlertKind,
-    _channels: Option<Vec<CriticalErrorChannel>>,
-) {
-}
-
-#[cfg(feature = "enterprise")]
-pub async fn schedule_key_renewal(_http_client: &reqwest::Client, _db: &crate::db::DB) -> () {
-    // Implementation is not open source
-}
-
-#[cfg(feature = "enterprise")]
-pub async fn renew_license_key(
-    _http_client: &reqwest::Client,
-    _db: &crate::db::DB,
-    _key: Option<String>,
-    _manual: bool,
-) -> String {
-    // Implementation is not open source
-    "".to_string()
-}
-
-#[cfg(feature = "enterprise")]
-pub async fn create_customer_portal_session(
-    _http_client: &reqwest::Client,
-    _key: Option<String>,
-) -> error::Result<String> {
-    // Implementation is not open source
-    Ok("".to_string())
-}
-
-#[cfg(feature = "enterprise")]
-pub async fn worker_groups_alerts(_db: &DB) {}

backend/windmill-common/src/ee.rs~main_0

@@ -1,76 +0,0 @@
-#[cfg(feature = "enterprise")]
-use crate::db::DB;
-use crate::ee::LicensePlan::Community;
-#[cfg(feature = "enterprise")]
-use crate::error;
-use serde::Deserialize;
-use std::sync::Arc;
-use tokio::sync::RwLock;
-
-lazy_static::lazy_static! {
-  pub static ref LICENSE_KEY_VALID: Arc<RwLock<bool>> = Arc::new(RwLock::new(true));
-  pub static ref LICENSE_KEY_ID: Arc<RwLock<String>> = Arc::new(RwLock::new("".to_string()));
-  pub static ref LICENSE_KEY: Arc<RwLock<String>> = Arc::new(RwLock::new("".to_string()));
-}
-
-pub enum LicensePlan {
-    Community,
-    Pro,
-    Enterprise,
-}
-
-pub async fn get_license_plan() -> LicensePlan {
-    // Implementation is not open source
-    return Community;
-}
-
-#[derive(Deserialize)]
-#[serde(untagged)]
-pub enum CriticalErrorChannel {
-    Email { email: String },
-    Slack { slack_channel: String },
-}
-
-pub enum CriticalAlertKind {
-    #[cfg(feature = "enterprise")]
-    CriticalError,
-    #[cfg(feature = "enterprise")]
-    RecoveredCriticalError,
-}
-
-#[cfg(feature = "enterprise")]
-pub async fn send_critical_alert(
-    _error_message: String,
-    _db: &DB,
-    _kind: CriticalAlertKind,
-    _channels: Option<Vec<CriticalErrorChannel>>,
-) {
-}
-
-#[cfg(feature = "enterprise")]
-pub async fn schedule_key_renewal(_http_client: &reqwest::Client, _db: &crate::db::DB) -> () {
-    // Implementation is not open source
-}
-
-#[cfg(feature = "enterprise")]
-pub async fn renew_license_key(
-    _http_client: &reqwest::Client,
-    _db: &crate::db::DB,
-    _key: Option<String>,
-    _manual: bool,
-) -> String {
-    // Implementation is not open source
-    "".to_string()
-}
-
-#[cfg(feature = "enterprise")]
-pub async fn create_customer_portal_session(
-    _http_client: &reqwest::Client,
-    _key: Option<String>,
-) -> error::Result<String> {
-    // Implementation is not open source
-    Ok("".to_string())
-}
-
-#[cfg(feature = "enterprise")]
-pub async fn worker_groups_alerts(_db: &DB) {}

backend/windmill-common/src/flow_status.rs

@@ -128,7 +128,6 @@ struct UntaggedFlowStatusModule {
     while_loop: Option<bool>,
     approvers: Option<Vec<Approval>>,
     failed_retries: Option<Vec<Uuid>>,
-    skipped: Option<bool>,
 }
 
 #[derive(Serialize, Debug, Clone)]
@@ -180,7 +179,6 @@ pub enum FlowStatusModule {
         approvers: Vec<Approval>,
         #[serde(skip_serializing_if = "Vec::is_empty")]
         failed_retries: Vec<Uuid>,
-        skipped: bool,
     },
     Failure {
         id: String,
@@ -257,7 +255,6 @@ impl<'de> Deserialize<'de> for FlowStatusModule {
                 branch_chosen: untagged.branch_chosen,
                 approvers: untagged.approvers.unwrap_or_default(),
                 failed_retries: untagged.failed_retries.unwrap_or_default(),
-                skipped: untagged.skipped.unwrap_or(false),
             }),
             "Failure" => Ok(FlowStatusModule::Failure {
                 id: untagged

backend/windmill-common/src/flows.rs

@@ -269,13 +269,6 @@ pub struct FlowModule {
     pub delete_after_use: Option<bool>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub continue_on_error: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub skip_if: Option<SkipIf>,
-}
-
-#[derive(Deserialize, Serialize, Debug, Clone)]
-pub struct SkipIf {
-    pub expr: String,
 }
 
 #[derive(Deserialize)]
@@ -292,23 +285,6 @@ pub struct FlowModuleValueWithParallel {
     pub parallelism: Option<u16>,
 }
 
-#[derive(Deserialize)]
-pub struct FlowModuleValueWithSkipFailures {
-    pub skip_failures: Option<bool>,
-    pub parallel: Option<bool>,
-    pub parallelism: Option<u16>,
-}
-
-#[derive(Deserialize)]
-pub struct BranchWithSkipFailures {
-    pub skip_failure: Option<bool>,
-}
-
-#[derive(Deserialize)]
-pub struct FlowModuleWithBranches {
-    pub branches: Vec<BranchWithSkipFailures>,
-}
-
 impl FlowModule {
     pub fn id_append(&mut self, s: &str) {
         self.id = format!("{}-{}", self.id, s);
@@ -317,16 +293,6 @@ impl FlowModule {
         serde_json::from_str::<FlowModuleValue>(self.value.get()).map_err(crate::error::to_anyhow)
     }
 
-    pub fn get_value_with_skip_failures(&self) -> anyhow::Result<FlowModuleValueWithSkipFailures> {
-        serde_json::from_str::<FlowModuleValueWithSkipFailures>(self.value.get())
-            .map_err(crate::error::to_anyhow)
-    }
-
-    pub fn get_branches_skip_failures(&self) -> anyhow::Result<FlowModuleWithBranches> {
-        serde_json::from_str::<FlowModuleWithBranches>(self.value.get())
-            .map_err(crate::error::to_anyhow)
-    }
-
     pub fn is_flow(&self) -> bool {
         self.get_type().is_ok_and(|x| x == "flow")
     }
@@ -638,7 +604,6 @@ pub fn add_virtual_items_if_necessary(modules: &mut Vec<FlowModule>) {
             priority: None,
             delete_after_use: None,
             continue_on_error: None,
-            skip_if: None,
         });
     }
 }

backend/windmill-common/src/lib.rs

@@ -17,8 +17,6 @@ use scripts::ScriptLang;
 use sqlx::{Pool, Postgres};
 
 pub mod apps;
-#[cfg(feature = "benchmark")]
-pub mod bench;
 pub mod db;
 pub mod ee;
 pub mod error;
@@ -53,17 +51,6 @@ pub const DEFAULT_MAX_CONNECTIONS_INDEXER: u32 = 5;
 
 pub const DEFAULT_HUB_BASE_URL: &str = "https://hub.windmill.dev";
 
-#[macro_export]
-macro_rules! add_time {
-    ($bench:expr, $name:expr) => {
-        #[cfg(feature = "benchmark")]
-        {
-            $bench.add_timing($name);
-            // println!("{}: {:?}", $z, $y.elapsed());
-        }
-    };
-}
-
 lazy_static::lazy_static! {
     pub static ref METRICS_PORT: u16 = std::env::var("METRICS_PORT")
     .ok()
@@ -95,8 +82,6 @@ lazy_static::lazy_static! {
 
     pub static ref JOB_RETENTION_SECS: Arc<RwLock<i64>> = Arc::new(RwLock::new(0));
 
-    pub static ref INSTANCE_NAME: String = rd_string(5);
-
 }
 
 pub async fn shutdown_signal(
@@ -143,7 +128,6 @@ pub async fn shutdown_signal(
 use tokio::sync::RwLock;
 #[cfg(feature = "prometheus")]
 use tokio::task::JoinHandle;
-use utils::rd_string;
 
 #[cfg(feature = "prometheus")]
 pub async fn serve_metrics(

backend/windmill-common/src/stats_ee.rs

@@ -8,7 +8,11 @@ pub async fn get_disable_stats_setting(_db: &DB) -> bool {
     false
 }
 
-pub async fn schedule_stats(_db: &DB, _http_client: &reqwest::Client) -> () {
+pub async fn schedule_stats(
+    _instance_name: String,
+    _db: &DB,
+    _http_client: &reqwest::Client,
+) -> () {
     // stats details are closed source
 }
 
@@ -19,17 +23,10 @@ struct JobsUsage {
     count: i64,
 }
 
-pub enum SendStatsReason {
-    Manual,
-    Schedule,
-    OnStart,
-}
-
 pub async fn send_stats(
+    _instance_name: &String,
     _http_client: &reqwest::Client,
     _db: &DB,
-    _skip_job_usage: bool,
-    _reason: SendStatsReason,
 ) -> Result<()> {
     // stats details are closed source
     Ok(())

backend/windmill-common/src/utils.rs

@@ -38,7 +38,7 @@ lazy_static::lazy_static! {
         .build().unwrap();
 }
 
-#[derive(Deserialize, Clone)]
+#[derive(Deserialize)]
 pub struct Pagination {
     pub page: Option<usize>,
     pub per_page: Option<usize>,
@@ -82,12 +82,6 @@ pub fn paginate(pagination: Pagination) -> (usize, usize) {
     (per_page, offset)
 }
 
-pub fn paginate_without_limits(pagination: Pagination) -> (usize, usize) {
-    let per_page = pagination.per_page.unwrap_or(MAX_PER_PAGE);
-    let offset = (pagination.page.unwrap_or(1).max(1) - 1) * per_page;
-    (per_page, offset)
-}
-
 pub async fn now_from_db<'c, E: sqlx::PgExecutor<'c>>(
     db: E,
 ) -> Result<chrono::DateTime<chrono::Utc>> {
@@ -189,7 +183,7 @@ pub fn calculate_hash(s: &str) -> String {
     format!("{:x}", hasher.finalize())
 }
 
-pub async fn get_uid<'c, E: sqlx::Executor<'c, Database = Postgres>>(db: E) -> Result<String> {
+pub async fn get_uid(db: &DB) -> Result<String> {
     let mut uid = LICENSE_KEY_ID.read().await.clone();
 
     if uid == "" {
@@ -228,13 +222,6 @@ impl std::fmt::Display for Mode {
     }
 }
 
-// inspired from rails: https://github.com/rails/rails/blob/6e49cc77ab3d16c06e12f93158eaf3e507d4120e/activerecord/lib/active_record/migration.rb#L1308
-pub fn generate_lock_id(database_name: &str) -> i64 {
-    const CRC_IEEE: crc::Crc<u32> = crc::Crc::<u32>::new(&crc::CRC_32_ISO_HDLC);
-    // 0x3d32ad9e chosen by fair dice roll
-    0x3d32ad9e * (CRC_IEEE.checksum(database_name.as_bytes()) as i64)
-}
-
 pub async fn send_email(
     subject: &str,
     content: &str,

backend/windmill-common/src/worker.rs

@@ -303,14 +303,14 @@ fn parse_file<T: FromStr>(path: &str) -> Option<T> {
         .flatten()
 }
 
-pub struct TypeScriptAnnotations {
+pub struct Annotations {
     pub npm_mode: bool,
     pub nodejs_mode: bool,
     pub native_mode: bool,
     pub nobundling: bool,
 }
 
-pub fn get_annotation_ts(inner_content: &str) -> TypeScriptAnnotations {
+pub fn get_annotation(inner_content: &str) -> Annotations {
     let annotations = inner_content
         .lines()
         .take_while(|x| x.starts_with("//"))
@@ -324,41 +324,7 @@ pub fn get_annotation_ts(inner_content: &str) -> TypeScriptAnnotations {
     let nobundling: bool =
         annotations.contains(&"nobundling".to_string()) || nodejs_mode || *DISABLE_BUNDLING;
 
-    TypeScriptAnnotations { npm_mode, nodejs_mode, native_mode, nobundling }
-}
-
-pub struct PythonAnnotations {
-    pub no_uv: bool,
-    pub no_cache: bool,
-}
-
-pub fn get_annotation_python(inner_content: &str) -> PythonAnnotations {
-    let annotations = inner_content
-        .lines()
-        .take_while(|x| x.starts_with("#"))
-        .map(|x| x.to_string().replace("#", "").trim().to_string())
-        .collect_vec();
-
-    let no_uv: bool = annotations.contains(&"no_uv".to_string());
-    let no_cache: bool = annotations.contains(&"no_cache".to_string());
-
-    PythonAnnotations { no_uv, no_cache }
-}
-
-pub struct SqlAnnotations {
-    pub return_last_result: bool,
-}
-
-pub fn get_sql_annotations(inner_content: &str) -> SqlAnnotations {
-    let annotations = inner_content
-        .lines()
-        .take_while(|x| x.starts_with("--"))
-        .map(|x| x.to_string().replace("--", "").trim().to_string())
-        .collect_vec();
-
-    let return_last_result: bool = annotations.contains(&"return_last_result".to_string());
-
-    SqlAnnotations { return_last_result }
+    Annotations { npm_mode, nodejs_mode, native_mode, nobundling }
 }
 
 pub async fn load_cache(bin_path: &str, _remote_path: &str) -> (bool, String) {
@@ -467,13 +433,10 @@ pub async fn save_cache(
 fn write_binary_file(main_path: &str, byts: &mut bytes::Bytes) -> error::Result<()> {
     use std::fs::{File, Permissions};
     use std::io::Write;
-
-    #[cfg(unix)]
     use std::os::unix::fs::PermissionsExt;
 
     let mut file = File::create(main_path)?;
     file.write_all(byts)?;
-    #[cfg(unix)]
     file.set_permissions(Permissions::from_mode(0o755))?;
     file.flush()?;
     Ok(())

backend/windmill-indexer/src/indexer_ee.rs

@@ -1,6 +1,6 @@
-use anyhow::anyhow;
 use sqlx::{Pool, Postgres};
 use windmill_common::error::Error;
+use anyhow::anyhow;
 
 #[derive(Clone)]
 pub struct IndexReader;

backend/windmill-queue/Cargo.toml

@@ -12,7 +12,7 @@ path = "src/lib.rs"
 default = []
 enterprise = ["windmill-common/enterprise"]
 cloud = []
-benchmark = ["windmill-common/benchmark"]
+benchmark = []
 prometheus = ["dep:prometheus"]
 
 [dependencies]

backend/windmill-queue/src/jobs.rs

@@ -37,9 +37,9 @@ use ulid::Ulid;
 use uuid::Uuid;
 use windmill_audit::audit_ee::{audit_log, AuditAuthor};
 use windmill_audit::ActionKind;
-
+#[cfg(not(feature = "enterprise"))]
+use windmill_common::worker::PriorityTags;
 use windmill_common::{
-    add_time,
     auth::{fetch_authed_from_permissioned_as, permissioned_as_to_username},
     db::{Authed, UserDB},
     error::{self, to_anyhow, Error},
@@ -62,12 +62,9 @@ use windmill_common::{
         to_raw_value, DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES, NO_LOGS, WORKER_CONFIG,
         WORKER_PULL_QUERIES, WORKER_SUSPENDED_PULL_QUERY,
     },
-    DB, METRICS_ENABLED,
+    BASE_URL, DB, METRICS_ENABLED,
 };
 
-#[cfg(feature = "enterprise")]
-use windmill_common::BASE_URL;
-
 #[cfg(feature = "cloud")]
 use windmill_common::users::SUPERADMIN_SYNC_EMAIL;
 
@@ -128,12 +125,10 @@ const MAX_FREE_CONCURRENT_RUNS: i32 = 30;
 
 const ERROR_HANDLER_USERNAME: &str = "error_handler";
 const SCHEDULE_ERROR_HANDLER_USERNAME: &str = "schedule_error_handler";
-#[cfg(feature = "enterprise")]
 const SCHEDULE_RECOVERY_HANDLER_USERNAME: &str = "schedule_recovery_handler";
 const ERROR_HANDLER_USER_GROUP: &str = "g/error_handler";
 const ERROR_HANDLER_USER_EMAIL: &str = "error_handler@windmill.dev";
 const SCHEDULE_ERROR_HANDLER_USER_EMAIL: &str = "schedule_error_handler@windmill.dev";
-#[cfg(feature = "enterprise")]
 const SCHEDULE_RECOVERY_HANDLER_USER_EMAIL: &str = "schedule_recovery_handler@windmill.dev";
 
 #[derive(Clone, Debug)]
@@ -179,8 +174,6 @@ pub async fn cancel_single_job<'c>(
                 rsmq.clone(),
                 "server",
                 false,
-                #[cfg(feature = "benchmark")]
-                &mut windmill_common::bench::BenchmarkIter::new(),
             )
             .await;
 
@@ -482,10 +475,8 @@ where
     }
 }
 
-#[cfg(feature = "enterprise")]
 #[derive(Deserialize)]
 struct RawFlowFailureModule {
-    #[cfg(feature = "enterprise")]
     failure_module: Option<Box<RawValue>>,
 }
 
@@ -499,7 +490,6 @@ pub async fn add_completed_job_error<R: rsmq_async::RsmqConnection + Clone + Sen
     rsmq: Option<R>,
     _worker_name: &str,
     flow_is_done: bool,
-    #[cfg(feature = "benchmark")] bench: &mut windmill_common::bench::BenchmarkIter,
 ) -> Result<WrappedError, Error> {
     #[cfg(feature = "prometheus")]
     register_metric(
@@ -537,8 +527,6 @@ pub async fn add_completed_job_error<R: rsmq_async::RsmqConnection + Clone + Sen
         canceled_by,
         rsmq,
         flow_is_done,
-        #[cfg(feature = "benchmark")]
-        bench,
     )
     .await?;
     Ok(result)
@@ -562,12 +550,10 @@ pub async fn add_completed_job<
     canceled_by: Option<CanceledBy>,
     rsmq: Option<R>,
     flow_is_done: bool,
-    #[cfg(feature = "benchmark")] bench: &mut windmill_common::bench::BenchmarkIter,
 ) -> Result<Uuid, Error> {
     // tracing::error!("Start");
     // let start = tokio::time::Instant::now();
 
-    add_time!(bench, "add_completed_job start");
     if !result.is_valid_json() {
         return Err(Error::InternalErr(
             "Result of job is invalid json (empty)".to_string(),
@@ -575,7 +561,6 @@ pub async fn add_completed_job<
     }
 
     let mut tx: QueueTransaction<'_, R> = (rsmq.clone(), db.begin().await?).into();
-
     let job_id = queued_job.id;
     // tracing::error!("1 {:?}", start.elapsed());
 
@@ -586,7 +571,6 @@ pub async fn add_completed_job<
     );
 
     let mem_peak = mem_peak.max(queued_job.mem_peak.unwrap_or(0));
-    add_time!(bench, "add_completed_job query START");
     let _duration: i64 = sqlx::query_scalar!(
         "INSERT INTO completed_job AS cj
                    ( workspace_id
@@ -658,8 +642,6 @@ pub async fn add_completed_job<
     .map_err(|e| Error::InternalErr(format!("Could not add completed job {job_id}: {e:#}")))?;
     // tracing::error!("2 {:?}", start.elapsed());
 
-    add_time!(bench, "add_completed_job query END");
-
     if !queued_job.is_flow_step {
         if _duration > 500
             && (queued_job.job_kind == JobKind::Script || queued_job.job_kind == JobKind::Preview)
@@ -689,7 +671,6 @@ pub async fn add_completed_job<
         }
     }
     // tracing::error!("Added completed job {:#?}", queued_job);
-    #[cfg(feature = "enterprise")]
     let mut skip_downstream_error_handlers = false;
     tx = delete_job(tx, &queued_job.workspace_id, job_id).await?;
     // tracing::error!("3 {:?}", start.elapsed());
@@ -735,10 +716,7 @@ pub async fn add_completed_job<
             .await?;
 
             if let Some(schedule) = schedule {
-                #[cfg(feature = "enterprise")]
-                {
-                    skip_downstream_error_handlers = schedule.ws_error_handler_muted;
-                }
+                skip_downstream_error_handlers = schedule.ws_error_handler_muted;
 
                 // script or flow that failed on start and might not have been rescheduled
                 let schedule_next_tick = !queued_job.is_flow()
@@ -771,7 +749,6 @@ pub async fn add_completed_job<
                     };
                 }
 
-                #[cfg(feature = "enterprise")]
                 if let Err(err) = apply_schedule_handlers(
                     rsmq.clone(),
                     db,
@@ -1347,8 +1324,6 @@ struct CompletedJobSubset {
     result: Option<sqlx::types::Json<Box<RawValue>>>,
     started_at: chrono::DateTime<chrono::Utc>,
 }
-
-#[cfg(feature = "enterprise")]
 async fn apply_schedule_handlers<
     'a,
     'c,
@@ -1367,6 +1342,7 @@ async fn apply_schedule_handlers<
     job_priority: Option<i16>,
 ) -> windmill_common::error::Result<()> {
     if !success {
+        #[cfg(feature = "enterprise")]
         if let Some(on_failure_path) = schedule.on_failure.clone() {
             let times = schedule.on_failure_times.unwrap_or(1).max(1);
             let exact = schedule.on_failure_exact.unwrap_or(false);
@@ -1416,6 +1392,7 @@ async fn apply_schedule_handlers<
             .await?;
         }
     } else {
+        #[cfg(feature = "enterprise")]
         if let Some(ref on_success_path) = schedule.on_success {
             handle_successful_schedule(
                 db,
@@ -1433,6 +1410,7 @@ async fn apply_schedule_handlers<
             .await?;
         }
 
+        #[cfg(feature = "enterprise")]
         if let Some(ref on_recovery_path) = schedule.on_recovery.clone() {
             let tx: QueueTransaction<'_, R> = (rsmq.clone(), db.begin().await?).into();
             let times = schedule.on_recovery_times.unwrap_or(1).max(1);
@@ -1601,7 +1579,6 @@ fn sanitize_result<T: Serialize + Send + Sync>(result: Json<&T>) -> HashMap<Stri
 //     is_flow: boolean,
 //     extra_args: serde_json::Value
 // }
-#[cfg(feature = "enterprise")]
 async fn handle_recovered_schedule<
     'a,
     'c,
@@ -1694,7 +1671,6 @@ async fn handle_recovered_schedule<
     Ok(())
 }
 
-#[cfg(feature = "enterprise")]
 async fn handle_successful_schedule<
     'a,
     'c,
@@ -1779,9 +1755,9 @@ pub async fn pull<R: rsmq_async::RsmqConnection + Send + Clone>(
     db: &Pool<Postgres>,
     rsmq: Option<R>,
     suspend_first: bool,
-) -> windmill_common::error::Result<(Option<QueuedJob>, bool)> {
+) -> windmill_common::error::Result<Option<QueuedJob>> {
     loop {
-        let (job, suspended) = pull_single_job_and_mark_as_running_no_concurrency_limit(
+        let job = pull_single_job_and_mark_as_running_no_concurrency_limit(
             db,
             rsmq.clone(),
             suspend_first,
@@ -1789,7 +1765,7 @@ pub async fn pull<R: rsmq_async::RsmqConnection + Send + Clone>(
         .await?;
 
         if job.is_none() {
-            return Ok((None, suspended));
+            return Ok(None);
         }
 
         let has_concurent_limit = job.as_ref().unwrap().concurrent_limit.is_some();
@@ -1809,7 +1785,7 @@ pub async fn pull<R: rsmq_async::RsmqConnection + Send + Clone>(
             if METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed) {
                 QUEUE_PULL_COUNT.inc();
             }
-            return Ok((Option::Some(pulled_job), suspended));
+            return Ok(Option::Some(pulled_job));
         }
 
         let itx = db.begin().await?;
@@ -1910,7 +1886,7 @@ pub async fn pull<R: rsmq_async::RsmqConnection + Send + Clone>(
                 QUEUE_PULL_COUNT.inc();
             }
             tx.commit().await?;
-            return Ok((Option::Some(pulled_job), suspended));
+            return Ok(Option::Some(pulled_job));
         }
         let x = sqlx::query_scalar!(
             "UPDATE concurrency_counter SET job_uuids = job_uuids - $2 WHERE concurrency_id = $1 RETURNING (SELECT COUNT(*) FROM jsonb_object_keys(job_uuids))",
@@ -2037,8 +2013,8 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<
     db: &Pool<Postgres>,
     rsmq: Option<R>,
     suspend_first: bool,
-) -> windmill_common::error::Result<(Option<QueuedJob>, bool)> {
-    let job_and_suspended: (Option<QueuedJob>, bool) = if let Some(mut rsmq) = rsmq {
+) -> windmill_common::error::Result<Option<QueuedJob>> {
+    let job: Option<QueuedJob> = if let Some(mut rsmq) = rsmq {
         #[cfg(feature = "benchmark")]
         let instant = Instant::now();
 
@@ -2096,9 +2072,9 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<
             #[cfg(feature = "benchmark")]
             println!("rsmq 2: {:?}", instant.elapsed());
 
-            (m2r, false)
+            m2r
         } else {
-            (None, false)
+            None
         }
     } else {
         /* Jobs can be started if they:
@@ -2112,7 +2088,7 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<
 
         if query.is_empty() {
             tracing::warn!("No suspended pull queries available");
-            return Ok((None, false));
+            return Ok(None);
         }
 
         let r = if suspend_first {
@@ -2132,7 +2108,7 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<
 
             if queries.is_empty() {
                 tracing::warn!("No pull queries available");
-                return Ok((None, false));
+                return Ok(None);
             }
 
             for query in queries.iter() {
@@ -2150,12 +2126,12 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<
 
             // #[cfg(feature = "benchmark")]
             // println!("pull query: {:?}", instant.elapsed());
-            (highest_priority_job, false)
+            highest_priority_job
         } else {
-            (r, true)
+            r
         }
     };
-    Ok(job_and_suspended)
+    Ok(job)
 }
 
 pub async fn custom_concurrency_key(
@@ -3536,7 +3512,6 @@ pub async fn push<'c, 'd, R: rsmq_async::RsmqConnection + Send + 'c>(
                     priority: None,
                     delete_after_use: None,
                     continue_on_error: None,
-                    skip_if: None,
                 }],
                 same_worker: false,
                 failure_module: None,

backend/windmill-worker/Cargo.toml

@@ -12,13 +12,12 @@ path = "src/lib.rs"
 default = []
 prometheus = ["dep:prometheus", "windmill-common/prometheus"]
 enterprise = ["windmill-queue/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "dep:gcp_auth", "dep:pem", "dep:tiberius", "dep:tokio-util", "dep:openidconnect"]
-benchmark = ["windmill-queue/benchmark", "windmill-common/benchmark"]
+benchmark = ["windmill-queue/benchmark"]
 flamegraph = []
 parquet = ["windmill-common/parquet", "dep:object_store"]
 flow_testing = []
 cloud = []
 sqlx = []
-deno_core = ["dep:deno_fetch", "dep:deno_webidl", "dep:deno_web", "dep:deno_net", "dep:deno_console", "dep:deno_url", "dep:deno_core", "dep:deno_ast", "dep:deno_tls"]
 
 [dependencies]
 windmill-queue.workspace = true
@@ -60,15 +59,15 @@ once_cell.workspace = true
 rsmq_async.workspace = true
 tokio-postgres.workspace = true
 bit-vec.workspace = true
-deno_fetch = { workspace = true, optional = true }
-deno_webidl = { workspace = true, optional = true }
-deno_web = { workspace = true, optional = true }
-deno_net = { workspace = true, optional = true }
-deno_console = { workspace = true, optional = true }
-deno_url = { workspace = true, optional = true }
-deno_core = { workspace = true, optional = true }
-deno_ast = { workspace = true, optional = true }
-deno_tls = { workspace = true, optional = true }
+deno_fetch.workspace = true
+deno_webidl.workspace = true
+deno_web.workspace = true
+deno_net.workspace = true
+deno_console.workspace = true
+deno_url.workspace = true
+deno_core.workspace = true
+deno_ast.workspace = true
+deno_tls.workspace = true
 postgres-native-tls.workspace = true
 native-tls.workspace = true
 mysql_async.workspace = true
@@ -90,17 +89,13 @@ tar.workspace = true
 object_store = { workspace = true, optional = true}
 convert_case.workspace = true
 yaml-rust.workspace = true
-swc_ecma_parser.workspace = true
-
 
 [build-dependencies]
-deno_fetch = { workspace = true, optional = true }
-deno_webidl = { workspace = true, optional = true }
-deno_web = { workspace = true, optional = true }
-deno_net = { workspace = true, optional = true }
-deno_console = { workspace = true, optional = true }
-deno_url = { workspace = true, optional = true }
-deno_core = { workspace = true, optional = true }
-deno_ast = { workspace = true, optional = true }
-deno_tls = { workspace = true, optional = true }
+deno_fetch.workspace = true
+deno_webidl.workspace = true
+deno_web.workspace = true
+deno_console.workspace = true
+deno_url.workspace = true
+deno_core.workspace = true
+deno_net.workspace = true
 zstd.workspace = true

backend/windmill-worker/build.rs

@@ -1,22 +1,13 @@
-#[cfg(feature = "deno_core")]
 use deno_fetch::FetchPermissions;
-#[cfg(feature = "deno_core")]
 use deno_net::NetPermissions;
-#[cfg(feature = "deno_core")]
 use deno_web::{BlobStore, TimersPermission};
-#[cfg(feature = "deno_core")]
 use std::env;
-#[cfg(feature = "deno_core")]
 use std::io::Write;
-#[cfg(feature = "deno_core")]
 use std::path::PathBuf;
-#[cfg(feature = "deno_core")]
 use std::sync::Arc;
 
-#[cfg(feature = "deno_core")]
 pub struct PermissionsContainer;
 
-#[cfg(feature = "deno_core")]
 impl FetchPermissions for PermissionsContainer {
     #[inline(always)]
     fn check_net_url(
@@ -37,7 +28,6 @@ impl FetchPermissions for PermissionsContainer {
     }
 }
 
-#[cfg(feature = "deno_core")]
 impl TimersPermission for PermissionsContainer {
     #[inline(always)]
     fn allow_hrtime(&mut self) -> bool {
@@ -45,7 +35,6 @@ impl TimersPermission for PermissionsContainer {
     }
 }
 
-#[cfg(feature = "deno_core")]
 impl NetPermissions for PermissionsContainer {
     fn check_read(
         &mut self,
@@ -72,14 +61,12 @@ impl NetPermissions for PermissionsContainer {
     }
 }
 
-#[cfg(feature = "deno_core")]
 deno_core::extension!(
     fetch,
     esm_entry_point = "ext:fetch/src/runtime.js",
     esm = ["src/runtime.js"],
 );
 
-#[cfg(feature = "deno_core")]
 fn main() {
     println!("cargo:rustc-env=TARGET={}", env::var("TARGET").unwrap());
     println!("cargo:rustc-env=PROFILE={}", env::var("PROFILE").unwrap());
@@ -135,6 +122,3 @@ fn main() {
         println!("cargo:rerun-if-changed={}", path.display());
     }
 }
-
-#[cfg(not(feature = "deno_core"))]
-fn main() {}

backend/windmill-worker/nsjail/download.py.config.proto

@@ -80,12 +80,6 @@ mount {
     rw: true
 }
 
-mount {
-    src: "/dev/urandom"
-    dst: "/dev/urandom"
-    is_bind: true
-}
-
 exec_bin {
  	path: "/bin/sh"
  	arg: "/download_deps.sh"

backend/windmill-worker/nsjail/download_deps.py.sh

@@ -19,6 +19,6 @@ then
       echo "\$TRUSTED_HOST is set to $TRUSTED_HOST"
 fi
 
-CMD="/usr/local/bin/python3 -m pip install -v \"$REQ\" -I -t \"$TARGET\" --no-cache --no-color --no-deps --isolated --no-warn-conflicts --disable-pip-version-check $INDEX_URL_ARG $EXTRA_INDEX_URL_ARG $TRUSTED_HOST_ARG"
+CMD="/usr/local/bin/python3 -m pip install -v $REQ -I -t $TARGET --no-cache --no-color --no-deps --isolated --no-warn-conflicts --disable-pip-version-check $INDEX_URL_ARG $EXTRA_INDEX_URL_ARG $TRUSTED_HOST_ARG"
 echo $CMD
 eval $CMD

backend/windmill-worker/nsjail/run.bun.config.proto

@@ -120,7 +120,6 @@ mount {
     src: "{JOB_DIR}/args.json"
     dst: "/tmp/{LANG}/args.json"
     is_bind: true
-    rw: true
 }
 
 mount {

backend/windmill-worker/nsjail/run.python3.config.proto

@@ -81,7 +81,6 @@ mount {
     src: "{JOB_DIR}/args.json"
     dst: "/tmp/args.json"
     is_bind: true
-    rw: true
 }
 
 mount {

backend/windmill-worker/src/ansible_executor.rs

@@ -1,4 +1,3 @@
-#[cfg(unix)]
 use std::{
     collections::HashMap,
     os::unix::fs::PermissionsExt,
@@ -6,13 +5,6 @@ use std::{
     process::Stdio,
 };
 
-#[cfg(windows)]
-use std::{
-    collections::HashMap,
-    path::{Path, PathBuf},
-    process::Stdio,
-};
-
 use anyhow::anyhow;
 use itertools::Itertools;
 use serde_json::value::RawValue;
@@ -29,11 +21,10 @@ use windmill_queue::{append_logs, CanceledBy};
 use crate::{
     bash_executor::BIN_BASH,
     common::{
-        get_reserved_variables, read_and_check_result, start_child_process, transform_json,
-        OccupancyMetrics,
+        get_reserved_variables, handle_child, read_and_check_result, start_child_process,
+        transform_json,
     },
-    handle_child::handle_child,
-    python_executor::{create_dependencies_dir, handle_python_reqs, uv_pip_compile},
+    python_executor::{create_dependencies_dir, handle_python_reqs, pip_compile},
     AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV,
     TZ_ENV,
 };
@@ -59,7 +50,6 @@ async fn handle_ansible_python_deps(
     worker_dir: &str,
     mem_peak: &mut i32,
     canceled_by: &mut Option<CanceledBy>,
-    occupancy_metrics: &mut OccupancyMetrics,
 ) -> error::Result<Vec<String>> {
     create_dependencies_dir(job_dir).await;
 
@@ -80,7 +70,7 @@ async fn handle_ansible_python_deps(
             if requirements.is_empty() {
                 "".to_string()
             } else {
-                uv_pip_compile(
+                pip_compile(
                     job_id,
                     &requirements,
                     mem_peak,
@@ -89,9 +79,6 @@ async fn handle_ansible_python_deps(
                     db,
                     worker_name,
                     w_id,
-                    &mut Some(occupancy_metrics),
-                    false,
-                    false,
                 )
                 .await
                 .map_err(|e| {
@@ -115,7 +102,6 @@ async fn handle_ansible_python_deps(
             worker_name,
             job_dir,
             worker_dir,
-            &mut Some(occupancy_metrics),
         )
         .await?;
         additional_python_paths.append(&mut venv_path);
@@ -132,7 +118,6 @@ async fn install_galaxy_collections(
     mem_peak: &mut i32,
     canceled_by: &mut Option<CanceledBy>,
     db: &sqlx::Pool<sqlx::Postgres>,
-    occupancy_metrics: &mut OccupancyMetrics,
 ) -> anyhow::Result<()> {
     write_file(job_dir, "requirements.yml", collections_yml)?;
 
@@ -175,7 +160,6 @@ async fn install_galaxy_collections(
         "ansible galaxy install",
         None,
         false,
-        &mut Some(occupancy_metrics),
     )
     .await?;
 
@@ -214,7 +198,6 @@ pub async fn handle_ansible_job(
     shared_mount: &str,
     base_internal_url: &str,
     envs: HashMap<String, String>,
-    occupancy_metrics: &mut OccupancyMetrics,
 ) -> windmill_common::error::Result<Box<RawValue>> {
     check_ansible_exists()?;
 
@@ -233,7 +216,6 @@ pub async fn handle_ansible_job(
         worker_dir,
         mem_peak,
         canceled_by,
-        occupancy_metrics,
     )
     .await?;
 
@@ -307,7 +289,6 @@ pub async fn handle_ansible_job(
                 mem_peak,
                 canceled_by,
                 db,
-                occupancy_metrics,
             )
             .await?;
         }
@@ -388,7 +369,6 @@ fi
 
         let file = write_file(job_dir, "wrapper.sh", &wrapper)?;
 
-        #[cfg(unix)]
         file.metadata()?.permissions().set_mode(0o777);
         // let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str());
         let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str());
@@ -444,7 +424,6 @@ fi
         "python run",
         job.timeout,
         false,
-        &mut Some(occupancy_metrics),
     )
     .await?;
     read_and_check_result(job_dir).await

backend/windmill-worker/src/bash_executor.rs

@@ -24,17 +24,13 @@ lazy_static::lazy_static! {
 
 use crate::{
     common::{
-        build_args_map, get_reserved_variables, read_file, read_file_content, start_child_process,
-        OccupancyMetrics,
+        build_args_map, get_reserved_variables, handle_child, read_file, read_file_content,
+        start_child_process,
     },
-    handle_child::handle_child,
     AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV,
     POWERSHELL_CACHE_DIR, POWERSHELL_PATH, TZ_ENV,
 };
 
-#[cfg(windows)]
-use crate::SYSTEM_ROOT;
-
 lazy_static::lazy_static! {
 
     pub static ref ANSI_ESCAPE_RE: Regex = Regex::new(r"\x1b\[[0-9;]*m").unwrap();
@@ -53,7 +49,6 @@ pub async fn handle_bash_job(
     base_internal_url: &str,
     worker_name: &str,
     envs: HashMap<String, String>,
-    occupancy_metrics: &mut OccupancyMetrics,
 ) -> Result<Box<RawValue>, Error> {
     let logs1 = "\n\n--- BASH CODE EXECUTION ---\n".to_string();
     append_logs(&job.id, &job.workspace_id, logs1, db).await;
@@ -147,7 +142,6 @@ pub async fn handle_bash_job(
         "bash run",
         job.timeout,
         true,
-        &mut Some(occupancy_metrics),
     )
     .await?;
 
@@ -200,7 +194,6 @@ pub async fn handle_powershell_job(
     base_internal_url: &str,
     worker_name: &str,
     envs: HashMap<String, String>,
-    occupancy_metrics: &mut OccupancyMetrics,
 ) -> Result<Box<RawValue>, Error> {
     let pwsh_args = {
         let args = build_args_map(job, client, db).await?.map(Json);
@@ -229,19 +222,13 @@ pub async fn handle_powershell_job(
             .collect::<Vec<_>>()
     };
 
-    #[cfg(windows)]
-    let split_char = '\\';
-
-    #[cfg(unix)]
-    let split_char = '/';
-
     let installed_modules = fs::read_dir(POWERSHELL_CACHE_DIR)?
         .filter_map(|x| {
             x.ok().map(|x| {
                 x.path()
                     .display()
                     .to_string()
-                    .split(split_char)
+                    .split('/')
                     .last()
                     .unwrap_or_default()
                     .to_lowercase()
@@ -288,7 +275,6 @@ pub async fn handle_powershell_job(
             "powershell install",
             job.timeout,
             false,
-            &mut Some(occupancy_metrics),
         )
         .await?;
     }
@@ -298,26 +284,14 @@ pub async fn handle_powershell_job(
     append_logs(&job.id, &job.workspace_id, logs2, db).await;
 
     // make sure default (only allhostsallusers) modules are loaded, disable autoload (cache can be large to explore especially on cloud) and add /tmp/windmill/cache to PSModulePath
-    #[cfg(unix)]
     let profile = format!(
         "$PSModuleAutoloadingPreference = 'None'
 $PSModulePathBackup = $env:PSModulePath
-$env:PSModulePath = \"$PSHome/Modules\"
+$env:PSModulePath = ($Env:PSModulePath -split ':')[-1]
 Get-Module -ListAvailable | Import-Module
 $env:PSModulePath = \"{}:$PSModulePathBackup\"",
         POWERSHELL_CACHE_DIR
     );
-
-    #[cfg(windows)]
-    let profile = format!(
-        "$PSModuleAutoloadingPreference = 'None'
-$PSModulePathBackup = $env:PSModulePath
-$env:PSModulePath = \"C:\\Program Files\\PowerShell\\7\\Modules\"
-Get-Module -ListAvailable | Import-Module
-$env:PSModulePath = \"{};$PSModulePathBackup\"",
-        POWERSHELL_CACHE_DIR
-    );
-
     // make sure param() is first
     let param_match = windmill_parser_bash::RE_POWERSHELL_PARAM.find(&content);
     let content: String = if let Some(param_match) = param_match {
@@ -333,29 +307,11 @@ $env:PSModulePath = \"{};$PSModulePathBackup\"",
     };
 
     write_file(job_dir, "main.ps1", content.as_str())?;
-
-    #[cfg(unix)]
     write_file(
         job_dir,
         "wrapper.sh",
         &format!("set -o pipefail\nset -e\nmkfifo bp\ncat bp | tail -1 > ./result2.out &\n{} -F ./main.ps1 \"$@\" 2>&1 | tee bp\nwait $!", POWERSHELL_PATH.as_str()),
     )?;
-
-    #[cfg(windows)]
-    write_file(
-        job_dir,
-        "wrapper.ps1",
-        &format!(
-            "param([string[]]$args)\n\
-    $ErrorActionPreference = 'Stop'\n\
-    $pipe = New-TemporaryFile\n\
-    & \"{}\" -File ./main.ps1 @args 2>&1 | Tee-Object -FilePath $pipe\n\
-    Get-Content -Path $pipe | Select-Object -Last 1 | Set-Content -Path './result2.out'\n\
-    Remove-Item $pipe\n",
-            POWERSHELL_PATH.as_str()
-        ),
-    )?;
-
     let token = client.get_token().await;
     let mut reserved_variables = get_reserved_variables(job, &token, db).await?;
     reserved_variables.insert("RUST_LOG".to_string(), "info".to_string());
@@ -394,24 +350,10 @@ $env:PSModulePath = \"{};$PSModulePathBackup\"",
             .stderr(Stdio::piped())
             .spawn()?
     } else {
-        let mut cmd;
-        let mut cmd_args;
-
-        #[cfg(unix)]
-        {
-            cmd_args = vec!["wrapper.sh"];
-            cmd_args.extend(pwsh_args.iter().map(|x| x.as_str()));
-            cmd = Command::new(BIN_BASH.as_str());
-        }
-
-        #[cfg(windows)]
-        {
-            cmd_args = vec![r".\wrapper.ps1".to_string()];
-            cmd_args.extend(pwsh_args.iter().map(|x| x.replace("--", "-")));
-            cmd = Command::new(POWERSHELL_PATH.as_str());
-        }
-
-        cmd.current_dir(job_dir)
+        let mut cmd_args = vec!["wrapper.sh"];
+        cmd_args.extend(pwsh_args.iter().map(|x| x.as_str()));
+        Command::new(BIN_BASH.as_str())
+            .current_dir(job_dir)
             .env_clear()
             .envs(envs)
             .envs(reserved_variables)
@@ -419,53 +361,11 @@ $env:PSModulePath = \"{};$PSModulePathBackup\"",
             .env("PATH", PATH_ENV.as_str())
             .env("BASE_INTERNAL_URL", base_internal_url)
             .env("HOME", HOME_ENV.as_str())
-            .args(&cmd_args)
+            .args(cmd_args)
             .stdout(Stdio::piped())
-            .stderr(Stdio::piped());
-
-        #[cfg(windows)]
-        {
-            cmd.env("SystemRoot", SYSTEM_ROOT.as_str())
-                .env(
-                    "LOCALAPPDATA",
-                    std::env::var("LOCALAPPDATA")
-                        .unwrap_or_else(|_| format!("{}\\AppData\\Local", HOME_ENV.as_str())),
-                )
-                .env(
-                    "ProgramData",
-                    std::env::var("ProgramData")
-                        .unwrap_or_else(|_| String::from("C:\\ProgramData")),
-                )
-                .env(
-                    "ProgramFiles",
-                    std::env::var("ProgramFiles")
-                        .unwrap_or_else(|_| String::from("C:\\Program Files")),
-                )
-                .env(
-                    "ProgramFiles(x86)",
-                    std::env::var("ProgramFiles(x86)")
-                        .unwrap_or_else(|_| String::from("C:\\Program Files (x86)")),
-                )
-                .env(
-                    "ProgramW6432",
-                    std::env::var("ProgramW6432")
-                        .unwrap_or_else(|_| String::from("C:\\Program Files")),
-                )
-                .env(
-                    "TMP",
-                    std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")),
-                )
-                .env(
-                    "PATHEXT",
-                    std::env::var("PATHEXT").unwrap_or_else(|_| {
-                        String::from(".COM;.EXE;.BAT;.CMD;.VBS;.VBE;.JS;.JSE;.WSF;.WSH;.MSC;.CPL")
-                    }),
-                );
-        }
-
-        cmd.spawn()?
+            .stderr(Stdio::piped())
+            .spawn()?
     };
-
     handle_child(
         &job.id,
         db,
@@ -478,7 +378,6 @@ $env:PSModulePath = \"{};$PSModulePathBackup\"",
         "powershell run",
         job.timeout,
         false,
-        &mut Some(occupancy_metrics),
     )
     .await?;
 

backend/windmill-worker/src/bigquery_executor.rs

@@ -5,7 +5,6 @@ use futures::{FutureExt, TryFutureExt};
 use serde_json::{json, value::RawValue, Value};
 use windmill_common::error::to_anyhow;
 use windmill_common::jobs::QueuedJob;
-use windmill_common::worker::get_sql_annotations;
 use windmill_common::{error::Error, worker::to_raw_value};
 use windmill_parser_sql::{
     parse_bigquery_sig, parse_db_resource, parse_sql_blocks, parse_sql_statement_named_params,
@@ -14,8 +13,7 @@ use windmill_queue::{CanceledBy, HTTP_CLIENT};
 
 use serde::Deserialize;
 
-use crate::common::OccupancyMetrics;
-use crate::handle_child::run_future_with_polling_update_job_poller;
+use crate::common::run_future_with_polling_update_job_poller;
 use crate::{
     common::{build_args_values, resolve_job_timeout},
     AuthedClientBackgroundTask,
@@ -71,7 +69,6 @@ fn do_bigquery_inner<'a>(
     token: &'a str,
     timeout_ms: i32,
     column_order: Option<&'a mut Option<Vec<String>>>,
-    skip_collect: bool,
 ) -> windmill_common::error::Result<BoxFuture<'a, windmill_common::error::Result<Box<RawValue>>>> {
     let param_names = parse_sql_statement_named_params(query, '@');
 
@@ -109,80 +106,76 @@ fn do_bigquery_inner<'a>(
 
         match response.error_for_status_ref() {
             Ok(_) => {
-                if skip_collect {
-                    return Ok(to_raw_value(&Value::Array(vec![])));
-                } else {
-                    let result = response.json::<BigqueryResponse>().await.map_err(|e| {
-                        Error::ExecutionErr(format!(
-                            "BigQuery API response could not be parsed: {}",
-                            e.to_string()
-                        ))
-                    })?;
+                let result = response.json::<BigqueryResponse>().await.map_err(|e| {
+                    Error::ExecutionErr(format!(
+                        "BigQuery API response could not be parsed: {}",
+                        e.to_string()
+                    ))
+                })?;
 
-                    if !result.jobComplete {
-                        return Err(Error::ExecutionErr(
-                            "BigQuery API did not answer query in time".to_string(),
-                        ));
-                    }
-
-                    if result.rows.is_none() || result.rows.as_ref().unwrap().len() == 0 {
-                        return Ok(serde_json::from_str("[]").unwrap());
-                    }
-
-                    if result.schema.is_none() {
-                        return Err(Error::ExecutionErr(
-                            "Incomplete response from BigQuery API".to_string(),
-                        ));
-                    }
-
-                    if result
-                        .totalRows
-                        .unwrap_or(json!(""))
-                        .as_str()
-                        .unwrap_or("")
-                        .parse::<i64>()
-                        .unwrap_or(0)
-                        > 10000
-                    {
-                        return Err(Error::ExecutionErr(
-                        "More than 10000 rows were requested, use LIMIT 10000 to limit the number of rows".to_string(),
+                if !result.jobComplete {
+                    return Err(Error::ExecutionErr(
+                        "BigQuery API did not answer query in time".to_string(),
                     ));
-                    }
-
-                    if let Some(column_order) = column_order {
-                        *column_order = Some(
-                            result
-                                .schema
-                                .as_ref()
-                                .unwrap()
-                                .fields
-                                .iter()
-                                .map(|x| x.name.clone())
-                                .collect::<Vec<String>>(),
-                        );
-                    }
-
-                    let rows = result
-                        .rows
-                        .unwrap()
-                        .iter()
-                        .map(|row| {
-                            let mut row_map = serde_json::Map::new();
-                            row.f
-                                .iter()
-                                .zip(result.schema.as_ref().unwrap().fields.iter())
-                                .for_each(|(field, schema)| {
-                                    row_map.insert(
-                                        schema.name.clone(),
-                                        parse_val(&field.v, &schema.r#type, &schema),
-                                    );
-                                });
-                            Value::from(row_map)
-                        })
-                        .collect::<Vec<_>>();
-
-                    Ok(to_raw_value(&rows))
                 }
+
+                if result.rows.is_none() || result.rows.as_ref().unwrap().len() == 0 {
+                    return Ok(serde_json::from_str("[]").unwrap());
+                }
+
+                if result.schema.is_none() {
+                    return Err(Error::ExecutionErr(
+                        "Incomplete response from BigQuery API".to_string(),
+                    ));
+                }
+
+                if result
+                    .totalRows
+                    .unwrap_or(json!(""))
+                    .as_str()
+                    .unwrap_or("")
+                    .parse::<i64>()
+                    .unwrap_or(0)
+                    > 10000
+                {
+                    return Err(Error::ExecutionErr(
+                    "More than 10000 rows were requested, use LIMIT 10000 to limit the number of rows".to_string(),
+                ));
+                }
+
+                if let Some(column_order) = column_order {
+                    *column_order = Some(
+                        result
+                            .schema
+                            .as_ref()
+                            .unwrap()
+                            .fields
+                            .iter()
+                            .map(|x| x.name.clone())
+                            .collect::<Vec<String>>(),
+                    );
+                }
+
+                let rows = result
+                    .rows
+                    .unwrap()
+                    .iter()
+                    .map(|row| {
+                        let mut row_map = serde_json::Map::new();
+                        row.f
+                            .iter()
+                            .zip(result.schema.as_ref().unwrap().fields.iter())
+                            .for_each(|(field, schema)| {
+                                row_map.insert(
+                                    schema.name.clone(),
+                                    parse_val(&field.v, &schema.r#type, &schema),
+                                );
+                            });
+                        Value::from(row_map)
+                    })
+                    .collect::<Vec<_>>();
+
+                return Ok(to_raw_value(&rows));
             }
             Err(e) => match response.json::<BigqueryErrorResponse>().await {
                 Ok(bq_err) => Err(Error::ExecutionErr(format!(
@@ -211,7 +204,6 @@ pub async fn do_bigquery(
     canceled_by: &mut Option<CanceledBy>,
     worker_name: &str,
     column_order: &mut Option<Vec<String>>,
-    occupancy_metrics: &mut OccupancyMetrics,
 ) -> windmill_common::error::Result<Box<RawValue>> {
     let bigquery_args = build_args_values(job, client, db).await?;
 
@@ -238,8 +230,6 @@ pub async fn do_bigquery(
         return Err(Error::BadRequest("Missing database argument".to_string()));
     };
 
-    let annotations = get_sql_annotations(query);
-
     let service_account = CustomServiceAccount::from_json(&database)
         .map_err(|e| Error::ExecutionErr(e.to_string()))?;
 
@@ -316,8 +306,7 @@ pub async fn do_bigquery(
     let result_f = if queries.len() > 1 {
         let futures = queries
             .iter()
-            .enumerate()
-            .map(|(i, x)| {
+            .map(|x| {
                 do_bigquery_inner(
                     x,
                     &statement_values,
@@ -325,23 +314,17 @@ pub async fn do_bigquery(
                     token.as_str(),
                     timeout_ms,
                     None,
-                    annotations.return_last_result && i < queries.len() - 1,
                 )
             })
             .collect::<windmill_common::error::Result<Vec<_>>>()?;
 
         let f = async {
             let mut res: Vec<Box<RawValue>> = vec![];
-
             for fut in futures {
                 let r = fut.await?;
                 res.push(r);
             }
-            if annotations.return_last_result && res.len() > 0 {
-                Ok(res.pop().unwrap())
-            } else {
-                Ok(to_raw_value(&res))
-            }
+            Ok(to_raw_value(&res))
         };
 
         f.boxed()
@@ -353,7 +336,6 @@ pub async fn do_bigquery(
             token.as_str(),
             timeout_ms,
             Some(column_order),
-            false,
         )?
     };
 
@@ -366,7 +348,6 @@ pub async fn do_bigquery(
         result_f.map_err(to_anyhow),
         worker_name,
         &job.workspace_id,
-        &mut Some(occupancy_metrics),
     )
     .await?;
 

backend/windmill-worker/src/bun_executor.rs

@@ -1,14 +1,8 @@
-#[cfg(feature = "deno_core")]
-use std::time::Instant;
-use std::{collections::HashMap, fs, io, path::Path, process::Stdio};
+use std::{collections::HashMap, fs, io, path::Path, process::Stdio, time::Instant};
 
 use base64::Engine;
 use itertools::Itertools;
-
-#[cfg(not(feature = "deno_core"))]
-use serde_json::value::to_raw_value;
 use serde_json::value::RawValue;
-
 use sha2::Digest;
 use uuid::Uuid;
 use windmill_parser_ts::remove_pinned_imports;
@@ -19,19 +13,15 @@ use crate::common::build_envs_map;
 
 use crate::{
     common::{
-        create_args_and_out_file, get_main_override, get_reserved_variables, parse_npm_config,
-        read_file, read_file_content, read_result, start_child_process, write_file_binary,
-        OccupancyMetrics,
+        create_args_and_out_file, get_main_override, get_reserved_variables, handle_child,
+        parse_npm_config, read_file, read_file_content, read_result, start_child_process,
+        write_file_binary,
     },
-    handle_child::handle_child,
     AuthedClientBackgroundTask, BUNFIG_INSTALL_SCOPES, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR,
     BUN_DEPSTAR_CACHE_DIR, BUN_PATH, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NODE_BIN_PATH,
     NODE_PATH, NPM_CONFIG_REGISTRY, NPM_PATH, NSJAIL_PATH, PATH_ENV, TZ_ENV,
 };
 
-#[cfg(windows)]
-use crate::SYSTEM_ROOT;
-
 use tokio::{fs::File, process::Command};
 
 use tokio::io::AsyncReadExt;
@@ -47,7 +37,7 @@ use windmill_common::{
     get_latest_hash_for_path,
     jobs::{QueuedJob, PREPROCESSOR_FAKE_ENTRYPOINT},
     scripts::ScriptLang,
-    worker::{exists_in_cache, get_annotation_ts, save_cache, write_file},
+    worker::{exists_in_cache, get_annotation, save_cache, write_file},
     DB,
 };
 
@@ -79,7 +69,6 @@ pub async fn gen_bun_lockfile(
     export_pkg: bool,
     raw_deps: Option<String>,
     npm_mode: bool,
-    occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
 ) -> Result<Option<String>> {
     let common_bun_proc_envs: HashMap<String, String> = get_common_bun_proc_envs(None).await;
 
@@ -121,9 +110,6 @@ pub async fn gen_bun_lockfile(
             .stdout(Stdio::piped())
             .stderr(Stdio::piped());
 
-        #[cfg(windows)]
-        child_cmd.env("SystemRoot", SYSTEM_ROOT.as_str());
-
         let mut child_process = start_child_process(child_cmd, &*BUN_PATH).await?;
 
         if let Some(db) = db {
@@ -139,7 +125,6 @@ pub async fn gen_bun_lockfile(
                 "bun build",
                 None,
                 false,
-                occupancy_metrics,
             )
             .await?;
         } else {
@@ -164,7 +149,6 @@ pub async fn gen_bun_lockfile(
             worker_name,
             common_bun_proc_envs,
             npm_mode,
-            occupancy_metrics,
         )
         .await?;
     } else {
@@ -246,7 +230,6 @@ pub async fn install_bun_lockfile(
     worker_name: &str,
     common_bun_proc_envs: HashMap<String, String>,
     npm_mode: bool,
-    occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
 ) -> Result<()> {
     let mut child_cmd = Command::new(if npm_mode { &*NPM_PATH } else { &*BUN_PATH });
     child_cmd
@@ -257,9 +240,6 @@ pub async fn install_bun_lockfile(
         .stdout(Stdio::piped())
         .stderr(Stdio::piped());
 
-    #[cfg(windows)]
-    child_cmd.env("SystemRoot", SYSTEM_ROOT.as_str());
-
     let mut npm_logs = if npm_mode {
         "NPM mode\n".to_string()
     } else {
@@ -316,7 +296,6 @@ pub async fn install_bun_lockfile(
             "bun install",
             None,
             false,
-            occupancy_metrics,
         )
         .await?
     } else {
@@ -457,7 +436,6 @@ pub async fn generate_wrapper_mjs(
     mem_peak: &mut i32,
     canceled_by: &mut Option<CanceledBy>,
     common_bun_proc_envs: &HashMap<String, String>,
-    occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
 ) -> Result<()> {
     let mut child = Command::new(&*BUN_PATH);
     child
@@ -468,10 +446,6 @@ pub async fn generate_wrapper_mjs(
         .args(vec!["run", "node_builder.ts"])
         .stdout(Stdio::piped())
         .stderr(Stdio::piped());
-
-    #[cfg(windows)]
-    child.env("SystemRoot", SYSTEM_ROOT.as_str());
-
     let child_process = start_child_process(child, &*BUN_PATH).await?;
     handle_child(
         job_id,
@@ -485,7 +459,6 @@ pub async fn generate_wrapper_mjs(
         "bun build",
         timeout,
         false,
-        occupancy_metrics,
     )
     .await?;
     fs::rename(
@@ -506,7 +479,6 @@ pub async fn generate_bun_bundle(
     mem_peak: &mut i32,
     canceled_by: &mut Option<CanceledBy>,
     common_bun_proc_envs: &HashMap<String, String>,
-    occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
 ) -> Result<()> {
     let mut child = Command::new(&*BUN_PATH);
     child
@@ -517,10 +489,6 @@ pub async fn generate_bun_bundle(
         .args(vec!["run", "node_builder.ts"])
         .stdout(Stdio::piped())
         .stderr(Stdio::piped());
-
-    #[cfg(windows)]
-    child.env("SystemRoot", SYSTEM_ROOT.as_str());
-
     let mut child_process = start_child_process(child, &*BUN_PATH).await?;
     if let Some(db) = db {
         handle_child(
@@ -535,7 +503,6 @@ pub async fn generate_bun_bundle(
             "bun build",
             timeout,
             false,
-            occupancy_metrics,
         )
         .await?;
     } else {
@@ -563,11 +530,7 @@ pub async fn pull_codebase(w_id: &str, id: &str, job_dir: &str) -> Result<()> {
         if is_tar {
             extract_tar(fs::read(bun_cache_path)?.into(), job_dir).await?;
         } else {
-            #[cfg(unix)]
             tokio::fs::symlink(&bun_cache_path, dst).await?;
-
-            #[cfg(windows)]
-            std::os::windows::fs::symlink_dir(&bun_cache_path, &dst)?;
         }
     } else if let Some(os) = windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS
         .read()
@@ -580,11 +543,7 @@ pub async fn pull_codebase(w_id: &str, id: &str, job_dir: &str) -> Result<()> {
         if is_tar {
             extract_tar(bytes, job_dir).await?;
         } else {
-            #[cfg(unix)]
             tokio::fs::symlink(bun_cache_path, dst).await?;
-
-            #[cfg(windows)]
-            std::os::windows::fs::symlink_dir(&bun_cache_path, &dst)?;
         }
 
         // extract_tar(bytes, job_dir).await?;
@@ -650,7 +609,6 @@ pub async fn prebundle_bun_script(
     base_internal_url: &str,
     worker_name: &str,
     token: &str,
-    occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
 ) -> Result<()> {
     let (local_path, remote_path) = compute_bundle_local_and_remote_path(
         inner_content,
@@ -663,7 +621,7 @@ pub async fn prebundle_bun_script(
     if exists_in_cache(&local_path, &remote_path).await {
         return Ok(());
     }
-    let annotation = get_annotation_ts(inner_content);
+    let annotation = get_annotation(inner_content);
     if annotation.nobundling {
         return Ok(());
     }
@@ -698,7 +656,6 @@ pub async fn prebundle_bun_script(
         &mut 0,
         &mut None,
         &common_bun_proc_envs,
-        occupancy_metrics,
     )
     .await?;
 
@@ -753,10 +710,6 @@ async fn compute_bundle_local_and_remote_path(
 
     let hash = windmill_common::utils::calculate_hash(&input_src);
     let local_path = format!("{BUN_BUNDLE_CACHE_DIR}/{hash}");
-
-    #[cfg(windows)]
-    let local_path = local_path.replace("/tmp", r"C:\tmp").replace("/", r"\");
-
     let remote_path = format!("{BUN_BUNDLE_OBJECT_STORE_PREFIX}{hash}");
     (local_path, remote_path)
 }
@@ -798,9 +751,8 @@ pub async fn handle_bun_job(
     envs: HashMap<String, String>,
     shared_mount: &str,
     new_args: &mut Option<HashMap<String, Box<RawValue>>>,
-    occupancy_metrics: &mut OccupancyMetrics,
 ) -> error::Result<Box<RawValue>> {
-    let mut annotation = windmill_common::worker::get_annotation_ts(inner_content);
+    let mut annotation = windmill_common::worker::get_annotation(inner_content);
 
     let (mut has_bundle_cache, cache_logs, local_path, remote_path) =
         if requirements_o.is_some() && !annotation.nobundling && codebase.is_none() {
@@ -850,23 +802,10 @@ pub async fn handle_bun_job(
         ));
     }
 
-    let mut gbuntar_name: Option<String> = None;
+    let mut gbuntar_name = None;
     if has_bundle_cache {
-        let target;
-        let symlink;
-
-        #[cfg(unix)]
-        {
-            target = format!("{job_dir}/main.js");
-            symlink = std::os::unix::fs::symlink(&local_path, &target);
-        }
-        #[cfg(windows)]
-        {
-            target = format!("{job_dir}\\main.js");
-            symlink = std::os::windows::fs::symlink_dir(&local_path, &target);
-        }
-
-        symlink.map_err(|e| {
+        let target = format!("{job_dir}/main.js");
+        std::os::unix::fs::symlink(&local_path, &target).map_err(|e| {
             error::Error::ExecutionErr(format!(
                 "could not copy cached binary from {local_path} to {job_dir}/main: {e:?}"
             ))
@@ -922,7 +861,6 @@ pub async fn handle_bun_job(
                     worker_name,
                     common_bun_proc_envs.clone(),
                     annotation.npm_mode,
-                    &mut Some(occupancy_metrics),
                 )
                 .await?;
 
@@ -950,6 +888,7 @@ pub async fn handle_bun_job(
         // if !*DISABLE_NSJAIL || !empty_trusted_deps || has_custom_config_registry {
         let logs1 = "\n\n--- BUN INSTALL ---\n".to_string();
         append_logs(&job.id, &job.workspace_id, logs1, db).await;
+
         let _ = gen_bun_lockfile(
             mem_peak,
             canceled_by,
@@ -964,7 +903,6 @@ pub async fn handle_bun_job(
             false,
             None,
             annotation.npm_mode,
-            &mut Some(occupancy_metrics),
         )
         .await?;
 
@@ -1190,7 +1128,6 @@ try {{
                 mem_peak,
                 canceled_by,
                 &common_bun_proc_envs,
-                &mut Some(occupancy_metrics),
             )
             .await?;
             if !local_path.is_empty() {
@@ -1232,64 +1169,55 @@ try {{
                 mem_peak,
                 canceled_by,
                 &common_bun_proc_envs,
-                &mut Some(occupancy_metrics),
             )
             .await?;
         }
     }
     if annotation.native_mode {
-        #[cfg(not(feature = "deno_core"))]
-        return Ok(to_raw_value("").unwrap());
-
-        #[cfg(feature = "deno_core")]
-        {
-            let env_code = format!(
+        let env_code = format!(
             "const process = {{ env: {{}} }};\nconst BASE_URL = '{base_internal_url}';\nconst BASE_INTERNAL_URL = '{base_internal_url}';\nprocess.env['BASE_URL'] = BASE_URL;process.env['BASE_INTERNAL_URL'] = BASE_INTERNAL_URL;\n{}",
             reserved_variables
                 .iter()
                 .map(|(k, v)| format!("process.env['{}'] = '{}';\n", k, v))
                 .collect::<Vec<String>>()
                 .join("\n"));
-            let js_code = read_file_content(&format!("{job_dir}/main.js")).await?;
-            let started_at = Instant::now();
-            let args = crate::common::build_args_map(job, client, db)
-                .await?
-                .map(sqlx::types::Json);
-            let job_args = if args.is_some() {
-                args.as_ref()
-            } else {
-                job.args.as_ref()
-            };
-
-            let result = crate::js_eval::eval_fetch_timeout(
-                env_code,
-                inner_content.clone(),
-                js_code,
-                job_args,
-                job.id,
-                job.timeout,
-                db,
-                mem_peak,
-                canceled_by,
-                worker_name,
-                &job.workspace_id,
-                false,
-                occupancy_metrics,
-            )
-            .await?;
-            tracing::info!(
-                "Executed native code in {}ms",
-                started_at.elapsed().as_millis()
-            );
-            append_logs(
-                &job.id,
-                &job.workspace_id,
-                format!("{}\n{}", init_logs, result.1),
-                db,
-            )
-            .await;
-            return Ok(result.0);
-        }
+        let js_code = read_file_content(&format!("{job_dir}/main.js")).await?;
+        let started_at = Instant::now();
+        let args = crate::common::build_args_map(job, client, db)
+            .await?
+            .map(sqlx::types::Json);
+        let job_args = if args.is_some() {
+            args.as_ref()
+        } else {
+            job.args.as_ref()
+        };
+        let result = crate::js_eval::eval_fetch_timeout(
+            env_code,
+            inner_content.clone(),
+            js_code,
+            job_args,
+            job.id,
+            job.timeout,
+            db,
+            mem_peak,
+            canceled_by,
+            worker_name,
+            &job.workspace_id,
+            false,
+        )
+        .await?;
+        tracing::info!(
+            "Executed native code in {}ms",
+            started_at.elapsed().as_millis()
+        );
+        append_logs(
+            &job.id,
+            &job.workspace_id,
+            format!("{}\n{}", init_logs, result.1),
+            db,
+        )
+        .await;
+        return Ok(result.0);
     }
     append_logs(&job.id, &job.workspace_id, init_logs, db).await;
 
@@ -1381,10 +1309,6 @@ try {{
                 .args(vec!["--preserve-symlinks", &script_path])
                 .stdout(Stdio::piped())
                 .stderr(Stdio::piped());
-
-            #[cfg(windows)]
-            bun_cmd.env("SystemRoot", SYSTEM_ROOT.as_str());
-
             bun_cmd
         } else {
             let script_path = format!("{job_dir}/wrapper.mjs");
@@ -1411,13 +1335,8 @@ try {{
                 .args(args)
                 .stdout(Stdio::piped())
                 .stderr(Stdio::piped());
-
-            #[cfg(windows)]
-            bun_cmd.env("SystemRoot", SYSTEM_ROOT.as_str());
-
             bun_cmd
         };
-
         start_child_process(
             cmd,
             if annotation.nodejs_mode {
@@ -1441,7 +1360,6 @@ try {{
         "bun run",
         job.timeout,
         false,
-        &mut Some(occupancy_metrics),
     )
     .await?;
 
@@ -1525,7 +1443,7 @@ pub async fn start_worker(
     let common_bun_proc_envs: HashMap<String, String> =
         get_common_bun_proc_envs(Some(&base_internal_url)).await;
 
-    let mut annotation = windmill_common::worker::get_annotation_ts(inner_content);
+    let mut annotation = windmill_common::worker::get_annotation(inner_content);
 
     //TODO: remove this when bun dedicated workers work without issues
     annotation.nodejs_mode = true;
@@ -1583,7 +1501,6 @@ pub async fn start_worker(
                 worker_name,
                 common_bun_proc_envs.clone(),
                 annotation.npm_mode,
-                &mut None,
             )
             .await?;
             tracing::info!("dedicated worker requirements installed: {reqs}");
@@ -1604,7 +1521,6 @@ pub async fn start_worker(
             false,
             None,
             annotation.npm_mode,
-            &mut None,
         )
         .await?;
     }
@@ -1701,7 +1617,6 @@ for await (const line of Readline.createInterface({{ input: process.stdin }})) {
             &mut mem_peak,
             &mut canceled_by,
             &common_bun_proc_envs,
-            &mut None,
         )
         .await?;
     }

backend/windmill-worker/src/common.rs

@@ -1,7 +1,15 @@
 use async_recursion::async_recursion;
-
+use deno_ast::swc::parser::lexer::util::CharExt;
+use futures::Future;
 use itertools::Itertools;
 
+#[cfg(any(target_os = "linux", target_os = "macos"))]
+use nix::sys::signal::{self, Signal};
+#[cfg(any(target_os = "linux", target_os = "macos"))]
+use nix::unistd::Pid;
+
+#[cfg(all(feature = "enterprise", feature = "parquet"))]
+use object_store::path::Path;
 use regex::Regex;
 use serde::{Deserialize, Serialize};
 use serde_json::value::RawValue;
@@ -10,15 +18,18 @@ use sqlx::types::Json;
 use sqlx::{Pool, Postgres};
 use tokio::process::Command;
 use tokio::{fs::File, io::AsyncReadExt};
+use windmill_common::error::to_anyhow;
 use windmill_common::jobs::ENTRYPOINT_OVERRIDE;
-
+#[cfg(all(feature = "enterprise", feature = "parquet"))]
+use windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS;
 #[cfg(feature = "parquet")]
 use windmill_common::s3_helpers::{
     get_etag_or_empty, LargeFileStorage, ObjectStoreResource, S3Object,
 };
 use windmill_common::variables::{build_crypt_with_key_suffix, decrypt_value_with_mc};
 use windmill_common::worker::{
-    to_raw_value, write_file, CLOUD_HOSTED, ROOT_CACHE_DIR, WORKER_CONFIG,
+    get_windmill_memory_usage, get_worker_memory_usage, to_raw_value, write_file, CLOUD_HOSTED,
+    ROOT_CACHE_DIR, TMP_DIR, WORKER_CONFIG,
 };
 use windmill_common::{
     error::{self, Error},
@@ -27,21 +38,43 @@ use windmill_common::{
 };
 
 use anyhow::{anyhow, Result};
+use windmill_queue::{append_logs, CanceledBy};
 
+#[cfg(any(target_os = "linux", target_os = "macos"))]
+use std::os::unix::process::ExitStatusExt;
+
+use std::process::ExitStatus;
+use std::sync::atomic::AtomicU32;
+use std::sync::Arc;
 use std::{
     collections::{hash_map::DefaultHasher, HashMap},
     hash::{Hash, Hasher},
+    io, panic,
     time::Duration,
 };
 
+use tracing::{trace_span, Instrument};
 use uuid::Uuid;
 use windmill_common::{variables, DB};
 
-use tokio::{io::AsyncWriteExt, process::Child, time::Instant};
+#[cfg(feature = "enterprise")]
+use windmill_common::job_metrics;
+
+use tokio::{
+    io::{AsyncBufReadExt, AsyncWriteExt, BufReader},
+    process::Child,
+    sync::{broadcast, watch},
+    time::{interval, sleep, Instant, MissedTickBehavior},
+};
+
+use futures::{
+    future::{self, ready, FutureExt},
+    stream, StreamExt,
+};
 
 use crate::{
     AuthedClient, AuthedClientBackgroundTask, JOB_DEFAULT_TIMEOUT, MAX_RESULT_SIZE,
-    MAX_TIMEOUT_DURATION,
+    MAX_TIMEOUT_DURATION, MAX_WAIT_FOR_SIGINT, MAX_WAIT_FOR_SIGTERM,
 };
 
 pub async fn build_args_map<'a>(
@@ -433,6 +466,36 @@ pub fn get_main_override(args: Option<&Json<HashMap<String, Box<RawValue>>>>) ->
         .flatten();
 }
 
+async fn get_mem_peak(pid: Option<u32>, nsjail: bool) -> i32 {
+    if pid.is_none() {
+        return -1;
+    }
+    let pid = if nsjail {
+        // This is a bit hacky, but the process id of the nsjail process is the pid of nsjail + 1.
+        // Ideally, we would get the number from fork() itself. This works in MOST cases.
+        pid.unwrap() + 1
+    } else {
+        pid.unwrap()
+    };
+
+    if let Ok(file) = File::open(format!("/proc/{}/status", pid)).await {
+        let mut lines = BufReader::new(file).lines();
+        while let Some(line) = lines.next_line().await.unwrap_or(None) {
+            if line.starts_with("VmHWM:") {
+                return line
+                    .split_whitespace()
+                    .nth(1)
+                    .and_then(|s| s.parse::<i32>().ok())
+                    .unwrap_or(-1);
+            };
+        }
+        -2
+    } else {
+        // rand::random::<i32>() % 100 // to remove - used to fake memory data on MacOS
+        -3
+    }
+}
+
 pub fn sizeof_val(v: &serde_json::Value) -> usize {
     std::mem::size_of::<serde_json::Value>()
         + match v {
@@ -453,6 +516,63 @@ pub fn sizeof_val(v: &serde_json::Value) -> usize {
         }
 }
 
+pub async fn run_future_with_polling_update_job_poller<Fut, T>(
+    job_id: Uuid,
+    timeout: Option<i32>,
+    db: &DB,
+    mem_peak: &mut i32,
+    canceled_by_ref: &mut Option<CanceledBy>,
+    result_f: Fut,
+    worker_name: &str,
+    w_id: &str,
+) -> error::Result<T>
+where
+    Fut: Future<Output = anyhow::Result<T>>,
+{
+    let (tx, rx) = broadcast::channel::<()>(3);
+
+    let update_job = update_job_poller(
+        job_id,
+        db,
+        mem_peak,
+        canceled_by_ref,
+        || async { 0 },
+        worker_name,
+        w_id,
+        rx,
+    );
+
+    let timeout_ms = u64::try_from(
+        resolve_job_timeout(&db, &w_id, job_id, timeout)
+            .await
+            .0
+            .as_millis(),
+    )
+    .unwrap_or(200000);
+
+    let rows = tokio::select! {
+        biased;
+        result = tokio::time::timeout(std::time::Duration::from_millis(timeout_ms), result_f) => result
+        .map_err(|e| {
+            tracing::error!("Query timeout: {}", e);
+            Error::ExecutionErr(format!("Query timeout after (>{}s)", timeout_ms/1000))
+        })?,
+        ex = update_job, if job_id != Uuid::nil() => {
+            match ex {
+                UpdateJobPollingExit::Done => Err(Error::ExecutionErr("Job cancelled".to_string())).map_err(to_anyhow)?,
+                UpdateJobPollingExit::AlreadyCompleted => Err(Error::AlreadyCompleted("Job already completed".to_string())).map_err(to_anyhow)?,
+            }
+        }
+    }?;
+    drop(tx);
+    Ok(rows)
+}
+
+pub enum UpdateJobPollingExit {
+    Done,
+    AlreadyCompleted,
+}
+
 pub async fn update_worker_ping_for_failed_init_script(
     db: &DB,
     worker_name: &str,
@@ -474,83 +594,672 @@ pub async fn update_worker_ping_for_failed_init_script(
         tracing::error!("Error updating worker ping for failed init script: {e:?}");
     }
 }
-pub struct OccupancyMetrics {
-    pub running_job_started_at: Option<Instant>,
-    pub total_duration_of_running_jobs: f32,
-    pub worker_occupancy_rate_history: Vec<(f32, f32)>,
-    pub start_time: Instant,
+
+pub async fn update_job_poller<F, Fut>(
+    job_id: Uuid,
+    db: &DB,
+    mem_peak: &mut i32,
+    canceled_by_ref: &mut Option<CanceledBy>,
+    get_mem: F,
+    worker_name: &str,
+    w_id: &str,
+    mut rx: broadcast::Receiver<()>,
+) -> UpdateJobPollingExit
+where
+    F: Fn() -> Fut,
+    Fut: Future<Output = i32>,
+{
+    let update_job_interval = Duration::from_millis(500);
+
+    let db = db.clone();
+
+    let mut interval = interval(update_job_interval);
+    interval.set_missed_tick_behavior(MissedTickBehavior::Skip);
+
+    let mut i = 0;
+
+    #[cfg(feature = "enterprise")]
+    let mut memory_metric_id: Result<String, Error> =
+        Err(Error::NotFound("not yet initialized".to_string()));
+
+    loop {
+        tokio::select!(
+            _ = rx.recv() => break,
+            _ = interval.tick() => {
+                // update the last_ping column every 5 seconds
+                i+=1;
+                if i == 1 || i % 10 == 0 {
+                    let memory_usage = get_worker_memory_usage();
+                    let wm_memory_usage = get_windmill_memory_usage();
+                    tracing::info!("job {job_id} on {worker_name} in {w_id} worker memory snapshot {}kB/{}kB", memory_usage.unwrap_or_default()/1024, wm_memory_usage.unwrap_or_default()/1024);
+                    if job_id != Uuid::nil() {
+                        sqlx::query!(
+                            "UPDATE worker_ping SET ping_at = now(), current_job_id = $1, current_job_workspace_id = $2, memory_usage = $3, wm_memory_usage = $4 WHERE worker = $5",
+                            &job_id,
+                            &w_id,
+                            memory_usage,
+                            wm_memory_usage,
+                            &worker_name
+                        )
+                        .execute(&db)
+                        .await
+                        .expect("update worker ping");
+                    }
+                }
+                let current_mem = get_mem().await;
+                if current_mem > *mem_peak {
+                    *mem_peak = current_mem
+                }
+                tracing::info!("job {job_id} on {worker_name} in {w_id} still running.  mem: {current_mem}kB, peak mem: {mem_peak}kB");
+
+
+                let update_job_row = i == 2 || (!*SLOW_LOGS && (i < 20 || (i < 120 && i % 5 == 0) || i % 10 == 0)) || i % 20 == 0;
+                if update_job_row {
+                #[cfg(feature = "enterprise")]
+                {
+                    if job_id != Uuid::nil() {
+
+                        // tracking metric starting at i >= 2 b/c first point it useless and we don't want to track metric for super fast jobs
+                        if i == 2 {
+                            memory_metric_id = job_metrics::register_metric_for_job(
+                                &db,
+                                w_id.to_string(),
+                                job_id,
+                                "memory_kb".to_string(),
+                                job_metrics::MetricKind::TimeseriesInt,
+                                Some("Job Memory Footprint (kB)".to_string()),
+                            )
+                            .await;
+                        }
+                        if let Ok(ref metric_id) = memory_metric_id {
+                            if let Err(err) = job_metrics::record_metric(&db, w_id.to_string(), job_id, metric_id.to_owned(), job_metrics::MetricNumericValue::Integer(current_mem)).await {
+                                tracing::error!("Unable to save memory stat for job {} in workspace {}. Error was: {:?}", job_id, w_id, err);
+                            }
+                        }
+                    }
+                }
+                if job_id != Uuid::nil() {
+                    let (canceled, canceled_by, canceled_reason, already_completed) = sqlx::query_as::<_, (bool, Option<String>, Option<String>, bool)>("UPDATE queue SET mem_peak = $1, last_ping = now() WHERE id = $2 RETURNING canceled, canceled_by, canceled_reason, false")
+                        .bind(*mem_peak)
+                        .bind(job_id)
+                        .fetch_optional(&db)
+                        .await
+                        .unwrap_or_else(|e| {
+                            tracing::error!(%e, "error updating job {job_id}: {e:#}");
+                            Some((false, None, None, false))
+                        })
+                        .unwrap_or_else(|| {
+                            // if the job is not in queue, it can only be in the completed_job so it is already complete
+                            (false, None, None, true)
+                        });
+                    if already_completed {
+                        return UpdateJobPollingExit::AlreadyCompleted
+                    }
+                    if canceled {
+                        canceled_by_ref.replace(CanceledBy {
+                            username: canceled_by.clone(),
+                            reason: canceled_reason.clone(),
+                        });
+                        break
+                    }
+                }
+            }
+            },
+        );
+    }
+    tracing::info!("job {job_id} finished");
+
+    UpdateJobPollingExit::Done
 }
 
-impl OccupancyMetrics {
-    pub fn new(start_time: Instant) -> Self {
-        OccupancyMetrics {
-            running_job_started_at: None,
-            total_duration_of_running_jobs: 0.0,
-            worker_occupancy_rate_history: Vec::new(),
-            start_time,
+pub enum CompactLogs {
+    NotEE,
+    NoS3,
+    S3,
+}
+
+async fn compact_logs(
+    job_id: Uuid,
+    w_id: &str,
+    db: &DB,
+    nlogs: String,
+    total_size: Arc<AtomicU32>,
+    compact_kind: CompactLogs,
+    _worker_name: &str,
+) -> error::Result<(String, String)> {
+    let mut prev_logs = sqlx::query_scalar!(
+        "SELECT logs FROM job_logs WHERE job_id = $1 AND workspace_id = $2",
+        job_id,
+        w_id
+    )
+    .fetch_optional(db)
+    .await?
+    .flatten()
+    .unwrap_or_default();
+    let size = prev_logs.char_indices().count() as i32;
+    let nlogs_len = nlogs.char_indices().count();
+    let to_keep_in_db = usize::max(
+        usize::min(nlogs_len, 3000),
+        nlogs_len % LARGE_LOG_THRESHOLD_SIZE,
+    );
+    let extra_split = to_keep_in_db < nlogs_len;
+    let stored_in_storage_len = if extra_split {
+        nlogs_len - to_keep_in_db
+    } else {
+        0
+    };
+    let extra_to_newline = nlogs
+        .chars()
+        .skip(stored_in_storage_len)
+        .find_position(|x| x.is_line_break())
+        .map(|(i, _)| i)
+        .unwrap_or(to_keep_in_db);
+    let stored_in_storage_to_newline = stored_in_storage_len + extra_to_newline;
+
+    let (append_to_storage, stored_in_db) = if extra_split {
+        if stored_in_storage_to_newline == nlogs.len() {
+            (nlogs.as_ref(), "".to_string())
+        } else {
+            let split_idx = nlogs
+                .char_indices()
+                .nth(stored_in_storage_to_newline)
+                .map(|(i, _)| i)
+                .unwrap_or(0);
+            let (append_to_storage, stored_in_db) = nlogs.split_at(split_idx);
+            // tracing::error!("{append_to_storage} ||||| {stored_in_db}");
+            // tracing::error!(
+            //     "{:?} {:?} {} {}",
+            //     excess_prev_logs.lines().last(),
+            //     current_logs.lines().next(),
+            //     split_idx,
+            //     excess_size_modulo
+            // );
+            (append_to_storage, stored_in_db.to_string())
+        }
+    } else {
+        // tracing::error!("{:?}", nlogs.lines().last());
+        ("", nlogs.to_string())
+    };
+
+    let new_size_with_excess = size + stored_in_storage_to_newline as i32;
+
+    let new_size = total_size.fetch_add(
+        new_size_with_excess as u32,
+        std::sync::atomic::Ordering::SeqCst,
+    ) + new_size_with_excess as u32;
+
+    let path = format!(
+        "logs/{job_id}/{}_{new_size}.txt",
+        chrono::Utc::now().timestamp_millis()
+    );
+
+    let mut new_current_logs = match compact_kind {
+        CompactLogs::NoS3 => format!("\n[windmill] No object storage set in instance settings. Previous logs have been saved to disk at {path}"),
+        CompactLogs::S3 => format!("\n[windmill] Previous logs have been saved to object storage at {path}"),
+        CompactLogs::NotEE => format!("\n[windmill] Previous logs have been saved to disk at {path}"),
+    };
+    new_current_logs.push_str(&stored_in_db);
+
+    sqlx::query!(
+        "UPDATE job_logs SET logs = $1, log_offset = $2, 
+        log_file_index = array_append(coalesce(log_file_index, array[]::text[]), $3) 
+        WHERE workspace_id = $4 AND job_id = $5",
+        new_current_logs,
+        new_size as i32,
+        path,
+        w_id,
+        job_id
+    )
+    .execute(db)
+    .await?;
+    prev_logs.push_str(&append_to_storage);
+
+    return Ok((prev_logs, path));
+}
+
+async fn default_disk_log_storage(
+    job_id: Uuid,
+    w_id: &str,
+    db: &DB,
+    nlogs: String,
+    total_size: Arc<AtomicU32>,
+    compact_kind: CompactLogs,
+    worker_name: &str,
+) {
+    match compact_logs(
+        job_id,
+        &w_id,
+        &db,
+        nlogs,
+        total_size,
+        compact_kind,
+        worker_name,
+    )
+    .await
+    {
+        Err(e) => tracing::error!("Could not compact logs for job {job_id}: {e:?}",),
+        Ok((prev_logs, path)) => {
+            let path = format!("{}/{}", TMP_DIR, path);
+            let splitted = &path.split("/").collect_vec();
+            tokio::fs::create_dir_all(splitted.into_iter().take(splitted.len() - 1).join("/"))
+                .await
+                .map_err(|e| {
+                    tracing::error!("Could not create logs directory: {e:?}",);
+                    e
+                })
+                .ok();
+            let created = tokio::fs::File::create(&path).await;
+            if let Err(e) = created {
+                tracing::error!("Could not create logs file {path}: {e:?}",);
+                return;
+            }
+            if let Err(e) = tokio::fs::write(&path, prev_logs).await {
+                tracing::error!("Could not write to logs file {path}: {e:?}");
+            } else {
+                tracing::info!("Logs length of {job_id} has exceeded a threshold. Previous logs have been saved to disk at {path}");
+            }
         }
     }
+}
 
-    pub fn update_occupancy_metrics(&mut self) -> (f32, Option<f32>, Option<f32>, Option<f32>) {
-        let metrics = self;
-        let current_occupied_duration = metrics
-            .running_job_started_at
-            .map(|started_at| started_at.elapsed().as_secs_f32())
-            .unwrap_or(0.0);
-        let total_occupation = metrics.total_duration_of_running_jobs + current_occupied_duration;
+async fn append_job_logs(
+    job_id: Uuid,
+    w_id: String,
+    logs: String,
+    db: DB,
+    must_compact_logs: bool,
+    total_size: Arc<AtomicU32>,
+    worker_name: String,
+) -> () {
+    if must_compact_logs {
+        #[cfg(all(feature = "enterprise", feature = "parquet"))]
+        if let Some(os) = OBJECT_STORE_CACHE_SETTINGS.read().await.clone() {
+            match compact_logs(
+                job_id,
+                &w_id,
+                &db,
+                logs,
+                total_size,
+                CompactLogs::S3,
+                &worker_name,
+            )
+            .await
+            {
+                Err(e) => tracing::error!("Could not compact logs for job {job_id}: {e:?}",),
+                Ok((prev_logs, path)) => {
+                    tracing::info!("Logs length of {job_id} has exceeded a threshold. Previous logs have been saved to object storage at {path}");
+                    let path2 = path.clone();
+                    if let Err(e) = os
+                        .put(&Path::from(path), prev_logs.to_string().into_bytes().into())
+                        .await
+                    {
+                        tracing::error!("Could not save logs to s3: {e:?}");
+                    }
+                    tracing::info!("Logs of {job_id} saved to object storage at {path2}");
+                }
+            }
+        } else {
+            default_disk_log_storage(
+                job_id,
+                &w_id,
+                &db,
+                logs,
+                total_size,
+                CompactLogs::NoS3,
+                &worker_name,
+            )
+            .await;
+        }
 
-        let elapsed = metrics.start_time.elapsed().as_secs_f32();
+        #[cfg(not(all(feature = "enterprise", feature = "parquet")))]
+        {
+            default_disk_log_storage(
+                job_id,
+                &w_id,
+                &db,
+                logs,
+                total_size,
+                CompactLogs::NotEE,
+                &worker_name,
+            )
+            .await;
+        }
+    } else {
+        append_logs(&job_id, w_id, logs, db).await;
+    }
+}
 
-        let (occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m) =
-            if !metrics.worker_occupancy_rate_history.is_empty() {
-                let mut total_occupation_15s = 0.0;
-                let mut total_occupation_5m = 0.0;
-                let mut total_occupation_30m = 0.0;
-                let mut index30m = 0;
-                for (i, (past_total_occupation, time)) in
-                    metrics.worker_occupancy_rate_history.iter().enumerate()
+pub const LARGE_LOG_THRESHOLD_SIZE: usize = 9000;
+/// - wait until child exits and return with exit status
+/// - read lines from stdout and stderr and append them to the "queue"."logs"
+///   quitting early if output exceedes MAX_LOG_SIZE characters (not bytes)
+/// - update the `last_line` and `logs` strings with the program output
+/// - update "queue"."last_ping" every five seconds
+/// - kill process if we exceed timeout or "queue"."canceled" is set
+#[tracing::instrument(level = "trace", skip_all)]
+pub async fn handle_child(
+    job_id: &Uuid,
+    db: &Pool<Postgres>,
+    mem_peak: &mut i32,
+    canceled_by_ref: &mut Option<CanceledBy>,
+    mut child: Child,
+    nsjail: bool,
+    worker: &str,
+    w_id: &str,
+    child_name: &str,
+    custom_timeout: Option<i32>,
+    sigterm: bool,
+) -> error::Result<()> {
+    let start = Instant::now();
+
+    let pid = child.id();
+    #[cfg(target_os = "linux")]
+    if let Some(pid) = pid {
+        //set the highest oom priority
+        if let Some(mut file) = File::create(format!("/proc/{pid}/oom_score_adj"))
+            .await
+            .map_err(|e| {
+                tracing::error!("Could not create oom_score_file to pid {pid}: {e:#}");
+                e
+            })
+            .ok()
+        {
+            let _ = file.write_all(b"1000").await;
+            let _ = file.sync_all().await;
+        }
+    } else {
+        tracing::info!("could not get child pid");
+    }
+    let (set_too_many_logs, mut too_many_logs) = watch::channel::<bool>(false);
+    let (tx, rx) = broadcast::channel::<()>(3);
+    let mut rx2 = tx.subscribe();
+
+    let output = child_joined_output_stream(&mut child);
+
+    let job_id = job_id.clone();
+
+    /* the cancellation future is polled on by `wait_on_child` while
+     * waiting for the child to exit normally */
+    let update_job = update_job_poller(
+        job_id,
+        db,
+        mem_peak,
+        canceled_by_ref,
+        || get_mem_peak(pid, nsjail),
+        worker,
+        w_id,
+        rx,
+    );
+
+    #[derive(PartialEq, Debug)]
+    enum KillReason {
+        TooManyLogs,
+        Timeout,
+        Cancelled,
+        AlreadyCompleted,
+    }
+
+    let (timeout_duration, timeout_warn_msg) =
+        resolve_job_timeout(&db, w_id, job_id, custom_timeout).await;
+    if let Some(msg) = timeout_warn_msg {
+        append_logs(&job_id, w_id, msg.as_str(), db).await;
+    }
+
+    /* a future that completes when the child process exits */
+    let wait_on_child = async {
+        let db = db.clone();
+
+        let kill_reason = tokio::select! {
+            biased;
+            result = child.wait() => return result.map(Ok),
+            Ok(()) = too_many_logs.changed() => KillReason::TooManyLogs,
+            _ = sleep(timeout_duration) => KillReason::Timeout,
+            ex = update_job, if job_id != Uuid::nil() => match ex {
+                UpdateJobPollingExit::Done => KillReason::Cancelled,
+                UpdateJobPollingExit::AlreadyCompleted => KillReason::AlreadyCompleted,
+            },
+        };
+        tx.send(()).expect("rx should never be dropped");
+        drop(tx);
+
+        let set_reason = async {
+            if kill_reason == KillReason::Timeout {
+                if let Err(err) = sqlx::query(
+                    r#"
+                       UPDATE queue
+                          SET canceled = true
+                            , canceled_by = 'timeout'
+                            , canceled_reason = $1
+                        WHERE id = $2
+                    "#,
+                )
+                .bind(format!("duration > {}", timeout_duration.as_secs()))
+                .bind(job_id)
+                .execute(&db)
+                .await
                 {
-                    let diff = elapsed - time;
-                    if diff < 1800.0 && total_occupation_30m == 0.0 {
-                        total_occupation_30m = (total_occupation - past_total_occupation) / diff;
-                        index30m = i;
+                    tracing::error!(%job_id, %err, "error setting cancelation reason for job {job_id}: {err}");
+                }
+            }
+        };
+
+        if let Some(id) = child.id() {
+            if *MAX_WAIT_FOR_SIGINT > 0 {
+                #[cfg(any(target_os = "linux", target_os = "macos"))]
+                signal::kill(Pid::from_raw(id as i32), Signal::SIGINT).unwrap();
+
+                for _ in 0..*MAX_WAIT_FOR_SIGINT {
+                    if child.try_wait().is_ok_and(|x| x.is_some()) {
+                        break;
                     }
-                    if diff < 300.0 && total_occupation_5m == 0.0 {
-                        total_occupation_5m = (total_occupation - past_total_occupation) / diff;
+                    sleep(Duration::from_secs(1)).await;
+                }
+                if child.try_wait().is_ok_and(|x| x.is_some()) {
+                    set_reason.await;
+                    return Ok(Err(kill_reason));
+                }
+            }
+            if sigterm {
+                #[cfg(any(target_os = "linux", target_os = "macos"))]
+                signal::kill(Pid::from_raw(id as i32), Signal::SIGTERM).unwrap();
+
+                for _ in 0..*MAX_WAIT_FOR_SIGTERM {
+                    if child.try_wait().is_ok_and(|x| x.is_some()) {
+                        break;
                     }
-                    if diff < 15.0 {
-                        total_occupation_15s = (total_occupation - past_total_occupation) / diff;
+                    sleep(Duration::from_secs(1)).await;
+                }
+                if child.try_wait().is_ok_and(|x| x.is_some()) {
+                    set_reason.await;
+                    return Ok(Err(kill_reason));
+                }
+            }
+        }
+        /* send SIGKILL and reap child process */
+        let (_, kill) = future::join(set_reason, child.kill()).await;
+        kill.map(|()| Err(kill_reason))
+    };
+
+    /* a future that reads output from the child and appends to the database */
+    let lines = async move {
+
+        let max_log_size = if *CLOUD_HOSTED {
+            MAX_RESULT_SIZE
+        } else {
+            usize::MAX
+        };
+
+        /* log_remaining is zero when output limit was reached */
+        let mut log_remaining =  if *CLOUD_HOSTED {
+            max_log_size
+        } else {
+            usize::MAX
+        };
+        let mut result = io::Result::Ok(());
+        let mut output = output.take_until(async {
+            let _ = rx2.recv().await;
+            //wait at most 50ms after end of a script for output stream to end
+            tokio::time::sleep(Duration::from_millis(50)).await;
+         }).boxed();
+        /* `do_write` resolves the task, but does not contain the Result.
+         * It's useful to know if the task completed. */
+        let (mut do_write, mut write_result) = tokio::spawn(ready(())).remote_handle();
+
+        let mut log_total_size: u64 = 0;
+        let pg_log_total_size = Arc::new(AtomicU32::new(0));
+
+        while let Some(line) =  output.by_ref().next().await {
+
+            let do_write_ = do_write.shared();
+
+            let delay = if start.elapsed() < Duration::from_secs(10) {
+                Duration::from_millis(500)
+            } else if start.elapsed() < Duration::from_secs(60){
+                Duration::from_millis(2500)
+            } else {
+                Duration::from_millis(5000)
+            };
+
+            let delay = if *SLOW_LOGS {
+                delay * 10
+            } else {
+                delay
+            };
+
+            let mut read_lines = stream::once(async { line })
+                .chain(output.by_ref())
+                /* after receiving a line, continue until some delay has passed
+                 * _and_ the previous database write is complete */
+                .take_until(future::join(sleep(delay), do_write_.clone()))
+                .boxed();
+
+            /* Read up until an error is encountered,
+             * handle log lines first and then the error... */
+            let mut joined = String::new();
+
+            while let Some(line) = read_lines.next().await {
+
+                match line {
+                    Ok(line) => {
+                        if line.is_empty() {
+                            continue;
+                        }
+                        append_with_limit(&mut joined, &line, &mut log_remaining);
+                        if log_remaining == 0 {
+                            tracing::info!(%job_id, "Too many logs lines for job {job_id}");
+                            let _ = set_too_many_logs.send(true);
+                            joined.push_str(&format!(
+                                "Job logs or result reached character limit of {MAX_RESULT_SIZE}; killing job."
+                            ));
+                            /* stop reading and drop our streams fairly quickly */
+                            break;
+                        }
+                    }
+                    Err(err) => {
+                        result = Err(err);
                         break;
                     }
                 }
+            }
 
-                //drop all elements before the oldest one in 30m windows
-                metrics.worker_occupancy_rate_history.drain(..index30m);
 
-                (
-                    Some(total_occupation_15s),
-                    Some(total_occupation_5m),
-                    Some(total_occupation_30m),
-                )
-            } else {
-                (None, None, None)
-            };
-        let occupancy_rate = total_occupation / elapsed;
+            /* Ensure the last flush completed before starting a new one.
+             *
+             * This shouldn't pause since `take_until()` reads lines until `do_write`
+             * resolves. We only stop reading lines before `take_until()` resolves if we reach
+             * EOF or a read error.  In those cases, waiting on a database query to complete is
+             * fine because we're done. */
 
-        //push the current occupancy rate and the timestamp
-        metrics
-            .worker_occupancy_rate_history
-            .push((total_occupation, elapsed));
+            if let Some(Ok(p)) = do_write_
+                .then(|()| write_result)
+                .await
+                .err()
+                .map(|err| err.try_into_panic())
+            {
+                panic::resume_unwind(p);
+            }
 
-        (
-            occupancy_rate,
-            occupancy_rate_15s,
-            occupancy_rate_5m,
-            occupancy_rate_30m,
-        )
+
+            let joined_len = joined.len() as u64;
+            log_total_size += joined_len;
+            let compact_logs = log_total_size > LARGE_LOG_THRESHOLD_SIZE as u64;
+            if compact_logs {
+                log_total_size = 0;
+            }
+
+            let worker_name = worker.to_string();
+            let w_id2 = w_id.to_string();
+            (do_write, write_result) = tokio::spawn(append_job_logs(job_id, w_id2, joined, db.clone(), compact_logs, pg_log_total_size.clone(), worker_name)).remote_handle();
+
+
+
+            if let Err(err) = result {
+                tracing::error!(%job_id, %err, "error reading output for job {job_id} '{child_name}': {err}");
+                break;
+            }
+
+            if *set_too_many_logs.borrow() {
+                break;
+            }
+        }
+
+        /* drop our end of the pipe */
+        drop(output);
+
+        if let Some(Ok(p)) = do_write
+            .then(|()| write_result)
+            .await
+            .err()
+            .map(|err| err.try_into_panic())
+        {
+            panic::resume_unwind(p);
+        }
+    }.instrument(trace_span!("child_lines"));
+
+    let (wait_result, _) = tokio::join!(wait_on_child, lines);
+
+    let success = wait_result.is_ok()
+        && wait_result.as_ref().unwrap().is_ok()
+        && wait_result.as_ref().unwrap().as_ref().unwrap().success();
+    tracing::info!(%job_id, %success, %mem_peak, %worker, "child process '{child_name}' took {}ms", start.elapsed().as_millis());
+
+    match wait_result {
+        _ if *too_many_logs.borrow() => Err(Error::ExecutionErr(format!(
+            "logs or result reached limit. (current max size: {MAX_RESULT_SIZE} characters)"
+        ))),
+        Ok(Ok(status)) => process_status(status),
+        Ok(Err(kill_reason)) => match kill_reason {
+            KillReason::AlreadyCompleted => {
+                Err(Error::AlreadyCompleted("Job already completed".to_string()))
+            }
+            _ => Err(Error::ExecutionErr(format!(
+                "job process killed because {kill_reason:#?}"
+            ))),
+        },
+        Err(err) => Err(Error::ExecutionErr(format!("job process io error: {err}"))),
     }
 }
 
+pub fn process_status(status: ExitStatus) -> error::Result<()> {
+    if status.success() {
+        Ok(())
+    } else if let Some(code) = status.code() {
+        Err(error::Error::ExitStatus(code))
+    } else {
+        #[cfg(any(target_os = "linux", target_os = "macos"))]
+        return Err(error::Error::ExecutionErr(format!(
+            "process terminated by signal: {:#?}, stopped_signal: {:#?}, core_dumped: {}",
+            status.signal(),
+            status.stopped_signal(),
+            status.core_dumped()
+        )));
+
+        #[cfg(not(any(target_os = "linux", target_os = "macos")))]
+        return Err(error::Error::ExecutionErr(String::from(
+            "process terminated by signal",
+        )));
+    }
+}
 pub async fn start_child_process(mut cmd: Command, executable: &str) -> Result<Child, Error> {
     return cmd
         .spawn()
@@ -615,6 +1324,80 @@ pub async fn resolve_job_timeout(
     }
 }
 
+/// takes stdout and stderr from Child, panics if either are not present
+///
+/// builds a stream joining both stdout and stderr each read line by line
+fn child_joined_output_stream(
+    child: &mut Child,
+) -> impl stream::FusedStream<Item = io::Result<String>> {
+    let stderr = child
+        .stderr
+        .take()
+        .expect("child did not have a handle to stdout");
+
+    let stdout = child
+        .stdout
+        .take()
+        .expect("child did not have a handle to stdout");
+
+    let stdout = BufReader::new(stdout).lines();
+    let stderr = BufReader::new(stderr).lines();
+    stream::select(lines_to_stream(stderr), lines_to_stream(stdout))
+}
+
+pub fn lines_to_stream<R: tokio::io::AsyncBufRead + Unpin>(
+    mut lines: tokio::io::Lines<R>,
+) -> impl futures::Stream<Item = io::Result<String>> {
+    stream::poll_fn(move |cx| {
+        std::pin::Pin::new(&mut lines)
+            .poll_next_line(cx)
+            .map(|result| result.transpose())
+    })
+}
+
+lazy_static::lazy_static! {
+    static ref RE_00: Regex = Regex::new('\u{00}'.to_string().as_str()).unwrap();
+    pub static ref NO_LOGS_AT_ALL: bool = std::env::var("NO_LOGS_AT_ALL").ok().is_some_and(|x| x == "1" || x == "true");
+    pub static ref SLOW_LOGS: bool = std::env::var("SLOW_LOGS").ok().is_some_and(|x| x == "1" || x == "true");
+}
+// as a detail, `BufReader::lines()` removes \n and \r\n from the strings it yields,
+// so this pushes \n to thd destination string in each call
+fn append_with_limit(dst: &mut String, src: &str, limit: &mut usize) {
+    if *NO_LOGS_AT_ALL {
+        return;
+    }
+    let src_str;
+    let src = {
+        src_str = RE_00.replace_all(src, "");
+        src_str.as_ref()
+    };
+    if !*CLOUD_HOSTED {
+        dst.push('\n');
+        dst.push_str(&src);
+        return;
+    } else {
+        if *limit > 0 {
+            dst.push('\n');
+        }
+        *limit -= 1;
+    }
+
+    let src_len = src.chars().count();
+    if src_len <= *limit {
+        dst.push_str(&src);
+        *limit -= src_len;
+    } else {
+        let byte_pos = src
+            .char_indices()
+            .skip(*limit)
+            .next()
+            .map(|(byte_pos, _)| byte_pos)
+            .unwrap_or(0);
+        dst.push_str(&src[0..byte_pos]);
+        *limit = 0;
+    }
+}
+
 pub async fn hash_args(
     _db: &DB,
     _client: &AuthedClient,

backend/windmill-worker/src/dedicated_worker.rs

@@ -29,7 +29,8 @@ use windmill_queue::append_logs;
 use anyhow::Context;
 
 use crate::{
-    common::start_child_process, JobCompleted, JobCompletedSender, MAX_BUFFERED_DEDICATED_JOBS,
+    common::{process_status, start_child_process},
+    JobCompleted, JobCompletedSender, MAX_BUFFERED_DEDICATED_JOBS,
 };
 
 use futures::{future, Future};
@@ -77,8 +78,6 @@ pub async fn handle_dedicated_process(
 ) -> std::result::Result<(), error::Error> {
     //do not cache local dependencies
 
-    use crate::handle_child::process_status;
-
     let mut child = {
         let mut cmd = Command::new(command_path);
         cmd.current_dir(job_dir)
@@ -128,7 +127,7 @@ pub async fn handle_dedicated_process(
         if let Err(e) = process_status(status) {
             tracing::error!("child exit status was not success: {e:#}");
         } else {
-            tracing::info!("child exit status was success");
+            tracing::info!("child exist status was success");
         }
     });
 

backend/windmill-worker/src/deno_executor.rs

@@ -7,10 +7,9 @@ use windmill_queue::{append_logs, CanceledBy};
 
 use crate::{
     common::{
-        create_args_and_out_file, get_main_override, get_reserved_variables, parse_npm_config,
-        read_file, read_result, start_child_process, OccupancyMetrics,
+        create_args_and_out_file, get_main_override, get_reserved_variables, handle_child,
+        parse_npm_config, read_file, read_result, start_child_process,
     },
-    handle_child::handle_child,
     AuthedClientBackgroundTask, DENO_CACHE_DIR, DENO_PATH, DISABLE_NSJAIL, HOME_ENV,
     NPM_CONFIG_REGISTRY, PATH_ENV, TZ_ENV,
 };
@@ -95,7 +94,6 @@ pub async fn generate_deno_lock(
     w_id: &str,
     worker_name: &str,
     base_internal_url: &str,
-    occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
 ) -> error::Result<String> {
     let _ = write_file(job_dir, "main.ts", code)?;
 
@@ -148,7 +146,6 @@ pub async fn generate_deno_lock(
             "deno cache",
             None,
             false,
-            occupancy_metrics,
         )
         .await?;
     } else {
@@ -176,7 +173,6 @@ pub async fn handle_deno_job(
     worker_name: &str,
     envs: HashMap<String, String>,
     new_args: &mut Option<HashMap<String, Box<RawValue>>>,
-    occupancy_metrics: &mut OccupancyMetrics,
 ) -> error::Result<Box<RawValue>> {
     // let mut start = Instant::now();
     let logs1 = "\n\n--- DENO CODE EXECUTION ---\n".to_string();
@@ -401,7 +397,6 @@ try {{
         "deno run",
         job.timeout,
         false,
-        &mut Some(occupancy_metrics),
     )
     .await?;
     // logs.push_str(format!("execute: {:?}\n", start.elapsed().as_millis()).as_str());

backend/windmill-worker/src/go_executor.rs

@@ -15,10 +15,9 @@ use windmill_queue::{append_logs, CanceledBy};
 
 use crate::{
     common::{
-        capitalize, create_args_and_out_file, get_reserved_variables, read_result,
-        start_child_process, OccupancyMetrics,
+        capitalize, create_args_and_out_file, get_reserved_variables, handle_child, read_result,
+        start_child_process,
     },
-    handle_child::handle_child,
     AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, GOPRIVATE, GOPROXY,
     GO_BIN_CACHE_DIR, GO_CACHE_DIR, HOME_ENV, NSJAIL_PATH, PATH_ENV, TZ_ENV,
 };
@@ -45,7 +44,6 @@ pub async fn handle_go_job(
     base_internal_url: &str,
     worker_name: &str,
     envs: HashMap<String, String>,
-    occupation_metrics: &mut OccupancyMetrics,
 ) -> Result<Box<RawValue>, Error> {
     //go does not like executing modules at temp root
     let job_dir = &format!("{job_dir}/go");
@@ -90,7 +88,6 @@ pub async fn handle_go_job(
             skip_tidy,
             worker_name,
             &job.workspace_id,
-            occupation_metrics,
         )
         .await?;
 
@@ -205,7 +202,6 @@ func Run(req Req) (interface{{}}, error){{
             "go build",
             None,
             false,
-            &mut Some(occupation_metrics),
         )
         .await?;
 
@@ -225,12 +221,7 @@ func Run(req Req) (interface{{}}, error){{
         }
     } else {
         let target = format!("{job_dir}/main");
-        #[cfg(unix)]
-        let symlink = std::os::unix::fs::symlink(&bin_path, &target);
-        #[cfg(windows)]
-        let symlink = std::os::windows::fs::symlink_dir(&bin_path, &target);
-
-        symlink.map_err(|e| {
+        std::os::unix::fs::symlink(&bin_path, &target).map_err(|e| {
             Error::ExecutionErr(format!(
                 "could not copy cached binary from {bin_path} to {job_dir}/main: {e:?}"
             ))
@@ -306,7 +297,6 @@ func Run(req Req) (interface{{}}, error){{
         "go run",
         job.timeout,
         false,
-        &mut Some(occupation_metrics),
     )
     .await?;
 
@@ -346,7 +336,6 @@ pub async fn install_go_dependencies(
     has_sum: bool,
     worker_name: &str,
     w_id: &str,
-    occupation_metrics: &mut OccupancyMetrics,
 ) -> error::Result<String> {
     if !skip_go_mod {
         gen_go_mymod(code, job_dir).await?;
@@ -370,7 +359,6 @@ pub async fn install_go_dependencies(
             "go init",
             None,
             false,
-            &mut Some(occupation_metrics),
         )
         .await?;
 
@@ -436,9 +424,9 @@ pub async fn install_go_dependencies(
         &format!("go {mod_command}"),
         None,
         false,
-        &mut Some(occupation_metrics),
     )
-    .await?;
+    .await
+    .map_err(|e| Error::ExecutionErr(format!("Lockfile generation failed: {e:?}")))?;
 
     if (!new_lockfile || has_sum) && non_dep_job {
         return Ok("".to_string());

backend/windmill-worker/src/graphql_executor.rs

@@ -12,8 +12,7 @@ use windmill_queue::{CanceledBy, HTTP_CLIENT};
 
 use serde::Deserialize;
 
-use crate::common::OccupancyMetrics;
-use crate::handle_child::run_future_with_polling_update_job_poller;
+use crate::common::run_future_with_polling_update_job_poller;
 use crate::{common::build_args_map, AuthedClientBackgroundTask};
 
 #[derive(Deserialize)]
@@ -42,7 +41,6 @@ pub async fn do_graphql(
     mem_peak: &mut i32,
     canceled_by: &mut Option<CanceledBy>,
     worker_name: &str,
-    occupation_metrics: &mut OccupancyMetrics,
 ) -> windmill_common::error::Result<Box<RawValue>> {
     let args = build_args_map(job, client, db).await?.map(Json);
     let job_args = if args.is_some() {
@@ -153,7 +151,6 @@ pub async fn do_graphql(
         result_f,
         worker_name,
         &job.workspace_id,
-        &mut Some(occupation_metrics),
     )
     .await?;
 

backend/windmill-worker/src/handle_child.rs

@@ -1,675 +0,0 @@
-use futures::Future;
-
-#[cfg(any(target_os = "linux", target_os = "macos"))]
-use nix::sys::signal::{self, Signal};
-#[cfg(any(target_os = "linux", target_os = "macos"))]
-use nix::unistd::Pid;
-
-use sqlx::{Pool, Postgres};
-#[cfg(windows)]
-use std::process::Stdio;
-use tokio::fs::File;
-#[cfg(windows)]
-use tokio::process::Command;
-use windmill_common::error::to_anyhow;
-
-use windmill_common::error::{self, Error};
-
-use windmill_common::worker::{get_windmill_memory_usage, get_worker_memory_usage, CLOUD_HOSTED};
-
-use windmill_queue::{append_logs, CanceledBy};
-
-#[cfg(any(target_os = "linux", target_os = "macos"))]
-use std::os::unix::process::ExitStatusExt;
-
-use std::process::ExitStatus;
-use std::sync::atomic::AtomicU32;
-use std::sync::Arc;
-use std::{io, panic, time::Duration};
-
-use tracing::{trace_span, Instrument};
-use uuid::Uuid;
-use windmill_common::DB;
-
-#[cfg(feature = "enterprise")]
-use windmill_common::job_metrics;
-
-use tokio::{
-    io::{AsyncBufReadExt, AsyncWriteExt, BufReader},
-    process::Child,
-    sync::{broadcast, watch},
-    time::{interval, sleep, Instant, MissedTickBehavior},
-};
-
-use futures::{
-    future::{self, ready, FutureExt},
-    stream, StreamExt,
-};
-
-use crate::common::{resolve_job_timeout, OccupancyMetrics};
-use crate::job_logger::{append_job_logs, append_with_limit, LARGE_LOG_THRESHOLD_SIZE};
-use crate::{MAX_RESULT_SIZE, MAX_WAIT_FOR_SIGINT, MAX_WAIT_FOR_SIGTERM};
-
-lazy_static::lazy_static! {
-    pub static ref SLOW_LOGS: bool = std::env::var("SLOW_LOGS").ok().is_some_and(|x| x == "1" || x == "true");
-}
-
-//  - kill windows process along with all child processes
-#[cfg(windows)]
-async fn kill_process_tree(pid: Option<u32>) -> Result<(), String> {
-    let pid = match pid {
-        Some(pid) => pid,
-        None => return Err("No PID provided to kill.".to_string()),
-    };
-
-    let output = Command::new("cmd")
-        .args(&["/C", "taskkill", "/PID", &pid.to_string(), "/T", "/F"])
-        .stdout(Stdio::piped())
-        .stderr(Stdio::piped())
-        .output()
-        .await
-        .map_err(|e| format!("Failed to execute taskkill: {}", e))?;
-
-    if output.status.success() {
-        Ok(())
-    } else {
-        Err(format!(
-            "Failed to kill process tree. Error: {}",
-            String::from_utf8_lossy(&output.stderr)
-        ))
-    }
-}
-
-/// - wait until child exits and return with exit status
-/// - read lines from stdout and stderr and append them to the "queue"."logs"
-///   quitting early if output exceedes MAX_LOG_SIZE characters (not bytes)
-/// - update the `last_line` and `logs` strings with the program output
-/// - update "queue"."last_ping" every five seconds
-/// - kill process if we exceed timeout or "queue"."canceled" is set
-#[tracing::instrument(level = "trace", skip_all)]
-pub async fn handle_child(
-    job_id: &Uuid,
-    db: &Pool<Postgres>,
-    mem_peak: &mut i32,
-    canceled_by_ref: &mut Option<CanceledBy>,
-    mut child: Child,
-    nsjail: bool,
-    worker: &str,
-    w_id: &str,
-    child_name: &str,
-    custom_timeout: Option<i32>,
-    sigterm: bool,
-    occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
-) -> error::Result<()> {
-    let start = Instant::now();
-
-    let pid = child.id();
-    #[cfg(target_os = "linux")]
-    if let Some(pid) = pid {
-        //set the highest oom priority
-        if let Some(mut file) = File::create(format!("/proc/{pid}/oom_score_adj"))
-            .await
-            .map_err(|e| {
-                tracing::error!("Could not create oom_score_file to pid {pid}: {e:#}");
-                e
-            })
-            .ok()
-        {
-            let _ = file.write_all(b"1000").await;
-            let _ = file.sync_all().await;
-        }
-    } else {
-        tracing::info!("could not get child pid");
-    }
-    let (set_too_many_logs, mut too_many_logs) = watch::channel::<bool>(false);
-    let (tx, rx) = broadcast::channel::<()>(3);
-    let mut rx2 = tx.subscribe();
-
-    let output = child_joined_output_stream(&mut child);
-
-    let job_id = job_id.clone();
-
-    /* the cancellation future is polled on by `wait_on_child` while
-     * waiting for the child to exit normally */
-    let update_job = update_job_poller(
-        job_id,
-        db,
-        mem_peak,
-        canceled_by_ref,
-        || get_mem_peak(pid, nsjail),
-        worker,
-        w_id,
-        rx,
-        occupancy_metrics,
-    );
-
-    #[derive(PartialEq, Debug)]
-    enum KillReason {
-        TooManyLogs,
-        Timeout,
-        Cancelled,
-        AlreadyCompleted,
-    }
-
-    let (timeout_duration, timeout_warn_msg) =
-        resolve_job_timeout(&db, w_id, job_id, custom_timeout).await;
-    if let Some(msg) = timeout_warn_msg {
-        append_logs(&job_id, w_id, msg.as_str(), db).await;
-    }
-
-    /* a future that completes when the child process exits */
-    let wait_on_child = async {
-        let db = db.clone();
-
-        let kill_reason = tokio::select! {
-            biased;
-            result = child.wait() => return result.map(Ok),
-            Ok(()) = too_many_logs.changed() => KillReason::TooManyLogs,
-            _ = sleep(timeout_duration) => KillReason::Timeout,
-            ex = update_job, if job_id != Uuid::nil() => match ex {
-                UpdateJobPollingExit::Done => KillReason::Cancelled,
-                UpdateJobPollingExit::AlreadyCompleted => KillReason::AlreadyCompleted,
-            },
-        };
-        tx.send(()).expect("rx should never be dropped");
-        drop(tx);
-
-        let set_reason = async {
-            if kill_reason == KillReason::Timeout {
-                if let Err(err) = sqlx::query(
-                    r#"
-                       UPDATE queue
-                          SET canceled = true
-                            , canceled_by = 'timeout'
-                            , canceled_reason = $1
-                        WHERE id = $2
-                    "#,
-                )
-                .bind(format!("duration > {}", timeout_duration.as_secs()))
-                .bind(job_id)
-                .execute(&db)
-                .await
-                {
-                    tracing::error!(%job_id, %err, "error setting cancelation reason for job {job_id}: {err}");
-                }
-            }
-        };
-
-        if let Some(id) = child.id() {
-            if *MAX_WAIT_FOR_SIGINT > 0 {
-                #[cfg(any(target_os = "linux", target_os = "macos"))]
-                signal::kill(Pid::from_raw(id as i32), Signal::SIGINT).unwrap();
-
-                for _ in 0..*MAX_WAIT_FOR_SIGINT {
-                    if child.try_wait().is_ok_and(|x| x.is_some()) {
-                        break;
-                    }
-                    sleep(Duration::from_secs(1)).await;
-                }
-                if child.try_wait().is_ok_and(|x| x.is_some()) {
-                    set_reason.await;
-                    return Ok(Err(kill_reason));
-                }
-            }
-            if sigterm {
-                #[cfg(any(target_os = "linux", target_os = "macos"))]
-                signal::kill(Pid::from_raw(id as i32), Signal::SIGTERM).unwrap();
-
-                for _ in 0..*MAX_WAIT_FOR_SIGTERM {
-                    if child.try_wait().is_ok_and(|x| x.is_some()) {
-                        break;
-                    }
-                    sleep(Duration::from_secs(1)).await;
-                }
-                if child.try_wait().is_ok_and(|x| x.is_some()) {
-                    set_reason.await;
-                    return Ok(Err(kill_reason));
-                }
-            }
-        }
-        #[cfg(windows)]
-        {
-            let pid_to_kill = child.id();
-            match kill_process_tree(pid_to_kill).await {
-                Ok(_) => tracing::debug!(
-                    "successfully killed process tree with PID: {:?}",
-                    pid_to_kill
-                ),
-                Err(e) => tracing::error!("failed to kill process tree: {:?}", e),
-            };
-            set_reason.await;
-            return Ok(Err(kill_reason));
-        }
-
-        #[cfg(unix)]
-        {
-            /* send SIGKILL and reap child process */
-            let (_, kill) = future::join(set_reason, child.kill()).await;
-            kill.map(|()| Err(kill_reason))
-        }
-    };
-
-    /* a future that reads output from the child and appends to the database */
-    let lines = async move {
-
-        let max_log_size = if *CLOUD_HOSTED {
-            MAX_RESULT_SIZE
-        } else {
-            usize::MAX
-        };
-
-        /* log_remaining is zero when output limit was reached */
-        let mut log_remaining =  if *CLOUD_HOSTED {
-            max_log_size
-        } else {
-            usize::MAX
-        };
-        let mut result = io::Result::Ok(());
-        let mut output = output.take_until(async {
-            let _ = rx2.recv().await;
-            //wait at most 50ms after end of a script for output stream to end
-            tokio::time::sleep(Duration::from_millis(50)).await;
-         }).boxed();
-        /* `do_write` resolves the task, but does not contain the Result.
-         * It's useful to know if the task completed. */
-        let (mut do_write, mut write_result) = tokio::spawn(ready(())).remote_handle();
-
-        let mut log_total_size: u64 = 0;
-        let pg_log_total_size = Arc::new(AtomicU32::new(0));
-
-        while let Some(line) =  output.by_ref().next().await {
-
-            let do_write_ = do_write.shared();
-
-            let delay = if start.elapsed() < Duration::from_secs(10) {
-                Duration::from_millis(500)
-            } else if start.elapsed() < Duration::from_secs(60){
-                Duration::from_millis(2500)
-            } else {
-                Duration::from_millis(5000)
-            };
-
-            let delay = if *SLOW_LOGS {
-                delay * 10
-            } else {
-                delay
-            };
-
-            let mut read_lines = stream::once(async { line })
-                .chain(output.by_ref())
-                /* after receiving a line, continue until some delay has passed
-                 * _and_ the previous database write is complete */
-                .take_until(future::join(sleep(delay), do_write_.clone()))
-                .boxed();
-
-            /* Read up until an error is encountered,
-             * handle log lines first and then the error... */
-            let mut joined = String::new();
-
-            while let Some(line) = read_lines.next().await {
-
-                match line {
-                    Ok(line) => {
-                        if line.is_empty() {
-                            continue;
-                        }
-                        append_with_limit(&mut joined, &line, &mut log_remaining);
-                        if log_remaining == 0 {
-                            tracing::info!(%job_id, "Too many logs lines for job {job_id}");
-                            let _ = set_too_many_logs.send(true);
-                            joined.push_str(&format!(
-                                "Job logs or result reached character limit of {MAX_RESULT_SIZE}; killing job."
-                            ));
-                            /* stop reading and drop our streams fairly quickly */
-                            break;
-                        }
-                    }
-                    Err(err) => {
-                        result = Err(err);
-                        break;
-                    }
-                }
-            }
-
-
-            /* Ensure the last flush completed before starting a new one.
-             *
-             * This shouldn't pause since `take_until()` reads lines until `do_write`
-             * resolves. We only stop reading lines before `take_until()` resolves if we reach
-             * EOF or a read error.  In those cases, waiting on a database query to complete is
-             * fine because we're done. */
-
-            if let Some(Ok(p)) = do_write_
-                .then(|()| write_result)
-                .await
-                .err()
-                .map(|err| err.try_into_panic())
-            {
-                panic::resume_unwind(p);
-            }
-
-
-            let joined_len = joined.len() as u64;
-            log_total_size += joined_len;
-            let compact_logs = log_total_size > LARGE_LOG_THRESHOLD_SIZE as u64;
-            if compact_logs {
-                log_total_size = 0;
-            }
-
-            let worker_name = worker.to_string();
-            let w_id2 = w_id.to_string();
-            (do_write, write_result) = tokio::spawn(append_job_logs(job_id, w_id2, joined, db.clone(), compact_logs, pg_log_total_size.clone(), worker_name)).remote_handle();
-
-
-
-            if let Err(err) = result {
-                tracing::error!(%job_id, %err, "error reading output for job {job_id} '{child_name}': {err}");
-                break;
-            }
-
-            if *set_too_many_logs.borrow() {
-                break;
-            }
-        }
-
-        /* drop our end of the pipe */
-        drop(output);
-
-        if let Some(Ok(p)) = do_write
-            .then(|()| write_result)
-            .await
-            .err()
-            .map(|err| err.try_into_panic())
-        {
-            panic::resume_unwind(p);
-        }
-    }.instrument(trace_span!("child_lines"));
-
-    let (wait_result, _) = tokio::join!(wait_on_child, lines);
-
-    let success = wait_result.is_ok()
-        && wait_result.as_ref().unwrap().is_ok()
-        && wait_result.as_ref().unwrap().as_ref().unwrap().success();
-    tracing::info!(%job_id, %success, %mem_peak, %worker, "child process '{child_name}' took {}ms", start.elapsed().as_millis());
-
-    match wait_result {
-        _ if *too_many_logs.borrow() => Err(Error::ExecutionErr(format!(
-            "logs or result reached limit. (current max size: {MAX_RESULT_SIZE} characters)"
-        ))),
-        Ok(Ok(status)) => process_status(status),
-        Ok(Err(kill_reason)) => match kill_reason {
-            KillReason::AlreadyCompleted => {
-                Err(Error::AlreadyCompleted("Job already completed".to_string()))
-            }
-            _ => Err(Error::ExecutionErr(format!(
-                "job process killed because {kill_reason:#?}"
-            ))),
-        },
-        Err(err) => Err(Error::ExecutionErr(format!("job process io error: {err}"))),
-    }
-}
-
-async fn get_mem_peak(pid: Option<u32>, nsjail: bool) -> i32 {
-    if pid.is_none() {
-        return -1;
-    }
-    let pid = if nsjail {
-        // This is a bit hacky, but the process id of the nsjail process is the pid of nsjail + 1.
-        // Ideally, we would get the number from fork() itself. This works in MOST cases.
-        pid.unwrap() + 1
-    } else {
-        pid.unwrap()
-    };
-
-    if let Ok(file) = File::open(format!("/proc/{}/status", pid)).await {
-        let mut lines = BufReader::new(file).lines();
-        while let Some(line) = lines.next_line().await.unwrap_or(None) {
-            if line.starts_with("VmHWM:") {
-                return line
-                    .split_whitespace()
-                    .nth(1)
-                    .and_then(|s| s.parse::<i32>().ok())
-                    .unwrap_or(-1);
-            };
-        }
-        -2
-    } else {
-        // rand::random::<i32>() % 100 // to remove - used to fake memory data on MacOS
-        -3
-    }
-}
-
-pub async fn run_future_with_polling_update_job_poller<Fut, T>(
-    job_id: Uuid,
-    timeout: Option<i32>,
-    db: &DB,
-    mem_peak: &mut i32,
-    canceled_by_ref: &mut Option<CanceledBy>,
-    result_f: Fut,
-    worker_name: &str,
-    w_id: &str,
-    occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
-) -> error::Result<T>
-where
-    Fut: Future<Output = anyhow::Result<T>>,
-{
-    let (tx, rx) = broadcast::channel::<()>(3);
-
-    let update_job = update_job_poller(
-        job_id,
-        db,
-        mem_peak,
-        canceled_by_ref,
-        || async { 0 },
-        worker_name,
-        w_id,
-        rx,
-        occupancy_metrics,
-    );
-
-    let timeout_ms = u64::try_from(
-        resolve_job_timeout(&db, &w_id, job_id, timeout)
-            .await
-            .0
-            .as_millis(),
-    )
-    .unwrap_or(200000);
-
-    let rows = tokio::select! {
-        biased;
-        result = tokio::time::timeout(std::time::Duration::from_millis(timeout_ms), result_f) => result
-        .map_err(|e| {
-            tracing::error!("Query timeout: {}", e);
-            Error::ExecutionErr(format!("Query timeout after (>{}s)", timeout_ms/1000))
-        })?,
-        ex = update_job, if job_id != Uuid::nil() => {
-            match ex {
-                UpdateJobPollingExit::Done => Err(Error::ExecutionErr("Job cancelled".to_string())).map_err(to_anyhow)?,
-                UpdateJobPollingExit::AlreadyCompleted => Err(Error::AlreadyCompleted("Job already completed".to_string())).map_err(to_anyhow)?,
-            }
-        }
-    }?;
-    drop(tx);
-    Ok(rows)
-}
-
-pub enum UpdateJobPollingExit {
-    Done,
-    AlreadyCompleted,
-}
-
-pub async fn update_job_poller<F, Fut>(
-    job_id: Uuid,
-    db: &DB,
-    mem_peak: &mut i32,
-    canceled_by_ref: &mut Option<CanceledBy>,
-    get_mem: F,
-    worker_name: &str,
-    w_id: &str,
-    mut rx: broadcast::Receiver<()>,
-    occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
-) -> UpdateJobPollingExit
-where
-    F: Fn() -> Fut,
-    Fut: Future<Output = i32>,
-{
-    let update_job_interval = Duration::from_millis(500);
-
-    let db = db.clone();
-
-    let mut interval = interval(update_job_interval);
-    interval.set_missed_tick_behavior(MissedTickBehavior::Skip);
-
-    let mut i = 0;
-
-    #[cfg(feature = "enterprise")]
-    let mut memory_metric_id: Result<String, Error> =
-        Err(Error::NotFound("not yet initialized".to_string()));
-
-    loop {
-        tokio::select!(
-            _ = rx.recv() => break,
-            _ = interval.tick() => {
-                // update the last_ping column every 5 seconds
-                i+=1;
-                if i == 1 || i % 10 == 0 {
-                    let memory_usage = get_worker_memory_usage();
-                    let wm_memory_usage = get_windmill_memory_usage();
-                    tracing::info!("job {job_id} on {worker_name} in {w_id} worker memory snapshot {}kB/{}kB", memory_usage.unwrap_or_default()/1024, wm_memory_usage.unwrap_or_default()/1024);
-                    let occupancy = occupancy_metrics.as_mut().map(|x| x.update_occupancy_metrics());
-                    if job_id != Uuid::nil() {
-                        sqlx::query!(
-                            "UPDATE worker_ping SET ping_at = now(), current_job_id = $1, current_job_workspace_id = $2, memory_usage = $3, wm_memory_usage = $4,
-                            occupancy_rate = $6, occupancy_rate_15s = $7, occupancy_rate_5m = $8, occupancy_rate_30m = $9 WHERE worker = $5",
-                            &job_id,
-                            &w_id,
-                            memory_usage,
-                            wm_memory_usage,
-                            &worker_name,
-                            occupancy.map(|x| x.0),
-                            occupancy.and_then(|x| x.1),
-                            occupancy.and_then(|x| x.2),
-                            occupancy.and_then(|x| x.3),
-                        )
-                        .execute(&db)
-                        .await
-                        .expect("update worker ping");
-                    }
-                }
-                let current_mem = get_mem().await;
-                if current_mem > *mem_peak {
-                    *mem_peak = current_mem
-                }
-                tracing::info!("job {job_id} on {worker_name} in {w_id} still running.  mem: {current_mem}kB, peak mem: {mem_peak}kB");
-
-
-                let update_job_row = i == 2 || (!*SLOW_LOGS && (i < 20 || (i < 120 && i % 5 == 0) || i % 10 == 0)) || i % 20 == 0;
-                if update_job_row {
-                #[cfg(feature = "enterprise")]
-                {
-                    if job_id != Uuid::nil() {
-
-                        // tracking metric starting at i >= 2 b/c first point it useless and we don't want to track metric for super fast jobs
-                        if i == 2 {
-                            memory_metric_id = job_metrics::register_metric_for_job(
-                                &db,
-                                w_id.to_string(),
-                                job_id,
-                                "memory_kb".to_string(),
-                                job_metrics::MetricKind::TimeseriesInt,
-                                Some("Job Memory Footprint (kB)".to_string()),
-                            )
-                            .await;
-                        }
-                        if let Ok(ref metric_id) = memory_metric_id {
-                            if let Err(err) = job_metrics::record_metric(&db, w_id.to_string(), job_id, metric_id.to_owned(), job_metrics::MetricNumericValue::Integer(current_mem)).await {
-                                tracing::error!("Unable to save memory stat for job {} in workspace {}. Error was: {:?}", job_id, w_id, err);
-                            }
-                        }
-                    }
-                }
-                if job_id != Uuid::nil() {
-                    let (canceled, canceled_by, canceled_reason, already_completed) = sqlx::query_as::<_, (bool, Option<String>, Option<String>, bool)>("UPDATE queue SET mem_peak = $1, last_ping = now() WHERE id = $2 RETURNING canceled, canceled_by, canceled_reason, false")
-                        .bind(*mem_peak)
-                        .bind(job_id)
-                        .fetch_optional(&db)
-                        .await
-                        .unwrap_or_else(|e| {
-                            tracing::error!(%e, "error updating job {job_id}: {e:#}");
-                            Some((false, None, None, false))
-                        })
-                        .unwrap_or_else(|| {
-                            // if the job is not in queue, it can only be in the completed_job so it is already complete
-                            (false, None, None, true)
-                        });
-                    if already_completed {
-                        return UpdateJobPollingExit::AlreadyCompleted
-                    }
-                    if canceled {
-                        canceled_by_ref.replace(CanceledBy {
-                            username: canceled_by.clone(),
-                            reason: canceled_reason.clone(),
-                        });
-                        break
-                    }
-                }
-            }
-            },
-        );
-    }
-    tracing::info!("job {job_id} finished");
-
-    UpdateJobPollingExit::Done
-}
-
-/// takes stdout and stderr from Child, panics if either are not present
-///
-/// builds a stream joining both stdout and stderr each read line by line
-fn child_joined_output_stream(
-    child: &mut Child,
-) -> impl stream::FusedStream<Item = io::Result<String>> {
-    let stderr = child
-        .stderr
-        .take()
-        .expect("child did not have a handle to stdout");
-
-    let stdout = child
-        .stdout
-        .take()
-        .expect("child did not have a handle to stdout");
-
-    let stdout = BufReader::new(stdout).lines();
-    let stderr = BufReader::new(stderr).lines();
-    stream::select(lines_to_stream(stderr), lines_to_stream(stdout))
-}
-
-pub fn lines_to_stream<R: tokio::io::AsyncBufRead + Unpin>(
-    mut lines: tokio::io::Lines<R>,
-) -> impl futures::Stream<Item = io::Result<String>> {
-    stream::poll_fn(move |cx| {
-        std::pin::Pin::new(&mut lines)
-            .poll_next_line(cx)
-            .map(|result| result.transpose())
-    })
-}
-
-pub fn process_status(status: ExitStatus) -> error::Result<()> {
-    if status.success() {
-        Ok(())
-    } else if let Some(code) = status.code() {
-        Err(error::Error::ExitStatus(code))
-    } else {
-        #[cfg(any(target_os = "linux", target_os = "macos"))]
-        return Err(error::Error::ExecutionErr(format!(
-            "process terminated by signal: {:#?}, stopped_signal: {:#?}, core_dumped: {}",
-            status.signal(),
-            status.stopped_signal(),
-            status.core_dumped()
-        )));
-
-        #[cfg(not(any(target_os = "linux", target_os = "macos")))]
-        return Err(error::Error::ExecutionErr(String::from(
-            "process terminated by signal",
-        )));
-    }
-}

backend/windmill-worker/src/job_logger.rs

@@ -1,285 +0,0 @@
-use itertools::Itertools;
-use swc_ecma_parser::lexer::util::CharExt;
-
-#[cfg(all(feature = "enterprise", feature = "parquet"))]
-use object_store::path::Path;
-use regex::Regex;
-
-#[cfg(all(feature = "enterprise", feature = "parquet"))]
-use windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS;
-
-use windmill_common::error::{self};
-use windmill_common::worker::{CLOUD_HOSTED, TMP_DIR};
-
-use windmill_queue::append_logs;
-
-use std::sync::atomic::AtomicU32;
-use std::sync::Arc;
-
-use uuid::Uuid;
-use windmill_common::DB;
-
-pub enum CompactLogs {
-    #[cfg(not(all(feature = "enterprise", feature = "parquet")))]
-    NotEE,
-    #[allow(dead_code)]
-    NoS3,
-    #[allow(dead_code)]
-    S3,
-}
-
-async fn compact_logs(
-    job_id: Uuid,
-    w_id: &str,
-    db: &DB,
-    nlogs: String,
-    total_size: Arc<AtomicU32>,
-    compact_kind: CompactLogs,
-    _worker_name: &str,
-) -> error::Result<(String, String)> {
-    let mut prev_logs = sqlx::query_scalar!(
-        "SELECT logs FROM job_logs WHERE job_id = $1 AND workspace_id = $2",
-        job_id,
-        w_id
-    )
-    .fetch_optional(db)
-    .await?
-    .flatten()
-    .unwrap_or_default();
-    let size = prev_logs.char_indices().count() as i32;
-    let nlogs_len = nlogs.char_indices().count();
-    let to_keep_in_db = usize::max(
-        usize::min(nlogs_len, 3000),
-        nlogs_len % LARGE_LOG_THRESHOLD_SIZE,
-    );
-    let extra_split = to_keep_in_db < nlogs_len;
-    let stored_in_storage_len = if extra_split {
-        nlogs_len - to_keep_in_db
-    } else {
-        0
-    };
-    let extra_to_newline = nlogs
-        .chars()
-        .skip(stored_in_storage_len)
-        .find_position(|x| x.is_line_break())
-        .map(|(i, _)| i)
-        .unwrap_or(to_keep_in_db);
-    let stored_in_storage_to_newline = stored_in_storage_len + extra_to_newline;
-
-    let (append_to_storage, stored_in_db) = if extra_split {
-        if stored_in_storage_to_newline == nlogs.len() {
-            (nlogs.as_ref(), "".to_string())
-        } else {
-            let split_idx = nlogs
-                .char_indices()
-                .nth(stored_in_storage_to_newline)
-                .map(|(i, _)| i)
-                .unwrap_or(0);
-            let (append_to_storage, stored_in_db) = nlogs.split_at(split_idx);
-            // tracing::error!("{append_to_storage} ||||| {stored_in_db}");
-            // tracing::error!(
-            //     "{:?} {:?} {} {}",
-            //     excess_prev_logs.lines().last(),
-            //     current_logs.lines().next(),
-            //     split_idx,
-            //     excess_size_modulo
-            // );
-            (append_to_storage, stored_in_db.to_string())
-        }
-    } else {
-        // tracing::error!("{:?}", nlogs.lines().last());
-        ("", nlogs.to_string())
-    };
-
-    let new_size_with_excess = size + stored_in_storage_to_newline as i32;
-
-    let new_size = total_size.fetch_add(
-        new_size_with_excess as u32,
-        std::sync::atomic::Ordering::SeqCst,
-    ) + new_size_with_excess as u32;
-
-    let path = format!(
-        "logs/{job_id}/{}_{new_size}.txt",
-        chrono::Utc::now().timestamp_millis()
-    );
-
-    let mut new_current_logs = match compact_kind {
-        CompactLogs::NoS3 => format!("\n[windmill] No object storage set in instance settings. Previous logs have been saved to disk at {path}"),
-        CompactLogs::S3 => format!("\n[windmill] Previous logs have been saved to object storage at {path}"),
-        #[cfg(not(all(feature = "enterprise", feature = "parquet")))]
-        CompactLogs::NotEE => format!("\n[windmill] Previous logs have been saved to disk at {path}"),
-    };
-    new_current_logs.push_str(&stored_in_db);
-
-    sqlx::query!(
-        "UPDATE job_logs SET logs = $1, log_offset = $2, 
-        log_file_index = array_append(coalesce(log_file_index, array[]::text[]), $3) 
-        WHERE workspace_id = $4 AND job_id = $5",
-        new_current_logs,
-        new_size as i32,
-        path,
-        w_id,
-        job_id
-    )
-    .execute(db)
-    .await?;
-    prev_logs.push_str(&append_to_storage);
-
-    return Ok((prev_logs, path));
-}
-
-async fn default_disk_log_storage(
-    job_id: Uuid,
-    w_id: &str,
-    db: &DB,
-    nlogs: String,
-    total_size: Arc<AtomicU32>,
-    compact_kind: CompactLogs,
-    worker_name: &str,
-) {
-    match compact_logs(
-        job_id,
-        &w_id,
-        &db,
-        nlogs,
-        total_size,
-        compact_kind,
-        worker_name,
-    )
-    .await
-    {
-        Err(e) => tracing::error!("Could not compact logs for job {job_id}: {e:?}",),
-        Ok((prev_logs, path)) => {
-            let path = format!("{}/{}", TMP_DIR, path);
-            let splitted = &path.split("/").collect_vec();
-            tokio::fs::create_dir_all(splitted.into_iter().take(splitted.len() - 1).join("/"))
-                .await
-                .map_err(|e| {
-                    tracing::error!("Could not create logs directory: {e:?}",);
-                    e
-                })
-                .ok();
-            let created = tokio::fs::File::create(&path).await;
-            if let Err(e) = created {
-                tracing::error!("Could not create logs file {path}: {e:?}",);
-                return;
-            }
-            if let Err(e) = tokio::fs::write(&path, prev_logs).await {
-                tracing::error!("Could not write to logs file {path}: {e:?}");
-            } else {
-                tracing::info!("Logs length of {job_id} has exceeded a threshold. Previous logs have been saved to disk at {path}");
-            }
-        }
-    }
-}
-
-pub(crate) async fn append_job_logs(
-    job_id: Uuid,
-    w_id: String,
-    logs: String,
-    db: DB,
-    must_compact_logs: bool,
-    total_size: Arc<AtomicU32>,
-    worker_name: String,
-) -> () {
-    if must_compact_logs {
-        #[cfg(all(feature = "enterprise", feature = "parquet"))]
-        if let Some(os) = OBJECT_STORE_CACHE_SETTINGS.read().await.clone() {
-            match compact_logs(
-                job_id,
-                &w_id,
-                &db,
-                logs,
-                total_size,
-                CompactLogs::S3,
-                &worker_name,
-            )
-            .await
-            {
-                Err(e) => tracing::error!("Could not compact logs for job {job_id}: {e:?}",),
-                Ok((prev_logs, path)) => {
-                    tracing::info!("Logs length of {job_id} has exceeded a threshold. Previous logs have been saved to object storage at {path}");
-                    let path2 = path.clone();
-                    if let Err(e) = os
-                        .put(&Path::from(path), prev_logs.to_string().into_bytes().into())
-                        .await
-                    {
-                        tracing::error!("Could not save logs to s3: {e:?}");
-                    }
-                    tracing::info!("Logs of {job_id} saved to object storage at {path2}");
-                }
-            }
-        } else {
-            default_disk_log_storage(
-                job_id,
-                &w_id,
-                &db,
-                logs,
-                total_size,
-                CompactLogs::NoS3,
-                &worker_name,
-            )
-            .await;
-        }
-
-        #[cfg(not(all(feature = "enterprise", feature = "parquet")))]
-        {
-            default_disk_log_storage(
-                job_id,
-                &w_id,
-                &db,
-                logs,
-                total_size,
-                CompactLogs::NotEE,
-                &worker_name,
-            )
-            .await;
-        }
-    } else {
-        append_logs(&job_id, w_id, logs, db).await;
-    }
-}
-
-pub const LARGE_LOG_THRESHOLD_SIZE: usize = 9000;
-
-lazy_static::lazy_static! {
-    static ref RE_00: Regex = Regex::new('\u{00}'.to_string().as_str()).unwrap();
-    pub static ref NO_LOGS_AT_ALL: bool = std::env::var("NO_LOGS_AT_ALL").ok().is_some_and(|x| x == "1" || x == "true");
-}
-// as a detail, `BufReader::lines()` removes \n and \r\n from the strings it yields,
-// so this pushes \n to thd destination string in each call
-pub fn append_with_limit(dst: &mut String, src: &str, limit: &mut usize) {
-    if *NO_LOGS_AT_ALL {
-        return;
-    }
-    let src_str;
-    let src = {
-        src_str = RE_00.replace_all(src, "");
-        src_str.as_ref()
-    };
-    if !*CLOUD_HOSTED {
-        dst.push('\n');
-        dst.push_str(&src);
-        return;
-    } else {
-        if *limit > 0 {
-            dst.push('\n');
-        }
-        *limit -= 1;
-    }
-
-    let src_len = src.chars().count();
-    if src_len <= *limit {
-        dst.push_str(&src);
-        *limit -= src_len;
-    } else {
-        let byte_pos = src
-            .char_indices()
-            .skip(*limit)
-            .next()
-            .map(|(byte_pos, _)| byte_pos)
-            .unwrap_or(0);
-        dst.push_str(&src[0..byte_pos]);
-        *limit = 0;
-    }
-}

backend/windmill-worker/src/js_eval.rs

@@ -6,72 +6,55 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
-#[cfg(feature = "deno_core")]
 use std::{
     cell::RefCell,
+    collections::HashMap,
     env,
     io::{self, BufReader},
     rc::Rc,
+    sync::Arc,
 };
 
-use std::{collections::HashMap, sync::Arc};
-
-#[cfg(feature = "deno_core")]
 use deno_ast::ParseParams;
-#[cfg(feature = "deno_core")]
 use deno_core::{
     error::AnyError,
     op2, serde_v8, url,
     v8::{self, IsolateHandle},
     Extension, JsRuntime, OpState, PollEventLoopOptions, RuntimeOptions,
 };
-#[cfg(feature = "deno_core")]
 use deno_fetch::FetchPermissions;
-#[cfg(feature = "deno_core")]
 use deno_net::NetPermissions;
-#[cfg(feature = "deno_core")]
 use deno_tls::{rustls::RootCertStore, rustls_pemfile};
-#[cfg(feature = "deno_core")]
 use deno_web::{BlobStore, TimersPermission};
-#[cfg(feature = "deno_core")]
 use itertools::Itertools;
 use lazy_static::lazy_static;
 use regex::Regex;
 use serde_json::value::RawValue;
 use sqlx::types::Json;
-
-#[cfg(feature = "deno_core")]
 use tokio::{
     sync::{mpsc, oneshot},
     time::timeout,
 };
 use uuid::Uuid;
-
-#[cfg(feature = "deno_core")]
-use windmill_common::error::Error;
-
-use windmill_common::{flow_status::JobResult, DB};
+use windmill_common::{error::Error, flow_status::JobResult, DB};
 use windmill_queue::CanceledBy;
 
-use crate::{common::OccupancyMetrics, AuthedClient};
-
-#[cfg(feature = "deno_core")]
-use crate::{common::unsafe_raw, handle_child::run_future_with_polling_update_job_poller};
+use crate::{
+    common::{run_future_with_polling_update_job_poller, unsafe_raw},
+    AuthedClient,
+};
 
 #[derive(Debug, Clone)]
 pub struct IdContext {
     pub flow_job: Uuid,
-    #[allow(dead_code)]
     pub steps_results: HashMap<String, JobResult>,
     pub previous_id: String,
 }
 
-#[cfg(feature = "deno_core")]
 pub struct ContainerRootCertStoreProvider {
     root_cert_store: RootCertStore,
 }
 
-#[cfg(feature = "deno_core")]
 impl ContainerRootCertStoreProvider {
     fn new() -> ContainerRootCertStoreProvider {
         return ContainerRootCertStoreProvider {
@@ -89,17 +72,14 @@ impl ContainerRootCertStoreProvider {
     }
 }
 
-#[cfg(feature = "deno_core")]
 impl deno_tls::RootCertStoreProvider for ContainerRootCertStoreProvider {
     fn get_or_try_init(&self) -> Result<&RootCertStore, AnyError> {
         Ok(&self.root_cert_store)
     }
 }
 
-#[cfg(feature = "deno_core")]
 pub struct PermissionsContainer;
 
-#[cfg(feature = "deno_core")]
 impl FetchPermissions for PermissionsContainer {
     #[inline(always)]
     fn check_net_url(
@@ -120,7 +100,6 @@ impl FetchPermissions for PermissionsContainer {
     }
 }
 
-#[cfg(feature = "deno_core")]
 impl TimersPermission for PermissionsContainer {
     #[inline(always)]
     fn allow_hrtime(&mut self) -> bool {
@@ -128,7 +107,6 @@ impl TimersPermission for PermissionsContainer {
     }
 }
 
-#[cfg(feature = "deno_core")]
 impl NetPermissions for PermissionsContainer {
     fn check_read(
         &mut self,
@@ -155,7 +133,6 @@ impl NetPermissions for PermissionsContainer {
     }
 }
 
-#[cfg(feature = "deno_core")]
 pub struct OptAuthedClient(Option<AuthedClient>);
 
 pub async fn eval_timeout(
@@ -164,7 +141,7 @@ pub async fn eval_timeout(
     flow_input: Option<mappable_rc::Marc<HashMap<String, Box<RawValue>>>>,
     authed_client: Option<&AuthedClient>,
     by_id: Option<IdContext>,
-    #[allow(unused_variables)] ctx: Option<Vec<(String, String)>>,
+    ctx: Option<Vec<(String, String)>>,
 ) -> anyhow::Result<Box<RawValue>> {
     let expr = expr.trim().to_string();
 
@@ -234,133 +211,121 @@ pub async fn eval_timeout(
         }
     }
 
-    #[cfg(not(feature = "deno_core"))]
-    {
-        #[allow(unreachable_code)]
-        return todo!();
-    }
+    let expr2 = expr.clone();
+    let (sender, mut receiver) = oneshot::channel::<IsolateHandle>();
+    let has_client = authed_client.is_some();
+    let authed_client = authed_client.cloned();
+    timeout(
+        std::time::Duration::from_millis(10000),
+        tokio::task::spawn_blocking(move || {
+            let mut ops = vec![op_get_context()];
 
-    #[cfg(feature = "deno_core")]
-    {
-        let expr2 = expr.clone();
-        let (sender, mut receiver) = oneshot::channel::<IsolateHandle>();
-        let has_client = authed_client.is_some();
-        let authed_client = authed_client.cloned();
-        return timeout(
-            std::time::Duration::from_millis(10000),
-            tokio::task::spawn_blocking(move || {
-                let mut ops = vec![op_get_context()];
+            if authed_client.is_some() {
+                ops.extend([
+                    // An op for summing an array of numbers
+                    // The op-layer automatically deserializes inputs
+                    // and serializes the returned Result & value
+                    op_variable(),
+                    op_resource(),
+                ])
+            }
 
-                if authed_client.is_some() {
-                    ops.extend([
-                        // An op for summing an array of numbers
-                        // The op-layer automatically deserializes inputs
-                        // and serializes the returned Result & value
-                        op_variable(),
-                        op_resource(),
-                    ])
-                }
+            if by_id.is_some() && authed_client.is_some() {
+                ops.push(op_get_result());
+                ops.push(op_get_id());
+            }
 
-                if by_id.is_some() && authed_client.is_some() {
-                    ops.push(op_get_result());
-                    ops.push(op_get_id());
-                }
-
-                let ext = Extension { name: "js_eval", ops: ops.into(), ..Default::default() };
-                let exts = vec![ext];
-                // Use our snapshot to provision our new runtime
-                let options = RuntimeOptions {
-                    extensions: exts,
-                    //                startup_snapshot: Some(Snapshot::Static(buffer)),
-                    ..Default::default()
-                };
-
-                let mut context_keys = transform_context
-                    .keys()
-                    .filter(|x| expr.contains(&x.to_string()))
-                    .map(|x| x.clone())
-                    .collect_vec();
-
-                if !context_keys.contains(&"previous_result".to_string())
-                    && (p_ids.is_some() && p_ids.as_ref().unwrap().iter().any(|x| expr.contains(x)))
-                    || expr.contains("error")
-                {
-                    // tracing::error!("PREVIOUS_RESULT");
-                    context_keys.push("previous_result".to_string());
-                }
-                let has_flow_input = expr.contains("flow_input");
-                if has_flow_input {
-                    context_keys.push("flow_input".to_string())
-                }
-
-                let mut js_runtime = JsRuntime::new(options);
-                {
-                    let op_state = js_runtime.op_state();
-                    let mut op_state = op_state.borrow_mut();
-                    let mut client = authed_client.clone();
-                    if let Some(client) = client.as_mut() {
-                        client.force_client = Some(
-                            reqwest::ClientBuilder::new()
-                                .user_agent("windmill/beta")
-                                .danger_accept_invalid_certs(
-                                    std::env::var("ACCEPT_INVALID_CERTS").is_ok(),
-                                )
-                                .build()
-                                .unwrap(),
-                        );
-                    }
-                    op_state.put(OptAuthedClient(client));
-                    op_state.put(TransformContext {
-                        flow_input: if has_flow_input { flow_input } else { None },
-                        envs: transform_context
-                            .into_iter()
-                            .filter(|(a, _)| context_keys.contains(a))
-                            .collect(),
-                    })
-                }
-
-                sender
-                    .send(js_runtime.v8_isolate().thread_safe_handle())
-                    .map_err(|_| {
-                        Error::ExecutionErr("impossible to send v8 isolate".to_string())
-                    })?;
-
-                let runtime = tokio::runtime::Builder::new_current_thread()
-                    .enable_all()
-                    .build()?;
-
-                // pretty frail but this it to make the expr more user friendly and not require the user to write await
-                let expr = ["variable", "resource"]
-                    .into_iter()
-                    .fold(expr, replace_with_await);
-
-                let expr = replace_with_await_result(expr);
-
-                let r = runtime.block_on(eval(
-                    &mut js_runtime,
-                    &expr,
-                    context_keys,
-                    by_id,
-                    has_client,
-                    ctx,
-                ))?;
-
-                Ok(r) as anyhow::Result<Box<RawValue>>
-            }),
-        )
-        .await
-        .map_err(|_| {
-            if let Ok(isolate) = receiver.try_recv() {
-                isolate.terminate_execution();
+            let ext = Extension { name: "js_eval", ops: ops.into(), ..Default::default() };
+            let exts = vec![ext];
+            // Use our snapshot to provision our new runtime
+            let options = RuntimeOptions {
+                extensions: exts,
+                //                startup_snapshot: Some(Snapshot::Static(buffer)),
+                ..Default::default()
             };
-            Error::ExecutionErr(format!(
-                "The expression of evaluation `{expr2}` took too long to execute (>10000ms)"
-            ))
-        })??;
-    }
+
+            let mut context_keys = transform_context
+                .keys()
+                .filter(|x| expr.contains(&x.to_string()))
+                .map(|x| x.clone())
+                .collect_vec();
+
+            if !context_keys.contains(&"previous_result".to_string())
+                && (p_ids.is_some() && p_ids.as_ref().unwrap().iter().any(|x| expr.contains(x)))
+                || expr.contains("error")
+            {
+                // tracing::error!("PREVIOUS_RESULT");
+                context_keys.push("previous_result".to_string());
+            }
+            let has_flow_input = expr.contains("flow_input");
+            if has_flow_input {
+                context_keys.push("flow_input".to_string())
+            }
+
+            let mut js_runtime = JsRuntime::new(options);
+            {
+                let op_state = js_runtime.op_state();
+                let mut op_state = op_state.borrow_mut();
+                let mut client = authed_client.clone();
+                if let Some(client) = client.as_mut() {
+                    client.force_client = Some(
+                        reqwest::ClientBuilder::new()
+                            .user_agent("windmill/beta")
+                            .danger_accept_invalid_certs(
+                                std::env::var("ACCEPT_INVALID_CERTS").is_ok(),
+                            )
+                            .build()
+                            .unwrap(),
+                    );
+                }
+                op_state.put(OptAuthedClient(client));
+                op_state.put(TransformContext {
+                    flow_input: if has_flow_input { flow_input } else { None },
+                    envs: transform_context
+                        .into_iter()
+                        .filter(|(a, _)| context_keys.contains(a))
+                        .collect(),
+                })
+            }
+
+            sender
+                .send(js_runtime.v8_isolate().thread_safe_handle())
+                .map_err(|_| Error::ExecutionErr("impossible to send v8 isolate".to_string()))?;
+
+            let runtime = tokio::runtime::Builder::new_current_thread()
+                .enable_all()
+                .build()?;
+
+            // pretty frail but this it to make the expr more user friendly and not require the user to write await
+            let expr = ["variable", "resource"]
+                .into_iter()
+                .fold(expr, replace_with_await);
+
+            let expr = replace_with_await_result(expr);
+
+            let r = runtime.block_on(eval(
+                &mut js_runtime,
+                &expr,
+                context_keys,
+                by_id,
+                has_client,
+                ctx,
+            ))?;
+
+            Ok(r) as anyhow::Result<Box<RawValue>>
+        }),
+    )
+    .await
+    .map_err(|_| {
+        if let Ok(isolate) = receiver.try_recv() {
+            isolate.terminate_execution();
+        };
+        Error::ExecutionErr(format!(
+            "The expression of evaluation `{expr2}` took too long to execute (>10000ms)"
+        ))
+    })??
 }
 
-#[cfg(feature = "deno_core")]
 fn replace_with_await(expr: String, fn_name: &str) -> String {
     let sep = format!("{}(", fn_name);
     let mut split = expr.split(&sep);
@@ -379,12 +344,10 @@ lazy_static! {
         Regex::new(r"^(https?)://(([^:@\s]+):([^:@\s]+)@)?([^:@\s]+)(:(\d+))?$").unwrap();
 }
 
-#[cfg(feature = "deno_core")]
 fn replace_with_await_result(expr: String) -> String {
     RE.replace_all(&expr, "(await $r)").to_string()
 }
 
-#[cfg(feature = "deno_core")]
 fn add_closing_bracket(s: &str) -> String {
     let mut s = s.to_string();
     let mut level = 1;
@@ -404,7 +367,6 @@ fn add_closing_bracket(s: &str) -> String {
     s
 }
 
-#[cfg(feature = "deno_core")]
 async fn eval(
     context: &mut JsRuntime,
     expr: &str,
@@ -545,7 +507,6 @@ function get_from_env(name) {{
 // }
 
 // TODO: Can we a) share the api configuration here somehow or b) just implement this natively in deno, via the deno client?
-#[cfg(feature = "deno_core")]
 #[op2(async)]
 #[string]
 async fn op_variable(
@@ -560,7 +521,6 @@ async fn op_variable(
     }
 }
 
-#[cfg(feature = "deno_core")]
 #[op2(async)]
 #[string]
 async fn op_get_result(
@@ -579,7 +539,6 @@ async fn op_get_result(
     }
 }
 
-#[cfg(feature = "deno_core")]
 #[op2(async)]
 #[string]
 async fn op_get_id(
@@ -603,7 +562,6 @@ async fn op_get_id(
     }
 }
 
-#[cfg(feature = "deno_core")]
 #[op2(async)]
 #[string]
 async fn op_resource(
@@ -621,13 +579,11 @@ async fn op_resource(
     }
 }
 
-#[cfg(feature = "deno_core")]
 pub struct TransformContext {
     pub envs: HashMap<String, Arc<Box<RawValue>>>,
     pub flow_input: Option<mappable_rc::Marc<HashMap<String, Box<RawValue>>>>,
 }
 
-#[cfg(feature = "deno_core")]
 #[op2]
 #[string]
 fn op_get_context(op_state: Rc<RefCell<OpState>>, #[string] id: &str) -> String {
@@ -648,7 +604,6 @@ fn op_get_context(op_state: Rc<RefCell<OpState>>, #[string] id: &str) -> String
     }
 }
 
-#[cfg(feature = "deno_core")]
 pub fn transpile_ts(expr: String) -> anyhow::Result<String> {
     let parsed = deno_ast::parse_module(ParseParams {
         specifier: url::Url::parse("file:///eval.ts")?,
@@ -665,30 +620,21 @@ pub fn transpile_ts(expr: String) -> anyhow::Result<String> {
         .text)
 }
 
-#[cfg(not(feature = "deno_core"))]
-pub fn transpile_ts(_expr: String) -> anyhow::Result<String> {
-    Ok("require deno".to_string())
-}
-
-#[cfg(feature = "deno_core")]
 static RUNTIME_SNAPSHOT: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/FETCH_SNAPSHOT.bin"));
 
-#[cfg(feature = "deno_core")]
 pub struct MainArgs {
     args: Vec<Option<Box<RawValue>>>,
 }
 
-#[cfg(feature = "deno_core")]
 pub struct LogString {
     pub s: String,
 }
 
-#[cfg(feature = "deno_core")]
 pub struct NativeAnnotation {
     pub useragent: Option<String>,
     pub proxy: Option<(String, Option<(String, String)>)>,
 }
-#[cfg(feature = "deno_core")]
+
 pub fn get_annotation(inner_content: &str) -> NativeAnnotation {
     let mut res = NativeAnnotation { useragent: None, proxy: None };
 
@@ -708,7 +654,6 @@ pub fn get_annotation(inner_content: &str) -> NativeAnnotation {
     res
 }
 
-#[cfg(feature = "deno_core")]
 fn capture_proxy(s: &str) -> Option<(String, Option<(String, String)>)> {
     RE_PROXY.captures(s).map(|x| {
         (
@@ -729,27 +674,7 @@ fn capture_proxy(s: &str) -> Option<(String, Option<(String, String)>)> {
         )
     })
 }
-#[cfg(not(feature = "deno_core"))]
-pub async fn eval_fetch_timeout(
-    _env_code: String,
-    _ts_expr: String,
-    _js_expr: String,
-    _args: Option<&Json<HashMap<String, Box<RawValue>>>>,
-    _job_id: Uuid,
-    _job_timeout: Option<i32>,
-    _db: &DB,
-    _mem_peak: &mut i32,
-    _canceled_by: &mut Option<CanceledBy>,
-    _worker_name: &str,
-    _w_id: &str,
-    _load_client: bool,
-    _occupation_metrics: &mut OccupancyMetrics,
-) -> anyhow::Result<(Box<RawValue>, String)> {
-    use serde_json::value::to_raw_value;
-    Ok((to_raw_value("require deno_core").unwrap(), "".to_string()))
-}
 
-#[cfg(feature = "deno_core")]
 pub async fn eval_fetch_timeout(
     env_code: String,
     ts_expr: String,
@@ -763,7 +688,6 @@ pub async fn eval_fetch_timeout(
     worker_name: &str,
     w_id: &str,
     load_client: bool,
-    occupation_metrics: &mut OccupancyMetrics,
 ) -> anyhow::Result<(Box<RawValue>, String)> {
     let (sender, mut receiver) = oneshot::channel::<IsolateHandle>();
 
@@ -912,7 +836,6 @@ pub async fn eval_fetch_timeout(
         async { result_f.await? },
         worker_name,
         w_id,
-        &mut Some(occupation_metrics),
     )
     .await
     .map_err(|e| {
@@ -925,10 +848,8 @@ pub async fn eval_fetch_timeout(
     Ok((res, format!("{extra_logs}{logs}")))
 }
 
-#[cfg(feature = "deno_core")]
 const WINDMILL_CLIENT: &str = include_str!("./windmill-client.js");
 
-#[cfg(feature = "deno_core")]
 async fn eval_fetch(
     js_runtime: &mut JsRuntime,
     expr: &str,
@@ -974,7 +895,6 @@ import("file:///eval.ts").then((module) => module.main(...args)).then(JSON.strin
     Ok(unsafe_raw(r.unwrap_or_else(|| "null".to_string())))
 }
 
-#[cfg(feature = "deno_core")]
 #[op2]
 #[serde]
 fn op_get_static_args(op_state: Rc<RefCell<OpState>>) -> Vec<Option<String>> {
@@ -987,7 +907,6 @@ fn op_get_static_args(op_state: Rc<RefCell<OpState>>) -> Vec<Option<String>> {
         .collect_vec()
 }
 
-#[cfg(feature = "deno_core")]
 #[op2(fast)]
 fn op_log(op_state: Rc<RefCell<OpState>>, #[string] log: &str) {
     // tracing::error!("log: |{}|", log);
@@ -998,7 +917,6 @@ fn op_log(op_state: Rc<RefCell<OpState>>, #[string] log: &str) {
         .push_str(log);
 }
 
-#[cfg(feature = "deno_core")]
 #[cfg(test)]
 mod tests {
 

backend/windmill-worker/src/lib.rs

@@ -5,9 +5,7 @@ mod mssql_executor;
 #[cfg(feature = "enterprise")]
 mod snowflake_executor;
 
-mod ansible_executor;
 mod bash_executor;
-
 mod bun_executor;
 pub mod common;
 mod config;
@@ -17,8 +15,6 @@ mod deno_executor;
 mod global_cache;
 mod go_executor;
 mod graphql_executor;
-mod handle_child;
-mod job_logger;
 mod js_eval;
 mod mysql_executor;
 mod pg_executor;
@@ -29,12 +25,10 @@ mod rust_executor;
 mod worker;
 mod worker_flow;
 mod worker_lockfiles;
-
+mod ansible_executor;
 pub use worker::*;
 
 pub use result_processor::handle_job_error;
 
-pub use bun_executor::{
-    get_common_bun_proc_envs, install_bun_lockfile, prebundle_bun_script, prepare_job_dir,
-};
+pub use bun_executor::{get_common_bun_proc_envs, install_bun_lockfile, prepare_job_dir};
 pub use deno_executor::generate_deno_lock;

backend/windmill-worker/src/mssql_executor.rs

@@ -1,5 +1,6 @@
 use base64::{engine::general_purpose, Engine as _};
 use chrono::{DateTime, NaiveDate, NaiveDateTime, NaiveTime, Utc};
+use futures::TryFutureExt;
 use regex::Regex;
 use serde::Deserialize;
 use serde_json::value::RawValue;
@@ -9,13 +10,12 @@ use tokio::net::TcpStream;
 use tokio_util::compat::TokioAsyncWriteCompatExt;
 use uuid::Uuid;
 use windmill_common::error::{self, Error};
-use windmill_common::worker::{get_sql_annotations, to_raw_value};
+use windmill_common::worker::to_raw_value;
 use windmill_common::{error::to_anyhow, jobs::QueuedJob};
 use windmill_parser_sql::{parse_db_resource, parse_mssql_sig};
 use windmill_queue::{append_logs, CanceledBy};
 
-use crate::common::{build_args_values, OccupancyMetrics};
-use crate::handle_child::run_future_with_polling_update_job_poller;
+use crate::common::{build_args_values, run_future_with_polling_update_job_poller};
 use crate::AuthedClientBackgroundTask;
 
 #[derive(Deserialize)]
@@ -40,7 +40,6 @@ pub async fn do_mssql(
     mem_peak: &mut i32,
     canceled_by: &mut Option<CanceledBy>,
     worker_name: &str,
-    occupancy_metrics: &mut OccupancyMetrics,
 ) -> error::Result<Box<RawValue>> {
     let mssql_args = build_args_values(job, client, db).await?;
 
@@ -68,8 +67,6 @@ pub async fn do_mssql(
         return Err(Error::BadRequest("Missing database argument".to_string()));
     };
 
-    let annotations = get_sql_annotations(query);
-
     let mut config = Config::new();
 
     config.host(database.host);
@@ -127,45 +124,37 @@ pub async fn do_mssql(
         // polled to the end before querying again. Using streams allows
         // fetching data in an asynchronous manner, if needed.
         let stream = prepared_query.query(&mut client).await.map_err(to_anyhow)?;
-
-        let results = stream.into_results().await.map_err(to_anyhow)?;
-        let len = results.len();
-        let mut json_results = vec![];
-        for (i, statement_result) in results.into_iter().enumerate() {
-            if annotations.return_last_result && i < len - 1 {
-                continue;
-            }
-            let mut json_rows = vec![];
-            for row in statement_result {
-                let row = row_to_json(row)?;
-                json_rows.push(row);
-            }
-            json_results.push(json_rows);
-        }
-
-        if annotations.return_last_result && json_results.len() > 0 {
-            Ok(to_raw_value(&json_results.pop().unwrap()))
-        } else {
-            Ok(to_raw_value(&json_results))
-        }
+        stream
+            .into_results()
+            .await
+            .map_err(to_anyhow)?
+            .into_iter()
+            .map(|rows| {
+                let result = rows
+                    .into_iter()
+                    .map(|row| row_to_json(row))
+                    .collect::<Result<Vec<Map<String, Value>>, Error>>();
+                result
+            })
+            .collect::<Result<Vec<Vec<Map<String, Value>>>, Error>>()
     };
 
-    let raw_result = run_future_with_polling_update_job_poller(
+    let rows = run_future_with_polling_update_job_poller(
         job.id,
         job.timeout,
         db,
         mem_peak,
         canceled_by,
-        result_f,
+        result_f.map_err(to_anyhow),
         worker_name,
         &job.workspace_id,
-        &mut Some(occupancy_metrics),
     )
     .await?;
 
-    *mem_peak = (raw_result.get().len() / 1000) as i32;
+    let r = to_raw_value(&rows);
+    *mem_peak = (r.get().len() / 1000) as i32;
 
-    Ok(raw_result)
+    return Ok(to_raw_value(&rows));
 }
 
 fn json_value_to_sql<'a>(
@@ -232,7 +221,7 @@ fn json_value_to_sql<'a>(
     Ok(())
 }
 
-fn row_to_json(row: Row) -> Result<Value, Error> {
+fn row_to_json(row: Row) -> Result<Map<String, Value>, Error> {
     let cols = row
         .columns()
         .iter()
@@ -242,7 +231,7 @@ fn row_to_json(row: Row) -> Result<Value, Error> {
     for (col, val) in cols.iter().zip(row.into_iter()) {
         map.insert(col.name().to_string(), sql_to_json_value(val)?);
     }
-    Ok(Value::Object(map))
+    Ok(map)
 }
 
 fn value_or_null<T>(

backend/windmill-worker/src/mysql_executor.rs

@@ -13,7 +13,6 @@ use tokio::sync::Mutex;
 use windmill_common::{
     error::{to_anyhow, Error},
     jobs::QueuedJob,
-    worker::{get_sql_annotations, to_raw_value},
 };
 use windmill_parser_sql::{
     parse_db_resource, parse_mysql_sig, parse_sql_blocks, parse_sql_statement_named_params,
@@ -22,8 +21,7 @@ use windmill_parser_sql::{
 use windmill_queue::CanceledBy;
 
 use crate::{
-    common::{build_args_map, OccupancyMetrics},
-    handle_child::run_future_with_polling_update_job_poller,
+    common::{build_args_map, run_future_with_polling_update_job_poller},
     AuthedClientBackgroundTask,
 };
 
@@ -42,8 +40,7 @@ pub fn do_mysql_inner<'a>(
     all_statement_values: &Params,
     conn: Arc<Mutex<mysql_async::Conn>>,
     column_order: Option<&'a mut Option<Vec<String>>>,
-    skip_collect: bool,
-) -> windmill_common::error::Result<BoxFuture<'a, anyhow::Result<Box<RawValue>>>> {
+) -> windmill_common::error::Result<BoxFuture<'a, anyhow::Result<Vec<Value>>>> {
     let param_names = parse_sql_statement_named_params(query, ':')
         .into_iter()
         .map(|x| x.into_bytes())
@@ -61,42 +58,31 @@ pub fn do_mysql_inner<'a>(
     };
 
     let result_f = async move {
-        if skip_collect {
-            conn.lock()
-                .await
-                .exec_drop(query, statement_values)
-                .await
-                .map_err(to_anyhow)?;
+        let rows: Vec<Row> = conn
+            .lock()
+            .await
+            .exec(query, statement_values)
+            .await
+            .map_err(to_anyhow)?;
 
-            Ok(to_raw_value(&Value::Array(vec![])))
-        } else {
-            let rows: Vec<Row> = conn
-                .lock()
-                .await
-                .exec(query, statement_values)
-                .await
-                .map_err(to_anyhow)?;
-
-            if let Some(column_order) = column_order {
-                *column_order = Some(
-                    rows.first()
-                        .map(|x| {
-                            x.columns()
-                                .iter()
-                                .map(|x| x.name_str().to_string())
-                                .collect::<Vec<String>>()
-                        })
-                        .unwrap_or_default(),
-                );
-            }
-
-            Ok(to_raw_value(
-                &rows
-                    .into_iter()
-                    .map(|x| convert_row_to_value(x))
-                    .collect::<Vec<serde_json::Value>>(),
-            ))
+        if let Some(column_order) = column_order {
+            *column_order = Some(
+                rows.first()
+                    .map(|x| {
+                        x.columns()
+                            .iter()
+                            .map(|x| x.name_str().to_string())
+                            .collect::<Vec<String>>()
+                    })
+                    .unwrap_or_default(),
+            );
         }
+
+        Ok(rows
+            .into_iter()
+            .map(|x| convert_row_to_value(x))
+            .collect::<Vec<serde_json::Value>>())
+            as Result<Vec<serde_json::Value>, anyhow::Error>
     };
 
     Ok(result_f.boxed())
@@ -111,7 +97,6 @@ pub async fn do_mysql(
     canceled_by: &mut Option<CanceledBy>,
     worker_name: &str,
     column_order: &mut Option<Vec<String>>,
-    occupancy_metrics: &mut OccupancyMetrics,
 ) -> windmill_common::error::Result<Box<RawValue>> {
     let args = build_args_map(job, client, db).await?.map(Json);
     let job_args = if args.is_some() {
@@ -148,8 +133,6 @@ pub async fn do_mysql(
         return Err(Error::BadRequest("Missing database argument".to_string()));
     };
 
-    let annotations = get_sql_annotations(query);
-
     let opts = OptsBuilder::default()
         .db_name(Some(database.database))
         .user(database.user)
@@ -252,40 +235,21 @@ pub async fn do_mysql(
     let result_f = if queries.len() > 1 {
         let futures = queries
             .iter()
-            .enumerate()
-            .map(|(i, x)| {
-                do_mysql_inner(
-                    x,
-                    &statement_values,
-                    conn_a.clone(),
-                    None,
-                    annotations.return_last_result && i < queries.len() - 1,
-                )
-            })
+            .map(|x| do_mysql_inner(x, &statement_values, conn_a.clone(), None))
             .collect::<windmill_common::error::Result<Vec<_>>>()?;
 
         let f = async {
-            let mut res: Vec<Box<RawValue>> = vec![];
+            let mut res: Vec<serde_json::Value> = vec![];
             for fut in futures {
                 let r = fut.await?;
-                res.push(r);
-            }
-            if annotations.return_last_result && res.len() > 0 {
-                Ok(res.pop().unwrap())
-            } else {
-                Ok(to_raw_value(&res))
+                res.push(serde_json::to_value(r).map_err(to_anyhow)?);
             }
+            Ok(res)
         };
 
         f.boxed()
     } else {
-        do_mysql_inner(
-            query,
-            &statement_values,
-            conn_a.clone(),
-            Some(column_order),
-            false,
-        )?
+        do_mysql_inner(query, &statement_values, conn_a.clone(), Some(column_order))?
     };
 
     let result = run_future_with_polling_update_job_poller(
@@ -297,7 +261,6 @@ pub async fn do_mysql(
         result_f,
         worker_name,
         &job.workspace_id,
-        &mut Some(occupancy_metrics),
     )
     .await?;
 

backend/windmill-worker/src/pg_executor.rs

@@ -30,7 +30,7 @@ use tokio_postgres::{
 };
 use uuid::Uuid;
 use windmill_common::error::{self, Error};
-use windmill_common::worker::{get_sql_annotations, to_raw_value, CLOUD_HOSTED};
+use windmill_common::worker::{to_raw_value, CLOUD_HOSTED};
 use windmill_common::{error::to_anyhow, jobs::QueuedJob};
 use windmill_parser::{Arg, Typ};
 use windmill_parser_sql::{
@@ -38,8 +38,7 @@ use windmill_parser_sql::{
 };
 use windmill_queue::CanceledBy;
 
-use crate::common::{build_args_values, sizeof_val, OccupancyMetrics};
-use crate::handle_child::run_future_with_polling_update_job_poller;
+use crate::common::{build_args_values, run_future_with_polling_update_job_poller, sizeof_val};
 use crate::{AuthedClientBackgroundTask, MAX_RESULT_SIZE};
 use bytes::{Buf, BytesMut};
 use lazy_static::lazy_static;
@@ -69,8 +68,7 @@ fn do_postgresql_inner<'a>(
     client: &'a Client,
     column_order: Option<&'a mut Option<Vec<String>>>,
     siz: &'a AtomicUsize,
-    skip_collect: bool,
-) -> error::Result<BoxFuture<'a, anyhow::Result<Box<RawValue>>>> {
+) -> error::Result<BoxFuture<'a, anyhow::Result<Vec<Value>>>> {
     let mut query_params = vec![];
 
     let arg_indices = parse_pg_statement_arg_indices(&query);
@@ -95,60 +93,51 @@ fn do_postgresql_inner<'a>(
 
     let result_f = async move {
         // Now we can execute a simple statement that just returns its parameter.
+        let rows = client
+            .query_raw(&query, query_params)
+            .await
+            .map_err(to_anyhow)?;
+
+        let rows = rows.try_collect::<Vec<Row>>().await.map_err(to_anyhow)?;
+
+        if let Some(column_order) = column_order {
+            *column_order = Some(
+                rows.first()
+                    .map(|x| {
+                        x.columns()
+                            .iter()
+                            .map(|x| x.name().to_string())
+                            .collect::<Vec<String>>()
+                    })
+                    .unwrap_or_default(),
+            );
+        }
 
         let mut res: Vec<serde_json::Value> = vec![];
-
-        if skip_collect {
-            client
-                .execute_raw(&query, query_params)
-                .await
-                .map_err(to_anyhow)?;
-        } else {
-            let rows = client
-                .query_raw(&query, query_params)
-                .await
-                .map_err(to_anyhow)?;
-
-            let rows = rows.try_collect::<Vec<Row>>().await.map_err(to_anyhow)?;
-
-            if let Some(column_order) = column_order {
-                *column_order = Some(
-                    rows.first()
-                        .map(|x| {
-                            x.columns()
-                                .iter()
-                                .map(|x| x.name().to_string())
-                                .collect::<Vec<String>>()
-                        })
-                        .unwrap_or_default(),
-                );
+        for row in rows.into_iter() {
+            let r = postgres_row_to_json_value(row);
+            if let Ok(v) = r.as_ref() {
+                let size = sizeof_val(v);
+                siz.fetch_add(size, Ordering::Relaxed);
             }
-
-            for row in rows.into_iter() {
-                let r = postgres_row_to_json_value(row);
-                if let Ok(v) = r.as_ref() {
-                    let size = sizeof_val(v);
-                    siz.fetch_add(size, Ordering::Relaxed);
-                }
-                if *CLOUD_HOSTED {
-                    let siz = siz.load(Ordering::Relaxed);
-                    if siz > MAX_RESULT_SIZE * 4 {
-                        return Err(anyhow::anyhow!(
-                            "Query result too large for cloud (size = {} > {})",
-                            siz,
-                            MAX_RESULT_SIZE & 4
-                        ));
-                    }
-                }
-                if let Ok(v) = r {
-                    res.push(v);
-                } else {
-                    return Err(to_anyhow(r.err().unwrap()));
+            if *CLOUD_HOSTED {
+                let siz = siz.load(Ordering::Relaxed);
+                if siz > MAX_RESULT_SIZE * 4 {
+                    return Err(anyhow::anyhow!(
+                        "Query result too large for cloud (size = {} > {})",
+                        siz,
+                        MAX_RESULT_SIZE & 4
+                    ));
                 }
             }
+            if let Ok(v) = r {
+                res.push(v);
+            } else {
+                return Err(to_anyhow(r.err().unwrap()));
+            }
         }
 
-        Ok(to_raw_value(&res))
+        Ok(res)
     };
 
     Ok(result_f.boxed())
@@ -163,7 +152,6 @@ pub async fn do_postgresql(
     canceled_by: &mut Option<CanceledBy>,
     worker_name: &str,
     column_order: &mut Option<Vec<String>>,
-    occupancy_metrics: &mut OccupancyMetrics,
 ) -> error::Result<Box<RawValue>> {
     let pg_args = build_args_values(job, client, db).await?;
 
@@ -190,9 +178,6 @@ pub async fn do_postgresql(
     } else {
         return Err(Error::BadRequest("Missing database argument".to_string()));
     };
-
-    let annotations = get_sql_annotations(query);
-
     let sslmode = match database.sslmode.as_deref() {
         Some("allow") => "prefer".to_string(),
         Some("verify-ca") | Some("verify-full") => "require".to_string(),
@@ -306,30 +291,24 @@ pub async fn do_postgresql(
     let result_f = if queries.len() > 1 {
         let futures = queries
             .iter()
-            .enumerate()
-            .map(|(i, x)| {
+            .map(|x| {
                 do_postgresql_inner(
                     x.to_string(),
                     &param_idx_to_arg_and_value,
                     client,
                     None,
                     &size,
-                    annotations.return_last_result && i < queries.len() - 1,
                 )
             })
             .collect::<error::Result<Vec<_>>>()?;
 
         let f = async {
-            let mut res: Vec<Box<RawValue>> = vec![];
+            let mut res: Vec<serde_json::Value> = vec![];
             for fut in futures {
                 let r = fut.await?;
-                res.push(r);
-            }
-            if annotations.return_last_result && res.len() > 0 {
-                Ok(res.pop().unwrap())
-            } else {
-                Ok(to_raw_value(&res))
+                res.push(serde_json::to_value(r).map_err(to_anyhow)?);
             }
+            Ok(res)
         };
 
         f.boxed()
@@ -340,7 +319,6 @@ pub async fn do_postgresql(
             client,
             Some(column_order),
             &size,
-            false,
         )?
     };
 
@@ -353,7 +331,6 @@ pub async fn do_postgresql(
         result_f,
         worker_name,
         &job.workspace_id,
-        &mut Some(occupancy_metrics),
     )
     .await?;
 

backend/windmill-worker/src/php_executor.rs

@@ -15,10 +15,9 @@ use windmill_queue::{append_logs, CanceledBy};
 
 use crate::{
     common::{
-        create_args_and_out_file, get_main_override, get_reserved_variables, read_result,
-        start_child_process, OccupancyMetrics,
+        create_args_and_out_file, get_main_override, get_reserved_variables, handle_child,
+        read_result, start_child_process,
     },
-    handle_child::handle_child,
     AuthedClientBackgroundTask, COMPOSER_CACHE_DIR, COMPOSER_PATH, DISABLE_NSJAIL, DISABLE_NUSER,
     NSJAIL_PATH, PHP_PATH,
 };
@@ -71,7 +70,6 @@ pub async fn composer_install(
     worker_name: &str,
     requirements: String,
     lock: Option<String>,
-    occupancy_metrics: &mut OccupancyMetrics,
 ) -> Result<String> {
     check_php_exists()?;
 
@@ -103,7 +101,6 @@ pub async fn composer_install(
         "composer install",
         None,
         false,
-        &mut Some(occupancy_metrics),
     )
     .await?;
 
@@ -163,7 +160,6 @@ pub async fn handle_php_job(
     worker_name: &str,
     envs: HashMap<String, String>,
     shared_mount: &str,
-    occupancy_metrics: &mut OccupancyMetrics,
 ) -> error::Result<Box<RawValue>> {
     check_php_exists()?;
 
@@ -194,7 +190,6 @@ pub async fn handle_php_job(
             worker_name,
             composer_json,
             composer_lock,
-            occupancy_metrics,
         )
         .await?;
         "require './vendor/autoload.php';"
@@ -351,7 +346,6 @@ try {{
         "php run",
         job.timeout,
         false,
-        &mut Some(occupancy_metrics),
     )
     .await?;
     read_result(job_dir).await

backend/windmill-worker/src/python_executor.rs

@@ -29,9 +29,6 @@ lazy_static::lazy_static! {
     static ref PYTHON_PATH: String =
     std::env::var("PYTHON_PATH").unwrap_or_else(|_| "/usr/local/bin/python3".to_string());
 
-    static ref UV_PATH: String =
-    std::env::var("UV_PATH").unwrap_or_else(|_| "/usr/local/bin/uv".to_string());
-
     static ref FLOCK_PATH: String =
     std::env::var("FLOCK_PATH").unwrap_or_else(|_| "/usr/bin/flock".to_string());
     static ref NON_ALPHANUM_CHAR: Regex = regex::Regex::new(r"[^0-9A-Za-z=.-]").unwrap();
@@ -39,9 +36,6 @@ lazy_static::lazy_static! {
     static ref PIP_TRUSTED_HOST: Option<String> = std::env::var("PIP_TRUSTED_HOST").ok();
     static ref PIP_INDEX_CERT: Option<String> = std::env::var("PIP_INDEX_CERT").ok();
 
-    static ref USE_PIP_COMPILE: bool = std::env::var("USE_PIP_COMPILE")
-        .ok().map(|flag| flag == "true").unwrap_or(false);
-
 
     static ref RELATIVE_IMPORT_REGEX: Regex = Regex::new(r#"(import|from)\s(((u|f)\.)|\.)"#).unwrap();
 
@@ -61,18 +55,14 @@ use windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS;
 
 use crate::{
     common::{
-        create_args_and_out_file, get_main_override, get_reserved_variables, read_file,
-        read_result, start_child_process, OccupancyMetrics,
+        create_args_and_out_file, get_main_override, get_reserved_variables, handle_child,
+        read_file, read_result, start_child_process,
     },
-    handle_child::handle_child,
     AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, HTTPS_PROXY, HTTP_PROXY,
     LOCK_CACHE_DIR, NO_PROXY, NSJAIL_PATH, PATH_ENV, PIP_CACHE_DIR, PIP_EXTRA_INDEX_URL,
-    PIP_INDEX_URL, TZ_ENV, UV_CACHE_DIR,
+    PIP_INDEX_URL, TZ_ENV,
 };
 
-#[cfg(windows)]
-use crate::SYSTEM_ROOT;
-
 pub async fn create_dependencies_dir(job_dir: &str) {
     DirBuilder::new()
         .recursive(true)
@@ -102,7 +92,7 @@ pub fn handle_ephemeral_token(x: String) -> String {
     x
 }
 
-pub async fn uv_pip_compile(
+pub async fn pip_compile(
     job_id: &Uuid,
     requirements: &str,
     mem_peak: &mut i32,
@@ -111,11 +101,6 @@ pub async fn uv_pip_compile(
     db: &Pool<Postgres>,
     worker_name: &str,
     w_id: &str,
-    occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
-    // Fallback to pip-compile. Will be removed in future
-    mut no_uv: bool,
-    // Debug-only flag
-    no_cache: bool,
 ) -> error::Result<String> {
     let mut logs = String::new();
     logs.push_str(&format!("\nresolving dependencies..."));
@@ -152,184 +137,82 @@ pub async fn uv_pip_compile(
     #[cfg(feature = "enterprise")]
     let requirements = replace_pip_secret(db, w_id, &requirements, worker_name, job_id).await?;
 
-    let mut req_hash = format!("py-{}", calculate_hash(&requirements));
-
-    if no_uv || *USE_PIP_COMPILE {
-        logs.push_str(&format!("\nFallback to pip-compile (Deprecated!)"));
-        // Set no_uv if not setted
-        no_uv = true;
-        // Make sure that if we put #no_uv (switch to pip-compile) to python code or used `USE_PIP_COMPILE=true` variable.
-        // Windmill will recalculate lockfile using pip-compile and dont take potentially broken lockfile (generated by uv) from cache (our db).
-        // It will recalculate lockfile even if inputs have not been changed.
-        req_hash.push_str("-no_uv");
-        // Will be in format:
-        //     py-000..000-no_uv
-    }
-    if !no_cache {
-        if let Some(cached) = sqlx::query_scalar!(
-            "SELECT lockfile FROM pip_resolution_cache WHERE hash = $1",
-            req_hash
-        )
-        .fetch_optional(db)
-        .await?
-        {
-            logs.push_str(&format!("\nfound cached resolution: {req_hash}"));
-            return Ok(cached);
-        }
+    let req_hash = format!("py-{}", calculate_hash(&requirements));
+    if let Some(cached) = sqlx::query_scalar!(
+        "SELECT lockfile FROM pip_resolution_cache WHERE hash = $1",
+        req_hash
+    )
+    .fetch_optional(db)
+    .await?
+    {
+        logs.push_str(&format!("\nfound cached resolution: {req_hash}"));
+        return Ok(cached);
     }
     let file = "requirements.in";
 
     write_file(job_dir, file, &requirements)?;
 
-    // Fallback pip-compile. Will be removed in future
-    if no_uv {
-        tracing::debug!("Fallback to pip-compile");
-
-        let mut args = vec![
-            "-q",
-            "--no-header",
-            file,
-            "--resolver=backtracking",
-            "--strip-extras",
-        ];
-        let mut pip_args = vec![];
-        let pip_extra_index_url = PIP_EXTRA_INDEX_URL
-            .read()
-            .await
-            .clone()
-            .map(handle_ephemeral_token);
-        if let Some(url) = pip_extra_index_url.as_ref() {
-            args.extend(["--extra-index-url", url, "--no-emit-index-url"]);
-            pip_args.push(format!("--extra-index-url {}", url));
-        }
-        let pip_index_url = PIP_INDEX_URL
-            .read()
-            .await
-            .clone()
-            .map(handle_ephemeral_token);
-        if let Some(url) = pip_index_url.as_ref() {
-            args.extend(["--index-url", url, "--no-emit-index-url"]);
-            pip_args.push(format!("--index-url {}", url));
-        }
-        if let Some(host) = PIP_TRUSTED_HOST.as_ref() {
-            args.extend(["--trusted-host", host]);
-        }
-        if let Some(cert_path) = PIP_INDEX_CERT.as_ref() {
-            args.extend(["--cert", cert_path]);
-        }
-        let pip_args_str = pip_args.join(" ");
-        if pip_args.len() > 0 {
-            args.extend(["--pip-args", &pip_args_str]);
-        }
-        tracing::debug!("pip-compile args: {:?}", args);
-
-        let mut child_cmd = Command::new("pip-compile");
-        child_cmd
-            .current_dir(job_dir)
-            .args(args)
-            .stdout(Stdio::piped())
-            .stderr(Stdio::piped());
-        let child_process = start_child_process(child_cmd, "pip-compile").await?;
-        append_logs(&job_id, &w_id, logs, db).await;
-        handle_child(
-            job_id,
-            db,
-            mem_peak,
-            canceled_by,
-            child_process,
-            false,
-            worker_name,
-            &w_id,
-            "pip-compile",
-            None,
-            false,
-            occupancy_metrics,
-        )
+    let mut args = vec![
+        "-q",
+        "--no-header",
+        file,
+        "--resolver=backtracking",
+        "--strip-extras",
+    ];
+    let mut pip_args = vec![];
+    let pip_extra_index_url = PIP_EXTRA_INDEX_URL
+        .read()
         .await
-        .map_err(|e| Error::ExecutionErr(format!("Lock file generation failed: {e:?}")))?;
-    } else {
-        let mut args = vec![
-            "pip",
-            "compile",
-            "-q",
-            "--no-header",
-            file,
-            "--strip-extras",
-            "-o",
-            "requirements.txt",
-            // Prefer main index over extra
-            // https://docs.astral.sh/uv/pip/compatibility/#packages-that-exist-on-multiple-indexes
-            // TODO: Use env variable that can be toggled from UI
-            "--index-strategy",
-            "unsafe-best-match",
-            // Target to /tmp/windmill/cache/uv
-            "--cache-dir",
-            UV_CACHE_DIR,
-            // We dont want UV to manage python installations
-            "--python-preference",
-            "only-system",
-            "--no-python-downloads",
-        ];
-        if no_cache {
-            args.extend(["--no-cache"]);
-        }
-        let pip_extra_index_url = PIP_EXTRA_INDEX_URL
-            .read()
-            .await
-            .clone()
-            .map(handle_ephemeral_token);
-        if let Some(url) = pip_extra_index_url.as_ref() {
-            args.extend(["--extra-index-url", url]);
-        }
-        let pip_index_url = PIP_INDEX_URL
-            .read()
-            .await
-            .clone()
-            .map(handle_ephemeral_token);
-        if let Some(url) = pip_index_url.as_ref() {
-            args.extend(["--index-url", url]);
-        }
-        if let Some(host) = PIP_TRUSTED_HOST.as_ref() {
-            args.extend(["--trusted-host", host]);
-        }
-        if let Some(cert_path) = PIP_INDEX_CERT.as_ref() {
-            args.extend(["--cert", cert_path]);
-        }
-        tracing::debug!("uv args: {:?}", args);
-
-        #[cfg(windows)]
-        let uv_cmd = "uv";
-
-        #[cfg(unix)]
-        let uv_cmd = UV_PATH.as_str();
-
-        let mut child_cmd = Command::new(uv_cmd);
-        child_cmd
-            .current_dir(job_dir)
-            .args(args)
-            .stdout(Stdio::piped())
-            .stderr(Stdio::piped());
-        let child_process = start_child_process(child_cmd, "/usr/local/bin/uv").await?;
-        append_logs(&job_id, &w_id, logs, db).await;
-        handle_child(
-            job_id,
-            db,
-            mem_peak,
-            canceled_by,
-            child_process,
-            false,
-            worker_name,
-            &w_id,
-            // TODO: Rename to uv-pip-compile?
-            "uv",
-            None,
-            false,
-            occupancy_metrics,
-        )
-        .await
-        .map_err(|e| Error::ExecutionErr(format!("Lock file generation failed: {e:?}")))?;
+        .clone()
+        .map(handle_ephemeral_token);
+    if let Some(url) = pip_extra_index_url.as_ref() {
+        args.extend(["--extra-index-url", url, "--no-emit-index-url"]);
+        pip_args.push(format!("--extra-index-url {}", url));
     }
+    let pip_index_url = PIP_INDEX_URL
+        .read()
+        .await
+        .clone()
+        .map(handle_ephemeral_token);
+    if let Some(url) = pip_index_url.as_ref() {
+        args.extend(["--index-url", url, "--no-emit-index-url"]);
+        pip_args.push(format!("--index-url {}", url));
+    }
+    if let Some(host) = PIP_TRUSTED_HOST.as_ref() {
+        args.extend(["--trusted-host", host]);
+    }
+    if let Some(cert_path) = PIP_INDEX_CERT.as_ref() {
+        args.extend(["--cert", cert_path]);
+    }
+    let pip_args_str = pip_args.join(" ");
+    if pip_args.len() > 0 {
+        args.extend(["--pip-args", &pip_args_str]);
+    }
+    tracing::debug!("pip-compile args: {:?}", args);
 
+    let mut child_cmd = Command::new("pip-compile");
+    child_cmd
+        .current_dir(job_dir)
+        .args(args)
+        .stdout(Stdio::piped())
+        .stderr(Stdio::piped());
+    let child_process = start_child_process(child_cmd, "pip-compile").await?;
+    append_logs(&job_id, &w_id, logs, db).await;
+    handle_child(
+        job_id,
+        db,
+        mem_peak,
+        canceled_by,
+        child_process,
+        false,
+        worker_name,
+        &w_id,
+        "pip-compile",
+        None,
+        false,
+    )
+    .await
+    .map_err(|e| Error::ExecutionErr(format!("Lock file generation failed: {e:?}")))?;
     let path_lock = format!("{job_dir}/requirements.txt");
     let mut file = File::open(path_lock).await?;
     let mut req_content = "".to_string();
@@ -364,7 +247,6 @@ pub async fn handle_python_job(
     base_internal_url: &str,
     envs: HashMap<String, String>,
     new_args: &mut Option<HashMap<String, Box<RawValue>>>,
-    occupancy_metrics: &mut OccupancyMetrics,
 ) -> windmill_common::error::Result<Box<RawValue>> {
     let script_path = crate::common::use_flow_root_path(job.script_path());
     let additional_python_paths = handle_python_deps(
@@ -379,7 +261,6 @@ pub async fn handle_python_job(
         worker_dir,
         mem_peak,
         canceled_by,
-        &mut Some(occupancy_metrics),
     )
     .await?;
 
@@ -513,9 +394,6 @@ except BaseException as e:
     let mut reserved_variables = get_reserved_variables(job, &client.token, db).await?;
     let additional_python_paths_folders = additional_python_paths.iter().join(":");
 
-    #[cfg(windows)]
-    let additional_python_paths_folders = additional_python_paths_folders.replace(":", ";");
-
     if !*DISABLE_NSJAIL {
         let shared_deps = additional_python_paths
             .into_iter()
@@ -592,10 +470,6 @@ mount {{
             .args(vec!["-u", "-m", "wrapper"])
             .stdout(Stdio::piped())
             .stderr(Stdio::piped());
-
-        #[cfg(windows)]
-        python_cmd.env("SystemRoot", SYSTEM_ROOT.as_str());
-
         start_child_process(python_cmd, PYTHON_PATH.as_str()).await?
     };
 
@@ -611,7 +485,6 @@ mount {{
         "python run",
         job.timeout,
         false,
-        &mut Some(occupancy_metrics),
     )
     .await?;
 
@@ -880,7 +753,6 @@ async fn handle_python_deps(
     worker_dir: &str,
     mem_peak: &mut i32,
     canceled_by: &mut Option<CanceledBy>,
-    occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
 ) -> error::Result<Vec<String>> {
     create_dependencies_dir(job_dir).await;
 
@@ -895,7 +767,6 @@ async fn handle_python_deps(
     let requirements = match requirements_o {
         Some(r) => r,
         None => {
-            let annotation = windmill_common::worker::get_annotation_python(inner_content);
             let mut already_visited = vec![];
 
             let requirements = windmill_parser_py_imports::parse_python_imports(
@@ -910,7 +781,7 @@ async fn handle_python_deps(
             if requirements.is_empty() {
                 "".to_string()
             } else {
-                uv_pip_compile(
+                pip_compile(
                     job_id,
                     &requirements,
                     mem_peak,
@@ -919,9 +790,6 @@ async fn handle_python_deps(
                     db,
                     worker_name,
                     w_id,
-                    occupancy_metrics,
-                    annotation.no_uv,
-                    annotation.no_cache,
                 )
                 .await
                 .map_err(|e| {
@@ -945,7 +813,6 @@ async fn handle_python_deps(
             worker_name,
             job_dir,
             worker_dir,
-            occupancy_metrics,
         )
         .await?;
         additional_python_paths.append(&mut venv_path);
@@ -967,7 +834,6 @@ pub async fn handle_python_reqs(
     worker_name: &str,
     job_dir: &str,
     worker_dir: &str,
-    occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
 ) -> error::Result<Vec<String>> {
     let mut req_paths: Vec<String> = vec![];
     let mut vars = vec![("PATH", PATH_ENV.as_str())];
@@ -1140,12 +1006,7 @@ pub async fn handle_python_reqs(
             start_child_process(nsjail_cmd, NSJAIL_PATH.as_str()).await?
         } else {
             let fssafe_req = NON_ALPHANUM_CHAR.replace_all(&req, "_").to_string();
-            #[cfg(unix)]
             let req = format!("'{}'", req);
-
-            #[cfg(windows)]
-            let req = format!("{}", req);
-
             let mut command_args = vec![
                 PYTHON_PATH.as_str(),
                 "-m",
@@ -1201,35 +1062,19 @@ pub async fn handle_python_reqs(
 
             tracing::debug!("pip install command: {:?}", command_args);
 
-            #[cfg(unix)]
-            {
-                let mut flock_cmd = Command::new(FLOCK_PATH.as_str());
-                flock_cmd
-                    .env_clear()
-                    .envs(envs)
-                    .args([
-                        "-x",
-                        &format!("{}/pip-{}.lock", LOCK_CACHE_DIR, fssafe_req),
-                        "--command",
-                        &command_args.join(" "),
-                    ])
-                    .stdout(Stdio::piped())
-                    .stderr(Stdio::piped());
-                start_child_process(flock_cmd, FLOCK_PATH.as_str()).await?
-            }
-
-            #[cfg(windows)]
-            {
-                let mut pip_cmd = Command::new(PYTHON_PATH.as_str());
-                pip_cmd
-                    .env_clear()
-                    .envs(envs)
-                    .env("SystemRoot", SYSTEM_ROOT.as_str())
-                    .args(&command_args[1..])
-                    .stdout(Stdio::piped())
-                    .stderr(Stdio::piped());
-                start_child_process(pip_cmd, PYTHON_PATH.as_str()).await?
-            }
+            let mut flock_cmd = Command::new(FLOCK_PATH.as_str());
+            flock_cmd
+                .env_clear()
+                .envs(envs)
+                .args([
+                    "-x",
+                    &format!("{}/pip-{}.lock", LOCK_CACHE_DIR, fssafe_req),
+                    "--command",
+                    &command_args.join(" "),
+                ])
+                .stdout(Stdio::piped())
+                .stderr(Stdio::piped());
+            start_child_process(flock_cmd, FLOCK_PATH.as_str()).await?
         };
 
         let child = handle_child(
@@ -1244,7 +1089,6 @@ pub async fn handle_python_reqs(
             &format!("pip install {req}"),
             None,
             false,
-            occupancy_metrics,
         )
         .await;
         tracing::info!(
@@ -1329,7 +1173,6 @@ pub async fn start_worker(
         job_dir,
         &mut mem_peak,
         &mut canceled_by,
-        &mut None,
     )
     .await?;
 

backend/windmill-worker/src/result_processor.rs

@@ -1,26 +1,16 @@
 use serde::Serialize;
 use sqlx::{types::Json, Pool, Postgres};
-use std::{
-    collections::HashMap,
-    sync::{
-        atomic::{AtomicBool, AtomicU16, Ordering},
-        Arc,
-    },
-};
+use std::{collections::HashMap, sync::Arc};
 
 use uuid::Uuid;
 
 use windmill_common::{
-    add_time,
     error::{self, Error},
-    jobs::{JobKind, QueuedJob},
-    worker::{to_raw_value, WORKER_GROUP},
+    jobs::QueuedJob,
+    worker::to_raw_value,
     DB,
 };
 
-#[cfg(feature = "benchmark")]
-use windmill_common::bench::{BenchmarkInfo, BenchmarkIter};
-
 use windmill_queue::{append_logs, get_queued_job, CanceledBy, WrappedError};
 
 #[cfg(feature = "prometheus")]
@@ -28,13 +18,7 @@ use windmill_queue::register_metric;
 
 use serde_json::{json, value::RawValue};
 
-use tokio::{
-    sync::{
-        self,
-        mpsc::{Receiver, Sender},
-    },
-    task::JoinHandle,
-};
+use tokio::sync::mpsc::Sender;
 
 use windmill_queue::{add_completed_job, add_completed_job_error};
 
@@ -42,149 +26,9 @@ use crate::{
     bash_executor::ANSI_ESCAPE_RE,
     common::{read_result, save_in_cache},
     worker_flow::update_flow_status_after_job_completion,
-    AuthedClient, JobCompleted, JobCompletedSender, SameWorkerSender, SendResult, INIT_SCRIPT_TAG,
+    AuthedClient, Histo, JobCompleted, JobCompletedSender, SameWorkerSender, SendResult,
 };
 
-pub fn start_background_processor<R>(
-    mut job_completed_rx: Receiver<SendResult>,
-    job_completed_sender: Sender<SendResult>,
-    same_worker_queue_size: Arc<AtomicU16>,
-    job_completed_processor_is_done: Arc<AtomicBool>,
-    base_internal_url: String,
-    db: DB,
-    worker_dir: String,
-    same_worker_tx: SameWorkerSender,
-    rsmq: Option<R>,
-    worker_name: String,
-    killpill_tx: sync::broadcast::Sender<()>,
-    is_dedicated_worker: bool,
-) -> JoinHandle<()>
-where
-    R: rsmq_async::RsmqConnection + Send + Sync + Clone + 'static,
-{
-    tokio::spawn(async move {
-        let mut has_been_killed = false;
-
-        #[cfg(feature = "benchmark")]
-        let mut infos = BenchmarkInfo::new();
-
-        //if we have been killed, we want to drain the queue of jobs
-        while let Some(sr) = {
-            if has_been_killed && same_worker_queue_size.load(Ordering::SeqCst) == 0 {
-                job_completed_rx.try_recv().ok()
-            } else {
-                job_completed_rx.recv().await
-            }
-        } {
-            #[cfg(feature = "benchmark")]
-            let mut bench = BenchmarkIter::new();
-
-            match sr {
-                SendResult::JobCompleted(jc) => {
-                    let rsmq = rsmq.clone();
-
-                    let is_init_script_and_failure =
-                        !jc.success && jc.job.tag.as_str() == INIT_SCRIPT_TAG;
-                    let is_dependency_job = matches!(
-                        jc.job.job_kind,
-                        JobKind::Dependencies | JobKind::FlowDependencies
-                    );
-
-                    handle_receive_completed_job(
-                        jc,
-                        &base_internal_url,
-                        &db,
-                        &worker_dir,
-                        &same_worker_tx,
-                        rsmq,
-                        &worker_name,
-                        job_completed_sender.clone(),
-                        #[cfg(feature = "benchmark")]
-                        &mut bench,
-                    )
-                    .await;
-
-                    if is_init_script_and_failure {
-                        tracing::error!("init script errored, exiting");
-                        killpill_tx.send(()).unwrap_or_default();
-                        break;
-                    }
-                    if is_dependency_job && is_dedicated_worker {
-                        tracing::error!("Dedicated worker executed a dependency job, a new script has been deployed. Exiting expecting to be restarted.");
-                        sqlx::query!(
-                                "UPDATE config SET config = config WHERE name = $1",
-                                format!("worker__{}", *WORKER_GROUP)
-                            )
-                            .execute(&db)
-                            .await
-                            .expect("update config to trigger restart of all dedicated workers at that config");
-                        killpill_tx.send(()).unwrap_or_default();
-                    }
-                    add_time!(bench, "job completed processed");
-
-                    #[cfg(feature = "benchmark")]
-                    {
-                        infos.add_iter(bench, true);
-                    }
-                }
-                SendResult::UpdateFlow {
-                    flow,
-                    w_id,
-                    success,
-                    result,
-                    worker_dir,
-                    stop_early_override,
-                    token,
-                } => {
-                    // let r;
-                    tracing::info!(parent_flow = %flow, "updating flow status");
-                    if let Err(e) = update_flow_status_after_job_completion(
-                        &db,
-                        &AuthedClient {
-                            base_internal_url: base_internal_url.to_string(),
-                            workspace: w_id.clone(),
-                            token: token.clone(),
-                            force_client: None,
-                        },
-                        flow,
-                        &Uuid::nil(),
-                        &w_id,
-                        success,
-                        Arc::new(result),
-                        true,
-                        same_worker_tx.clone(),
-                        &worker_dir,
-                        stop_early_override,
-                        rsmq.clone(),
-                        &worker_name,
-                        job_completed_sender.clone(),
-                        #[cfg(feature = "benchmark")]
-                        &mut bench,
-                    )
-                    .await
-                    {
-                        tracing::error!("Error updating flow status after job completion for {flow} on {worker_name}: {e:#}");
-                    }
-                }
-                SendResult::Kill => {
-                    has_been_killed = true;
-                }
-            }
-        }
-
-        job_completed_processor_is_done.store(true, Ordering::SeqCst);
-
-        tracing::info!("finished processing all completed jobs");
-
-        #[cfg(feature = "benchmark")]
-        {
-            infos
-                .write_to_file("profiling_result_processor.json")
-                .expect("write to file profiling");
-        }
-    })
-}
-
 async fn send_job_completed(
     job_completed_tx: JobCompletedSender,
     job: Arc<QueuedJob>,
@@ -324,8 +168,9 @@ pub async fn handle_receive_completed_job<
     same_worker_tx: &SameWorkerSender,
     rsmq: Option<R>,
     worker_name: &str,
+    worker_save_completed_job_duration: Option<Histo>,
+    worker_flow_transition_duration: Option<Histo>,
     job_completed_tx: Sender<SendResult>,
-    #[cfg(feature = "benchmark")] bench: &mut BenchmarkIter,
 ) {
     let token = jc.token.clone();
     let workspace = jc.job.workspace_id.clone();
@@ -346,9 +191,9 @@ pub async fn handle_receive_completed_job<
         same_worker_tx.clone(),
         rsmq.clone(),
         worker_name,
+        worker_save_completed_job_duration,
+        worker_flow_transition_duration,
         job_completed_tx.clone(),
-        #[cfg(feature = "benchmark")]
-        bench,
     )
     .await
     {
@@ -365,8 +210,6 @@ pub async fn handle_receive_completed_job<
             rsmq.clone(),
             worker_name,
             job_completed_tx,
-            #[cfg(feature = "benchmark")]
-            bench,
         )
         .await;
     }
@@ -381,8 +224,9 @@ pub async fn process_completed_job<R: rsmq_async::RsmqConnection + Send + Sync +
     same_worker_tx: SameWorkerSender,
     rsmq: Option<R>,
     worker_name: &str,
+    _worker_save_completed_job_duration: Option<Histo>,
+    _worker_flow_transition_duration: Option<Histo>,
     job_completed_tx: Sender<SendResult>,
-    #[cfg(feature = "benchmark")] bench: &mut BenchmarkIter,
 ) -> windmill_common::error::Result<()> {
     if success {
         // println!("bef completed job{:?}",  SystemTime::now());
@@ -394,7 +238,10 @@ pub async fn process_completed_job<R: rsmq_async::RsmqConnection + Send + Sync +
         let parent_job = job.parent_job.clone();
         let job_id = job.id.clone();
         let workspace_id = job.workspace_id.clone();
-
+        #[cfg(feature = "prometheus")]
+        let timer = _worker_save_completed_job_duration
+            .as_ref()
+            .map(|x| x.start_timer());
         add_completed_job(
             db,
             &job,
@@ -405,16 +252,19 @@ pub async fn process_completed_job<R: rsmq_async::RsmqConnection + Send + Sync +
             canceled_by,
             rsmq.clone(),
             false,
-            #[cfg(feature = "benchmark")]
-            bench,
         )
         .await?;
         drop(job);
 
-        add_time!(bench, "add_completed_job END");
+        #[cfg(feature = "prometheus")]
+        timer.map(|x| x.stop_and_record());
 
         if is_flow_step {
             if let Some(parent_job) = parent_job {
+                #[cfg(feature = "prometheus")]
+                let timer = _worker_flow_transition_duration
+                    .as_ref()
+                    .map(|x| x.start_timer());
                 tracing::info!(parent_flow = %parent_job, subflow = %job_id, "updating flow status (2)");
                 update_flow_status_after_job_completion(
                     db,
@@ -431,13 +281,12 @@ pub async fn process_completed_job<R: rsmq_async::RsmqConnection + Send + Sync +
                     rsmq.clone(),
                     worker_name,
                     job_completed_tx,
-                    #[cfg(feature = "benchmark")]
-                    bench,
                 )
                 .await?;
+                #[cfg(feature = "prometheus")]
+                timer.map(|x| x.stop_and_record());
             }
         }
-        add_time!(bench, "updated flow status END");
     } else {
         let result = add_completed_job_error(
             db,
@@ -450,8 +299,6 @@ pub async fn process_completed_job<R: rsmq_async::RsmqConnection + Send + Sync +
             rsmq.clone(),
             worker_name,
             false,
-            #[cfg(feature = "benchmark")]
-            bench,
         )
         .await?;
         if job.is_flow_step {
@@ -472,8 +319,6 @@ pub async fn process_completed_job<R: rsmq_async::RsmqConnection + Send + Sync +
                     rsmq,
                     worker_name,
                     job_completed_tx,
-                    #[cfg(feature = "benchmark")]
-                    bench,
                 )
                 .await?;
             }
@@ -496,7 +341,6 @@ pub async fn handle_job_error<R: rsmq_async::RsmqConnection + Send + Sync + Clon
     rsmq: Option<R>,
     worker_name: &str,
     job_completed_tx: Sender<SendResult>,
-    #[cfg(feature = "benchmark")] bench: &mut BenchmarkIter,
 ) {
     let err = match err {
         Error::JsonErr(err) => err,
@@ -521,8 +365,6 @@ pub async fn handle_job_error<R: rsmq_async::RsmqConnection + Send + Sync + Clon
             rsmq_2,
             worker_name,
             false,
-            #[cfg(feature = "benchmark")]
-            bench,
         )
         .await
     };
@@ -557,8 +399,6 @@ pub async fn handle_job_error<R: rsmq_async::RsmqConnection + Send + Sync + Clon
             rsmq.clone(),
             worker_name,
             job_completed_tx.clone(),
-            #[cfg(feature = "benchmark")]
-            bench,
         )
         .await;
 
@@ -584,8 +424,6 @@ pub async fn handle_job_error<R: rsmq_async::RsmqConnection + Send + Sync + Clon
                         rsmq,
                         worker_name,
                         false,
-                        #[cfg(feature = "benchmark")]
-                        bench,
                     )
                     .await;
                 }

backend/windmill-worker/src/rust_executor.rs

@@ -15,36 +15,19 @@ use windmill_queue::{append_logs, CanceledBy};
 
 use crate::{
     common::{
-        create_args_and_out_file, get_reserved_variables, read_result, start_child_process,
-        OccupancyMetrics,
+        create_args_and_out_file, get_reserved_variables, handle_child, read_result,
+        start_child_process,
     },
-    handle_child::handle_child,
     AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV,
     RUST_CACHE_DIR, TZ_ENV,
 };
 
-#[cfg(windows)]
-use crate::SYSTEM_ROOT;
-
 const NSJAIL_CONFIG_RUN_RUST_CONTENT: &str = include_str!("../nsjail/run.rust.config.proto");
 
 lazy_static::lazy_static! {
-    static ref HOME_DIR: String = std::env::var("HOME").expect("Could not find the HOME environment variable");
-    static ref CARGO_HOME: String = std::env::var("CARGO_HOME").unwrap_or_else(|_| { CARGO_HOME_DEFAULT.clone() });
-    static ref RUSTUP_HOME: String = std::env::var("RUSTUP_HOME").unwrap_or_else(|_| { RUSTUP_HOME_DEFAULT.clone() });
-    static ref CARGO_PATH: String = format!("{}/bin/cargo", CARGO_HOME.as_str());
-}
-
-#[cfg(windows)]
-lazy_static::lazy_static! {
-    static ref CARGO_HOME_DEFAULT: String = format!("{}\\.cargo", *HOME_DIR);
-    static ref RUSTUP_HOME_DEFAULT: String = format!("{}\\.rustup", *HOME_DIR);
-}
-
-#[cfg(unix)]
-lazy_static::lazy_static! {
-    static ref CARGO_HOME_DEFAULT: String = "/usr/local/cargo".to_string();
-    static ref RUSTUP_HOME_DEFAULT: String = "/usr/local/rustup".to_string();
+    static ref CARGO_HOME: String = std::env::var("CARGO_HOME").unwrap_or_else(|_| "/usr/local/cargo".to_string());
+    static ref RUSTUP_HOME: String = std::env::var("RUSTUP_HOME").unwrap_or_else(|_| "/usr/local/rustup".to_string());
+    static ref CARGO_PATH: String = format!("{}/bin/cargo", std::env::var("CARGO_HOME").unwrap_or("/usr/local/cargo/bin/cargo".to_string()));
 }
 
 const RUST_OBJECT_STORE_PREFIX: &str = "rustbin/";
@@ -130,7 +113,6 @@ pub async fn generate_cargo_lockfile(
     db: &sqlx::Pool<sqlx::Postgres>,
     worker_name: &str,
     w_id: &str,
-    occupancy_metrics: &mut OccupancyMetrics,
 ) -> error::Result<String> {
     check_cargo_exists()?;
 
@@ -142,14 +124,6 @@ pub async fn generate_cargo_lockfile(
         .args(vec!["generate-lockfile"])
         .stdout(Stdio::piped())
         .stderr(Stdio::piped());
-    #[cfg(windows)]
-    {
-        gen_lockfile_cmd.env("SystemRoot", SYSTEM_ROOT.as_str());
-        gen_lockfile_cmd.env(
-            "TMP",
-            std::env::var("TMP").unwrap_or_else(|_| "C:\\tmp".to_string()),
-        );
-    }
     let gen_lockfile_process = start_child_process(gen_lockfile_cmd, CARGO_PATH.as_str()).await?;
     handle_child(
         job_id,
@@ -163,7 +137,6 @@ pub async fn generate_cargo_lockfile(
         "cargo generate-lockfile",
         None,
         false,
-        &mut Some(occupancy_metrics),
     )
     .await?;
 
@@ -184,7 +157,6 @@ pub async fn build_rust_crate(
     w_id: &str,
     base_internal_url: &str,
     hash: &str,
-    occupancy_metrics: &mut OccupancyMetrics,
 ) -> error::Result<String> {
     let bin_path = format!("{}/{hash}", RUST_CACHE_DIR);
 
@@ -200,16 +172,6 @@ pub async fn build_rust_crate(
         .args(vec!["build", "--release"])
         .stdout(Stdio::piped())
         .stderr(Stdio::piped());
-
-    #[cfg(windows)]
-    {
-        build_rust_cmd.env("SystemRoot", SYSTEM_ROOT.as_str());
-        build_rust_cmd.env(
-            "TMP",
-            std::env::var("TMP").unwrap_or_else(|_| "C:\\tmp".to_string()),
-        );
-    }
-
     let build_rust_process = start_child_process(build_rust_cmd, CARGO_PATH.as_str()).await?;
     handle_child(
         job_id,
@@ -223,7 +185,6 @@ pub async fn build_rust_crate(
         "rust build",
         None,
         false,
-        &mut Some(occupancy_metrics),
     )
     .await?;
     append_logs(job_id, w_id, "\n\n", db).await;
@@ -301,7 +262,6 @@ pub async fn handle_rust_job(
     base_internal_url: &str,
     worker_name: &str,
     envs: HashMap<String, String>,
-    occupancy_metrics: &mut OccupancyMetrics,
 ) -> Result<Box<RawValue>, Error> {
     check_cargo_exists()?;
 
@@ -313,13 +273,7 @@ pub async fn handle_rust_job(
 
     let cache_logs = if cache {
         let target = format!("{job_dir}/main");
-
-        #[cfg(unix)]
-        let symlink = std::os::unix::fs::symlink(&bin_path, &target);
-        #[cfg(windows)]
-        let symlink = std::os::windows::fs::symlink_dir(&bin_path, &target);
-
-        symlink.map_err(|e| {
+        std::os::unix::fs::symlink(&bin_path, &target).map_err(|e| {
             Error::ExecutionErr(format!(
                 "could not copy cached binary from {bin_path} to {job_dir}/main: {e:?}"
             ))
@@ -351,7 +305,6 @@ pub async fn handle_rust_job(
             &job.workspace_id,
             base_internal_url,
             &hash,
-            occupancy_metrics,
         )
         .await?
     };
@@ -400,9 +353,6 @@ pub async fn handle_rust_job(
             .stdout(Stdio::piped())
             .stderr(Stdio::piped());
 
-        #[cfg(windows)]
-        run_rust.env("SystemRoot", SYSTEM_ROOT.as_str());
-
         start_child_process(run_rust, compiled_executable_name).await?
     };
     handle_child(
@@ -417,7 +367,6 @@ pub async fn handle_rust_job(
         "rust run",
         job.timeout,
         false,
-        &mut Some(occupancy_metrics),
     )
     .await?;
     read_result(job_dir).await

backend/windmill-worker/src/snowflake_executor.rs

@@ -9,7 +9,6 @@ use serde_json::{json, value::RawValue, Value};
 use sha2::{Digest, Sha256};
 use std::collections::HashMap;
 use windmill_common::error::to_anyhow;
-use windmill_common::worker::get_sql_annotations;
 
 use windmill_common::jobs::QueuedJob;
 use windmill_common::{error::Error, worker::to_raw_value};
@@ -18,8 +17,7 @@ use windmill_queue::{CanceledBy, HTTP_CLIENT};
 
 use serde::{Deserialize, Serialize};
 
-use crate::common::{resolve_job_timeout, OccupancyMetrics};
-use crate::handle_child::run_future_with_polling_update_job_poller;
+use crate::common::{resolve_job_timeout, run_future_with_polling_update_job_poller};
 use crate::{common::build_args_values, AuthedClientBackgroundTask};
 
 #[derive(Serialize)]
@@ -76,41 +74,34 @@ struct SnowflakeError {
 }
 
 trait SnowflakeResponseExt {
-    async fn parse_snowflake_response<T: for<'a> Deserialize<'a>>(
+    async fn get_snowflake_response<T: for<'a> Deserialize<'a>>(
         self,
     ) -> windmill_common::error::Result<T>;
 }
 
-async fn handle_snowflake_result(
-    result: Result<Response, reqwest::Error>,
-) -> windmill_common::error::Result<Response> {
-    match result {
-        Ok(response) => match response.error_for_status_ref() {
-            Ok(_) => Ok(response),
-            Err(e) => {
-                let resp = response.text().await.unwrap_or("".to_string());
-                match serde_json::from_str::<SnowflakeError>(&resp) {
-                    Ok(sf_err) => return Err(Error::ExecutionErr(sf_err.message)),
-                    Err(_) => return Err(Error::ExecutionErr(e.to_string())),
-                }
-            }
-        },
-        Err(e) => Err(Error::ExecutionErr(format!(
-            "Could not send request: {:?}",
-            e
-        ))),
-    }
-}
-
 impl SnowflakeResponseExt for Result<Response, reqwest::Error> {
-    async fn parse_snowflake_response<T: for<'a> Deserialize<'a>>(
+    async fn get_snowflake_response<T: for<'a> Deserialize<'a>>(
         self,
     ) -> windmill_common::error::Result<T> {
-        let response = handle_snowflake_result(self).await?;
-        response
-            .json::<T>()
-            .await
-            .map_err(|e| Error::ExecutionErr(e.to_string()))
+        match self {
+            Ok(response) => match response.error_for_status_ref() {
+                Ok(_) => response
+                    .json::<T>()
+                    .await
+                    .map_err(|e| Error::ExecutionErr(e.to_string())),
+                Err(e) => {
+                    let resp = response.text().await.unwrap_or("".to_string());
+                    match serde_json::from_str::<SnowflakeError>(&resp) {
+                        Ok(sf_err) => return Err(Error::ExecutionErr(sf_err.message)),
+                        Err(_) => return Err(Error::ExecutionErr(e.to_string())),
+                    }
+                }
+            },
+            Err(e) => Err(Error::ExecutionErr(format!(
+                "Could not send request: {:?}",
+                e
+            ))),
+        }
     }
 }
 
@@ -121,7 +112,6 @@ fn do_snowflake_inner<'a>(
     account_identifier: &'a str,
     token: &'a str,
     column_order: Option<&'a mut Option<Vec<String>>>,
-    skip_collect: bool,
 ) -> windmill_common::error::Result<BoxFuture<'a, windmill_common::error::Result<Box<RawValue>>>> {
     body.insert("statement".to_string(), json!(query));
 
@@ -145,7 +135,7 @@ fn do_snowflake_inner<'a>(
     }
 
     let result_f = async move {
-        let result = HTTP_CLIENT
+        let response = HTTP_CLIENT
             .post(format!(
                 "https://{}.snowflakecomputing.com/api/v2/statements/",
                 account_identifier.to_uppercase()
@@ -154,76 +144,67 @@ fn do_snowflake_inner<'a>(
             .header("X-Snowflake-Authorization-Token-Type", "KEYPAIR_JWT")
             .json(&body)
             .send()
-            .await;
+            .await
+            .get_snowflake_response::<SnowflakeResponse>()
+            .await?;
 
-        if skip_collect {
-            handle_snowflake_result(result).await?;
-            Ok(to_raw_value(&Value::Array(vec![])))
-        } else {
-            let response = result
-                .parse_snowflake_response::<SnowflakeResponse>()
-                .await?;
-
-            if response.resultSetMetaData.numRows > 10000 {
-                return Err(Error::ExecutionErr(
-                    "More than 10000 rows were requested, use LIMIT 10000 to limit the number of rows"
-                        .to_string(),
-                ));
-            }
-            if let Some(column_order) = column_order {
-                *column_order = Some(
-                    response
-                        .resultSetMetaData
-                        .rowType
-                        .iter()
-                        .map(|x| x.name.clone())
-                        .collect::<Vec<String>>(),
-                );
-            }
-
-            let mut rows = response.data;
-
-            if response.resultSetMetaData.partitionInfo.len() > 1 {
-                for idx in 1..response.resultSetMetaData.partitionInfo.len() {
-                    let url = format!(
-                        "https://{}.snowflakecomputing.com/api/v2/statements/{}",
-                        account_identifier.to_uppercase(),
-                        response.statementHandle
-                    );
-                    let response = HTTP_CLIENT
-                        .get(url)
-                        .bearer_auth(token)
-                        .header("X-Snowflake-Authorization-Token-Type", "KEYPAIR_JWT")
-                        .query(&[("partition", idx.to_string())])
-                        .send()
-                        .await
-                        .parse_snowflake_response::<SnowflakeDataOnlyResponse>()
-                        .await?;
-
-                    rows.extend(response.data);
-                }
-            }
-
-            let rows = to_raw_value(
-                &rows
-                    .iter()
-                    .map(|row| {
-                        let mut row_map = serde_json::Map::new();
-                        row.iter()
-                            .zip(response.resultSetMetaData.rowType.iter())
-                            .for_each(|(val, row_type)| {
-                                row_map.insert(
-                                    row_type.name.clone(),
-                                    parse_val(&val, &row_type.r#type),
-                                );
-                            });
-                        row_map
-                    })
-                    .collect::<Vec<_>>(),
-            );
-
-            Ok(rows)
+        if response.resultSetMetaData.numRows > 10000 {
+            return Err(Error::ExecutionErr(
+                "More than 10000 rows were requested, use LIMIT 10000 to limit the number of rows"
+                    .to_string(),
+            ));
         }
+        if let Some(column_order) = column_order {
+            *column_order = Some(
+                response
+                    .resultSetMetaData
+                    .rowType
+                    .iter()
+                    .map(|x| x.name.clone())
+                    .collect::<Vec<String>>(),
+            );
+        }
+
+        let mut rows = response.data;
+
+        if response.resultSetMetaData.partitionInfo.len() > 1 {
+            for idx in 1..response.resultSetMetaData.partitionInfo.len() {
+                let url = format!(
+                    "https://{}.snowflakecomputing.com/api/v2/statements/{}",
+                    account_identifier.to_uppercase(),
+                    response.statementHandle
+                );
+                let response = HTTP_CLIENT
+                    .get(url)
+                    .bearer_auth(token)
+                    .header("X-Snowflake-Authorization-Token-Type", "KEYPAIR_JWT")
+                    .query(&[("partition", idx.to_string())])
+                    .send()
+                    .await
+                    .get_snowflake_response::<SnowflakeDataOnlyResponse>()
+                    .await?;
+
+                rows.extend(response.data);
+            }
+        }
+
+        let rows = to_raw_value(
+            &rows
+                .iter()
+                .map(|row| {
+                    let mut row_map = serde_json::Map::new();
+                    row.iter()
+                        .zip(response.resultSetMetaData.rowType.iter())
+                        .for_each(|(val, row_type)| {
+                            row_map
+                                .insert(row_type.name.clone(), parse_val(&val, &row_type.r#type));
+                        });
+                    row_map
+                })
+                .collect::<Vec<_>>(),
+        );
+
+        Ok(rows)
     };
 
     Ok(result_f.boxed())
@@ -238,7 +219,6 @@ pub async fn do_snowflake(
     canceled_by: &mut Option<CanceledBy>,
     worker_name: &str,
     column_order: &mut Option<Vec<String>>,
-    occupancy_metrics: &mut OccupancyMetrics,
 ) -> windmill_common::error::Result<Box<RawValue>> {
     let snowflake_args = build_args_values(job, client, db).await?;
 
@@ -266,8 +246,6 @@ pub async fn do_snowflake(
         return Err(Error::BadRequest("Missing database argument".to_string()));
     };
 
-    let annotations = get_sql_annotations(query);
-
     let qualified_username = format!(
         "{}.{}",
         database.account_identifier.split('.').next().unwrap_or(""), // get first part of account identifier
@@ -337,8 +315,7 @@ pub async fn do_snowflake(
     let result_f = if queries.len() > 1 {
         let futures = queries
             .iter()
-            .enumerate()
-            .map(|(i, x)| {
+            .map(|x| {
                 do_snowflake_inner(
                     x,
                     &snowflake_args,
@@ -346,7 +323,6 @@ pub async fn do_snowflake(
                     &database.account_identifier,
                     &token,
                     None,
-                    annotations.return_last_result && i < queries.len() - 1,
                 )
             })
             .collect::<windmill_common::error::Result<Vec<_>>>()?;
@@ -357,11 +333,7 @@ pub async fn do_snowflake(
                 let r = fut.await?;
                 res.push(r);
             }
-            if annotations.return_last_result && res.len() > 0 {
-                Ok(res.pop().unwrap())
-            } else {
-                Ok(to_raw_value(&res))
-            }
+            Ok(to_raw_value(&res))
         };
 
         f.boxed()
@@ -373,7 +345,6 @@ pub async fn do_snowflake(
             &database.account_identifier,
             &token,
             Some(column_order),
-            false,
         )?
     };
     let r = run_future_with_polling_update_job_poller(
@@ -385,7 +356,6 @@ pub async fn do_snowflake(
         result_f.map_err(to_anyhow),
         worker_name,
         &job.workspace_id,
-        &mut Some(occupancy_metrics),
     )
     .await?;
     *mem_peak = (r.get().len() / 1000) as i32;

backend/windmill-worker/src/worker_flow.rs

@@ -29,10 +29,7 @@ use sqlx::FromRow;
 use tokio::sync::mpsc::Sender;
 use tracing::instrument;
 use uuid::Uuid;
-use windmill_common::add_time;
 use windmill_common::auth::JobPerms;
-#[cfg(feature = "benchmark")]
-use windmill_common::bench::BenchmarkIter;
 use windmill_common::db::Authed;
 use windmill_common::flow_status::{
     ApprovalConditions, FlowStatusModuleWParent, Iterator, JobResult,
@@ -53,8 +50,8 @@ use windmill_common::{
 };
 use windmill_queue::schedule::get_schedule_opt;
 use windmill_queue::{
-    add_completed_job, add_completed_job_error, append_logs, handle_maybe_scheduled_job,
-    CanceledBy, PushArgs, PushIsolationLevel, WrappedError,
+    add_completed_job, add_completed_job_error, append_logs, get_queued_job,
+    handle_maybe_scheduled_job, CanceledBy, PushArgs, PushIsolationLevel, WrappedError,
 };
 
 type DB = sqlx::Pool<sqlx::Postgres>;
@@ -79,7 +76,6 @@ pub async fn update_flow_status_after_job_completion<
     rsmq: Option<R>,
     worker_name: &str,
     job_completed_tx: Sender<SendResult>,
-    #[cfg(feature = "benchmark")] bench: &mut BenchmarkIter,
 ) -> error::Result<()> {
     // this is manual tailrecursion because async_recursion blows up the stack
     // todo!();
@@ -101,8 +97,6 @@ pub async fn update_flow_status_after_job_completion<
         rsmq.clone(),
         worker_name,
         job_completed_tx.clone(),
-        #[cfg(feature = "benchmark")]
-        bench,
     )
     .await?;
     while let Some(nrec) = rec {
@@ -123,8 +117,6 @@ pub async fn update_flow_status_after_job_completion<
             rsmq.clone(),
             worker_name,
             job_completed_tx.clone(),
-            #[cfg(feature = "benchmark")]
-            bench,
         )
         .await
         {
@@ -149,8 +141,6 @@ pub async fn update_flow_status_after_job_completion<
                     rsmq.clone(),
                     worker_name,
                     job_completed_tx.clone(),
-                    #[cfg(feature = "benchmark")]
-                    bench,
                 )
                 .await?
             }
@@ -168,7 +158,10 @@ pub struct RecUpdateFlowStatusAfterJobCompletion {
 }
 
 #[derive(FromRow)]
-pub struct RowArgs {
+pub struct SkipIfStopped {
+    pub skip_if_stopped: Option<bool>,
+    pub stop_early_expr: Option<String>,
+    pub continue_on_error: Option<bool>,
     pub args: Option<Json<HashMap<String, Box<RawValue>>>>,
 }
 
@@ -180,7 +173,6 @@ struct RecoveryObject {
 #[derive(sqlx::FromRow, Deserialize)]
 pub struct RowFlowStatus {
     pub flow_status: sqlx::types::Json<Box<serde_json::value::RawValue>>,
-    pub current_module: Option<sqlx::types::Json<Box<serde_json::value::RawValue>>>,
 }
 // #[instrument(level = "trace", skip_all)]
 pub async fn update_flow_status_after_job_completion_internal<
@@ -201,9 +193,7 @@ pub async fn update_flow_status_after_job_completion_internal<
     rsmq: Option<R>,
     worker_name: &str,
     job_completed_tx: Sender<SendResult>,
-    #[cfg(feature = "benchmark")] bench: &mut BenchmarkIter,
 ) -> error::Result<Option<RecUpdateFlowStatusAfterJobCompletion>> {
-    add_time!(bench, "update flow status internal START");
     let (
         should_continue_flow,
         flow_job,
@@ -216,7 +206,7 @@ pub async fn update_flow_status_after_job_completion_internal<
         // tracing::debug!("UPDATE FLOW STATUS: {flow:?} {success} {result:?} {w_id} {depth}");
 
         let old_status_json = sqlx::query_as::<_, RowFlowStatus>(
-            "SELECT flow_status, raw_flow->'modules'->(flow_status->'step')::int as current_module FROM queue WHERE id = $1 AND workspace_id = $2",
+            "SELECT flow_status FROM queue WHERE id = $1 AND workspace_id = $2",
         )
         .bind(flow)
         .bind(w_id)
@@ -235,16 +225,6 @@ pub async fn update_flow_status_after_job_completion_internal<
                 )))
             })?;
 
-        let current_module = if let Some(x) = old_status_json.current_module {
-            Some(serde_json::from_str::<FlowModule>(x.0.get()).or_else(|e| {
-                Err(Error::InternalErr(format!(
-                    "requiring current module to be parsable as FlowModule: {e:?}"
-                )))
-            })?)
-        } else {
-            None
-        };
-
         let module_step = Step::from_i32_and_len(old_status.step, old_status.modules.len());
 
         let module_status = match module_step {
@@ -267,17 +247,9 @@ pub async fn update_flow_status_after_job_completion_internal<
             module_status,
             FlowStatusModule::InProgress { iterator: Some(_), .. }
         ) {
-            let value = current_module
-                .as_ref()
-                .and_then(|x| x.get_value_with_skip_failures().ok());
-            (
-                true,
-                value
-                    .as_ref()
-                    .and_then(|x| x.skip_failures)
-                    .unwrap_or(false),
-                value.as_ref().and_then(|x| x.parallelism),
-            )
+            let (loop_failures, parallelism) =
+                compute_skip_loop_failures_and_parallelism(flow, old_status.step, db).await?;
+            (true, loop_failures.unwrap_or(false), parallelism)
         } else {
             (false, false, None)
         };
@@ -295,43 +267,37 @@ pub async fn update_flow_status_after_job_completion_internal<
             stop_early_override
         {
             //do not stop early if module is a flow step
-            let step = match module_step {
-                Step::PreprocessorStep => None,
-                Step::FailureStep => None,
-                Step::Step(i) => Some(i),
-            };
+            let flow_job = get_queued_job(&flow, w_id, db)
+                .await?
+                .ok_or_else(|| Error::InternalErr(format!("requiring flow to be in the queue")))?;
+            let module = get_module(&flow_job, &module_step);
 
-            let is_flow = if let Some(step) = step {
-                sqlx::query_scalar!(
-                    "SELECT raw_flow->'modules'->($1)->'value'->>'type' = 'flow' FROM queue WHERE id = $2",
-                    step as i32,
-                    &flow
-                )
-                    .fetch_one(db)
-                    .await
-                    .map_err(|e| {
-                        Error::InternalErr(format!("error during retrieval of step's type: {e:#}"))
-                    })?
-                    .unwrap_or(false)
-            } else {
-                false
-            };
-
-            if is_flow {
+            if module.is_some_and(|x| x.is_flow()) {
                 (false, false, false)
             } else {
                 (true, se, false)
             }
         } else if is_failure_step || matches!(module_step, Step::PreprocessorStep) {
             (false, false, false)
-        } else if let Some(current_module) = current_module.as_ref() {
+        } else {
+            let r = sqlx::query_as::<_, SkipIfStopped>(
+            "SELECT 
+                    raw_flow->'modules'->$1::int->'stop_after_if'->>'expr' as stop_early_expr,
+                    (raw_flow->'modules'->$1::int->'stop_after_if'->>'skip_if_stopped')::bool as skip_if_stopped,
+                    (raw_flow->'modules'->$1::int->'continue_on_error')::bool as continue_on_error,
+                    args 
+                FROM queue
+                WHERE id = $2"
+            )
+            .bind(old_status.step)
+            .bind(flow)
+            .fetch_one(db)
+            .await
+            .map_err(|e| Error::InternalErr(format!("retrieval of stop_early_expr from state: {e:#}")))?;
+
             let stop_early = success
                 && !is_branch_all
-                && if let Some(ref expr) = current_module
-                    .stop_after_if
-                    .as_ref()
-                    .map(|x| x.expr.clone())
-                {
+                && if let Some(expr) = r.stop_early_expr.clone() {
                     let all_iters = match &module_status {
                         FlowStatusModule::InProgress { flow_jobs: Some(flow_jobs), .. }
                             if expr.contains("all_iters") =>
@@ -342,22 +308,14 @@ pub async fn update_flow_status_after_job_completion_internal<
                         }
                         _ => None,
                     };
-                    let args = sqlx::query_as::<_, RowArgs>(
-                        "SELECT
-                                        args
-                                    FROM queue
-                                    WHERE id = $2",
-                    )
-                    .bind(old_status.step)
-                    .bind(flow)
-                    .fetch_one(db)
-                    .await
-                    .map_err(|e| {
-                        Error::InternalErr(format!("retrieval of args from state: {e:#}"))
-                    })?;
                     compute_bool_from_expr(
-                        expr.to_string(),
-                        Marc::new(args.args.unwrap_or_default().0),
+                        expr,
+                        Marc::new(
+                            r.args
+                                .map(|x| x.0)
+                                .unwrap_or_else(|| serde_json::from_str("{}").unwrap())
+                                .to_owned(),
+                        ),
                         result.clone(),
                         all_iters,
                         None,
@@ -371,15 +329,9 @@ pub async fn update_flow_status_after_job_completion_internal<
                 };
             (
                 stop_early,
-                current_module
-                    .stop_after_if
-                    .as_ref()
-                    .map(|x| x.skip_if_stopped)
-                    .unwrap_or(false),
-                current_module.continue_on_error.unwrap_or(false),
+                r.skip_if_stopped.unwrap_or(false),
+                r.continue_on_error.unwrap_or(false),
             )
-        } else {
-            (false, false, false)
         };
 
         let skip_branch_failure = match module_status {
@@ -388,23 +340,26 @@ pub async fn update_flow_status_after_job_completion_internal<
                 parallel,
                 ..
             } => compute_skip_branchall_failure(
+                flow,
                 job_id_for_status,
+                old_status.step,
                 *branch,
                 *parallel,
                 db,
-                current_module.as_ref(),
             )
             .await?
             .unwrap_or(false),
             _ => false,
         };
 
+        let mut tx: QueueTransaction<'_, _> = (rsmq.clone(), db.begin().await?).into();
+
         if matches!(module_step, Step::PreprocessorStep) {
             sqlx::query!(
                 "UPDATE queue SET args = (select result FROM completed_job WHERE id = $1) WHERE id = $2",
                 job_id_for_status,
                 flow
-            ).execute(db).await.map_err(|e| {
+            ).execute(&mut tx).await.map_err(|e| {
                 Error::InternalErr(format!("error while updating args in preprocessing step: {e:#}"))
             })?;
 
@@ -412,7 +367,7 @@ pub async fn update_flow_status_after_job_completion_internal<
                 r#"UPDATE completed_job SET args = '{"reason":"PREPROCESSOR_ARGS_ARE_DISCARDED"}'::jsonb WHERE id = $1"#,
                 job_id_for_status
             )
-            .execute(db)
+            .execute(&mut tx)
             .await
             .map_err(|e| {
                 Error::InternalErr(format!(
@@ -421,10 +376,6 @@ pub async fn update_flow_status_after_job_completion_internal<
             })?;
         }
 
-        let mut tx: QueueTransaction<'_, _> = (rsmq.clone(), db.begin().await?).into();
-
-        add_time!(bench, "process module status START");
-
         let (inc_step_counter, new_status) = match module_status {
             FlowStatusModule::InProgress {
                 iterator,
@@ -436,38 +387,25 @@ pub async fn update_flow_status_after_job_completion_internal<
             } if *parallel => {
                 let (nindex, len) = match (iterator, branchall) {
                     (Some(Iterator { itered, .. }), _) => {
-                        let position = if flow_jobs_success.is_some() {
-                            find_flow_job_index(jobs, job_id_for_status)
-                        } else {
-                            None
-                        };
+                        set_success_in_flow_job_success(
+                            flow_jobs_success,
+                            jobs,
+                            job_id_for_status,
+                            &old_status,
+                            flow,
+                            success,
+                            &mut tx,
+                        )
+                        .await?;
 
-                        let nindex = if let Some(position) = position { 
-                            sqlx::query_scalar!(
+                        let nindex = sqlx::query_scalar!(
                             "UPDATE queue
-                            SET flow_status = JSONB_SET(
-                                JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4),
-                                ARRAY['modules', $1::TEXT, 'iterator', 'index'],
-                                ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb
-                            ),
-                            last_ping = NULL
+                            SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'iterator', 'index'], ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb)
                             WHERE id = $2
                             RETURNING (flow_status->'modules'->$1::int->'iterator'->>'index')::int",
                             old_status.step,
-                            flow,
-                            position as i32,
-                            json!(success)
-                        )} else {
-                            sqlx::query_scalar!(
-                                "UPDATE queue
-                                SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'iterator', 'index'], ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb),
-                                last_ping = NULL
-                                WHERE id = $2
-                                RETURNING (flow_status->'modules'->$1::int->'iterator'->>'index')::int",
-                                old_status.step,
-                                flow
-                            )
-                        }
+                            flow
+                        )
                         .fetch_one(&mut tx)
                         .await.map_err(|e| {
                             Error::InternalErr(format!(
@@ -483,35 +421,24 @@ pub async fn update_flow_status_after_job_completion_internal<
                         (nindex, itered.len() as i32)
                     }
                     (_, Some(BranchAllStatus { len, .. })) => {
-                        let position = if flow_jobs_success.is_some() {
-                            find_flow_job_index(jobs, job_id_for_status)
-                        } else {
-                            None
-                        };
-
-                        let nindex = if let Some(position) = position { 
-                            sqlx::query_scalar!(
-                                "UPDATE queue
-                                SET flow_status = JSONB_SET(
-                                JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4),
-                                ARRAY['modules', $1::TEXT, 'branchall', 'branch'], ((flow_status->'modules'->$1::int->'branchall'->>'branch')::int + 1)::text::jsonb),
-                                last_ping = NULL
-                                WHERE id = $2
-                                RETURNING (flow_status->'modules'->$1::int->'branchall'->>'branch')::int",
-                                old_status.step,
-                                flow,
-                                position as i32,
-                                json!(success)
-                            )
-                        } else { sqlx::query_scalar!(
+                        set_success_in_flow_job_success(
+                            flow_jobs_success,
+                            jobs,
+                            job_id_for_status,
+                            &old_status,
+                            flow,
+                            success,
+                            &mut tx,
+                        )
+                        .await?;
+                        let nindex = sqlx::query_scalar!(
                             "UPDATE queue
-                            SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'branchall', 'branch'], ((flow_status->'modules'->$1::int->'branchall'->>'branch')::int + 1)::text::jsonb),
-                            last_ping = NULL
+                            SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'branchall', 'branch'], ((flow_status->'modules'->$1::int->'branchall'->>'branch')::int + 1)::text::jsonb)
                             WHERE id = $2
                             RETURNING (flow_status->'modules'->$1::int->'branchall'->>'branch')::int",
                             old_status.step,
                             flow
-                        )}
+                        )
                         .fetch_one(&mut tx)
                         .await
                         .map_err(|e| {
@@ -561,7 +488,6 @@ pub async fn update_flow_status_after_job_completion_internal<
                             branch_chosen: None,
                             approvers: vec![],
                             failed_retries: vec![],
-                            skipped: false,
                         }
                     } else {
                         success = false;
@@ -591,9 +517,9 @@ pub async fn update_flow_status_after_job_completion_internal<
                     tracing::info!(
                         "parallel iteration {job_id_for_status} of flow {flow} has finished",
                     );
+
                     (true, Some(new_status))
                 } else {
-                    add_time!(bench, "handle parallel flow start");
                     tx.commit().await?;
 
                     if parallelism.is_some() {
@@ -611,6 +537,18 @@ pub async fn update_flow_status_after_job_completion_internal<
                         })?;
                     }
 
+                    sqlx::query!(
+                        "UPDATE queue
+                        SET last_ping = null
+                        WHERE id = $1",
+                        flow
+                    )
+                    .execute(db)
+                    .await
+                    .map_err(|e| {
+                        Error::InternalErr(format!("error while setting last ping to null: {e:#}"))
+                    })?;
+
                     let r = sqlx::query_scalar!(
                         "DELETE FROM parallel_monitor_lock WHERE parent_flow_id = $1 and job_id = $2 RETURNING last_ping",
                         flow,
@@ -624,7 +562,7 @@ pub async fn update_flow_status_after_job_completion_internal<
                             r.unwrap()
                         );
                     }
-                    add_time!(bench, "non final parallel flow finished");
+
                     return Ok(None);
                 }
             }
@@ -699,20 +637,6 @@ pub async fn update_flow_status_after_job_completion_internal<
                     }
                 }
                 if success || (flow_jobs.is_some() && (skip_loop_failures || skip_branch_failure)) {
-                    let is_skipped = if current_module.as_ref().is_some_and(|m| m.skip_if.is_some()) {
-                        sqlx::query_scalar!(
-                            "SELECT job_kind = 'identity' FROM completed_job WHERE id = $1",
-                            job_id_for_status
-                        )
-                        .fetch_one(db)
-                        .await
-                        .map_err(|e| {
-                            Error::InternalErr(format!("error during skip check: {e:#}"))
-                        })?
-                        .unwrap_or(false)
-                    } else {
-                        false
-                    };
                     success = true;
                     (
                         true,
@@ -724,17 +648,29 @@ pub async fn update_flow_status_after_job_completion_internal<
                             branch_chosen,
                             approvers: vec![],
                             failed_retries: old_status.retry.failed_jobs.clone(),
-                            skipped: is_skipped,
                         }),
                     )
                 } else {
                     let inc = if continue_on_error {
-                        let retry = current_module
-                            .as_ref()
-                            .and_then(|x| x.retry.clone())
-                            .unwrap_or_default();
+                        let retry = sqlx::query_scalar!(
+                            "SELECT raw_flow->'modules'->$2::int->'retry' FROM queue WHERE id = $1",
+                            flow,
+                            old_status.step
+                        )
+                        .fetch_optional(&mut tx)
+                        .await
+                        .map_err(|e| {
+                            Error::InternalErr(format!(
+                                "error while getting retry from step: {e:#}"
+                            ))
+                        })?
+                        .flatten();
 
-                        tracing::info!("update flow status on rety: {retry:#?} ");
+                        let retry = retry
+                            .map(|x| serde_json::from_value::<Retry>(x).ok())
+                            .flatten()
+                            .unwrap_or_default();
+                        tracing::info!("update flow status  on rety: {retry:#?} ");
                         next_retry(&retry, &old_status.retry).is_none()
                     } else {
                         false
@@ -866,27 +802,30 @@ pub async fn update_flow_status_after_job_completion_internal<
 
         match &new_status {
             Some(FlowStatusModule::Success { .. }) if is_loop || is_branch_all => {
-                if let Some(ref expr) = current_module
-                    .as_ref()
-                    .and_then(|m| m.stop_after_all_iters_if.as_ref().map(|x| x.expr.clone()))
-                {
-                    let args = sqlx::query_as::<_, RowArgs>(
-                        "SELECT
+                let r_after_all_iters = sqlx::query_as::<_, SkipIfStopped>(
+                    "SELECT
+                            raw_flow->'modules'->$1::int->'stop_after_all_iters_if'->>'expr' as stop_early_expr,
+                            (raw_flow->'modules'->$1::int->'stop_after_all_iters_if'->>'skip_if_stopped')::bool as skip_if_stopped,
+                            NULL as continue_on_error,
                             args
                         FROM queue
-                        WHERE id = $2",
+                        WHERE id = $2"
                     )
                     .bind(old_status.step)
                     .bind(flow)
                     .fetch_one(db)
                     .await
-                    .map_err(|e| {
-                        Error::InternalErr(format!("retrieval of args from state: {e:#}"))
-                    })?;
-
+                    .map_err(|e| Error::InternalErr(format!("retrieval of stop_early_expr from state: {e:#}")))?;
+                if let Some(expr) = r_after_all_iters.stop_early_expr {
                     let should_stop = compute_bool_from_expr(
-                        expr.to_string(),
-                        Marc::new(args.args.unwrap_or_default().0),
+                        expr,
+                        Marc::new(
+                            r_after_all_iters
+                                .args
+                                .map(|x| x.0)
+                                .unwrap_or_else(|| serde_json::from_str("{}").unwrap())
+                                .to_owned(),
+                        ),
                         nresult.clone(),
                         None,
                         None,
@@ -898,14 +837,7 @@ pub async fn update_flow_status_after_job_completion_internal<
 
                     if should_stop {
                         stop_early = should_stop;
-                        skip_if_stop_early = current_module
-                            .as_ref()
-                            .and_then(|m| {
-                                m.stop_after_all_iters_if
-                                    .as_ref()
-                                    .map(|x| x.skip_if_stopped)
-                            })
-                            .unwrap_or(false);
+                        skip_if_stop_early = r_after_all_iters.skip_if_stopped.unwrap_or(false);
                     }
                 }
             }
@@ -930,7 +862,6 @@ pub async fn update_flow_status_after_job_completion_internal<
         let flow_job = get_queued_job_tx(flow, w_id, tx.transaction_mut())
             .await?
             .ok_or_else(|| Error::InternalErr(format!("requiring flow to be in the queue")))?;
-        tx.commit().await?;
 
         let job_root = flow_job
             .root_job
@@ -963,13 +894,14 @@ pub async fn update_flow_status_after_job_completion_internal<
             false
                 if !is_failure_step
                     && !skip_error_handler
-                    && has_failure_module(flow, db).await? =>
+                    && has_failure_module(flow, tx.transaction_mut()).await? =>
             {
                 true
             }
             false => false,
         };
 
+        tx.commit().await?;
         tracing::debug!(id = %flow_job.id, root_id = %job_root, "flow status updated");
 
         (
@@ -1032,8 +964,6 @@ pub async fn update_flow_status_after_job_completion_internal<
                 rsmq.clone(),
                 worker_name,
                 true,
-                #[cfg(feature = "benchmark")]
-                bench,
             )
             .await?;
         } else {
@@ -1064,8 +994,6 @@ pub async fn update_flow_status_after_job_completion_internal<
             let success = success
                 && (!is_failure_step || result_has_recover_true(nresult.clone()))
                 && !skip_error_handler;
-
-            add_time!(bench, "flow status update 1");
             if success {
                 add_completed_job(
                     db,
@@ -1077,8 +1005,6 @@ pub async fn update_flow_status_after_job_completion_internal<
                     None,
                     rsmq.clone(),
                     true,
-                    #[cfg(feature = "benchmark")]
-                    bench,
                 )
                 .await?;
             } else {
@@ -1096,8 +1022,6 @@ pub async fn update_flow_status_after_job_completion_internal<
                     None,
                     rsmq.clone(),
                     true,
-                    #[cfg(feature = "benchmark")]
-                    bench,
                 )
                 .await?;
             }
@@ -1135,8 +1059,6 @@ pub async fn update_flow_status_after_job_completion_internal<
                     rsmq.clone(),
                     worker_name,
                     true,
-                    #[cfg(feature = "benchmark")]
-                    bench,
                 )
                 .await;
                 true
@@ -1174,10 +1096,6 @@ pub async fn update_flow_status_after_job_completion_internal<
     }
 }
 
-fn find_flow_job_index(flow_jobs: &Vec<Uuid>, job_id_for_status: &Uuid) -> Option<usize> {
-    flow_jobs.iter().position(|x| x == job_id_for_status)
-}
-
 async fn set_success_in_flow_job_success<'c, R: rsmq_async::RsmqConnection + Send>(
     flow_jobs_success: &Option<Vec<Option<bool>>>,
     flow_jobs: &Vec<Uuid>,
@@ -1187,8 +1105,10 @@ async fn set_success_in_flow_job_success<'c, R: rsmq_async::RsmqConnection + Sen
     success: bool,
     tx: &mut QueueTransaction<'c, R>,
 ) -> error::Result<()> {
+    let flow_jobs_success = flow_jobs_success.clone();
+
     if flow_jobs_success.is_some() {
-        let position = find_flow_job_index(flow_jobs, job_id_for_status);
+        let position = flow_jobs.iter().position(|x| x == job_id_for_status);
         if let Some(position) = position {
             sqlx::query!(
             "UPDATE queue SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4) WHERE id = $2",
@@ -1252,12 +1172,31 @@ fn get_module(flow_job: &QueuedJob, module_step: &Step) -> Option<FlowModule> {
     }
 }
 
+async fn compute_skip_loop_failures_and_parallelism(
+    flow: Uuid,
+    step: i32,
+    db: &DB,
+) -> Result<(Option<bool>, Option<i32>), Error> {
+    sqlx::query_as(
+        "SELECT (raw_flow->'modules'->$1->'value'->>'skip_failures')::bool, (raw_flow->'modules'->$1->'value'->>'parallelism')::int
+        FROM queue
+        WHERE id = $2",
+    )
+    .bind(step)
+    .bind(flow)
+    .fetch_one(db)
+    .await
+    .map(|(v, n)| (v,n))
+    .map_err(|e| Error::InternalErr(format!("error during retrieval of skip_loop_failures: {e:#}")))
+}
+
 async fn compute_skip_branchall_failure<'c>(
+    flow: Uuid,
     job: &Uuid,
+    step: i32,
     branch: usize,
     parallel: bool,
     db: &DB,
-    flow_module: Option<&FlowModule>,
 ) -> Result<Option<bool>, Error> {
     let branch = if parallel {
         sqlx::query_scalar!("SELECT script_path FROM completed_job WHERE id = $1", job)
@@ -1281,23 +1220,35 @@ async fn compute_skip_branchall_failure<'c>(
     } else {
         branch as i32
     };
-    Ok(flow_module
-        .and_then(|x| x.get_branches_skip_failures().ok())
-        .and_then(|x| {
-            x.branches
-                .get(branch as usize)
-                .map(|x| x.skip_failure.unwrap_or(false))
-        }))
+    sqlx::query_as(
+        "SELECT (raw_flow->'modules'->$1->'value'->'branches'->$2->>'skip_failure')::bool
+        FROM queue
+        WHERE id = $3",
+    )
+    .bind(step)
+    .bind(branch)
+    .bind(flow)
+    .fetch_one(db)
+    .await
+    .map(|(v,)| v)
+    .map_err(|e| {
+        Error::InternalErr(format!(
+            "error during retrieval of skip_loop_failures: {e:#}"
+        ))
+    })
 }
 
-async fn has_failure_module<'c>(flow: Uuid, db: &DB) -> Result<bool, Error> {
+async fn has_failure_module<'c>(
+    flow: Uuid,
+    tx: &mut sqlx::Transaction<'c, sqlx::Postgres>,
+) -> Result<bool, Error> {
     sqlx::query_scalar::<_, Option<bool>>(
         "SELECT raw_flow->'failure_module' != 'null'::jsonb
         FROM queue
         WHERE id = $1",
     )
     .bind(flow)
-    .fetch_one(db)
+    .fetch_one(&mut **tx)
     .await
     .map_err(|e| {
         Error::InternalErr(format!(
@@ -2226,23 +2177,6 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
 
     drop(resume_messages);
 
-    let is_skipped = if let Some(skip_if) = &module.skip_if {
-        let idcontext = get_transform_context(&flow_job, previous_id.as_str(), &status).await?;
-        compute_bool_from_expr(
-            skip_if.expr.to_string(),
-            arc_flow_job_args.clone(),
-            arc_last_job_result.clone(),
-            None,
-            Some(idcontext.clone()),
-            Some(client),
-            Some((resumes.clone(), resume.clone(), approvers.clone())),
-            None,
-        )
-        .await?
-    } else {
-        false
-    };
-
     let args: windmill_common::error::Result<_> =
         if module.mock.is_some() && module.mock.as_ref().unwrap().enabled {
             let mut hm = HashMap::new();
@@ -2282,16 +2216,7 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
             );
             Ok(Marc::new(hm))
         } else {
-            let value = module.get_value();
-            match &value {
-                Ok(_) if matches!(value, Ok(FlowModuleValue::Identity)) || is_skipped => serde_json::from_str(
-                    &serde_json::to_string(&PreviousResult {
-                        previous_result: Some(&arc_last_job_result),
-                    })
-                    .unwrap(),
-                )
-                .map(Marc::new)
-                .map_err(|e| error::Error::InternalErr(format!("identity: {e:#}"))),
+            match &module.get_value() {
                 Ok(
                     FlowModuleValue::Script { input_transforms, .. }
                     | FlowModuleValue::RawScript { input_transforms, .. }
@@ -2312,7 +2237,16 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
                     )
                     .await
                     .map(Marc::new)
-                },
+                }
+                Ok(FlowModuleValue::Identity) => serde_json::from_str(
+                    &serde_json::to_string(&PreviousResult {
+                        previous_result: Some(&arc_last_job_result),
+                    })
+                    .unwrap(),
+                )
+                .map(Marc::new)
+                .map_err(|e| error::Error::InternalErr(format!("identity: {e:#}"))),
+
                 Ok(_) => Ok(arc_flow_job_args.clone()),
                 Err(e) => {
                     return Err(error::Error::InternalErr(format!(
@@ -2338,7 +2272,6 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
         resumes.clone(),
         resume.clone(),
         approvers.clone(),
-        is_skipped,
     )
     .await?;
     tracing::info!(id = %flow_job.id, root_id = %job_root, "next flow transform computed");
@@ -2360,7 +2293,6 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
                 branch_chosen: None,
                 approvers: vec![],
                 failed_retries: vec![],
-                skipped: false,
             }))
             .bind(flow_job.id)
             .execute(db)
@@ -3006,7 +2938,6 @@ async fn compute_next_flow_transform(
     resumes: Arc<Box<RawValue>>,
     resume: Arc<Box<RawValue>>,
     approvers: Arc<Box<RawValue>>,
-    is_skipped: bool,
 ) -> error::Result<NextFlowTransform> {
     if module.mock.is_some() && module.mock.as_ref().unwrap().enabled {
         return Ok(NextFlowTransform::Continue(
@@ -3033,9 +2964,6 @@ async fn compute_next_flow_transform(
     let delete_after_use = module.delete_after_use.unwrap_or(false);
 
     tracing::debug!(id = %flow_job.id, "computing next flow transform for {:?}", &module.value);
-    if is_skipped {
-        return trivial_next_job(JobPayload::Identity);
-    }
     match &module.get_value()? {
         FlowModuleValue::Identity => trivial_next_job(JobPayload::Identity),
         FlowModuleValue::Flow { path, .. } => {
@@ -3529,7 +3457,6 @@ fn is_simple_modules(modules: &Vec<FlowModule>, flow: &FlowValue) -> bool {
         && modules[0].retry.is_none()
         && modules[0].stop_after_if.is_none()
         && modules[0].stop_after_all_iters_if.is_none()
-        && modules[0].skip_if.is_none()
         && (modules[0].mock.is_none() || modules[0].mock.as_ref().is_some_and(|m| !m.enabled))
         && flow.failure_module.is_none();
     is_simple

backend/windmill-worker/src/worker_lockfiles.rs

@@ -12,7 +12,7 @@ use windmill_common::flows::{FlowModule, FlowModuleValue};
 use windmill_common::get_latest_deployed_hash_for_path;
 use windmill_common::jobs::JobPayload;
 use windmill_common::scripts::ScriptHash;
-use windmill_common::worker::{get_annotation_ts, to_raw_value, to_raw_value_owned, write_file};
+use windmill_common::worker::{get_annotation, to_raw_value, to_raw_value_owned, write_file};
 use windmill_common::{
     error::{self, to_anyhow},
     flows::FlowValue,
@@ -25,8 +25,7 @@ use windmill_parser_py_imports::parse_relative_imports;
 use windmill_parser_ts::parse_expr_for_imports;
 use windmill_queue::{append_logs, CanceledBy, PushIsolationLevel};
 
-use crate::common::OccupancyMetrics;
-use crate::python_executor::{create_dependencies_dir, handle_python_reqs, uv_pip_compile};
+use crate::python_executor::{create_dependencies_dir, handle_python_reqs, pip_compile};
 use crate::rust_executor::{build_rust_crate, compute_rust_hash, generate_cargo_lockfile};
 use crate::{
     bun_executor::gen_bun_lockfile,
@@ -213,7 +212,6 @@ pub async fn handle_dependency_job<R: rsmq_async::RsmqConnection + Send + Sync +
     base_internal_url: &str,
     token: &str,
     rsmq: Option<R>,
-    occupancy_metrics: &mut OccupancyMetrics,
 ) -> error::Result<Box<RawValue>> {
     let raw_code = match job.raw_code {
         Some(ref code) => code.to_owned(),
@@ -275,7 +273,6 @@ pub async fn handle_dependency_job<R: rsmq_async::RsmqConnection + Send + Sync +
         script_path,
         raw_deps,
         npm_mode,
-        occupancy_metrics,
     )
     .await;
 
@@ -536,7 +533,6 @@ pub async fn handle_flow_dependency_job<R: rsmq_async::RsmqConnection + Send + S
     base_internal_url: &str,
     token: &str,
     rsmq: Option<R>,
-    occupancy_metrics: &mut OccupancyMetrics,
 ) -> error::Result<Box<serde_json::value::RawValue>> {
     let job_path = job.script_path.clone().ok_or_else(|| {
         error::Error::InternalErr(
@@ -609,7 +605,6 @@ pub async fn handle_flow_dependency_job<R: rsmq_async::RsmqConnection + Send + S
         base_internal_url,
         token,
         &nodes_to_relock,
-        occupancy_metrics,
     )
     .await?;
     let new_flow_value = serde_json::to_value(flow).map_err(to_anyhow)?;
@@ -714,7 +709,6 @@ async fn lock_modules<'c>(
     base_internal_url: &str,
     token: &str,
     locks_to_reload: &Option<Vec<String>>,
-    occupancy_metrics: &mut OccupancyMetrics,
     // (modules to replace old seq (even unmmodified ones), new transaction, modified ids) )
 ) -> Result<(
     Vec<FlowModule>,
@@ -760,7 +754,6 @@ async fn lock_modules<'c>(
                         base_internal_url,
                         token,
                         locks_to_reload,
-                        occupancy_metrics,
                     ))
                     .await?;
                     e.value = FlowModuleValue::ForloopFlow {
@@ -792,7 +785,6 @@ async fn lock_modules<'c>(
                             base_internal_url,
                             token,
                             locks_to_reload,
-                            occupancy_metrics,
                         ))
                         .await?;
                         nmodified_ids.extend(inner_modified_ids);
@@ -817,7 +809,6 @@ async fn lock_modules<'c>(
                         base_internal_url,
                         token,
                         locks_to_reload,
-                        occupancy_metrics,
                     ))
                     .await?;
                     e.value =
@@ -844,7 +835,6 @@ async fn lock_modules<'c>(
                             base_internal_url,
                             token,
                             locks_to_reload,
-                            occupancy_metrics,
                         ))
                         .await?;
                         nmodified_ids.extend(inner_modified_ids);
@@ -866,7 +856,6 @@ async fn lock_modules<'c>(
                         base_internal_url,
                         token,
                         locks_to_reload,
-                        occupancy_metrics,
                     ))
                     .await?;
                     e.value = FlowModuleValue::BranchOne { branches: nbranches, default: ndefault }
@@ -915,7 +904,6 @@ async fn lock_modules<'c>(
             ),
             false,
             None,
-            occupancy_metrics,
         )
         .await;
         //
@@ -953,7 +941,7 @@ async fn lock_modules<'c>(
                 }
 
                 if language == ScriptLang::Bun || language == ScriptLang::Bunnative {
-                    let anns = get_annotation_ts(&content);
+                    let anns = get_annotation(&content);
                     if anns.native_mode && language == ScriptLang::Bun {
                         language = ScriptLang::Bunnative;
                     } else if !anns.native_mode && language == ScriptLang::Bunnative {
@@ -1003,7 +991,7 @@ async fn lock_modules<'c>(
 
 fn skip_creating_new_lock(language: &ScriptLang, content: &str) -> bool {
     if language == &ScriptLang::Bun || language == &ScriptLang::Bunnative {
-        let anns = get_annotation_ts(&content);
+        let anns = get_annotation(&content);
         if anns.native_mode && language == &ScriptLang::Bun {
             return false;
         } else if !anns.native_mode && language == &ScriptLang::Bunnative {
@@ -1026,7 +1014,6 @@ async fn lock_modules_app(
     job_path: &str,
     base_internal_url: &str,
     token: &str,
-    occupancy_metrics: &mut OccupancyMetrics,
 ) -> Result<Value> {
     match value {
         Value::Object(mut m) => {
@@ -1071,13 +1058,12 @@ async fn lock_modules_app(
                                 &format!("{}/app", job.script_path()),
                                 false,
                                 None,
-                                occupancy_metrics,
                             )
                             .await;
                             match new_lock {
                                 Ok(new_lock) => {
                                     append_logs(&job.id, &job.workspace_id, logs, db).await;
-                                    let anns = get_annotation_ts(&content);
+                                    let anns = get_annotation(&content);
                                     let nlang = if anns.native_mode && language == ScriptLang::Bun {
                                         Some(ScriptLang::Bunnative)
                                     } else if !anns.native_mode && language == ScriptLang::Bunnative
@@ -1127,7 +1113,6 @@ async fn lock_modules_app(
                         job_path,
                         base_internal_url,
                         token,
-                        occupancy_metrics,
                     )
                     .await?,
                 );
@@ -1150,7 +1135,6 @@ async fn lock_modules_app(
                         job_path,
                         base_internal_url,
                         token,
-                        occupancy_metrics,
                     )
                     .await?,
                 );
@@ -1172,7 +1156,6 @@ pub async fn handle_app_dependency_job<R: rsmq_async::RsmqConnection + Send + Sy
     base_internal_url: &str,
     token: &str,
     rsmq: Option<R>,
-    occupancy_metrics: &mut OccupancyMetrics,
 ) -> error::Result<()> {
     let job_path = job.script_path.clone().ok_or_else(|| {
         error::Error::InternalErr(
@@ -1202,7 +1185,6 @@ pub async fn handle_app_dependency_job<R: rsmq_async::RsmqConnection + Send + Sy
             &job_path,
             base_internal_url,
             token,
-            occupancy_metrics,
         )
         .await?;
 
@@ -1277,10 +1259,9 @@ async fn python_dep(
     worker_name: &str,
     w_id: &str,
     worker_dir: &str,
-    occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
 ) -> std::result::Result<String, Error> {
     create_dependencies_dir(job_dir).await;
-    let req: std::result::Result<String, Error> = uv_pip_compile(
+    let req: std::result::Result<String, Error> = pip_compile(
         job_id,
         &reqs,
         mem_peak,
@@ -1289,9 +1270,6 @@ async fn python_dep(
         db,
         worker_name,
         w_id,
-        occupancy_metrics,
-        false,
-        false,
     )
     .await;
     // install the dependencies to pre-fill the cache
@@ -1306,7 +1284,6 @@ async fn python_dep(
             worker_name,
             job_dir,
             worker_dir,
-            occupancy_metrics,
         )
         .await;
 
@@ -1336,7 +1313,6 @@ async fn capture_dependency_job(
     script_path: &str,
     raw_deps: bool,
     npm_mode: Option<bool>,
-    occupancy_metrics: &mut OccupancyMetrics,
 ) -> error::Result<String> {
     match job_language {
         ScriptLang::Python3 => {
@@ -1366,7 +1342,6 @@ async fn capture_dependency_job(
                 worker_name,
                 w_id,
                 worker_dir,
-                &mut Some(occupancy_metrics),
             )
             .await
         }
@@ -1389,7 +1364,6 @@ async fn capture_dependency_job(
                 worker_name,
                 w_id,
                 worker_dir,
-                &mut Some(occupancy_metrics),
             )
             .await
         }
@@ -1411,7 +1385,6 @@ async fn capture_dependency_job(
                 false,
                 worker_name,
                 w_id,
-                occupancy_metrics,
             )
             .await
         }
@@ -1431,14 +1404,12 @@ async fn capture_dependency_job(
                 w_id,
                 worker_name,
                 base_internal_url,
-                &mut Some(occupancy_metrics),
             )
             .await
         }
         ScriptLang::Bun | ScriptLang::Bunnative => {
-            let npm_mode = npm_mode.unwrap_or_else(|| {
-                windmill_common::worker::get_annotation_ts(job_raw_code).npm_mode
-            });
+            let npm_mode = npm_mode
+                .unwrap_or_else(|| windmill_common::worker::get_annotation(job_raw_code).npm_mode);
             if !raw_deps {
                 let _ = write_file(job_dir, "main.ts", job_raw_code)?;
             }
@@ -1460,7 +1431,6 @@ async fn capture_dependency_job(
                     None
                 },
                 npm_mode,
-                &mut Some(occupancy_metrics),
             )
             .await?;
             if req.is_some() && !raw_deps {
@@ -1475,7 +1445,6 @@ async fn capture_dependency_job(
                     base_internal_url,
                     worker_name,
                     &token,
-                    &mut Some(occupancy_metrics),
                 )
                 .await?;
             }
@@ -1506,7 +1475,6 @@ async fn capture_dependency_job(
                 worker_name,
                 reqs,
                 None,
-                occupancy_metrics,
             )
             .await
         }
@@ -1526,7 +1494,6 @@ async fn capture_dependency_job(
                 db,
                 worker_name,
                 w_id,
-                occupancy_metrics,
             )
             .await?;
 
@@ -1540,7 +1507,6 @@ async fn capture_dependency_job(
                 w_id,
                 base_internal_url,
                 &compute_rust_hash(&job_raw_code, Some(&lockfile)),
-                occupancy_metrics,
             )
             .await?;
             Ok(lockfile)

benchmarks/lib.ts

@@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts";
 import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts";
 import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts";
 
-export const VERSION = "v1.405.4";
+export const VERSION = "v1.399.0";
 
 export async function login(email: string, password: string): Promise<string> {
   return await windmill.UserService.login({

cli/gen/core/OpenAPI.ts

@@ -54,7 +54,7 @@ export const OpenAPI: OpenAPIConfig = {
 	PASSWORD: undefined,
 	TOKEN: getEnv("WM_TOKEN"),
 	USERNAME: undefined,
-	VERSION: '1.401.0',
+	VERSION: '1.398.1',
 	WITH_CREDENTIALS: true,
 	interceptors: {
 		request: new Interceptors(),