Windmill debugger binary

.github/change-versions-mac.sh

@@ -7,7 +7,7 @@ VERSION=$1
 echo "Updating versions to: $VERSION"
 
 sed -i '' -e "/^version =/s/= .*/= \"$VERSION\"/" ${root_dirpath}/backend/Cargo.toml
-sed -i '' -e "/^export const VERSION =/s/= .*/= \"v$VERSION\";/"  ${root_dirpath}/cli/src/main.ts
+sed -i '' -e "/^export const VERSION =/s/= .*/= \"v$VERSION\";/"  ${root_dirpath}/cli/main.ts
 sed -i '' -e "/^export const VERSION =/s/= .*/= \"v$VERSION\";/" ${root_dirpath}/benchmarks/lib.ts
 sed -i '' -e "/version: /s/: .*/: $VERSION/" ${root_dirpath}/backend/windmill-api/openapi.yaml
 sed -i '' -e "/version: /s/: .*/: $VERSION/" ${root_dirpath}/openflow.openapi.yaml

.github/change-versions.sh

@@ -7,7 +7,7 @@ VERSION=$1
 echo "Updating versions to: $VERSION"
 
 sed -i -e "/^version =/s/= .*/= \"$VERSION\"/" ${root_dirpath}/backend/Cargo.toml
-sed -i -e "/^export const VERSION =/s/= .*/= \"$VERSION\";/" ${root_dirpath}/cli/src/main.ts
+sed -i -e "/^export const VERSION =/s/= .*/= \"$VERSION\";/" ${root_dirpath}/cli/main.ts
 sed -i -e "/^export const VERSION =/s/= .*/= \"v$VERSION\";/" ${root_dirpath}/benchmarks/lib.ts
 sed -i -e "/version: /s/: .*/: $VERSION/" ${root_dirpath}/backend/windmill-api/openapi.yaml
 sed -i -e "/version: /s/: .*/: $VERSION/" ${root_dirpath}/openflow.openapi.yaml

.gitignore

@@ -7,7 +7,7 @@ CaddyfileRemoteMalo
 *.swp
 **/.idea/
 .direnv
-/.vscode
+.vscode
 .dev-docker-wrapper*
 backend/.minio-data
 .aider*

CHANGELOG.md

@@ -1,92 +1,5 @@
 # Changelog
 
-## [1.519.2](https://github.com/windmill-labs/windmill/compare/v1.519.1...v1.519.2) (2025-08-07)
-
-
-### Bug Fixes
-
-* native text response streaming in agent workers ([545d3ce](https://github.com/windmill-labs/windmill/commit/545d3ce74c1edcf588289dc67aa4c365dd48ed10))
-
-## [1.519.1](https://github.com/windmill-labs/windmill/compare/v1.519.0...v1.519.1) (2025-08-06)
-
-
-### Bug Fixes
-
-* fix python-client f formatting error ([fb9d7d4](https://github.com/windmill-labs/windmill/commit/fb9d7d4c95601965a0c00fe07458ea1b77a0fe69))
-
-## [1.519.0](https://github.com/windmill-labs/windmill/compare/v1.518.2...v1.519.0) (2025-08-06)
-
-
-### Features
-
-* add native result streaming ([#6242](https://github.com/windmill-labs/windmill/issues/6242)) ([161dbbc](https://github.com/windmill-labs/windmill/commit/161dbbc6d60ad8b4e7f1ea7afdcae35b2128eeda))
-* Ducklake native support ([#6268](https://github.com/windmill-labs/windmill/issues/6268)) ([e6f1211](https://github.com/windmill-labs/windmill/commit/e6f1211d317ca5c5b33ced324bf914b080d3ce2c))
-* dynamic select in flow ([#6315](https://github.com/windmill-labs/windmill/issues/6315)) ([1bcb043](https://github.com/windmill-labs/windmill/commit/1bcb0431c44be5cc7b99e2131f73f456c1bd91ac))
-* git sync v2 + cli git_branches support ([#6327](https://github.com/windmill-labs/windmill/issues/6327)) ([cb649b2](https://github.com/windmill-labs/windmill/commit/cb649b26893d8829aae0ed8cdf896607a3331804))
-
-
-### Bug Fixes
-
-* **rust-sdk:** revert `openapi-generator-cli` to `7.10.0` ([#6325](https://github.com/windmill-labs/windmill/issues/6325)) ([83aa0d0](https://github.com/windmill-labs/windmill/commit/83aa0d0267fbf2c467c050d21583ddcb49909a8f))
-
-## [1.518.2](https://github.com/windmill-labs/windmill/compare/v1.518.1...v1.518.2) (2025-08-05)
-
-
-### Bug Fixes
-
-* fix edit button for script in flow module ([c351b7b](https://github.com/windmill-labs/windmill/commit/c351b7ba87c68ec7428b7bc4f901b6422c7c2fb3))
-* github apps popup appear for superadmins ([86df8ce](https://github.com/windmill-labs/windmill/commit/86df8ce240d1b69f35a1b7d132e30e815e992b31))
-
-## [1.518.1](https://github.com/windmill-labs/windmill/compare/v1.518.0...v1.518.1) (2025-08-04)
-
-
-### Bug Fixes
-
-* **cli:** revert renaming of inline scripts ([#6320](https://github.com/windmill-labs/windmill/issues/6320)) ([82ac747](https://github.com/windmill-labs/windmill/commit/82ac74779114af837cb3eccdd46962656f06ae08))
-
-## [1.518.0](https://github.com/windmill-labs/windmill/compare/v1.517.0...v1.518.0) (2025-08-01)
-
-
-### Features
-
-* **utils:** add flow.yaml validation function ([#6316](https://github.com/windmill-labs/windmill/issues/6316)) ([4937076](https://github.com/windmill-labs/windmill/commit/493707668b2b723115568e9bb96e7d57b5246ac4))
-
-
-### Bug Fixes
-
-* add disabled support to resource picker in schema forms ([c279154](https://github.com/windmill-labs/windmill/commit/c279154409643da493606da6d401228281966583))
-* add wm_labels to tracing spans ([d45ddec](https://github.com/windmill-labs/windmill/commit/d45ddecf8fdd14fe5ef759b7e836caacf1b78a52))
-* cleanup concurrency_counter automatically + remove orphans keys automatically ([87dd522](https://github.com/windmill-labs/windmill/commit/87dd52296f4bb8a585fffd08a6d13e9b1e4856b8))
-* delete empty git connection ([#6318](https://github.com/windmill-labs/windmill/issues/6318)) ([8d05696](https://github.com/windmill-labs/windmill/commit/8d05696809cae7590ffedee1f30a76efcc8f3c66))
-* sanitize XSS on login error ([523bc20](https://github.com/windmill-labs/windmill/commit/523bc2023b6789fe986532eb78f42db2c94f0c58))
-
-## [1.517.0](https://github.com/windmill-labs/windmill/compare/v1.516.0...v1.517.0) (2025-07-31)
-
-
-### Features
-
-* **cli:** wmill-lock.yaml v2 for easier git merge diffs ([ef3e235](https://github.com/windmill-labs/windmill/commit/ef3e2353a76d096847b3f10b1daa6767fc4baa0d))
-
-
-### Bug Fixes
-
-* use with_capacity back presusre for tantivy directory multipart writes ([#6313](https://github.com/windmill-labs/windmill/issues/6313)) ([8887707](https://github.com/windmill-labs/windmill/commit/8887707d41456889371e471c996773e605088a88))
-
-## [1.516.0](https://github.com/windmill-labs/windmill/compare/v1.515.1...v1.516.0) (2025-07-31)
-
-
-### Features
-
-* add CA certificate update at startup via environment variable ([#6280](https://github.com/windmill-labs/windmill/issues/6280)) ([a460e13](https://github.com/windmill-labs/windmill/commit/a460e131c71a0105fb14812ea2fabaa6bea1e0df))
-* prevent too large results (>500Mb) from OOMing database ([4b9683f](https://github.com/windmill-labs/windmill/commit/4b9683f1462e9c8a577cc9e65a79fcdcd3894da0))
-
-
-### Bug Fixes
-
-* indexer collection of job logs before indexing ([#6300](https://github.com/windmill-labs/windmill/issues/6300)) ([77c8f17](https://github.com/windmill-labs/windmill/commit/77c8f17fdf88821951af36786e28aed9a270d476))
-* no process relative imports for scripts with codebase ([576156b](https://github.com/windmill-labs/windmill/commit/576156b0cc89c8a6ccb94234c59307ab8c41fed4))
-* sqs oidc authentication disconnect [#6307](https://github.com/windmill-labs/windmill/issues/6307) ([993e809](https://github.com/windmill-labs/windmill/commit/993e80955b23098d7075ed5279e3f18cd8a633b9))
-
 ## [1.515.1](https://github.com/windmill-labs/windmill/compare/v1.515.0...v1.515.1) (2025-07-29)
 
 

README.md

@@ -364,8 +364,6 @@ you to have it being synced automatically everyday.
 | DISABLE_RESPONSE_LOGS               | false                  | Disable response logs                                                                                                                                                                              | Server                |
 | CREATE_WORKSPACE_REQUIRE_SUPERADMIN | true                   | If true, only superadmins can create new workspaces                                                                                                                                                | Server                |
 | MIN_FREE_DISK_SPACE_MB              | 15000                  | Minimum amount of free space on worker. Sends critical alert if worker has less free space.                                                                                                        | Worker                |
-| RUN_UPDATE_CA_CERTIFICATE_AT_START  | false                  | If true, runs CA certificate update command at startup before other initialization                                                                                                                  | All                   |
-| RUN_UPDATE_CA_CERTIFICATE_PATH      | /usr/sbin/update-ca-certificates | Path to the CA certificate update command/script to run when RUN_UPDATE_CA_CERTIFICATE_AT_START is true                                                                                             | All                   |
 
 ## Run a local dev setup
 

backend/.sqlx/query-02ecdcc882931d5cbb2243e32805c8a1291a5106fff46ceba85fa27d50a0354c.json

@@ -1,22 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\n            SELECT jsonb_object_keys(ws.ducklake->'ducklakes') AS ducklake_name\n            FROM workspace_settings ws\n            WHERE ws.workspace_id = $1\n        ",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "ducklake_name",
-        "type_info": "Text"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Text"
-      ]
-    },
-    "nullable": [
-      null
-    ]
-  },
-  "hash": "02ecdcc882931d5cbb2243e32805c8a1291a5106fff46ceba85fa27d50a0354c"
-}

backend/.sqlx/query-081dc94a7d0fdaade77cfb593a025d8c48d7eab3dbb30ca0b43fb1ef45d8d8bd.json

@@ -1,25 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "INSERT INTO flow (\n        workspace_id, path, summary, description,\n        dependency_job, lock_error_logs, draft_only, tag,\n        dedicated_worker, visible_to_runner_only, on_behalf_of_email,\n        value, schema, edited_by, edited_at\n    ) VALUES (\n        $1, $2, $3, $4,\n        NULL, '', $5, $6,\n        $7, $8, $9,\n        $10, $11::text::json, $12, now()\n    )",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Varchar",
-        "Varchar",
-        "Text",
-        "Text",
-        "Bool",
-        "Varchar",
-        "Bool",
-        "Bool",
-        "Text",
-        "Jsonb",
-        "Text",
-        "Varchar"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "081dc94a7d0fdaade77cfb593a025d8c48d7eab3dbb30ca0b43fb1ef45d8d8bd"
-}

backend/.sqlx/query-08f288d2781d823e109a9e5b8848234ca7d1efeee9661f3901f298da375e73f7.json

@@ -142,11 +142,6 @@
         "ordinal": 27,
         "name": "git_app_installations",
         "type_info": "Jsonb"
-      },
-      {
-        "ordinal": 28,
-        "name": "ducklake",
-        "type_info": "Jsonb"
       }
     ],
     "parameters": {
@@ -182,8 +177,7 @@
       true,
       true,
       true,
-      false,
-      true
+      false
     ]
   },
   "hash": "08f288d2781d823e109a9e5b8848234ca7d1efeee9661f3901f298da375e73f7"

backend/.sqlx/query-0a1c10bd2232b0770a7816e1bd8d758dc393f797890d597e5996146247f512ac.json

@@ -1,38 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\nWITH lockable_counters AS (\n    SELECT concurrency_id, job_uuids\n    FROM concurrency_counter\n    WHERE job_uuids != '{}'::jsonb\n    FOR UPDATE SKIP LOCKED\n),\nall_job_uuids AS (\n    SELECT DISTINCT jsonb_object_keys(job_uuids) AS job_uuid\n    FROM lockable_counters\n),\norphaned_job_uuids AS (\n    SELECT job_uuid\n    FROM all_job_uuids\n    WHERE job_uuid NOT IN (\n        SELECT id::text \n        FROM v2_job_queue \n        FOR SHARE SKIP LOCKED\n    )\n),\norphaned_array AS (\n    SELECT ARRAY(SELECT job_uuid FROM orphaned_job_uuids) AS orphaned_keys\n),\nbefore_update AS (\n    SELECT lc.concurrency_id, lc.job_uuids, oa.orphaned_keys\n    FROM lockable_counters lc, orphaned_array oa\n    WHERE lc.job_uuids ?| oa.orphaned_keys\n),\naffected_rows AS (\n    UPDATE concurrency_counter \n    SET job_uuids = job_uuids - orphaned_array.orphaned_keys\n    FROM orphaned_array\n    WHERE concurrency_counter.concurrency_id IN (\n        SELECT concurrency_id FROM before_update\n    )\n    RETURNING concurrency_id, job_uuids AS updated_job_uuids\n),\nexpanded_orphaned AS (\n    SELECT bu.concurrency_id, \n           bu.job_uuids AS original_job_uuids,\n           unnest(bu.orphaned_keys) AS orphaned_key\n    FROM before_update bu\n)\nSELECT \n    eo.concurrency_id,\n    eo.orphaned_key,\n    eo.original_job_uuids,\n    ar.updated_job_uuids\nFROM expanded_orphaned eo\nJOIN affected_rows ar ON eo.concurrency_id = ar.concurrency_id\nWHERE eo.original_job_uuids ? eo.orphaned_key\nORDER BY eo.concurrency_id, eo.orphaned_key\n",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "concurrency_id",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 1,
-        "name": "orphaned_key",
-        "type_info": "Text"
-      },
-      {
-        "ordinal": 2,
-        "name": "original_job_uuids",
-        "type_info": "Jsonb"
-      },
-      {
-        "ordinal": 3,
-        "name": "updated_job_uuids",
-        "type_info": "Jsonb"
-      }
-    ],
-    "parameters": {
-      "Left": []
-    },
-    "nullable": [
-      false,
-      null,
-      false,
-      false
-    ]
-  },
-  "hash": "0a1c10bd2232b0770a7816e1bd8d758dc393f797890d597e5996146247f512ac"
-}

backend/.sqlx/query-278bc6b4f149f824b5db32dacfaa714ee3852dc2ddf2d661dfdd5a986a9bb62b.json

@@ -0,0 +1,75 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT\n                c.id IS NOT NULL AS completed,\n                CASE \n                    WHEN q.id IS NOT NULL THEN (CASE WHEN NOT $5 AND q.running THEN true ELSE null END)\n                    ELSE false\n                END AS running,\n                SUBSTR(logs, GREATEST($1 - log_offset, 0)) AS logs,\n                COALESCE(r.memory_peak, c.memory_peak) AS mem_peak,\n                COALESCE(c.flow_status, f.flow_status) AS \"flow_status: sqlx::types::Json<Box<RawValue>>\",\n                COALESCE(c.workflow_as_code_status, f.workflow_as_code_status) AS \"workflow_as_code_status: sqlx::types::Json<Box<RawValue>>\",\n                job_logs.log_offset + CHAR_LENGTH(job_logs.logs) + 1 AS log_offset,\n                created_by AS \"created_by!\",\n                CASE WHEN $4::BOOLEAN THEN (\n                    SELECT scalar_int FROM job_stats WHERE job_id = $3 AND metric_id = 'progress_perc'\n                ) END AS progress\n            FROM v2_job j\n                LEFT JOIN v2_job_queue q USING (id)\n                LEFT JOIN v2_job_runtime r USING (id)\n                LEFT JOIN v2_job_status f USING (id)\n                LEFT JOIN v2_job_completed c USING (id)\n                LEFT JOIN job_logs ON job_logs.job_id =  $3\n            WHERE j.workspace_id = $2 AND j.id = $3\n            AND ($6::text[] IS NULL OR j.tag = ANY($6))",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "completed",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 1,
+        "name": "running",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 2,
+        "name": "logs",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 3,
+        "name": "mem_peak",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 4,
+        "name": "flow_status: sqlx::types::Json<Box<RawValue>>",
+        "type_info": "Jsonb"
+      },
+      {
+        "ordinal": 5,
+        "name": "workflow_as_code_status: sqlx::types::Json<Box<RawValue>>",
+        "type_info": "Jsonb"
+      },
+      {
+        "ordinal": 6,
+        "name": "log_offset",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 7,
+        "name": "created_by!",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 8,
+        "name": "progress",
+        "type_info": "Int4"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int4",
+        "Text",
+        "Uuid",
+        "Bool",
+        "Bool",
+        "TextArray"
+      ]
+    },
+    "nullable": [
+      null,
+      null,
+      null,
+      null,
+      null,
+      null,
+      null,
+      false,
+      null
+    ]
+  },
+  "hash": "278bc6b4f149f824b5db32dacfaa714ee3852dc2ddf2d661dfdd5a986a9bb62b"
+}

backend/.sqlx/query-2b9607ed838c8c62eb0f2856420389f7be648f52edbb875ff52c96219ed3ba84.json

@@ -0,0 +1,25 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE flow SET path = $1, summary = $2, description = $3,dependency_job = NULL, lock_error_logs = '', draft_only = NULL, tag = $4, dedicated_worker = $5, visible_to_runner_only = $6, on_behalf_of_email = $7, value = $8, schema = $9::text::json, edited_by = $10, edited_at = now()\n        WHERE path = $11 AND workspace_id = $12",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Text",
+        "Text",
+        "Varchar",
+        "Bool",
+        "Bool",
+        "Text",
+        "Jsonb",
+        "Text",
+        "Varchar",
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "2b9607ed838c8c62eb0f2856420389f7be648f52edbb875ff52c96219ed3ba84"
+}

backend/.sqlx/query-2bf5f7f2cf9d85a5d23e5db2f7616fb41fece9b3d46fde2d546d70b46f9008e3.json

@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO usage (id, is_workspace, month_, usage) \n                    VALUES ($1, FALSE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), $2) \n                    ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + $2",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Int4"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "2bf5f7f2cf9d85a5d23e5db2f7616fb41fece9b3d46fde2d546d70b46f9008e3"
+}

backend/.sqlx/query-2c0ab7571e1a7c4290315bc3efccb4db9e0c9aee05596a594f81975a0cdb74d1.json

@@ -1,23 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT value FROM variable WHERE workspace_id = $1 AND path = $2",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "value",
-        "type_info": "Varchar"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Text",
-        "Text"
-      ]
-    },
-    "nullable": [
-      false
-    ]
-  },
-  "hash": "2c0ab7571e1a7c4290315bc3efccb4db9e0c9aee05596a594f81975a0cdb74d1"
-}

backend/.sqlx/query-40d372c253bef624ad07010b164269f95904b5a1aa3560aa372621154ddbb28c.json

@@ -1,15 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "UPDATE workspace_settings SET ducklake = $1 WHERE workspace_id = $2",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Jsonb",
-        "Text"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "40d372c253bef624ad07010b164269f95904b5a1aa3560aa372621154ddbb28c"
-}

backend/.sqlx/query-4968e9edac534657c808b891cbf93c8c0a57f93b7b445171b1cc3f4428ee6e53.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "INSERT INTO deployment_metadata (workspace_id, path, flow_version, callback_job_ids, deployment_msg) VALUES ($1, $2, $3, $4, $5)\n                        ON CONFLICT (workspace_id, path) DO UPDATE SET callback_job_ids = $4, deployment_msg = $5",
+  "query": "INSERT INTO deployment_metadata (workspace_id, path, flow_version, callback_job_ids, deployment_msg) VALUES ($1, $2, $3, $4, $5)",
   "describe": {
     "columns": [],
     "parameters": {
@@ -14,5 +14,5 @@
     },
     "nullable": []
   },
-  "hash": "434d8dfbc25cf7e92de51d763d3a2904ccc2e95ecc3d90b43a6394a7bb4d26ab"
+  "hash": "4968e9edac534657c808b891cbf93c8c0a57f93b7b445171b1cc3f4428ee6e53"
 }

backend/.sqlx/query-4bc533074c720820cebff8d97a203df52520b7606378ecca267e88383a45b49b.json

@@ -1,16 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\n            INSERT INTO job_result_stream (workspace_id, job_id, stream)\n            VALUES ($1, $2, $3)\n            ON CONFLICT (job_id) DO UPDATE SET stream = job_result_stream.stream || $3\n            ",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Text",
-        "Uuid",
-        "Text"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "4bc533074c720820cebff8d97a203df52520b7606378ecca267e88383a45b49b"
-}

backend/.sqlx/query-4f372d047c78532907adf2d2dc114352aa7b5b28dccc50a1231a7f6539397da7.json

@@ -1,95 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT\n                c.id IS NOT NULL AS completed,\n                CASE \n                    WHEN q.id IS NOT NULL THEN (CASE WHEN NOT $5 AND q.running THEN true ELSE null END)\n                    ELSE false\n                END AS running,\n                CASE WHEN $7::BOOLEAN THEN NULL ELSE SUBSTR(logs, GREATEST($1 - log_offset, 0)) END AS logs,\n                SUBSTR(rs.stream, $8) AS new_result_stream,\n                COALESCE(r.memory_peak, c.memory_peak) AS mem_peak,\n                COALESCE(c.flow_status, f.flow_status) AS \"flow_status: sqlx::types::Json<Box<RawValue>>\",\n                COALESCE(c.workflow_as_code_status, f.workflow_as_code_status) AS \"workflow_as_code_status: sqlx::types::Json<Box<RawValue>>\",\n                CASE WHEN $7::BOOLEAN THEN NULL ELSE job_logs.log_offset + CHAR_LENGTH(job_logs.logs) + 1 END AS log_offset,\n                CHAR_LENGTH(rs.stream) + 1 AS stream_offset,\n                created_by AS \"created_by!\",\n                CASE WHEN $4::BOOLEAN THEN (\n                    SELECT scalar_int FROM job_stats WHERE job_id = $3 AND metric_id = 'progress_perc'\n                ) END AS progress,\n                rs.stream AS \"result_stream: Option<String>\"\n            FROM v2_job j\n                LEFT JOIN v2_job_queue q USING (id)\n                LEFT JOIN v2_job_runtime r USING (id)\n                LEFT JOIN v2_job_status f USING (id)\n                LEFT JOIN v2_job_completed c USING (id)\n                LEFT JOIN job_result_stream rs ON rs.job_id = $3\n                LEFT JOIN job_logs ON job_logs.job_id =  $3\n            WHERE j.workspace_id = $2 AND j.id = $3\n            AND ($6::text[] IS NULL OR j.tag = ANY($6))",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "completed",
-        "type_info": "Bool"
-      },
-      {
-        "ordinal": 1,
-        "name": "running",
-        "type_info": "Bool"
-      },
-      {
-        "ordinal": 2,
-        "name": "logs",
-        "type_info": "Text"
-      },
-      {
-        "ordinal": 3,
-        "name": "new_result_stream",
-        "type_info": "Text"
-      },
-      {
-        "ordinal": 4,
-        "name": "mem_peak",
-        "type_info": "Int4"
-      },
-      {
-        "ordinal": 5,
-        "name": "flow_status: sqlx::types::Json<Box<RawValue>>",
-        "type_info": "Jsonb"
-      },
-      {
-        "ordinal": 6,
-        "name": "workflow_as_code_status: sqlx::types::Json<Box<RawValue>>",
-        "type_info": "Jsonb"
-      },
-      {
-        "ordinal": 7,
-        "name": "log_offset",
-        "type_info": "Int4"
-      },
-      {
-        "ordinal": 8,
-        "name": "stream_offset",
-        "type_info": "Int4"
-      },
-      {
-        "ordinal": 9,
-        "name": "created_by!",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 10,
-        "name": "progress",
-        "type_info": "Int4"
-      },
-      {
-        "ordinal": 11,
-        "name": "result_stream: Option<String>",
-        "type_info": "Text"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Int4",
-        "Text",
-        "Uuid",
-        "Bool",
-        "Bool",
-        "TextArray",
-        "Bool",
-        "Int4"
-      ]
-    },
-    "nullable": [
-      null,
-      null,
-      null,
-      null,
-      null,
-      null,
-      null,
-      null,
-      null,
-      false,
-      null,
-      false
-    ]
-  },
-  "hash": "4f372d047c78532907adf2d2dc114352aa7b5b28dccc50a1231a7f6539397da7"
-}

backend/.sqlx/query-543859cf1c8d9e3bf2c2b23d21d096d01fe7a72d749229f21634b549c6b1241a.json

@@ -1,22 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT elem FROM (SELECT unnest($1::TEXT[]) AS elem)\n        WHERE elem NOT IN (SELECT datname FROM pg_catalog.pg_database);",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "elem",
-        "type_info": "Text"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "TextArray"
-      ]
-    },
-    "nullable": [
-      null
-    ]
-  },
-  "hash": "543859cf1c8d9e3bf2c2b23d21d096d01fe7a72d749229f21634b549c6b1241a"
-}

backend/.sqlx/query-55cb03040bc2a8c53dd7fbb42bbdcc40f463cbc52d94ed9315cf9a547d4c89f2.json

@@ -142,11 +142,6 @@
         "ordinal": 27,
         "name": "git_app_installations",
         "type_info": "Jsonb"
-      },
-      {
-        "ordinal": 28,
-        "name": "ducklake",
-        "type_info": "Jsonb"
       }
     ],
     "parameters": {
@@ -182,8 +177,7 @@
       true,
       true,
       true,
-      false,
-      true
+      false
     ]
   },
   "hash": "55cb03040bc2a8c53dd7fbb42bbdcc40f463cbc52d94ed9315cf9a547d4c89f2"

backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json

@@ -15,7 +15,7 @@
       ]
     },
     "nullable": [
-      null
+      true
     ]
   },
   "hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55"

backend/.sqlx/query-69924462c788dbc8f31aacc7f8ae588d76bf1f25d631833ce4b194818a7d1437.json

@@ -1,48 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT result as \"result: sqlx::types::Json<Box<RawValue>>\", v2_job.tag,\n                v2_job_queue.running as \"running: Option<bool>\", SUBSTR(rs.stream, $3) AS \"result_stream: Option<String>\", CHAR_LENGTH(rs.stream) AS stream_offset\n                FROM v2_job\n                LEFT JOIN v2_job_queue USING (id)\n                LEFT JOIN v2_job_completed USING (id)\n                LEFT JOIN job_result_stream rs ON rs.job_id = $2\n                WHERE v2_job.id = $2 AND v2_job.workspace_id = $1",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "result: sqlx::types::Json<Box<RawValue>>",
-        "type_info": "Jsonb"
-      },
-      {
-        "ordinal": 1,
-        "name": "tag",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 2,
-        "name": "running: Option<bool>",
-        "type_info": "Bool"
-      },
-      {
-        "ordinal": 3,
-        "name": "result_stream: Option<String>",
-        "type_info": "Text"
-      },
-      {
-        "ordinal": 4,
-        "name": "stream_offset",
-        "type_info": "Int4"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Text",
-        "Uuid",
-        "Int4"
-      ]
-    },
-    "nullable": [
-      true,
-      false,
-      false,
-      null,
-      null
-    ]
-  },
-  "hash": "69924462c788dbc8f31aacc7f8ae588d76bf1f25d631833ce4b194818a7d1437"
-}

backend/.sqlx/query-6bb4fab9e74c421fb30cfd856a1a9b44c76f1cf1485ceba50aeede9daec94c01.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "INSERT INTO deployment_metadata (workspace_id, path, script_hash, callback_job_ids, deployment_msg) VALUES ($1, $2, $3, $4, $5) \n                        ON CONFLICT (workspace_id, path) DO UPDATE SET callback_job_ids = $4, deployment_msg = $5",
+  "query": "INSERT INTO deployment_metadata (workspace_id, path, script_hash, callback_job_ids, deployment_msg) VALUES ($1, $2, $3, $4, $5)",
   "describe": {
     "columns": [],
     "parameters": {
@@ -14,5 +14,5 @@
     },
     "nullable": []
   },
-  "hash": "e77fcf4e0d58855542605d13177df61671334418820ca942b442adfab413cbae"
+  "hash": "6bb4fab9e74c421fb30cfd856a1a9b44c76f1cf1485ceba50aeede9daec94c01"
 }

backend/.sqlx/query-6e2564dc37ee967c634deade67ceabc4c516418e595dee9cd7d651acc1f4af6e.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "WITH job_sizes AS (\n                            SELECT \n                                cj.id,\n                                cj.workspace_id,\n                                cj.parent_job,\n                                cj.created_by,\n                                cj.duration_ms,\n                                cj.success,\n                                cj.script_hash,\n                                cj.script_path,\n                                cj.args,\n                                cj.result,\n                                cj.deleted,\n                                cj.canceled,\n                                cj.canceled_by,\n                                cj.canceled_reason,\n                                cj.job_kind,\n                                cj.schedule_path,\n                                cj.permissioned_as,\n                                cj.is_flow_step,\n                                cj.language,\n                                cj.is_skipped,\n                                cj.email,\n                                cj.visible_to_owner,\n                                cj.mem_peak,\n                                cj.tag,\n                                cj.created_at,\n                                cj.started_at,\n                                job_logs.logs,\n                                job_logs.log_offset,\n                                job_logs.log_file_index,\n                                -- Estimate size in bytes based on actual data characteristics\n                                (36 + -- UUID\n                                 COALESCE(LENGTH(cj.workspace_id), 0) + \n                                 COALESCE(LENGTH(cj.created_by), 0) + \n                                 COALESCE(LENGTH(cj.script_path), 0) + \n                                 COALESCE(LENGTH(cj.args::text), 0) + \n                                 COALESCE(LENGTH(cj.result::text), 0) + \n                                 COALESCE(LENGTH(job_logs.logs), 0) + \n                                 200) AS estimated_size_bytes -- Other fields overhead\n                            FROM v2_as_completed_job AS cj\n                            LEFT JOIN job_logs ON cj.id = job_logs.job_id\n                            WHERE cj.created_at < $1\n                            ORDER BY created_at ASC\n                            LIMIT 5000\n                        ),\n                        cumulative_sizes AS (\n                            SELECT \n                                *,\n                                SUM(estimated_size_bytes) OVER (\n                                    ORDER BY created_at ASC \n                                    ROWS UNBOUNDED PRECEDING\n                                ) AS cumulative_size_bytes,\n                                ROW_NUMBER() OVER (ORDER BY created_at ASC) AS row_num\n                            FROM job_sizes\n                        )\n                        SELECT\n                            id AS \"id!\",\n                            workspace_id AS \"workspace_id!\",\n                            parent_job,\n                            created_by AS \"created_by!\",\n                            duration_ms AS \"duration_ms!\",\n                            success AS \"success!\",\n                            script_hash AS \"script_hash!: Option<ScriptHash>\",\n                            script_path,\n                            args AS \"args: sqlx::types::Json<HashMap<String, Box<RawValue>>>\",\n                            result AS \"result: sqlx::types::Json<Box<RawValue>>\",\n                            deleted AS \"deleted!\",\n                            canceled AS \"canceled!\",\n                            canceled_by,\n                            canceled_reason,\n                            job_kind AS \"job_kind!: JobKind\",\n                            schedule_path,\n                            permissioned_as AS \"permissioned_as!\",\n                            is_flow_step AS \"is_flow_step!\",\n                            language AS \"language: ScriptLang\",\n                            is_skipped AS \"is_skipped!\",\n                            email AS \"email!\",\n                            visible_to_owner AS \"visible_to_owner!\",\n                            mem_peak,\n                            tag AS \"tag!\",\n                            created_at AS \"created_at!\",\n                            started_at,\n                            logs,\n                            log_offset AS \"log_offset?\",\n                            log_file_index\n                        FROM cumulative_sizes\n                        WHERE cumulative_size_bytes <= $2 OR row_num = 1\n                        ORDER BY created_at ASC",
+  "query": "SELECT\n                            cj.id AS \"id!\",\n                            cj.workspace_id AS \"workspace_id!\",\n                            cj.parent_job,\n                            cj.created_by AS \"created_by!\",\n                            cj.duration_ms AS \"duration_ms!\",\n                            cj.success AS \"success!\",\n                            cj.script_hash AS \"script_hash!: Option<ScriptHash>\",\n                            cj.script_path,\n                            cj.args AS \"args: sqlx::types::Json<HashMap<String, Box<RawValue>>>\",\n                            cj.result AS \"result: sqlx::types::Json<Box<RawValue>>\",\n                            cj.deleted AS \"deleted!\",\n                            cj.canceled AS \"canceled!\",\n                            cj.canceled_by,\n                            cj.canceled_reason,\n                            cj.job_kind AS \"job_kind!: JobKind\",\n                            cj.schedule_path,\n                            cj.permissioned_as AS \"permissioned_as!\",\n                            cj.is_flow_step AS \"is_flow_step!\",\n                            cj.language AS \"language: ScriptLang\",\n                            cj.is_skipped AS \"is_skipped!\",\n                            cj.email AS \"email!\",\n                            cj.visible_to_owner AS \"visible_to_owner!\",\n                            cj.mem_peak,\n                            cj.tag AS \"tag!\",\n                            cj.created_at AS \"created_at!\",\n                            cj.started_at,\n                            job_logs.logs,\n                            job_logs.log_offset AS \"log_offset?\",\n                            job_logs.log_file_index\n\n                        FROM v2_as_completed_job AS cj\n                        LEFT JOIN job_logs ON cj.id = job_logs.job_id\n                        WHERE (cj.created_at > $1 AND cj.created_at < $3)\n                               OR cj.id = ANY($2)\n                        ORDER BY cj.created_at ASC LIMIT $4",
   "describe": {
     "columns": [
       {
@@ -207,6 +207,8 @@
     ],
     "parameters": {
       "Left": [
+        "Timestamptz",
+        "UuidArray",
         "Timestamptz",
         "Int8"
       ]
@@ -243,5 +245,5 @@
       true
     ]
   },
-  "hash": "a6608d47b96d851eb7b04d2e4b472889ff257db0f0e3e9252adda2e4ef2039d6"
+  "hash": "6e2564dc37ee967c634deade67ceabc4c516418e595dee9cd7d651acc1f4af6e"
 }

backend/.sqlx/query-71a040866adbd192080da165eb120abf7531b2542a2da225152e19144400f950.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT workspace_id, slack_team_id, teams_team_id, teams_team_name, slack_name, slack_command_script, teams_command_script, slack_email, auto_invite_domain, auto_invite_operator, auto_add, customer_id, plan, webhook, deploy_to, ai_config, error_handler, error_handler_extra_args, error_handler_muted_on_cancel, large_file_storage, ducklake, git_sync, deploy_ui, default_app, default_scripts, mute_critical_alerts, color, operator_settings, git_app_installations FROM workspace_settings WHERE workspace_id = $1",
+  "query": "SELECT workspace_id, slack_team_id, teams_team_id, teams_team_name, slack_name, slack_command_script, teams_command_script, slack_email, auto_invite_domain, auto_invite_operator, auto_add, customer_id, plan, webhook, deploy_to, ai_config, error_handler, error_handler_extra_args, error_handler_muted_on_cancel, large_file_storage, git_sync, deploy_ui, default_app, default_scripts, mute_critical_alerts, color, operator_settings, git_app_installations FROM workspace_settings WHERE workspace_id = $1",
   "describe": {
     "columns": [
       {
@@ -105,46 +105,41 @@
       },
       {
         "ordinal": 20,
-        "name": "ducklake",
-        "type_info": "Jsonb"
-      },
-      {
-        "ordinal": 21,
         "name": "git_sync",
         "type_info": "Jsonb"
       },
       {
-        "ordinal": 22,
+        "ordinal": 21,
         "name": "deploy_ui",
         "type_info": "Jsonb"
       },
       {
-        "ordinal": 23,
+        "ordinal": 22,
         "name": "default_app",
         "type_info": "Varchar"
       },
       {
-        "ordinal": 24,
+        "ordinal": 23,
         "name": "default_scripts",
         "type_info": "Jsonb"
       },
       {
-        "ordinal": 25,
+        "ordinal": 24,
         "name": "mute_critical_alerts",
         "type_info": "Bool"
       },
       {
-        "ordinal": 26,
+        "ordinal": 25,
         "name": "color",
         "type_info": "Varchar"
       },
       {
-        "ordinal": 27,
+        "ordinal": 26,
         "name": "operator_settings",
         "type_info": "Jsonb"
       },
       {
-        "ordinal": 28,
+        "ordinal": 27,
         "name": "git_app_installations",
         "type_info": "Jsonb"
       }
@@ -182,9 +177,8 @@
       true,
       true,
       true,
-      true,
       false
     ]
   },
-  "hash": "6e60068b582af46afbb93010355cb5e7cc038042a6fa7089f3f73cd137a63956"
+  "hash": "71a040866adbd192080da165eb120abf7531b2542a2da225152e19144400f950"
 }

backend/.sqlx/query-7274e9489b18d7ab82bad1fbff89ffe41d162b482adb04b7b898a19576af2a5e.json

@@ -1,249 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "WITH job_sizes AS (\n                            SELECT \n                                cj.id,\n                                cj.workspace_id,\n                                cj.parent_job,\n                                cj.created_by,\n                                cj.duration_ms,\n                                cj.success,\n                                cj.script_hash,\n                                cj.script_path,\n                                cj.args,\n                                cj.result,\n                                cj.deleted,\n                                cj.canceled,\n                                cj.canceled_by,\n                                cj.canceled_reason,\n                                cj.job_kind,\n                                cj.schedule_path,\n                                cj.permissioned_as,\n                                cj.is_flow_step,\n                                cj.language,\n                                cj.is_skipped,\n                                cj.email,\n                                cj.visible_to_owner,\n                                cj.mem_peak,\n                                cj.tag,\n                                cj.created_at,\n                                cj.started_at,\n                                job_logs.logs,\n                                job_logs.log_offset,\n                                job_logs.log_file_index,\n                                -- Estimate size in bytes based on actual data characteristics\n                                (36 + -- UUID\n                                 COALESCE(LENGTH(cj.workspace_id), 0) + \n                                 COALESCE(LENGTH(cj.created_by), 0) + \n                                 COALESCE(LENGTH(cj.script_path), 0) + \n                                 COALESCE(LENGTH(cj.args::text), 0) + \n                                 COALESCE(LENGTH(cj.result::text), 0) + \n                                 COALESCE(LENGTH(job_logs.logs), 0) + \n                                 200) AS estimated_size_bytes -- Other fields overhead\n                            FROM v2_as_completed_job AS cj\n                            LEFT JOIN job_logs ON cj.id = job_logs.job_id\n                            WHERE (cj.created_at > $1 AND cj.created_at < $3)\n                                   OR cj.id = ANY($2)\n                            ORDER BY created_at ASC\n                            LIMIT 5000\n                        ),\n                        cumulative_sizes AS (\n                            SELECT \n                                *,\n                                SUM(estimated_size_bytes) OVER (\n                                    ORDER BY created_at ASC \n                                    ROWS UNBOUNDED PRECEDING\n                                ) AS cumulative_size_bytes,\n                                ROW_NUMBER() OVER (ORDER BY created_at ASC) AS row_num\n                            FROM job_sizes\n                        )\n                        SELECT\n                            id AS \"id!\",\n                            workspace_id AS \"workspace_id!\",\n                            parent_job,\n                            created_by AS \"created_by!\",\n                            duration_ms AS \"duration_ms!\",\n                            success AS \"success!\",\n                            script_hash AS \"script_hash!: Option<ScriptHash>\",\n                            script_path,\n                            args AS \"args: sqlx::types::Json<HashMap<String, Box<RawValue>>>\",\n                            result AS \"result: sqlx::types::Json<Box<RawValue>>\",\n                            deleted AS \"deleted!\",\n                            canceled AS \"canceled!\",\n                            canceled_by,\n                            canceled_reason,\n                            job_kind AS \"job_kind!: JobKind\",\n                            schedule_path,\n                            permissioned_as AS \"permissioned_as!\",\n                            is_flow_step AS \"is_flow_step!\",\n                            language AS \"language: ScriptLang\",\n                            is_skipped AS \"is_skipped!\",\n                            email AS \"email!\",\n                            visible_to_owner AS \"visible_to_owner!\",\n                            mem_peak,\n                            tag AS \"tag!\",\n                            created_at AS \"created_at!\",\n                            started_at,\n                            logs,\n                            log_offset AS \"log_offset?\",\n                            log_file_index\n                        FROM cumulative_sizes\n                        WHERE cumulative_size_bytes <= $4 OR row_num = 1\n                        ORDER BY created_at ASC",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "id!",
-        "type_info": "Uuid"
-      },
-      {
-        "ordinal": 1,
-        "name": "workspace_id!",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 2,
-        "name": "parent_job",
-        "type_info": "Uuid"
-      },
-      {
-        "ordinal": 3,
-        "name": "created_by!",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 4,
-        "name": "duration_ms!",
-        "type_info": "Int8"
-      },
-      {
-        "ordinal": 5,
-        "name": "success!",
-        "type_info": "Bool"
-      },
-      {
-        "ordinal": 6,
-        "name": "script_hash!: Option<ScriptHash>",
-        "type_info": "Int8"
-      },
-      {
-        "ordinal": 7,
-        "name": "script_path",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 8,
-        "name": "args: sqlx::types::Json<HashMap<String, Box<RawValue>>>",
-        "type_info": "Jsonb"
-      },
-      {
-        "ordinal": 9,
-        "name": "result: sqlx::types::Json<Box<RawValue>>",
-        "type_info": "Jsonb"
-      },
-      {
-        "ordinal": 10,
-        "name": "deleted!",
-        "type_info": "Bool"
-      },
-      {
-        "ordinal": 11,
-        "name": "canceled!",
-        "type_info": "Bool"
-      },
-      {
-        "ordinal": 12,
-        "name": "canceled_by",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 13,
-        "name": "canceled_reason",
-        "type_info": "Text"
-      },
-      {
-        "ordinal": 14,
-        "name": "job_kind!: JobKind",
-        "type_info": {
-          "Custom": {
-            "name": "job_kind",
-            "kind": {
-              "Enum": [
-                "script",
-                "preview",
-                "flow",
-                "dependencies",
-                "flowpreview",
-                "script_hub",
-                "identity",
-                "flowdependencies",
-                "http",
-                "graphql",
-                "postgresql",
-                "noop",
-                "appdependencies",
-                "deploymentcallback",
-                "singlescriptflow",
-                "flowscript",
-                "flownode",
-                "appscript"
-              ]
-            }
-          }
-        }
-      },
-      {
-        "ordinal": 15,
-        "name": "schedule_path",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 16,
-        "name": "permissioned_as!",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 17,
-        "name": "is_flow_step!",
-        "type_info": "Bool"
-      },
-      {
-        "ordinal": 18,
-        "name": "language: ScriptLang",
-        "type_info": {
-          "Custom": {
-            "name": "script_lang",
-            "kind": {
-              "Enum": [
-                "python3",
-                "deno",
-                "go",
-                "bash",
-                "postgresql",
-                "nativets",
-                "bun",
-                "mysql",
-                "bigquery",
-                "snowflake",
-                "graphql",
-                "powershell",
-                "mssql",
-                "php",
-                "bunnative",
-                "rust",
-                "ansible",
-                "csharp",
-                "oracledb",
-                "nu",
-                "java",
-                "duckdb"
-              ]
-            }
-          }
-        }
-      },
-      {
-        "ordinal": 19,
-        "name": "is_skipped!",
-        "type_info": "Bool"
-      },
-      {
-        "ordinal": 20,
-        "name": "email!",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 21,
-        "name": "visible_to_owner!",
-        "type_info": "Bool"
-      },
-      {
-        "ordinal": 22,
-        "name": "mem_peak",
-        "type_info": "Int4"
-      },
-      {
-        "ordinal": 23,
-        "name": "tag!",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 24,
-        "name": "created_at!",
-        "type_info": "Timestamptz"
-      },
-      {
-        "ordinal": 25,
-        "name": "started_at",
-        "type_info": "Timestamptz"
-      },
-      {
-        "ordinal": 26,
-        "name": "logs",
-        "type_info": "Text"
-      },
-      {
-        "ordinal": 27,
-        "name": "log_offset?",
-        "type_info": "Int4"
-      },
-      {
-        "ordinal": 28,
-        "name": "log_file_index",
-        "type_info": "TextArray"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Timestamptz",
-        "UuidArray",
-        "Timestamptz",
-        "Int8"
-      ]
-    },
-    "nullable": [
-      true,
-      true,
-      true,
-      true,
-      true,
-      true,
-      true,
-      true,
-      true,
-      true,
-      true,
-      true,
-      true,
-      true,
-      true,
-      true,
-      true,
-      true,
-      true,
-      true,
-      true,
-      true,
-      true,
-      true,
-      true,
-      true,
-      true,
-      false,
-      true
-    ]
-  },
-  "hash": "7274e9489b18d7ab82bad1fbff89ffe41d162b482adb04b7b898a19576af2a5e"
-}

backend/.sqlx/query-74a2871aba7e35527dcefb2538b8cf41c35618f7c15ea047c5082f8feb7a8464.json

@@ -14,8 +14,7 @@
               "Enum": [
                 "s3object",
                 "resource",
-                "variable",
-                "ducklake"
+                "variable"
               ]
             }
           }

backend/.sqlx/query-7b48820a08fbb3bee0fb2dc802a0b4f28ed5507797165e8c577ce6a95d11694c.json

@@ -1,15 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "INSERT INTO usage (id, is_workspace, month_, usage) \n                        VALUES ($1, FALSE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), $2) \n                        ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + $2",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Varchar",
-        "Int4"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "7b48820a08fbb3bee0fb2dc802a0b4f28ed5507797165e8c577ce6a95d11694c"
-}

backend/.sqlx/query-9f4811fe735d401b62f4b7bf3db2b5cd13eb3364a8a8546007dec7ab528b1f9d.json

@@ -1,20 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\nWITH rows_to_delete AS (\n    SELECT concurrency_id\n    FROM concurrency_counter\n    WHERE job_uuids = '{}'::jsonb\n    FOR UPDATE SKIP LOCKED\n)\nDELETE FROM concurrency_counter\nWHERE concurrency_id IN (SELECT concurrency_id FROM rows_to_delete)  RETURNING concurrency_id",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "concurrency_id",
-        "type_info": "Varchar"
-      }
-    ],
-    "parameters": {
-      "Left": []
-    },
-    "nullable": [
-      false
-    ]
-  },
-  "hash": "9f4811fe735d401b62f4b7bf3db2b5cd13eb3364a8a8546007dec7ab528b1f9d"
-}

backend/.sqlx/query-9f5b677a02690d3e4b4a5f5e141c7107077bbe90423102b5469e219f2a8b9293.json

@@ -1,36 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT result as \"result: sqlx::types::Json<Box<RawValue>>\", SUBSTR(rs.stream, $3) AS \"result_stream: Option<String>\", CHAR_LENGTH(rs.stream) + 1 AS stream_offset\n                 FROM v2_job_completed  FULL OUTER JOIN job_result_stream rs ON rs.job_id = v2_job_completed.id WHERE (v2_job_completed.id = $2 AND v2_job_completed.workspace_id = $1 OR rs.workspace_id = $1)",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "result: sqlx::types::Json<Box<RawValue>>",
-        "type_info": "Jsonb"
-      },
-      {
-        "ordinal": 1,
-        "name": "result_stream: Option<String>",
-        "type_info": "Text"
-      },
-      {
-        "ordinal": 2,
-        "name": "stream_offset",
-        "type_info": "Int4"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Text",
-        "Uuid",
-        "Int4"
-      ]
-    },
-    "nullable": [
-      true,
-      null,
-      null
-    ]
-  },
-  "hash": "9f5b677a02690d3e4b4a5f5e141c7107077bbe90423102b5469e219f2a8b9293"
-}

backend/.sqlx/query-a22e166746ee10943c737668aafd548e3adefe5847d2e35331edb61442d0dd92.json

@@ -1,20 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT trim(both '\"' from value::text) FROM global_settings WHERE name = 'ducklake_user_pg_pwd';",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "btrim",
-        "type_info": "Text"
-      }
-    ],
-    "parameters": {
-      "Left": []
-    },
-    "nullable": [
-      null
-    ]
-  },
-  "hash": "a22e166746ee10943c737668aafd548e3adefe5847d2e35331edb61442d0dd92"
-}

backend/.sqlx/query-b51b1e024f6dc7f30e4fbab26243f702af06663df6ddce22dd7e3d8402eb1d94.json

@@ -1,23 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\n            SELECT ws.ducklake->'ducklakes'->$2 AS config\n            FROM workspace_settings ws\n            WHERE ws.workspace_id = $1\n        ",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "config",
-        "type_info": "Jsonb"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Text",
-        "Text"
-      ]
-    },
-    "nullable": [
-      null
-    ]
-  },
-  "hash": "b51b1e024f6dc7f30e4fbab26243f702af06663df6ddce22dd7e3d8402eb1d94"
-}

backend/.sqlx/query-b83e58972d8f8285eb604de94c3d363102f5f6705d6debb442a85126ed59d863.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "INSERT INTO deployment_metadata (workspace_id, path, app_version, callback_job_ids, deployment_msg) VALUES ($1, $2, $3, $4, $5)\n                        ON CONFLICT (workspace_id, path) DO UPDATE SET callback_job_ids = $4, deployment_msg = $5",
+  "query": "INSERT INTO deployment_metadata (workspace_id, path, app_version, callback_job_ids, deployment_msg) VALUES ($1, $2, $3, $4, $5)",
   "describe": {
     "columns": [],
     "parameters": {
@@ -14,5 +14,5 @@
     },
     "nullable": []
   },
-  "hash": "f04632c3a8e0d7c5b48cdd26a99bb1dc5bd12df221f82405d663b8f15f5c0c3a"
+  "hash": "b83e58972d8f8285eb604de94c3d363102f5f6705d6debb442a85126ed59d863"
 }

backend/.sqlx/query-b91b692f070c6f1e651870fbd5aa54bc215b6fac86d8b92569d0074969530e1a.json

@@ -1,23 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT value AS \"value!: _\" FROM resource WHERE workspace_id = $1 AND path = $2",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "value!: _",
-        "type_info": "Jsonb"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Text",
-        "Text"
-      ]
-    },
-    "nullable": [
-      true
-    ]
-  },
-  "hash": "b91b692f070c6f1e651870fbd5aa54bc215b6fac86d8b92569d0074969530e1a"
-}

backend/.sqlx/query-b9b3c341fe452da916ee29637e14b5c1ad75462eba17083c6f81ff6ef35af77f.json

@@ -0,0 +1,29 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT result as \"result: sqlx::types::Json<Box<RawValue>>\", v2_job_queue.running as \"running: Option<bool>\" FROM v2_job_completed FULL OUTER JOIN v2_job_queue USING (id) WHERE (v2_job_queue.id = $1 AND v2_job_queue.workspace_id = $2) OR (v2_job_completed.id = $1 AND v2_job_completed.workspace_id = $2)",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "result: sqlx::types::Json<Box<RawValue>>",
+        "type_info": "Jsonb"
+      },
+      {
+        "ordinal": 1,
+        "name": "running: Option<bool>",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Uuid",
+        "Text"
+      ]
+    },
+    "nullable": [
+      true,
+      false
+    ]
+  },
+  "hash": "b9b3c341fe452da916ee29637e14b5c1ad75462eba17083c6f81ff6ef35af77f"
+}

backend/.sqlx/query-ca3ba808e020c8c7a35eaef842b20cfeee64fd47ded72fce55cc75e0bbb291a8.json

@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO usage (id, is_workspace, month_, usage) \n                VALUES ($1, TRUE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), $2) \n                ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + $2",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Int4"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "ca3ba808e020c8c7a35eaef842b20cfeee64fd47ded72fce55cc75e0bbb291a8"
+}

backend/.sqlx/query-ceb8c2607023883e1eebd4b9539e36ed202a6ecd12e3f90cb070341c38886de4.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT result as \"result: sqlx::types::Json<Box<RawValue>>\" FROM v2_job_completed WHERE id = $2 AND workspace_id = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "result: sqlx::types::Json<Box<RawValue>>",
+        "type_info": "Jsonb"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Uuid"
+      ]
+    },
+    "nullable": [
+      true
+    ]
+  },
+  "hash": "ceb8c2607023883e1eebd4b9539e36ed202a6ecd12e3f90cb070341c38886de4"
+}

backend/.sqlx/query-d1f505524793c03a63967d058ed1fd381094070dd7707edac81e1b5e7d1d958b.json

@@ -1,23 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT 1 FROM resource WHERE workspace_id = $1 AND path = $2",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "?column?",
-        "type_info": "Int4"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Text",
-        "Text"
-      ]
-    },
-    "nullable": [
-      null
-    ]
-  },
-  "hash": "d1f505524793c03a63967d058ed1fd381094070dd7707edac81e1b5e7d1d958b"
-}

backend/.sqlx/query-d2acc380c45df31f741bbd420c2f4a5b962f7803def9ecd6e33d2aec6abce772.json

@@ -1,25 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\n        UPDATE \n            flow \n        SET\n            path = $1,\n            summary = $2,\n            description = $3,\n            dependency_job = NULL,\n            lock_error_logs = '',\n            draft_only = NULL,\n            tag = $4,\n            dedicated_worker = $5,\n            visible_to_runner_only = $6,\n            on_behalf_of_email = $7,\n            value = $8,\n            schema = $9::text::json,\n            edited_by = $10,\n            edited_at = now()\n        WHERE \n            path = $11 AND workspace_id = $12",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Varchar",
-        "Text",
-        "Text",
-        "Varchar",
-        "Bool",
-        "Bool",
-        "Text",
-        "Jsonb",
-        "Text",
-        "Varchar",
-        "Text",
-        "Text"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "d2acc380c45df31f741bbd420c2f4a5b962f7803def9ecd6e33d2aec6abce772"
-}

backend/.sqlx/query-e4f1ee1568ce3c186b569421c6c8a3039f73d04fc53c67c70e67371f06416ef3.json

@@ -0,0 +1,25 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO flow (workspace_id, path, summary, description, dependency_job, lock_error_logs, draft_only, tag, dedicated_worker, visible_to_runner_only, on_behalf_of_email, value, schema, edited_by, edited_at) \n         VALUES ($1, $2, $3, $4, NULL, '', $5, $6, $7, $8, $9, $10, $11::text::json, $12, now())",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Varchar",
+        "Text",
+        "Text",
+        "Bool",
+        "Varchar",
+        "Bool",
+        "Bool",
+        "Text",
+        "Jsonb",
+        "Text",
+        "Varchar"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "e4f1ee1568ce3c186b569421c6c8a3039f73d04fc53c67c70e67371f06416ef3"
+}

backend/.sqlx/query-e653d36b607a16c0dfc0324690942ab25883b53a81ebb581fe019af2ec5eb567.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "\n        SELECT\n            id AS \"id!\", workspace_id AS \"workspace_id!\", parent_job, is_flow_step,\n            flow_status AS \"flow_status: Box<str>\", last_ping, same_worker\n        FROM v2_as_queue\n        WHERE running = true AND suspend = 0 AND suspend_until IS null AND scheduled_for <= now()\n            AND (job_kind = 'flow' OR job_kind = 'flowpreview' OR job_kind = 'flownode')\n            AND last_ping IS NOT NULL AND last_ping < NOW() - ($1 || ' seconds')::interval\n            AND canceled = false\n            \n        ",
+  "query": "\n        SELECT\n            id AS \"id!\", workspace_id AS \"workspace_id!\", parent_job, is_flow_step,\n            flow_status AS \"flow_status: Box<str>\", last_ping, same_worker\n        FROM v2_as_queue\n        WHERE running = true AND suspend = 0 AND suspend_until IS null AND scheduled_for <= now()\n            AND (job_kind = 'flow' OR job_kind = 'flowpreview' OR job_kind = 'flownode')\n            AND last_ping IS NOT NULL AND last_ping < NOW() - ($1 || ' seconds')::interval\n            AND canceled = false\n        ",
   "describe": {
     "columns": [
       {
@@ -54,5 +54,5 @@
       true
     ]
   },
-  "hash": "72956f508f66312807738399b57aa01a048fa4f9281327cf5b78111178424b43"
+  "hash": "e653d36b607a16c0dfc0324690942ab25883b53a81ebb581fe019af2ec5eb567"
 }

backend/.sqlx/query-ec0f8fa36328507e51c1974dbef884b755504a6cefa4af34fa4659fb95a7ee9a.json

@@ -1,42 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT \n                        result as \"result: sqlx::types::Json<Box<RawValue>>\",\n                        v2_job_queue.running as \"running: Option<bool>\",\n                        SUBSTR(rs.stream, $3) AS \"result_stream: Option<String>\",\n                        CHAR_LENGTH(rs.stream) + 1 AS stream_offset\n                    FROM v2_job_completed FULL OUTER JOIN v2_job_queue USING (id) \n                    LEFT JOIN job_result_stream rs ON rs.job_id = $1\n                    WHERE (v2_job_queue.id = $1 AND v2_job_queue.workspace_id = $2) OR (v2_job_completed.id = $1 AND v2_job_completed.workspace_id = $2)",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "result: sqlx::types::Json<Box<RawValue>>",
-        "type_info": "Jsonb"
-      },
-      {
-        "ordinal": 1,
-        "name": "running: Option<bool>",
-        "type_info": "Bool"
-      },
-      {
-        "ordinal": 2,
-        "name": "result_stream: Option<String>",
-        "type_info": "Text"
-      },
-      {
-        "ordinal": 3,
-        "name": "stream_offset",
-        "type_info": "Int4"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Uuid",
-        "Text",
-        "Int4"
-      ]
-    },
-    "nullable": [
-      true,
-      false,
-      null,
-      null
-    ]
-  },
-  "hash": "ec0f8fa36328507e51c1974dbef884b755504a6cefa4af34fa4659fb95a7ee9a"
-}

backend/.sqlx/query-f22964772dc2d67aee437bbbd08b64792c00da1d713d7ca8f9904ccce7bfdae7.json

@@ -0,0 +1,247 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT\n                            cj.id AS \"id!\",\n                            cj.workspace_id AS \"workspace_id!\",\n                            cj.parent_job,\n                            cj.created_by AS \"created_by!\",\n                            cj.duration_ms AS \"duration_ms!\",\n                            cj.success AS \"success!\",\n                            cj.script_hash AS \"script_hash!: Option<ScriptHash>\",\n                            cj.script_path,\n                            cj.args AS \"args: sqlx::types::Json<HashMap<String, Box<RawValue>>>\",\n                            cj.result AS \"result: sqlx::types::Json<Box<RawValue>>\",\n                            cj.deleted AS \"deleted!\",\n                            cj.canceled AS \"canceled!\",\n                            cj.canceled_by,\n                            cj.canceled_reason,\n                            cj.job_kind AS \"job_kind!: JobKind\",\n                            cj.schedule_path,\n                            cj.permissioned_as AS \"permissioned_as!\",\n                            cj.is_flow_step AS \"is_flow_step!\",\n                            cj.language AS \"language: ScriptLang\",\n                            cj.is_skipped AS \"is_skipped!\",\n                            cj.email AS \"email!\",\n                            cj.visible_to_owner AS \"visible_to_owner!\",\n                            cj.mem_peak,\n                            cj.tag AS \"tag!\",\n                            cj.created_at AS \"created_at!\",\n                            cj.started_at,\n                            job_logs.logs,\n                            job_logs.log_offset AS \"log_offset?\",\n                            job_logs.log_file_index\n\n                        FROM v2_as_completed_job AS cj\n                        LEFT JOIN job_logs ON cj.id = job_logs.job_id\n                        WHERE cj.created_at < $1\n                        ORDER BY cj.created_at ASC LIMIT $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "id!",
+        "type_info": "Uuid"
+      },
+      {
+        "ordinal": 1,
+        "name": "workspace_id!",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 2,
+        "name": "parent_job",
+        "type_info": "Uuid"
+      },
+      {
+        "ordinal": 3,
+        "name": "created_by!",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 4,
+        "name": "duration_ms!",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 5,
+        "name": "success!",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 6,
+        "name": "script_hash!: Option<ScriptHash>",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 7,
+        "name": "script_path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 8,
+        "name": "args: sqlx::types::Json<HashMap<String, Box<RawValue>>>",
+        "type_info": "Jsonb"
+      },
+      {
+        "ordinal": 9,
+        "name": "result: sqlx::types::Json<Box<RawValue>>",
+        "type_info": "Jsonb"
+      },
+      {
+        "ordinal": 10,
+        "name": "deleted!",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 11,
+        "name": "canceled!",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 12,
+        "name": "canceled_by",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 13,
+        "name": "canceled_reason",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 14,
+        "name": "job_kind!: JobKind",
+        "type_info": {
+          "Custom": {
+            "name": "job_kind",
+            "kind": {
+              "Enum": [
+                "script",
+                "preview",
+                "flow",
+                "dependencies",
+                "flowpreview",
+                "script_hub",
+                "identity",
+                "flowdependencies",
+                "http",
+                "graphql",
+                "postgresql",
+                "noop",
+                "appdependencies",
+                "deploymentcallback",
+                "singlescriptflow",
+                "flowscript",
+                "flownode",
+                "appscript"
+              ]
+            }
+          }
+        }
+      },
+      {
+        "ordinal": 15,
+        "name": "schedule_path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 16,
+        "name": "permissioned_as!",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 17,
+        "name": "is_flow_step!",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 18,
+        "name": "language: ScriptLang",
+        "type_info": {
+          "Custom": {
+            "name": "script_lang",
+            "kind": {
+              "Enum": [
+                "python3",
+                "deno",
+                "go",
+                "bash",
+                "postgresql",
+                "nativets",
+                "bun",
+                "mysql",
+                "bigquery",
+                "snowflake",
+                "graphql",
+                "powershell",
+                "mssql",
+                "php",
+                "bunnative",
+                "rust",
+                "ansible",
+                "csharp",
+                "oracledb",
+                "nu",
+                "java",
+                "duckdb"
+              ]
+            }
+          }
+        }
+      },
+      {
+        "ordinal": 19,
+        "name": "is_skipped!",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 20,
+        "name": "email!",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 21,
+        "name": "visible_to_owner!",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 22,
+        "name": "mem_peak",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 23,
+        "name": "tag!",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 24,
+        "name": "created_at!",
+        "type_info": "Timestamptz"
+      },
+      {
+        "ordinal": 25,
+        "name": "started_at",
+        "type_info": "Timestamptz"
+      },
+      {
+        "ordinal": 26,
+        "name": "logs",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 27,
+        "name": "log_offset?",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 28,
+        "name": "log_file_index",
+        "type_info": "TextArray"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Timestamptz",
+        "Int8"
+      ]
+    },
+    "nullable": [
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      false,
+      true
+    ]
+  },
+  "hash": "f22964772dc2d67aee437bbbd08b64792c00da1d713d7ca8f9904ccce7bfdae7"
+}

backend/.sqlx/query-f9e1334ecb17b313924587d96fc8b0310b14d75e85a558c8cb70f2761923d175.json

@@ -1,15 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "INSERT INTO usage (id, is_workspace, month_, usage) \n                    VALUES ($1, TRUE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), $2) \n                    ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + $2",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Varchar",
-        "Int4"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "f9e1334ecb17b313924587d96fc8b0310b14d75e85a558c8cb70f2761923d175"
-}

backend/.sqlx/query-fab257c4e20aa51b8f785b1882aa0b16fde33b246cbf0749ffa0e4ed63504451.json

@@ -0,0 +1,35 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT result as \"result: sqlx::types::Json<Box<RawValue>>\", v2_job.tag,\n                v2_job_queue.running as \"running: Option<bool>\"\n                FROM v2_job\n                LEFT JOIN v2_job_queue USING (id)\n                LEFT JOIN v2_job_completed USING (id)\n                WHERE v2_job.id = $2 AND v2_job.workspace_id = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "result: sqlx::types::Json<Box<RawValue>>",
+        "type_info": "Jsonb"
+      },
+      {
+        "ordinal": 1,
+        "name": "tag",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 2,
+        "name": "running: Option<bool>",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Uuid"
+      ]
+    },
+    "nullable": [
+      true,
+      false,
+      false
+    ]
+  },
+  "hash": "fab257c4e20aa51b8f785b1882aa0b16fde33b246cbf0749ffa0e4ed63504451"
+}

backend/Cargo.lock

@@ -175,9 +175,9 @@ dependencies = [
 
 [[package]]
 name = "anstream"
-version = "0.6.20"
+version = "0.6.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3ae563653d1938f79b1ab1b5e668c87c76a9930414574a6583a7b7e11a8e6192"
+checksum = "301af1932e46185686725e0fad2f8f2aa7da69dd70bf6ecc44d6b703844a3933"
 dependencies = [
  "anstyle",
  "anstyle-parse",
@@ -205,22 +205,22 @@ dependencies = [
 
 [[package]]
 name = "anstyle-query"
-version = "1.1.4"
+version = "1.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9e231f6134f61b71076a3eab506c379d4f36122f2af15a9ff04415ea4c3339e2"
+checksum = "6c8bdeb6047d8983be085bab0ba1472e6dc604e7041dbf6fcd5e71523014fae9"
 dependencies = [
- "windows-sys 0.60.2",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
 name = "anstyle-wincon"
-version = "3.0.10"
+version = "3.0.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3e0633414522a32ffaac8ac6cc8f748e090c5717661fddeea04219e2344f5f2a"
+checksum = "403f75924867bb1033c59fbf0797484329750cfbe3c4325cd33127941fabc882"
 dependencies = [
  "anstyle",
  "once_cell_polyfill",
- "windows-sys 0.60.2",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -776,9 +776,9 @@ dependencies = [
 
 [[package]]
 name = "aws-credential-types"
-version = "1.2.5"
+version = "1.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1541072f81945fa1251f8795ef6c92c4282d74d59f88498ae7d4bf00f0ebdad9"
+checksum = "b68c2194a190e1efc999612792e25b1ab3abfefe4306494efaaabc25933c0cbe"
 dependencies = [
  "aws-smithy-async",
  "aws-smithy-runtime-api",
@@ -811,9 +811,9 @@ dependencies = [
 
 [[package]]
 name = "aws-runtime"
-version = "1.5.10"
+version = "1.5.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c034a1bc1d70e16e7f4e4caf7e9f7693e4c9c24cd91cf17c2a0b21abaebc7c8b"
+checksum = "b2090e664216c78e766b6bac10fe74d2f451c02441d43484cd76ac9a295075f7"
 dependencies = [
  "aws-credential-types",
  "aws-sigv4",
@@ -924,9 +924,9 @@ dependencies = [
 
 [[package]]
 name = "aws-sigv4"
-version = "1.3.4"
+version = "1.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "084c34162187d39e3740cb635acd73c4e3a551a36146ad6fe8883c929c9f876c"
+checksum = "ddfb9021f581b71870a17eac25b52335b82211cdc092e02b6876b2bcefa61666"
 dependencies = [
  "aws-credential-types",
  "aws-smithy-http",
@@ -957,9 +957,9 @@ dependencies = [
 
 [[package]]
 name = "aws-smithy-http"
-version = "0.62.3"
+version = "0.62.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7c4dacf2d38996cf729f55e7a762b30918229917eca115de45dfa8dfb97796c9"
+checksum = "43c82ba4cab184ea61f6edaafc1072aad3c2a17dcf4c0fce19ac5694b90d8b5f"
 dependencies = [
  "aws-smithy-runtime-api",
  "aws-smithy-types",
@@ -985,7 +985,7 @@ dependencies = [
  "aws-smithy-runtime-api",
  "aws-smithy-types",
  "h2 0.3.27",
- "h2 0.4.12",
+ "h2 0.4.11",
  "http 0.2.12",
  "http 1.3.1",
  "http-body 0.4.6",
@@ -1034,9 +1034,9 @@ dependencies = [
 
 [[package]]
 name = "aws-smithy-runtime"
-version = "1.8.6"
+version = "1.8.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9e107ce0783019dbff59b3a244aa0c114e4a8c9d93498af9162608cd5474e796"
+checksum = "660f70d9d8af6876b4c9aa8dcb0dbaf0f89b04ee9a4455bea1b4ba03b15f26f6"
 dependencies = [
  "aws-smithy-async",
  "aws-smithy-http",
@@ -1058,9 +1058,9 @@ dependencies = [
 
 [[package]]
 name = "aws-smithy-runtime-api"
-version = "1.8.7"
+version = "1.8.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "75d52251ed4b9776a3e8487b2a01ac915f73b2da3af8fc1e77e0fce697a550d4"
+checksum = "937a49ecf061895fca4a6dd8e864208ed9be7546c0527d04bc07d502ec5fba1c"
 dependencies = [
  "aws-smithy-async",
  "aws-smithy-types",
@@ -1196,9 +1196,9 @@ checksum = "7b7e4c2464d97fe331d41de9d5db0def0a96f4d823b8b32a2efd503578988973"
 
 [[package]]
 name = "backon"
-version = "1.5.2"
+version = "1.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "592277618714fbcecda9a02ba7a8781f319d26532a88553bbacc77ba5d2b3a8d"
+checksum = "302eaff5357a264a2c42f127ecb8bac761cf99749fc3dc95677e2743991f99e7"
 dependencies = [
  "fastrand",
  "gloo-timers",
@@ -1531,31 +1531,6 @@ dependencies = [
  "serde_with",
 ]
 
-[[package]]
-name = "bon"
-version = "3.6.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "33d9ef19ae5263a138da9a86871eca537478ab0332a7770bac7e3f08b801f89f"
-dependencies = [
- "bon-macros",
- "rustversion",
-]
-
-[[package]]
-name = "bon-macros"
-version = "3.6.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "577ae008f2ca11ca7641bd44601002ee5ab49ef0af64846ce1ab6057218a5cc1"
-dependencies = [
- "darling 0.21.1",
- "ident_case",
- "prettyplease",
- "proc-macro2",
- "quote",
- "rustversion",
- "syn 2.0.104",
-]
-
 [[package]]
 name = "borsh"
 version = "1.5.7"
@@ -1714,9 +1689,9 @@ dependencies = [
 
 [[package]]
 name = "bytemuck_derive"
-version = "1.10.1"
+version = "1.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4f154e572231cb6ba2bd1176980827e3d5dc04cc183a75dea38109fbdd672d29"
+checksum = "441473f2b4b0459a68628c744bc61d23e730fb00128b841d30fa4bb3972257e4"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1883,9 +1858,9 @@ dependencies = [
 
 [[package]]
 name = "cc"
-version = "1.2.31"
+version = "1.2.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c3a42d84bb6b69d3a8b3eaacf0d88f179e1929695e1ad012b6cf64d9caaa5fd2"
+checksum = "deec109607ca693028562ed836a5f1c4b8bd77755c4e132fc5ce11b0b6211ae7"
 dependencies = [
  "jobserver",
  "libc",
@@ -1992,9 +1967,9 @@ dependencies = [
 
 [[package]]
 name = "clap"
-version = "4.5.43"
+version = "4.5.41"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "50fd97c9dc2399518aa331917ac6f274280ec5eb34e555dd291899745c48ec6f"
+checksum = "be92d32e80243a54711e5d7ce823c35c41c9d929dc4ab58e1276f625841aadf9"
 dependencies = [
  "clap_builder",
  "clap_derive",
@@ -2002,9 +1977,9 @@ dependencies = [
 
 [[package]]
 name = "clap_builder"
-version = "4.5.43"
+version = "4.5.41"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c35b5830294e1fa0462034af85cc95225a4cb07092c088c55bda3147cfcd8f65"
+checksum = "707eab41e9622f9139419d573eca0900137718000c517d47da73045f54331c3d"
 dependencies = [
  "anstream",
  "anstyle",
@@ -2502,16 +2477,6 @@ dependencies = [
  "darling_macro 0.20.11",
 ]
 
-[[package]]
-name = "darling"
-version = "0.21.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d6b136475da5ef7b6ac596c0e956e37bad51b85b987ff3d5e230e964936736b2"
-dependencies = [
- "darling_core 0.21.1",
- "darling_macro 0.21.1",
-]
-
 [[package]]
 name = "darling_core"
 version = "0.13.4"
@@ -2554,20 +2519,6 @@ dependencies = [
  "syn 2.0.104",
 ]
 
-[[package]]
-name = "darling_core"
-version = "0.21.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b44ad32f92b75fb438b04b68547e521a548be8acc339a6dacc4a7121488f53e6"
-dependencies = [
- "fnv",
- "ident_case",
- "proc-macro2",
- "quote",
- "strsim 0.11.1",
- "syn 2.0.104",
-]
-
 [[package]]
 name = "darling_macro"
 version = "0.13.4"
@@ -2601,17 +2552,6 @@ dependencies = [
  "syn 2.0.104",
 ]
 
-[[package]]
-name = "darling_macro"
-version = "0.21.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2b5be8a7a562d315a5b92a630c30cec6bcf663e6673f00fbb69cca66a6f521b9"
-dependencies = [
- "darling_core 0.21.1",
- "quote",
- "syn 2.0.104",
-]
-
 [[package]]
 name = "dashmap"
 version = "5.5.3"
@@ -3559,7 +3499,7 @@ dependencies = [
  "deno_tls",
  "dyn-clone",
  "error_reporter",
- "h2 0.4.12",
+ "h2 0.4.11",
  "hickory-resolver",
  "http 1.3.1",
  "http-body-util",
@@ -3840,7 +3780,7 @@ dependencies = [
  "elliptic-curve",
  "errno",
  "faster-hex",
- "h2 0.4.12",
+ "h2 0.4.11",
  "hkdf",
  "http 1.3.1",
  "http-body-util",
@@ -4296,7 +4236,7 @@ dependencies = [
  "deno_permissions",
  "deno_tls",
  "fastwebsockets",
- "h2 0.4.12",
+ "h2 0.4.11",
  "http 1.3.1",
  "http-body-util",
  "hyper 1.6.0",
@@ -4600,7 +4540,7 @@ checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab"
 dependencies = [
  "libc",
  "option-ext",
- "redox_users 0.5.2",
+ "redox_users 0.5.0",
  "windows-sys 0.60.2",
 ]
 
@@ -4684,9 +4624,9 @@ checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b"
 
 [[package]]
 name = "downcast-rs"
-version = "2.0.1"
+version = "1.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ea8a8b81cacc08888170eef4d13b775126db426d0b348bee9d18c2c1eaf123cf"
+checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2"
 
 [[package]]
 name = "dprint-swc-ext"
@@ -4825,9 +4765,9 @@ dependencies = [
 
 [[package]]
 name = "ecow"
-version = "0.2.6"
+version = "0.2.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "78e4f79b296fbaab6ce2e22d52cb4c7f010fe0ebe7a32e34fa25885fd797bd02"
+checksum = "b92b481eb5d59fd8e80e92ff11d057d1ca8d144b2cd8c66cc8d5bd177a3c0dc5"
 dependencies = [
  "serde",
 ]
@@ -5022,9 +4962,9 @@ checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0"
 
 [[package]]
 name = "event-listener"
-version = "5.4.1"
+version = "5.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab"
+checksum = "3492acde4c3fc54c845eaab3eed8bd00c7a7d881f78bfc801e43a93dec1331ae"
 dependencies = [
  "concurrent-queue",
  "parking",
@@ -5308,12 +5248,12 @@ dependencies = [
 
 [[package]]
 name = "fs4"
-version = "0.13.1"
+version = "0.8.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8640e34b88f7652208ce9e88b1a37a2ae95227d84abec377ccd3c5cfeb141ed4"
+checksum = "f7e180ac76c23b45e767bd7ae9579bc0bb458618c4bc71835926e098e61d15f8"
 dependencies = [
- "rustix 1.0.8",
- "windows-sys 0.59.0",
+ "rustix 0.38.44",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]
@@ -5408,9 +5348,9 @@ checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6"
 
 [[package]]
 name = "futures-lite"
-version = "2.6.1"
+version = "2.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad"
+checksum = "f5edaec856126859abb19ed65f39e90fea3a9574b9707f13539acf4abf7eb532"
 dependencies = [
  "fastrand",
  "futures-core",
@@ -6072,9 +6012,9 @@ dependencies = [
 
 [[package]]
 name = "h2"
-version = "0.4.12"
+version = "0.4.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f3c0b69cfcb4e1b9f1bf2f53f95f766e4661169728ec61cd3fe5a0166f2d1386"
+checksum = "17da50a276f1e01e0ba6c029e47b7100754904ee8a278f886546e98575380785"
 dependencies = [
  "atomic-waker",
  "bytes",
@@ -6453,7 +6393,7 @@ dependencies = [
  "bytes",
  "futures-channel",
  "futures-util",
- "h2 0.4.12",
+ "h2 0.4.11",
  "http 1.3.1",
  "http-body 1.0.1",
  "httparse",
@@ -6597,15 +6537,6 @@ dependencies = [
  "tower-service",
 ]
 
-[[package]]
-name = "hyperloglogplus"
-version = "0.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "621debdf94dcac33e50475fdd76d34d5ea9c0362a834b9db08c3024696c1fbe3"
-dependencies = [
- "serde",
-]
-
 [[package]]
 name = "iana-time-zone"
 version = "0.1.63"
@@ -7361,9 +7292,9 @@ dependencies = [
 
 [[package]]
 name = "libredox"
-version = "0.1.9"
+version = "0.1.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "391290121bad3d37fbddad76d8f5d1c1c314cfc646d143d7e07a3086ddff0ce3"
+checksum = "360e552c93fa0e8152ab463bc4c4837fce76a225df11dfaeea66c313de5e61f7"
 dependencies = [
  "bitflags 2.9.1",
  "libc",
@@ -7750,10 +7681,11 @@ dependencies = [
 
 [[package]]
 name = "measure_time"
-version = "0.9.0"
+version = "0.8.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "51c55d61e72fc3ab704396c5fa16f4c184db37978ae4e94ca8959693a235fc0e"
+checksum = "dbefd235b0aadd181626f281e1d684e116972988c14c264e42069d5e8a5775cc"
 dependencies = [
+ "instant",
  "log",
 ]
 
@@ -8039,9 +7971,9 @@ dependencies = [
 
 [[package]]
 name = "mysql_common"
-version = "0.35.5"
+version = "0.35.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fbb9f371618ce723f095c61fbcdc36e8936956d2b62832f9c7648689b338e052"
+checksum = "6e0ec195e788c95f36b7cf88127d538465fc2f7773e6e47af01834738eab0aee"
 dependencies = [
  "base64 0.22.1",
  "bitflags 2.9.1",
@@ -9012,8 +8944,9 @@ checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"
 
 [[package]]
 name = "ownedbytes"
-version = "0.9.0"
-source = "git+https://github.com/windmill-labs/tantivy?rev=6a24621231202ccd77bec90d8787e2281fb94e4e#6a24621231202ccd77bec90d8787e2281fb94e4e"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c3a059efb063b8f425b948e042e6b9bd85edfe60e913630ed727b23e2dfcc558"
 dependencies = [
  "stable_deref_trait",
 ]
@@ -9908,9 +9841,9 @@ dependencies = [
 
 [[package]]
 name = "quick_cache"
-version = "0.6.16"
+version = "0.6.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9ad6644cb07b7f3488b9f3d2fde3b4c0a7fa367cafefb39dff93a659f76eb786"
+checksum = "8565e62e02af316570d4b492f17af1481d6c07cea60f4e7edd71700da5052ba9"
 dependencies = [
  "ahash 0.8.12",
  "equivalent",
@@ -10243,9 +10176,9 @@ dependencies = [
 
 [[package]]
 name = "redox_users"
-version = "0.5.2"
+version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac"
+checksum = "dd6f9d3d47bdd2ad6945c5015a226ec6155d0bcdfd8f7cd29f86b71f8de99d2b"
 dependencies = [
  "getrandom 0.2.16",
  "libredox",
@@ -10391,7 +10324,7 @@ dependencies = [
  "encoding_rs",
  "futures-core",
  "futures-util",
- "h2 0.4.12",
+ "h2 0.4.11",
  "http 1.3.1",
  "http-body 1.0.1",
  "http-body-util",
@@ -10875,7 +10808,7 @@ dependencies = [
  "openssl-probe",
  "rustls-pki-types",
  "schannel",
- "security-framework 3.3.0",
+ "security-framework 3.2.0",
 ]
 
 [[package]]
@@ -11247,9 +11180,9 @@ dependencies = [
 
 [[package]]
 name = "security-framework"
-version = "3.3.0"
+version = "3.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "80fb1d92c5028aa318b4b8bd7302a5bfcf48be96a37fc6fc790f806b0004ee0c"
+checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316"
 dependencies = [
  "bitflags 2.9.1",
  "core-foundation 0.10.1",
@@ -11370,9 +11303,9 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.142"
+version = "1.0.141"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "030fedb782600dcbd6f02d479bf0d817ac3bb40d644745b769d6a96bc3afc5a7"
+checksum = "30b9eff21ebe718216c6ec64e1d9ac57087aad11efc64e32002bce4a0d4c03d3"
 dependencies = [
  "indexmap 2.10.0",
  "itoa",
@@ -11593,9 +11526,9 @@ dependencies = [
 
 [[package]]
 name = "signal-hook-registry"
-version = "1.4.6"
+version = "1.4.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b2a4719bff48cee6b39d12c020eeb490953ad2443b7055bd0b21fca26bd8c28b"
+checksum = "9203b8055f63a2a00e2f593bb0510367fe707d7ff1e5c872de2f537b339e5410"
 dependencies = [
  "libc",
 ]
@@ -11690,9 +11623,9 @@ checksum = "1b6709c7b6754dca1311b3c73e79fcce40dd414c782c66d88e8823030093b02b"
 
 [[package]]
 name = "sketches-ddsketch"
-version = "0.3.0"
+version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c1e9a774a6c28142ac54bb25d25562e6bcf957493a184f15ad4eebccb23e410a"
+checksum = "85636c14b73d81f541e525f585c0a2109e6744e1565b5c1668e31c70c10ed65c"
 dependencies = [
  "serde",
 ]
@@ -11931,7 +11864,7 @@ dependencies = [
  "crc",
  "crossbeam-queue",
  "either",
- "event-listener 5.4.1",
+ "event-listener 5.4.0",
  "futures-core",
  "futures-intrusive",
  "futures-io",
@@ -12817,14 +12750,14 @@ checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417"
 
 [[package]]
 name = "tantivy"
-version = "0.24.0"
-source = "git+https://github.com/windmill-labs/tantivy?rev=6a24621231202ccd77bec90d8787e2281fb94e4e#6a24621231202ccd77bec90d8787e2281fb94e4e"
+version = "0.22.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "96599ea6fccd844fc833fed21d2eecac2e6a7c1afd9e044057391d78b1feb141"
 dependencies = [
  "aho-corasick",
  "arc-swap",
  "base64 0.22.1",
  "bitpacking",
- "bon",
  "byteorder",
  "census",
  "crc32fast",
@@ -12834,20 +12767,20 @@ dependencies = [
  "fnv",
  "fs4",
  "htmlescape",
- "hyperloglogplus",
- "itertools 0.14.0",
+ "itertools 0.12.1",
  "levenshtein_automata",
  "log",
  "lru 0.12.5",
  "lz4_flex",
  "measure_time",
  "memmap2 0.9.7",
+ "num_cpus",
  "once_cell",
  "oneshot",
  "rayon",
  "regex",
  "rust-stemmers",
- "rustc-hash 2.1.1",
+ "rustc-hash 1.1.0",
  "serde",
  "serde_json",
  "sketches-ddsketch",
@@ -12860,7 +12793,7 @@ dependencies = [
  "tantivy-stacker",
  "tantivy-tokenizer-api",
  "tempfile",
- "thiserror 2.0.12",
+ "thiserror 1.0.69",
  "time",
  "uuid",
  "winapi",
@@ -12868,20 +12801,22 @@ dependencies = [
 
 [[package]]
 name = "tantivy-bitpacker"
-version = "0.8.0"
-source = "git+https://github.com/windmill-labs/tantivy?rev=6a24621231202ccd77bec90d8787e2281fb94e4e#6a24621231202ccd77bec90d8787e2281fb94e4e"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "284899c2325d6832203ac6ff5891b297fc5239c3dc754c5bc1977855b23c10df"
 dependencies = [
  "bitpacking",
 ]
 
 [[package]]
 name = "tantivy-columnar"
-version = "0.5.0"
-source = "git+https://github.com/windmill-labs/tantivy?rev=6a24621231202ccd77bec90d8787e2281fb94e4e#6a24621231202ccd77bec90d8787e2281fb94e4e"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "12722224ffbe346c7fec3275c699e508fd0d4710e629e933d5736ec524a1f44e"
 dependencies = [
  "downcast-rs",
  "fastdivide",
- "itertools 0.14.0",
+ "itertools 0.12.1",
  "serde",
  "tantivy-bitpacker",
  "tantivy-common",
@@ -12891,8 +12826,9 @@ dependencies = [
 
 [[package]]
 name = "tantivy-common"
-version = "0.9.0"
-source = "git+https://github.com/windmill-labs/tantivy?rev=6a24621231202ccd77bec90d8787e2281fb94e4e#6a24621231202ccd77bec90d8787e2281fb94e4e"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8019e3cabcfd20a1380b491e13ff42f57bb38bf97c3d5fa5c07e50816e0621f4"
 dependencies = [
  "async-trait",
  "byteorder",
@@ -12914,21 +12850,19 @@ dependencies = [
 
 [[package]]
 name = "tantivy-query-grammar"
-version = "0.24.0"
-source = "git+https://github.com/windmill-labs/tantivy?rev=6a24621231202ccd77bec90d8787e2281fb94e4e#6a24621231202ccd77bec90d8787e2281fb94e4e"
+version = "0.22.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "847434d4af57b32e309f4ab1b4f1707a6c566656264caa427ff4285c4d9d0b82"
 dependencies = [
  "nom 7.1.3",
- "serde",
- "serde_json",
 ]
 
 [[package]]
 name = "tantivy-sstable"
-version = "0.5.0"
-source = "git+https://github.com/windmill-labs/tantivy?rev=6a24621231202ccd77bec90d8787e2281fb94e4e#6a24621231202ccd77bec90d8787e2281fb94e4e"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c69578242e8e9fc989119f522ba5b49a38ac20f576fc778035b96cc94f41f98e"
 dependencies = [
- "futures-util",
- "itertools 0.14.0",
  "tantivy-bitpacker",
  "tantivy-common",
  "tantivy-fst",
@@ -12937,8 +12871,9 @@ dependencies = [
 
 [[package]]
 name = "tantivy-stacker"
-version = "0.5.0"
-source = "git+https://github.com/windmill-labs/tantivy?rev=6a24621231202ccd77bec90d8787e2281fb94e4e#6a24621231202ccd77bec90d8787e2281fb94e4e"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c56d6ff5591fc332739b3ce7035b57995a3ce29a93ffd6012660e0949c956ea8"
 dependencies = [
  "murmurhash32",
  "rand_distr 0.4.3",
@@ -12947,8 +12882,9 @@ dependencies = [
 
 [[package]]
 name = "tantivy-tokenizer-api"
-version = "0.5.0"
-source = "git+https://github.com/windmill-labs/tantivy?rev=6a24621231202ccd77bec90d8787e2281fb94e4e#6a24621231202ccd77bec90d8787e2281fb94e4e"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2a0dcade25819a89cfe6f17d932c9cedff11989936bf6dd4f336d50392053b04"
 dependencies = [
  "serde",
 ]
@@ -13475,9 +13411,9 @@ dependencies = [
 
 [[package]]
 name = "tokio-util"
-version = "0.7.16"
+version = "0.7.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "14307c986784f72ef81c89db7d9e28d6ac26d16213b109ea501696195e6e3ce5"
+checksum = "66a539a9ad6d5d281510d5bd368c973d636c02dbf8a67300bfb6b950696ad7df"
 dependencies = [
  "bytes",
  "futures-core",
@@ -13568,7 +13504,7 @@ dependencies = [
  "base64 0.22.1",
  "bytes",
  "flate2",
- "h2 0.4.12",
+ "h2 0.4.11",
  "http 1.3.1",
  "http-body 1.0.1",
  "http-body-util",
@@ -14749,7 +14685,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
 
 [[package]]
 name = "windmill"
-version = "1.519.2"
+version = "1.515.1"
 dependencies = [
  "anyhow",
  "axum",
@@ -14801,7 +14737,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api"
-version = "1.519.2"
+version = "1.515.1"
 dependencies = [
  "anyhow",
  "argon2",
@@ -14888,7 +14824,6 @@ dependencies = [
  "tokio",
  "tokio-native-tls",
  "tokio-postgres 0.7.11",
- "tokio-postgres 0.7.13",
  "tokio-stream",
  "tokio-tar",
  "tokio-tungstenite",
@@ -14917,7 +14852,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-client"
-version = "1.519.2"
+version = "1.515.1"
 dependencies = [
  "base64 0.22.1",
  "chrono",
@@ -14932,7 +14867,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-audit"
-version = "1.519.2"
+version = "1.515.1"
 dependencies = [
  "chrono",
  "serde",
@@ -14945,7 +14880,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-autoscaling"
-version = "1.519.2"
+version = "1.515.1"
 dependencies = [
  "anyhow",
  "serde",
@@ -14959,10 +14894,9 @@ dependencies = [
 
 [[package]]
 name = "windmill-common"
-version = "1.519.2"
+version = "1.515.1"
 dependencies = [
  "anyhow",
- "async-recursion",
  "async-stream",
  "aws-config",
  "aws-sdk-sts",
@@ -15039,13 +14973,12 @@ dependencies = [
 
 [[package]]
 name = "windmill-git-sync"
-version = "1.519.2"
+version = "1.515.1"
 dependencies = [
  "regex",
  "serde",
  "serde_json",
  "sqlx",
- "tokio",
  "tracing",
  "uuid",
  "windmill-common",
@@ -15054,7 +14987,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-indexer"
-version = "1.519.2"
+version = "1.515.1"
 dependencies = [
  "anyhow",
  "bytes",
@@ -15078,7 +15011,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-macros"
-version = "1.519.2"
+version = "1.515.1"
 dependencies = [
  "itertools 0.14.0",
  "lazy_static",
@@ -15090,7 +15023,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser"
-version = "1.519.2"
+version = "1.515.1"
 dependencies = [
  "convert_case 0.6.0",
  "serde",
@@ -15099,7 +15032,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-bash"
-version = "1.519.2"
+version = "1.515.1"
 dependencies = [
  "anyhow",
  "lazy_static",
@@ -15111,7 +15044,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-csharp"
-version = "1.519.2"
+version = "1.515.1"
 dependencies = [
  "anyhow",
  "serde_json",
@@ -15123,7 +15056,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-go"
-version = "1.519.2"
+version = "1.515.1"
 dependencies = [
  "anyhow",
  "gosyn",
@@ -15135,7 +15068,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-graphql"
-version = "1.519.2"
+version = "1.515.1"
 dependencies = [
  "anyhow",
  "lazy_static",
@@ -15147,7 +15080,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-java"
-version = "1.519.2"
+version = "1.515.1"
 dependencies = [
  "anyhow",
  "serde_json",
@@ -15159,7 +15092,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-nu"
-version = "1.519.2"
+version = "1.515.1"
 dependencies = [
  "anyhow",
  "nu-parser",
@@ -15170,7 +15103,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-php"
-version = "1.519.2"
+version = "1.515.1"
 dependencies = [
  "anyhow",
  "itertools 0.14.0",
@@ -15181,7 +15114,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-py"
-version = "1.519.2"
+version = "1.515.1"
 dependencies = [
  "anyhow",
  "itertools 0.14.0",
@@ -15193,7 +15126,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-py-imports"
-version = "1.519.2"
+version = "1.515.1"
 dependencies = [
  "anyhow",
  "async-recursion",
@@ -15216,7 +15149,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-rust"
-version = "1.519.2"
+version = "1.515.1"
 dependencies = [
  "anyhow",
  "convert_case 0.6.0",
@@ -15233,7 +15166,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-sql"
-version = "1.519.2"
+version = "1.515.1"
 dependencies = [
  "anyhow",
  "lazy_static",
@@ -15247,7 +15180,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-ts"
-version = "1.519.2"
+version = "1.515.1"
 dependencies = [
  "anyhow",
  "lazy_static",
@@ -15265,7 +15198,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-wasm"
-version = "1.519.2"
+version = "1.515.1"
 dependencies = [
  "anyhow",
  "getrandom 0.2.16",
@@ -15289,7 +15222,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-yaml"
-version = "1.519.2"
+version = "1.515.1"
 dependencies = [
  "anyhow",
  "serde_json",
@@ -15299,7 +15232,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-queue"
-version = "1.519.2"
+version = "1.515.1"
 dependencies = [
  "anyhow",
  "async-recursion",
@@ -15332,7 +15265,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-sql-datatype-parser-wasm"
-version = "1.519.2"
+version = "1.515.1"
 dependencies = [
  "wasm-bindgen",
  "wasm-bindgen-test",
@@ -15342,7 +15275,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-worker"
-version = "1.519.2"
+version = "1.515.1"
 dependencies = [
  "anyhow",
  "async-recursion",
@@ -16146,9 +16079,9 @@ dependencies = [
 
 [[package]]
 name = "zerovec"
-version = "0.11.4"
+version = "0.11.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e7aa2bd55086f1ab526693ecbe444205da57e25f4489879da80635a46d90e73b"
+checksum = "4a05eb080e015ba39cc9e23bbe5e7fb04d5fb040350f99f34e338d5fdd294428"
 dependencies = [
  "yoke 0.8.0",
  "zerofrom",

backend/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "windmill"
-version = "1.519.2"
+version = "1.515.1"
 authors.workspace = true
 edition.workspace = true
 
@@ -32,7 +32,7 @@ members = [
 ]
 
 [workspace.package]
-version = "1.519.2"
+version = "1.515.1"
 authors = ["Ruben Fiszel <ruben@windmill.dev>"]
 edition = "2021"
 
@@ -243,7 +243,7 @@ json-pointer = "^0"
 itertools = "^0"
 regex = "^1"
 semver = "^1"
-duckdb = { version = "1.3.2", features = ["bundled"] }
+duckdb = { version = "1.2.2", features = ["bundled"] }
 
 v8 = "=130.0.7"  # Exact version NOTE: Do not forget to update version and hash in flake.nix
 deno_fetch = "0.214.0"
@@ -386,7 +386,7 @@ tikv-jemalloc-ctl = { version = "^0.5" }
 triomphe = "^0"
 pin-project-lite = "^0"
 
-tantivy = { git="https://github.com/windmill-labs/tantivy", rev="6a24621231202ccd77bec90d8787e2281fb94e4e" }
+tantivy = "0.22.0"
 
 backon = "1.3.0"
 systemstat = "0.2.4"

backend/ee-repo-ref.txt

@@ -1 +1 @@
-dbc87a4966118e5486226d1a1863e4d97ba82ecc
+cef7486dfa765f1cdfaddba59564c4bb92fe864e

backend/migrations/20250719095639_add_variable_asset_kind.down.sql

@@ -1,6 +1 @@
--- Remove 'variable' kind
-DELETE FROM asset WHERE kind = 'variable';
-ALTER TABLE asset ALTER column kind TYPE VARCHAR;
-DROP TYPE asset_kind;
-CREATE TYPE ASSET_KIND AS ENUM ('s3object', 'resource');
-ALTER TABLE asset ALTER column kind TYPE ASSET_KIND using kind::ASSET_KIND;
+-- Add down migration script here

backend/migrations/20250724084100_ducklake.down.sql

@@ -1,9 +0,0 @@
-ALTER TABLE workspace_settings
-DROP COLUMN ducklake;
-
--- Remove 'ducklake' kind
-DELETE FROM asset WHERE kind = 'ducklake';
-ALTER TABLE asset ALTER column kind TYPE VARCHAR;
-DROP TYPE asset_kind;
-CREATE TYPE ASSET_KIND AS ENUM ('s3object', 'resource', 'variable');
-ALTER TABLE asset ALTER column kind TYPE ASSET_KIND using kind::ASSET_KIND;

backend/migrations/20250724084100_ducklake.up.sql

@@ -1,4 +0,0 @@
-ALTER TYPE asset_kind ADD VALUE 'ducklake';
-
-ALTER TABLE workspace_settings
-ADD COLUMN ducklake JSONB;

backend/migrations/20250731132157_ducklake_instance_settings.down.sql

@@ -1,2 +0,0 @@
-DELETE FROM global_settings
-WHERE name = 'ducklake_user_pg_pwd';

backend/migrations/20250731132157_ducklake_instance_settings.up.sql

@@ -1,23 +0,0 @@
-INSERT INTO global_settings (name, value)
-VALUES ('ducklake_user_pg_pwd', ('"' || gen_random_uuid()::text || '"')::jsonb)
-ON CONFLICT DO NOTHING;
-
--- Cannot simply create the user because Postgres expect a static string for the password
--- Also we cannot drop the user easily in the down migration because databases will depend on it
--- And we cannot drop databases in transactions (migrations)
-
-DO $$
-DECLARE
-    pwd text;
-BEGIN
-    SELECT trim(both '"' from value::text) INTO pwd FROM global_settings WHERE name = 'ducklake_user_pg_pwd';
-    IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'ducklake_user') THEN
-        EXECUTE format('CREATE USER ducklake_user WITH PASSWORD %L', pwd);
-    ELSE
-      EXECUTE format('ALTER USER ducklake_user WITH PASSWORD %L', pwd);
-    END IF;
-EXCEPTION
-  WHEN others THEN
-    RAISE NOTICE 'ducklake_user migration error, skipping.';
-END
-$$;

backend/migrations/20250804155709_add_stream_result.down.sql

@@ -1,2 +0,0 @@
--- Add down migration script here
-DROP TABLE job_result_stream;

backend/migrations/20250804155709_add_stream_result.up.sql

@@ -1,11 +0,0 @@
--- Add up migration script here
-CREATE TABLE job_result_stream (
-    job_id UUID NOT NULL PRIMARY KEY,
-    workspace_id TEXT NOT NULL,
-    stream TEXT NOT NULL
-);
-
-ALTER TABLE job_result_stream ADD CONSTRAINT fk_job_result_stream_job_id FOREIGN KEY (job_id) REFERENCES v2_job_queue(id) ON DELETE CASCADE;
-
-GRANT ALL ON TABLE job_result_stream TO windmill_admin;
-GRANT ALL ON TABLE job_result_stream TO windmill_user;

backend/parsers/windmill-parser-sql/src/asset_parser.rs

@@ -7,8 +7,6 @@ use nom::{
     branch::alt,
     bytes::complete::{tag, tag_no_case, take_while},
     character::complete::{char, multispace0},
-    combinator::opt,
-    sequence::preceded,
     IResult, Parser,
 };
 
@@ -54,11 +52,6 @@ fn parse_asset(input: &str) -> IResult<&str, ParseAssetsResult<&str>> {
             kind: AssetKind::Resource,
             access_type: None,
         }),
-        parse_ducklake_lit.map(|path| ParseAssetsResult {
-            path,
-            kind: AssetKind::Ducklake,
-            access_type: None,
-        }),
     ))
     .parse(input)
 }
@@ -121,14 +114,6 @@ fn parse_resource_lit(input: &str) -> IResult<&str, &str> {
     Ok((input, path))
 }
 
-fn parse_ducklake_lit(input: &str) -> IResult<&str, &str> {
-    let (input, _) = quote(input)?;
-    let (input, _) = tag("ducklake").parse(input)?;
-    let (input, path) =
-        opt(preceded(tag("://"), take_while(|c| c != '\'' && c != '"'))).parse(input)?;
-    let (input, _) = quote(input)?;
-    Ok((input, path.unwrap_or("main")))
-}
 fn parse_comment(input: &str) -> IResult<&str, &str> {
     let (input, _) = tag("--").parse(input)?;
     let (input, comment) = take_while(|c| c != '\n')(input)?;

backend/parsers/windmill-parser/src/asset_parser.rs

@@ -15,7 +15,6 @@ use AssetUsageAccessType::*;
 pub enum AssetKind {
     S3Object,
     Resource,
-    Ducklake,
 }
 
 #[derive(Serialize)]
@@ -57,8 +56,6 @@ pub fn parse_asset_syntax(s: &str) -> Option<(AssetKind, &str)> {
         Some((AssetKind::Resource, &s[6..]))
     } else if s.starts_with("$res:") {
         Some((AssetKind::Resource, &s[5..]))
-    } else if s.starts_with("ducklake://") {
-        Some((AssetKind::Ducklake, &s[11..]))
     } else {
         None
     }

backend/parsers/windmill-sql-datatype-parser-wasm/src/lib.rs

@@ -38,8 +38,3 @@ pub fn parse_snowflake(typ: &str) -> String {
 pub fn parse_mssql(typ: &str) -> String {
     to_str(windmill_parser_sql::parse_mssql_typ(typ))
 }
-
-#[wasm_bindgen]
-pub fn parse_duckdb(typ: &str) -> String {
-    to_str(windmill_parser_sql::parse_duckdb_typ(typ))
-}

backend/src/main.rs

@@ -95,7 +95,7 @@ use crate::monitor::{
     reload_job_default_timeout_setting, reload_jwt_secret_setting, reload_license_key,
     reload_npm_config_registry_setting, reload_pip_index_url_setting,
     reload_retention_period_setting, reload_scim_token_setting, reload_smtp_config,
-    reload_worker_config, MonitorIteration,
+    reload_worker_config,
 };
 
 #[cfg(feature = "parquet")]
@@ -135,44 +135,6 @@ pub fn setup_deno_runtime() -> anyhow::Result<()> {
     Ok(())
 }
 
-fn update_ca_certificates_if_requested() {
-    if std::env::var("RUN_UPDATE_CA_CERTIFICATE_AT_START")
-        .ok()
-        .map(|v| v.to_lowercase() == "true")
-        .unwrap_or(false)
-    {
-        let ca_cert_path = std::env::var("RUN_UPDATE_CA_CERTIFICATE_PATH")
-            .unwrap_or_else(|_| "/usr/sbin/update-ca-certificates".to_string());
-
-        println!(
-            "RUN_UPDATE_CA_CERTIFICATE_AT_START=true, running: {}",
-            ca_cert_path
-        );
-
-        let output = std::process::Command::new(&ca_cert_path).output();
-
-        match output {
-            Ok(result) => {
-                if result.status.success() {
-                    println!("Successfully updated CA certificates");
-                } else {
-                    let stderr = String::from_utf8_lossy(&result.stderr);
-                    println!(
-                        "Failed to update CA certificates, but continuing startup: {}",
-                        stderr.trim()
-                    );
-                }
-            }
-            Err(e) => {
-                println!(
-                    "Could not run update-ca-certificates command, but continuing startup: {}",
-                    e
-                );
-            }
-        }
-    }
-}
-
 #[inline(always)]
 fn create_and_run_current_thread_inner<F, R>(future: F) -> R
 where
@@ -301,14 +263,12 @@ async fn cache_hub_scripts(file_path: Option<String>) -> anyhow::Result<()> {
 async fn windmill_main() -> anyhow::Result<()> {
     dotenv::dotenv().ok();
 
-    update_ca_certificates_if_requested();
-
     if std::env::var("RUST_LOG").is_err() {
         std::env::set_var("RUST_LOG", "info")
     }
 
     if let Err(_e) = rustls::crypto::ring::default_provider().install_default() {
-        println!("Failed to install rustls crypto provider");
+        tracing::error!("Failed to install rustls crypto provider");
     }
 
     #[cfg(feature = "enterprise")]
@@ -320,9 +280,9 @@ async fn windmill_main() -> anyhow::Result<()> {
             .arg("iptables -A OUTPUT -d 169.254.169.254 -j DROP && iptables -A FORWARD -d 169.254.169.254 -j DROP")
             .status()
         {
-            println!("Failed to run iptables to block metadata endpoint: {e}");
+            tracing::warn!("Failed to run iptables to block metadata endpoint: {e}");
         } else {
-            println!("Successfully blocked metadata endpoint using iptables");
+            tracing::info!("Successfully blocked metadata endpoint using iptables");
         }
     }
 
@@ -417,7 +377,7 @@ async fn windmill_main() -> anyhow::Result<()> {
 
         let num_version = sqlx::query_scalar!("SELECT version()").fetch_one(&db).await;
 
-        println!(
+        tracing::info!(
             "PostgreSQL version: {} (windmill require PG >= 14)",
             num_version
                 .ok()
@@ -426,7 +386,7 @@ async fn windmill_main() -> anyhow::Result<()> {
         );
         load_otel(&db).await;
 
-        println!("Database connected");
+        tracing::info!("Database connected");
         (Connection::Sql(db), None)
     };
 
@@ -573,7 +533,6 @@ Windmill Community Edition {GIT_VERSION}
             worker_mode,
             true,
             killpill_tx.clone(),
-            None,
         )
         .await;
 
@@ -801,8 +760,6 @@ Windmill Community Edition {GIT_VERSION}
                     let h = tokio::spawn(async move {
                         let mut listener = retry_listen_pg(&db_url).await;
                         let mut last_listener_refresh = Instant::now();
-                        let mut monitor_iteration: u64 = 0;
-                        let rd_shift: u8 = rand::rng().random_range(0..200);
                         loop {
                             let db = db.clone();
                             tokio::select! {
@@ -1138,13 +1095,8 @@ Windmill Community Edition {GIT_VERSION}
                                         worker_mode,
                                         false,
                                         tx.clone(),
-                                        Some(MonitorIteration {
-                                            rd_shift,
-                                            iter: monitor_iteration,
-                                        }),
                                     )
                                     .await;
-                                    monitor_iteration += 1;
                                     if server_mode {
                                         if !*windmill_common::QUIET_LOGS {
                                             tracing::info!("monitor task finished");

backend/src/monitor.rs

@@ -1337,17 +1337,6 @@ pub async fn monitor_pool(db: &DB) {
     }
 }
 
-pub struct MonitorIteration {
-    pub rd_shift: u8,
-    pub iter: u64,
-}
-
-impl MonitorIteration {
-    pub fn should_run(&self, period: u8) -> bool {
-        self.iter % (period as u64) == self.rd_shift as u64
-    }
-}
-
 pub async fn monitor_db(
     conn: &Connection,
     base_internal_url: &str,
@@ -1355,7 +1344,6 @@ pub async fn monitor_db(
     _worker_mode: bool,
     initial_load: bool,
     _killpill_tx: KillpillSender,
-    iteration: Option<MonitorIteration>,
 ) {
     let zombie_jobs_f = async {
         if server_mode && !initial_load && !*DISABLE_ZOMBIE_JOBS_MONITORING {
@@ -1370,29 +1358,6 @@ pub async fn monitor_db(
             }
         }
     };
-
-    // run every 5 minutes
-    let cleanup_concurrency_counters_f = async {
-        if server_mode && iteration.is_some() && iteration.as_ref().unwrap().should_run(10) {
-            if let Some(db) = conn.as_sql() {
-                if let Err(e) = cleanup_concurrency_counters_orphaned_keys(&db).await {
-                    tracing::error!("Error cleaning up concurrency counters: {:?}", e);
-                }
-            }
-        }
-    };
-
-    // run every 10 minutes
-    let cleanup_concurrency_counters_empty_keys_f = async {
-        if server_mode && iteration.is_some() && iteration.as_ref().unwrap().should_run(20) {
-            if let Some(db) = conn.as_sql() {
-                if let Err(e) = cleanup_concurrency_counters_empty_keys(&db).await {
-                    tracing::error!("Error cleaning up concurrency counters: {:?}", e);
-                }
-            }
-        }
-    };
-
     let expired_items_f = async {
         if server_mode && !initial_load {
             if let Some(db) = conn.as_sql() {
@@ -1476,8 +1441,6 @@ pub async fn monitor_db(
         low_disk_alerts_f,
         apply_autoscaling_f,
         update_min_worker_version_f,
-        cleanup_concurrency_counters_f,
-        cleanup_concurrency_counters_empty_keys_f,
     );
 }
 
@@ -2020,104 +1983,6 @@ async fn handle_zombie_jobs(db: &Pool<Postgres>, base_internal_url: &str, worker
     }
 }
 
-async fn cleanup_concurrency_counters_orphaned_keys(db: &DB) -> error::Result<()> {
-    let result = sqlx::query!(
-        "
-WITH lockable_counters AS (
-    SELECT concurrency_id, job_uuids
-    FROM concurrency_counter
-    WHERE job_uuids != '{}'::jsonb
-    FOR UPDATE SKIP LOCKED
-),
-all_job_uuids AS (
-    SELECT DISTINCT jsonb_object_keys(job_uuids) AS job_uuid
-    FROM lockable_counters
-),
-orphaned_job_uuids AS (
-    SELECT job_uuid
-    FROM all_job_uuids
-    WHERE job_uuid NOT IN (
-        SELECT id::text 
-        FROM v2_job_queue 
-        FOR SHARE SKIP LOCKED
-    )
-),
-orphaned_array AS (
-    SELECT ARRAY(SELECT job_uuid FROM orphaned_job_uuids) AS orphaned_keys
-),
-before_update AS (
-    SELECT lc.concurrency_id, lc.job_uuids, oa.orphaned_keys
-    FROM lockable_counters lc, orphaned_array oa
-    WHERE lc.job_uuids ?| oa.orphaned_keys
-),
-affected_rows AS (
-    UPDATE concurrency_counter 
-    SET job_uuids = job_uuids - orphaned_array.orphaned_keys
-    FROM orphaned_array
-    WHERE concurrency_counter.concurrency_id IN (
-        SELECT concurrency_id FROM before_update
-    )
-    RETURNING concurrency_id, job_uuids AS updated_job_uuids
-),
-expanded_orphaned AS (
-    SELECT bu.concurrency_id, 
-           bu.job_uuids AS original_job_uuids,
-           unnest(bu.orphaned_keys) AS orphaned_key
-    FROM before_update bu
-)
-SELECT 
-    eo.concurrency_id,
-    eo.orphaned_key,
-    eo.original_job_uuids,
-    ar.updated_job_uuids
-FROM expanded_orphaned eo
-JOIN affected_rows ar ON eo.concurrency_id = ar.concurrency_id
-WHERE eo.original_job_uuids ? eo.orphaned_key
-ORDER BY eo.concurrency_id, eo.orphaned_key
-",
-    )
-    .fetch_all(db)
-    .await?;
-
-    if result.len() > 0 {
-        tracing::info!("Cleaned up {} concurrency counters", result.len());
-        for row in result {
-            tracing::info!("Concurrency counter cleaned up: concurrency_id: {}, orphaned_key: {:?}, original_job_uuids: {:?}, updated_job_uuids: {:?}", row.concurrency_id, row.orphaned_key, row.original_job_uuids, row.updated_job_uuids);
-        }
-    }
-    Ok(())
-}
-
-async fn cleanup_concurrency_counters_empty_keys(db: &DB) -> error::Result<()> {
-    let result = sqlx::query!(
-        "
-WITH rows_to_delete AS (
-    SELECT concurrency_id
-    FROM concurrency_counter
-    WHERE job_uuids = '{}'::jsonb
-    FOR UPDATE SKIP LOCKED
-)
-DELETE FROM concurrency_counter
-WHERE concurrency_id IN (SELECT concurrency_id FROM rows_to_delete)  RETURNING concurrency_id",
-    )
-    .fetch_all(db)
-    .await?;
-
-    if result.len() > 0 {
-        tracing::info!(
-            "Cleaned up {} empty concurrency counters: {:?}",
-            result.len(),
-            result
-                .iter()
-                .map(|x| x.concurrency_id.to_string())
-                .collect::<Vec<_>>()
-                .join(",")
-        );
-    }
-
-    Ok(())
-}
-
 async fn handle_zombie_flows(db: &DB) -> error::Result<()> {
     let flows = sqlx::query!(
         r#"
@@ -2129,7 +1994,6 @@ async fn handle_zombie_flows(db: &DB) -> error::Result<()> {
             AND (job_kind = 'flow' OR job_kind = 'flowpreview' OR job_kind = 'flownode')
             AND last_ping IS NOT NULL AND last_ping < NOW() - ($1 || ' seconds')::interval
             AND canceled = false
-            
         "#,
         FLOW_ZOMBIE_TRANSITION_TIMEOUT.as_str()
     )

backend/tests/worker.rs

@@ -27,7 +27,7 @@ use serde::Serialize;
 use windmill_common::flows::InputTransform;
 use windmill_common::worker::WORKER_CONFIG;
 
-#[cfg(any(feature = "python", feature = "deno_core"))]
+#[cfg(feature = "python")]
 use windmill_common::flow_status::{FlowStatus, FlowStatusModule, RestartedFrom};
 
 use windmill_common::{

backend/windmill-api/Cargo.toml

@@ -92,7 +92,6 @@ magic-crypt.workspace = true
 tempfile.workspace = true
 tokio-util.workspace = true
 tokio-tar.workspace = true
-tokio-postgres.workspace = true
 hmac.workspace = true
 cookie.workspace = true
 sha2.workspace = true

backend/windmill-api/openapi.yaml

@@ -1,7 +1,7 @@
 openapi: "3.0.3"
 
 info:
-  version: 1.519.2
+  version: 1.515.1
   title: Windmill API
 
   contact:
@@ -728,50 +728,6 @@ paths:
               schema:
                 type: boolean
 
-  /settings/databases_exist:
-    post:
-      summary: checks that all given databases exist or else return the ones that don't
-      operationId: databasesExist
-      tags:
-        - setting
-      requestBody:
-        required: true
-        content:
-          application/json:
-            schema:
-              type: array
-              items:
-                type: string
-      responses:
-        "200":
-          description: databases that do not exist
-          content:
-            application/json:
-              schema:
-                type: array
-                items:
-                  type: string
-
-  /settings/create_ducklake_database/{name}:
-    post:
-      summary: Runs CREATE DATABASE on the Windmill Postgres and grants access to the ducklake_user
-      operationId: createDucklakeDatabase
-      tags:
-        - setting
-      parameters:
-        - in: path
-          name: name
-          required: true
-          schema:
-            type: string
-          description: The name of the database to create
-      responses:
-        "200":
-          description: status
-          content:
-            application/json:
-              schema: {}
-
   /settings/global/{key}:
     get:
       summary: get global settings
@@ -1962,8 +1918,6 @@ paths:
                     type: boolean
                   large_file_storage:
                     $ref: "#/components/schemas/LargeFileStorage"
-                  ducklake:
-                    $ref: "#/components/schemas/DucklakeSettings"
                   git_sync:
                     $ref: "#/components/schemas/WorkspaceGitSyncSettings"
                   deploy_ui:
@@ -2467,51 +2421,6 @@ paths:
               properties:
                 large_file_storage:
                   $ref: "#/components/schemas/LargeFileStorage"
-      responses:
-        "200":
-          description: status
-          content:
-            application/json:
-              schema: {}
-
-
-  /w/{workspace}/workspaces/list_ducklakes:
-    get:
-      summary: list ducklakes
-      operationId: listDucklakes
-      tags:
-        - workspace
-      parameters:
-        - $ref: "#/components/parameters/WorkspaceId"
-      responses:
-        "200":
-          description: status
-          content:
-            application/json:
-              schema:
-                type: array
-                items:
-                  type: string
-
-  /w/{workspace}/workspaces/edit_ducklake_config:
-    post:
-      summary: edit ducklake settings
-      operationId: editDucklakeConfig
-      tags:
-        - workspace
-      parameters:
-        - $ref: "#/components/parameters/WorkspaceId"
-      requestBody:
-        description: Ducklake settings
-        required: true
-        content:
-          application/json:
-            schema:
-              type: object
-              required: [settings]
-              properties:
-                settings:
-                  $ref: "#/components/schemas/DucklakeSettings"
 
       responses:
         "200":
@@ -2546,65 +2455,6 @@ paths:
             application/json:
               schema: {}
 
-  /w/{workspace}/workspaces/edit_git_sync_repository:
-    post:
-      summary: add or update individual git sync repository
-      operationId: editGitSyncRepository
-      tags:
-        - workspace
-      parameters:
-        - $ref: "#/components/parameters/WorkspaceId"
-      requestBody:
-        description: Git sync repository settings to add or update
-        required: true
-        content:
-          application/json:
-            schema:
-              type: object
-              properties:
-                git_repo_resource_path:
-                  type: string
-                  description: The resource path of the git repository to update
-                repository:
-                  $ref: "#/components/schemas/GitRepositorySettings"
-              required:
-                - git_repo_resource_path
-                - repository
-      responses:
-        "200":
-          description: status
-          content:
-            application/json:
-              schema: {}
-
-  /w/{workspace}/workspaces/delete_git_sync_repository:
-    delete:
-      summary: delete individual git sync repository
-      operationId: deleteGitSyncRepository
-      tags:
-        - workspace
-      parameters:
-        - $ref: "#/components/parameters/WorkspaceId"
-      requestBody:
-        description: Git sync repository to delete
-        required: true
-        content:
-          application/json:
-            schema:
-              type: object
-              properties:
-                git_repo_resource_path:
-                  type: string
-                  description: The resource path of the git repository to delete
-              required:
-                - git_repo_resource_path
-      responses:
-        "200":
-          description: status
-          content:
-            application/json:
-              schema: {}
-
   /w/{workspace}/workspaces/edit_deploy_ui_config:
     post:
       summary: edit workspace deploy ui settings
@@ -7798,18 +7648,10 @@ paths:
           in: query
           schema:
             type: integer
-        - name: stream_offset
-          in: query
-          schema:
-            type: integer
         - name: get_progress
           in: query
           schema:
             type: boolean
-        - name: no_logs
-          in: query
-          schema:
-            type: boolean
 
       responses:
         "200":
@@ -7831,10 +7673,6 @@ paths:
                     type: integer
                   progress:
                     type: integer
-                  stream_offset:
-                    type: integer
-                  new_result_stream:
-                    type: string
                   flow_status:
                     $ref: "../../openflow.openapi.yaml#/components/schemas/FlowStatus"
                   workflow_as_code_status:
@@ -7865,10 +7703,6 @@ paths:
           in: query
           schema:
             type: boolean
-        - name: no_logs
-          in: query
-          schema:
-            type: boolean
 
       responses:
         "200":
@@ -13815,24 +13649,6 @@ components:
           customai,
         ]
 
-    GitSyncObjectType:
-      type: string
-      enum:
-        - script
-        - flow
-        - app
-        - folder
-        - resource
-        - variable
-        - secret
-        - resourcetype
-        - schedule
-        - user
-        - group
-        - trigger
-        - settings
-        - key
-
     AIProviderModel:
       type: object
       properties:
@@ -14083,7 +13899,10 @@ components:
               path:
                 type: string
               kind:
-                $ref: "#/components/schemas/AssetKind"
+                type: string
+                enum:
+                  - s3object
+                  - resource
               access_type:
                 type: string
                 enum: [r, w, rw]
@@ -14856,7 +14675,7 @@ components:
                                 - key
                                 - typ
                     required:
-                      - object
+                      - object 
                   - type: object
                     properties:
                       list:
@@ -16836,7 +16655,6 @@ components:
       additionalProperties:
         type: boolean
 
-
     FlowMetadata:
       type: object
       properties:
@@ -17237,39 +17055,6 @@ components:
               public_resource:
                 type: boolean
 
-    DucklakeSettings:
-      type: object
-      required: [ducklakes]
-      properties:
-        ducklakes:
-          type: object
-          additionalProperties:
-            type: object
-            required: [catalog, storage]
-            properties:
-              catalog:
-                type: object
-                properties:
-                  resource_type:
-                    type: string
-                    enum:
-                      - postgresql
-                      - mysql
-                      - instance
-                  resource_path:
-                    type: string
-                required:
-                  - resource_type
-              storage:
-                type: object
-                properties:
-                  storage:
-                    type: string
-                  path:
-                    type: string
-                required:
-                  - path
-
     WindmillLargeFile:
       type: object
       properties:
@@ -17349,7 +17134,15 @@ components:
         include_type:
           type: array
           items:
-            $ref: "#/components/schemas/GitSyncObjectType"
+            type: string
+            enum:
+              - script
+              - flow
+              - app
+              - resource
+              - variable
+              - secret
+              - trigger
 
     WorkspaceDefaultScripts:
       type: object
@@ -17389,7 +17182,22 @@ components:
             include_type:
               type: array
               items:
-                $ref: "#/components/schemas/GitSyncObjectType"
+                type: string
+                enum:
+                  - script
+                  - flow
+                  - app
+                  - folder
+                  - resource
+                  - variable
+                  - secret
+                  - resourcetype
+                  - schedule
+                  - user
+                  - group
+                  - trigger
+                  - settings
+                  - key
             exclude_path:
               type: array
               items:
@@ -17401,7 +17209,22 @@ components:
         exclude_types_override:
           type: array
           items:
-            $ref: "#/components/schemas/GitSyncObjectType"
+            type: string
+            enum:
+              - script
+              - flow
+              - app
+              - folder
+              - resource
+              - variable
+              - secret
+              - resourcetype
+              - schedule
+              - user
+              - group
+              - trigger
+              - settings
+              - key
       required:
         - script_path
         - git_repo_resource_path
@@ -17848,7 +17671,6 @@ components:
       enum:
         - s3object
         - resource
-        - ducklake
     Asset:
       type: object
       properties:

backend/windmill-api/src/approvals.rs

@@ -1,24 +1,18 @@
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+use uuid::Uuid;
+use std::str::FromStr;
+use regex::Regex;
+use serde_json::Value;
 use crate::auth::OptTokened;
 use crate::db::{ApiAuthed, DB};
-use crate::jobs::{
-    cancel_suspended_job, get_resume_urls_internal, resume_suspended_job, QueryApprover,
-    QueryOrBody, ResumeUrls,
-};
-use axum::{
-    extract::{Path, Query},
-    Extension,
-};
-use regex::Regex;
-use serde::{Deserialize, Serialize};
-use serde_json::value::RawValue;
-use serde_json::Value;
-use std::collections::HashMap;
-use std::str::FromStr;
-use uuid::Uuid;
-use windmill_common::cache;
+use crate::jobs::{cancel_suspended_job, resume_suspended_job, QueryApprover, QueryOrBody, ResumeUrls, get_resume_urls_internal};
+use axum::{extract::{Path, Query}, Extension};
 use windmill_common::error::Error;
+use windmill_common::cache;
 use windmill_common::jobs::JobKind;
 use windmill_common::scripts::ScriptHash;
+use serde_json::value::RawValue;
 
 #[derive(Debug, Deserialize, Serialize)]
 pub struct ResumeSchema {
@@ -240,7 +234,7 @@ pub async fn get_approval_form_details(
     .ok_or_else(|| Error::BadRequest("This workflow is no longer running and has either already timed out or been cancelled or completed.".to_string()))
     .map(|r| (r.job_kind, r.script_hash, r.raw_flow, r.parent_job, r.created_at, r.created_by, r.script_path, r.args))?;
 
-    let flow_data = match cache::job::fetch_flow(&db, &job_kind, script_hash).await {
+    let flow_data = match cache::job::fetch_flow(&db, job_kind, script_hash).await {
         Ok(data) => data,
         Err(_) => {
             if let Some(parent_job_id) = parent_job_id.as_ref() {

backend/windmill-api/src/flows.rs

@@ -409,18 +409,11 @@ async fn create_flow(
     check_schedule_conflict(&mut tx, &w_id, &nf.path).await?;
 
     let schema_str = nf.schema.and_then(|x| serde_json::to_string(&x.0).ok());
+
     sqlx::query!(
-        r#"INSERT INTO flow (
-        workspace_id, path, summary, description,
-        dependency_job, lock_error_logs, draft_only, tag,
-        dedicated_worker, visible_to_runner_only, on_behalf_of_email,
-        value, schema, edited_by, edited_at
-    ) VALUES (
-        $1, $2, $3, $4,
-        NULL, '', $5, $6,
-        $7, $8, $9,
-        $10, $11::text::json, $12, now()
-    )"#,
+        "INSERT INTO flow (workspace_id, path, summary, description, \
+         dependency_job, lock_error_logs, draft_only, tag, dedicated_worker, visible_to_runner_only, on_behalf_of_email, value, schema, edited_by, edited_at) 
+         VALUES ($1, $2, $3, $4, NULL, '', $5, $6, $7, $8, $9, $10, $11::text::json, $12, now())",
         w_id,
         nf.path,
         nf.summary,
@@ -751,27 +744,11 @@ async fn update_flow(
     let schema_str = schema.and_then(|x| serde_json::to_string(&x).ok());
 
     sqlx::query!(
-        "
-        UPDATE 
-            flow 
-        SET
-            path = $1,
-            summary = $2,
-            description = $3,
-            dependency_job = NULL,
-            lock_error_logs = '',
-            draft_only = NULL,
-            tag = $4,
-            dedicated_worker = $5,
-            visible_to_runner_only = $6,
-            on_behalf_of_email = $7,
-            value = $8,
-            schema = $9::text::json,
-            edited_by = $10,
-            edited_at = now()
-        WHERE 
-            path = $11 AND workspace_id = $12",
-        if is_new_path { flow_path } else { &nf.path },
+        "UPDATE flow SET path = $1, summary = $2, description = $3,\
+        dependency_job = NULL, lock_error_logs = '', draft_only = NULL, tag = $4, dedicated_worker = $5, visible_to_runner_only = $6, on_behalf_of_email = $7, \
+        value = $8, schema = $9::text::json, edited_by = $10, edited_at = now()
+        WHERE path = $11 AND workspace_id = $12",
+        if is_new_path { flow_path } else { &nf.path }, // if new path, do not rename directly (to avoid flow_version foreign key constraint)
         nf.summary,
         nf.description.unwrap_or_else(String::new),
         nf.tag,
@@ -784,15 +761,12 @@ async fn update_flow(
         },
         nf.value,
         schema_str,
-        authed.username,
+        &authed.username,
         flow_path,
         w_id,
     )
     .execute(&mut *tx)
-    .await
-    .map_err(|e| {
-        error::Error::internal_err(format!("Error updating flow due to flow update: {e:#}"))
-    })?;
+    .await.map_err(|e| error::Error::internal_err(format!("Error updating flow due to flow update: {e:#}")))?;
 
     if is_new_path {
         // if new path, must clone flow to new path and delete old flow for flow_version foreign key constraint
@@ -1073,80 +1047,36 @@ async fn get_flow_by_path(
     let path = path.to_path();
     check_scopes(&authed, || format!("flows:read:{}", path))?;
     let mut tx = user_db.begin(&authed).await?;
+
     let flow_o = if query.with_starred_info.unwrap_or(false) {
         sqlx::query_as::<_, FlowWithStarred>(
-            r#"
-        SELECT 
-            flow.workspace_id, 
-            flow.path, 
-            flow.lock_error_logs, 
-            flow.summary, 
-            flow.description, 
-            flow.archived, 
-            flow.extra_perms, 
-            flow.draft_only, 
-            flow.dedicated_worker, 
-            flow.tag, 
-            flow.ws_error_handler_muted, 
-            flow.timeout, 
-            flow.visible_to_runner_only, 
-            flow.on_behalf_of_email, 
-            flow_version.schema, 
-            flow_version.value, 
-            flow_version.created_at AS edited_at, 
-            flow_version.created_by AS edited_by,
-            favorite.path IS NOT NULL AS starred
-        FROM flow
-        LEFT JOIN favorite
+            "SELECT flow.workspace_id, flow.path, flow.lock_error_logs, flow.summary, flow.description, flow.archived, flow.extra_perms, flow.draft_only, flow.dedicated_worker, flow.tag, flow.ws_error_handler_muted, flow.timeout, flow.visible_to_runner_only, flow.on_behalf_of_email, flow_version.schema, flow_version.value, flow_version.created_at as edited_at, flow_version.created_by as edited_by, favorite.path IS NOT NULL as starred
+            FROM flow
+            LEFT JOIN favorite
             ON favorite.favorite_kind = 'flow' 
-            AND favorite.workspace_id = flow.workspace_id 
-            AND favorite.path = flow.path 
-            AND favorite.usr = $3
-        LEFT JOIN flow_version 
-            ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]
-        WHERE flow.path = $1 AND flow.workspace_id = $2
-        "#,
+                AND favorite.workspace_id = flow.workspace_id 
+                AND favorite.path = flow.path 
+                AND favorite.usr = $3
+            LEFT JOIN flow_version ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]
+            WHERE flow.path = $1 AND flow.workspace_id = $2"
         )
-        .bind(path)
-        .bind(w_id)
-        .bind(&authed.username)
-        .fetch_optional(&mut *tx)
-        .await?
+            .bind(path)
+            .bind(w_id)
+            .bind(&authed.username)
+            .fetch_optional(&mut *tx)
+            .await?
     } else {
         sqlx::query_as::<_, FlowWithStarred>(
-            r#"
-        SELECT 
-            flow.workspace_id, 
-            flow.path, 
-            flow.lock_error_logs, 
-            flow.summary, 
-            flow.description, 
-            flow.archived, 
-            flow.extra_perms, 
-            flow.draft_only, 
-            flow.dedicated_worker, 
-            flow.tag, 
-            flow.ws_error_handler_muted, 
-            flow.timeout, 
-            flow.visible_to_runner_only, 
-            flow.on_behalf_of_email, 
-            flow_version.schema, 
-            flow_version.value, 
-            flow_version.created_at AS edited_at, 
-            flow_version.created_by AS edited_by, 
-            NULL AS starred
-        FROM flow
-        LEFT JOIN flow_version 
-            ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]
-        WHERE flow.path = $1 AND flow.workspace_id = $2
-        "#,
+            "SELECT flow.workspace_id, flow.path, flow.lock_error_logs, flow.summary, flow.description, flow.archived, flow.extra_perms, flow.draft_only, flow.dedicated_worker, flow.tag, flow.ws_error_handler_muted, flow.timeout, flow.visible_to_runner_only, flow.on_behalf_of_email, flow_version.schema, flow_version.value, flow_version.created_at as edited_at, flow_version.created_by as edited_by, NULL as starred
+            FROM flow
+            LEFT JOIN flow_version ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]
+            WHERE flow.path = $1 AND flow.workspace_id = $2"
         )
-        .bind(path)
-        .bind(w_id)
-        .fetch_optional(&mut *tx)
-        .await?
+            .bind(path)
+            .bind(w_id)
+            .fetch_optional(&mut *tx)
+            .await?
     };
-
     tx.commit().await?;
 
     let flow = not_found_if_none(flow_o, "Flow", path)?;
@@ -1185,36 +1115,20 @@ async fn get_flow_by_path_w_draft(
     let path = path.to_path();
     check_scopes(&authed, || format!("flows:read:{}", path))?;
     let mut tx = user_db.begin(&authed).await?;
+
     let flow_o = sqlx::query_as::<_, FlowWDraft>(
-        "SELECT
-            flow.path,
-            flow.summary,
-            flow.description,
-            flow_version.schema,
-            flow_version.value,
-            flow.extra_perms,
-            flow.draft_only,
-            flow.ws_error_handler_muted,
-            flow.dedicated_worker,
-            draft.value AS draft,
-            flow.tag,
-            flow.visible_to_runner_only,
-            flow.on_behalf_of_email
-        FROM flow
+        "SELECT flow.path, flow.summary, flow.description, flow_version.schema, flow_version.value, flow.extra_perms, flow.draft_only, flow.ws_error_handler_muted, flow.dedicated_worker, draft.value as draft, flow.tag, flow.visible_to_runner_only, flow.on_behalf_of_email
+         FROM flow
         LEFT JOIN draft
-            ON flow.path = draft.path
-            AND draft.workspace_id = $2
-            AND draft.typ = 'flow'
-        LEFT JOIN flow_version
+            ON flow.path = draft.path AND draft.workspace_id = $2 AND draft.typ = 'flow' 
+        LEFT JOIN flow_version 
             ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]
-        WHERE flow.path = $1
-        AND flow.workspace_id = $2",
+        WHERE flow.path = $1 AND flow.workspace_id = $2",
     )
     .bind(path)
     .bind(w_id)
     .fetch_optional(&mut *tx)
     .await?;
-
     tx.commit().await?;
 
     let flow = not_found_if_none(flow_o, "Flow", path)?;

backend/windmill-api/src/folders.rs

@@ -216,18 +216,6 @@ async fn create_folder(
     .execute(&mut *tx)
     .await?;
 
-    audit_log(
-        &mut *tx,
-        &authed,
-        "folder.create",
-        ActionKind::Create,
-        &w_id,
-        Some(&ng.name.to_string()),
-        None,
-    )
-    .await?;
-    tx.commit().await?;
-
     handle_deployment_metadata(
         &authed.email,
         &authed.username,
@@ -239,6 +227,17 @@ async fn create_folder(
     )
     .await?;
 
+    audit_log(
+        &mut *tx,
+        &authed,
+        "folder.create",
+        ActionKind::Create,
+        &w_id,
+        Some(&ng.name.to_string()),
+        None,
+    )
+    .await?;
+    tx.commit().await?;
     webhook.send_message(
         w_id.clone(),
         WebhookMessage::CreateFolder { workspace: w_id, name: ng.name.clone() },
@@ -368,18 +367,6 @@ async fn update_folder(
         }
     }
 
-    audit_log(
-        &mut *tx,
-        &authed,
-        "folder.update",
-        ActionKind::Update,
-        &w_id,
-        Some(&name.to_string()),
-        None,
-    )
-    .await?;
-    tx.commit().await?;
-
     handle_deployment_metadata(
         &authed.email,
         &authed.username,
@@ -391,6 +378,17 @@ async fn update_folder(
     )
     .await?;
 
+    audit_log(
+        &mut *tx,
+        &authed,
+        "folder.update",
+        ActionKind::Update,
+        &w_id,
+        Some(&name.to_string()),
+        None,
+    )
+    .await?;
+    tx.commit().await?;
     webhook.send_message(
         w_id.clone().clone(),
         WebhookMessage::UpdateFolder { workspace: w_id, name: name.to_owned() },

backend/windmill-api/src/jobs.rs

@@ -900,10 +900,11 @@ impl<'a> GetQuery<'a> {
     /// when pushed from an un-updated workers.
     /// This function is used to make the above change transparent for the API, as the returned jobs
     /// will have the raw values as if they were still in the tables.
-    async fn resolve_raw_values<T: JobCommon>(
+    async fn resolve_raw_values<T>(
         &self,
         db: &DB,
         id: Uuid,
+        kind: JobKind,
         hash: Option<ScriptHash>,
         job: &mut JobExtended<T>,
     ) {
@@ -916,18 +917,18 @@ impl<'a> GetQuery<'a> {
             // Try to fetch the flow from the cache, fallback to the preview flow.
             // NOTE: This could check for the job kinds instead of the `or_else` but it's not
             // necessary as `fetch_flow` return early if the job kind is not a preview one.
-            cache::job::fetch_flow(db, job.job_kind(), hash)
+            cache::job::fetch_flow(db, kind, hash)
                 .or_else(|_| cache::job::fetch_preview_flow(db, &id, raw_flow))
                 .await
                 .ok()
                 .inspect(|data| job.raw_flow = Some(sqlx::types::Json(data.raw_flow.clone())));
         }
-        if self.with_code && job.job_kind() == &JobKind::Preview {
+        if self.with_code {
             // Try to fetch the code from the cache, fallback to the preview code.
             // NOTE: This could check for the job kinds instead of the `or_else` but it's not
             // necessary as `fetch_script` return early if the job kind is not a preview one.
             let conn = Connection::from(db.clone());
-            cache::job::fetch_script(db.clone(), job.job_kind(), hash)
+            cache::job::fetch_script(db.clone(), kind, hash)
                 .or_else(|_| cache::job::fetch_preview_script(&conn, &id, raw_lock, raw_code))
                 .await
                 .ok()
@@ -957,7 +958,7 @@ impl<'a> GetQuery<'a> {
 
         self.check_auth(job.as_ref().map(|job| job.created_by.as_str()))?;
         if let Some(job) = job.as_mut() {
-            self.resolve_raw_values(&db, job.id, job.script_hash, job)
+            self.resolve_raw_values(&db, job.id, job.job_kind, job.script_hash, job)
                 .await;
         }
         if self.with_flow {
@@ -991,7 +992,7 @@ impl<'a> GetQuery<'a> {
 
         self.check_auth(cjob.as_ref().map(|job| job.created_by.as_str()))?;
         if let Some(job) = cjob.as_mut() {
-            self.resolve_raw_values(db, job.id, job.script_hash, job)
+            self.resolve_raw_values(db, job.id, job.job_kind, job.script_hash, job)
                 .await;
         }
 
@@ -2641,7 +2642,7 @@ pub async fn get_resume_urls_internal(
 }
 
 #[derive(sqlx::FromRow, Debug, Serialize)]
-pub struct JobExtended<T: JobCommon> {
+pub struct JobExtended<T> {
     #[sqlx(flatten)]
     #[serde(flatten)]
     inner: T,
@@ -2664,23 +2665,7 @@ pub struct JobExtended<T: JobCommon> {
     pub aggregate_wait_time_ms: Option<i64>,
 }
 
-pub trait JobCommon {
-    fn job_kind(&self) -> &JobKind;
-}
-
-impl JobCommon for QueuedJob {
-    fn job_kind(&self) -> &JobKind {
-        &self.job_kind
-    }
-}
-
-impl JobCommon for CompletedJob {
-    fn job_kind(&self) -> &JobKind {
-        &self.job_kind
-    }
-}
-
-impl<T: JobCommon> JobExtended<T> {
+impl<T> JobExtended<T> {
     pub fn new(
         self_wait_time_ms: Option<i64>,
         aggregate_wait_time_ms: Option<i64>,
@@ -2698,7 +2683,7 @@ impl<T: JobCommon> JobExtended<T> {
     }
 }
 
-impl<T: JobCommon> Deref for JobExtended<T> {
+impl<T> Deref for JobExtended<T> {
     type Target = T;
 
     fn deref(&self) -> &Self::Target {
@@ -2706,7 +2691,7 @@ impl<T: JobCommon> Deref for JobExtended<T> {
     }
 }
 
-impl<T: JobCommon> DerefMut for JobExtended<T> {
+impl<T> DerefMut for JobExtended<T> {
     fn deref_mut(&mut self) -> &mut Self::Target {
         &mut self.inner
     }
@@ -4553,13 +4538,8 @@ pub async fn run_wait_result_script_by_path_internal(
     check_queue_too_long(&db, QUEUE_LIMIT_WAIT_RESULT.or(run_query.queue_limit)).await?;
 
     let mut tx = user_db.clone().begin(&authed).await?;
-    let (job_payload, tag, delete_after_use, timeout, on_behalf_of) = script_path_to_payload(
-        script_path.to_path(),
-        &mut *tx,
-        &w_id,
-        run_query.skip_preprocessor,
-    )
-    .await?;
+    let (job_payload, tag, delete_after_use, timeout, on_behalf_of) =
+        script_path_to_payload(script_path.to_path(), &mut *tx, &w_id, run_query.skip_preprocessor).await?;
     drop(tx);
 
     let tag = run_query.tag.clone().or(tag);
@@ -5700,38 +5680,22 @@ pub async fn run_job_by_hash_inner(
 pub struct JobUpdateQuery {
     pub running: Option<bool>,
     pub log_offset: Option<i32>,
-    pub stream_offset: Option<i32>,
     pub get_progress: Option<bool>,
-    pub no_logs: Option<bool>,
     pub only_result: Option<bool>,
     pub fast: Option<bool>,
 }
 
 #[derive(Serialize, Debug)]
 pub struct JobUpdate {
-    #[serde(skip_serializing_if = "Option::is_none")]
     pub running: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
     pub completed: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
     pub new_logs: Option<String>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub new_result_stream: Option<String>,
-    #[serde(skip_serializing_if = "Option::is_none")]
     pub log_offset: Option<i32>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub stream_offset: Option<i32>,
-    #[serde(skip_serializing_if = "Option::is_none")]
     pub mem_peak: Option<i32>,
-    #[serde(skip_serializing_if = "Option::is_none")]
     pub progress: Option<i32>,
-    #[serde(skip_serializing_if = "Option::is_none")]
     pub flow_status: Option<Box<serde_json::value::RawValue>>,
-    #[serde(skip_serializing_if = "Option::is_none")]
     pub workflow_as_code_status: Option<Box<serde_json::value::RawValue>>,
-    #[serde(skip_serializing_if = "Option::is_none")]
     pub job: Option<Job>,
-    #[serde(skip_serializing_if = "Option::is_none")]
     pub only_result: Option<Box<serde_json::value::RawValue>>,
 }
 
@@ -5750,7 +5714,6 @@ impl Hash for JobUpdate {
         self.log_offset.hash(state);
         self.mem_peak.hash(state);
         self.progress.hash(state);
-        self.stream_offset.hash(state);
         if !self.completed.unwrap_or(false) {
             self.flow_status.as_ref().map(|x| x.get().hash(state));
             self.workflow_as_code_status
@@ -5816,7 +5779,7 @@ async fn get_job_update(
     opt_tokened: OptTokened,
     Extension(db): Extension<DB>,
     Path((w_id, job_id)): Path<(String, Uuid)>,
-    Query(JobUpdateQuery { log_offset, stream_offset, get_progress, running, only_result, no_logs, .. }): Query<
+    Query(JobUpdateQuery { log_offset, get_progress, running, only_result, .. }): Query<
         JobUpdateQuery,
     >,
 ) -> JsonResult<JobUpdate> {
@@ -5828,13 +5791,11 @@ async fn get_job_update(
             &w_id,
             &job_id,
             log_offset,
-            stream_offset,
             get_progress,
             running,
             true,
             false,
             only_result,
-            no_logs,
         )
         .await?,
     ))
@@ -5845,7 +5806,7 @@ async fn get_job_update_sse(
     opt_tokened: OptTokened,
     Extension(db): Extension<DB>,
     Path((w_id, job_id)): Path<(String, Uuid)>,
-    Query(JobUpdateQuery { log_offset, stream_offset, get_progress, running, no_logs, only_result, fast }): Query<
+    Query(JobUpdateQuery { log_offset, get_progress, running, only_result, fast }): Query<
         JobUpdateQuery,
     >,
 ) -> Response {
@@ -5856,12 +5817,10 @@ async fn get_job_update_sse(
         w_id,
         job_id,
         log_offset,
-        stream_offset,
         get_progress,
         running,
         only_result,
         fast,
-        no_logs,
     )
     .map(|x| {
         format!(
@@ -5898,24 +5857,19 @@ fn get_job_update_sse_stream(
     w_id: String,
     job_id: Uuid,
     initial_log_offset: Option<i32>,
-    initial_stream_offset: Option<i32>,
     get_progress: Option<bool>,
     running: Option<bool>,
     only_result: Option<bool>,
     fast: Option<bool>,
-    no_logs: Option<bool>,
 ) -> impl futures::Stream<Item = JobUpdateSSEStream> {
     let (tx, rx) = tokio::sync::mpsc::channel(32);
 
     tokio::spawn(async move {
         let mut log_offset = initial_log_offset;
-        let mut stream_offset = initial_stream_offset;
         let mut last_update_hash: Option<String> = None;
 
         // Send initial update immediately
         let mut running = running;
-        let mut mem_peak = 0;
-
         match get_job_update_data(
             &opt_authed,
             &opt_tokened,
@@ -5923,38 +5877,22 @@ fn get_job_update_sse_stream(
             &w_id,
             &job_id,
             log_offset,
-            stream_offset,
             get_progress,
             running,
             true,
             true,
             only_result,
-            no_logs,
         )
         .await
         {
-            Ok(mut update) => {
+            Ok(update) => {
                 last_update_hash = Some(update.hash_str());
                 let completion_sent = update.completed.unwrap_or(false);
                 if running.is_some() && update.running.is_some_and(|x| x) {
                     running = Some(true);
                 }
-                if let Some(new_mem_peak) = update.mem_peak {
-                    mem_peak = new_mem_peak;
-                }
                 if let Some(new_offset) = update.log_offset {
-                    if new_offset != log_offset.unwrap_or(0) {
-                        log_offset = Some(new_offset);
-                    } else {
-                        update.log_offset = None;
-                    }
-                }
-                if let Some(new_stream_offset) = update.stream_offset {
-                    if new_stream_offset != stream_offset.unwrap_or(0) {
-                        stream_offset = Some(new_stream_offset);
-                    } else {
-                        update.stream_offset = None;
-                    }
+                    log_offset = Some(new_offset);
                 }
                 if tx.send(JobUpdateSSEStream::Update(update)).await.is_err() {
                     tracing::warn!("Failed to send initial job update for job {job_id}");
@@ -5980,7 +5918,6 @@ fn get_job_update_sse_stream(
         let mut i = 0;
         let start = Instant::now();
         let mut last_ping = Instant::now();
-
         loop {
             i += 1;
             let ms_duration = if i > 100 || !fast.unwrap_or(false) {
@@ -6013,27 +5950,18 @@ fn get_job_update_sse_stream(
                 &w_id,
                 &job_id,
                 log_offset,
-                stream_offset,
                 get_progress,
                 running,
                 false,
                 true,
                 only_result,
-                no_logs,
             )
             .await
             {
-                Ok(mut update) => {
+                Ok(update) => {
                     if running.is_some() && update.running.is_some_and(|x| x) {
                         running = Some(true);
                     }
-                    if update.completed.is_some_and(|x| !x) {
-                        update.completed = None;
-                    }
-                    if update.new_logs.as_ref().is_some_and(|x| x.is_empty()) {
-                        update.new_logs = None;
-                    }
-
                     // if !only_result.unwrap_or(false) {
                     //     tracing::error!("update {:?}", update);
                     // }
@@ -6042,25 +5970,7 @@ fn get_job_update_sse_stream(
                     if last_update_hash.as_ref() != Some(&update_last_status) {
                         // Update log offset if available
                         if let Some(new_offset) = update.log_offset {
-                            if new_offset != log_offset.unwrap_or(0) {
-                                log_offset = Some(new_offset);
-                            } else {
-                                update.log_offset = None;
-                            }
-                        }
-                        if let Some(new_stream_offset) = update.stream_offset {
-                            if new_stream_offset != stream_offset.unwrap_or(0) {
-                                stream_offset = Some(new_stream_offset);
-                            } else {
-                                update.stream_offset = None;
-                            }
-                        }
-                        if let Some(new_mem_peak) = update.mem_peak {
-                            if new_mem_peak != mem_peak {
-                                mem_peak = new_mem_peak;
-                            } else {
-                                update.mem_peak = None;
-                            }
+                            log_offset = Some(new_offset);
                         }
                         let completed = update.completed.unwrap_or(false);
                         if tx.send(JobUpdateSSEStream::Update(update)).await.is_err() {
@@ -6073,8 +5983,7 @@ fn get_job_update_sse_stream(
                         last_update_hash = Some(update_last_status);
                     }
                 }
-                Err(e) => {
-                    tracing::error!("Error getting job update: {:?}", e);
+                Err(_) => {
                     if tx.send(JobUpdateSSEStream::NotFound).await.is_err() {
                         tracing::warn!("Failed to send job not found for job {job_id}");
                     }
@@ -6094,13 +6003,11 @@ async fn get_job_update_data(
     w_id: &str,
     job_id: &Uuid,
     log_offset: Option<i32>,
-    stream_offset: Option<i32>,
     get_progress: Option<bool>,
     running: Option<bool>,
     log_view: bool,
     get_full_job_on_completion: bool,
     only_result: Option<bool>,
-    no_logs: Option<bool>,
 ) -> error::Result<JobUpdate> {
     let tags = if log_view {
         log_job_view(
@@ -6121,22 +6028,19 @@ async fn get_job_update_data(
 
     if only_result.unwrap_or(false) {
         let result = if let Some(tags) = tags {
-            let r = 
-                sqlx::query!(
-                    "SELECT result as \"result: sqlx::types::Json<Box<RawValue>>\", v2_job.tag,
-                v2_job_queue.running as \"running: Option<bool>\", SUBSTR(rs.stream, $3) AS \"result_stream: Option<String>\", CHAR_LENGTH(rs.stream) AS stream_offset
+            let r = sqlx::query!(
+                "SELECT result as \"result: sqlx::types::Json<Box<RawValue>>\", v2_job.tag,
+                v2_job_queue.running as \"running: Option<bool>\"
                 FROM v2_job
                 LEFT JOIN v2_job_queue USING (id)
                 LEFT JOIN v2_job_completed USING (id)
-                LEFT JOIN job_result_stream rs ON rs.job_id = $2
                 WHERE v2_job.id = $2 AND v2_job.workspace_id = $1",
-                    w_id,
-                    job_id,
-                    stream_offset.unwrap_or(0),
-                )
-                .fetch_optional(db)
-                .await?
-                .ok_or_else(|| Error::NotFound(format!("Job not found: {}", job_id)))?;
+                w_id,
+                job_id,
+            )
+            .fetch_optional(db)
+            .await?
+            .ok_or_else(|| Error::NotFound(format!("Job not found: {}", job_id)))?;
 
             if !tags.contains(&r.tag.as_str()) {
                 return Err(Error::NotAuthorized(format!(
@@ -6146,44 +6050,27 @@ async fn get_job_update_data(
                 )));
             }
             let running = r.running.as_ref().map(|x| *x);
-            (r.result.map(|x| x.0), running, r.result_stream.flatten(), r.stream_offset)
+            (r.result.map(|x| x.0), running)
         } else {
             if running.is_some_and(|x| !x) {
                 let r = sqlx::query!(
-                    "SELECT 
-                        result as \"result: sqlx::types::Json<Box<RawValue>>\",
-                        v2_job_queue.running as \"running: Option<bool>\",
-                        SUBSTR(rs.stream, $3) AS \"result_stream: Option<String>\",
-                        CHAR_LENGTH(rs.stream) + 1 AS stream_offset
-                    FROM v2_job_completed FULL OUTER JOIN v2_job_queue USING (id) 
-                    LEFT JOIN job_result_stream rs ON rs.job_id = $1
-                    WHERE (v2_job_queue.id = $1 AND v2_job_queue.workspace_id = $2) OR (v2_job_completed.id = $1 AND v2_job_completed.workspace_id = $2)",
+                    "SELECT result as \"result: sqlx::types::Json<Box<RawValue>>\", v2_job_queue.running as \"running: Option<bool>\" FROM v2_job_completed FULL OUTER JOIN v2_job_queue USING (id) WHERE (v2_job_queue.id = $1 AND v2_job_queue.workspace_id = $2) OR (v2_job_completed.id = $1 AND v2_job_completed.workspace_id = $2)",
                     job_id,
                     w_id,
-                    stream_offset.unwrap_or(0),
                 ).fetch_optional(db).await?;
                 if let Some(r) = r {
                     let running = r.running.as_ref().map(|x| *x);
-                    (r.result.map(|x| x.0), running, r.result_stream.flatten(), r.stream_offset)
+                    (r.result.map(|x| x.0), running)
                 } else {
-                    (None, None, None, None)
+                    (None, None)
                 }
             } else {
-                let q =  sqlx::query!(
-                    "SELECT result as \"result: sqlx::types::Json<Box<RawValue>>\", SUBSTR(rs.stream, $3) AS \"result_stream: Option<String>\", CHAR_LENGTH(rs.stream) + 1 AS stream_offset
-                 FROM v2_job_completed  FULL OUTER JOIN job_result_stream rs ON rs.job_id = v2_job_completed.id WHERE (v2_job_completed.id = $2 AND v2_job_completed.workspace_id = $1 OR rs.workspace_id = $1)",
+                (sqlx::query_scalar!(
+                    "SELECT result as \"result: sqlx::types::Json<Box<RawValue>>\" FROM v2_job_completed WHERE id = $2 AND workspace_id = $1",
                     w_id,
                     job_id,
-                    stream_offset.unwrap_or(0),
-                )
-                .fetch_optional(db)
-                .await?;
-                tracing::error!("q {:?}", q);
-                if let Some(r) = q {
-                    (r.result.map(|x| x.0), running, r.result_stream.flatten(), r.stream_offset)
-                } else {
-                    (None, None, None, None)
-                }
+                ).fetch_optional(db).await?.flatten()
+                .map(|x| x.0), running)
             }
         };
         Ok(JobUpdate {
@@ -6191,8 +6078,6 @@ async fn get_job_update_data(
             completed: if result.0.is_some() { Some(true) } else { None },
             log_offset: None,
             new_logs: None,
-            new_result_stream: result.2,
-            stream_offset: result.3,
             mem_peak: None,
             progress: None,
             job: None,
@@ -6208,24 +6093,20 @@ async fn get_job_update_data(
                     WHEN q.id IS NOT NULL THEN (CASE WHEN NOT $5 AND q.running THEN true ELSE null END)
                     ELSE false
                 END AS running,
-                CASE WHEN $7::BOOLEAN THEN NULL ELSE SUBSTR(logs, GREATEST($1 - log_offset, 0)) END AS logs,
-                SUBSTR(rs.stream, $8) AS new_result_stream,
+                SUBSTR(logs, GREATEST($1 - log_offset, 0)) AS logs,
                 COALESCE(r.memory_peak, c.memory_peak) AS mem_peak,
                 COALESCE(c.flow_status, f.flow_status) AS \"flow_status: sqlx::types::Json<Box<RawValue>>\",
                 COALESCE(c.workflow_as_code_status, f.workflow_as_code_status) AS \"workflow_as_code_status: sqlx::types::Json<Box<RawValue>>\",
-                CASE WHEN $7::BOOLEAN THEN NULL ELSE job_logs.log_offset + CHAR_LENGTH(job_logs.logs) + 1 END AS log_offset,
-                CHAR_LENGTH(rs.stream) + 1 AS stream_offset,
+                job_logs.log_offset + CHAR_LENGTH(job_logs.logs) + 1 AS log_offset,
                 created_by AS \"created_by!\",
                 CASE WHEN $4::BOOLEAN THEN (
                     SELECT scalar_int FROM job_stats WHERE job_id = $3 AND metric_id = 'progress_perc'
-                ) END AS progress,
-                rs.stream AS \"result_stream: Option<String>\"
+                ) END AS progress
             FROM v2_job j
                 LEFT JOIN v2_job_queue q USING (id)
                 LEFT JOIN v2_job_runtime r USING (id)
                 LEFT JOIN v2_job_status f USING (id)
                 LEFT JOIN v2_job_completed c USING (id)
-                LEFT JOIN job_result_stream rs ON rs.job_id = $3
                 LEFT JOIN job_logs ON job_logs.job_id =  $3
             WHERE j.workspace_id = $2 AND j.id = $3
             AND ($6::text[] IS NULL OR j.tag = ANY($6))",
@@ -6235,8 +6116,6 @@ async fn get_job_update_data(
             get_progress.unwrap_or(false),
             running,
             tags.as_ref().map(|v| v.as_slice()) as Option<&[&str]>,
-            no_logs.unwrap_or(false),
-            stream_offset.unwrap_or(0),
         )
         .fetch_optional(db)
         .await?
@@ -6260,8 +6139,6 @@ async fn get_job_update_data(
             completed: record.completed,
             log_offset: record.log_offset,
             new_logs: record.logs,
-            new_result_stream: record.new_result_stream,
-            stream_offset: record.stream_offset,
             mem_peak: record.mem_peak,
             progress: record.progress,
             workflow_as_code_status: record

backend/windmill-api/src/resources.rs

@@ -1069,18 +1069,6 @@ async fn create_resource_type(
     .execute(&mut *tx)
     .await?;
 
-    audit_log(
-        &mut *tx,
-        &authed,
-        "resource_types.create",
-        ActionKind::Create,
-        &w_id,
-        Some(&resource_type.name),
-        None,
-    )
-    .await?;
-    tx.commit().await?;
-
     handle_deployment_metadata(
         &authed.email,
         &authed.username,
@@ -1095,6 +1083,18 @@ async fn create_resource_type(
     )
     .await?;
 
+    audit_log(
+        &mut *tx,
+        &authed,
+        "resource_types.create",
+        ActionKind::Create,
+        &w_id,
+        Some(&resource_type.name),
+        None,
+    )
+    .await?;
+    tx.commit().await?;
+
     webhook.send_message(
         w_id.clone(),
         WebhookMessage::CreateResourceType { name: resource_type.name.clone() },

backend/windmill-api/src/schedule.rs

@@ -25,11 +25,7 @@ use std::str::FromStr;
 use windmill_audit::audit_oss::audit_log;
 use windmill_audit::ActionKind;
 use windmill_common::{
-    db::UserDB,
-    error::{Error, JsonResult, Result},
-    schedule::Schedule,
-    utils::{not_found_if_none, paginate, Pagination, ScheduleType, StripPath},
-    worker::to_raw_value,
+    db::UserDB, error::{Error, JsonResult, Result}, schedule::Schedule, utils::{not_found_if_none, paginate, Pagination, ScheduleType, StripPath}, worker::to_raw_value
 };
 use windmill_git_sync::{handle_deployment_metadata, DeployedObject};
 use windmill_queue::schedule::push_scheduled_job;
@@ -257,6 +253,17 @@ async fn create_schedule(
     .await
     .map_err(|e| Error::internal_err(format!("inserting schedule in {w_id}: {e:#}")))?;
 
+    handle_deployment_metadata(
+        &authed.email,
+        &authed.username,
+        &db,
+        &w_id,
+        DeployedObject::Schedule { path: ns.path.clone() },
+        Some(format!("Schedule '{}' created", ns.path.clone())),
+        true,
+    )
+    .await?;
+
     audit_log(
         &mut *tx,
         &authed,
@@ -281,17 +288,6 @@ async fn create_schedule(
     }
     tx.commit().await?;
 
-    handle_deployment_metadata(
-        &authed.email,
-        &authed.username,
-        &db,
-        &w_id,
-        DeployedObject::Schedule { path: ns.path.clone() },
-        Some(format!("Schedule '{}' created", ns.path.clone())),
-        true,
-    )
-    .await?;
-
     Ok(ns.path.to_string())
 }
 
@@ -402,6 +398,17 @@ async fn edit_schedule(
     .await
     .map_err(|e| Error::internal_err(format!("updating schedule in {w_id}: {e:#}")))?;
 
+    handle_deployment_metadata(
+        &authed.email,
+        &authed.username,
+        &db,
+        &w_id,
+        DeployedObject::Schedule { path: path.to_string() },
+        None,
+        true,
+    )
+    .await?;
+
     audit_log(
         &mut *tx,
         &authed,
@@ -423,17 +430,6 @@ async fn edit_schedule(
     }
     tx.commit().await?;
 
-    handle_deployment_metadata(
-        &authed.email,
-        &authed.username,
-        &db,
-        &w_id,
-        DeployedObject::Schedule { path: path.to_string() },
-        None,
-        true,
-    )
-    .await?;
-
     Ok(path.to_string())
 }
 
@@ -671,6 +667,17 @@ pub async fn set_enabled(
 
     clear_schedule(&mut tx, path, &w_id).await?;
 
+    handle_deployment_metadata(
+        &authed.email,
+        &authed.username,
+        &db,
+        &w_id,
+        DeployedObject::Schedule { path: path.to_string() },
+        None,
+        true,
+    )
+    .await?;
+
     audit_log(
         &mut *tx,
         &authed,
@@ -687,17 +694,6 @@ pub async fn set_enabled(
     }
     tx.commit().await?;
 
-    handle_deployment_metadata(
-        &authed.email,
-        &authed.username,
-        &db,
-        &w_id,
-        DeployedObject::Schedule { path: path.to_string() },
-        None,
-        true,
-    )
-    .await?;
-
     Ok(format!(
         "succesfully updated schedule at path {} to status {}",
         path, payload.enabled
@@ -794,6 +790,17 @@ async fn delete_schedule(
         )));
     }
 
+    handle_deployment_metadata(
+        &authed.email,
+        &authed.username,
+        &db,
+        &w_id,
+        DeployedObject::Schedule { path: path.to_string() },
+        Some(format!("Schedule '{}' deleted", path)),
+        true,
+    )
+    .await?;
+
     audit_log(
         &mut *tx,
         &authed,
@@ -807,17 +814,6 @@ async fn delete_schedule(
 
     tx.commit().await?;
 
-    handle_deployment_metadata(
-        &authed.email,
-        &authed.username,
-        &db,
-        &w_id,
-        DeployedObject::Schedule { path: path.to_string() },
-        Some(format!("Schedule '{}' deleted", path)),
-        true,
-    )
-    .await?;
-
     Ok(format!("schedule {} deleted", path))
 }
 

backend/windmill-api/src/scripts.rs

@@ -395,7 +395,7 @@ async fn create_snapshot_script(
     let mut script_hash = None;
     let mut tx = None;
     let mut uploaded = false;
-    let mut handle_deployment_metadata = None;
+
     while let Some(field) = multipart.next_field().await.unwrap() {
         let name = field.name().unwrap().to_string();
         let data = field.bytes().await.unwrap();
@@ -403,7 +403,7 @@ async fn create_snapshot_script(
             let ns: NewScript = Some(serde_json::from_slice(&data).map_err(to_anyhow)?).unwrap();
             let is_tar = ns.codebase.as_ref().is_some_and(|x| x.ends_with(".tar"));
 
-            let (new_hash, ntx, hdm) = create_script_internal(
+            let (new_hash, ntx) = create_script_internal(
                 ns,
                 w_id.clone(),
                 authed.clone(),
@@ -415,7 +415,6 @@ async fn create_snapshot_script(
             let nh = new_hash.to_string();
             script_hash = Some(if is_tar { format!("{nh}.tar") } else { nh });
             tx = Some(ntx);
-            handle_deployment_metadata = hdm;
         }
         if name == "file" {
             let hash = script_hash.as_ref().ok_or_else(|| {
@@ -478,9 +477,6 @@ async fn create_snapshot_script(
     }
 
     tx.unwrap().commit().await?;
-    if let Some(hdm) = handle_deployment_metadata {
-        hdm.handle(&db).await?;
-    }
     return Ok((StatusCode::CREATED, format!("{}", script_hash.unwrap())));
 }
 
@@ -510,38 +506,11 @@ async fn create_script(
     Path(w_id): Path<String>,
     Json(ns): Json<NewScript>,
 ) -> Result<(StatusCode, String)> {
-    let (hash, tx, hdm) =
-        create_script_internal(ns, w_id, authed, db.clone(), user_db, webhook).await?;
+    let (hash, tx) = create_script_internal(ns, w_id, authed, db, user_db, webhook).await?;
     tx.commit().await?;
-    if let Some(hdm) = hdm {
-        hdm.handle(&db).await?;
-    }
     Ok((StatusCode::CREATED, format!("{}", hash)))
 }
 
-struct HandleDeploymentMetadata {
-    email: String,
-    created_by: String,
-    w_id: String,
-    obj: DeployedObject,
-    deployment_message: Option<String>,
-}
-
-impl HandleDeploymentMetadata {
-    async fn handle(self, db: &DB) -> Result<()> {
-        handle_deployment_metadata(
-            &self.email,
-            &self.created_by,
-            &db,
-            &self.w_id,
-            self.obj,
-            self.deployment_message,
-            false,
-        )
-        .await
-    }
-}
-
 async fn create_script_internal<'c>(
     ns: NewScript,
     w_id: String,
@@ -549,11 +518,7 @@ async fn create_script_internal<'c>(
     db: sqlx::Pool<Postgres>,
     user_db: UserDB,
     webhook: WebhookShared,
-) -> Result<(
-    ScriptHash,
-    Transaction<'c, Postgres>,
-    Option<HandleDeploymentMetadata>,
-)> {
+) -> Result<(ScriptHash, Transaction<'c, Postgres>)> {
     check_scopes(&authed, || format!("scripts:write:{}", ns.path))?;
 
     let codebase = ns.codebase.as_ref();
@@ -1024,74 +989,57 @@ async fn create_script_internal<'c>(
             Some(&authed.clone().into()),
         )
         .await?;
-        Ok((hash, new_tx, None))
+        Ok((hash, new_tx))
     } else {
-        if codebase.is_none() {
-            let db2 = db.clone();
-            let w_id2 = w_id.clone();
-            let authed2 = authed.clone();
-            let permissioned_as2 = permissioned_as.clone();
-            let script_path2 = script_path.clone();
-            let parent_path = p_path_opt.clone();
-            let lock = ns.lock.clone();
-            let deployment_message = ns.deployment_message.clone();
-            let content = ns.content.clone();
-            let language = ns.language.clone();
-            tokio::spawn(async move {
-                // wait for 10 seconds to make sure the script is deployed and that the CLI sync that pushed it (f one) is complete
-                tokio::time::sleep(std::time::Duration::from_secs(10)).await;
-                if let Err(e) = process_relative_imports(
-                    &db2,
-                    None,
-                    None,
-                    &w_id2,
-                    &script_path2,
-                    parent_path,
-                    deployment_message,
-                    &content,
-                    &Some(language),
-                    &authed2.email,
-                    &authed2.username,
-                    &permissioned_as2,
-                    lock,
-                )
-                .await
-                {
-                    tracing::error!(%e, "error processing relative imports");
-                }
-            });
-        }
+        let db2 = db.clone();
+        let w_id2 = w_id.clone();
+        let authed2 = authed.clone();
+        let permissioned_as2 = permissioned_as.clone();
+        let script_path2 = script_path.clone();
+        let parent_path = p_path_opt.clone();
+        let lock = ns.lock.clone();
+        let deployment_message = ns.deployment_message.clone();
+        let content = ns.content.clone();
+        let language = ns.language.clone();
+        tokio::spawn(async move {
+            // wait for 10 seconds to make sure the script is deployed and that the CLI sync that pushed it (f one) is complete
+            tokio::time::sleep(std::time::Duration::from_secs(10)).await;
+            if let Err(e) = process_relative_imports(
+                &db2,
+                None,
+                None,
+                &w_id2,
+                &script_path2,
+                parent_path,
+                deployment_message,
+                &content,
+                &Some(language),
+                &authed2.email,
+                &authed2.username,
+                &permissioned_as2,
+                lock,
+            )
+            .await
+            {
+                tracing::error!(%e, "error processing relative imports");
+            }
+        });
 
-        // handle_deployment_metadata(
-        //     &authed.email,
-        //     &authed.username,
-        //     &db,
-        //     &w_id,
-        //     DeployedObject::Script {
-        //         hash: hash.clone(),
-        //         path: script_path.clone(),
-        //         parent_path: p_path_opt,
-        //     },
-        //     ns.deployment_message,
-        //     false,
-        // )
-        // .await?;
-
-        Ok((
-            hash,
-            tx,
-            Some(HandleDeploymentMetadata {
-                email: authed.email,
-                created_by: authed.username,
-                w_id,
-                obj: DeployedObject::Script {
-                    hash: hash.clone(),
-                    path: script_path.clone(),
-                    parent_path: p_path_opt,
-                },
-                deployment_message: ns.deployment_message,
-            }),
-        ))
+        handle_deployment_metadata(
+            &authed.email,
+            &authed.username,
+            &db,
+            &w_id,
+            DeployedObject::Script {
+                hash: hash.clone(),
+                path: script_path.clone(),
+                parent_path: p_path_opt,
+            },
+            ns.deployment_message,
+            false,
+        )
+        .await?;
+        Ok((hash, tx))
     }
 }
 

backend/windmill-api/src/settings.rs

@@ -30,20 +30,19 @@ use crate::utils::require_devops_role;
 use serde::Deserialize;
 #[cfg(feature = "enterprise")]
 use windmill_common::ee_oss::{send_critical_alert, CriticalAlertKind, CriticalErrorChannel};
-use windmill_common::error::to_anyhow;
 use windmill_common::{
     email_oss::send_email,
     error::{self, JsonResult, Result},
-    get_database_url,
     global_settings::{
         AUTOMATE_USERNAME_CREATION_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING, EMAIL_DOMAIN_SETTING,
         ENV_SETTINGS, HUB_ACCESSIBLE_URL_SETTING, HUB_BASE_URL_SETTING,
     },
-    parse_postgres_url,
     server::Smtp,
-    utils::build_arg_str,
 };
 
+#[cfg(feature = "parquet")]
+use windmill_common::error::to_anyhow;
+
 pub fn global_service() -> Router {
     #[warn(unused_mut)]
     let r = Router::new()
@@ -68,11 +67,6 @@ pub fn global_service() -> Router {
             "/critical_alerts/:id/acknowledge",
             post(acknowledge_critical_alert),
         )
-        .route("/databases_exist", post(databases_exist))
-        .route(
-            "/create_ducklake_database/:name",
-            post(create_ducklake_database),
-        )
         .route(
             "/critical_alerts/acknowledge_all",
             post(acknowledge_all_critical_alerts),
@@ -504,92 +498,3 @@ pub async fn acknowledge_all_critical_alerts(
 pub async fn acknowledge_all_critical_alerts() -> error::Error {
     error::Error::NotFound("Critical Alerts require EE".to_string())
 }
-
-async fn databases_exist(
-    _authed: ApiAuthed,
-    Extension(db): Extension<DB>,
-    Json(database_names): Json<Vec<String>>,
-) -> JsonResult<Vec<String>> {
-    let result = sqlx::query_scalar!(
-        r#"SELECT elem FROM (SELECT unnest($1::TEXT[]) AS elem)
-        WHERE elem NOT IN (SELECT datname FROM pg_catalog.pg_database);"#,
-        database_names.as_slice()
-    )
-    .fetch_all(&db)
-    .await?
-    .into_iter()
-    .filter_map(|x| x)
-    .collect();
-
-    Ok(Json(result))
-}
-
-async fn create_ducklake_database(
-    authed: ApiAuthed,
-    Extension(db): Extension<DB>,
-    Path(dbname): Path<String>,
-) -> Result<()> {
-    require_super_admin(&db, &authed.email).await?;
-
-    // Validate name to ensure it only contains alphanumeric characters
-    // Prevents SQL injection on the instance database
-    let valid_name = regex::Regex::new(r"^[a-zA-Z0-9_]+$")
-        .map_err(|_| error::Error::internal_err("Failed to compile regex".to_string()))?;
-    if !valid_name.is_match(&dbname) {
-        return Err(error::Error::BadRequest(
-            "Invalid database name".to_string(),
-        ));
-    }
-
-    sqlx::query(&format!("CREATE DATABASE \"{dbname}\""))
-        .execute(&db)
-        .await?;
-
-    sqlx::query(&format!(
-        "GRANT CONNECT ON DATABASE \"{dbname}\" TO ducklake_user"
-    ))
-    .execute(&db)
-    .await?;
-
-    // We have to connect to the newly created database as admin to grant permissions
-    let pg_creds = parse_postgres_url(&get_database_url().await?)?;
-    let Some(wm_pg_pwd) = pg_creds.password else {
-        return Err(error::Error::BadRequest("Password not found".to_string()));
-    };
-    let conn_str: String = build_arg_str(
-        &[
-            ("host", Some(&pg_creds.host)),
-            ("port", pg_creds.port.map(|p| p.to_string()).as_deref()),
-            ("password", Some(&wm_pg_pwd)),
-            ("user", pg_creds.username.as_deref()),
-            ("dbname", Some(&dbname)),
-        ],
-        " ",
-        "=",
-    );
-    let (client, connection) = tokio::time::timeout(
-        std::time::Duration::from_secs(20),
-        tokio_postgres::connect(&conn_str, tokio_postgres::NoTls),
-    )
-    .await
-    .map_err(to_anyhow)?
-    .map_err(to_anyhow)?;
-
-    tokio::spawn(async move {
-        if let Err(e) = connection.await {
-            eprintln!("connection error: {}", e);
-        }
-    });
-
-    client
-        .batch_execute(&format!(
-            "GRANT USAGE ON SCHEMA public TO ducklake_user;
-            GRANT CREATE ON SCHEMA public TO ducklake_user;
-            ALTER DEFAULT PRIVILEGES IN SCHEMA public 
-                GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO ducklake_user;"
-        ))
-        .await
-        .map_err(to_anyhow)?;
-
-    Ok(())
-}

backend/windmill-api/src/utils.rs

@@ -452,6 +452,3 @@ pub async fn update_rw_lock<T>(lock: std::sync::Arc<tokio::sync::RwLock<T>>, val
     let mut w = lock.write().await;
     *w = value;
 }
-lazy_static::lazy_static! {
-    static ref DUCKLAKE_INSTANCE_PG_PASSWORD: std::sync::RwLock<Option<String>> = std::sync::RwLock::new(None);
-}

backend/windmill-api/src/workspaces.rs

@@ -38,12 +38,9 @@ use windmill_common::users::username_to_permissioned_as;
 use windmill_common::variables::{build_crypt, decrypt, encrypt};
 use windmill_common::worker::{to_raw_value, CLOUD_HOSTED};
 #[cfg(feature = "enterprise")]
-use windmill_common::workspaces::GitRepositorySettings;
-#[cfg(feature = "enterprise")]
 use windmill_common::workspaces::WorkspaceDeploymentUISettings;
 #[cfg(feature = "enterprise")]
 use windmill_common::workspaces::WorkspaceGitSyncSettings;
-use windmill_common::workspaces::{Ducklake, DucklakeCatalogResourceType};
 use windmill_common::{
     error::{Error, JsonResult, Result},
     global_settings::AUTOMATE_USERNAME_CREATION_SETTING,
@@ -115,14 +112,7 @@ pub fn workspaced_service() -> Router {
             "/edit_large_file_storage_config",
             post(edit_large_file_storage_config),
         )
-        .route("/edit_ducklake_config", post(edit_ducklake_config))
-        .route("/list_ducklakes", get(list_ducklakes))
         .route("/edit_git_sync_config", post(edit_git_sync_config))
-        .route("/edit_git_sync_repository", post(edit_git_sync_repository))
-        .route(
-            "/delete_git_sync_repository",
-            delete(delete_git_sync_repository),
-        )
         .route("/edit_deploy_ui_config", post(edit_deploy_ui_config))
         .route("/edit_default_app", post(edit_default_app))
         .route("/default_app", get(get_default_app))
@@ -235,8 +225,6 @@ pub struct WorkspaceSettings {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub large_file_storage: Option<serde_json::Value>, // effectively: DatasetsStorage
     #[serde(skip_serializing_if = "Option::is_none")]
-    pub ducklake: Option<serde_json::Value>,
-    #[serde(skip_serializing_if = "Option::is_none")]
     pub git_sync: Option<serde_json::Value>, // effectively: WorkspaceGitSyncSettings
     #[serde(skip_serializing_if = "Option::is_none")]
     pub deploy_ui: Option<serde_json::Value>, // effectively: WorkspaceDeploymentUISettings
@@ -304,21 +292,11 @@ struct LargeFileStorageWithSecondary {
     secondary_storage: HashMap<String, LargeFileStorage>,
 }
 
-#[derive(Deserialize, Serialize, Debug)]
-pub struct DucklakeSettings {
-    pub ducklakes: HashMap<String, Ducklake>,
-}
-
 #[derive(Deserialize, Debug)]
 struct EditLargeFileStorageConfig {
     large_file_storage: Option<LargeFileStorageWithSecondary>,
 }
 
-#[derive(Deserialize, Debug)]
-struct EditDucklakeConfig {
-    settings: DucklakeSettings,
-}
-
 #[derive(Deserialize)]
 struct CreateWorkspace {
     id: String,
@@ -457,7 +435,7 @@ async fn get_settings(
     let mut tx = user_db.begin(&authed).await?;
     let settings = sqlx::query_as!(
         WorkspaceSettings,
-        "SELECT workspace_id, slack_team_id, teams_team_id, teams_team_name, slack_name, slack_command_script, teams_command_script, slack_email, auto_invite_domain, auto_invite_operator, auto_add, customer_id, plan, webhook, deploy_to, ai_config, error_handler, error_handler_extra_args, error_handler_muted_on_cancel, large_file_storage, ducklake, git_sync, deploy_ui, default_app, default_scripts, mute_critical_alerts, color, operator_settings, git_app_installations FROM workspace_settings WHERE workspace_id = $1",
+        "SELECT workspace_id, slack_team_id, teams_team_id, teams_team_name, slack_name, slack_command_script, teams_command_script, slack_email, auto_invite_domain, auto_invite_operator, auto_add, customer_id, plan, webhook, deploy_to, ai_config, error_handler, error_handler_extra_args, error_handler_muted_on_cancel, large_file_storage, git_sync, deploy_ui, default_app, default_scripts, mute_critical_alerts, color, operator_settings, git_app_installations FROM workspace_settings WHERE workspace_id = $1",
         &w_id
     )
     .fetch_optional(&mut *tx)
@@ -885,184 +863,12 @@ async fn edit_large_file_storage_config(
     ))
 }
 
-async fn list_ducklakes(
-    _authed: ApiAuthed,
-    Extension(db): Extension<DB>,
-    Path(w_id): Path<String>,
-) -> JsonResult<Vec<String>> {
-    let ducklakes = sqlx::query_scalar!(
-        r#"
-            SELECT jsonb_object_keys(ws.ducklake->'ducklakes') AS ducklake_name
-            FROM workspace_settings ws
-            WHERE ws.workspace_id = $1
-        "#,
-        &w_id
-    )
-    .fetch_all(&db)
-    .await?
-    .into_iter()
-    .filter_map(|s| s)
-    .collect();
-
-    Ok(Json(ducklakes))
-}
-
-async fn edit_ducklake_config(
-    authed: ApiAuthed,
-    Extension(db): Extension<DB>,
-    Path(w_id): Path<String>,
-    ApiAuthed { is_admin, username, .. }: ApiAuthed,
-    Json(new_config): Json<EditDucklakeConfig>,
-) -> Result<String> {
-    require_admin(is_admin, &username)?;
-
-    let mut tx = db.begin().await?;
-
-    let args_for_audit = format!("{:?}", new_config.settings);
-    audit_log(
-        &mut *tx,
-        &authed,
-        "workspaces.edit_ducklake_config",
-        ActionKind::Update,
-        &w_id,
-        Some(&authed.email),
-        Some([("ducklake", args_for_audit.as_str())].into()),
-    )
-    .await?;
-
-    // Check that all ducklake catalog resources exist to prevent
-    // exploiting the shared property to see any resource
-    for dl in new_config.settings.ducklakes.values() {
-        if dl.catalog.resource_type == DucklakeCatalogResourceType::Instance {
-            continue;
-        }
-        let catalog_res = sqlx::query_scalar!(
-            "SELECT 1 FROM resource WHERE workspace_id = $1 AND path = $2",
-            &w_id,
-            &dl.catalog.resource_path
-        )
-        .fetch_optional(&mut *tx)
-        .await?
-        .flatten();
-
-        if catalog_res.is_none() {
-            return Err(Error::BadRequest(format!(
-                "Ducklake catalog resource {} not found in workspace {}",
-                dl.catalog.resource_path, &w_id
-            )));
-        }
-    }
-
-    let config: serde_json::Value = serde_json::to_value(new_config.settings)
-        .map_err(|err| Error::internal_err(err.to_string()))?;
-
-    sqlx::query!(
-        "UPDATE workspace_settings SET ducklake = $1 WHERE workspace_id = $2",
-        config,
-        &w_id
-    )
-    .execute(&mut *tx)
-    .await?;
-
-    tx.commit().await?;
-
-    Ok(format!("Edit ducklake config for workspace {}", &w_id))
-}
-
 #[derive(Deserialize)]
 pub struct EditGitSyncConfig {
     #[cfg(feature = "enterprise")]
     pub git_sync_settings: Option<WorkspaceGitSyncSettings>,
 }
 
-#[cfg(feature = "enterprise")]
-#[derive(Deserialize, Debug)]
-pub struct EditGitSyncRepository {
-    pub git_repo_resource_path: String,
-    pub repository: GitRepositorySettings,
-}
-
-#[cfg(feature = "enterprise")]
-#[derive(Deserialize, Debug)]
-pub struct DeleteGitSyncRepositoryRequest {
-    pub git_repo_resource_path: String,
-}
-
-#[cfg(feature = "enterprise")]
-fn validate_git_repo_resource_path(path: &str) -> Result<()> {
-    // Resource paths should follow the pattern: $res:f/<folder>/<name> or $res:u/<username>/<name>
-    if path.is_empty() {
-        return Err(Error::BadRequest(
-            "Resource path cannot be empty".to_string(),
-        ));
-    }
-
-    // Must start with $res: prefix
-    if !path.starts_with("$res:") {
-        return Err(Error::BadRequest(
-            "Resource path must start with '$res:'".to_string(),
-        ));
-    }
-
-    // Extract the actual path after $res:
-    let actual_path = &path[5..]; // Remove "$res:" prefix
-
-    // Basic validation: must start with f/ or u/ and contain at least one slash
-    if !actual_path.starts_with("f/") && !actual_path.starts_with("u/") {
-        return Err(Error::BadRequest(
-            "Resource path must start with '$res:f/' or '$res:u/'".to_string(),
-        ));
-    }
-
-    // Must have at least 3 parts (type, folder/user, name)
-    let parts: Vec<&str> = actual_path.split('/').collect();
-    if parts.len() < 3 || parts.iter().any(|part| part.is_empty()) {
-        return Err(Error::BadRequest(
-            "Invalid resource path format".to_string(),
-        ));
-    }
-
-    // Resource name validation (last part)
-    let resource_name = parts.last().unwrap();
-    if !resource_name
-        .chars()
-        .all(|c| c.is_alphanumeric() || c == '_' || c == '-')
-    {
-        return Err(Error::BadRequest(
-            "Resource name can only contain alphanumeric characters, underscores, and hyphens"
-                .to_string(),
-        ));
-    }
-
-    Ok(())
-}
-
-#[cfg(feature = "enterprise")]
-fn cleanup_legacy_git_sync_settings_in_memory(
-    git_sync_settings: &mut windmill_common::workspaces::WorkspaceGitSyncSettings,
-    workspace_id: &str,
-) {
-    // Check if all repositories are in new format (have settings field)
-    let all_repos_migrated = git_sync_settings
-        .repositories
-        .iter()
-        .all(|repo| repo.settings.is_some());
-
-    // If all repos are migrated and we still have legacy workspace-level settings
-    if all_repos_migrated
-        && (git_sync_settings.include_path.is_some() || git_sync_settings.include_type.is_some())
-    {
-        tracing::info!(
-            workspace_id = workspace_id,
-            "All git sync repositories migrated to new format, cleaning up legacy workspace-level settings"
-        );
-
-        // Remove workspace-level legacy fields
-        git_sync_settings.include_path = None;
-        git_sync_settings.include_type = None;
-    }
-}
-
 #[cfg(not(feature = "enterprise"))]
 async fn edit_git_sync_config(
     _authed: ApiAuthed,
@@ -1099,10 +905,7 @@ async fn edit_git_sync_config(
     )
     .await?;
 
-    if let Some(mut git_sync_settings) = new_config.git_sync_settings {
-        // Clean up legacy workspace-level settings if all repos are migrated
-        cleanup_legacy_git_sync_settings_in_memory(&mut git_sync_settings, &w_id);
-
+    if let Some(git_sync_settings) = new_config.git_sync_settings {
         let serialized_config = serde_json::to_value::<WorkspaceGitSyncSettings>(git_sync_settings)
             .map_err(|err| Error::internal_err(err.to_string()))?;
 
@@ -1121,7 +924,6 @@ async fn edit_git_sync_config(
         .execute(&mut *tx)
         .await?;
     }
-
     tx.commit().await?;
 
     // Trigger git sync for git sync settings changes
@@ -1139,247 +941,6 @@ async fn edit_git_sync_config(
     Ok(format!("Edit git sync config for workspace {}", &w_id))
 }
 
-#[cfg(not(feature = "enterprise"))]
-async fn edit_git_sync_repository(
-    _authed: ApiAuthed,
-    Extension(_db): Extension<DB>,
-    Path(_w_id): Path<String>,
-    Json(_new_config): Json<serde_json::Value>,
-) -> Result<String> {
-    return Err(Error::BadRequest(
-        "Git sync is only available on Windmill Enterprise Edition".to_string(),
-    ));
-}
-
-#[cfg(feature = "enterprise")]
-async fn edit_git_sync_repository(
-    authed: ApiAuthed,
-    Extension(db): Extension<DB>,
-    Path(w_id): Path<String>,
-    ApiAuthed { is_admin, username, .. }: ApiAuthed,
-    Json(new_config): Json<EditGitSyncRepository>,
-) -> Result<String> {
-    require_admin(is_admin, &username)?;
-
-    // Validate the resource path format
-    validate_git_repo_resource_path(&new_config.git_repo_resource_path)?;
-
-    let mut tx = db.begin().await?;
-
-    // First, get the current git sync settings
-    let current_settings = sqlx::query!(
-        "SELECT git_sync FROM workspace_settings WHERE workspace_id = $1",
-        &w_id
-    )
-    .fetch_optional(&mut *tx)
-    .await?;
-
-    let mut git_sync_settings = if let Some(row) = current_settings {
-        if let Some(git_sync) = row.git_sync {
-            serde_json::from_value::<WorkspaceGitSyncSettings>(git_sync)
-                .map_err(|err| Error::internal_err(err.to_string()))?
-        } else {
-            WorkspaceGitSyncSettings::default()
-        }
-    } else {
-        WorkspaceGitSyncSettings::default()
-    };
-
-    // Audit log before we move the repository
-    audit_log(
-        &mut *tx,
-        &authed,
-        "workspaces.edit_git_sync_repository",
-        ActionKind::Update,
-        &w_id,
-        Some(&authed.email),
-        Some(
-            [
-                (
-                    "repository_path",
-                    new_config.git_repo_resource_path.as_str(),
-                ),
-                ("repository_data", &format!("{:?}", new_config.repository)),
-            ]
-            .into(),
-        ),
-    )
-    .await?;
-
-    // Check if repository exists before modifying
-    let repo_exists = git_sync_settings
-        .repositories
-        .iter()
-        .any(|repo| repo.git_repo_resource_path == new_config.git_repo_resource_path);
-
-    // Find and update the specific repository, or add it if it doesn't exist
-    let repo_found = git_sync_settings
-        .repositories
-        .iter_mut()
-        .find(|repo| repo.git_repo_resource_path == new_config.git_repo_resource_path);
-
-    if let Some(existing_repo) = repo_found {
-        // Update existing repository
-        *existing_repo = new_config.repository;
-    } else {
-        // Repository doesn't exist, add it as a new repository
-        git_sync_settings.repositories.push(new_config.repository);
-    }
-
-    // Clean up legacy workspace-level settings if all repos are migrated
-    cleanup_legacy_git_sync_settings_in_memory(&mut git_sync_settings, &w_id);
-
-    // Save the updated configuration
-    let serialized_config = serde_json::to_value::<WorkspaceGitSyncSettings>(git_sync_settings)
-        .map_err(|err| Error::internal_err(err.to_string()))?;
-
-    sqlx::query!(
-        "UPDATE workspace_settings SET git_sync = $1 WHERE workspace_id = $2",
-        serialized_config,
-        &w_id
-    )
-    .execute(&mut *tx)
-    .await?;
-
-    tx.commit().await?;
-
-    // Trigger git sync for individual repository update/add
-    handle_deployment_metadata(
-        &authed.email,
-        &authed.username,
-        &db,
-        &w_id,
-        windmill_git_sync::DeployedObject::Settings { setting_type: "git_sync".to_string() },
-        Some(format!(
-            "Git sync repository '{}' {}",
-            new_config.git_repo_resource_path,
-            if repo_exists { "updated" } else { "added" }
-        )),
-        false,
-    )
-    .await?;
-
-    Ok(format!(
-        "{} git sync repository '{}' for workspace {}",
-        if repo_exists { "Updated" } else { "Added" },
-        new_config.git_repo_resource_path,
-        &w_id
-    ))
-}
-
-#[cfg(not(feature = "enterprise"))]
-async fn delete_git_sync_repository(
-    _authed: ApiAuthed,
-    Extension(_db): Extension<DB>,
-    Path(_w_id): Path<String>,
-    Json(_request): Json<serde_json::Value>,
-) -> Result<String> {
-    return Err(Error::BadRequest(
-        "Git sync is only available on Windmill Enterprise Edition".to_string(),
-    ));
-}
-
-#[cfg(feature = "enterprise")]
-async fn delete_git_sync_repository(
-    authed: ApiAuthed,
-    Extension(db): Extension<DB>,
-    Path(w_id): Path<String>,
-    ApiAuthed { is_admin, username, .. }: ApiAuthed,
-    Json(request): Json<DeleteGitSyncRepositoryRequest>,
-) -> Result<String> {
-    require_admin(is_admin, &username)?;
-
-    // For deletion, only validate that path is not empty to allow cleanup of malformed entries
-    if request.git_repo_resource_path.is_empty() {
-        return Err(Error::BadRequest(
-            "Resource path cannot be empty".to_string(),
-        ));
-    }
-
-    let mut tx = db.begin().await?;
-
-    // First, get the current git sync settings
-    let current_settings = sqlx::query!(
-        "SELECT git_sync FROM workspace_settings WHERE workspace_id = $1",
-        &w_id
-    )
-    .fetch_optional(&mut *tx)
-    .await?;
-
-    let mut git_sync_settings = if let Some(row) = current_settings {
-        if let Some(git_sync) = row.git_sync {
-            serde_json::from_value::<WorkspaceGitSyncSettings>(git_sync)
-                .map_err(|err| Error::internal_err(err.to_string()))?
-        } else {
-            WorkspaceGitSyncSettings::default()
-        }
-    } else {
-        WorkspaceGitSyncSettings::default()
-    };
-
-    // Check if repository exists and remove it
-    let original_count = git_sync_settings.repositories.len();
-    git_sync_settings
-        .repositories
-        .retain(|repo| repo.git_repo_resource_path != request.git_repo_resource_path);
-
-    if git_sync_settings.repositories.len() == original_count {
-        return Err(Error::BadRequest(format!(
-            "Repository with path '{}' not found in git sync configuration",
-            request.git_repo_resource_path
-        )));
-    }
-
-    // Audit log
-    audit_log(
-        &mut *tx,
-        &authed,
-        "workspaces.delete_git_sync_repository",
-        ActionKind::Delete,
-        &w_id,
-        Some(&authed.email),
-        Some([("repository_path", request.git_repo_resource_path.as_str())].into()),
-    )
-    .await?;
-
-    // Clean up legacy workspace-level settings if all repos are migrated
-    cleanup_legacy_git_sync_settings_in_memory(&mut git_sync_settings, &w_id);
-
-    // Save the updated configuration
-    let serialized_config = serde_json::to_value::<WorkspaceGitSyncSettings>(git_sync_settings)
-        .map_err(|err| Error::internal_err(err.to_string()))?;
-
-    sqlx::query!(
-        "UPDATE workspace_settings SET git_sync = $1 WHERE workspace_id = $2",
-        serialized_config,
-        &w_id
-    )
-    .execute(&mut *tx)
-    .await?;
-
-    tx.commit().await?;
-
-    // Trigger git sync for repository deletion
-    handle_deployment_metadata(
-        &authed.email,
-        &authed.username,
-        &db,
-        &w_id,
-        windmill_git_sync::DeployedObject::Settings { setting_type: "git_sync".to_string() },
-        Some(format!(
-            "Git sync repository '{}' deleted",
-            request.git_repo_resource_path
-        )),
-        false,
-    )
-    .await?;
-
-    Ok(format!(
-        "Deleted git sync repository '{}' from workspace {}",
-        request.git_repo_resource_path, &w_id
-    ))
-}
-
 #[derive(Debug, Deserialize)]
 struct EditDeployUIConfig {
     #[cfg(feature = "enterprise")]
@@ -2765,25 +2326,15 @@ pub async fn mute_critical_alerts() -> Error {
 
 #[derive(Deserialize, Serialize)]
 struct ChangeOperatorSettings {
-    #[serde(default)]
     runs: bool,
-    #[serde(default)]
     schedules: bool,
-    #[serde(default)]
     resources: bool,
-    #[serde(default)]
     variables: bool,
-    #[serde(default)]
     assets: bool,
-    #[serde(default)]
     triggers: bool,
-    #[serde(default)]
     audit_logs: bool,
-    #[serde(default)]
     groups: bool,
-    #[serde(default)]
     folders: bool,
-    #[serde(default)]
     workers: bool,
 }
 

backend/windmill-common/Cargo.toml

@@ -83,7 +83,6 @@ openidconnect = { workspace = true, optional = true }
 strum.workspace = true
 strum_macros.workspace = true
 url.workspace = true
-async-recursion.workspace = true
 
 semver.workspace = true
 croner = "2.2.0"

backend/windmill-common/src/assets.rs

@@ -11,7 +11,6 @@ pub enum AssetKind {
     Resource,
     // Avoid unnexpected crashes when deserializing old assets
     Variable, // Deprecated
-    Ducklake,
 }
 
 #[derive(Serialize, Deserialize, Debug, PartialEq, Copy, Clone, Hash, Eq, sqlx::Type)]

backend/windmill-common/src/cache.rs

@@ -797,12 +797,11 @@ pub mod job {
     #[track_caller]
     pub fn fetch_script(
         db: DB,
-        kind: &JobKind,
+        kind: JobKind,
         hash: Option<ScriptHash>,
     ) -> impl Future<Output = error::Result<Arc<ScriptData>>> {
         use JobKind::*;
         let loc = Location::caller();
-        let kind = kind.clone();
         async move {
             match (kind, hash.map(|ScriptHash(id)| id)) {
                 (FlowScript, Some(id)) => {
@@ -826,12 +825,11 @@ pub mod job {
     #[track_caller]
     pub fn fetch_flow<'c>(
         db: &'c DB,
-        kind: &JobKind,
+        kind: JobKind,
         hash: Option<ScriptHash>,
     ) -> impl Future<Output = error::Result<Arc<FlowData>>> + 'c {
         use JobKind::*;
         let loc = Location::caller();
-        let kind = kind.clone();
         async move {
             match (kind, hash.map(|ScriptHash(id)| id)) {
                 (FlowDependencies, Some(id)) => flow::fetch_version(db, id).await,

backend/windmill-common/src/flows.rs

@@ -27,7 +27,7 @@ use crate::{
     DB,
 };
 
-#[derive(Debug, Serialize, Deserialize, sqlx::FromRow)]
+#[derive(Serialize, Deserialize, sqlx::FromRow)]
 pub struct Flow {
     pub workspace_id: String,
     pub path: String,
@@ -91,7 +91,7 @@ pub struct ListableFlow {
     pub deployment_msg: Option<String>,
 }
 
-#[derive(Debug, Deserialize, sqlx::FromRow)]
+#[derive(Deserialize, sqlx::FromRow)]
 pub struct NewFlow {
     pub path: String,
     pub summary: String,

backend/windmill-common/src/indexer.rs

@@ -21,7 +21,7 @@ impl Default for TantivyIndexerSettings {
     fn default() -> Self {
         TantivyIndexerSettings {
             writer_memory_budget: 300_000_000,
-            commit_job_max_batch_size: 50_000,
+            commit_job_max_batch_size: 100_000,
             commit_log_max_batch_size: 10_000,
             refresh_index_period: 300,
             refresh_log_index_period: 300,

backend/windmill-common/src/lib.rs

@@ -82,7 +82,6 @@ pub mod variables;
 pub mod worker;
 pub mod workspaces;
 pub mod triggers;
-pub mod result_stream;
 pub mod stream;
 
 pub const DEFAULT_MAX_CONNECTIONS_SERVER: u32 = 50;
@@ -276,43 +275,6 @@ async fn reset() -> () {
     todo!()
 }
 
-pub struct PostgresUrlComponents {
-    pub scheme: String,
-    pub username: Option<String>,
-    pub password: Option<String>,
-    pub host: String,
-    pub port: Option<u16>,
-    pub database: String,
-    pub ssl_mode: Option<String>,
-}
-
-pub fn parse_postgres_url(url: &str) -> Result<PostgresUrlComponents, Error> {
-    let parsed_url = url::Url::parse(url).map_err(|_| Error::BadConfig("Invalid PostgreSQL URL".to_string()))?;
-
-    let scheme = parsed_url.scheme().to_string();
-    let username = parsed_url.username().to_string();
-    let password = parsed_url.password().map(|p| p.to_string());
-    let host = parsed_url.host_str().ok_or_else(|| Error::BadConfig("Missing host in PostgreSQL URL".to_string()))?.to_string();
-    let port = parsed_url.port();
-    let database = parsed_url.path().trim_start_matches('/').to_string();
-    let mut ssl_mode = None;
-    for query in parsed_url.query_pairs() {
-        if query.0 == "sslmode" {
-            ssl_mode = Some(query.1.to_string());
-        }
-    }
-
-    Ok(PostgresUrlComponents {
-        scheme,
-        username: if username.is_empty() { None } else { Some(username) },
-        password,
-        host,
-        port,
-        database,
-        ssl_mode,
-    })
-}
-
 pub async fn get_database_url() -> Result<String, Error> {
     use std::env::var;
     use tokio::fs::File;

backend/windmill-common/src/result_stream.rs

@@ -1,33 +0,0 @@
-use uuid::Uuid;
-use crate::{error, DB};
-
-pub const STREAM_PREFIX: &str = "WM_STREAM: ";
-
-pub fn extract_stream_from_logs(line: &str) -> Option<String> {
-    if line.starts_with(STREAM_PREFIX) {
-        // Extract the content after "WM_STREAM:" prefix
-        let stream_content = line.strip_prefix(STREAM_PREFIX).unwrap_or("");
-        if !stream_content.is_empty() {
-            return Some(stream_content.to_string().replace("\\n", "\n"));
-        }
-    }
-    None
-}
-
-
-
-pub async fn append_result_stream_db(db: &DB, workspace_id: &str, job_id: &Uuid, nstream: &str) -> error::Result<()> {
-    if !nstream.is_empty() {
-        sqlx::query!(
-            r#"
-            INSERT INTO job_result_stream (workspace_id, job_id, stream)
-            VALUES ($1, $2, $3)
-            ON CONFLICT (job_id) DO UPDATE SET stream = job_result_stream.stream || $3
-            "#,
-            workspace_id,
-            job_id,
-            nstream,
-        ).execute(db).await?;
-    }
-    Ok(())
-}

backend/windmill-common/src/s3_helpers.rs

@@ -1,3 +1,4 @@
+#[cfg(feature = "parquet")]
 use crate::error;
 #[cfg(feature = "parquet")]
 use aws_sdk_sts::config::ProvideCredentials;
@@ -226,18 +227,6 @@ pub enum LargeFileStorage {
     // TODO: Add a filesystem type here in the future if needed
 }
 
-impl LargeFileStorage {
-    pub fn get_s3_resource_path(&self) -> &str {
-        match self {
-            LargeFileStorage::S3Storage(s3_lfs) => &s3_lfs.s3_resource_path,
-            LargeFileStorage::S3AwsOidc(s3_lfs) => &s3_lfs.s3_resource_path,
-            LargeFileStorage::AzureBlobStorage(az_lfs) => &az_lfs.azure_blob_resource_path,
-            LargeFileStorage::AzureWorkloadIdentity(az_lfs) => &az_lfs.azure_blob_resource_path,
-            LargeFileStorage::GoogleCloudStorage(gcs_lfs) => &gcs_lfs.gcs_resource_path,
-        }
-    }
-}
-
 #[derive(Serialize, Deserialize, Debug)]
 pub struct S3Storage {
     pub s3_resource_path: String,
@@ -980,7 +969,7 @@ pub async fn convert_json_line_stream<E: Into<anyhow::Error>>(
     Ok(tokio_stream::wrappers::ReceiverStream::new(rx))
 }
 
-#[derive(Deserialize, Serialize, Clone, Debug)]
+#[derive(Deserialize, Serialize)]
 pub struct DuckdbConnectionSettingsResponse {
     pub connection_settings_str: String,
     #[serde(skip_serializing_if = "Option::is_none")]
@@ -996,88 +985,3 @@ pub struct DuckdbConnectionSettingsQueryV2 {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub storage: Option<String>,
 }
-
-pub fn lfs_to_object_store_resource(
-    lfs: &LargeFileStorage,
-    resource_value: serde_json::Value,
-) -> error::Result<ObjectStoreResource> {
-    match lfs {
-        LargeFileStorage::S3Storage(_) | LargeFileStorage::S3AwsOidc(_) => {
-            let s3_resource: S3Resource = serde_json::from_value(resource_value).map_err(|e| {
-                error::Error::internal_err(format!("Error parsing S3 resource: {}", e))
-            })?;
-            Ok(ObjectStoreResource::S3(s3_resource))
-        }
-        LargeFileStorage::AzureBlobStorage(_) | LargeFileStorage::AzureWorkloadIdentity(_) => {
-            let azure_blob_resource: AzureBlobResource = serde_json::from_value(resource_value)
-                .map_err(|e| {
-                    error::Error::internal_err(format!("Error parsing Azure Blob resource: {}", e))
-                })?;
-            Ok(ObjectStoreResource::Azure(azure_blob_resource))
-        }
-        LargeFileStorage::GoogleCloudStorage(_) => {
-            let gcs_resource: GcsResource =
-                serde_json::from_value(resource_value).map_err(|e| {
-                    error::Error::internal_err(format!("Error parsing GCS resource: {}", e))
-                })?;
-            Ok(ObjectStoreResource::Gcs(gcs_resource))
-        }
-    }
-}
-
-pub fn format_duckdb_connection_settings(
-    object_store_resource: ObjectStoreResource,
-) -> error::Result<DuckdbConnectionSettingsResponse> {
-    match object_store_resource {
-        ObjectStoreResource::S3(s3_resource) => duckdb_connection_settings_internal(s3_resource),
-        ObjectStoreResource::Azure(azure_resource) => {
-            let connection_string = format!(
-                "CREATE SECRET az_secret (TYPE AZURE, CONNECTION_STRING 'DefaultEndpointsProtocol=https;AccountName={};AccountKey={};EndpointSuffix=core.windows.net');",
-                azure_resource.account_name,
-                azure_resource.access_key.unwrap_or_default()
-            );
-            let response = DuckdbConnectionSettingsResponse {
-                connection_settings_str: connection_string,
-                azure_container_path: Some(format!("az://{}", azure_resource.container_name)),
-                s3_bucket: None,
-            };
-            Ok(response)
-        }
-        ObjectStoreResource::Gcs(_) => {
-            return Err(error::Error::BadRequest(
-                "GCS is not supported in DuckDB".to_string(),
-            ));
-        }
-    }
-}
-
-pub fn duckdb_connection_settings_internal(
-    s3_resource: S3Resource,
-) -> error::Result<DuckdbConnectionSettingsResponse> {
-    let mut duckdb_settings: String = String::new();
-
-    duckdb_settings.push_str("SET home_directory='./';\n"); // TODO: make this configurable maybe, or point to a temporary folder
-    duckdb_settings.push_str("INSTALL 'httpfs';\n");
-    if s3_resource.path_style.unwrap_or(true) {
-        duckdb_settings.push_str("SET s3_url_style='path';\n");
-    }
-    duckdb_settings.push_str(format!("SET s3_region='{}';\n", s3_resource.region).as_str());
-    duckdb_settings.push_str(format!("SET s3_endpoint='{}';\n", s3_resource.endpoint).as_str());
-    if !s3_resource.use_ssl {
-        duckdb_settings.push_str("SET s3_use_ssl=0;\n"); // default is true for DuckDB
-    }
-    if let Some(access_key_id) = s3_resource.access_key {
-        duckdb_settings.push_str(format!("SET s3_access_key_id='{}';\n", access_key_id).as_str());
-    }
-    if let Some(secret_access_key) = s3_resource.secret_key {
-        duckdb_settings
-            .push_str(format!("SET s3_secret_access_key='{}';\n", secret_access_key).as_str());
-    }
-
-    let response = DuckdbConnectionSettingsResponse {
-        connection_settings_str: duckdb_settings,
-        azure_container_path: None,
-        s3_bucket: Some(s3_resource.bucket),
-    };
-    return Ok(response);
-}

backend/windmill-common/src/utils.rs

@@ -19,7 +19,6 @@ use git_version::git_version;
 
 use chrono::Utc;
 use croner::Cron;
-use itertools::Itertools;
 use rand::{distr::Alphanumeric, rng, Rng};
 use reqwest::Client;
 use semver::Version;
@@ -596,30 +595,6 @@ pub async fn fetch_mute_workspace(_db: &DB, workspace_id: &str) -> Result<bool>
     }
 }
 
-// build_arg_str(&[("name", Some("value")), ("name2", None)], " ", "=")
-pub fn build_arg_str(args: &[(&str, Option<&str>)], sep: &str, eq: &str) -> String {
-    args.iter()
-        .filter_map(|(k, v)| {
-            if let Some(value) = v {
-                Some(format!("{}{}{}", k, eq, value))
-            } else {
-                None
-            }
-        })
-        .join(sep)
-}
-
-// Some errors (duckdb) leak the password in the error message
-pub fn sanitize_string_from_password(s: &str, passwd: &str) -> Option<String> {
-    if s.contains(passwd) {
-        return Some(s.replace(passwd, "******"));
-    }
-    // Do NOT check substrings
-    // In the case the user finds a string and notices that it gets substituted,
-    // He can very easily find the next character in O(1) and thus the entire password
-    None
-}
-
 pub enum ScheduleType {
     Croner(Cron),
     Cron(cron::Schedule),
@@ -884,23 +859,3 @@ impl Display for RunnableKind {
         write!(f, "{}", runnable_kind)
     }
 }
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    #[test]
-    fn test_build_arg_str() {
-        let r = build_arg_str(
-            &[
-                ("host", Some("localhost")),
-                ("port", Some("5432")),
-                ("password", None),
-                ("user", Some("postgres")),
-                ("dbname", Some("test_db")),
-            ],
-            " ",
-            "=",
-        );
-        assert_eq!(r, "host=localhost port=5432 user=postgres dbname=test_db");
-    }
-}

backend/windmill-common/src/workspaces.rs

@@ -1,19 +1,5 @@
-use async_recursion::async_recursion;
 use quick_cache::sync::Cache;
 use serde::{Deserialize, Serialize};
-use serde_json::json;
-use strum::AsRefStr;
-
-use crate::{
-    error::{to_anyhow, Error, Result},
-    get_database_url, parse_postgres_url,
-    s3_helpers::{
-        format_duckdb_connection_settings, lfs_to_object_store_resource,
-        DuckdbConnectionSettingsResponse, LargeFileStorage,
-    },
-    variables::{build_crypt, decrypt},
-    DB,
-};
 
 #[derive(Serialize, Deserialize, Debug, Default)]
 pub struct WorkspaceGitSyncSettings {
@@ -103,176 +89,3 @@ pub async fn is_premium_workspace(_db: &crate::DB, _w_id: &str) -> bool {
     IS_PREMIUM_CACHE.insert(_w_id.to_string(), premium);
     premium
 }
-
-#[derive(Deserialize, Serialize, Debug)]
-pub struct Ducklake {
-    pub catalog: DucklakeCatalog,
-    pub storage: DucklakeStorage,
-}
-
-#[derive(Deserialize, Serialize, Debug)]
-pub struct DucklakeCatalog {
-    pub resource_type: DucklakeCatalogResourceType,
-    pub resource_path: String,
-}
-
-#[derive(Deserialize, Serialize, Debug)]
-pub struct DucklakeStorage {
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub storage: Option<String>,
-    pub path: String,
-}
-
-#[derive(Deserialize, Serialize, Debug, PartialEq)]
-#[serde(rename_all = "lowercase")]
-#[derive(AsRefStr)]
-#[strum(serialize_all = "lowercase")]
-pub enum DucklakeCatalogResourceType {
-    #[strum(serialize = "postgres")]
-    Postgresql,
-    Mysql,
-    Instance,
-}
-
-#[derive(Deserialize, Serialize)]
-pub struct DucklakeWithConnData {
-    pub catalog: DucklakeCatalog,
-    pub catalog_resource: serde_json::Value,
-    pub storage: DucklakeStorage,
-    pub storage_settings: DuckdbConnectionSettingsResponse,
-}
-
-pub async fn get_ducklake_from_db_unchecked(
-    name: &str,
-    w_id: &str,
-    db: &DB,
-) -> Result<DucklakeWithConnData> {
-    let ducklake = sqlx::query_scalar!(
-        r#"
-            SELECT ws.ducklake->'ducklakes'->$2 AS config
-            FROM workspace_settings ws
-            WHERE ws.workspace_id = $1
-        "#,
-        &w_id,
-        name
-    )
-    .fetch_one(db)
-    .await
-    .map_err(|err| Error::internal_err(format!("getting ducklake {name}: {err}")))?
-    .ok_or_else(|| Error::internal_err(format!("ducklake {name} not found")))?;
-
-    let ducklake = serde_json::from_value::<Ducklake>(ducklake)?;
-
-    let lfs = if let Some(storage) = &ducklake.storage.storage {
-        sqlx::query_scalar!("SELECT large_file_storage->'secondary_storage'->$2 FROM workspace_settings WHERE workspace_id = $1", w_id, storage)
-    } else {
-        sqlx::query_scalar!("SELECT large_file_storage FROM workspace_settings WHERE workspace_id = $1", w_id)
-    }.fetch_optional(db)
-    .await?
-    .flatten()
-    .map(serde_json::from_value::<LargeFileStorage>)
-    .ok_or_else(|| Error::ExecutionErr("Ducklake storage not found".to_string()))??;
-
-    let s3_resource = transform_json_unchecked(
-        &serde_json::Value::String(lfs.get_s3_resource_path().to_string()),
-        w_id,
-        db,
-    )
-    .await?;
-    let object_store_resource = lfs_to_object_store_resource(&lfs, s3_resource)?;
-
-    let catalog_resource =
-        if ducklake.catalog.resource_type == DucklakeCatalogResourceType::Instance {
-            let pg_creds = parse_postgres_url(&get_database_url().await?)?;
-            json!({
-                "dbname": ducklake.catalog.resource_path,
-                "host": pg_creds.host,
-                "port": pg_creds.port,
-                "user": "ducklake_user",
-                "sslmode": pg_creds.ssl_mode,
-                "password": get_ducklake_instance_pg_catalog_password(&db).await?,
-            })
-        } else {
-            transform_json_unchecked(
-                &serde_json::Value::String(format!("$res:{}", ducklake.catalog.resource_path)),
-                w_id,
-                db,
-            )
-            .await?
-        };
-    let ducklake = DucklakeWithConnData {
-        catalog_resource,
-        storage_settings: format_duckdb_connection_settings(object_store_resource)?,
-        catalog: ducklake.catalog,
-        storage: ducklake.storage,
-    };
-    Ok(ducklake)
-}
-
-pub async fn get_ducklake_instance_pg_catalog_password(db: &DB) -> Result<String> {
-    sqlx::query_scalar!(
-        "SELECT trim(both '\"' from value::text) FROM global_settings WHERE name = 'ducklake_user_pg_pwd';"
-    )
-    .fetch_optional(db)
-    .await?
-    .flatten().ok_or_else(||
-        Error::BadRequest(format!(
-            "Ducklake instance catalog password not found, did you run migrations ?"
-        ))
-    )
-}
-
-// This does not check for any permission. Should never be displayed to a user.
-#[async_recursion]
-async fn transform_json_unchecked(
-    value: &serde_json::Value,
-    w_id: &str,
-    db: &DB,
-) -> Result<serde_json::Value> {
-    let value = match value {
-        serde_json::Value::Object(map) => {
-            let mut transformed_map = serde_json::Map::new();
-            for (key, val) in map {
-                let transformed_val = transform_json_unchecked(val, w_id, db).await?;
-                transformed_map.insert(key.clone(), serde_json::to_value(transformed_val)?);
-            }
-            serde_json::Value::Object(transformed_map)
-        }
-        serde_json::Value::Array(arr) => {
-            let mut transformed_array = Vec::new();
-            for val in arr {
-                let transformed_val = transform_json_unchecked(val, w_id, db).await?;
-                transformed_array.push(serde_json::to_value(transformed_val)?);
-            }
-            serde_json::Value::Array(transformed_array)
-        }
-        serde_json::Value::String(s) if s.starts_with("$res:") => {
-            let resource = sqlx::query_scalar!(
-                "SELECT value AS \"value!: _\" FROM resource WHERE workspace_id = $1 AND path = $2",
-                &w_id,
-                &s[5..]
-            )
-            .fetch_one(db)
-            .await
-            .map_err(to_anyhow)?;
-            transform_json_unchecked(&resource, w_id, db).await?
-        }
-        serde_json::Value::String(s) if s.starts_with("$var:") => {
-            let variable = sqlx::query_scalar!(
-                "SELECT value FROM variable WHERE workspace_id = $1 AND path = $2",
-                &w_id,
-                &s[5..]
-            )
-            .fetch_one(db)
-            .await
-            .map_err(to_anyhow)?;
-            let mc = build_crypt(&db, &w_id).await?;
-            let variable = decrypt(&mc, variable)?;
-            serde_json::Value::String(variable)
-        }
-        s @ serde_json::Value::String(_) => s.clone(),
-        x => x.clone(),
-    };
-
-    Ok(value)
-}

backend/windmill-git-sync/Cargo.toml

@@ -21,5 +21,4 @@ serde_json.workspace = true
 tracing.workspace = true
 windmill-common = { workspace = true, default-features = false }
 windmill-queue.workspace = true
-regex = "1.10.3"
-tokio = { workspace = true, features = ["full"] }
+regex = "1.10.3"

backend/windmill-indexer/Cargo.toml

@@ -8,8 +8,12 @@ edition.workspace = true
 name = "windmill_indexer"
 path = "src/lib.rs"
 
+[[bin]]
+name = "windmill_indexer_debug"
+path = "./src/main.rs"
+
 [features]
-default = []
+default = ["enterprise", "private", "parquet"]
 parquet = ["dep:object_store"]
 private = []
 enterprise = []
@@ -33,3 +37,4 @@ tokio-tar.workspace = true
 lazy_static.workspace = true
 const_format.workspace = true
 flume.workspace = true
+gethostname.workspace = true

backend/windmill-indexer/src/main.rs

@@ -0,0 +1,131 @@
+use anyhow::anyhow;
+use const_format::concatcp;
+use gethostname::gethostname;
+use std::{str::FromStr, sync::{Arc, Mutex}};
+use tantivy::{directory::MmapDirectory, schema::Schema};
+use windmill_common::{
+    indexer::TantivyIndexerSettings, utils::{rd_string, Mode}, worker::TMP_DIR,
+};
+use windmill_indexer::{
+    completed_runs_ee::{
+        fill_schema, IndexWriter, IndexedJobTrackerData, INDEXED_JOB_TRACKER_FILENAME,
+    },
+    indexer_ee::{clean_dir_all_if_exists, IndexedDBTracker, S3BackedMMapDirectory},
+};
+
+lazy_static::lazy_static! {
+    static ref HOSTNAME :String = std::env::var("FORCE_HOSTNAME").unwrap_or_else(|_| {
+        gethostname()
+            .to_str()
+            .map(|x| x.to_string())
+            .unwrap_or_else(|| rd_string(5))
+    });
+}
+
+const S3_JOB_INDEX_PATH: &str = "search-dbg/completed_job_index";
+const JOB_INDEX_DIRECTORY_PATH: &str = concatcp!(TMP_DIR, "/search-dbg/completed_jobs_index");
+
+fn init_index() -> anyhow::Result<IndexWriter> {
+    let mut schema_builder = Schema::builder();
+    let fields = fill_schema(&mut schema_builder);
+    let schema = schema_builder.build();
+
+    std::fs::create_dir_all(JOB_INDEX_DIRECTORY_PATH)?;
+
+    let idx_tantivy_dir = MmapDirectory::open(JOB_INDEX_DIRECTORY_PATH)
+        .map_err(|e| anyhow!("Failed to create MMapDirectory: {e}"))?;
+
+    #[cfg(all(feature = "enterprise", feature = "parquet"))]
+    let idx_tantivy_dir = S3BackedMMapDirectory::new(
+        std::path::PathBuf::from_str(JOB_INDEX_DIRECTORY_PATH).map_err(|e| anyhow!("{}", e))?,
+        idx_tantivy_dir,
+        S3_JOB_INDEX_PATH.to_string(),
+        tokio::runtime::Handle::current(),
+    );
+
+    clean_dir_all_if_exists(JOB_INDEX_DIRECTORY_PATH)?;
+    let index = tantivy::Index::open_or_create(idx_tantivy_dir, schema)
+        .map_err(|e| anyhow!("Failed to open index {e}"))?;
+
+    let job_tracker: IndexedDBTracker<IndexedJobTrackerData> = IndexedDBTracker::new(
+        std::path::PathBuf::from_str(INDEXED_JOB_TRACKER_FILENAME)
+            .map_err(|e| anyhow!("Failed to read {}: {e}", INDEXED_JOB_TRACKER_FILENAME))?,
+        index.directory().clone(),
+    );
+
+    Ok(IndexWriter { index, fields, job_tracker })
+}
+#[tokio::main]
+pub async fn main() -> anyhow::Result<()> {
+    let hostname = HOSTNAME.to_owned();
+
+    if std::env::var("RUST_LOG").is_err() {
+        std::env::set_var("RUST_LOG", "info")
+    }
+
+    let (_guard, _jaja) = windmill_common::tracing_init::initialize_tracing(&hostname, &Mode::Indexer, "");
+
+    let db = windmill_common::initial_connection().await?;
+
+    let mut idx_writer = init_index()?;
+
+    let max_created_at = std::env::var("INDEX_STARTING_FROM")
+        .ok()
+        .map(|start_from| chrono::DateTime::parse_from_rfc3339(&start_from))
+        .transpose()
+        .map_err(|e| anyhow!("Failed to parse date for INDEX_STARTING_FROM"))?;
+
+    if let Some(max_created_at) = max_created_at {
+        tracing::info!("Date to start indexing aka INDEX_STARTING_FROM is set to {:?}", max_created_at);
+        idx_writer.job_tracker.data =
+            Some(IndexedJobTrackerData { max_created_at: max_created_at.into(), queued_uuids_remainder: vec![] });
+    } else {
+        tracing::info!("INDEX_STARTING_FROM env var not found, indexing from the very first job");
+    }
+
+    let writer = idx_writer
+        .index
+        .writer(300 * 1024 * 1024)?;
+
+    let w = Arc::new(Mutex::new(writer));
+    idx_writer.refresh_jobs(w.clone(), &db, &TantivyIndexerSettings::default()).await?;
+
+    tokio::task::spawn_blocking(move || {
+        if let Err(e) = Arc::try_unwrap(w)
+            .map_err(|_| {
+                anyhow!(
+                    "There was more than 1 refrence to the writer. This should not be possible."
+                )
+            })
+            .unwrap()
+            .into_inner()
+            .unwrap()
+            .wait_merging_threads()
+        {
+            tracing::error!("Error while waiting for index writer merging threads: {e}");
+        }
+    })
+    .await?;
+
+    tracing::info!("All merging threads completed, releasing lock");
+
+    Ok(())
+}
+
+// let (index_reader, index_writer) = {
+//     let mut indexer_rx = killpill_rx.resubscribe();
+//
+//     let (mut reader, mut writer) = (None, None);
+//     tokio::select! {
+//     _ = indexer_rx.recv() => {
+//         tracing::info!("Received killpill, aborting index initialization");
+//     },
+//     res = windmill_indexer::completed_runs_oss::init_index(&db) => {
+//             let res = res?;
+//             reader = Some(res.0);
+//             writer = Some(res.1);
+//     }
+//
+//     }
+//     (reader, writer)
+// };

backend/windmill-worker/src/agent_workers.rs

@@ -1,8 +1,6 @@
 use reqwest::header::HeaderMap;
 use uuid::Uuid;
-use windmill_common::{
-    agent_workers::QueueInitJob, worker::HttpClient, workspaces::DucklakeWithConnData,
-};
+use windmill_common::{agent_workers::QueueInitJob, worker::HttpClient};
 use windmill_queue::{JobAndPerms, JobCompleted};
 
 pub async fn queue_init_job(client: &HttpClient, content: &str) -> anyhow::Result<Uuid> {
@@ -50,18 +48,4 @@ pub async fn send_result(client: &HttpClient, jc: JobCompleted) -> anyhow::Resul
         .await
 }
 
-#[allow(dead_code)]
-pub async fn get_ducklake_from_agent_http(
-    client: &HttpClient,
-    name: &str,
-    w_id: &str,
-) -> anyhow::Result<DucklakeWithConnData> {
-    client
-        .get(&format!(
-            "/api/w/{}/agent_workers/get_ducklake/{}",
-            w_id, &name
-        ))
-        .await
-}
-
 pub const UPDATE_PING_URL: &str = "/api/agent_workers/update_ping";

backend/windmill-worker/src/bun_executor.rs

@@ -364,7 +364,7 @@ pub async fn install_bun_lockfile(
             occupancy_metrics,
             None,
         )
-        .await?;
+        .await?
     } else {
         Box::into_pin(child_process.wait()).await?;
     }
@@ -1081,10 +1081,6 @@ function argsObjToArr({{ {spread} }}) {{
     return [ {spread} ];
 }}
 
-function isAsyncIterable(obj) {{
-    return obj != null && typeof obj[Symbol.asyncIterator] === 'function';
-}}
-
 BigInt.prototype.toJSON = function () {{
     return this.toString();
 }};
@@ -1097,12 +1093,6 @@ async function run() {{
         throw new Error("{main_name} function is missing");
     }}
     let res = await Main.{main_name}(...argsArr);
-    if (isAsyncIterable(res)) {{
-        for await (const chunk of res) {{
-            console.log("WM_STREAM: " + chunk.replace('\n', '\\n'));
-        }}
-        res = null;
-    }}
     const res_json = JSON.stringify(res ?? null, (key, value) => typeof value === 'undefined' ? null : value);
     await fs.writeFile("result.json", res_json);
     process.exit(0);
@@ -1451,7 +1441,7 @@ try {{
         .await?
     };
 
-    let handle_result = handle_child(
+    handle_child(
         &job.id,
         conn,
         mem_peak,
@@ -1484,7 +1474,7 @@ try {{
             })?;
         *new_args = Some(args.clone());
     }
-    read_result(job_dir, handle_result.result_stream).await
+    read_result(job_dir).await
 }
 
 pub async fn get_common_bun_proc_envs(base_internal_url: Option<&str>) -> HashMap<String, String> {

backend/windmill-worker/src/common.rs

@@ -359,41 +359,11 @@ pub async fn read_file(path: &str) -> error::Result<Box<RawValue>> {
     return Ok(r);
 }
 
-pub async fn merge_result_stream(
-    result: error::Result<Box<RawValue>>,
-    result_stream: Option<String>,
-) -> error::Result<Box<RawValue>> {
-    if let Some(result_stream) = result_stream {
-        result.and_then(|x| {
-            let mut value: Value = serde_json::from_str(x.get())?;
-
-            // Insert the string at the "wm_stream" field
-            if let Value::Object(ref mut map) = value {
-                map.insert("wm_stream".to_string(), Value::String(result_stream));
-            } else if value.is_null() {
-                // return Ok(unsafe_raw(json))
-                return Ok(to_raw_value(&json!(result_stream)));
-            } else {
-                return Ok(x);
-            }
-
-            // Convert back to RawValue
-            let json_string = serde_json::to_string(&value)?;
-            Ok(RawValue::from_string(json_string)?)
-        })
-    } else {
-        result
-    }
-}
 /// Read the `result.json` file. This function assumes that the file contains valid json and will
 /// result in undefined behaviour if it isn't. If the result.json is user generated or otherwise
 /// not guaranteed to be valid, use `read_and_check_result`
-pub async fn read_result(
-    job_dir: &str,
-    result_stream: Option<String>,
-) -> error::Result<Box<RawValue>> {
-    let rf = read_file(&format!("{job_dir}/result.json")).await;
-    merge_result_stream(rf, result_stream).await
+pub async fn read_result(job_dir: &str) -> error::Result<Box<RawValue>> {
+    return read_file(&format!("{job_dir}/result.json")).await;
 }
 
 pub async fn read_and_check_file(path: &str) -> error::Result<Box<RawValue>> {

backend/windmill-worker/src/csharp_executor.rs

@@ -641,5 +641,5 @@ pub async fn handle_csharp_job(
         None,
     )
     .await?;
-    read_result(job_dir, None).await
+    read_result(job_dir).await
 }

backend/windmill-worker/src/dedicated_worker.rs

@@ -126,7 +126,7 @@ pub async fn handle_dedicated_process(
         let status = Box::into_pin(child.wait())
             .await
             .expect("child process encountered an error");
-        if let Err(e) = process_status(&cmd_name, status, vec![]) {
+        if let Err(e) = process_status(&cmd_name, status) {
             tracing::error!("child exit status was not success: {e:#}");
         } else {
             tracing::info!("child exit status was success");

backend/windmill-worker/src/deno_executor.rs

@@ -283,10 +283,6 @@ BigInt.prototype.toJSON = function () {{
     return this.toString();
 }};
 
-function isAsyncIterable(obj) {{
-    return obj != null && typeof obj[Symbol.asyncIterator] === 'function';
-}}
-
 async function run() {{
     {dates}
     {preprocessor}
@@ -295,12 +291,6 @@ async function run() {{
         throw new Error("{main_name} function is missing");
     }}
     let res: any = await {main_name}(...argsArr);
-    if (isAsyncIterable(res)) {{
-        for await (const chunk of res) {{
-            console.log("WM_STREAM: " + chunk.replace('\n', '\\n'));
-        }}
-        res = null;
-    }}
     const res_json = JSON.stringify(res ?? null, (key, value) => typeof value === 'undefined' ? null : value);
     await Deno.writeTextFile("result.json", res_json);
     Deno.exit(0);
@@ -418,7 +408,7 @@ try {{
     };
     // logs.push_str(format!("prepare: {:?}\n", start.elapsed().as_micros()).as_str());
     // start = Instant::now();
-    let handle_result = handle_child(
+    handle_child(
         &job.id,
         conn,
         mem_peak,
@@ -455,7 +445,7 @@ try {{
             })?;
         *new_args = Some(args.clone());
     }
-    read_result(job_dir, handle_result.result_stream).await
+    read_result(job_dir).await
 }
 
 async fn build_import_map(

backend/windmill-worker/src/duckdb_executor.rs

@@ -1,6 +1,5 @@
 use std::collections::HashMap;
 use std::env;
-use std::sync::{Arc, Mutex};
 
 use duckdb::types::TimeUnit;
 use duckdb::{params_from_iter, Row};
@@ -8,19 +7,17 @@ use rust_decimal::prelude::FromPrimitive;
 use rust_decimal::Decimal;
 use serde_json::value::RawValue;
 use serde_json::{json, Value};
+use tokio::fs::remove_file;
 use tokio::task;
 use uuid::Uuid;
 use windmill_common::error::{to_anyhow, Error, Result};
 use windmill_common::s3_helpers::{
     DuckdbConnectionSettingsQueryV2, DuckdbConnectionSettingsResponse, S3Object,
 };
-use windmill_common::utils::sanitize_string_from_password;
 use windmill_common::worker::{to_raw_value, Connection};
-use windmill_common::workspaces::{get_ducklake_from_db_unchecked, DucklakeCatalogResourceType};
 use windmill_parser_sql::{parse_duckdb_sig, parse_sql_blocks};
 use windmill_queue::{CanceledBy, MiniPulledJob};
 
-use crate::agent_workers::get_ducklake_from_agent_http;
 use crate::common::{build_args_values, OccupancyMetrics};
 use crate::handle_child::run_future_with_polling_update_job_poller;
 #[cfg(feature = "mysql")]
@@ -97,19 +94,18 @@ pub async fn do_duckdb(
     column_order_ref: &mut Option<Vec<String>>,
     occupancy_metrics: &mut OccupancyMetrics,
 ) -> Result<Box<RawValue>> {
-    let hidden_passwords = Arc::new(Mutex::new(Vec::<String>::new()));
-
     let result_f = async {
-        let mut hidden_passwords = hidden_passwords.clone();
-        let mut bigquery_credentials = None;
-        let mut duckdb_connection_settings_cache =
-            HashMap::<Option<String>, DuckdbConnectionSettingsResponse>::new();
-
         let sig = parse_duckdb_sig(query)?.args;
         let mut job_args = build_args_values(job, client, conn).await?;
 
         let (query, _) = &sanitize_and_interpolate_unsafe_sql_args(query, &sig, &job_args)?;
-        let query = transform_s3_uris(query, client, &mut duckdb_connection_settings_cache).await?;
+        // Prevent interpolate_named_args from detecting argument identifiers in the signature for
+        // the first query block
+        let query = trunc_sig(query);
+
+        let (_query_with_transformed_s3_uris, mut used_storages) =
+            transform_s3_uris(query, client).await?;
+        let query = _query_with_transformed_s3_uris.as_deref().unwrap_or(query);
 
         let job_args = {
             let mut m: HashMap<String, duckdb::types::Value> = HashMap::new();
@@ -123,17 +119,28 @@ pub async fn do_duckdb(
                     let s3_obj = serde_json::from_value::<S3Object>(json_value).map_err(|e| {
                         Error::ExecutionErr(format!("Failed to deserialize S3Object: {}", e))
                     })?;
-                    let duckdb_conn_settings: DuckdbConnectionSettingsResponse =
-                        get_duckdb_connection_settings(
-                            &s3_obj.storage,
-                            &mut duckdb_connection_settings_cache,
-                            client,
-                        )
+                    let duckdb_conn_settings: windmill_common::s3_helpers::DuckdbConnectionSettingsResponse = client
+                        .get_duckdb_connection_settings(&DuckdbConnectionSettingsQueryV2 {
+                            s3_resource_path: None,
+                            storage: s3_obj.storage.clone(),
+                        })
                         .await?;
 
-                    let uri =
-                        duckdb_conn_settings_to_s3_network_uri(&duckdb_conn_settings, &s3_obj.s3)?;
+                    let uri = match (
+                        &duckdb_conn_settings.s3_bucket,
+                        &duckdb_conn_settings.azure_container_path,
+                    ) {
+                        (Some(s3_bucket), None) => format!("s3://{}/{}", s3_bucket, &s3_obj.s3),
+                        (None, Some(az_container)) => format!("{}/{}", az_container, &s3_obj.s3),
+                        _ => {
+                            return Err(Error::ExecutionErr(
+                                "S3Object must have either s3_bucket or azure_container_path"
+                                    .to_string(),
+                            ));
+                        }
+                    };
                     m.insert(sig_arg.name, duckdb::types::Value::Text(uri));
+                    used_storages.insert(s3_obj.storage, duckdb_conn_settings);
                 } else {
                     let duckdb_value = json_value_to_duckdb_value(
                         &json_value,
@@ -149,67 +156,36 @@ pub async fn do_duckdb(
             m
         };
 
-        let query_block_list = parse_sql_blocks(&query);
+        let query_block_list = parse_sql_blocks(query);
 
         // Replace windmill resource ATTACH statements with the real instructions
         let query_block_list = {
             let mut v = vec![];
             for query_block in query_block_list.iter() {
-                let query_block = remove_comments(&query_block);
                 match parse_attach_db_resource(query_block) {
-                    Some(parsed) => {
-                        v.extend(
-                            transform_attach_db_resource_query(
-                                &parsed,
-                                &job.id,
-                                client,
-                                &mut hidden_passwords,
-                            )
-                            .await?,
-                        );
-                        if parsed.db_type == "bigquery" {
-                            bigquery_credentials = Some(UseBigQueryCredentialsFile::new(
-                                job.id,
-                                parsed.resource_path,
-                            )?);
-                        }
-                    }
-                    None => match transform_attach_ducklake(
-                        &query_block,
-                        conn,
-                        &mut duckdb_connection_settings_cache,
-                        &mut hidden_passwords,
-                        &job.workspace_id,
-                    )
-                    .await?
-                    {
-                        Some(ducklake_query) => v.extend(ducklake_query),
-                        None => v.push(query_block.to_string()),
-                    },
+                    Some(parsed) => v.extend(
+                        transform_attach_db_resource_query(&parsed, &job.id, client).await?,
+                    ),
+                    None => v.push(query_block.to_string()),
                 };
             }
             v
         };
 
-        // duckdb::Connection is not Send so we run the queries in a single blocking task
+        // duckdb::Connection is not Send so we do it in a single blocking task
         let (result, column_order) = task::spawn_blocking(move || {
             let conn = duckdb::Connection::open_in_memory()
                 .map_err(|e| Error::ConnectingToDatabase(e.to_string()))?;
 
-            for DuckdbConnectionSettingsResponse { connection_settings_str, .. } in
-                duckdb_connection_settings_cache.values()
+            for (_, DuckdbConnectionSettingsResponse { connection_settings_str, .. }) in
+                used_storages.into_iter()
             {
-                hidden_passwords
-                    .lock()
-                    .unwrap()
-                    .push(connection_settings_str.clone());
                 conn.execute_batch(&connection_settings_str)
                     .map_err(|e| Error::ExecutionErr(e.to_string()))?;
             }
 
             let mut result: Option<Box<RawValue>> = None;
             let mut column_order = None;
-
             for (query_block_index, query_block) in query_block_list.iter().enumerate() {
                 result = Some(
                     do_duckdb_inner(
@@ -228,9 +204,14 @@ pub async fn do_duckdb(
         .await
         .map_err(to_anyhow)??;
 
-        drop(bigquery_credentials);
-
         *column_order_ref = column_order;
+
+        // BigQuery cleanup
+        let bq_credentials_path = make_bq_credentials_path(&job.id);
+        env::remove_var("GOOGLE_APPLICATION_CREDENTIALS");
+        if matches!(tokio::fs::try_exists(&bq_credentials_path).await, Ok(true)) {
+            remove_file(&bq_credentials_path).await.map_err(to_anyhow)?;
+        }
         Ok(result)
     };
 
@@ -246,21 +227,9 @@ pub async fn do_duckdb(
         &mut Some(occupancy_metrics),
         Box::pin(futures::stream::once(async { 0 })),
     )
-    .await;
+    .await?;
 
-    match result {
-        Ok(result) => Ok(result),
-        Err(e) => {
-            // Passwords might appear in the error message
-            let mut err_str = e.to_string();
-            for pwd in hidden_passwords.lock().unwrap().iter() {
-                if let Some(sanitized) = sanitize_string_from_password(&err_str, &pwd.clone()) {
-                    err_str = sanitized;
-                }
-            }
-            Err(Error::ExecutionErr(err_str))
-        }
-    }
+    Ok(result)
 }
 
 fn row_to_value(row: &Row<'_>, column_names: &[String]) -> Result<Box<RawValue>> {
@@ -429,8 +398,7 @@ fn string_to_duckdb_timestamp(s: &str) -> Result<duckdb::types::Value> {
 
 fn string_to_duckdb_date(s: &str) -> Result<duckdb::types::Value> {
     use chrono::Datelike;
-    let date = chrono::NaiveDate::parse_from_str(s, "%Y-%m-%d")
-        .map_err(|e| Error::ExecutionErr(format!("Invalid date format: {}", e)))?;
+    let date = chrono::NaiveDate::parse_from_str(s, "%Y-%m-%d").unwrap();
     Ok(duckdb::types::Value::Date32(date.num_days_from_ce()))
 }
 
@@ -451,7 +419,7 @@ struct ParsedAttachDbResource<'a> {
 }
 fn parse_attach_db_resource<'a>(query: &'a str) -> Option<ParsedAttachDbResource<'a>> {
     lazy_static::lazy_static! {
-        static ref RE: regex::Regex = regex::Regex::new(r"(?i)ATTACH '(\$res:|res://)([^']+)' AS (\S+) \(TYPE (\w+)(.*)\)").unwrap();
+        static ref RE: regex::Regex = regex::Regex::new(r"ATTACH '(\$res:|res://)([^']+)' AS (\S+) \(TYPE (\w+)(.*)\)").unwrap();
     }
 
     for cap in RE.captures_iter(query) {
@@ -470,32 +438,25 @@ fn parse_attach_db_resource<'a>(query: &'a str) -> Option<ParsedAttachDbResource
     None
 }
 
-fn format_attach_db_conn_str(db_resource: Value, db_type: &str) -> Result<String> {
-    let s = match db_type.to_lowercase().as_str() {
-        "postgres" | "postgresql" => {
-            let res: PgDatabase = serde_json::from_value(db_resource)?;
-            format!(
-                "dbname={} {} host={} {} {}",
-                res.dbname,
-                res.user.map(|u| format!("user={}", u)).unwrap_or_default(),
-                res.host,
-                res.password
-                    .map(|p| format!("password={}", p))
-                    .unwrap_or_default(),
-                res.port.map(|p| format!("port={}", p)).unwrap_or_default(),
-            )
-        }
-        #[cfg(feature = "mysql")]
-        "mysql" => {
-            let resource: MysqlDatabase = serde_json::from_value(db_resource)?;
-            format!(
-                "database={} host={} ssl_mode={} {} {} {}",
-                resource.database,
-                resource.host,
+async fn transform_attach_db_resource_query(
+    parsed: &ParsedAttachDbResource<'_>,
+    job_id: &Uuid,
+    client: &AuthedClient,
+) -> Result<Vec<String>> {
+    match parsed.db_type.to_lowercase().as_str() {
+        "postgres" => {
+            let resource: PgDatabase = client
+                .get_resource_value_interpolated(parsed.resource_path, Some(job_id.to_string()))
+                .await?;
+
+            let attach_str = format!(
+                "ATTACH 'dbname={} {} host={} {} {}' AS {} (TYPE postgres{});",
+                resource.dbname,
                 resource
-                    .ssl
-                    .map(|ssl| if ssl { "required" } else { "disabled" })
-                    .unwrap_or("preferred"),
+                    .user
+                    .map(|u| format!("user={}", u))
+                    .unwrap_or_default(),
+                resource.host,
                 resource
                     .password
                     .map(|p| format!("password={}", p))
@@ -504,15 +465,76 @@ fn format_attach_db_conn_str(db_resource: Value, db_type: &str) -> Result<String
                     .port
                     .map(|p| format!("port={}", p))
                     .unwrap_or_default(),
-                resource
-                    .user
-                    .map(|u| format!("user={}", u))
-                    .unwrap_or_default(),
-            )
+                parsed.name,
+                parsed.extra_args.unwrap_or("")
+            );
+
+            Ok(vec![
+                "INSTALL postgres;".to_string(),
+                "LOAD postgres;".to_string(),
+                attach_str,
+            ])
+        }
+        "mysql" => {
+            #[cfg(not(feature = "mysql"))]
+            return Err(Error::ExecutionErr(
+                "MySQL feature is not enabled".to_string(),
+            ));
+
+            #[cfg(feature = "mysql")]
+            {
+                let resource: MysqlDatabase = client
+                    .get_resource_value_interpolated(parsed.resource_path, Some(job_id.to_string()))
+                    .await?;
+
+                let attach_str = format!(
+                    "ATTACH 'database={} host={} ssl_mode={} {} {} {}' AS {} (TYPE mysql{});",
+                    resource.database,
+                    resource.host,
+                    resource
+                        .ssl
+                        .map(|ssl| if ssl { "required" } else { "disabled" })
+                        .unwrap_or("preferred"),
+                    resource
+                        .password
+                        .map(|p| format!("password={}", p))
+                        .unwrap_or_default(),
+                    resource
+                        .port
+                        .map(|p| format!("port={}", p))
+                        .unwrap_or_default(),
+                    resource
+                        .user
+                        .map(|u| format!("user={}", u))
+                        .unwrap_or_default(),
+                    parsed.name,
+                    parsed.extra_args.unwrap_or("")
+                );
+
+                Ok(vec![
+                    "INSTALL mysql;".to_string(),
+                    "LOAD mysql;".to_string(),
+                    attach_str,
+                ])
+            }
         }
         "bigquery" => {
+            let resource: Value = client
+                .get_resource_value_interpolated(parsed.resource_path, Some(job_id.to_string()))
+                .await?;
+            // duckdb's bigquery extension requires a json file as credentials
+            let bq_credentials_path = make_bq_credentials_path(job_id);
+            env::set_var("GOOGLE_APPLICATION_CREDENTIALS", &bq_credentials_path);
+            tokio::fs::write(&bq_credentials_path, resource.to_string())
+                .await
+                .map_err(|e| {
+                    Error::ExecutionErr(format!(
+                        "Failed to write BigQuery credentials to {}: {}",
+                        &bq_credentials_path, e
+                    ))
+                })?;
             let project_id: String = serde_json::from_value(
-                db_resource
+                resource
                     .get("project_id")
                     .ok_or_else(|| {
                         Error::ExecutionErr("BigQuery resource must contain project_id".to_string())
@@ -520,133 +542,39 @@ fn format_attach_db_conn_str(db_resource: Value, db_type: &str) -> Result<String
                     .to_owned(),
             )
             .map_err(|_e| Error::ExecutionErr("failed project_id deserialize".to_string()))?;
-            format!("project={}", project_id,)
+            let attach_str = format!(
+                "ATTACH 'project={}' as {} (TYPE bigquery{});",
+                project_id,
+                parsed.name,
+                parsed.extra_args.unwrap_or("")
+            )
+            .to_string();
+            Ok(vec![
+                "INSTALL bigquery FROM community;".to_string(),
+                "LOAD bigquery;".to_string(),
+                attach_str,
+            ])
         }
-        _ => {
-            return Err(Error::ExecutionErr(format!(
-                "Unsupported db type in DuckDB ATTACH: {db_type}",
-            )))
-        }
-    };
-    Ok(s)
-}
-
-fn get_attach_db_install_str(db_type: &str) -> Result<&str> {
-    match db_type.to_lowercase().as_str() {
-        "postgres" => Ok("INSTALL postgres;"),
-        "mysql" => {
-            #[cfg(not(feature = "mysql"))]
-            return Err(Error::ExecutionErr(
-                "MySQL feature is not enabled".to_string(),
-            ));
-            #[cfg(feature = "mysql")]
-            Ok("INSTALL mysql;")
-        }
-        "bigquery" => Ok("INSTALL bigquery FROM community;"),
         _ => Err(Error::ExecutionErr(format!(
             "Unsupported db type in DuckDB ATTACH: {}",
-            db_type
+            parsed.db_type
         ))),
     }
 }
 
-async fn transform_attach_db_resource_query(
-    parsed: &ParsedAttachDbResource<'_>,
-    job_id: &Uuid,
-    client: &AuthedClient,
-    hidden_passwords: &mut Arc<Mutex<Vec<String>>>,
-) -> Result<Vec<String>> {
-    let db_resource: Value = client
-        .get_resource_value_interpolated(parsed.resource_path, Some(job_id.to_string()))
-        .await?;
-    if let Some(pwd) = db_resource.get("password").and_then(|p| p.as_str()) {
-        hidden_passwords.lock().unwrap().push(pwd.to_string());
-    }
-    let attach_str = format!(
-        "ATTACH '{}' as {} (TYPE {}{});",
-        format_attach_db_conn_str(db_resource, parsed.db_type)?,
-        parsed.name,
-        parsed.db_type,
-        parsed.extra_args.unwrap_or("")
-    )
-    .to_string();
-
-    Ok(vec![
-        get_attach_db_install_str(parsed.db_type)?.to_string(),
-        format!("LOAD {};", parsed.db_type),
-        attach_str,
-    ])
-}
-
-async fn transform_attach_ducklake(
-    query: &str,
-    conn: &Connection,
-    duckdb_connection_settings_cache: &mut DuckDbConnectionSettingsCache,
-    hidden_passwords: &mut Arc<Mutex<Vec<String>>>,
-    w_id: &str,
-) -> Result<Option<Vec<String>>> {
-    lazy_static::lazy_static! {
-        static ref RE: regex::Regex = regex::Regex::new(r"(?i)ATTACH\s*'ducklake(://[^':]+)?'\s*AS\s+([^ ;]+)\s*(\([^)]*\))?").unwrap();
-    }
-    let Some(cap) = RE.captures(query) else {
-        return Ok(None);
-    };
-    let name = cap.get(1).map(|m| &m.as_str()[3..]).unwrap_or("main");
-    let alias_name = cap.get(2).map(|m| m.as_str()).unwrap_or("");
-    let extra_args = cap
-        .get(3)
-        .map(|m| format!(", {}", &m.as_str()[1..m.as_str().len() - 1]))
-        .unwrap_or("".to_string());
-
-    let ducklake = match conn {
-        Connection::Http(client) => get_ducklake_from_agent_http(client, name, w_id).await?,
-        Connection::Sql(db) => get_ducklake_from_db_unchecked(name, w_id, db).await?,
-    };
-    let db_type = match ducklake.catalog.resource_type {
-        DucklakeCatalogResourceType::Instance => "postgres",
-        _ => ducklake.catalog.resource_type.as_ref(),
-    };
-
-    if let Some(pwd) = ducklake
-        .catalog_resource
-        .get("password")
-        .and_then(|p| p.as_str())
-    {
-        hidden_passwords.lock().unwrap().push(pwd.to_string());
-    }
-
-    let db_conn_str = format_attach_db_conn_str(ducklake.catalog_resource, db_type)?;
-
-    let storage_settings = ducklake.storage_settings;
-    let storage = ducklake.storage.storage;
-    if !duckdb_connection_settings_cache.contains_key(&storage) {
-        duckdb_connection_settings_cache.insert(storage.clone(), storage_settings.clone());
-    };
-    let s3_network_uri =
-        duckdb_conn_settings_to_s3_network_uri(&storage_settings, &ducklake.storage.path)?;
-
-    let attach_str = format!(
-        "ATTACH 'ducklake:{db_type}:{db_conn_str}' AS {alias_name} (DATA_PATH '{s3_network_uri}'{extra_args});",
-    );
-
-    let install_db_ext_str = get_attach_db_install_str(db_type)?;
-    Ok(Some(vec![
-        "INSTALL ducklake;".to_string(),
-        install_db_ext_str.to_string(),
-        attach_str,
-    ]))
-}
-
-// Replaces all s3 URIs in the windmill syntax with the actual S3 network URIs
+// Returns the transformed query and the set of storages used
 async fn transform_s3_uris(
     query: &str,
     client: &AuthedClient,
-    duckdb_connection_settings_cache: &mut DuckDbConnectionSettingsCache,
-) -> Result<String> {
+) -> Result<(
+    Option<String>,
+    HashMap<Option<String>, DuckdbConnectionSettingsResponse>,
+)> {
     let mut transformed_query = None;
     lazy_static::lazy_static! {
-        static ref RE: regex::Regex = regex::Regex::new(r"'s3://([^'/]*)/([^']*)'").unwrap();
+        static ref RE: regex::Regex = regex::Regex::new(r"'s3://([^'/]*)/([^']+)'").unwrap();
     }
+    let mut used_storages = HashMap::new();
     for cap in RE.captures_iter(query) {
         if let (storage, Some(s3_path)) = (cap.get(1), cap.get(2)) {
             let s3_path = s3_path.as_str();
@@ -656,63 +584,42 @@ async fn transform_s3_uris(
             };
             let original_str_lit =
                 format!("'s3://{}/{}'", storage.as_deref().unwrap_or(""), s3_path);
-            let duckdb_conn_settings =
-                get_duckdb_connection_settings(&storage, duckdb_connection_settings_cache, client)
-                    .await?;
-            let url = duckdb_conn_settings_to_s3_network_uri(&duckdb_conn_settings, s3_path)?;
-            let url = format!("'{url}'");
+            let duckdb_conn_settings = client
+                .get_duckdb_connection_settings(&DuckdbConnectionSettingsQueryV2 {
+                    s3_resource_path: None,
+                    storage: storage.clone(),
+                })
+                .await?;
+            let url = match &duckdb_conn_settings {
+                DuckdbConnectionSettingsResponse { s3_bucket: Some(bucket), .. } => {
+                    format!("'s3://{bucket}/{s3_path}'")
+                }
+                DuckdbConnectionSettingsResponse { azure_container_path: Some(base), .. } => {
+                    format!("'{base}/{s3_path}'")
+                }
+                _ => {
+                    return Err(Error::ExecutionErr(
+                        "DuckDB connection settings response must have either s3_bucket or azure_container_path".to_string(),
+                    ))?;
+                }
+            };
             transformed_query = Some(
                 transformed_query
                     .unwrap_or(query.to_string())
                     .replace(&original_str_lit, &url),
             );
+            used_storages.insert(storage, duckdb_conn_settings);
         }
     }
-    Ok(transformed_query.unwrap_or(query.to_string()))
-}
-
-pub fn duckdb_conn_settings_to_s3_network_uri(
-    s: &DuckdbConnectionSettingsResponse,
-    s3_path: &str,
-) -> Result<String> {
-    match &s {
-        DuckdbConnectionSettingsResponse { s3_bucket: Some(bucket), .. } => {
-            Ok(format!("s3://{bucket}/{s3_path}"))
-        }
-        DuckdbConnectionSettingsResponse { azure_container_path: Some(base), .. } => {
-            Ok(format!("{base}/{s3_path}"))
-        }
-        _ => {
-            Err(Error::ExecutionErr(
-                "DuckDB connection settings response must have either s3_bucket or azure_container_path".to_string(),
-            ))
-        }
-    }
+    Ok((transformed_query, used_storages))
 }
 
 // BigQuery extension requires a json file as credentials
 // The file path is set as an env var by do_duckdb
 // It is created by transform_attach_db_resource_query (when bigquery is detected)
 // and deleted by do_duckdb after the query is executed
-pub struct UseBigQueryCredentialsFile {
-    path: String,
-}
-impl UseBigQueryCredentialsFile {
-    fn new(job_id: Uuid, bigquery_resource: &str) -> Result<Self> {
-        let path = format!("/tmp/service-account-credentials-{}.json", job_id);
-        env::set_var("GOOGLE_APPLICATION_CREDENTIALS", &path);
-        std::fs::write(&path, bigquery_resource)
-            .map_err(|e| Error::ExecutionErr(format!("Failed to write BigQuery creds: {e}")))?;
-        Ok(Self { path })
-    }
-}
-impl Drop for UseBigQueryCredentialsFile {
-    fn drop(&mut self) {
-        env::remove_var("GOOGLE_APPLICATION_CREDENTIALS");
-        if matches!(std::fs::exists(&self.path), Ok(true)) {
-            let _ = std::fs::remove_file(&self.path);
-        }
-    }
+fn make_bq_credentials_path(job_id: &Uuid) -> String {
+    format!("/tmp/service-account-credentials-{}.json", job_id)
 }
 
 // duckdb-rs does not support named parameters,
@@ -737,87 +644,9 @@ fn interpolate_named_args<'a>(
     (query, values)
 }
 
-// input should contain a single statement. remove all comments before and after it
-fn remove_comments(stmt: &str) -> &str {
-    let mut in_stmt = false;
-    let mut in_comment = false;
-    let mut start = None;
-    let mut end = stmt.len();
-
-    let mut c = ' ';
-    for (next_i, next_char) in stmt.char_indices() {
-        if next_i > 0 {
-            let i = next_i - 1;
-            if !in_comment && in_stmt && c == ';' {
-                end = i + 1;
-                break;
-            } else if in_comment && c == '\n' {
-                in_comment = false;
-            } else if c == '-' && next_char == '-' {
-                in_comment = true;
-            } else if !in_comment && !c.is_whitespace() && start == None {
-                start = Some(i);
-                in_stmt = true;
-            }
-        }
-        c = next_char;
-    }
-
-    return &stmt[start.unwrap_or(0)..end];
+fn trunc_sig(query: &str) -> &str {
+    let idx = query.rfind("-- $").unwrap_or(query.len());
+    // find next \n starting from idx and return everything after it
+    let idx = query[idx..].find('\n').map(|i| i + idx).unwrap_or(0);
+    &query[idx..]
 }
-
-async fn get_duckdb_connection_settings(
-    storage: &Option<String>,
-    cache: &mut DuckDbConnectionSettingsCache,
-    client: &AuthedClient,
-) -> Result<DuckdbConnectionSettingsResponse> {
-    if let Some(settings) = cache.get(storage) {
-        return Ok(settings.clone());
-    } else {
-        let settings = client
-            .get_duckdb_connection_settings(&DuckdbConnectionSettingsQueryV2 {
-                s3_resource_path: None,
-                storage: storage.clone(),
-            })
-            .await?;
-        cache.insert(storage.clone(), settings.clone());
-        return Ok(settings);
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    #[test]
-    fn test_remove_comments_single_line() {
-        let sql = "-- This is a comment\nSELECT * FROM table;";
-        assert_eq!(remove_comments(sql), "SELECT * FROM table;");
-    }
-    #[test]
-    fn test_remove_comments_multi_line() {
-        let sql = "-- This is a comment\nSELECT * FROM table;\n-- Another comment";
-        assert_eq!(remove_comments(sql), "SELECT * FROM table;");
-    }
-    #[test]
-    fn test_remove_comments_inline_comment() {
-        let sql = "   SELECT * FROM table;    -- This is an inline comment  ";
-        assert_eq!(remove_comments(sql), "SELECT * FROM table;");
-    }
-    #[test]
-    fn test_remove_comments_no_comments() {
-        let sql = "SELECT * FROM table;";
-        assert_eq!(remove_comments(sql), "SELECT * FROM table;");
-    }
-    #[test]
-    fn test_remove_comments_empty_string() {
-        let sql = "";
-        assert_eq!(remove_comments(sql), "");
-    }
-    #[test]
-    fn test_remove_comments_with_whitespace() {
-        let sql = "   -- Comment\n  -- Comment2\n  -- Comment3\n   SELECT\n\n * FROM\n table\n;\n\n -- end comment   ";
-        assert_eq!(remove_comments(sql), "SELECT\n\n * FROM\n table\n;");
-    }
-}
-
-type DuckDbConnectionSettingsCache = HashMap<Option<String>, DuckdbConnectionSettingsResponse>;

backend/windmill-worker/src/go_executor.rs

@@ -408,7 +408,7 @@ func Run(req Req) (interface{{}}, error){{
         run_go.stdout(Stdio::piped()).stderr(Stdio::piped());
         start_child_process(run_go, &compiled_executable_name).await?
     };
-    let handle_result = handle_child(
+    handle_child(
         &job.id,
         conn,
         mem_peak,
@@ -425,7 +425,7 @@ func Run(req Req) (interface{{}}, error){{
     )
     .await?;
 
-    read_result(job_dir, handle_result.result_stream).await
+    read_result(job_dir).await
 }
 
 async fn gen_go_mod(

backend/windmill-worker/src/handle_child.rs

@@ -7,7 +7,6 @@ use nix::unistd::Pid;
 use process_wrap::tokio::TokioChildWrapper;
 use windmill_common::agent_workers::PingJobStatusResponse;
 use windmill_common::jobs::LARGE_LOG_THRESHOLD_SIZE;
-use windmill_common::result_stream::extract_stream_from_logs;
 
 #[cfg(windows)]
 use std::process::Stdio;
@@ -53,7 +52,7 @@ use futures::{
 };
 
 use crate::common::{resolve_job_timeout, OccupancyMetrics};
-use crate::job_logger::{append_job_logs, append_result_stream, append_with_limit};
+use crate::job_logger::{append_job_logs, append_with_limit};
 use crate::job_logger_oss::process_streaming_log_lines;
 use crate::worker_utils::{ping_job_status, update_worker_ping_from_job};
 use crate::{MAX_RESULT_SIZE, MAX_WAIT_FOR_SIGINT, MAX_WAIT_FOR_SIGTERM};
@@ -88,10 +87,6 @@ async fn kill_process_tree(pid: Option<u32>) -> Result<(), String> {
     }
 }
 
-pub struct HandleChildResult {
-    pub result_stream: Option<String>,
-}
-
 /// - wait until child exits and return with exit status
 /// - read lines from stdout and stderr and append them to the "queue"."logs"
 ///   quitting early if output exceedes MAX_LOG_SIZE characters (not bytes)
@@ -114,7 +109,7 @@ pub async fn handle_child(
     occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
     // Do not print logs to output, but instead save to string.
     pipe_stdout: Option<&mut String>,
-) -> error::Result<HandleChildResult> {
+) -> error::Result<()> {
     let start = Instant::now();
 
     let pid = child.id();
@@ -301,7 +296,6 @@ pub async fn handle_child(
         }
     };
 
-    let mut stream_result = Vec::new();
     /* a future that reads output from the child and appends to the database */
     let lines = write_lines(
         output,
@@ -314,7 +308,6 @@ pub async fn handle_child(
         pipe_stdout,
         &mut rx2,
         child_name,
-        &mut stream_result,
     )
     .instrument(trace_span!("child_lines"));
 
@@ -329,7 +322,7 @@ pub async fn handle_child(
         _ if *too_many_logs.borrow() => Err(Error::ExecutionErr(format!(
             "logs or result reached limit. (current max size: {MAX_RESULT_SIZE} characters)"
         ))),
-        Ok(Ok(status)) => process_status(&child_name, status, stream_result),
+        Ok(Ok(status)) => process_status(&child_name, status),
         Ok(Err(kill_reason)) => match kill_reason {
             KillReason::AlreadyCompleted => {
                 Err(Error::AlreadyCompleted("Job already completed".to_string()))
@@ -353,7 +346,6 @@ pub async fn write_lines(
     pipe_stdout: Option<&mut String>,
     rx2: &mut broadcast::Receiver<()>,
     child_name: &str,
-    stream_result: &mut Vec<String>,
 ) {
     let max_log_size = if *CLOUD_HOSTED {
         MAX_RESULT_SIZE
@@ -409,25 +401,13 @@ pub async fn write_lines(
         let mut joined = String::new();
 
         let job_id = job_id.clone();
-        let mut nstream = String::new();
         while let Some(line) = read_lines.next().await {
             match line {
                 Ok(line) => {
                     if line.is_empty() {
                         continue;
                     }
-                    if let Some(stream) = extract_stream_from_logs(&line) {
-                        let len = stream.len();
-                        if log_remaining >= len {
-                            log_remaining -= len;
-                            nstream.push_str(&stream);
-                            stream_result.push(stream);
-                        } else {
-                            log_remaining = 0;
-                        }
-                    } else {
-                        append_with_limit(&mut joined, &line, &mut log_remaining);
-                    }
+                    append_with_limit(&mut joined, &line, &mut log_remaining);
                     if log_remaining == 0 {
                         tracing::info!(%job_id, "Too many logs lines for job {job_id}");
                         let _ = set_too_many_logs.send(true);
@@ -480,14 +460,6 @@ pub async fn write_lines(
             let job_id = job_id.clone();
             let pg_log_total_size = pg_log_total_size.clone();
             (do_write, write_result) = tokio::spawn(async move {
-                if !nstream.is_empty() {
-                    if let Err(err) = append_result_stream(&conn, &w_id, &job_id, &nstream).await {
-                        tracing::error!(
-                            "Unable to send result stream for job {job_id}. Error was: {:?}",
-                            err
-                        );
-                    }
-                }
                 append_job_logs(
                     &job_id,
                     &w_id,
@@ -790,19 +762,9 @@ pub fn lines_to_stream<R: tokio::io::AsyncBufRead + Unpin>(
     })
 }
 
-pub fn process_status(
-    program: &str,
-    status: ExitStatus,
-    stream_result: Vec<String>,
-) -> error::Result<HandleChildResult> {
+pub fn process_status(program: &str, status: ExitStatus) -> error::Result<()> {
     if status.success() {
-        Ok(HandleChildResult {
-            result_stream: if stream_result.is_empty() {
-                None
-            } else {
-                Some(stream_result.join(""))
-            },
-        })
+        Ok(())
     } else if let Some(code) = status.code() {
         Err(error::Error::ExitStatus(program.to_string(), code))
     } else {

backend/windmill-worker/src/java_executor.rs

@@ -88,7 +88,7 @@ pub async fn handle_java_job<'a>(mut args: JobHandlerInput<'a>) -> Result<Box<Ra
     }
     // --- Retrieve results ---
     {
-        read_result(&args.job_dir, None).await
+        read_result(&args.job_dir).await
     }
 }
 
@@ -718,8 +718,7 @@ async fn run<'a>(
         &mut Some(occupancy_metrics),
         None,
     )
-    .await?;
-    Ok(())
+    .await
 }
 
 #[derive(Default, Debug)]

backend/windmill-worker/src/job_logger.rs

@@ -1,11 +1,10 @@
 use regex::Regex;
 
 pub use windmill_common::jobs::LARGE_LOG_THRESHOLD_SIZE;
-use windmill_common::result_stream::append_result_stream_db;
 use windmill_common::utils::WarnAfterExt;
 use windmill_common::worker::{Connection, CLOUD_HOSTED};
 
-use windmill_common::{error, DB};
+use windmill_common::DB;
 use windmill_queue::append_logs;
 
 use std::sync::atomic::AtomicU32;
@@ -62,32 +61,6 @@ pub async fn append_job_logs(
     }
 }
 
-pub async fn append_result_stream(
-    conn: &Connection,
-    workspace_id: &str,
-    job_id: &Uuid,
-    nstream: &str,
-) -> error::Result<()> {
-    match conn {
-        Connection::Sql(db) => {
-            append_result_stream_db(db, workspace_id, job_id, nstream).await?;
-        }
-        Connection::Http(client) => {
-            if let Err(e) = client
-                .post::<_, String>(
-                    &format!("/api/w/{}/agent_workers/push_logs/{}", workspace_id, job_id),
-                    None,
-                    &nstream,
-                )
-                .await
-            {
-                tracing::error!(%job_id, %e, "error sending result stream for  job {job_id}: {e}");
-            };
-        }
-    }
-    Ok(())
-}
-
 pub async fn append_logs_with_compaction(
     job_id: &Uuid,
     w_id: &str,

backend/windmill-worker/src/js_eval.rs

@@ -701,7 +701,7 @@ pub struct MainArgs {
 
 #[cfg(feature = "deno_core")]
 pub struct LogString {
-    pub s: mpsc::UnboundedSender<String>,
+    pub s: String,
 }
 
 #[cfg(feature = "deno_core")]
@@ -894,8 +894,6 @@ pub async fn eval_fetch_timeout(
             return y*2;
         });
 
-        let (log_sender, mut log_receiver) = mpsc::unbounded_channel::<String>();
-
         {
             let op_state = js_runtime.op_state();
             let mut op_state = op_state.borrow_mut();
@@ -903,7 +901,7 @@ pub async fn eval_fetch_timeout(
             //reqwest client seems to not be sharable between runtimes unfortunately
             // op_state.put(HTTP_CLIENT.clone());
             op_state.put(MainArgs { args: spread });
-            op_state.put(LogString { s: log_sender });
+            op_state.put(LogString { s: String::new() });
         }
 
         sender
@@ -915,51 +913,23 @@ pub async fn eval_fetch_timeout(
             .build()?;
 
         let future = async {
-            use crate::common::merge_result_stream;
-
-            if !extra_logs.is_empty() {
-                append_logs(&job_id, w_id_.as_str(), format!("{extra_logs}"), &conn_).await;
-            }
-            let w_id = w_id_.clone();
-            let handle = tokio::spawn(async move {
-                let mut result_stream = String::new();
-                while let Some(log) = log_receiver.recv().await {
-                    use windmill_common::result_stream::extract_stream_from_logs;
-
-                    if let Some(stream) = extract_stream_from_logs(&log.trim_end_matches("\n")) {
-                        use crate::job_logger::append_result_stream;
-
-                        result_stream.push_str(&stream);
-                        if let Err(e) = append_result_stream(&conn_, &w_id, &job_id, &stream).await
-                        {
-                            tracing::error!("failed to append result stream for job {job_id}: {e}");
-                        }
-                    } else {
-                        append_logs(&job_id, w_id_.as_str(), log, &conn_).await;
-                    }
-                }
-                if !result_stream.is_empty() {
-                    Some(result_stream)
-                } else {
-                    None
-                }
-            });
-
             let r = tokio::select! {
                 r = eval_fetch(&mut js_runtime, &js_expr, Some(env_code), script_entrypoint_override, load_client, &job_id) => Ok(r),
                 _ = memory_limit_rx.recv() => Err(Error::ExecutionErr("Memory limit reached, killing isolate".to_string()))
             };
-            drop(js_runtime);
-            if let Ok(r) = r {
-                match handle.await {
-                    Ok(Some(logs)) => Ok(merge_result_stream(r, Some(logs)).await),
-                    Ok(None) => Ok(r),
-                    Err(e) => Err(Error::ExecutionErr(e.to_string())),
-                }
-            } else {
-                r
-            }
-            // r
+
+            append_logs(
+                &job_id,
+                w_id_.as_str(),
+                format!(
+                    "{extra_logs}{}",
+                    js_runtime.op_state().borrow().borrow::<LogString>().s
+                ),
+                &conn_,
+            )
+            .await;
+
+            r
         };
         let r = runtime.block_on(future)?;
         // tracing::info!("total: {:?}", instant.elapsed());
@@ -1069,46 +1039,8 @@ async fn eval_fetch(
             "<anon>",
             format!(
                 r#"
-function isAsyncIterable(obj) {{
-    // return true;    // TODO: remove this
-    return obj != null && typeof obj[Symbol.asyncIterator] === 'function';
-}}
-
-function processStreamIterative(res) {{
-    const iterator = res[Symbol.asyncIterator]();
-    
-    function processLoop() {{
-        return new Promise(function(resolve) {{
-            function step() {{
-                iterator.next().then(function(result) {{
-                    if (!result.done) {{
-                        const chunk = result.value;
-                        console.log("WM_STREAM: " + chunk.replace('\n', '\\n'));
-                        // Continue the loop
-                        step();
-                    }} else {{
-                        resolve("null");
-                    }}
-                }}).catch(function(error) {{
-                    resolve("null");
-                }});
-            }}
-            step();
-        }});
-    }}
-    
-    return processLoop();
-}}
-
 let args = Deno.core.ops.op_get_static_args().map(JSON.parse)
-import("file:///eval.ts").then((module) => module.{main_override}(...args))
-    .then(res => {{
-        if (isAsyncIterable(res)) {{
-            return processStreamIterative(res)
-        }} else {{
-            return JSON.stringify(res ?? null);
-        }}
-    }})
+import("file:///eval.ts").then((module) => module.{main_override}(...args)).then(JSON.stringify)
 "#
             ),
         )
@@ -1188,14 +1120,11 @@ fn op_get_static_args(op_state: Rc<RefCell<OpState>>) -> Vec<Option<String>> {
 #[op2(fast)]
 fn op_log(op_state: Rc<RefCell<OpState>>, #[string] log: &str) {
     // tracing::error!("log: |{}|", log);
-    if let Err(e) = op_state
+    op_state
         .borrow_mut()
         .borrow_mut::<LogString>()
         .s
-        .send(log.to_string())
-    {
-        tracing::error!("failed to send log: {e}");
-    }
+        .push_str(log);
 }
 
 #[cfg(feature = "deno_core")]

backend/windmill-worker/src/nu_executor.rs

@@ -20,6 +20,7 @@ use crate::{
 };
 use windmill_common::client::AuthedClient;
 
+
 const NSJAIL_CONFIG_RUN_NU_CONTENT: &str = include_str!("../nsjail/run.nu.config.proto");
 lazy_static::lazy_static! {
     static ref NU_PATH: String = std::env::var("NU_PATH").unwrap_or_else(|_| "/usr/bin/nu".to_string());
@@ -68,7 +69,7 @@ pub async fn handle_nu_job<'a>(mut args: JobHandlerInput<'a>) -> Result<Box<RawV
     }
     // --- Retrieve results ---
     {
-        read_result(&args.job_dir, None).await
+        read_result(&args.job_dir).await
     }
 }
 
@@ -340,8 +341,7 @@ async fn run<'a>(
         &mut Some(occupancy_metrics),
         None,
     )
-    .await?;
-    Ok(())
+    .await
 }
 // #[cfg(test)]
 // mod test {

backend/windmill-worker/src/php_executor.rs

@@ -20,7 +20,8 @@ use crate::{
         read_result, start_child_process, OccupancyMetrics,
     },
     handle_child::handle_child,
-    COMPOSER_CACHE_DIR, COMPOSER_PATH, DISABLE_NSJAIL, DISABLE_NUSER, NSJAIL_PATH, PHP_PATH,
+    COMPOSER_CACHE_DIR, COMPOSER_PATH, DISABLE_NSJAIL, DISABLE_NUSER, NSJAIL_PATH,
+    PHP_PATH,
 };
 use windmill_common::client::AuthedClient;
 
@@ -344,5 +345,5 @@ try {{
         None,
     )
     .await?;
-    read_result(job_dir, None).await
+    read_result(job_dir).await
 }