all

refresh superadmin state on workspace list page

CHANGELOG.md

@@ -1,5 +1,79 @@
 # Changelog
 
+## [1.403.1](https://github.com/windmill-labs/windmill/compare/v1.403.0...v1.403.1) (2024-10-01)
+
+
+### Bug Fixes
+
+* fix new instance db setup ([73ab8e1](https://github.com/windmill-labs/windmill/commit/73ab8e1653d6e0c0c69fa7dcd96583f25d13ef86))
+
+## [1.403.0](https://github.com/windmill-labs/windmill/compare/v1.402.3...v1.403.0) (2024-10-01)
+
+
+### Features
+
+* flow step skipping ([#4461](https://github.com/windmill-labs/windmill/issues/4461)) ([0df169e](https://github.com/windmill-labs/windmill/commit/0df169e3f996ed54b91569b13cce15d7d019a213))
+
+
+### Bug Fixes
+
+* skip one migration to avoid using md5 for azure support ([630ae5d](https://github.com/windmill-labs/windmill/commit/630ae5d425cd9957d674befd2df96e2befec52a3))
+
+## [1.402.3](https://github.com/windmill-labs/windmill/compare/v1.402.2...v1.402.3) (2024-09-30)
+
+
+### Bug Fixes
+
+* improve allowed domains setting for sso ([24f4a7c](https://github.com/windmill-labs/windmill/commit/24f4a7caaafa93f51669dcf44a3dca09d5b228bb))
+
+## [1.402.2](https://github.com/windmill-labs/windmill/compare/v1.402.1...v1.402.2) (2024-09-28)
+
+
+### Bug Fixes
+
+* make form properties disablable ([0779d47](https://github.com/windmill-labs/windmill/commit/0779d47c1d39626d11bd3769cd787cb036df0a94))
+
+## [1.402.1](https://github.com/windmill-labs/windmill/compare/v1.402.0...v1.402.1) (2024-09-28)
+
+
+### Bug Fixes
+
+* allow preprocessor to write to args.json on nsjail ([#4455](https://github.com/windmill-labs/windmill/issues/4455)) ([0b9ec83](https://github.com/windmill-labs/windmill/commit/0b9ec83036e2a1d0773b4ec5856f907b383e9323))
+* **frontend:** Fix flow graph bg in dark mode on chrome ([#4454](https://github.com/windmill-labs/windmill/issues/4454)) ([6956a3a](https://github.com/windmill-labs/windmill/commit/6956a3a2ba6d189528cb34ab05f7137fdf4f840b))
+* improve suspend_first behavior and frequency ([b5e226b](https://github.com/windmill-labs/windmill/commit/b5e226b977e6d24ebd28bc1e7c867cb4888f77b2))
+
+## [1.402.0](https://github.com/windmill-labs/windmill/compare/v1.401.0...v1.402.0) (2024-09-26)
+
+
+### Features
+
+* **cli:** add queues, workers and worker-groups commands ([#4439](https://github.com/windmill-labs/windmill/issues/4439)) ([9f91b19](https://github.com/windmill-labs/windmill/commit/9f91b1995a98c9e096c6e599c4d5a8d5ea499ada))
+
+## [1.401.0](https://github.com/windmill-labs/windmill/compare/v1.400.0...v1.401.0) (2024-09-25)
+
+
+### Features
+
+* add return_last_result annotation to sql ([#4443](https://github.com/windmill-labs/windmill/issues/4443)) ([3ce5587](https://github.com/windmill-labs/windmill/commit/3ce5587faae3912ceedae4644732fa9704eb6d76))
+
+
+### Bug Fixes
+
+* fix flow rendering ([fd58e7e](https://github.com/windmill-labs/windmill/commit/fd58e7eb48c4fb66d199c33d0f8aaf2535485a2f))
+
+## [1.400.0](https://github.com/windmill-labs/windmill/compare/v1.399.0...v1.400.0) (2024-09-25)
+
+
+### Features
+
+* add static variable and resources support to ansible ([#4435](https://github.com/windmill-labs/windmill/issues/4435)) ([398a09b](https://github.com/windmill-labs/windmill/commit/398a09b7419c7adafa76e5f9f5a981bec6f0611d))
+* **frontend:** Catch flow errors in the UI ([#4429](https://github.com/windmill-labs/windmill/issues/4429)) ([84eefad](https://github.com/windmill-labs/windmill/commit/84eefadfcf06b4f38117b6c4c534f47fb2ef7cc7))
+
+
+### Bug Fixes
+
+* remove autocomplete for searchbar ([#4440](https://github.com/windmill-labs/windmill/issues/4440)) ([a1ac583](https://github.com/windmill-labs/windmill/commit/a1ac583f05c34a534c4ae78ca7d2a6850723ce85))
+
 ## [1.399.0](https://github.com/windmill-labs/windmill/compare/v1.398.1...v1.399.0) (2024-09-25)
 
 

backend/.sqlx/query-19f0abd79372698f378cb6deea3ee6d098a2758d16ede000809bd9a09660b604.json

@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT created_at FROM metrics WHERE id = 'telemetry' ORDER BY created_at DESC LIMIT 1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "created_at",
+        "type_info": "Timestamptz"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "19f0abd79372698f378cb6deea3ee6d098a2758d16ede000809bd9a09660b604"
+}

backend/.sqlx/query-1e188d8e427cab25dbe18aa900260e26e644a9d939e74a8317c4a09335f110fe.json

@@ -0,0 +1,25 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE queue\n                                SET flow_status = JSONB_SET(\n                                JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4),\n                                ARRAY['modules', $1::TEXT, 'branchall', 'branch'], ((flow_status->'modules'->$1::int->'branchall'->>'branch')::int + 1)::text::jsonb),\n                                last_ping = NULL\n                                WHERE id = $2\n                                RETURNING (flow_status->'modules'->$1::int->'branchall'->>'branch')::int",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "int4",
+        "type_info": "Int4"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int4",
+        "Uuid",
+        "Text",
+        "Jsonb"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "1e188d8e427cab25dbe18aa900260e26e644a9d939e74a8317c4a09335f110fe"
+}

backend/.sqlx/query-3e539fef054ad31bc1736e27276087775a721a6ee7ae35b03fd4ce3563ea3838.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT raw_flow->'modules'->($1)->'value'->>'type' = 'flow' FROM queue WHERE id = $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "?column?",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Uuid"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "3e539fef054ad31bc1736e27276087775a721a6ee7ae35b03fd4ce3563ea3838"
+}

backend/.sqlx/query-41f68f4ce5bed783cf69e42da115e9ad2c9fcbd75f55817b2114c04207f66e4a.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "INSERT INTO metrics (id, value)\n                            VALUES ($1, to_jsonb((SELECT EXTRACT(EPOCH FROM now() - scheduled_for)\n                            FROM queue WHERE tag = $2 AND running = false AND scheduled_for <= now() - ('3 seconds')::interval\n                            ORDER BY priority DESC NULLS LAST, scheduled_for, created_at LIMIT 1)))",
+  "query": "INSERT INTO metrics (id, value)\n                            VALUES ($1, to_jsonb((SELECT EXTRACT(EPOCH FROM now() - scheduled_for)\n                            FROM queue WHERE tag = $2 AND running = false AND scheduled_for <= now() - ('3 seconds')::interval\n                            ORDER BY priority DESC NULLS LAST, scheduled_for LIMIT 1)))",
   "describe": {
     "columns": [],
     "parameters": {
@@ -11,5 +11,5 @@
     },
     "nullable": []
   },
-  "hash": "9bf41c3161a02b7d0731c4e1d79519cef5255f5df1b759af3aa4985bb64313e5"
+  "hash": "41f68f4ce5bed783cf69e42da115e9ad2c9fcbd75f55817b2114c04207f66e4a"
 }

backend/.sqlx/query-6776dc50f184188756ad7fe263b0304333536768527525a43bdd45aedffa3c4f.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT pg_try_advisory_xact_lock($1)",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "pg_try_advisory_xact_lock",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int8"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "6776dc50f184188756ad7fe263b0304333536768527525a43bdd45aedffa3c4f"
+}

backend/.sqlx/query-6a497334c98bfaf70be44fced572a1cc0dde4141aa4c5002765a95432d0101ab.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT worker, worker_instance,  EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed, CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as last_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as last_job_workspace_id, custom_tags, worker_group, wm_version, occupancy_rate, memory, vcpus, memory_usage, wm_memory_usage\n        FROM worker_ping\n        WHERE ($1::integer IS NULL AND ping_at > now() - interval '5 minute') OR (ping_at > now() - ($1 || ' seconds')::interval)\n        ORDER BY ping_at desc LIMIT $2 OFFSET $3",
+  "query": "SELECT worker, worker_instance,  EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed,\n        CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as last_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as last_job_workspace_id, \n        custom_tags, worker_group, wm_version, occupancy_rate, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, memory, vcpus, memory_usage, wm_memory_usage\n        FROM worker_ping\n        WHERE ($1::integer IS NULL AND ping_at > now() - interval '5 minute') OR (ping_at > now() - ($1 || ' seconds')::interval)\n        ORDER BY ping_at desc LIMIT $2 OFFSET $3",
   "describe": {
     "columns": [
       {
@@ -65,21 +65,36 @@
       },
       {
         "ordinal": 12,
+        "name": "occupancy_rate_15s",
+        "type_info": "Float4"
+      },
+      {
+        "ordinal": 13,
+        "name": "occupancy_rate_5m",
+        "type_info": "Float4"
+      },
+      {
+        "ordinal": 14,
+        "name": "occupancy_rate_30m",
+        "type_info": "Float4"
+      },
+      {
+        "ordinal": 15,
         "name": "memory",
         "type_info": "Int8"
       },
       {
-        "ordinal": 13,
+        "ordinal": 16,
         "name": "vcpus",
         "type_info": "Int8"
       },
       {
-        "ordinal": 14,
+        "ordinal": 17,
         "name": "memory_usage",
         "type_info": "Int8"
       },
       {
-        "ordinal": 15,
+        "ordinal": 18,
         "name": "wm_memory_usage",
         "type_info": "Int8"
       }
@@ -108,8 +123,11 @@
       true,
       true,
       true,
+      true,
+      true,
+      true,
       true
     ]
   },
-  "hash": "8375c1efeb1e2a2d2803052a2899bf70f4a6434eb91b4b05b9fb8420beae26af"
+  "hash": "6a497334c98bfaf70be44fced572a1cc0dde4141aa4c5002765a95432d0101ab"
 }

backend/.sqlx/query-6e7f234267fbb4720b29f288fba82c1df21ba601ac0989e175f834c569962d46.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE queue\n                                SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'iterator', 'index'], ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb),\n                                last_ping = NULL\n                                WHERE id = $2\n                                RETURNING (flow_status->'modules'->$1::int->'iterator'->>'index')::int",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "int4",
+        "type_info": "Int4"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int4",
+        "Uuid"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "6e7f234267fbb4720b29f288fba82c1df21ba601ac0989e175f834c569962d46"
+}

backend/.sqlx/query-6f4817fad2739a11d89b6704edf62c3c267ca336a8b6bec5b29d4409030ed561.json

@@ -0,0 +1,73 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO queue (id, script_hash, script_path, job_kind, language, tag, created_by, permissioned_as, email, scheduled_for, workspace_id, raw_flow, flow_status) (SELECT gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12 FROM generate_series(1, 1))",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Int8",
+        "Varchar",
+        {
+          "Custom": {
+            "name": "job_kind",
+            "kind": {
+              "Enum": [
+                "script",
+                "preview",
+                "flow",
+                "dependencies",
+                "flowpreview",
+                "script_hub",
+                "identity",
+                "flowdependencies",
+                "http",
+                "graphql",
+                "postgresql",
+                "noop",
+                "appdependencies",
+                "deploymentcallback",
+                "singlescriptflow"
+              ]
+            }
+          }
+        },
+        {
+          "Custom": {
+            "name": "script_lang",
+            "kind": {
+              "Enum": [
+                "python3",
+                "deno",
+                "go",
+                "bash",
+                "postgresql",
+                "nativets",
+                "bun",
+                "mysql",
+                "bigquery",
+                "snowflake",
+                "graphql",
+                "powershell",
+                "mssql",
+                "php",
+                "bunnative",
+                "rust",
+                "ansible"
+              ]
+            }
+          }
+        },
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "Timestamptz",
+        "Varchar",
+        "Jsonb",
+        "Jsonb"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "6f4817fad2739a11d89b6704edf62c3c267ca336a8b6bec5b29d4409030ed561"
+}

backend/.sqlx/query-777190559e27c8c8fb6718b0a0c1d7db9b956abd88b94db3948f2c579c3826d0.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "UPDATE queue\n                            SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'branchall', 'branch'], ((flow_status->'modules'->$1::int->'branchall'->>'branch')::int + 1)::text::jsonb)\n                            WHERE id = $2\n                            RETURNING (flow_status->'modules'->$1::int->'branchall'->>'branch')::int",
+  "query": "UPDATE queue\n                            SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'branchall', 'branch'], ((flow_status->'modules'->$1::int->'branchall'->>'branch')::int + 1)::text::jsonb),\n                            last_ping = NULL\n                            WHERE id = $2\n                            RETURNING (flow_status->'modules'->$1::int->'branchall'->>'branch')::int",
   "describe": {
     "columns": [
       {
@@ -19,5 +19,5 @@
       null
     ]
   },
-  "hash": "a94dbd1f7aab20682548471c5cc06c7a496edac36e3de537b00b76ad93c7556c"
+  "hash": "777190559e27c8c8fb6718b0a0c1d7db9b956abd88b94db3948f2c579c3826d0"
 }

backend/.sqlx/query-804fc11e35f4afc0db194b6fe2594f91df7e588d4d2431bc85f4d8734920c8bf.json

@@ -0,0 +1,45 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO script (summary, description, dedicated_worker, content, workspace_id, path, hash, language, tag, created_by, lock) VALUES ('', '', true, $1, $2, $3, $4, $5, $6, $7, '') ON CONFLICT (workspace_id, hash) DO NOTHING",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Varchar",
+        "Varchar",
+        "Int8",
+        {
+          "Custom": {
+            "name": "script_lang",
+            "kind": {
+              "Enum": [
+                "python3",
+                "deno",
+                "go",
+                "bash",
+                "postgresql",
+                "nativets",
+                "bun",
+                "mysql",
+                "bigquery",
+                "snowflake",
+                "graphql",
+                "powershell",
+                "mssql",
+                "php",
+                "bunnative",
+                "rust",
+                "ansible"
+              ]
+            }
+          }
+        },
+        "Varchar",
+        "Varchar"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "804fc11e35f4afc0db194b6fe2594f91df7e588d4d2431bc85f4d8734920c8bf"
+}

backend/.sqlx/query-829130d74c107e4e1a86f3e772657ae030463c139c39072777e453bfb7e9c0c3.json

@@ -1,23 +1,22 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT raw_flow->'modules'->$2::int->'retry' FROM queue WHERE id = $1",
+  "query": "SELECT job_kind = 'identity' FROM completed_job WHERE id = $1",
   "describe": {
     "columns": [
       {
         "ordinal": 0,
         "name": "?column?",
-        "type_info": "Jsonb"
+        "type_info": "Bool"
       }
     ],
     "parameters": {
       "Left": [
-        "Uuid",
-        "Int4"
+        "Uuid"
       ]
     },
     "nullable": [
       null
     ]
   },
-  "hash": "ae2f005af8ab4b035a907e0c8fc9a9d035f3eb1d9d833041969fce967daa91a4"
+  "hash": "829130d74c107e4e1a86f3e772657ae030463c139c39072777e453bfb7e9c0c3"
 }

backend/.sqlx/query-9cf96fa6364b7f34dc83719b4a0e97e8494393c29f7c8d915aa54da7ab7eed51.json

@@ -1,21 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1, custom_tags = $2, occupancy_rate = $3, memory_usage = $4, wm_memory_usage = $5, vcpus = COALESCE($7, vcpus), memory = COALESCE($8, memory) WHERE worker = $6",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Int4",
-        "TextArray",
-        "Float4",
-        "Int8",
-        "Int8",
-        "Text",
-        "Int8",
-        "Int8"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "9cf96fa6364b7f34dc83719b4a0e97e8494393c29f7c8d915aa54da7ab7eed51"
-}

backend/.sqlx/query-a439552f74ed0ba305e3d9cb99ae9e5d24834082ebf2fe9fd3964fdd80b69ccb.json

@@ -0,0 +1,24 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1, custom_tags = $2,\n                 occupancy_rate = $3, memory_usage = $4, wm_memory_usage = $5, vcpus = COALESCE($7, vcpus),\n                 memory = COALESCE($8, memory), occupancy_rate_15s = $9, occupancy_rate_5m = $10, occupancy_rate_30m = $11 WHERE worker = $6",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Int4",
+        "TextArray",
+        "Float4",
+        "Int8",
+        "Int8",
+        "Text",
+        "Int8",
+        "Int8",
+        "Float4",
+        "Float4",
+        "Float4"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "a439552f74ed0ba305e3d9cb99ae9e5d24834082ebf2fe9fd3964fdd80b69ccb"
+}

backend/.sqlx/query-c3b5abbf2c9079d597a55f7c63bc83b8b4da98bda204a40f045a62172cfb4ebb.json

@@ -0,0 +1,29 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n        SELECT tag as \"tag!\", COUNT(*) as \"count!\"\n        FROM completed_job\n        WHERE started_at > NOW() - make_interval(secs => $1) AND ($2::text IS NULL OR workspace_id = $2)\n        GROUP BY tag\n        ORDER BY \"count!\" DESC\n        ",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "tag!",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "count!",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Float8",
+        "Text"
+      ]
+    },
+    "nullable": [
+      false,
+      null
+    ]
+  },
+  "hash": "c3b5abbf2c9079d597a55f7c63bc83b8b4da98bda204a40f045a62172cfb4ebb"
+}

backend/.sqlx/query-d0df57fc4cd0be7b541dee081ffbe86b869b01ca1ab10aa17634bb6dae879f12.json

@@ -1,14 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "UPDATE queue\n                        SET last_ping = null\n                        WHERE id = $1",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Uuid"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "d0df57fc4cd0be7b541dee081ffbe86b869b01ca1ab10aa17634bb6dae879f12"
-}

backend/.sqlx/query-d6c25421bb6513908697ebe74c159da0ef78b5252b6ac669a3c9e545d05c0d43.json

@@ -1,23 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "UPDATE queue\n                            SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'iterator', 'index'], ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb)\n                            WHERE id = $2\n                            RETURNING (flow_status->'modules'->$1::int->'iterator'->>'index')::int",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "int4",
-        "type_info": "Int4"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Int4",
-        "Uuid"
-      ]
-    },
-    "nullable": [
-      null
-    ]
-  },
-  "hash": "d6c25421bb6513908697ebe74c159da0ef78b5252b6ac669a3c9e545d05c0d43"
-}

backend/.sqlx/query-e968e879d3c52f7dd502c3cd15fc8fbd983a4a3ab25648c562497a27c74b5c8c.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "UPDATE worker_ping SET ping_at = now(), current_job_id = $1, current_job_workspace_id = $2, memory_usage = $3, wm_memory_usage = $4 WHERE worker = $5",
+  "query": "UPDATE worker_ping SET ping_at = now(), current_job_id = $1, current_job_workspace_id = $2, memory_usage = $3, wm_memory_usage = $4,\n                            occupancy_rate = $6, occupancy_rate_15s = $7, occupancy_rate_5m = $8, occupancy_rate_30m = $9 WHERE worker = $5",
   "describe": {
     "columns": [],
     "parameters": {
@@ -9,10 +9,14 @@
         "Varchar",
         "Int8",
         "Int8",
-        "Text"
+        "Text",
+        "Float4",
+        "Float4",
+        "Float4",
+        "Float4"
       ]
     },
     "nullable": []
   },
-  "hash": "95cb1fe8658f98fb736d899fa21cd7378b0c9d3b5f3d6bd6cafcba273f8277d4"
+  "hash": "e968e879d3c52f7dd502c3cd15fc8fbd983a4a3ab25648c562497a27c74b5c8c"
 }

backend/.sqlx/query-f916ec232837ece9323675e5f5142e7285f4266a1471e5ffdefadf421a67e44b.json

@@ -0,0 +1,25 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE queue\n                            SET flow_status = JSONB_SET(\n                                JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4),\n                                ARRAY['modules', $1::TEXT, 'iterator', 'index'],\n                                ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb\n                            ),\n                            last_ping = NULL\n                            WHERE id = $2\n                            RETURNING (flow_status->'modules'->$1::int->'iterator'->>'index')::int",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "int4",
+        "type_info": "Int4"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int4",
+        "Uuid",
+        "Text",
+        "Jsonb"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "f916ec232837ece9323675e5f5142e7285f4266a1471e5ffdefadf421a67e44b"
+}

backend/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "windmill"
-version = "1.399.0"
+version = "1.403.1"
 authors.workspace = true
 edition.workspace = true
 
@@ -27,7 +27,7 @@ members = [
 ]
 
 [workspace.package]
-version = "1.399.0"
+version = "1.403.1"
 authors = ["Ruben Fiszel <ruben@windmill.dev>"]
 edition = "2021"
 
@@ -44,7 +44,7 @@ default = []
 enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmill-api/enterprise", "windmill-git-sync/enterprise", "windmill-common/prometheus", "windmill-common/enterprise", "windmill-indexer/enterprise"]
 enterprise_saml = ["windmill-api/enterprise_saml"]
 stripe = ["windmill-api/stripe"]
-benchmark = ["windmill-api/benchmark", "windmill-worker/benchmark", "windmill-queue/benchmark"]
+benchmark = ["windmill-api/benchmark", "windmill-worker/benchmark", "windmill-queue/benchmark", "windmill-common/benchmark"]
 flamegraph = ["windmill-common/flamegraph", "windmill-worker/flamegraph"]
 loki = ["windmill-common/loki"]
 pg_embed = ["dep:pg-embed"]

backend/custom_migrations/create_workspace_without_md5.sql

@@ -0,0 +1,16 @@
+INSERT INTO workspace(id, name, owner) VALUES
+	('admins', 'Admins', 'admin@windmill.dev') ON CONFLICT DO NOTHING;
+
+INSERT INTO workspace_settings (workspace_id) VALUES
+	('admins') ON CONFLICT DO NOTHING;
+
+INSERT INTO workspace_key
+	(workspace_id, kind, key)
+	VALUES ('admins', 'cloud', array_to_string(
+    array(
+        SELECT chr( (trunc(65 + random() * 25)::int) + 
+                    CASE WHEN random() > 0.5 THEN 32 ELSE 0 END )  -- generates random uppercase/lowercase letters
+        FROM generate_series(1, 32) -- generates 32 characters
+    ),
+    ''
+)) ON CONFLICT DO NOTHING;

backend/ee-repo-ref.txt

@@ -1 +1 @@
-3d37b6c31155265d8d026ae9d6ced0b433078f87
+0f5f42d2f8f5f1af05c8086f3dc7ad38d83750df

backend/migrations/20240924213010_add_last_occupancy_rates.down.sql

@@ -0,0 +1,5 @@
+-- Add down migration script here
+ALTER TABLE worker_ping
+DROP COLUMN occupancy_rate_15s,
+DROP COLUMN occupancy_rate_5m,
+DROP COLUMN occupancy_rate_30m;

backend/migrations/20240924213010_add_last_occupancy_rates.up.sql

@@ -0,0 +1,5 @@
+-- Add up migration script here
+ALTER TABLE worker_ping
+ADD COLUMN occupancy_rate_15s REAL,
+ADD COLUMN occupancy_rate_5m REAL,
+ADD COLUMN occupancy_rate_30m REAL;

backend/plot.py

@@ -0,0 +1,89 @@
+import json
+import matplotlib.pyplot as plt
+
+# Function to load JSON data from a file
+def load_json_data(filepath):
+    with open(filepath, 'r') as file:
+        data = json.load(file)
+    return data
+
+# Function to plot two arrays of subarrays with tuples (step_name, duration)
+def plot_two_arrays_of_subarrays(arrays1, arrays2):
+    # Function to calculate sum of durations for each step
+    def calculate_sums(arrays):
+        steps = [step for step, _ in arrays[0]['timings']]  # Extract steps from the first iteration
+        sums = {step: 0.0 for step in steps}  # Initialize sums dictionary with step names
+
+        # Sum up the durations for each step across all subarrays
+        for subarray in arrays:
+            for step_name, duration in subarray['timings']:
+                if step_name not in sums:
+                    sums[step_name] = 0
+                sums[step_name] += duration
+
+        for step_name, duration in sums.items():
+            sums[step_name] = duration / 1000000000
+
+        # Convert the sums dictionary to two lists (for plotting)
+        step_names = list(sums.keys())
+        durations = list(sums.values())
+        return step_names, durations
+
+    # Calculate sums for both arrays of subarrays
+    step_names1, sums1 = calculate_sums(arrays1)
+    step_names2, sums2 = calculate_sums(arrays2)
+
+    # Create two subplots, one on top of the other
+    fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(10, 12))
+
+    # First plot (top) for the first array of subarrays
+    ax1.bar(step_names1, sums1, color='b')
+    ax1.set_title('Total Duration per Step - Main Loop')
+    ax1.set_xlabel('Step Name')
+    ax1.set_ylabel('Total Duration (s)')
+    ax1.grid(True, axis='y')
+    ax1.tick_params(axis='x', rotation=45)
+
+    # Second plot (bottom) for the second array of subarrays
+    ax2.bar(step_names2, sums2, color='r')
+    ax2.set_title('Total Duration per Step - Result Processor')
+    ax2.set_xlabel('Step Name')
+    ax2.set_ylabel('Total Duration (s)')
+    ax2.grid(True, axis='y')
+    ax2.tick_params(axis='x', rotation=45)
+
+    # Adjust layout so the plots don't overlap
+    plt.tight_layout()
+
+    # Display the plot
+    plt.show()
+
+# Load arrays from the JSON files
+main = load_json_data('/tmp/windmill/profiling_main.json')
+result_processor = load_json_data('/tmp/windmill/profiling_result_processor.json')
+
+arrays1 = main['timings']
+arrays2 = result_processor['timings']
+
+total_duration1 = main['total_duration']/1000
+total_duration2 = result_processor['total_duration']/1000
+
+print(f"Total duration for main: {total_duration1}s")
+print(f"Total duration for result processor: {total_duration2}s")
+
+iterations_total = sum(main['iter_durations']) / 1000000000
+iterations_total2 = sum(result_processor['iter_durations']) / 1000000000
+
+print(f"Number of iterations: {len(main['iter_durations'])}")
+print(f"Total iterations for main: {iterations_total}s")
+print(f"Total iterations for result processor: {iterations_total2}s")
+
+# Calculate RPS
+rps1 = len(main['iter_durations']) / total_duration1
+rps2 = len(result_processor['iter_durations']) / total_duration2
+
+print(f"RPS for main: {rps1}")
+print(f"RPS for result processor: {rps2}")
+
+# Plot the data
+plot_two_arrays_of_subarrays(arrays1, arrays2)

backend/src/ee.rs

@@ -1,16 +0,0 @@
-use anyhow::anyhow;
-#[cfg(feature = "enterprise")]
-use windmill_common::error::{Error, Result};
-
-pub async fn set_license_key(_license_key: String) -> anyhow::Result<()> {
-    // Implementation is not open source
-    Err(anyhow!("License cannot be set in Windmill CE"))
-}
-
-#[cfg(feature = "enterprise")]
-pub async fn verify_license_key() -> Result<()> {
-    // Implementation is not open source
-    Err(Error::InternalErr(
-        "License always invalid in Windmill CE".to_string(),
-    ))
-}

backend/src/ee.rs

@@ -0,0 +1 @@
+/git/windmill/../windmill-ee-private/src/ee.rs

backend/src/main.rs

@@ -158,6 +158,7 @@ async fn cache_hub_scripts(file_path: Option<String>) -> anyhow::Result<()> {
                 "global",
                 "global",
                 "",
+                &mut None,
             )
             .await?;
             tokio::fs::remove_dir_all(job_dir).await?;
@@ -178,6 +179,7 @@ async fn cache_hub_scripts(file_path: Option<String>) -> anyhow::Result<()> {
                     "cache_init",
                     windmill_worker::get_common_bun_proc_envs(None).await,
                     false,
+                    &mut None,
                 )
                 .await?;
             } else {
@@ -743,9 +745,8 @@ Windmill Community Edition {GIT_VERSION}
             Ok(()) as anyhow::Result<()>
         };
 
-        let instance_name = rd_string(8);
         if mode == Mode::Server || mode == Mode::Standalone {
-            schedule_stats(instance_name, &db, &HTTP_CLIENT).await;
+            schedule_stats(&db, &HTTP_CLIENT).await;
         }
 
         #[cfg(feature = "enterprise")]

backend/src/monitor.rs

@@ -525,7 +525,7 @@ fn read_log_counters(ts_str: String) -> (usize, usize) {
             ok_lines = counter.non_error_count;
             err_lines = counter.error_count;
         } else {
-            println!("no counter found for {ts_str}");
+            // println!("no counter found for {ts_str}");
         }
     } else {
         println!("Error reading log counters 2");
@@ -1071,73 +1071,68 @@ pub async fn monitor_db(
 }
 
 pub async fn expose_queue_metrics(db: &Pool<Postgres>) {
-    let tx = db.begin().await;
-    if let Ok(mut tx) = tx {
-        let last_check = sqlx::query_scalar!(
+    let last_check = sqlx::query_scalar!(
             "SELECT created_at FROM metrics WHERE id LIKE 'queue_count_%' ORDER BY created_at DESC LIMIT 1"
         )
         .fetch_optional(db)
         .await
         .unwrap_or(Some(chrono::Utc::now()));
 
-        let metrics_enabled = METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed);
-        let save_metrics = last_check
-            .map(|last_check| chrono::Utc::now() - last_check > chrono::Duration::seconds(25))
-            .unwrap_or(true);
+    let metrics_enabled = METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed);
+    let save_metrics = last_check
+        .map(|last_check| chrono::Utc::now() - last_check > chrono::Duration::seconds(25))
+        .unwrap_or(true);
 
-        if metrics_enabled || save_metrics {
-            let queue_counts = sqlx::query!(
-                "SELECT tag, count(*) as count FROM queue WHERE
+    if metrics_enabled || save_metrics {
+        let queue_counts = sqlx::query!(
+            "SELECT tag, count(*) as count FROM queue WHERE
                 scheduled_for <= now() - ('3 seconds')::interval AND running = false
                 GROUP BY tag"
-            )
-            .fetch_all(&mut *tx)
-            .await
-            .ok()
-            .unwrap_or_else(|| vec![]);
+        )
+        .fetch_all(db)
+        .await
+        .ok()
+        .unwrap_or_else(|| vec![]);
 
-            for q in queue_counts {
-                let count = q.count.unwrap_or(0);
-                let tag = q.tag;
-                if metrics_enabled {
-                    let metric = (*QUEUE_COUNT).with_label_values(&[&tag]);
-                    metric.set(count as i64);
-                }
+        for q in queue_counts {
+            let count = q.count.unwrap_or(0);
+            let tag = q.tag;
+            if metrics_enabled {
+                let metric = (*QUEUE_COUNT).with_label_values(&[&tag]);
+                metric.set(count as i64);
+            }
 
-                // save queue_count and delay metrics per tag
-                if save_metrics {
+            // save queue_count and delay metrics per tag
+            if save_metrics {
+                sqlx::query!(
+                    "INSERT INTO metrics (id, value) VALUES ($1, $2)",
+                    format!("queue_count_{}", tag),
+                    serde_json::json!(count)
+                )
+                .execute(db)
+                .await
+                .ok();
+                if count > 0 {
                     sqlx::query!(
-                        "INSERT INTO metrics (id, value) VALUES ($1, $2)",
-                        format!("queue_count_{}", tag),
-                        serde_json::json!(count)
-                    )
-                    .execute(&mut *tx)
-                    .await
-                    .ok();
-                    if count > 0 {
-                        sqlx::query!(
                             "INSERT INTO metrics (id, value)
                             VALUES ($1, to_jsonb((SELECT EXTRACT(EPOCH FROM now() - scheduled_for)
                             FROM queue WHERE tag = $2 AND running = false AND scheduled_for <= now() - ('3 seconds')::interval
-                            ORDER BY priority DESC NULLS LAST, scheduled_for, created_at LIMIT 1)))",
+                            ORDER BY priority DESC NULLS LAST, scheduled_for LIMIT 1)))",
                             format!("queue_delay_{}", tag),
                             tag
-                        ).execute(&mut *tx).await.ok();
-                    }
+                        ).execute(db).await.ok();
                 }
             }
         }
-
-        // clean queue metrics older than 14 days
-        sqlx::query!(
-            "DELETE FROM metrics WHERE id LIKE 'queue_%' AND created_at < NOW() - INTERVAL '14 day'"
-        )
-        .execute(&mut *tx)
-        .await
-        .ok();
-
-        tx.commit().await.ok();
     }
+
+    // clean queue metrics older than 14 days
+    sqlx::query!(
+        "DELETE FROM metrics WHERE id LIKE 'queue_%' AND created_at < NOW() - INTERVAL '14 day'"
+    )
+    .execute(db)
+    .await
+    .ok();
 }
 
 pub async fn reload_smtp_config(db: &Pool<Postgres>) {
@@ -1373,6 +1368,8 @@ async fn handle_zombie_jobs<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
             rsmq.clone(),
             worker_name,
             send_result_never_used,
+            #[cfg(feature = "benchmark")]
+            &mut windmill_common::bench::BenchmarkIter::new(),
         )
         .await;
     }

backend/tests/worker.rs

@@ -1126,6 +1126,7 @@ async fn test_deno_flow(db: Pool<Postgres>) {
                     priority: None,
                     delete_after_use: None,
                     continue_on_error: None,
+                    skip_if: None,
                 },
                 FlowModule {
                     id: "b".to_string(),
@@ -1166,6 +1167,7 @@ async fn test_deno_flow(db: Pool<Postgres>) {
                             priority: None,
                             delete_after_use: None,
                             continue_on_error: None,
+                            skip_if: None,
                         }],
                     }
                     .into(),
@@ -1181,6 +1183,7 @@ async fn test_deno_flow(db: Pool<Postgres>) {
                     priority: None,
                     delete_after_use: None,
                     continue_on_error: None,
+                    skip_if: None,
                 },
             ],
             same_worker: false,
@@ -1286,6 +1289,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
                     priority: None,
                     delete_after_use: None,
                     continue_on_error: None,
+                    skip_if: None,
                 },
                 FlowModule {
                     id: "b".to_string(),
@@ -1336,6 +1340,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
                                 priority: None,
                                 delete_after_use: None,
                                 continue_on_error: None,
+                                skip_if: None,
                             },
                             FlowModule {
                                 id: "e".to_string(),
@@ -1372,7 +1377,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
                                 priority: None,
                                 delete_after_use: None,
                                 continue_on_error: None,
-
+                                skip_if: None,
                             },
                         ],
                     }.into(),
@@ -1388,6 +1393,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
                     priority: None,
                     delete_after_use: None,
                     continue_on_error: None,
+                    skip_if: None,
                 },
                 FlowModule {
                     id: "c".to_string(),
@@ -1431,6 +1437,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
                     priority: None,
                     delete_after_use: None,
                     continue_on_error: None,
+                    skip_if: None,
                 },
             ],
             same_worker: true,

backend/windmill-api/Cargo.toml

@@ -97,7 +97,6 @@ jsonwebtoken = { workspace = true }
 matchit.workspace = true
 
 pin-project.workspace = true
-crc.workspace = true
 http.workspace = true
 async-stream.workspace = true
 ulid.workspace = true

backend/windmill-api/openapi.yaml

@@ -1,7 +1,7 @@
 openapi: "3.0.3"
 
 info:
-  version: 1.399.0
+  version: 1.403.1
   title: Windmill API
 
   contact:
@@ -5964,6 +5964,43 @@ paths:
               schema:
                 type: integer
 
+  /jobs/completed/count_by_tag:
+    get:
+      summary: Count jobs by tag
+      operationId: countJobsByTag
+      tags:
+        - job
+      parameters:
+        - name: horizon_secs
+          in: query
+          description: Past Time horizon in seconds (when to start the count = now - horizon) (default is 3600)
+          required: false
+          schema:
+            type: integer
+        - name: workspace_id
+          in: query
+          description: Specific workspace ID to filter results (optional)
+          required: false
+          schema:
+            type: string
+      responses:
+        "200":
+          description: Job counts by tag
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  type: object
+                  properties:
+                    tag:
+                      type: string
+                    count:
+                      type: integer
+                  required:
+                    - tag
+                    - count
+
   /w/{workspace}/jobs_u/get/{id}:
     get:
       summary: get job
@@ -6498,6 +6535,7 @@ paths:
               schema:
                 type: string
 
+
   /w/{workspace}/jobs_u/cancel/{id}/{resume_id}/{signature}:
     get:
       summary: cancel a job for a suspended flow
@@ -9949,6 +9987,8 @@ components:
           type: number
         aggregate_wait_time_ms:
           type: number
+        suspend:
+          type: number
       required:
         - id
         - running
@@ -11158,6 +11198,12 @@ components:
           type: string
         occupancy_rate:
           type: number
+        occupancy_rate_15s:
+          type: number
+        occupancy_rate_5m:
+          type: number
+        occupancy_rate_30m:
+          type: number
         memory:
           type: number
         vcpus:

backend/windmill-api/src/configs.rs

@@ -33,7 +33,7 @@ pub fn global_service() -> Router {
 
 #[derive(Serialize, Deserialize, FromRow)]
 struct Config {
-    name: String,
+    name: Option<String>,
     config: serde_json::Value,
 }
 
@@ -41,9 +41,18 @@ async fn list_worker_groups(
     authed: ApiAuthed,
     Extension(db): Extension<DB>,
 ) -> error::JsonResult<Vec<Config>> {
-    let configs_raw = sqlx::query_as!(Config, "SELECT * FROM config WHERE name LIKE 'worker__%'")
-        .fetch_all(&db)
-        .await?;
+    let mut configs_raw =
+        sqlx::query_as!(Config, "SELECT * FROM config WHERE name LIKE 'worker__%'")
+            .fetch_all(&db)
+            .await?;
+    // Remove the 'worker__' prefix from all config names
+    for config in configs_raw.iter_mut() {
+        if let Some(name) = &config.name {
+            if name.starts_with("worker__") {
+                config.name = Some(name.strip_prefix("worker__").unwrap().to_string());
+            }
+        }
+    }
     let configs = if !authed.is_admin {
         let mut obfuscated_configs: Vec<Config> = vec![];
         for config in configs_raw {

backend/windmill-api/src/db.rs

@@ -15,6 +15,7 @@ use sqlx::{
     PgConnection, Pool, Postgres,
 };
 use windmill_audit::audit_ee::{AuditAuthor, AuditAuthorable};
+use windmill_common::utils::generate_lock_id;
 use windmill_common::{
     db::{Authable, Authed},
     error::Error,
@@ -29,13 +30,6 @@ async fn current_database(conn: &mut PgConnection) -> Result<String, MigrateErro
         .await?)
 }
 
-// inspired from rails: https://github.com/rails/rails/blob/6e49cc77ab3d16c06e12f93158eaf3e507d4120e/activerecord/lib/active_record/migration.rb#L1308
-fn generate_lock_id(database_name: &str) -> i64 {
-    const CRC_IEEE: crc::Crc<u32> = crc::Crc::<u32>::new(&crc::CRC_32_ISO_HDLC);
-    // 0x3d32ad9e chosen by fair dice roll
-    0x3d32ad9e * (CRC_IEEE.checksum(database_name.as_bytes()) as i64)
-}
-
 struct CustomMigrator {
     inner: PoolConnection<Postgres>,
 }
@@ -136,9 +130,30 @@ impl Migrate for CustomMigrator {
                 migration.version,
                 migration.description
             );
-            let r = self.inner.apply(migration).await;
-            tracing::info!("Finished applying migration {}", migration.version);
-            r
+            if migration.version == 20221207103910 {
+                tracing::info!("Skipping migration 20221207103910 to avoid using md5");
+                self.inner
+                    .execute(include_str!(
+                        "../../custom_migrations/create_workspace_without_md5.sql"
+                    ))
+                    .await?;
+                let _ = sqlx::query(
+                    r#"
+                INSERT INTO _sqlx_migrations ( version, description, success, checksum, execution_time )
+                VALUES ( $1, $2, TRUE, $3, -1 ) ON CONFLICT DO NOTHING
+                            "#,
+                )
+                .bind(migration.version)
+                .bind(&*migration.description)
+                .bind(&*migration.checksum)
+                .execute(&mut *self.inner)
+                .await?;
+                return Ok(std::time::Duration::from_secs(0));
+            } else {
+                let r = self.inner.apply(migration).await;
+                tracing::info!("Finished applying migration {}", migration.version);
+                return r;
+            }
         }
         .boxed()
     }

backend/windmill-api/src/ee.rs

@@ -1,32 +0,0 @@
-use anyhow::anyhow;
-#[cfg(feature = "enterprise")]
-use std::sync::Arc;
-#[cfg(feature = "enterprise")]
-use tokio::sync::RwLock;
-
-pub async fn validate_license_key(_license_key: String) -> anyhow::Result<String> {
-    // Implementation is not open source
-    Err(anyhow!("License can't be validated in Windmill CE"))
-}
-
-#[cfg(feature = "enterprise")]
-pub async fn jwt_ext_auth(
-    _w_id: Option<&String>,
-    _token: &str,
-    _external_jwks: Option<Arc<RwLock<ExternalJwks>>>,
-) -> anyhow::Result<(crate::db::ApiAuthed, usize)> {
-    // Implementation is not open source
-
-    Err(anyhow!("External JWT auth is not open source"))
-}
-
-#[cfg(feature = "enterprise")]
-pub struct ExternalJwks;
-
-#[cfg(feature = "enterprise")]
-impl ExternalJwks {
-    pub async fn load() -> Option<Arc<RwLock<Self>>> {
-        // Implementation is not open source
-        None
-    }
-}

backend/windmill-api/src/ee.rs

@@ -0,0 +1 @@
+/git/windmill/../windmill-ee-private/windmill-api/src/ee.rs

backend/windmill-api/src/flows.rs

@@ -242,6 +242,7 @@ pub async fn get_hub_flow_by_id(
 
 #[derive(Deserialize)]
 pub struct ToggleWorkspaceErrorHandler {
+    #[cfg(feature = "enterprise")]
     pub muted: Option<bool>,
 }
 
@@ -1175,6 +1176,7 @@ mod tests {
                     priority: None,
                     delete_after_use: None,
                     continue_on_error: None,
+                    skip_if: None,
                 },
                 FlowModule {
                     id: "b".to_string(),
@@ -1204,6 +1206,7 @@ mod tests {
                     priority: None,
                     delete_after_use: None,
                     continue_on_error: None,
+                    skip_if: None,
                 },
                 FlowModule {
                     id: "c".to_string(),
@@ -1231,6 +1234,7 @@ mod tests {
                     priority: None,
                     delete_after_use: None,
                     continue_on_error: None,
+                    skip_if: None,
                 },
             ],
             failure_module: Some(Box::new(FlowModule {
@@ -1257,6 +1261,7 @@ mod tests {
                 priority: None,
                 delete_after_use: None,
                 continue_on_error: None,
+                skip_if: None,
             })),
             preprocessor_module: None,
             same_worker: false,

backend/windmill-api/src/indexer_ee.rs

@@ -1,5 +0,0 @@
-use axum::Router;
-
-pub fn workspaced_service() -> Router {
-    Router::new()
-}

backend/windmill-api/src/indexer_ee.rs

@@ -0,0 +1 @@
+/git/windmill/../windmill-ee-private/windmill-api/src/indexer_ee.rs

backend/windmill-api/src/job_helpers_ee.rs

@@ -1,5 +0,0 @@
-use axum::Router;
-
-pub fn workspaced_service() -> Router {
-    Router::new()
-}

backend/windmill-api/src/job_helpers_ee.rs

@@ -0,0 +1 @@
+/git/windmill/../windmill-ee-private/windmill-api/src/job_helpers_ee.rs

backend/windmill-api/src/jobs.rs

@@ -11,6 +11,7 @@ use axum::http::HeaderValue;
 use quick_cache::sync::Cache;
 use serde_json::value::RawValue;
 use sqlx::Pool;
+use windmill_common::error::JsonResult;
 use std::collections::HashMap;
 #[cfg(feature = "prometheus")]
 use std::sync::atomic::Ordering;
@@ -69,7 +70,7 @@ use windmill_common::{
     oauth2::HmacSha256,
     scripts::{ScriptHash, ScriptLang},
     users::username_to_permissioned_as,
-    utils::{not_found_if_none, now_from_db, paginate, require_admin, Pagination, StripPath},
+    utils::{not_found_if_none, now_from_db, paginate, paginate_without_limits, require_admin, Pagination, StripPath},
 };
 
 #[cfg(all(feature = "enterprise", feature = "parquet"))]
@@ -247,7 +248,7 @@ pub fn workspaced_service() -> Router {
         .route("/run/flow_dependencies", post(run_flow_dependencies_job))
 }
 
-pub fn global_service() -> Router {
+pub fn workspace_unauthed_service() -> Router {
     Router::new()
         .route(
             "/resume/:job_id/:resume_id/:secret",
@@ -291,7 +292,12 @@ pub fn global_service() -> Router {
 }
 
 pub fn global_root_service() -> Router {
-    Router::new().route("/db_clock", get(get_db_clock))
+    Router::new()
+    .route("/db_clock", get(get_db_clock))
+    .route(
+        "/completed/count_by_tag",
+        get(count_by_tag),
+    )
 }
 
 #[derive(Deserialize)]
@@ -1248,13 +1254,16 @@ pub fn list_queue_jobs_query(
     w_id: &str,
     lq: &ListQueueQuery,
     fields: &[&str],
+    pagination: Pagination,
     join_outstanding_wait_times: bool,
     tags: Option<Vec<&str>>,
 ) -> SqlBuilder {
+    let (limit, offset) = paginate_without_limits(pagination);
     let mut sqlb = SqlBuilder::select_from("queue")
         .fields(fields)
         .order_by("created_at", lq.order_desc.unwrap_or(true))
-        .limit(1000)
+        .limit(limit)
+        .offset(offset)
         .clone();
 
     if let Some(tags) = tags {
@@ -1267,6 +1276,7 @@ pub fn list_queue_jobs_query(
 #[derive(Serialize, FromRow)]
 struct ListableQueuedJob {
     pub id: Uuid,
+    pub running: bool,
     pub created_by: String,
     pub created_at: chrono::DateTime<chrono::Utc>,
     pub started_at: Option<chrono::DateTime<chrono::Utc>>,
@@ -1289,6 +1299,7 @@ async fn list_queue_jobs(
     authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
     Path(w_id): Path<String>,
+    Query(pagination): Query<Pagination>,
     Query(lq): Query<ListQueueQuery>,
 ) -> error::JsonResult<Vec<ListableQueuedJob>> {
     let sql = list_queue_jobs_query(
@@ -1296,6 +1307,7 @@ async fn list_queue_jobs(
         &lq,
         &[
             "id",
+            "running",
             "created_by",
             "created_at",
             "started_at",
@@ -1315,6 +1327,7 @@ async fn list_queue_jobs(
             "priority",
             "workspace_id",
         ],
+        pagination,
         false,
         get_scope_tags(&authed),
     )
@@ -1570,6 +1583,7 @@ async fn list_jobs(
 ) -> error::JsonResult<Vec<Job>> {
     check_scopes(&authed, || format!("jobs:listjobs"))?;
 
+    let limit = pagination.per_page.unwrap_or(1000);
     let (per_page, offset) = paginate(pagination);
     let lqc = lq.clone();
 
@@ -1601,6 +1615,7 @@ async fn list_jobs(
             &w_id,
             &ListQueueQuery { order_desc: Some(true), ..lq.into() },
             UnifiedJob::queued_job_fields(),
+            Pagination { per_page: Some(limit), page: None },
             true,
             get_scope_tags(&authed),
         );
@@ -5100,6 +5115,46 @@ async fn get_completed_job_result(
     Ok(Json(result).into_response())
 }
 
+
+
+#[derive(Deserialize)]
+struct CountByTagQuery {
+    horizon_secs: Option<i64>,
+    workspace_id: Option<String>,
+}
+
+#[derive(Serialize)]
+struct TagCount {
+    tag: String,
+    count: i64,
+}
+
+async fn count_by_tag(
+    ApiAuthed { email, ..}: ApiAuthed,
+    Extension(db): Extension<DB>,
+    Query(query): Query<CountByTagQuery>,
+) -> JsonResult<Vec<TagCount>> {
+    require_super_admin(&db, &email).await?;
+    let horizon = query.horizon_secs.unwrap_or(3600); // Default to 1 hour if not specified
+
+    let counts = sqlx::query_as!(
+        TagCount,
+        r#"
+        SELECT tag as "tag!", COUNT(*) as "count!"
+        FROM completed_job
+        WHERE started_at > NOW() - make_interval(secs => $1) AND ($2::text IS NULL OR workspace_id = $2)
+        GROUP BY tag
+        ORDER BY "count!" DESC
+        "#,
+        horizon as f64,
+        query.workspace_id
+    )
+    .fetch_all(&db)
+    .await?;
+
+    Ok(Json(counts))
+}
+
 #[derive(Serialize)]
 struct CompletedJobResult {
     started: Option<bool>,

backend/windmill-api/src/lib.rs

@@ -39,9 +39,8 @@ use tower_http::{
     trace::TraceLayer,
 };
 use windmill_common::db::UserDB;
-use windmill_common::utils::rd_string;
 use windmill_common::worker::ALL_TAGS;
-use windmill_common::BASE_URL;
+use windmill_common::{BASE_URL, INSTANCE_NAME};
 
 use crate::scim_ee::has_scim_token;
 use windmill_common::error::AppError;
@@ -330,7 +329,7 @@ pub async fn run_server(
                 )
                 .nest(
                     "/w/:workspace_id/jobs_u",
-                    jobs::global_service().layer(cors.clone()),
+                    jobs::workspace_unauthed_service().layer(cors.clone()),
                 )
                 .nest(
                     "/w/:workspace_id/resources_u",
@@ -373,8 +372,6 @@ pub async fn run_server(
         )
     };
 
-    let instance_name = rd_string(5);
-
     let listener = tokio::net::TcpListener::bind(addr).await.unwrap();
     let port = listener.local_addr().map(|x| x.port()).unwrap_or(8000);
     let ip = listener
@@ -385,7 +382,7 @@ pub async fn run_server(
     let server = axum::serve(listener, app.into_make_service());
 
     tracing::info!(
-        instance = %instance_name,
+        instance = %*INSTANCE_NAME,
         "server started on port={} and addr={}",
         port,
         ip

backend/windmill-api/src/oauth2_ee.rs

@@ -1,188 +0,0 @@
-/*
- * Author: Ruben Fiszel
- * Copyright: Windmill Labs, Inc 2022
- * This file and its contents are licensed under the AGPLv3 License.
- * Please see the included NOTICE for copyright information and
- * LICENSE-AGPL for a copy of the license.
- */
-
-use std::{collections::HashMap, fmt::Debug};
-
-use axum::{routing::get, Json, Router};
-use hmac::Mac;
-use hyper::HeaderMap;
-
-use itertools::Itertools;
-use oauth2::{Client as OClient, *};
-use serde::{Deserialize, Serialize};
-use sqlx::{Postgres, Transaction};
-use windmill_common::more_serde::maybe_number_opt;
-
-use crate::OAUTH_CLIENTS;
-use windmill_common::error;
-use windmill_common::oauth2::*;
-
-use crate::db::DB;
-use std::str;
-
-pub fn global_service() -> Router {
-    Router::new()
-        .route("/list_supabase", get(list_supabase))
-        .route("/list_logins", get(list_logins))
-        .route("/list_connects", get(list_connects))
-}
-
-pub fn workspaced_service() -> Router {
-    Router::new()
-}
-
-#[derive(Serialize)]
-#[serde(tag = "type")]
-pub enum InstanceEvent {
-    UserAdded { email: String },
-    // UserDeleted { email: String },
-    // UserDeletedWorkspace { workspace: String, email: String },
-    UserAddedWorkspace { workspace: String, email: String },
-    UserInvitedWorkspace { workspace: String, email: String },
-    UserJoinedWorkspace { workspace: String, email: String, username: String },
-}
-
-#[derive(Debug, Clone)]
-pub struct ClientWithScopes {
-    _client: OClient,
-    _scopes: Vec<String>,
-    _extra_params: Option<HashMap<String, String>>,
-    _extra_params_callback: Option<HashMap<String, String>>,
-    _allowed_domains: Option<Vec<String>>,
-    _userinfo_url: Option<String>,
-}
-
-pub type BasicClientsMap = HashMap<String, ClientWithScopes>;
-
-#[derive(Clone, Debug, Serialize, Deserialize)]
-pub struct OAuthConfig {
-    auth_url: String,
-    token_url: String,
-    userinfo_url: Option<String>,
-    scopes: Option<Vec<String>>,
-    extra_params: Option<HashMap<String, String>>,
-    extra_params_callback: Option<HashMap<String, String>>,
-    req_body_auth: Option<bool>,
-}
-
-#[derive(Clone, Debug, Serialize, Deserialize)]
-pub struct OAuthClient {
-    id: String,
-    secret: String,
-    allowed_domains: Option<Vec<String>>,
-    connect_config: Option<OAuthConfig>,
-    login_config: Option<OAuthConfig>,
-}
-
-#[derive(Debug)]
-pub struct AllClients {
-    pub logins: BasicClientsMap,
-    pub connects: BasicClientsMap,
-    pub slack: Option<OClient>,
-}
-
-pub fn build_oauth_clients(
-    _base_url: &str,
-    _oauths_from_config: Option<HashMap<String, OAuthClient>>,
-) -> anyhow::Result<AllClients> {
-    // Implementation is not open source
-    return Ok(AllClients {
-        logins: HashMap::default(),
-        connects: HashMap::default(),
-        slack: None,
-    });
-}
-
-#[derive(Clone, Debug, Deserialize, Serialize)]
-pub struct TokenResponse {
-    access_token: AccessToken,
-    #[serde(deserialize_with = "maybe_number_opt")]
-    #[serde(default)]
-    expires_in: Option<u64>,
-    refresh_token: Option<RefreshToken>,
-    #[serde(deserialize_with = "helpers::deserialize_space_delimited_vec")]
-    #[serde(serialize_with = "helpers::serialize_space_delimited_vec")]
-    #[serde(default)]
-    scope: Option<Vec<Scope>>,
-}
-
-#[derive(Serialize)]
-struct Logins {
-    oauth: Vec<String>,
-    saml: Option<String>,
-}
-async fn list_logins() -> error::JsonResult<Logins> {
-    // Implementation is not open source
-    return Ok(Json(Logins { oauth: vec![], saml: None }));
-}
-
-async fn list_connects() -> error::JsonResult<Vec<String>> {
-    Ok(Json(
-        (&OAUTH_CLIENTS.read().await.connects)
-            .keys()
-            .map(|x| x.to_owned())
-            .collect_vec(),
-    ))
-}
-
-pub async fn _refresh_token<'c>(
-    _tx: Transaction<'c, Postgres>,
-    _path: &str,
-    _w_id: &str,
-    _id: i32,
-    _db: &DB,
-) -> error::Result<String> {
-    // Implementation is not open source
-    Err(error::Error::BadRequest(
-        "Not implemented in Windmill's Open Source repository".to_string(),
-    ))
-}
-
-async fn list_supabase(_headers: HeaderMap) -> error::Result<String> {
-    // Implementation is not open source
-    Err(error::Error::BadRequest(
-        "Not implemented in Windmill's Open Source repository".to_string(),
-    ))
-}
-
-pub async fn check_nb_of_user(db: &DB) -> error::Result<()> {
-    let nb_users_sso =
-        sqlx::query_scalar!("SELECT COUNT(*) FROM password WHERE login_type != 'password'",)
-            .fetch_one(db)
-            .await?;
-    if nb_users_sso.unwrap_or(0) >= 10 {
-        return Err(error::Error::BadRequest(
-            "You have reached the maximum number of oauth users accounts (10) without an enterprise license"
-                .to_string(),
-        ));
-    }
-
-    let nb_users = sqlx::query_scalar!("SELECT COUNT(*) FROM password",)
-        .fetch_one(db)
-        .await?;
-    if nb_users.unwrap_or(0) >= 50 {
-        return Err(error::Error::BadRequest(
-            "You have reached the maximum number of accounts (50) without an enterprise license"
-                .to_string(),
-        ));
-    }
-    return Ok(());
-}
-
-#[derive(Clone, Debug)]
-pub struct SlackVerifier {
-    _mac: HmacSha256,
-}
-
-impl SlackVerifier {
-    pub fn new<S: AsRef<[u8]>>(secret: S) -> anyhow::Result<SlackVerifier> {
-        HmacSha256::new_from_slice(secret.as_ref())
-            .map(|mac| SlackVerifier { _mac: mac })
-            .map_err(|_| anyhow::anyhow!("invalid secret"))
-    }
-}

backend/windmill-api/src/oauth2_ee.rs

@@ -0,0 +1 @@
+/git/windmill/../windmill-ee-private/windmill-api/src/oauth2_ee.rs

backend/windmill-api/src/oidc_ee.rs

@@ -1,17 +0,0 @@
-/*
- * Author: Ruben Fiszel
- * Copyright: Windmill Labs, Inc 2023
- * This file and its contents are licensed under the AGPLv3 License.
- * Please see the included NOTICE for copyright information and
- * LICENSE-AGPL for a copy of the license.
- */
-
-use axum::Router;
-
-pub fn global_service() -> Router {
-    Router::new()
-}
-
-pub fn workspaced_service() -> Router {
-    Router::new()
-}

backend/windmill-api/src/oidc_ee.rs

@@ -0,0 +1 @@
+/git/windmill/../windmill-ee-private/windmill-api/src/oidc_ee.rs

backend/windmill-api/src/saml_ee.rs

@@ -1,25 +0,0 @@
-/*
- * Author: Ruben Fiszel
- * Copyright: Windmill Labs, Inc 2023
- * This file and its contents are licensed under the AGPLv3 License.
- * Please see the included NOTICE for copyright information and
- * LICENSE-AGPL for a copy of the license.
- */
-#![allow(non_snake_case)]
-
-use axum::{routing::post, Router};
-
-pub struct ServiceProviderExt();
-
-pub async fn build_sp_extension() -> anyhow::Result<ServiceProviderExt> {
-    return Ok(ServiceProviderExt());
-}
-
-pub fn global_service() -> Router {
-    Router::new().route("/acs", post(acs))
-}
-
-pub async fn acs() -> String {
-    // Implementation is not open source as it is a Windmill Enterprise Edition feature
-    "SAML available only in enterprise version".to_string()
-}

backend/windmill-api/src/saml_ee.rs

@@ -0,0 +1 @@
+/git/windmill/../windmill-ee-private/windmill-api/src/saml_ee.rs

backend/windmill-api/src/scim_ee.rs

@@ -1,23 +0,0 @@
-/*
- * Author: Ruben Fiszel
- * Copyright: Windmill Labs, Inc 2023
- * This file and its contents are licensed under the AGPLv3 License.
- * Please see the included NOTICE for copyright information and
- * LICENSE-AGPL for a copy of the license.
- */
-
-use axum::{middleware::Next, response::Response, routing::get, Router};
-use hyper::Request;
-
-pub fn global_service() -> Router {
-    Router::new().route("/ee", get(ee))
-}
-
-pub async fn ee() -> String {
-    return "Enterprise Edition".to_string();
-}
-
-pub async fn has_scim_token<B>(_request: Request<B>, _next: Next) -> Response {
-    //Not implemented in open-source version
-    todo!()
-}

backend/windmill-api/src/scim_ee.rs

@@ -0,0 +1 @@
+/git/windmill/../windmill-ee-private/windmill-api/src/scim_ee.rs

backend/windmill-api/src/scripts.rs

@@ -967,6 +967,7 @@ async fn list_paths(
 
 #[derive(Deserialize)]
 pub struct ToggleWorkspaceErrorHandler {
+    #[cfg(feature = "enterprise")]
     pub muted: Option<bool>,
 }
 

backend/windmill-api/src/settings.rs

@@ -24,9 +24,9 @@ use axum::{
 #[cfg(feature = "enterprise")]
 use axum::extract::Query;
 
+use serde::Deserialize;
 #[cfg(feature = "enterprise")]
 use windmill_common::ee::{send_critical_alert, CriticalAlertKind, CriticalErrorChannel};
-use serde::Deserialize;
 use windmill_common::{
     error::{self, JsonResult, Result},
     global_settings::{
@@ -298,7 +298,13 @@ async fn list_global_settings() -> JsonResult<String> {
 
 pub async fn send_stats(Extension(db): Extension<DB>, authed: ApiAuthed) -> Result<String> {
     require_super_admin(&db, &authed.email).await?;
-    windmill_common::stats_ee::send_stats(&"manual".to_string(), &HTTP_CLIENT, &db).await?;
+    windmill_common::stats_ee::send_stats(
+        &HTTP_CLIENT,
+        &db,
+        true,
+        windmill_common::stats_ee::SendStatsReason::Manual,
+    )
+    .await?;
 
     Ok("Sent stats".to_string())
 }
@@ -357,8 +363,13 @@ pub async fn renew_license_key(
     authed: ApiAuthed,
 ) -> Result<String> {
     require_super_admin(&db, &authed.email).await?;
-    windmill_common::stats_ee::send_stats(&"manual".to_string(), &HTTP_CLIENT, &db).await?;
-    let result = windmill_common::ee::renew_license_key(&HTTP_CLIENT, &db, license_key, true).await;
+    let result = windmill_common::ee::renew_license_key(
+        &HTTP_CLIENT,
+        &db,
+        license_key,
+        windmill_common::ee::RenewReason::Manual,
+    )
+    .await;
 
     if result != "success" {
         return Err(error::Error::BadRequest(format!(

backend/windmill-api/src/smtp_server_ee.rs

@@ -1,17 +0,0 @@
-use crate::{db::DB, users::AuthCache};
-use std::{net::SocketAddr, sync::Arc};
-use windmill_common::db::UserDB;
-
-pub struct SmtpServer {
-    pub auth_cache: Arc<AuthCache>,
-    pub db: DB,
-    pub user_db: UserDB,
-    pub rsmq: Option<rsmq_async::MultiplexedRsmq>,
-    pub base_internal_url: String,
-}
-
-impl SmtpServer {
-    pub async fn start_listener_thread(self: Arc<Self>, _addr: SocketAddr) -> anyhow::Result<()> {
-        Err(anyhow::anyhow!("Implementation not open source"))
-    }
-}

backend/windmill-api/src/smtp_server_ee.rs

@@ -0,0 +1 @@
+/git/windmill/../windmill-ee-private/windmill-api/src/smtp_server_ee.rs

backend/windmill-api/src/stripe_ee.rs

@@ -1,7 +0,0 @@
-#[cfg(feature = "stripe")]
-use axum::Router;
-
-#[cfg(feature = "stripe")]
-pub fn add_stripe_routes(router: Router) -> Router {
-    return router;
-}

backend/windmill-api/src/stripe_ee.rs

@@ -0,0 +1 @@
+/git/windmill/../windmill-ee-private/windmill-api/src/stripe_ee.rs

backend/windmill-api/src/workers.rs

@@ -54,6 +54,12 @@ struct WorkerPing {
     #[serde(skip_serializing_if = "Option::is_none")]
     occupancy_rate: Option<f32>,
     #[serde(skip_serializing_if = "Option::is_none")]
+    occupancy_rate_15s: Option<f32>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    occupancy_rate_5m: Option<f32>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    occupancy_rate_30m: Option<f32>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     memory: Option<i64>,
     #[serde(skip_serializing_if = "Option::is_none")]
     vcpus: Option<i64>,
@@ -88,7 +94,9 @@ async fn list_worker_pings(
 
     let rows = sqlx::query_as!(
         WorkerPing,
-        "SELECT worker, worker_instance,  EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed, CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as last_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as last_job_workspace_id, custom_tags, worker_group, wm_version, occupancy_rate, memory, vcpus, memory_usage, wm_memory_usage
+        "SELECT worker, worker_instance,  EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed,
+        CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as last_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as last_job_workspace_id, 
+        custom_tags, worker_group, wm_version, occupancy_rate, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, memory, vcpus, memory_usage, wm_memory_usage
         FROM worker_ping
         WHERE ($1::integer IS NULL AND ping_at > now() - interval '5 minute') OR (ping_at > now() - ($1 || ' seconds')::interval)
         ORDER BY ping_at desc LIMIT $2 OFFSET $3",

backend/windmill-api/src/workspaces.rs

@@ -42,7 +42,10 @@ use windmill_common::schedule::Schedule;
 use windmill_common::users::username_to_permissioned_as;
 use windmill_common::variables::build_crypt;
 use windmill_common::worker::{to_raw_value, CLOUD_HOSTED};
-use windmill_common::workspaces::{WorkspaceDeploymentUISettings, WorkspaceGitSyncSettings};
+#[cfg(feature = "enterprise")]
+use windmill_common::workspaces::WorkspaceDeploymentUISettings;
+#[cfg(feature = "enterprise")]
+use windmill_common::workspaces::WorkspaceGitSyncSettings;
 use windmill_common::{
     error::{to_anyhow, Error, JsonResult, Result},
     flows::Flow,
@@ -991,6 +994,7 @@ async fn edit_large_file_storage_config(
 
 #[derive(Deserialize)]
 pub struct EditGitSyncConfig {
+    #[cfg(feature = "enterprise")]
     pub git_sync_settings: Option<WorkspaceGitSyncSettings>,
 }
 
@@ -1056,6 +1060,7 @@ async fn edit_git_sync_config(
 
 #[derive(Deserialize)]
 struct EditDeployUIConfig {
+    #[cfg(feature = "enterprise")]
     deploy_ui_settings: Option<WorkspaceDeploymentUISettings>,
 }
 
@@ -1064,7 +1069,6 @@ async fn edit_deploy_ui_config(
     _authed: ApiAuthed,
     Extension(_db): Extension<DB>,
     Path(_w_id): Path<String>,
-    Json(_new_config): Json<EditDeployUIConfig>,
 ) -> Result<String> {
     return Err(Error::BadRequest(
         "Deployment UI is only available on Windmill Enterprise Edition".to_string(),
@@ -1122,6 +1126,7 @@ async fn edit_deploy_ui_config(
 
 #[derive(Deserialize)]
 pub struct EditDefaultApp {
+    #[cfg(feature = "enterprise")]
     pub default_app_path: Option<String>,
 }
 

backend/windmill-audit/src/audit_ee.rs

@@ -1,75 +0,0 @@
-/*
- * Author: Ruben Fiszel
- * Copyright: Windmill Labs, Inc 2022
- * This file and its contents are licensed under the AGPLv3 License.
- * Please see the included NOTICE for copyright information and
- * LICENSE-AGPL for a copy of the license.
- */
-use std::collections::HashMap;
-
-use windmill_common::{
-    error::{Error, Result},
-    utils::Pagination,
-};
-
-use crate::{ActionKind, AuditLog, ListAuditLogQuery};
-use sqlx::{Postgres, Transaction};
-
-#[derive(Clone)]
-pub struct AuditAuthor {
-    pub username: String,
-    pub email: String,
-    pub username_override: Option<String>,
-}
-
-impl AuditAuthorable for AuditAuthor {
-    fn email(&self) -> &str {
-        &self.email
-    }
-
-    fn username(&self) -> &str {
-        &self.username
-    }
-
-    fn username_override(&self) -> Option<&str> {
-        self.username_override.as_deref()
-    }
-}
-
-pub trait AuditAuthorable {
-    fn username(&self) -> &str;
-    fn email(&self) -> &str;
-    fn username_override(&self) -> Option<&str>;
-}
-
-#[tracing::instrument(level = "trace", skip_all)]
-pub async fn audit_log<'c, E: sqlx::Executor<'c, Database = Postgres>>(
-    _db: E,
-    _author: &impl AuditAuthorable,
-    mut _operation: &str,
-    _action_kind: ActionKind,
-    _w_id: &str,
-    mut _resource: Option<&str>,
-    _parameters: Option<HashMap<&str, &str>>,
-) -> Result<()> {
-    // Implementation is not open source as Audit logs is a Windmill Enterprise Edition feature
-    Ok(())
-}
-
-pub async fn list_audit(
-    _tx: Transaction<'_, Postgres>,
-    _w_id: String,
-    _pagination: Pagination,
-    _lq: ListAuditLogQuery,
-) -> Result<Vec<AuditLog>> {
-    // Implementation is not open source as Audit logs is a Windmill Enterprise Edition feature
-    return Ok(vec![]);
-}
-
-pub async fn get_audit(tx: Transaction<'_, Postgres>, _id: i32, _w_id: &str) -> Result<AuditLog> {
-    // Implementation is not open source as Audit logs is a Windmill Enterprise Edition feature
-    tx.commit().await?;
-    Err(Error::NotFound(
-        "Audit log not not available in Windmill Community edition".to_string(),
-    ))
-}

backend/windmill-audit/src/audit_ee.rs

@@ -0,0 +1 @@
+/git/windmill/../windmill-ee-private/windmill-audit/src/audit_ee.rs

backend/windmill-common/Cargo.toml

@@ -11,6 +11,7 @@ jemalloc = ["dep:tikv-jemalloc-ctl"]
 prometheus = ["dep:prometheus"]
 flamegraph = ["dep:tracing-flame"]
 loki = ["dep:tracing-loki"]
+benchmark = []
 parquet = ["dep:object_store", "dep:aws-config", "dep:aws-sdk-sts"]
 
 [lib]
@@ -55,6 +56,7 @@ mail-send.workspace = true
 futures-core.workspace = true
 async-stream.workspace = true
 const_format.workspace = true
+crc.workspace = true
 
 [target.'cfg(not(target_env = "msvc"))'.dependencies]
 tikv-jemalloc-ctl = { optional = true, workspace = true }

backend/windmill-common/src/bench.rs

@@ -0,0 +1,213 @@
+use crate::{
+    worker::{write_file, TMP_DIR},
+    DB,
+};
+use serde::Serialize;
+use tokio::time::Instant;
+
+#[derive(Serialize)]
+pub struct BenchmarkInfo {
+    #[serde(skip)]
+    pub start: Instant,
+    #[serde(skip)]
+    pub iters: u64,
+    timings: Vec<BenchmarkIter>,
+    pub iter_durations: Vec<u64>,
+    pub total_duration: Option<u64>,
+}
+
+impl BenchmarkInfo {
+    pub fn new() -> Self {
+        BenchmarkInfo {
+            iters: 0,
+            timings: vec![],
+            start: Instant::now(),
+            iter_durations: vec![],
+            total_duration: None,
+        }
+    }
+
+    pub fn add_iter(&mut self, bench: BenchmarkIter, inc_iters: bool) {
+        if inc_iters {
+            self.iters += 1;
+        }
+        let elapsed_total = bench.start.elapsed().as_nanos() as u64;
+        self.timings.push(bench);
+        self.iter_durations.push(elapsed_total);
+    }
+
+    pub fn write_to_file(&mut self, path: &str) -> anyhow::Result<()> {
+        let total_duration = self.start.elapsed().as_millis() as u64;
+        self.total_duration = Some(total_duration as u64);
+
+        println!(
+            "Writing benchmark {path}, duration of benchmark: {total_duration}s and RPS: {}",
+            self.iters as f64 / total_duration as f64
+        );
+        write_file(TMP_DIR, path, &serde_json::to_string(&self).unwrap()).expect("write profiling");
+        Ok(())
+    }
+}
+
+#[derive(Serialize)]
+pub struct BenchmarkIter {
+    #[serde(skip)]
+    pub start: Instant,
+    #[serde(skip)]
+    last_instant: Instant,
+    last_step: String,
+    timings: Vec<(String, u32)>,
+}
+
+impl BenchmarkIter {
+    pub fn new() -> Self {
+        BenchmarkIter {
+            last_instant: Instant::now(),
+            timings: vec![],
+            start: Instant::now(),
+            last_step: String::new(),
+        }
+    }
+
+    pub fn add_timing(&mut self, name: &str) {
+        let elapsed = self.last_instant.elapsed().as_nanos() as u32;
+        self.timings
+            .push((format!("{}->{}", self.last_step, name), elapsed));
+        self.last_instant = Instant::now();
+        self.last_step = name.to_string();
+    }
+}
+
+pub async fn benchmark_init(benchmark_jobs: i32, db: &DB) {
+    use crate::{jobs::JobKind, scripts::ScriptLang};
+
+    let benchmark_kind = std::env::var("BENCHMARK_KIND").unwrap_or("noop".to_string());
+
+    if benchmark_jobs > 0 {
+        match benchmark_kind.as_str() {
+            "dedicated" => {
+                // you need to create the script first, check https://github.com/windmill-labs/windmill/blob/b76a92cfe454c686f005c65f534e29e039f3c706/benchmarks/lib.ts#L47
+                let hash = sqlx::query_scalar!(
+                    "SELECT hash FROM script WHERE path = $1 AND workspace_id = $2",
+                    "f/benchmarks/dedicated",
+                    "admins"
+                )
+                .fetch_one(db)
+                .await
+                .unwrap_or_else(|_e| panic!("failed to insert dedicated jobs"));
+                sqlx::query!("INSERT INTO queue (id, script_hash, script_path, job_kind, language, tag, created_by, permissioned_as, email, scheduled_for, workspace_id) (SELECT gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8, $9, $10 FROM generate_series(1, $11))",
+                    hash,
+                    "f/benchmarks/dedicated",
+                    JobKind::Script as JobKind,
+                    ScriptLang::Bun as ScriptLang,
+                    "admins:f/benchmarks/dedicated",
+                    "admin",
+                    "u/admin",
+                    "admin@windmill.dev",
+                    chrono::Utc::now(),
+                    "admins",
+                    benchmark_jobs
+                )
+                .execute(db)
+                .await.unwrap_or_else(|_e| panic!("failed to insert dedicated jobs"));
+            }
+            "parallelflow" => {
+                //create dedicated script
+                sqlx::query!("INSERT INTO script (summary, description, dedicated_worker, content, workspace_id, path, hash, language, tag, created_by, lock) VALUES ('', '', true, $1, $2, $3, $4, $5, $6, $7, '') ON CONFLICT (workspace_id, hash) DO NOTHING",
+                "export async function main() {
+                    console.log('hello world');
+                }",
+                "admins",
+                "u/admin/parallelflow",
+                1234567890,
+                ScriptLang::Deno as ScriptLang,
+                "flow",
+                "admin",
+                )
+                .execute(db)
+                .await.unwrap_or_else(|_e| panic!("failed to insert parallelflow jobs {_e:#}"));
+                sqlx::query!("INSERT INTO queue (id, script_hash, script_path, job_kind, language, tag, created_by, permissioned_as, email, scheduled_for, workspace_id, raw_flow, flow_status) (SELECT gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12 FROM generate_series(1, 1))",
+                    None::<i64>,
+                    None::<String>,
+                    JobKind::FlowPreview as JobKind,
+                    ScriptLang::Deno as ScriptLang,
+                    "flow",
+                    "admin",
+                    "u/admin",
+                    "admin@windmill.dev",
+                    chrono::Utc::now(),
+                    "admins",
+                    serde_json::from_str::<serde_json::Value>(r#"
+{
+  "modules": [
+    {
+      "id": "a",
+      "value": {
+        "type": "forloopflow",
+        "modules": [
+          {
+            "id": "b",
+            "value": {
+              "path": "u/admin/parallelflow",
+              "type": "script",
+              "tag_override": "",
+              "input_transforms": {}
+            },
+            "summary": "calctest"
+          }
+        ],
+        "iterator": {
+          "expr": "[...new Array(300)]",
+          "type": "javascript"
+        },
+        "parallel": true,
+        "parallelism": 10,
+        "skip_failures": true
+      }
+    }
+  ],
+  "preprocessor_module": null
+}
+                    "#).unwrap(),
+                    serde_json::from_str::<serde_json::Value>(r#"
+{
+		"step": 0,
+		"modules": [
+			{
+				"id": "a",
+				"type": "WaitingForPriorSteps"
+			}
+		],
+		"cleanup_module": {},
+		"failure_module": {
+			"id": "failure",
+			"type": "WaitingForPriorSteps"
+		},
+		"preprocessor_module": null
+	}
+
+                "#).unwrap()
+                )
+                .execute(db)
+                .await.unwrap_or_else(|_e| panic!("failed to insert parallelflow jobs"));
+            }
+            _ => {
+                sqlx::query!("INSERT INTO queue (id, script_hash, script_path, job_kind, language, tag, created_by, permissioned_as, email, scheduled_for, workspace_id) (SELECT gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8, $9, $10 FROM generate_series(1, $11))",
+                    None::<i64>,
+                    None::<String>,
+                    JobKind::Noop as JobKind,
+                    ScriptLang::Deno as ScriptLang,
+                    "deno",
+                    "admin",
+                    "u/admin",
+                    "admin@windmill.dev",
+                    chrono::Utc::now(),
+                    "admins",
+                    benchmark_jobs
+                )
+                .execute(db)
+                .await.unwrap_or_else(|_e| panic!("failed to insert noop jobs"));
+            }
+        }
+    }
+}

backend/windmill-common/src/ee.rs

@@ -1,76 +0,0 @@
-#[cfg(feature = "enterprise")]
-use crate::db::DB;
-use crate::ee::LicensePlan::Community;
-#[cfg(feature = "enterprise")]
-use crate::error;
-use serde::Deserialize;
-use std::sync::Arc;
-use tokio::sync::RwLock;
-
-lazy_static::lazy_static! {
-  pub static ref LICENSE_KEY_VALID: Arc<RwLock<bool>> = Arc::new(RwLock::new(true));
-  pub static ref LICENSE_KEY_ID: Arc<RwLock<String>> = Arc::new(RwLock::new("".to_string()));
-  pub static ref LICENSE_KEY: Arc<RwLock<String>> = Arc::new(RwLock::new("".to_string()));
-}
-
-pub enum LicensePlan {
-    Community,
-    Pro,
-    Enterprise,
-}
-
-pub async fn get_license_plan() -> LicensePlan {
-    // Implementation is not open source
-    return Community;
-}
-
-#[derive(Deserialize)]
-#[serde(untagged)]
-pub enum CriticalErrorChannel {
-    Email { email: String },
-    Slack { slack_channel: String },
-}
-
-pub enum CriticalAlertKind {
-    #[cfg(feature = "enterprise")]
-    CriticalError,
-    #[cfg(feature = "enterprise")]
-    RecoveredCriticalError,
-}
-
-#[cfg(feature = "enterprise")]
-pub async fn send_critical_alert(
-    _error_message: String,
-    _db: &DB,
-    _kind: CriticalAlertKind,
-    _channels: Option<Vec<CriticalErrorChannel>>,
-) {
-}
-
-#[cfg(feature = "enterprise")]
-pub async fn schedule_key_renewal(_http_client: &reqwest::Client, _db: &crate::db::DB) -> () {
-    // Implementation is not open source
-}
-
-#[cfg(feature = "enterprise")]
-pub async fn renew_license_key(
-    _http_client: &reqwest::Client,
-    _db: &crate::db::DB,
-    _key: Option<String>,
-    _manual: bool,
-) -> String {
-    // Implementation is not open source
-    "".to_string()
-}
-
-#[cfg(feature = "enterprise")]
-pub async fn create_customer_portal_session(
-    _http_client: &reqwest::Client,
-    _key: Option<String>,
-) -> error::Result<String> {
-    // Implementation is not open source
-    Ok("".to_string())
-}
-
-#[cfg(feature = "enterprise")]
-pub async fn worker_groups_alerts(_db: &DB) {}

backend/windmill-common/src/ee.rs

@@ -0,0 +1 @@
+/git/windmill/../windmill-ee-private/windmill-common/src/ee.rs

backend/windmill-common/src/ee.rs~main

@@ -0,0 +1,73 @@
+#[cfg(feature = "enterprise")]
+use crate::db::DB;
+use crate::ee::LicensePlan::Community;
+#[cfg(feature = "enterprise")]
+use crate::error;
+use serde::Deserialize;
+use std::sync::Arc;
+use tokio::sync::RwLock;
+
+lazy_static::lazy_static! {
+  pub static ref LICENSE_KEY_VALID: Arc<RwLock<bool>> = Arc::new(RwLock::new(true));
+  pub static ref LICENSE_KEY_ID: Arc<RwLock<String>> = Arc::new(RwLock::new("".to_string()));
+  pub static ref LICENSE_KEY: Arc<RwLock<String>> = Arc::new(RwLock::new("".to_string()));
+}
+
+pub enum LicensePlan {
+    Community,
+    Pro,
+    Enterprise,
+}
+
+pub async fn get_license_plan() -> LicensePlan {
+    // Implementation is not open source
+    return Community;
+}
+
+#[derive(Deserialize)]
+#[serde(untagged)]
+pub enum CriticalErrorChannel {}
+
+pub enum CriticalAlertKind {
+    #[cfg(feature = "enterprise")]
+    CriticalError,
+    #[cfg(feature = "enterprise")]
+    RecoveredCriticalError,
+}
+
+#[cfg(feature = "enterprise")]
+pub async fn send_critical_alert(
+    _error_message: String,
+    _db: &DB,
+    _kind: CriticalAlertKind,
+    _channels: Option<Vec<CriticalErrorChannel>>,
+) {
+}
+
+#[cfg(feature = "enterprise")]
+pub async fn schedule_key_renewal(_http_client: &reqwest::Client, _db: &crate::db::DB) -> () {
+    // Implementation is not open source
+}
+
+#[cfg(feature = "enterprise")]
+pub async fn renew_license_key(
+    _http_client: &reqwest::Client,
+    _db: &crate::db::DB,
+    _key: Option<String>,
+    _manual: bool,
+) -> String {
+    // Implementation is not open source
+    "".to_string()
+}
+
+#[cfg(feature = "enterprise")]
+pub async fn create_customer_portal_session(
+    _http_client: &reqwest::Client,
+    _key: Option<String>,
+) -> error::Result<String> {
+    // Implementation is not open source
+    Ok("".to_string())
+}
+
+#[cfg(feature = "enterprise")]
+pub async fn worker_groups_alerts(_db: &DB) {}

backend/windmill-common/src/ee.rs~main_0

@@ -0,0 +1,76 @@
+#[cfg(feature = "enterprise")]
+use crate::db::DB;
+use crate::ee::LicensePlan::Community;
+#[cfg(feature = "enterprise")]
+use crate::error;
+use serde::Deserialize;
+use std::sync::Arc;
+use tokio::sync::RwLock;
+
+lazy_static::lazy_static! {
+  pub static ref LICENSE_KEY_VALID: Arc<RwLock<bool>> = Arc::new(RwLock::new(true));
+  pub static ref LICENSE_KEY_ID: Arc<RwLock<String>> = Arc::new(RwLock::new("".to_string()));
+  pub static ref LICENSE_KEY: Arc<RwLock<String>> = Arc::new(RwLock::new("".to_string()));
+}
+
+pub enum LicensePlan {
+    Community,
+    Pro,
+    Enterprise,
+}
+
+pub async fn get_license_plan() -> LicensePlan {
+    // Implementation is not open source
+    return Community;
+}
+
+#[derive(Deserialize)]
+#[serde(untagged)]
+pub enum CriticalErrorChannel {
+    Email { email: String },
+    Slack { slack_channel: String },
+}
+
+pub enum CriticalAlertKind {
+    #[cfg(feature = "enterprise")]
+    CriticalError,
+    #[cfg(feature = "enterprise")]
+    RecoveredCriticalError,
+}
+
+#[cfg(feature = "enterprise")]
+pub async fn send_critical_alert(
+    _error_message: String,
+    _db: &DB,
+    _kind: CriticalAlertKind,
+    _channels: Option<Vec<CriticalErrorChannel>>,
+) {
+}
+
+#[cfg(feature = "enterprise")]
+pub async fn schedule_key_renewal(_http_client: &reqwest::Client, _db: &crate::db::DB) -> () {
+    // Implementation is not open source
+}
+
+#[cfg(feature = "enterprise")]
+pub async fn renew_license_key(
+    _http_client: &reqwest::Client,
+    _db: &crate::db::DB,
+    _key: Option<String>,
+    _manual: bool,
+) -> String {
+    // Implementation is not open source
+    "".to_string()
+}
+
+#[cfg(feature = "enterprise")]
+pub async fn create_customer_portal_session(
+    _http_client: &reqwest::Client,
+    _key: Option<String>,
+) -> error::Result<String> {
+    // Implementation is not open source
+    Ok("".to_string())
+}
+
+#[cfg(feature = "enterprise")]
+pub async fn worker_groups_alerts(_db: &DB) {}

backend/windmill-common/src/flow_status.rs

@@ -128,6 +128,7 @@ struct UntaggedFlowStatusModule {
     while_loop: Option<bool>,
     approvers: Option<Vec<Approval>>,
     failed_retries: Option<Vec<Uuid>>,
+    skipped: Option<bool>,
 }
 
 #[derive(Serialize, Debug, Clone)]
@@ -179,6 +180,7 @@ pub enum FlowStatusModule {
         approvers: Vec<Approval>,
         #[serde(skip_serializing_if = "Vec::is_empty")]
         failed_retries: Vec<Uuid>,
+        skipped: bool,
     },
     Failure {
         id: String,
@@ -255,6 +257,7 @@ impl<'de> Deserialize<'de> for FlowStatusModule {
                 branch_chosen: untagged.branch_chosen,
                 approvers: untagged.approvers.unwrap_or_default(),
                 failed_retries: untagged.failed_retries.unwrap_or_default(),
+                skipped: untagged.skipped.unwrap_or(false),
             }),
             "Failure" => Ok(FlowStatusModule::Failure {
                 id: untagged

backend/windmill-common/src/flows.rs

@@ -269,6 +269,13 @@ pub struct FlowModule {
     pub delete_after_use: Option<bool>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub continue_on_error: Option<bool>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub skip_if: Option<SkipIf>,
+}
+
+#[derive(Deserialize, Serialize, Debug, Clone)]
+pub struct SkipIf {
+    pub expr: String,
 }
 
 #[derive(Deserialize)]
@@ -285,6 +292,23 @@ pub struct FlowModuleValueWithParallel {
     pub parallelism: Option<u16>,
 }
 
+#[derive(Deserialize)]
+pub struct FlowModuleValueWithSkipFailures {
+    pub skip_failures: Option<bool>,
+    pub parallel: Option<bool>,
+    pub parallelism: Option<u16>,
+}
+
+#[derive(Deserialize)]
+pub struct BranchWithSkipFailures {
+    pub skip_failure: Option<bool>,
+}
+
+#[derive(Deserialize)]
+pub struct FlowModuleWithBranches {
+    pub branches: Vec<BranchWithSkipFailures>,
+}
+
 impl FlowModule {
     pub fn id_append(&mut self, s: &str) {
         self.id = format!("{}-{}", self.id, s);
@@ -293,6 +317,16 @@ impl FlowModule {
         serde_json::from_str::<FlowModuleValue>(self.value.get()).map_err(crate::error::to_anyhow)
     }
 
+    pub fn get_value_with_skip_failures(&self) -> anyhow::Result<FlowModuleValueWithSkipFailures> {
+        serde_json::from_str::<FlowModuleValueWithSkipFailures>(self.value.get())
+            .map_err(crate::error::to_anyhow)
+    }
+
+    pub fn get_branches_skip_failures(&self) -> anyhow::Result<FlowModuleWithBranches> {
+        serde_json::from_str::<FlowModuleWithBranches>(self.value.get())
+            .map_err(crate::error::to_anyhow)
+    }
+
     pub fn is_flow(&self) -> bool {
         self.get_type().is_ok_and(|x| x == "flow")
     }
@@ -604,6 +638,7 @@ pub fn add_virtual_items_if_necessary(modules: &mut Vec<FlowModule>) {
             priority: None,
             delete_after_use: None,
             continue_on_error: None,
+            skip_if: None,
         });
     }
 }

backend/windmill-common/src/job_s3_helpers_ee.rs

@@ -1,18 +0,0 @@
-use std::future::Future;
-
-use crate::{
-    error::Error,
-    s3_helpers::{ObjectStoreResource, StorageResourceType},
-};
-
-pub async fn get_s3_resource_internal<'c, F, Fut>(
-    _resource_type: StorageResourceType,
-    _s3_resource_value_raw: serde_json::Value,
-    _gen_token: F,
-) -> crate::error::Result<ObjectStoreResource>
-where
-    F: FnOnce(String) -> Fut,
-    Fut: Future<Output = Result<String, Error>> + Send + 'static,
-{
-    todo!()
-}

backend/windmill-common/src/job_s3_helpers_ee.rs

@@ -0,0 +1 @@
+/git/windmill/../windmill-ee-private/windmill-common/src/job_s3_helpers_ee.rs

backend/windmill-common/src/lib.rs

@@ -17,6 +17,8 @@ use scripts::ScriptLang;
 use sqlx::{Pool, Postgres};
 
 pub mod apps;
+#[cfg(feature = "benchmark")]
+pub mod bench;
 pub mod db;
 pub mod ee;
 pub mod error;
@@ -51,6 +53,17 @@ pub const DEFAULT_MAX_CONNECTIONS_INDEXER: u32 = 5;
 
 pub const DEFAULT_HUB_BASE_URL: &str = "https://hub.windmill.dev";
 
+#[macro_export]
+macro_rules! add_time {
+    ($bench:expr, $name:expr) => {
+        #[cfg(feature = "benchmark")]
+        {
+            $bench.add_timing($name);
+            // println!("{}: {:?}", $z, $y.elapsed());
+        }
+    };
+}
+
 lazy_static::lazy_static! {
     pub static ref METRICS_PORT: u16 = std::env::var("METRICS_PORT")
     .ok()
@@ -82,6 +95,8 @@ lazy_static::lazy_static! {
 
     pub static ref JOB_RETENTION_SECS: Arc<RwLock<i64>> = Arc::new(RwLock::new(0));
 
+    pub static ref INSTANCE_NAME: String = rd_string(5);
+
 }
 
 pub async fn shutdown_signal(
@@ -128,6 +143,7 @@ pub async fn shutdown_signal(
 use tokio::sync::RwLock;
 #[cfg(feature = "prometheus")]
 use tokio::task::JoinHandle;
+use utils::rd_string;
 
 #[cfg(feature = "prometheus")]
 pub async fn serve_metrics(

backend/windmill-common/src/stats_ee.rs

@@ -1,45 +0,0 @@
-use sqlx::Postgres;
-
-use crate::{error::Result, scripts::ScriptLang, DB};
-
-pub async fn get_disable_stats_setting(_db: &DB) -> bool {
-    // stats details are closed source
-
-    false
-}
-
-pub async fn schedule_stats(
-    _instance_name: String,
-    _db: &DB,
-    _http_client: &reqwest::Client,
-) -> () {
-    // stats details are closed source
-}
-
-#[derive(Debug, sqlx::FromRow, serde::Serialize)]
-struct JobsUsage {
-    language: Option<ScriptLang>,
-    total_duration: i64,
-    count: i64,
-}
-
-pub async fn send_stats(
-    _instance_name: &String,
-    _http_client: &reqwest::Client,
-    _db: &DB,
-) -> Result<()> {
-    // stats details are closed source
-    Ok(())
-}
-
-pub struct ActiveUserUsage {
-    pub author_count: Option<i32>,
-    pub operator_count: Option<i32>,
-}
-
-pub async fn get_user_usage<'c, E: sqlx::Executor<'c, Database = Postgres>>(
-    _db: E,
-) -> Result<ActiveUserUsage> {
-    let usage = ActiveUserUsage { author_count: None, operator_count: None };
-    Ok(usage)
-}

backend/windmill-common/src/stats_ee.rs

@@ -0,0 +1 @@
+/git/windmill/../windmill-ee-private/windmill-common/src/stats_ee.rs

backend/windmill-common/src/utils.rs

@@ -38,7 +38,7 @@ lazy_static::lazy_static! {
         .build().unwrap();
 }
 
-#[derive(Deserialize)]
+#[derive(Deserialize, Clone)]
 pub struct Pagination {
     pub page: Option<usize>,
     pub per_page: Option<usize>,
@@ -82,6 +82,12 @@ pub fn paginate(pagination: Pagination) -> (usize, usize) {
     (per_page, offset)
 }
 
+pub fn paginate_without_limits(pagination: Pagination) -> (usize, usize) {
+    let per_page = pagination.per_page.unwrap_or(MAX_PER_PAGE);
+    let offset = (pagination.page.unwrap_or(1).max(1) - 1) * per_page;
+    (per_page, offset)
+}
+
 pub async fn now_from_db<'c, E: sqlx::PgExecutor<'c>>(
     db: E,
 ) -> Result<chrono::DateTime<chrono::Utc>> {
@@ -183,7 +189,7 @@ pub fn calculate_hash(s: &str) -> String {
     format!("{:x}", hasher.finalize())
 }
 
-pub async fn get_uid(db: &DB) -> Result<String> {
+pub async fn get_uid<'c, E: sqlx::Executor<'c, Database = Postgres>>(db: E) -> Result<String> {
     let mut uid = LICENSE_KEY_ID.read().await.clone();
 
     if uid == "" {
@@ -222,6 +228,13 @@ impl std::fmt::Display for Mode {
     }
 }
 
+// inspired from rails: https://github.com/rails/rails/blob/6e49cc77ab3d16c06e12f93158eaf3e507d4120e/activerecord/lib/active_record/migration.rb#L1308
+pub fn generate_lock_id(database_name: &str) -> i64 {
+    const CRC_IEEE: crc::Crc<u32> = crc::Crc::<u32>::new(&crc::CRC_32_ISO_HDLC);
+    // 0x3d32ad9e chosen by fair dice roll
+    0x3d32ad9e * (CRC_IEEE.checksum(database_name.as_bytes()) as i64)
+}
+
 pub async fn send_email(
     subject: &str,
     content: &str,

backend/windmill-common/src/worker.rs

@@ -327,6 +327,22 @@ pub fn get_annotation(inner_content: &str) -> Annotations {
     Annotations { npm_mode, nodejs_mode, native_mode, nobundling }
 }
 
+pub struct SqlAnnotations {
+    pub return_last_result: bool,
+}
+
+pub fn get_sql_annotations(inner_content: &str) -> SqlAnnotations {
+    let annotations = inner_content
+        .lines()
+        .take_while(|x| x.starts_with("--"))
+        .map(|x| x.to_string().replace("--", "").trim().to_string())
+        .collect_vec();
+
+    let return_last_result: bool = annotations.contains(&"return_last_result".to_string());
+
+    SqlAnnotations { return_last_result }
+}
+
 pub async fn load_cache(bin_path: &str, _remote_path: &str) -> (bool, String) {
     if tokio::fs::metadata(&bin_path).await.is_ok() {
         (true, format!("loaded from local cache: {}\n", bin_path))

backend/windmill-git-sync/src/git_sync_ee.rs

@@ -1,17 +0,0 @@
-use windmill_common::error::Result;
-
-use crate::{DeployedObject, DB};
-
-pub async fn handle_deployment_metadata<'c, R: rsmq_async::RsmqConnection + Send + Clone + 'c>(
-    _email: &str,
-    _created_by: &str,
-    _db: &DB,
-    _w_id: &str,
-    _obj: DeployedObject,
-    _deployment_message: Option<String>,
-    _rsmq: Option<R>,
-    _skip_db_insert: bool,
-) -> Result<()> {
-    // Git sync is an enterprise feature and not part of the open-source version
-    return Ok(());
-}

backend/windmill-git-sync/src/git_sync_ee.rs

@@ -0,0 +1 @@
+/git/windmill/../windmill-ee-private/windmill-git-sync/src/git_sync_ee.rs

backend/windmill-indexer/src/indexer_ee.rs

@@ -1,21 +0,0 @@
-use sqlx::{Pool, Postgres};
-use windmill_common::error::Error;
-use anyhow::anyhow;
-
-#[derive(Clone)]
-pub struct IndexReader;
-
-#[derive(Clone)]
-pub struct IndexWriter;
-
-pub async fn init_index() -> Result<(IndexReader, IndexWriter), Error> {
-    Err(anyhow!("Cannot initialize index: not in EE").into())
-}
-
-pub async fn run_indexer(
-    _db: Pool<Postgres>,
-    mut _index_writer: IndexWriter,
-    mut _killpill_rx: tokio::sync::broadcast::Receiver<()>,
-) {
-    tracing::error!("Cannot run indexer: not in EE");
-}

backend/windmill-indexer/src/indexer_ee.rs

@@ -0,0 +1 @@
+/git/windmill/../windmill-ee-private/windmill-indexer/src/indexer_ee.rs

backend/windmill-queue/Cargo.toml

@@ -12,7 +12,7 @@ path = "src/lib.rs"
 default = []
 enterprise = ["windmill-common/enterprise"]
 cloud = []
-benchmark = []
+benchmark = ["windmill-common/benchmark"]
 prometheus = ["dep:prometheus"]
 
 [dependencies]

backend/windmill-queue/src/jobs.rs

@@ -37,9 +37,9 @@ use ulid::Ulid;
 use uuid::Uuid;
 use windmill_audit::audit_ee::{audit_log, AuditAuthor};
 use windmill_audit::ActionKind;
-#[cfg(not(feature = "enterprise"))]
-use windmill_common::worker::PriorityTags;
+
 use windmill_common::{
+    add_time,
     auth::{fetch_authed_from_permissioned_as, permissioned_as_to_username},
     db::{Authed, UserDB},
     error::{self, to_anyhow, Error},
@@ -62,9 +62,12 @@ use windmill_common::{
         to_raw_value, DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES, NO_LOGS, WORKER_CONFIG,
         WORKER_PULL_QUERIES, WORKER_SUSPENDED_PULL_QUERY,
     },
-    BASE_URL, DB, METRICS_ENABLED,
+    DB, METRICS_ENABLED,
 };
 
+#[cfg(feature = "enterprise")]
+use windmill_common::BASE_URL;
+
 #[cfg(feature = "cloud")]
 use windmill_common::users::SUPERADMIN_SYNC_EMAIL;
 
@@ -125,10 +128,12 @@ const MAX_FREE_CONCURRENT_RUNS: i32 = 30;
 
 const ERROR_HANDLER_USERNAME: &str = "error_handler";
 const SCHEDULE_ERROR_HANDLER_USERNAME: &str = "schedule_error_handler";
+#[cfg(feature = "enterprise")]
 const SCHEDULE_RECOVERY_HANDLER_USERNAME: &str = "schedule_recovery_handler";
 const ERROR_HANDLER_USER_GROUP: &str = "g/error_handler";
 const ERROR_HANDLER_USER_EMAIL: &str = "error_handler@windmill.dev";
 const SCHEDULE_ERROR_HANDLER_USER_EMAIL: &str = "schedule_error_handler@windmill.dev";
+#[cfg(feature = "enterprise")]
 const SCHEDULE_RECOVERY_HANDLER_USER_EMAIL: &str = "schedule_recovery_handler@windmill.dev";
 
 #[derive(Clone, Debug)]
@@ -174,6 +179,8 @@ pub async fn cancel_single_job<'c>(
                 rsmq.clone(),
                 "server",
                 false,
+                #[cfg(feature = "benchmark")]
+                &mut windmill_common::bench::BenchmarkIter::new(),
             )
             .await;
 
@@ -475,8 +482,10 @@ where
     }
 }
 
+#[cfg(feature = "enterprise")]
 #[derive(Deserialize)]
 struct RawFlowFailureModule {
+    #[cfg(feature = "enterprise")]
     failure_module: Option<Box<RawValue>>,
 }
 
@@ -490,6 +499,7 @@ pub async fn add_completed_job_error<R: rsmq_async::RsmqConnection + Clone + Sen
     rsmq: Option<R>,
     _worker_name: &str,
     flow_is_done: bool,
+    #[cfg(feature = "benchmark")] bench: &mut windmill_common::bench::BenchmarkIter,
 ) -> Result<WrappedError, Error> {
     #[cfg(feature = "prometheus")]
     register_metric(
@@ -527,6 +537,8 @@ pub async fn add_completed_job_error<R: rsmq_async::RsmqConnection + Clone + Sen
         canceled_by,
         rsmq,
         flow_is_done,
+        #[cfg(feature = "benchmark")]
+        bench,
     )
     .await?;
     Ok(result)
@@ -550,10 +562,12 @@ pub async fn add_completed_job<
     canceled_by: Option<CanceledBy>,
     rsmq: Option<R>,
     flow_is_done: bool,
+    #[cfg(feature = "benchmark")] bench: &mut windmill_common::bench::BenchmarkIter,
 ) -> Result<Uuid, Error> {
     // tracing::error!("Start");
     // let start = tokio::time::Instant::now();
 
+    add_time!(bench, "add_completed_job start");
     if !result.is_valid_json() {
         return Err(Error::InternalErr(
             "Result of job is invalid json (empty)".to_string(),
@@ -561,6 +575,7 @@ pub async fn add_completed_job<
     }
 
     let mut tx: QueueTransaction<'_, R> = (rsmq.clone(), db.begin().await?).into();
+
     let job_id = queued_job.id;
     // tracing::error!("1 {:?}", start.elapsed());
 
@@ -571,6 +586,7 @@ pub async fn add_completed_job<
     );
 
     let mem_peak = mem_peak.max(queued_job.mem_peak.unwrap_or(0));
+    add_time!(bench, "add_completed_job query START");
     let _duration: i64 = sqlx::query_scalar!(
         "INSERT INTO completed_job AS cj
                    ( workspace_id
@@ -642,6 +658,8 @@ pub async fn add_completed_job<
     .map_err(|e| Error::InternalErr(format!("Could not add completed job {job_id}: {e:#}")))?;
     // tracing::error!("2 {:?}", start.elapsed());
 
+    add_time!(bench, "add_completed_job query END");
+
     if !queued_job.is_flow_step {
         if _duration > 500
             && (queued_job.job_kind == JobKind::Script || queued_job.job_kind == JobKind::Preview)
@@ -671,6 +689,7 @@ pub async fn add_completed_job<
         }
     }
     // tracing::error!("Added completed job {:#?}", queued_job);
+    #[cfg(feature = "enterprise")]
     let mut skip_downstream_error_handlers = false;
     tx = delete_job(tx, &queued_job.workspace_id, job_id).await?;
     // tracing::error!("3 {:?}", start.elapsed());
@@ -716,7 +735,10 @@ pub async fn add_completed_job<
             .await?;
 
             if let Some(schedule) = schedule {
-                skip_downstream_error_handlers = schedule.ws_error_handler_muted;
+                #[cfg(feature = "enterprise")]
+                {
+                    skip_downstream_error_handlers = schedule.ws_error_handler_muted;
+                }
 
                 // script or flow that failed on start and might not have been rescheduled
                 let schedule_next_tick = !queued_job.is_flow()
@@ -749,6 +771,7 @@ pub async fn add_completed_job<
                     };
                 }
 
+                #[cfg(feature = "enterprise")]
                 if let Err(err) = apply_schedule_handlers(
                     rsmq.clone(),
                     db,
@@ -1324,6 +1347,8 @@ struct CompletedJobSubset {
     result: Option<sqlx::types::Json<Box<RawValue>>>,
     started_at: chrono::DateTime<chrono::Utc>,
 }
+
+#[cfg(feature = "enterprise")]
 async fn apply_schedule_handlers<
     'a,
     'c,
@@ -1342,7 +1367,6 @@ async fn apply_schedule_handlers<
     job_priority: Option<i16>,
 ) -> windmill_common::error::Result<()> {
     if !success {
-        #[cfg(feature = "enterprise")]
         if let Some(on_failure_path) = schedule.on_failure.clone() {
             let times = schedule.on_failure_times.unwrap_or(1).max(1);
             let exact = schedule.on_failure_exact.unwrap_or(false);
@@ -1392,7 +1416,6 @@ async fn apply_schedule_handlers<
             .await?;
         }
     } else {
-        #[cfg(feature = "enterprise")]
         if let Some(ref on_success_path) = schedule.on_success {
             handle_successful_schedule(
                 db,
@@ -1410,7 +1433,6 @@ async fn apply_schedule_handlers<
             .await?;
         }
 
-        #[cfg(feature = "enterprise")]
         if let Some(ref on_recovery_path) = schedule.on_recovery.clone() {
             let tx: QueueTransaction<'_, R> = (rsmq.clone(), db.begin().await?).into();
             let times = schedule.on_recovery_times.unwrap_or(1).max(1);
@@ -1579,6 +1601,7 @@ fn sanitize_result<T: Serialize + Send + Sync>(result: Json<&T>) -> HashMap<Stri
 //     is_flow: boolean,
 //     extra_args: serde_json::Value
 // }
+#[cfg(feature = "enterprise")]
 async fn handle_recovered_schedule<
     'a,
     'c,
@@ -1671,6 +1694,7 @@ async fn handle_recovered_schedule<
     Ok(())
 }
 
+#[cfg(feature = "enterprise")]
 async fn handle_successful_schedule<
     'a,
     'c,
@@ -1755,9 +1779,9 @@ pub async fn pull<R: rsmq_async::RsmqConnection + Send + Clone>(
     db: &Pool<Postgres>,
     rsmq: Option<R>,
     suspend_first: bool,
-) -> windmill_common::error::Result<Option<QueuedJob>> {
+) -> windmill_common::error::Result<(Option<QueuedJob>, bool)> {
     loop {
-        let job = pull_single_job_and_mark_as_running_no_concurrency_limit(
+        let (job, suspended) = pull_single_job_and_mark_as_running_no_concurrency_limit(
             db,
             rsmq.clone(),
             suspend_first,
@@ -1765,7 +1789,7 @@ pub async fn pull<R: rsmq_async::RsmqConnection + Send + Clone>(
         .await?;
 
         if job.is_none() {
-            return Ok(None);
+            return Ok((None, suspended));
         }
 
         let has_concurent_limit = job.as_ref().unwrap().concurrent_limit.is_some();
@@ -1785,7 +1809,7 @@ pub async fn pull<R: rsmq_async::RsmqConnection + Send + Clone>(
             if METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed) {
                 QUEUE_PULL_COUNT.inc();
             }
-            return Ok(Option::Some(pulled_job));
+            return Ok((Option::Some(pulled_job), suspended));
         }
 
         let itx = db.begin().await?;
@@ -1886,7 +1910,7 @@ pub async fn pull<R: rsmq_async::RsmqConnection + Send + Clone>(
                 QUEUE_PULL_COUNT.inc();
             }
             tx.commit().await?;
-            return Ok(Option::Some(pulled_job));
+            return Ok((Option::Some(pulled_job), suspended));
         }
         let x = sqlx::query_scalar!(
             "UPDATE concurrency_counter SET job_uuids = job_uuids - $2 WHERE concurrency_id = $1 RETURNING (SELECT COUNT(*) FROM jsonb_object_keys(job_uuids))",
@@ -2013,8 +2037,8 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<
     db: &Pool<Postgres>,
     rsmq: Option<R>,
     suspend_first: bool,
-) -> windmill_common::error::Result<Option<QueuedJob>> {
-    let job: Option<QueuedJob> = if let Some(mut rsmq) = rsmq {
+) -> windmill_common::error::Result<(Option<QueuedJob>, bool)> {
+    let job_and_suspended: (Option<QueuedJob>, bool) = if let Some(mut rsmq) = rsmq {
         #[cfg(feature = "benchmark")]
         let instant = Instant::now();
 
@@ -2072,9 +2096,9 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<
             #[cfg(feature = "benchmark")]
             println!("rsmq 2: {:?}", instant.elapsed());
 
-            m2r
+            (m2r, false)
         } else {
-            None
+            (None, false)
         }
     } else {
         /* Jobs can be started if they:
@@ -2088,7 +2112,7 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<
 
         if query.is_empty() {
             tracing::warn!("No suspended pull queries available");
-            return Ok(None);
+            return Ok((None, false));
         }
 
         let r = if suspend_first {
@@ -2108,7 +2132,7 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<
 
             if queries.is_empty() {
                 tracing::warn!("No pull queries available");
-                return Ok(None);
+                return Ok((None, false));
             }
 
             for query in queries.iter() {
@@ -2126,12 +2150,12 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<
 
             // #[cfg(feature = "benchmark")]
             // println!("pull query: {:?}", instant.elapsed());
-            highest_priority_job
+            (highest_priority_job, false)
         } else {
-            r
+            (r, true)
         }
     };
-    Ok(job)
+    Ok(job_and_suspended)
 }
 
 pub async fn custom_concurrency_key(
@@ -3512,6 +3536,7 @@ pub async fn push<'c, 'd, R: rsmq_async::RsmqConnection + Send + 'c>(
                     priority: None,
                     delete_after_use: None,
                     continue_on_error: None,
+                    skip_if: None,
                 }],
                 same_worker: false,
                 failure_module: None,

backend/windmill-worker/Cargo.toml

@@ -12,7 +12,7 @@ path = "src/lib.rs"
 default = []
 prometheus = ["dep:prometheus", "windmill-common/prometheus"]
 enterprise = ["windmill-queue/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "dep:gcp_auth", "dep:pem", "dep:tiberius", "dep:tokio-util", "dep:openidconnect"]
-benchmark = ["windmill-queue/benchmark"]
+benchmark = ["windmill-queue/benchmark", "windmill-common/benchmark"]
 flamegraph = []
 parquet = ["windmill-common/parquet", "dep:object_store"]
 flow_testing = []

backend/windmill-worker/nsjail/download.py.config.proto

@@ -80,6 +80,12 @@ mount {
     rw: true
 }
 
+mount {
+    src: "/dev/urandom"
+    dst: "/dev/urandom"
+    is_bind: true
+}
+
 exec_bin {
  	path: "/bin/sh"
  	arg: "/download_deps.sh"

backend/windmill-worker/nsjail/download_deps.py.sh

@@ -19,6 +19,6 @@ then
       echo "\$TRUSTED_HOST is set to $TRUSTED_HOST"
 fi
 
-CMD="/usr/local/bin/python3 -m pip install -v $REQ -I -t $TARGET --no-cache --no-color --no-deps --isolated --no-warn-conflicts --disable-pip-version-check $INDEX_URL_ARG $EXTRA_INDEX_URL_ARG $TRUSTED_HOST_ARG"
+CMD="/usr/local/bin/python3 -m pip install -v \"$REQ\" -I -t \"$TARGET\" --no-cache --no-color --no-deps --isolated --no-warn-conflicts --disable-pip-version-check $INDEX_URL_ARG $EXTRA_INDEX_URL_ARG $TRUSTED_HOST_ARG"
 echo $CMD
 eval $CMD

backend/windmill-worker/nsjail/run.bun.config.proto

@@ -120,6 +120,7 @@ mount {
     src: "{JOB_DIR}/args.json"
     dst: "/tmp/{LANG}/args.json"
     is_bind: true
+    rw: true
 }
 
 mount {

backend/windmill-worker/nsjail/run.python3.config.proto

@@ -81,6 +81,7 @@ mount {
     src: "{JOB_DIR}/args.json"
     dst: "/tmp/args.json"
     is_bind: true
+    rw: true
 }
 
 mount {

backend/windmill-worker/src/ansible_executor.rs

@@ -21,9 +21,10 @@ use windmill_queue::{append_logs, CanceledBy};
 use crate::{
     bash_executor::BIN_BASH,
     common::{
-        get_reserved_variables, handle_child, read_and_check_result, start_child_process,
-        transform_json,
+        get_reserved_variables, read_and_check_result, start_child_process, transform_json,
+        OccupancyMetrics,
     },
+    handle_child::handle_child,
     python_executor::{create_dependencies_dir, handle_python_reqs, pip_compile},
     AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV,
     TZ_ENV,
@@ -50,6 +51,7 @@ async fn handle_ansible_python_deps(
     worker_dir: &str,
     mem_peak: &mut i32,
     canceled_by: &mut Option<CanceledBy>,
+    occupancy_metrics: &mut OccupancyMetrics,
 ) -> error::Result<Vec<String>> {
     create_dependencies_dir(job_dir).await;
 
@@ -79,6 +81,7 @@ async fn handle_ansible_python_deps(
                     db,
                     worker_name,
                     w_id,
+                    &mut Some(occupancy_metrics),
                 )
                 .await
                 .map_err(|e| {
@@ -102,6 +105,7 @@ async fn handle_ansible_python_deps(
             worker_name,
             job_dir,
             worker_dir,
+            &mut Some(occupancy_metrics),
         )
         .await?;
         additional_python_paths.append(&mut venv_path);
@@ -118,6 +122,7 @@ async fn install_galaxy_collections(
     mem_peak: &mut i32,
     canceled_by: &mut Option<CanceledBy>,
     db: &sqlx::Pool<sqlx::Postgres>,
+    occupancy_metrics: &mut OccupancyMetrics,
 ) -> anyhow::Result<()> {
     write_file(job_dir, "requirements.yml", collections_yml)?;
 
@@ -160,6 +165,7 @@ async fn install_galaxy_collections(
         "ansible galaxy install",
         None,
         false,
+        &mut Some(occupancy_metrics),
     )
     .await?;
 
@@ -198,6 +204,7 @@ pub async fn handle_ansible_job(
     shared_mount: &str,
     base_internal_url: &str,
     envs: HashMap<String, String>,
+    occupancy_metrics: &mut OccupancyMetrics,
 ) -> windmill_common::error::Result<Box<RawValue>> {
     check_ansible_exists()?;
 
@@ -216,6 +223,7 @@ pub async fn handle_ansible_job(
         worker_dir,
         mem_peak,
         canceled_by,
+        occupancy_metrics,
     )
     .await?;
 
@@ -289,6 +297,7 @@ pub async fn handle_ansible_job(
                 mem_peak,
                 canceled_by,
                 db,
+                occupancy_metrics,
             )
             .await?;
         }
@@ -424,6 +433,7 @@ fi
         "python run",
         job.timeout,
         false,
+        &mut Some(occupancy_metrics),
     )
     .await?;
     read_and_check_result(job_dir).await

backend/windmill-worker/src/bash_executor.rs

@@ -24,9 +24,10 @@ lazy_static::lazy_static! {
 
 use crate::{
     common::{
-        build_args_map, get_reserved_variables, handle_child, read_file, read_file_content,
-        start_child_process,
+        build_args_map, get_reserved_variables, read_file, read_file_content, start_child_process,
+        OccupancyMetrics,
     },
+    handle_child::handle_child,
     AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV,
     POWERSHELL_CACHE_DIR, POWERSHELL_PATH, TZ_ENV,
 };
@@ -49,6 +50,7 @@ pub async fn handle_bash_job(
     base_internal_url: &str,
     worker_name: &str,
     envs: HashMap<String, String>,
+    occupancy_metrics: &mut OccupancyMetrics,
 ) -> Result<Box<RawValue>, Error> {
     let logs1 = "\n\n--- BASH CODE EXECUTION ---\n".to_string();
     append_logs(&job.id, &job.workspace_id, logs1, db).await;
@@ -142,6 +144,7 @@ pub async fn handle_bash_job(
         "bash run",
         job.timeout,
         true,
+        &mut Some(occupancy_metrics),
     )
     .await?;
 
@@ -194,6 +197,7 @@ pub async fn handle_powershell_job(
     base_internal_url: &str,
     worker_name: &str,
     envs: HashMap<String, String>,
+    occupancy_metrics: &mut OccupancyMetrics,
 ) -> Result<Box<RawValue>, Error> {
     let pwsh_args = {
         let args = build_args_map(job, client, db).await?.map(Json);
@@ -275,6 +279,7 @@ pub async fn handle_powershell_job(
             "powershell install",
             job.timeout,
             false,
+            &mut Some(occupancy_metrics),
         )
         .await?;
     }
@@ -378,6 +383,7 @@ $env:PSModulePath = \"{}:$PSModulePathBackup\"",
         "powershell run",
         job.timeout,
         false,
+        &mut Some(occupancy_metrics),
     )
     .await?;
 

backend/windmill-worker/src/bigquery_executor.rs

@@ -5,6 +5,7 @@ use futures::{FutureExt, TryFutureExt};
 use serde_json::{json, value::RawValue, Value};
 use windmill_common::error::to_anyhow;
 use windmill_common::jobs::QueuedJob;
+use windmill_common::worker::get_sql_annotations;
 use windmill_common::{error::Error, worker::to_raw_value};
 use windmill_parser_sql::{
     parse_bigquery_sig, parse_db_resource, parse_sql_blocks, parse_sql_statement_named_params,
@@ -13,7 +14,8 @@ use windmill_queue::{CanceledBy, HTTP_CLIENT};
 
 use serde::Deserialize;
 
-use crate::common::run_future_with_polling_update_job_poller;
+use crate::common::OccupancyMetrics;
+use crate::handle_child::run_future_with_polling_update_job_poller;
 use crate::{
     common::{build_args_values, resolve_job_timeout},
     AuthedClientBackgroundTask,
@@ -69,6 +71,7 @@ fn do_bigquery_inner<'a>(
     token: &'a str,
     timeout_ms: i32,
     column_order: Option<&'a mut Option<Vec<String>>>,
+    skip_collect: bool,
 ) -> windmill_common::error::Result<BoxFuture<'a, windmill_common::error::Result<Box<RawValue>>>> {
     let param_names = parse_sql_statement_named_params(query, '@');
 
@@ -106,76 +109,80 @@ fn do_bigquery_inner<'a>(
 
         match response.error_for_status_ref() {
             Ok(_) => {
-                let result = response.json::<BigqueryResponse>().await.map_err(|e| {
-                    Error::ExecutionErr(format!(
-                        "BigQuery API response could not be parsed: {}",
-                        e.to_string()
-                    ))
-                })?;
+                if skip_collect {
+                    return Ok(to_raw_value(&Value::Array(vec![])));
+                } else {
+                    let result = response.json::<BigqueryResponse>().await.map_err(|e| {
+                        Error::ExecutionErr(format!(
+                            "BigQuery API response could not be parsed: {}",
+                            e.to_string()
+                        ))
+                    })?;
 
-                if !result.jobComplete {
-                    return Err(Error::ExecutionErr(
-                        "BigQuery API did not answer query in time".to_string(),
+                    if !result.jobComplete {
+                        return Err(Error::ExecutionErr(
+                            "BigQuery API did not answer query in time".to_string(),
+                        ));
+                    }
+
+                    if result.rows.is_none() || result.rows.as_ref().unwrap().len() == 0 {
+                        return Ok(serde_json::from_str("[]").unwrap());
+                    }
+
+                    if result.schema.is_none() {
+                        return Err(Error::ExecutionErr(
+                            "Incomplete response from BigQuery API".to_string(),
+                        ));
+                    }
+
+                    if result
+                        .totalRows
+                        .unwrap_or(json!(""))
+                        .as_str()
+                        .unwrap_or("")
+                        .parse::<i64>()
+                        .unwrap_or(0)
+                        > 10000
+                    {
+                        return Err(Error::ExecutionErr(
+                        "More than 10000 rows were requested, use LIMIT 10000 to limit the number of rows".to_string(),
                     ));
+                    }
+
+                    if let Some(column_order) = column_order {
+                        *column_order = Some(
+                            result
+                                .schema
+                                .as_ref()
+                                .unwrap()
+                                .fields
+                                .iter()
+                                .map(|x| x.name.clone())
+                                .collect::<Vec<String>>(),
+                        );
+                    }
+
+                    let rows = result
+                        .rows
+                        .unwrap()
+                        .iter()
+                        .map(|row| {
+                            let mut row_map = serde_json::Map::new();
+                            row.f
+                                .iter()
+                                .zip(result.schema.as_ref().unwrap().fields.iter())
+                                .for_each(|(field, schema)| {
+                                    row_map.insert(
+                                        schema.name.clone(),
+                                        parse_val(&field.v, &schema.r#type, &schema),
+                                    );
+                                });
+                            Value::from(row_map)
+                        })
+                        .collect::<Vec<_>>();
+
+                    Ok(to_raw_value(&rows))
                 }
-
-                if result.rows.is_none() || result.rows.as_ref().unwrap().len() == 0 {
-                    return Ok(serde_json::from_str("[]").unwrap());
-                }
-
-                if result.schema.is_none() {
-                    return Err(Error::ExecutionErr(
-                        "Incomplete response from BigQuery API".to_string(),
-                    ));
-                }
-
-                if result
-                    .totalRows
-                    .unwrap_or(json!(""))
-                    .as_str()
-                    .unwrap_or("")
-                    .parse::<i64>()
-                    .unwrap_or(0)
-                    > 10000
-                {
-                    return Err(Error::ExecutionErr(
-                    "More than 10000 rows were requested, use LIMIT 10000 to limit the number of rows".to_string(),
-                ));
-                }
-
-                if let Some(column_order) = column_order {
-                    *column_order = Some(
-                        result
-                            .schema
-                            .as_ref()
-                            .unwrap()
-                            .fields
-                            .iter()
-                            .map(|x| x.name.clone())
-                            .collect::<Vec<String>>(),
-                    );
-                }
-
-                let rows = result
-                    .rows
-                    .unwrap()
-                    .iter()
-                    .map(|row| {
-                        let mut row_map = serde_json::Map::new();
-                        row.f
-                            .iter()
-                            .zip(result.schema.as_ref().unwrap().fields.iter())
-                            .for_each(|(field, schema)| {
-                                row_map.insert(
-                                    schema.name.clone(),
-                                    parse_val(&field.v, &schema.r#type, &schema),
-                                );
-                            });
-                        Value::from(row_map)
-                    })
-                    .collect::<Vec<_>>();
-
-                return Ok(to_raw_value(&rows));
             }
             Err(e) => match response.json::<BigqueryErrorResponse>().await {
                 Ok(bq_err) => Err(Error::ExecutionErr(format!(
@@ -204,6 +211,7 @@ pub async fn do_bigquery(
     canceled_by: &mut Option<CanceledBy>,
     worker_name: &str,
     column_order: &mut Option<Vec<String>>,
+    occupancy_metrics: &mut OccupancyMetrics,
 ) -> windmill_common::error::Result<Box<RawValue>> {
     let bigquery_args = build_args_values(job, client, db).await?;
 
@@ -230,6 +238,8 @@ pub async fn do_bigquery(
         return Err(Error::BadRequest("Missing database argument".to_string()));
     };
 
+    let annotations = get_sql_annotations(query);
+
     let service_account = CustomServiceAccount::from_json(&database)
         .map_err(|e| Error::ExecutionErr(e.to_string()))?;
 
@@ -306,7 +316,8 @@ pub async fn do_bigquery(
     let result_f = if queries.len() > 1 {
         let futures = queries
             .iter()
-            .map(|x| {
+            .enumerate()
+            .map(|(i, x)| {
                 do_bigquery_inner(
                     x,
                     &statement_values,
@@ -314,17 +325,23 @@ pub async fn do_bigquery(
                     token.as_str(),
                     timeout_ms,
                     None,
+                    annotations.return_last_result && i < queries.len() - 1,
                 )
             })
             .collect::<windmill_common::error::Result<Vec<_>>>()?;
 
         let f = async {
             let mut res: Vec<Box<RawValue>> = vec![];
+
             for fut in futures {
                 let r = fut.await?;
                 res.push(r);
             }
-            Ok(to_raw_value(&res))
+            if annotations.return_last_result && res.len() > 0 {
+                Ok(res.pop().unwrap())
+            } else {
+                Ok(to_raw_value(&res))
+            }
         };
 
         f.boxed()
@@ -336,6 +353,7 @@ pub async fn do_bigquery(
             token.as_str(),
             timeout_ms,
             Some(column_order),
+            false,
         )?
     };
 
@@ -348,6 +366,7 @@ pub async fn do_bigquery(
         result_f.map_err(to_anyhow),
         worker_name,
         &job.workspace_id,
+        &mut Some(occupancy_metrics),
     )
     .await?;
 

backend/windmill-worker/src/bun_executor.rs

@@ -13,10 +13,11 @@ use crate::common::build_envs_map;
 
 use crate::{
     common::{
-        create_args_and_out_file, get_main_override, get_reserved_variables, handle_child,
-        parse_npm_config, read_file, read_file_content, read_result, start_child_process,
-        write_file_binary,
+        create_args_and_out_file, get_main_override, get_reserved_variables, parse_npm_config,
+        read_file, read_file_content, read_result, start_child_process, write_file_binary,
+        OccupancyMetrics,
     },
+    handle_child::handle_child,
     AuthedClientBackgroundTask, BUNFIG_INSTALL_SCOPES, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR,
     BUN_DEPSTAR_CACHE_DIR, BUN_PATH, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NODE_BIN_PATH,
     NODE_PATH, NPM_CONFIG_REGISTRY, NPM_PATH, NSJAIL_PATH, PATH_ENV, TZ_ENV,
@@ -69,6 +70,7 @@ pub async fn gen_bun_lockfile(
     export_pkg: bool,
     raw_deps: Option<String>,
     npm_mode: bool,
+    occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
 ) -> Result<Option<String>> {
     let common_bun_proc_envs: HashMap<String, String> = get_common_bun_proc_envs(None).await;
 
@@ -125,6 +127,7 @@ pub async fn gen_bun_lockfile(
                 "bun build",
                 None,
                 false,
+                occupancy_metrics,
             )
             .await?;
         } else {
@@ -149,6 +152,7 @@ pub async fn gen_bun_lockfile(
             worker_name,
             common_bun_proc_envs,
             npm_mode,
+            occupancy_metrics,
         )
         .await?;
     } else {
@@ -230,6 +234,7 @@ pub async fn install_bun_lockfile(
     worker_name: &str,
     common_bun_proc_envs: HashMap<String, String>,
     npm_mode: bool,
+    occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
 ) -> Result<()> {
     let mut child_cmd = Command::new(if npm_mode { &*NPM_PATH } else { &*BUN_PATH });
     child_cmd
@@ -296,6 +301,7 @@ pub async fn install_bun_lockfile(
             "bun install",
             None,
             false,
+            occupancy_metrics,
         )
         .await?
     } else {
@@ -436,6 +442,7 @@ pub async fn generate_wrapper_mjs(
     mem_peak: &mut i32,
     canceled_by: &mut Option<CanceledBy>,
     common_bun_proc_envs: &HashMap<String, String>,
+    occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
 ) -> Result<()> {
     let mut child = Command::new(&*BUN_PATH);
     child
@@ -459,6 +466,7 @@ pub async fn generate_wrapper_mjs(
         "bun build",
         timeout,
         false,
+        occupancy_metrics,
     )
     .await?;
     fs::rename(
@@ -479,6 +487,7 @@ pub async fn generate_bun_bundle(
     mem_peak: &mut i32,
     canceled_by: &mut Option<CanceledBy>,
     common_bun_proc_envs: &HashMap<String, String>,
+    occupancy_metrics: &mut OccupancyMetrics,
 ) -> Result<()> {
     let mut child = Command::new(&*BUN_PATH);
     child
@@ -503,6 +512,7 @@ pub async fn generate_bun_bundle(
             "bun build",
             timeout,
             false,
+            &mut Some(occupancy_metrics),
         )
         .await?;
     } else {
@@ -609,6 +619,7 @@ pub async fn prebundle_bun_script(
     base_internal_url: &str,
     worker_name: &str,
     token: &str,
+    occupancy_metrics: &mut OccupancyMetrics,
 ) -> Result<()> {
     let (local_path, remote_path) = compute_bundle_local_and_remote_path(
         inner_content,
@@ -656,6 +667,7 @@ pub async fn prebundle_bun_script(
         &mut 0,
         &mut None,
         &common_bun_proc_envs,
+        occupancy_metrics,
     )
     .await?;
 
@@ -751,6 +763,7 @@ pub async fn handle_bun_job(
     envs: HashMap<String, String>,
     shared_mount: &str,
     new_args: &mut Option<HashMap<String, Box<RawValue>>>,
+    occupancy_metrics: &mut OccupancyMetrics,
 ) -> error::Result<Box<RawValue>> {
     let mut annotation = windmill_common::worker::get_annotation(inner_content);
 
@@ -861,6 +874,7 @@ pub async fn handle_bun_job(
                     worker_name,
                     common_bun_proc_envs.clone(),
                     annotation.npm_mode,
+                    &mut Some(occupancy_metrics),
                 )
                 .await?;
 
@@ -888,7 +902,6 @@ pub async fn handle_bun_job(
         // if !*DISABLE_NSJAIL || !empty_trusted_deps || has_custom_config_registry {
         let logs1 = "\n\n--- BUN INSTALL ---\n".to_string();
         append_logs(&job.id, &job.workspace_id, logs1, db).await;
-
         let _ = gen_bun_lockfile(
             mem_peak,
             canceled_by,
@@ -903,6 +916,7 @@ pub async fn handle_bun_job(
             false,
             None,
             annotation.npm_mode,
+            &mut Some(occupancy_metrics),
         )
         .await?;
 
@@ -1128,6 +1142,7 @@ try {{
                 mem_peak,
                 canceled_by,
                 &common_bun_proc_envs,
+                occupancy_metrics,
             )
             .await?;
             if !local_path.is_empty() {
@@ -1169,6 +1184,7 @@ try {{
                 mem_peak,
                 canceled_by,
                 &common_bun_proc_envs,
+                &mut Some(occupancy_metrics),
             )
             .await?;
         }
@@ -1204,6 +1220,7 @@ try {{
             worker_name,
             &job.workspace_id,
             false,
+            occupancy_metrics,
         )
         .await?;
         tracing::info!(
@@ -1360,6 +1377,7 @@ try {{
         "bun run",
         job.timeout,
         false,
+        &mut Some(occupancy_metrics),
     )
     .await?;
 
@@ -1501,6 +1519,7 @@ pub async fn start_worker(
                 worker_name,
                 common_bun_proc_envs.clone(),
                 annotation.npm_mode,
+                &mut None,
             )
             .await?;
             tracing::info!("dedicated worker requirements installed: {reqs}");
@@ -1521,6 +1540,7 @@ pub async fn start_worker(
             false,
             None,
             annotation.npm_mode,
+            &mut None,
         )
         .await?;
     }
@@ -1617,6 +1637,7 @@ for await (const line of Readline.createInterface({{ input: process.stdin }})) {
             &mut mem_peak,
             &mut canceled_by,
             &common_bun_proc_envs,
+            &mut None,
         )
         .await?;
     }

backend/windmill-worker/src/common.rs

@@ -1,15 +1,7 @@
 use async_recursion::async_recursion;
-use deno_ast::swc::parser::lexer::util::CharExt;
-use futures::Future;
+
 use itertools::Itertools;
 
-#[cfg(any(target_os = "linux", target_os = "macos"))]
-use nix::sys::signal::{self, Signal};
-#[cfg(any(target_os = "linux", target_os = "macos"))]
-use nix::unistd::Pid;
-
-#[cfg(all(feature = "enterprise", feature = "parquet"))]
-use object_store::path::Path;
 use regex::Regex;
 use serde::{Deserialize, Serialize};
 use serde_json::value::RawValue;
@@ -18,18 +10,15 @@ use sqlx::types::Json;
 use sqlx::{Pool, Postgres};
 use tokio::process::Command;
 use tokio::{fs::File, io::AsyncReadExt};
-use windmill_common::error::to_anyhow;
 use windmill_common::jobs::ENTRYPOINT_OVERRIDE;
-#[cfg(all(feature = "enterprise", feature = "parquet"))]
-use windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS;
+
 #[cfg(feature = "parquet")]
 use windmill_common::s3_helpers::{
     get_etag_or_empty, LargeFileStorage, ObjectStoreResource, S3Object,
 };
 use windmill_common::variables::{build_crypt_with_key_suffix, decrypt_value_with_mc};
 use windmill_common::worker::{
-    get_windmill_memory_usage, get_worker_memory_usage, to_raw_value, write_file, CLOUD_HOSTED,
-    ROOT_CACHE_DIR, TMP_DIR, WORKER_CONFIG,
+    to_raw_value, write_file, CLOUD_HOSTED, ROOT_CACHE_DIR, WORKER_CONFIG,
 };
 use windmill_common::{
     error::{self, Error},
@@ -38,43 +27,21 @@ use windmill_common::{
 };
 
 use anyhow::{anyhow, Result};
-use windmill_queue::{append_logs, CanceledBy};
 
-#[cfg(any(target_os = "linux", target_os = "macos"))]
-use std::os::unix::process::ExitStatusExt;
-
-use std::process::ExitStatus;
-use std::sync::atomic::AtomicU32;
-use std::sync::Arc;
 use std::{
     collections::{hash_map::DefaultHasher, HashMap},
     hash::{Hash, Hasher},
-    io, panic,
     time::Duration,
 };
 
-use tracing::{trace_span, Instrument};
 use uuid::Uuid;
 use windmill_common::{variables, DB};
 
-#[cfg(feature = "enterprise")]
-use windmill_common::job_metrics;
-
-use tokio::{
-    io::{AsyncBufReadExt, AsyncWriteExt, BufReader},
-    process::Child,
-    sync::{broadcast, watch},
-    time::{interval, sleep, Instant, MissedTickBehavior},
-};
-
-use futures::{
-    future::{self, ready, FutureExt},
-    stream, StreamExt,
-};
+use tokio::{io::AsyncWriteExt, process::Child, time::Instant};
 
 use crate::{
     AuthedClient, AuthedClientBackgroundTask, JOB_DEFAULT_TIMEOUT, MAX_RESULT_SIZE,
-    MAX_TIMEOUT_DURATION, MAX_WAIT_FOR_SIGINT, MAX_WAIT_FOR_SIGTERM,
+    MAX_TIMEOUT_DURATION,
 };
 
 pub async fn build_args_map<'a>(
@@ -466,36 +433,6 @@ pub fn get_main_override(args: Option<&Json<HashMap<String, Box<RawValue>>>>) ->
         .flatten();
 }
 
-async fn get_mem_peak(pid: Option<u32>, nsjail: bool) -> i32 {
-    if pid.is_none() {
-        return -1;
-    }
-    let pid = if nsjail {
-        // This is a bit hacky, but the process id of the nsjail process is the pid of nsjail + 1.
-        // Ideally, we would get the number from fork() itself. This works in MOST cases.
-        pid.unwrap() + 1
-    } else {
-        pid.unwrap()
-    };
-
-    if let Ok(file) = File::open(format!("/proc/{}/status", pid)).await {
-        let mut lines = BufReader::new(file).lines();
-        while let Some(line) = lines.next_line().await.unwrap_or(None) {
-            if line.starts_with("VmHWM:") {
-                return line
-                    .split_whitespace()
-                    .nth(1)
-                    .and_then(|s| s.parse::<i32>().ok())
-                    .unwrap_or(-1);
-            };
-        }
-        -2
-    } else {
-        // rand::random::<i32>() % 100 // to remove - used to fake memory data on MacOS
-        -3
-    }
-}
-
 pub fn sizeof_val(v: &serde_json::Value) -> usize {
     std::mem::size_of::<serde_json::Value>()
         + match v {
@@ -516,63 +453,6 @@ pub fn sizeof_val(v: &serde_json::Value) -> usize {
         }
 }
 
-pub async fn run_future_with_polling_update_job_poller<Fut, T>(
-    job_id: Uuid,
-    timeout: Option<i32>,
-    db: &DB,
-    mem_peak: &mut i32,
-    canceled_by_ref: &mut Option<CanceledBy>,
-    result_f: Fut,
-    worker_name: &str,
-    w_id: &str,
-) -> error::Result<T>
-where
-    Fut: Future<Output = anyhow::Result<T>>,
-{
-    let (tx, rx) = broadcast::channel::<()>(3);
-
-    let update_job = update_job_poller(
-        job_id,
-        db,
-        mem_peak,
-        canceled_by_ref,
-        || async { 0 },
-        worker_name,
-        w_id,
-        rx,
-    );
-
-    let timeout_ms = u64::try_from(
-        resolve_job_timeout(&db, &w_id, job_id, timeout)
-            .await
-            .0
-            .as_millis(),
-    )
-    .unwrap_or(200000);
-
-    let rows = tokio::select! {
-        biased;
-        result = tokio::time::timeout(std::time::Duration::from_millis(timeout_ms), result_f) => result
-        .map_err(|e| {
-            tracing::error!("Query timeout: {}", e);
-            Error::ExecutionErr(format!("Query timeout after (>{}s)", timeout_ms/1000))
-        })?,
-        ex = update_job, if job_id != Uuid::nil() => {
-            match ex {
-                UpdateJobPollingExit::Done => Err(Error::ExecutionErr("Job cancelled".to_string())).map_err(to_anyhow)?,
-                UpdateJobPollingExit::AlreadyCompleted => Err(Error::AlreadyCompleted("Job already completed".to_string())).map_err(to_anyhow)?,
-            }
-        }
-    }?;
-    drop(tx);
-    Ok(rows)
-}
-
-pub enum UpdateJobPollingExit {
-    Done,
-    AlreadyCompleted,
-}
-
 pub async fn update_worker_ping_for_failed_init_script(
     db: &DB,
     worker_name: &str,
@@ -594,672 +474,83 @@ pub async fn update_worker_ping_for_failed_init_script(
         tracing::error!("Error updating worker ping for failed init script: {e:?}");
     }
 }
+pub struct OccupancyMetrics {
+    pub running_job_started_at: Option<Instant>,
+    pub total_duration_of_running_jobs: f32,
+    pub worker_occupancy_rate_history: Vec<(f32, f32)>,
+    pub start_time: Instant,
+}
 
-pub async fn update_job_poller<F, Fut>(
-    job_id: Uuid,
-    db: &DB,
-    mem_peak: &mut i32,
-    canceled_by_ref: &mut Option<CanceledBy>,
-    get_mem: F,
-    worker_name: &str,
-    w_id: &str,
-    mut rx: broadcast::Receiver<()>,
-) -> UpdateJobPollingExit
-where
-    F: Fn() -> Fut,
-    Fut: Future<Output = i32>,
-{
-    let update_job_interval = Duration::from_millis(500);
+impl OccupancyMetrics {
+    pub fn new(start_time: Instant) -> Self {
+        OccupancyMetrics {
+            running_job_started_at: None,
+            total_duration_of_running_jobs: 0.0,
+            worker_occupancy_rate_history: Vec::new(),
+            start_time,
+        }
+    }
 
-    let db = db.clone();
+    pub fn update_occupancy_metrics(&mut self) -> (f32, Option<f32>, Option<f32>, Option<f32>) {
+        let metrics = self;
+        let current_occupied_duration = metrics
+            .running_job_started_at
+            .map(|started_at| started_at.elapsed().as_secs_f32())
+            .unwrap_or(0.0);
+        let total_occupation = metrics.total_duration_of_running_jobs + current_occupied_duration;
 
-    let mut interval = interval(update_job_interval);
-    interval.set_missed_tick_behavior(MissedTickBehavior::Skip);
+        let elapsed = metrics.start_time.elapsed().as_secs_f32();
 
-    let mut i = 0;
-
-    #[cfg(feature = "enterprise")]
-    let mut memory_metric_id: Result<String, Error> =
-        Err(Error::NotFound("not yet initialized".to_string()));
-
-    loop {
-        tokio::select!(
-            _ = rx.recv() => break,
-            _ = interval.tick() => {
-                // update the last_ping column every 5 seconds
-                i+=1;
-                if i == 1 || i % 10 == 0 {
-                    let memory_usage = get_worker_memory_usage();
-                    let wm_memory_usage = get_windmill_memory_usage();
-                    tracing::info!("job {job_id} on {worker_name} in {w_id} worker memory snapshot {}kB/{}kB", memory_usage.unwrap_or_default()/1024, wm_memory_usage.unwrap_or_default()/1024);
-                    if job_id != Uuid::nil() {
-                        sqlx::query!(
-                            "UPDATE worker_ping SET ping_at = now(), current_job_id = $1, current_job_workspace_id = $2, memory_usage = $3, wm_memory_usage = $4 WHERE worker = $5",
-                            &job_id,
-                            &w_id,
-                            memory_usage,
-                            wm_memory_usage,
-                            &worker_name
-                        )
-                        .execute(&db)
-                        .await
-                        .expect("update worker ping");
-                    }
-                }
-                let current_mem = get_mem().await;
-                if current_mem > *mem_peak {
-                    *mem_peak = current_mem
-                }
-                tracing::info!("job {job_id} on {worker_name} in {w_id} still running.  mem: {current_mem}kB, peak mem: {mem_peak}kB");
-
-
-                let update_job_row = i == 2 || (!*SLOW_LOGS && (i < 20 || (i < 120 && i % 5 == 0) || i % 10 == 0)) || i % 20 == 0;
-                if update_job_row {
-                #[cfg(feature = "enterprise")]
+        let (occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m) =
+            if !metrics.worker_occupancy_rate_history.is_empty() {
+                let mut total_occupation_15s = 0.0;
+                let mut total_occupation_5m = 0.0;
+                let mut total_occupation_30m = 0.0;
+                let mut index30m = 0;
+                for (i, (past_total_occupation, time)) in
+                    metrics.worker_occupancy_rate_history.iter().enumerate()
                 {
-                    if job_id != Uuid::nil() {
-
-                        // tracking metric starting at i >= 2 b/c first point it useless and we don't want to track metric for super fast jobs
-                        if i == 2 {
-                            memory_metric_id = job_metrics::register_metric_for_job(
-                                &db,
-                                w_id.to_string(),
-                                job_id,
-                                "memory_kb".to_string(),
-                                job_metrics::MetricKind::TimeseriesInt,
-                                Some("Job Memory Footprint (kB)".to_string()),
-                            )
-                            .await;
-                        }
-                        if let Ok(ref metric_id) = memory_metric_id {
-                            if let Err(err) = job_metrics::record_metric(&db, w_id.to_string(), job_id, metric_id.to_owned(), job_metrics::MetricNumericValue::Integer(current_mem)).await {
-                                tracing::error!("Unable to save memory stat for job {} in workspace {}. Error was: {:?}", job_id, w_id, err);
-                            }
-                        }
+                    let diff = elapsed - time;
+                    if diff < 1800.0 && total_occupation_30m == 0.0 {
+                        total_occupation_30m = (total_occupation - past_total_occupation) / diff;
+                        index30m = i;
+                    }
+                    if diff < 300.0 && total_occupation_5m == 0.0 {
+                        total_occupation_5m = (total_occupation - past_total_occupation) / diff;
+                    }
+                    if diff < 15.0 {
+                        total_occupation_15s = (total_occupation - past_total_occupation) / diff;
+                        break;
                     }
                 }
-                if job_id != Uuid::nil() {
-                    let (canceled, canceled_by, canceled_reason, already_completed) = sqlx::query_as::<_, (bool, Option<String>, Option<String>, bool)>("UPDATE queue SET mem_peak = $1, last_ping = now() WHERE id = $2 RETURNING canceled, canceled_by, canceled_reason, false")
-                        .bind(*mem_peak)
-                        .bind(job_id)
-                        .fetch_optional(&db)
-                        .await
-                        .unwrap_or_else(|e| {
-                            tracing::error!(%e, "error updating job {job_id}: {e:#}");
-                            Some((false, None, None, false))
-                        })
-                        .unwrap_or_else(|| {
-                            // if the job is not in queue, it can only be in the completed_job so it is already complete
-                            (false, None, None, true)
-                        });
-                    if already_completed {
-                        return UpdateJobPollingExit::AlreadyCompleted
-                    }
-                    if canceled {
-                        canceled_by_ref.replace(CanceledBy {
-                            username: canceled_by.clone(),
-                            reason: canceled_reason.clone(),
-                        });
-                        break
-                    }
-                }
-            }
-            },
-        );
-    }
-    tracing::info!("job {job_id} finished");
 
-    UpdateJobPollingExit::Done
-}
+                //drop all elements before the oldest one in 30m windows
+                metrics.worker_occupancy_rate_history.drain(..index30m);
 
-pub enum CompactLogs {
-    NotEE,
-    NoS3,
-    S3,
-}
-
-async fn compact_logs(
-    job_id: Uuid,
-    w_id: &str,
-    db: &DB,
-    nlogs: String,
-    total_size: Arc<AtomicU32>,
-    compact_kind: CompactLogs,
-    _worker_name: &str,
-) -> error::Result<(String, String)> {
-    let mut prev_logs = sqlx::query_scalar!(
-        "SELECT logs FROM job_logs WHERE job_id = $1 AND workspace_id = $2",
-        job_id,
-        w_id
-    )
-    .fetch_optional(db)
-    .await?
-    .flatten()
-    .unwrap_or_default();
-    let size = prev_logs.char_indices().count() as i32;
-    let nlogs_len = nlogs.char_indices().count();
-    let to_keep_in_db = usize::max(
-        usize::min(nlogs_len, 3000),
-        nlogs_len % LARGE_LOG_THRESHOLD_SIZE,
-    );
-    let extra_split = to_keep_in_db < nlogs_len;
-    let stored_in_storage_len = if extra_split {
-        nlogs_len - to_keep_in_db
-    } else {
-        0
-    };
-    let extra_to_newline = nlogs
-        .chars()
-        .skip(stored_in_storage_len)
-        .find_position(|x| x.is_line_break())
-        .map(|(i, _)| i)
-        .unwrap_or(to_keep_in_db);
-    let stored_in_storage_to_newline = stored_in_storage_len + extra_to_newline;
-
-    let (append_to_storage, stored_in_db) = if extra_split {
-        if stored_in_storage_to_newline == nlogs.len() {
-            (nlogs.as_ref(), "".to_string())
-        } else {
-            let split_idx = nlogs
-                .char_indices()
-                .nth(stored_in_storage_to_newline)
-                .map(|(i, _)| i)
-                .unwrap_or(0);
-            let (append_to_storage, stored_in_db) = nlogs.split_at(split_idx);
-            // tracing::error!("{append_to_storage} ||||| {stored_in_db}");
-            // tracing::error!(
-            //     "{:?} {:?} {} {}",
-            //     excess_prev_logs.lines().last(),
-            //     current_logs.lines().next(),
-            //     split_idx,
-            //     excess_size_modulo
-            // );
-            (append_to_storage, stored_in_db.to_string())
-        }
-    } else {
-        // tracing::error!("{:?}", nlogs.lines().last());
-        ("", nlogs.to_string())
-    };
-
-    let new_size_with_excess = size + stored_in_storage_to_newline as i32;
-
-    let new_size = total_size.fetch_add(
-        new_size_with_excess as u32,
-        std::sync::atomic::Ordering::SeqCst,
-    ) + new_size_with_excess as u32;
-
-    let path = format!(
-        "logs/{job_id}/{}_{new_size}.txt",
-        chrono::Utc::now().timestamp_millis()
-    );
-
-    let mut new_current_logs = match compact_kind {
-        CompactLogs::NoS3 => format!("\n[windmill] No object storage set in instance settings. Previous logs have been saved to disk at {path}"),
-        CompactLogs::S3 => format!("\n[windmill] Previous logs have been saved to object storage at {path}"),
-        CompactLogs::NotEE => format!("\n[windmill] Previous logs have been saved to disk at {path}"),
-    };
-    new_current_logs.push_str(&stored_in_db);
-
-    sqlx::query!(
-        "UPDATE job_logs SET logs = $1, log_offset = $2, 
-        log_file_index = array_append(coalesce(log_file_index, array[]::text[]), $3) 
-        WHERE workspace_id = $4 AND job_id = $5",
-        new_current_logs,
-        new_size as i32,
-        path,
-        w_id,
-        job_id
-    )
-    .execute(db)
-    .await?;
-    prev_logs.push_str(&append_to_storage);
-
-    return Ok((prev_logs, path));
-}
-
-async fn default_disk_log_storage(
-    job_id: Uuid,
-    w_id: &str,
-    db: &DB,
-    nlogs: String,
-    total_size: Arc<AtomicU32>,
-    compact_kind: CompactLogs,
-    worker_name: &str,
-) {
-    match compact_logs(
-        job_id,
-        &w_id,
-        &db,
-        nlogs,
-        total_size,
-        compact_kind,
-        worker_name,
-    )
-    .await
-    {
-        Err(e) => tracing::error!("Could not compact logs for job {job_id}: {e:?}",),
-        Ok((prev_logs, path)) => {
-            let path = format!("{}/{}", TMP_DIR, path);
-            let splitted = &path.split("/").collect_vec();
-            tokio::fs::create_dir_all(splitted.into_iter().take(splitted.len() - 1).join("/"))
-                .await
-                .map_err(|e| {
-                    tracing::error!("Could not create logs directory: {e:?}",);
-                    e
-                })
-                .ok();
-            let created = tokio::fs::File::create(&path).await;
-            if let Err(e) = created {
-                tracing::error!("Could not create logs file {path}: {e:?}",);
-                return;
-            }
-            if let Err(e) = tokio::fs::write(&path, prev_logs).await {
-                tracing::error!("Could not write to logs file {path}: {e:?}");
-            } else {
-                tracing::info!("Logs length of {job_id} has exceeded a threshold. Previous logs have been saved to disk at {path}");
-            }
-        }
-    }
-}
-
-async fn append_job_logs(
-    job_id: Uuid,
-    w_id: String,
-    logs: String,
-    db: DB,
-    must_compact_logs: bool,
-    total_size: Arc<AtomicU32>,
-    worker_name: String,
-) -> () {
-    if must_compact_logs {
-        #[cfg(all(feature = "enterprise", feature = "parquet"))]
-        if let Some(os) = OBJECT_STORE_CACHE_SETTINGS.read().await.clone() {
-            match compact_logs(
-                job_id,
-                &w_id,
-                &db,
-                logs,
-                total_size,
-                CompactLogs::S3,
-                &worker_name,
-            )
-            .await
-            {
-                Err(e) => tracing::error!("Could not compact logs for job {job_id}: {e:?}",),
-                Ok((prev_logs, path)) => {
-                    tracing::info!("Logs length of {job_id} has exceeded a threshold. Previous logs have been saved to object storage at {path}");
-                    let path2 = path.clone();
-                    if let Err(e) = os
-                        .put(&Path::from(path), prev_logs.to_string().into_bytes().into())
-                        .await
-                    {
-                        tracing::error!("Could not save logs to s3: {e:?}");
-                    }
-                    tracing::info!("Logs of {job_id} saved to object storage at {path2}");
-                }
-            }
-        } else {
-            default_disk_log_storage(
-                job_id,
-                &w_id,
-                &db,
-                logs,
-                total_size,
-                CompactLogs::NoS3,
-                &worker_name,
-            )
-            .await;
-        }
-
-        #[cfg(not(all(feature = "enterprise", feature = "parquet")))]
-        {
-            default_disk_log_storage(
-                job_id,
-                &w_id,
-                &db,
-                logs,
-                total_size,
-                CompactLogs::NotEE,
-                &worker_name,
-            )
-            .await;
-        }
-    } else {
-        append_logs(&job_id, w_id, logs, db).await;
-    }
-}
-
-pub const LARGE_LOG_THRESHOLD_SIZE: usize = 9000;
-/// - wait until child exits and return with exit status
-/// - read lines from stdout and stderr and append them to the "queue"."logs"
-///   quitting early if output exceedes MAX_LOG_SIZE characters (not bytes)
-/// - update the `last_line` and `logs` strings with the program output
-/// - update "queue"."last_ping" every five seconds
-/// - kill process if we exceed timeout or "queue"."canceled" is set
-#[tracing::instrument(level = "trace", skip_all)]
-pub async fn handle_child(
-    job_id: &Uuid,
-    db: &Pool<Postgres>,
-    mem_peak: &mut i32,
-    canceled_by_ref: &mut Option<CanceledBy>,
-    mut child: Child,
-    nsjail: bool,
-    worker: &str,
-    w_id: &str,
-    child_name: &str,
-    custom_timeout: Option<i32>,
-    sigterm: bool,
-) -> error::Result<()> {
-    let start = Instant::now();
-
-    let pid = child.id();
-    #[cfg(target_os = "linux")]
-    if let Some(pid) = pid {
-        //set the highest oom priority
-        if let Some(mut file) = File::create(format!("/proc/{pid}/oom_score_adj"))
-            .await
-            .map_err(|e| {
-                tracing::error!("Could not create oom_score_file to pid {pid}: {e:#}");
-                e
-            })
-            .ok()
-        {
-            let _ = file.write_all(b"1000").await;
-            let _ = file.sync_all().await;
-        }
-    } else {
-        tracing::info!("could not get child pid");
-    }
-    let (set_too_many_logs, mut too_many_logs) = watch::channel::<bool>(false);
-    let (tx, rx) = broadcast::channel::<()>(3);
-    let mut rx2 = tx.subscribe();
-
-    let output = child_joined_output_stream(&mut child);
-
-    let job_id = job_id.clone();
-
-    /* the cancellation future is polled on by `wait_on_child` while
-     * waiting for the child to exit normally */
-    let update_job = update_job_poller(
-        job_id,
-        db,
-        mem_peak,
-        canceled_by_ref,
-        || get_mem_peak(pid, nsjail),
-        worker,
-        w_id,
-        rx,
-    );
-
-    #[derive(PartialEq, Debug)]
-    enum KillReason {
-        TooManyLogs,
-        Timeout,
-        Cancelled,
-        AlreadyCompleted,
-    }
-
-    let (timeout_duration, timeout_warn_msg) =
-        resolve_job_timeout(&db, w_id, job_id, custom_timeout).await;
-    if let Some(msg) = timeout_warn_msg {
-        append_logs(&job_id, w_id, msg.as_str(), db).await;
-    }
-
-    /* a future that completes when the child process exits */
-    let wait_on_child = async {
-        let db = db.clone();
-
-        let kill_reason = tokio::select! {
-            biased;
-            result = child.wait() => return result.map(Ok),
-            Ok(()) = too_many_logs.changed() => KillReason::TooManyLogs,
-            _ = sleep(timeout_duration) => KillReason::Timeout,
-            ex = update_job, if job_id != Uuid::nil() => match ex {
-                UpdateJobPollingExit::Done => KillReason::Cancelled,
-                UpdateJobPollingExit::AlreadyCompleted => KillReason::AlreadyCompleted,
-            },
-        };
-        tx.send(()).expect("rx should never be dropped");
-        drop(tx);
-
-        let set_reason = async {
-            if kill_reason == KillReason::Timeout {
-                if let Err(err) = sqlx::query(
-                    r#"
-                       UPDATE queue
-                          SET canceled = true
-                            , canceled_by = 'timeout'
-                            , canceled_reason = $1
-                        WHERE id = $2
-                    "#,
+                (
+                    Some(total_occupation_15s),
+                    Some(total_occupation_5m),
+                    Some(total_occupation_30m),
                 )
-                .bind(format!("duration > {}", timeout_duration.as_secs()))
-                .bind(job_id)
-                .execute(&db)
-                .await
-                {
-                    tracing::error!(%job_id, %err, "error setting cancelation reason for job {job_id}: {err}");
-                }
-            }
-        };
-
-        if let Some(id) = child.id() {
-            if *MAX_WAIT_FOR_SIGINT > 0 {
-                #[cfg(any(target_os = "linux", target_os = "macos"))]
-                signal::kill(Pid::from_raw(id as i32), Signal::SIGINT).unwrap();
-
-                for _ in 0..*MAX_WAIT_FOR_SIGINT {
-                    if child.try_wait().is_ok_and(|x| x.is_some()) {
-                        break;
-                    }
-                    sleep(Duration::from_secs(1)).await;
-                }
-                if child.try_wait().is_ok_and(|x| x.is_some()) {
-                    set_reason.await;
-                    return Ok(Err(kill_reason));
-                }
-            }
-            if sigterm {
-                #[cfg(any(target_os = "linux", target_os = "macos"))]
-                signal::kill(Pid::from_raw(id as i32), Signal::SIGTERM).unwrap();
-
-                for _ in 0..*MAX_WAIT_FOR_SIGTERM {
-                    if child.try_wait().is_ok_and(|x| x.is_some()) {
-                        break;
-                    }
-                    sleep(Duration::from_secs(1)).await;
-                }
-                if child.try_wait().is_ok_and(|x| x.is_some()) {
-                    set_reason.await;
-                    return Ok(Err(kill_reason));
-                }
-            }
-        }
-        /* send SIGKILL and reap child process */
-        let (_, kill) = future::join(set_reason, child.kill()).await;
-        kill.map(|()| Err(kill_reason))
-    };
-
-    /* a future that reads output from the child and appends to the database */
-    let lines = async move {
-
-        let max_log_size = if *CLOUD_HOSTED {
-            MAX_RESULT_SIZE
-        } else {
-            usize::MAX
-        };
-
-        /* log_remaining is zero when output limit was reached */
-        let mut log_remaining =  if *CLOUD_HOSTED {
-            max_log_size
-        } else {
-            usize::MAX
-        };
-        let mut result = io::Result::Ok(());
-        let mut output = output.take_until(async {
-            let _ = rx2.recv().await;
-            //wait at most 50ms after end of a script for output stream to end
-            tokio::time::sleep(Duration::from_millis(50)).await;
-         }).boxed();
-        /* `do_write` resolves the task, but does not contain the Result.
-         * It's useful to know if the task completed. */
-        let (mut do_write, mut write_result) = tokio::spawn(ready(())).remote_handle();
-
-        let mut log_total_size: u64 = 0;
-        let pg_log_total_size = Arc::new(AtomicU32::new(0));
-
-        while let Some(line) =  output.by_ref().next().await {
-
-            let do_write_ = do_write.shared();
-
-            let delay = if start.elapsed() < Duration::from_secs(10) {
-                Duration::from_millis(500)
-            } else if start.elapsed() < Duration::from_secs(60){
-                Duration::from_millis(2500)
             } else {
-                Duration::from_millis(5000)
+                (None, None, None)
             };
+        let occupancy_rate = total_occupation / elapsed;
 
-            let delay = if *SLOW_LOGS {
-                delay * 10
-            } else {
-                delay
-            };
+        //push the current occupancy rate and the timestamp
+        metrics
+            .worker_occupancy_rate_history
+            .push((total_occupation, elapsed));
 
-            let mut read_lines = stream::once(async { line })
-                .chain(output.by_ref())
-                /* after receiving a line, continue until some delay has passed
-                 * _and_ the previous database write is complete */
-                .take_until(future::join(sleep(delay), do_write_.clone()))
-                .boxed();
-
-            /* Read up until an error is encountered,
-             * handle log lines first and then the error... */
-            let mut joined = String::new();
-
-            while let Some(line) = read_lines.next().await {
-
-                match line {
-                    Ok(line) => {
-                        if line.is_empty() {
-                            continue;
-                        }
-                        append_with_limit(&mut joined, &line, &mut log_remaining);
-                        if log_remaining == 0 {
-                            tracing::info!(%job_id, "Too many logs lines for job {job_id}");
-                            let _ = set_too_many_logs.send(true);
-                            joined.push_str(&format!(
-                                "Job logs or result reached character limit of {MAX_RESULT_SIZE}; killing job."
-                            ));
-                            /* stop reading and drop our streams fairly quickly */
-                            break;
-                        }
-                    }
-                    Err(err) => {
-                        result = Err(err);
-                        break;
-                    }
-                }
-            }
-
-
-            /* Ensure the last flush completed before starting a new one.
-             *
-             * This shouldn't pause since `take_until()` reads lines until `do_write`
-             * resolves. We only stop reading lines before `take_until()` resolves if we reach
-             * EOF or a read error.  In those cases, waiting on a database query to complete is
-             * fine because we're done. */
-
-            if let Some(Ok(p)) = do_write_
-                .then(|()| write_result)
-                .await
-                .err()
-                .map(|err| err.try_into_panic())
-            {
-                panic::resume_unwind(p);
-            }
-
-
-            let joined_len = joined.len() as u64;
-            log_total_size += joined_len;
-            let compact_logs = log_total_size > LARGE_LOG_THRESHOLD_SIZE as u64;
-            if compact_logs {
-                log_total_size = 0;
-            }
-
-            let worker_name = worker.to_string();
-            let w_id2 = w_id.to_string();
-            (do_write, write_result) = tokio::spawn(append_job_logs(job_id, w_id2, joined, db.clone(), compact_logs, pg_log_total_size.clone(), worker_name)).remote_handle();
-
-
-
-            if let Err(err) = result {
-                tracing::error!(%job_id, %err, "error reading output for job {job_id} '{child_name}': {err}");
-                break;
-            }
-
-            if *set_too_many_logs.borrow() {
-                break;
-            }
-        }
-
-        /* drop our end of the pipe */
-        drop(output);
-
-        if let Some(Ok(p)) = do_write
-            .then(|()| write_result)
-            .await
-            .err()
-            .map(|err| err.try_into_panic())
-        {
-            panic::resume_unwind(p);
-        }
-    }.instrument(trace_span!("child_lines"));
-
-    let (wait_result, _) = tokio::join!(wait_on_child, lines);
-
-    let success = wait_result.is_ok()
-        && wait_result.as_ref().unwrap().is_ok()
-        && wait_result.as_ref().unwrap().as_ref().unwrap().success();
-    tracing::info!(%job_id, %success, %mem_peak, %worker, "child process '{child_name}' took {}ms", start.elapsed().as_millis());
-
-    match wait_result {
-        _ if *too_many_logs.borrow() => Err(Error::ExecutionErr(format!(
-            "logs or result reached limit. (current max size: {MAX_RESULT_SIZE} characters)"
-        ))),
-        Ok(Ok(status)) => process_status(status),
-        Ok(Err(kill_reason)) => match kill_reason {
-            KillReason::AlreadyCompleted => {
-                Err(Error::AlreadyCompleted("Job already completed".to_string()))
-            }
-            _ => Err(Error::ExecutionErr(format!(
-                "job process killed because {kill_reason:#?}"
-            ))),
-        },
-        Err(err) => Err(Error::ExecutionErr(format!("job process io error: {err}"))),
+        (
+            occupancy_rate,
+            occupancy_rate_15s,
+            occupancy_rate_5m,
+            occupancy_rate_30m,
+        )
     }
 }
 
-pub fn process_status(status: ExitStatus) -> error::Result<()> {
-    if status.success() {
-        Ok(())
-    } else if let Some(code) = status.code() {
-        Err(error::Error::ExitStatus(code))
-    } else {
-        #[cfg(any(target_os = "linux", target_os = "macos"))]
-        return Err(error::Error::ExecutionErr(format!(
-            "process terminated by signal: {:#?}, stopped_signal: {:#?}, core_dumped: {}",
-            status.signal(),
-            status.stopped_signal(),
-            status.core_dumped()
-        )));
-
-        #[cfg(not(any(target_os = "linux", target_os = "macos")))]
-        return Err(error::Error::ExecutionErr(String::from(
-            "process terminated by signal",
-        )));
-    }
-}
 pub async fn start_child_process(mut cmd: Command, executable: &str) -> Result<Child, Error> {
     return cmd
         .spawn()
@@ -1324,80 +615,6 @@ pub async fn resolve_job_timeout(
     }
 }
 
-/// takes stdout and stderr from Child, panics if either are not present
-///
-/// builds a stream joining both stdout and stderr each read line by line
-fn child_joined_output_stream(
-    child: &mut Child,
-) -> impl stream::FusedStream<Item = io::Result<String>> {
-    let stderr = child
-        .stderr
-        .take()
-        .expect("child did not have a handle to stdout");
-
-    let stdout = child
-        .stdout
-        .take()
-        .expect("child did not have a handle to stdout");
-
-    let stdout = BufReader::new(stdout).lines();
-    let stderr = BufReader::new(stderr).lines();
-    stream::select(lines_to_stream(stderr), lines_to_stream(stdout))
-}
-
-pub fn lines_to_stream<R: tokio::io::AsyncBufRead + Unpin>(
-    mut lines: tokio::io::Lines<R>,
-) -> impl futures::Stream<Item = io::Result<String>> {
-    stream::poll_fn(move |cx| {
-        std::pin::Pin::new(&mut lines)
-            .poll_next_line(cx)
-            .map(|result| result.transpose())
-    })
-}
-
-lazy_static::lazy_static! {
-    static ref RE_00: Regex = Regex::new('\u{00}'.to_string().as_str()).unwrap();
-    pub static ref NO_LOGS_AT_ALL: bool = std::env::var("NO_LOGS_AT_ALL").ok().is_some_and(|x| x == "1" || x == "true");
-    pub static ref SLOW_LOGS: bool = std::env::var("SLOW_LOGS").ok().is_some_and(|x| x == "1" || x == "true");
-}
-// as a detail, `BufReader::lines()` removes \n and \r\n from the strings it yields,
-// so this pushes \n to thd destination string in each call
-fn append_with_limit(dst: &mut String, src: &str, limit: &mut usize) {
-    if *NO_LOGS_AT_ALL {
-        return;
-    }
-    let src_str;
-    let src = {
-        src_str = RE_00.replace_all(src, "");
-        src_str.as_ref()
-    };
-    if !*CLOUD_HOSTED {
-        dst.push('\n');
-        dst.push_str(&src);
-        return;
-    } else {
-        if *limit > 0 {
-            dst.push('\n');
-        }
-        *limit -= 1;
-    }
-
-    let src_len = src.chars().count();
-    if src_len <= *limit {
-        dst.push_str(&src);
-        *limit -= src_len;
-    } else {
-        let byte_pos = src
-            .char_indices()
-            .skip(*limit)
-            .next()
-            .map(|(byte_pos, _)| byte_pos)
-            .unwrap_or(0);
-        dst.push_str(&src[0..byte_pos]);
-        *limit = 0;
-    }
-}
-
 pub async fn hash_args(
     _db: &DB,
     _client: &AuthedClient,

backend/windmill-worker/src/dedicated_worker.rs

@@ -29,8 +29,7 @@ use windmill_queue::append_logs;
 use anyhow::Context;
 
 use crate::{
-    common::{process_status, start_child_process},
-    JobCompleted, JobCompletedSender, MAX_BUFFERED_DEDICATED_JOBS,
+    common::start_child_process, JobCompleted, JobCompletedSender, MAX_BUFFERED_DEDICATED_JOBS,
 };
 
 use futures::{future, Future};
@@ -78,6 +77,8 @@ pub async fn handle_dedicated_process(
 ) -> std::result::Result<(), error::Error> {
     //do not cache local dependencies
 
+    use crate::handle_child::process_status;
+
     let mut child = {
         let mut cmd = Command::new(command_path);
         cmd.current_dir(job_dir)
@@ -127,7 +128,7 @@ pub async fn handle_dedicated_process(
         if let Err(e) = process_status(status) {
             tracing::error!("child exit status was not success: {e:#}");
         } else {
-            tracing::info!("child exist status was success");
+            tracing::info!("child exit status was success");
         }
     });
 

backend/windmill-worker/src/deno_executor.rs

@@ -7,9 +7,10 @@ use windmill_queue::{append_logs, CanceledBy};
 
 use crate::{
     common::{
-        create_args_and_out_file, get_main_override, get_reserved_variables, handle_child,
-        parse_npm_config, read_file, read_result, start_child_process,
+        create_args_and_out_file, get_main_override, get_reserved_variables, parse_npm_config,
+        read_file, read_result, start_child_process, OccupancyMetrics,
     },
+    handle_child::handle_child,
     AuthedClientBackgroundTask, DENO_CACHE_DIR, DENO_PATH, DISABLE_NSJAIL, HOME_ENV,
     NPM_CONFIG_REGISTRY, PATH_ENV, TZ_ENV,
 };
@@ -94,6 +95,7 @@ pub async fn generate_deno_lock(
     w_id: &str,
     worker_name: &str,
     base_internal_url: &str,
+    occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
 ) -> error::Result<String> {
     let _ = write_file(job_dir, "main.ts", code)?;
 
@@ -146,6 +148,7 @@ pub async fn generate_deno_lock(
             "deno cache",
             None,
             false,
+            occupancy_metrics,
         )
         .await?;
     } else {
@@ -173,6 +176,7 @@ pub async fn handle_deno_job(
     worker_name: &str,
     envs: HashMap<String, String>,
     new_args: &mut Option<HashMap<String, Box<RawValue>>>,
+    occupancy_metrics: &mut OccupancyMetrics,
 ) -> error::Result<Box<RawValue>> {
     // let mut start = Instant::now();
     let logs1 = "\n\n--- DENO CODE EXECUTION ---\n".to_string();
@@ -397,6 +401,7 @@ try {{
         "deno run",
         job.timeout,
         false,
+        &mut Some(occupancy_metrics),
     )
     .await?;
     // logs.push_str(format!("execute: {:?}\n", start.elapsed().as_millis()).as_str());

backend/windmill-worker/src/go_executor.rs

@@ -15,9 +15,10 @@ use windmill_queue::{append_logs, CanceledBy};
 
 use crate::{
     common::{
-        capitalize, create_args_and_out_file, get_reserved_variables, handle_child, read_result,
-        start_child_process,
+        capitalize, create_args_and_out_file, get_reserved_variables, read_result,
+        start_child_process, OccupancyMetrics,
     },
+    handle_child::handle_child,
     AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, GOPRIVATE, GOPROXY,
     GO_BIN_CACHE_DIR, GO_CACHE_DIR, HOME_ENV, NSJAIL_PATH, PATH_ENV, TZ_ENV,
 };
@@ -44,6 +45,7 @@ pub async fn handle_go_job(
     base_internal_url: &str,
     worker_name: &str,
     envs: HashMap<String, String>,
+    occupation_metrics: &mut OccupancyMetrics,
 ) -> Result<Box<RawValue>, Error> {
     //go does not like executing modules at temp root
     let job_dir = &format!("{job_dir}/go");
@@ -88,6 +90,7 @@ pub async fn handle_go_job(
             skip_tidy,
             worker_name,
             &job.workspace_id,
+            occupation_metrics,
         )
         .await?;
 
@@ -202,6 +205,7 @@ func Run(req Req) (interface{{}}, error){{
             "go build",
             None,
             false,
+            &mut Some(occupation_metrics),
         )
         .await?;
 
@@ -297,6 +301,7 @@ func Run(req Req) (interface{{}}, error){{
         "go run",
         job.timeout,
         false,
+        &mut Some(occupation_metrics),
     )
     .await?;
 
@@ -336,6 +341,7 @@ pub async fn install_go_dependencies(
     has_sum: bool,
     worker_name: &str,
     w_id: &str,
+    occupation_metrics: &mut OccupancyMetrics,
 ) -> error::Result<String> {
     if !skip_go_mod {
         gen_go_mymod(code, job_dir).await?;
@@ -359,6 +365,7 @@ pub async fn install_go_dependencies(
             "go init",
             None,
             false,
+            &mut Some(occupation_metrics),
         )
         .await?;
 
@@ -424,9 +431,9 @@ pub async fn install_go_dependencies(
         &format!("go {mod_command}"),
         None,
         false,
+        &mut Some(occupation_metrics),
     )
-    .await
-    .map_err(|e| Error::ExecutionErr(format!("Lockfile generation failed: {e:?}")))?;
+    .await?;
 
     if (!new_lockfile || has_sum) && non_dep_job {
         return Ok("".to_string());

backend/windmill-worker/src/graphql_executor.rs

@@ -12,7 +12,8 @@ use windmill_queue::{CanceledBy, HTTP_CLIENT};
 
 use serde::Deserialize;
 
-use crate::common::run_future_with_polling_update_job_poller;
+use crate::common::OccupancyMetrics;
+use crate::handle_child::run_future_with_polling_update_job_poller;
 use crate::{common::build_args_map, AuthedClientBackgroundTask};
 
 #[derive(Deserialize)]
@@ -41,6 +42,7 @@ pub async fn do_graphql(
     mem_peak: &mut i32,
     canceled_by: &mut Option<CanceledBy>,
     worker_name: &str,
+    occupation_metrics: &mut OccupancyMetrics,
 ) -> windmill_common::error::Result<Box<RawValue>> {
     let args = build_args_map(job, client, db).await?.map(Json);
     let job_args = if args.is_some() {
@@ -151,6 +153,7 @@ pub async fn do_graphql(
         result_f,
         worker_name,
         &job.workspace_id,
+        &mut Some(occupation_metrics),
     )
     .await?;
 

backend/windmill-worker/src/handle_child.rs

@@ -0,0 +1,627 @@
+use futures::Future;
+
+#[cfg(any(target_os = "linux", target_os = "macos"))]
+use nix::sys::signal::{self, Signal};
+#[cfg(any(target_os = "linux", target_os = "macos"))]
+use nix::unistd::Pid;
+
+use sqlx::{Pool, Postgres};
+use tokio::fs::File;
+use windmill_common::error::to_anyhow;
+
+use windmill_common::error::{self, Error};
+
+use windmill_common::worker::{get_windmill_memory_usage, get_worker_memory_usage, CLOUD_HOSTED};
+
+use windmill_queue::{append_logs, CanceledBy};
+
+#[cfg(any(target_os = "linux", target_os = "macos"))]
+use std::os::unix::process::ExitStatusExt;
+
+use std::process::ExitStatus;
+use std::sync::atomic::AtomicU32;
+use std::sync::Arc;
+use std::{io, panic, time::Duration};
+
+use tracing::{trace_span, Instrument};
+use uuid::Uuid;
+use windmill_common::DB;
+
+#[cfg(feature = "enterprise")]
+use windmill_common::job_metrics;
+
+use tokio::{
+    io::{AsyncBufReadExt, AsyncWriteExt, BufReader},
+    process::Child,
+    sync::{broadcast, watch},
+    time::{interval, sleep, Instant, MissedTickBehavior},
+};
+
+use futures::{
+    future::{self, ready, FutureExt},
+    stream, StreamExt,
+};
+
+use crate::common::{resolve_job_timeout, OccupancyMetrics};
+use crate::job_logger::{append_job_logs, append_with_limit, LARGE_LOG_THRESHOLD_SIZE};
+use crate::{MAX_RESULT_SIZE, MAX_WAIT_FOR_SIGINT, MAX_WAIT_FOR_SIGTERM};
+
+lazy_static::lazy_static! {
+    pub static ref SLOW_LOGS: bool = std::env::var("SLOW_LOGS").ok().is_some_and(|x| x == "1" || x == "true");
+}
+/// - wait until child exits and return with exit status
+/// - read lines from stdout and stderr and append them to the "queue"."logs"
+///   quitting early if output exceedes MAX_LOG_SIZE characters (not bytes)
+/// - update the `last_line` and `logs` strings with the program output
+/// - update "queue"."last_ping" every five seconds
+/// - kill process if we exceed timeout or "queue"."canceled" is set
+#[tracing::instrument(level = "trace", skip_all)]
+pub async fn handle_child(
+    job_id: &Uuid,
+    db: &Pool<Postgres>,
+    mem_peak: &mut i32,
+    canceled_by_ref: &mut Option<CanceledBy>,
+    mut child: Child,
+    nsjail: bool,
+    worker: &str,
+    w_id: &str,
+    child_name: &str,
+    custom_timeout: Option<i32>,
+    sigterm: bool,
+    occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
+) -> error::Result<()> {
+    let start = Instant::now();
+
+    let pid = child.id();
+    #[cfg(target_os = "linux")]
+    if let Some(pid) = pid {
+        //set the highest oom priority
+        if let Some(mut file) = File::create(format!("/proc/{pid}/oom_score_adj"))
+            .await
+            .map_err(|e| {
+                tracing::error!("Could not create oom_score_file to pid {pid}: {e:#}");
+                e
+            })
+            .ok()
+        {
+            let _ = file.write_all(b"1000").await;
+            let _ = file.sync_all().await;
+        }
+    } else {
+        tracing::info!("could not get child pid");
+    }
+    let (set_too_many_logs, mut too_many_logs) = watch::channel::<bool>(false);
+    let (tx, rx) = broadcast::channel::<()>(3);
+    let mut rx2 = tx.subscribe();
+
+    let output = child_joined_output_stream(&mut child);
+
+    let job_id = job_id.clone();
+
+    /* the cancellation future is polled on by `wait_on_child` while
+     * waiting for the child to exit normally */
+    let update_job = update_job_poller(
+        job_id,
+        db,
+        mem_peak,
+        canceled_by_ref,
+        || get_mem_peak(pid, nsjail),
+        worker,
+        w_id,
+        rx,
+        occupancy_metrics,
+    );
+
+    #[derive(PartialEq, Debug)]
+    enum KillReason {
+        TooManyLogs,
+        Timeout,
+        Cancelled,
+        AlreadyCompleted,
+    }
+
+    let (timeout_duration, timeout_warn_msg) =
+        resolve_job_timeout(&db, w_id, job_id, custom_timeout).await;
+    if let Some(msg) = timeout_warn_msg {
+        append_logs(&job_id, w_id, msg.as_str(), db).await;
+    }
+
+    /* a future that completes when the child process exits */
+    let wait_on_child = async {
+        let db = db.clone();
+
+        let kill_reason = tokio::select! {
+            biased;
+            result = child.wait() => return result.map(Ok),
+            Ok(()) = too_many_logs.changed() => KillReason::TooManyLogs,
+            _ = sleep(timeout_duration) => KillReason::Timeout,
+            ex = update_job, if job_id != Uuid::nil() => match ex {
+                UpdateJobPollingExit::Done => KillReason::Cancelled,
+                UpdateJobPollingExit::AlreadyCompleted => KillReason::AlreadyCompleted,
+            },
+        };
+        tx.send(()).expect("rx should never be dropped");
+        drop(tx);
+
+        let set_reason = async {
+            if kill_reason == KillReason::Timeout {
+                if let Err(err) = sqlx::query(
+                    r#"
+                       UPDATE queue
+                          SET canceled = true
+                            , canceled_by = 'timeout'
+                            , canceled_reason = $1
+                        WHERE id = $2
+                    "#,
+                )
+                .bind(format!("duration > {}", timeout_duration.as_secs()))
+                .bind(job_id)
+                .execute(&db)
+                .await
+                {
+                    tracing::error!(%job_id, %err, "error setting cancelation reason for job {job_id}: {err}");
+                }
+            }
+        };
+
+        if let Some(id) = child.id() {
+            if *MAX_WAIT_FOR_SIGINT > 0 {
+                #[cfg(any(target_os = "linux", target_os = "macos"))]
+                signal::kill(Pid::from_raw(id as i32), Signal::SIGINT).unwrap();
+
+                for _ in 0..*MAX_WAIT_FOR_SIGINT {
+                    if child.try_wait().is_ok_and(|x| x.is_some()) {
+                        break;
+                    }
+                    sleep(Duration::from_secs(1)).await;
+                }
+                if child.try_wait().is_ok_and(|x| x.is_some()) {
+                    set_reason.await;
+                    return Ok(Err(kill_reason));
+                }
+            }
+            if sigterm {
+                #[cfg(any(target_os = "linux", target_os = "macos"))]
+                signal::kill(Pid::from_raw(id as i32), Signal::SIGTERM).unwrap();
+
+                for _ in 0..*MAX_WAIT_FOR_SIGTERM {
+                    if child.try_wait().is_ok_and(|x| x.is_some()) {
+                        break;
+                    }
+                    sleep(Duration::from_secs(1)).await;
+                }
+                if child.try_wait().is_ok_and(|x| x.is_some()) {
+                    set_reason.await;
+                    return Ok(Err(kill_reason));
+                }
+            }
+        }
+        /* send SIGKILL and reap child process */
+        let (_, kill) = future::join(set_reason, child.kill()).await;
+        kill.map(|()| Err(kill_reason))
+    };
+
+    /* a future that reads output from the child and appends to the database */
+    let lines = async move {
+
+        let max_log_size = if *CLOUD_HOSTED {
+            MAX_RESULT_SIZE
+        } else {
+            usize::MAX
+        };
+
+        /* log_remaining is zero when output limit was reached */
+        let mut log_remaining =  if *CLOUD_HOSTED {
+            max_log_size
+        } else {
+            usize::MAX
+        };
+        let mut result = io::Result::Ok(());
+        let mut output = output.take_until(async {
+            let _ = rx2.recv().await;
+            //wait at most 50ms after end of a script for output stream to end
+            tokio::time::sleep(Duration::from_millis(50)).await;
+         }).boxed();
+        /* `do_write` resolves the task, but does not contain the Result.
+         * It's useful to know if the task completed. */
+        let (mut do_write, mut write_result) = tokio::spawn(ready(())).remote_handle();
+
+        let mut log_total_size: u64 = 0;
+        let pg_log_total_size = Arc::new(AtomicU32::new(0));
+
+        while let Some(line) =  output.by_ref().next().await {
+
+            let do_write_ = do_write.shared();
+
+            let delay = if start.elapsed() < Duration::from_secs(10) {
+                Duration::from_millis(500)
+            } else if start.elapsed() < Duration::from_secs(60){
+                Duration::from_millis(2500)
+            } else {
+                Duration::from_millis(5000)
+            };
+
+            let delay = if *SLOW_LOGS {
+                delay * 10
+            } else {
+                delay
+            };
+
+            let mut read_lines = stream::once(async { line })
+                .chain(output.by_ref())
+                /* after receiving a line, continue until some delay has passed
+                 * _and_ the previous database write is complete */
+                .take_until(future::join(sleep(delay), do_write_.clone()))
+                .boxed();
+
+            /* Read up until an error is encountered,
+             * handle log lines first and then the error... */
+            let mut joined = String::new();
+
+            while let Some(line) = read_lines.next().await {
+
+                match line {
+                    Ok(line) => {
+                        if line.is_empty() {
+                            continue;
+                        }
+                        append_with_limit(&mut joined, &line, &mut log_remaining);
+                        if log_remaining == 0 {
+                            tracing::info!(%job_id, "Too many logs lines for job {job_id}");
+                            let _ = set_too_many_logs.send(true);
+                            joined.push_str(&format!(
+                                "Job logs or result reached character limit of {MAX_RESULT_SIZE}; killing job."
+                            ));
+                            /* stop reading and drop our streams fairly quickly */
+                            break;
+                        }
+                    }
+                    Err(err) => {
+                        result = Err(err);
+                        break;
+                    }
+                }
+            }
+
+
+            /* Ensure the last flush completed before starting a new one.
+             *
+             * This shouldn't pause since `take_until()` reads lines until `do_write`
+             * resolves. We only stop reading lines before `take_until()` resolves if we reach
+             * EOF or a read error.  In those cases, waiting on a database query to complete is
+             * fine because we're done. */
+
+            if let Some(Ok(p)) = do_write_
+                .then(|()| write_result)
+                .await
+                .err()
+                .map(|err| err.try_into_panic())
+            {
+                panic::resume_unwind(p);
+            }
+
+
+            let joined_len = joined.len() as u64;
+            log_total_size += joined_len;
+            let compact_logs = log_total_size > LARGE_LOG_THRESHOLD_SIZE as u64;
+            if compact_logs {
+                log_total_size = 0;
+            }
+
+            let worker_name = worker.to_string();
+            let w_id2 = w_id.to_string();
+            (do_write, write_result) = tokio::spawn(append_job_logs(job_id, w_id2, joined, db.clone(), compact_logs, pg_log_total_size.clone(), worker_name)).remote_handle();
+
+
+
+            if let Err(err) = result {
+                tracing::error!(%job_id, %err, "error reading output for job {job_id} '{child_name}': {err}");
+                break;
+            }
+
+            if *set_too_many_logs.borrow() {
+                break;
+            }
+        }
+
+        /* drop our end of the pipe */
+        drop(output);
+
+        if let Some(Ok(p)) = do_write
+            .then(|()| write_result)
+            .await
+            .err()
+            .map(|err| err.try_into_panic())
+        {
+            panic::resume_unwind(p);
+        }
+    }.instrument(trace_span!("child_lines"));
+
+    let (wait_result, _) = tokio::join!(wait_on_child, lines);
+
+    let success = wait_result.is_ok()
+        && wait_result.as_ref().unwrap().is_ok()
+        && wait_result.as_ref().unwrap().as_ref().unwrap().success();
+    tracing::info!(%job_id, %success, %mem_peak, %worker, "child process '{child_name}' took {}ms", start.elapsed().as_millis());
+
+    match wait_result {
+        _ if *too_many_logs.borrow() => Err(Error::ExecutionErr(format!(
+            "logs or result reached limit. (current max size: {MAX_RESULT_SIZE} characters)"
+        ))),
+        Ok(Ok(status)) => process_status(status),
+        Ok(Err(kill_reason)) => match kill_reason {
+            KillReason::AlreadyCompleted => {
+                Err(Error::AlreadyCompleted("Job already completed".to_string()))
+            }
+            _ => Err(Error::ExecutionErr(format!(
+                "job process killed because {kill_reason:#?}"
+            ))),
+        },
+        Err(err) => Err(Error::ExecutionErr(format!("job process io error: {err}"))),
+    }
+}
+
+async fn get_mem_peak(pid: Option<u32>, nsjail: bool) -> i32 {
+    if pid.is_none() {
+        return -1;
+    }
+    let pid = if nsjail {
+        // This is a bit hacky, but the process id of the nsjail process is the pid of nsjail + 1.
+        // Ideally, we would get the number from fork() itself. This works in MOST cases.
+        pid.unwrap() + 1
+    } else {
+        pid.unwrap()
+    };
+
+    if let Ok(file) = File::open(format!("/proc/{}/status", pid)).await {
+        let mut lines = BufReader::new(file).lines();
+        while let Some(line) = lines.next_line().await.unwrap_or(None) {
+            if line.starts_with("VmHWM:") {
+                return line
+                    .split_whitespace()
+                    .nth(1)
+                    .and_then(|s| s.parse::<i32>().ok())
+                    .unwrap_or(-1);
+            };
+        }
+        -2
+    } else {
+        // rand::random::<i32>() % 100 // to remove - used to fake memory data on MacOS
+        -3
+    }
+}
+
+pub async fn run_future_with_polling_update_job_poller<Fut, T>(
+    job_id: Uuid,
+    timeout: Option<i32>,
+    db: &DB,
+    mem_peak: &mut i32,
+    canceled_by_ref: &mut Option<CanceledBy>,
+    result_f: Fut,
+    worker_name: &str,
+    w_id: &str,
+    occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
+) -> error::Result<T>
+where
+    Fut: Future<Output = anyhow::Result<T>>,
+{
+    let (tx, rx) = broadcast::channel::<()>(3);
+
+    let update_job = update_job_poller(
+        job_id,
+        db,
+        mem_peak,
+        canceled_by_ref,
+        || async { 0 },
+        worker_name,
+        w_id,
+        rx,
+        occupancy_metrics,
+    );
+
+    let timeout_ms = u64::try_from(
+        resolve_job_timeout(&db, &w_id, job_id, timeout)
+            .await
+            .0
+            .as_millis(),
+    )
+    .unwrap_or(200000);
+
+    let rows = tokio::select! {
+        biased;
+        result = tokio::time::timeout(std::time::Duration::from_millis(timeout_ms), result_f) => result
+        .map_err(|e| {
+            tracing::error!("Query timeout: {}", e);
+            Error::ExecutionErr(format!("Query timeout after (>{}s)", timeout_ms/1000))
+        })?,
+        ex = update_job, if job_id != Uuid::nil() => {
+            match ex {
+                UpdateJobPollingExit::Done => Err(Error::ExecutionErr("Job cancelled".to_string())).map_err(to_anyhow)?,
+                UpdateJobPollingExit::AlreadyCompleted => Err(Error::AlreadyCompleted("Job already completed".to_string())).map_err(to_anyhow)?,
+            }
+        }
+    }?;
+    drop(tx);
+    Ok(rows)
+}
+
+pub enum UpdateJobPollingExit {
+    Done,
+    AlreadyCompleted,
+}
+
+pub async fn update_job_poller<F, Fut>(
+    job_id: Uuid,
+    db: &DB,
+    mem_peak: &mut i32,
+    canceled_by_ref: &mut Option<CanceledBy>,
+    get_mem: F,
+    worker_name: &str,
+    w_id: &str,
+    mut rx: broadcast::Receiver<()>,
+    occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
+) -> UpdateJobPollingExit
+where
+    F: Fn() -> Fut,
+    Fut: Future<Output = i32>,
+{
+    let update_job_interval = Duration::from_millis(500);
+
+    let db = db.clone();
+
+    let mut interval = interval(update_job_interval);
+    interval.set_missed_tick_behavior(MissedTickBehavior::Skip);
+
+    let mut i = 0;
+
+    #[cfg(feature = "enterprise")]
+    let mut memory_metric_id: Result<String, Error> =
+        Err(Error::NotFound("not yet initialized".to_string()));
+
+    loop {
+        tokio::select!(
+            _ = rx.recv() => break,
+            _ = interval.tick() => {
+                // update the last_ping column every 5 seconds
+                i+=1;
+                if i == 1 || i % 10 == 0 {
+                    let memory_usage = get_worker_memory_usage();
+                    let wm_memory_usage = get_windmill_memory_usage();
+                    tracing::info!("job {job_id} on {worker_name} in {w_id} worker memory snapshot {}kB/{}kB", memory_usage.unwrap_or_default()/1024, wm_memory_usage.unwrap_or_default()/1024);
+                    let occupancy = occupancy_metrics.as_mut().map(|x| x.update_occupancy_metrics());
+                    if job_id != Uuid::nil() {
+                        sqlx::query!(
+                            "UPDATE worker_ping SET ping_at = now(), current_job_id = $1, current_job_workspace_id = $2, memory_usage = $3, wm_memory_usage = $4,
+                            occupancy_rate = $6, occupancy_rate_15s = $7, occupancy_rate_5m = $8, occupancy_rate_30m = $9 WHERE worker = $5",
+                            &job_id,
+                            &w_id,
+                            memory_usage,
+                            wm_memory_usage,
+                            &worker_name,
+                            occupancy.map(|x| x.0),
+                            occupancy.and_then(|x| x.1),
+                            occupancy.and_then(|x| x.2),
+                            occupancy.and_then(|x| x.3),
+                        )
+                        .execute(&db)
+                        .await
+                        .expect("update worker ping");
+                    }
+                }
+                let current_mem = get_mem().await;
+                if current_mem > *mem_peak {
+                    *mem_peak = current_mem
+                }
+                tracing::info!("job {job_id} on {worker_name} in {w_id} still running.  mem: {current_mem}kB, peak mem: {mem_peak}kB");
+
+
+                let update_job_row = i == 2 || (!*SLOW_LOGS && (i < 20 || (i < 120 && i % 5 == 0) || i % 10 == 0)) || i % 20 == 0;
+                if update_job_row {
+                #[cfg(feature = "enterprise")]
+                {
+                    if job_id != Uuid::nil() {
+
+                        // tracking metric starting at i >= 2 b/c first point it useless and we don't want to track metric for super fast jobs
+                        if i == 2 {
+                            memory_metric_id = job_metrics::register_metric_for_job(
+                                &db,
+                                w_id.to_string(),
+                                job_id,
+                                "memory_kb".to_string(),
+                                job_metrics::MetricKind::TimeseriesInt,
+                                Some("Job Memory Footprint (kB)".to_string()),
+                            )
+                            .await;
+                        }
+                        if let Ok(ref metric_id) = memory_metric_id {
+                            if let Err(err) = job_metrics::record_metric(&db, w_id.to_string(), job_id, metric_id.to_owned(), job_metrics::MetricNumericValue::Integer(current_mem)).await {
+                                tracing::error!("Unable to save memory stat for job {} in workspace {}. Error was: {:?}", job_id, w_id, err);
+                            }
+                        }
+                    }
+                }
+                if job_id != Uuid::nil() {
+                    let (canceled, canceled_by, canceled_reason, already_completed) = sqlx::query_as::<_, (bool, Option<String>, Option<String>, bool)>("UPDATE queue SET mem_peak = $1, last_ping = now() WHERE id = $2 RETURNING canceled, canceled_by, canceled_reason, false")
+                        .bind(*mem_peak)
+                        .bind(job_id)
+                        .fetch_optional(&db)
+                        .await
+                        .unwrap_or_else(|e| {
+                            tracing::error!(%e, "error updating job {job_id}: {e:#}");
+                            Some((false, None, None, false))
+                        })
+                        .unwrap_or_else(|| {
+                            // if the job is not in queue, it can only be in the completed_job so it is already complete
+                            (false, None, None, true)
+                        });
+                    if already_completed {
+                        return UpdateJobPollingExit::AlreadyCompleted
+                    }
+                    if canceled {
+                        canceled_by_ref.replace(CanceledBy {
+                            username: canceled_by.clone(),
+                            reason: canceled_reason.clone(),
+                        });
+                        break
+                    }
+                }
+            }
+            },
+        );
+    }
+    tracing::info!("job {job_id} finished");
+
+    UpdateJobPollingExit::Done
+}
+
+/// takes stdout and stderr from Child, panics if either are not present
+///
+/// builds a stream joining both stdout and stderr each read line by line
+fn child_joined_output_stream(
+    child: &mut Child,
+) -> impl stream::FusedStream<Item = io::Result<String>> {
+    let stderr = child
+        .stderr
+        .take()
+        .expect("child did not have a handle to stdout");
+
+    let stdout = child
+        .stdout
+        .take()
+        .expect("child did not have a handle to stdout");
+
+    let stdout = BufReader::new(stdout).lines();
+    let stderr = BufReader::new(stderr).lines();
+    stream::select(lines_to_stream(stderr), lines_to_stream(stdout))
+}
+
+pub fn lines_to_stream<R: tokio::io::AsyncBufRead + Unpin>(
+    mut lines: tokio::io::Lines<R>,
+) -> impl futures::Stream<Item = io::Result<String>> {
+    stream::poll_fn(move |cx| {
+        std::pin::Pin::new(&mut lines)
+            .poll_next_line(cx)
+            .map(|result| result.transpose())
+    })
+}
+
+pub fn process_status(status: ExitStatus) -> error::Result<()> {
+    if status.success() {
+        Ok(())
+    } else if let Some(code) = status.code() {
+        Err(error::Error::ExitStatus(code))
+    } else {
+        #[cfg(any(target_os = "linux", target_os = "macos"))]
+        return Err(error::Error::ExecutionErr(format!(
+            "process terminated by signal: {:#?}, stopped_signal: {:#?}, core_dumped: {}",
+            status.signal(),
+            status.stopped_signal(),
+            status.core_dumped()
+        )));
+
+        #[cfg(not(any(target_os = "linux", target_os = "macos")))]
+        return Err(error::Error::ExecutionErr(String::from(
+            "process terminated by signal",
+        )));
+    }
+}

backend/windmill-worker/src/job_logger.rs

@@ -0,0 +1,285 @@
+use deno_ast::swc::parser::lexer::util::CharExt;
+use itertools::Itertools;
+
+#[cfg(all(feature = "enterprise", feature = "parquet"))]
+use object_store::path::Path;
+use regex::Regex;
+
+#[cfg(all(feature = "enterprise", feature = "parquet"))]
+use windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS;
+
+use windmill_common::error::{self};
+use windmill_common::worker::{CLOUD_HOSTED, TMP_DIR};
+
+use windmill_queue::append_logs;
+
+use std::sync::atomic::AtomicU32;
+use std::sync::Arc;
+
+use uuid::Uuid;
+use windmill_common::DB;
+
+pub enum CompactLogs {
+    #[cfg(not(all(feature = "enterprise", feature = "parquet")))]
+    NotEE,
+    #[allow(dead_code)]
+    NoS3,
+    #[allow(dead_code)]
+    S3,
+}
+
+async fn compact_logs(
+    job_id: Uuid,
+    w_id: &str,
+    db: &DB,
+    nlogs: String,
+    total_size: Arc<AtomicU32>,
+    compact_kind: CompactLogs,
+    _worker_name: &str,
+) -> error::Result<(String, String)> {
+    let mut prev_logs = sqlx::query_scalar!(
+        "SELECT logs FROM job_logs WHERE job_id = $1 AND workspace_id = $2",
+        job_id,
+        w_id
+    )
+    .fetch_optional(db)
+    .await?
+    .flatten()
+    .unwrap_or_default();
+    let size = prev_logs.char_indices().count() as i32;
+    let nlogs_len = nlogs.char_indices().count();
+    let to_keep_in_db = usize::max(
+        usize::min(nlogs_len, 3000),
+        nlogs_len % LARGE_LOG_THRESHOLD_SIZE,
+    );
+    let extra_split = to_keep_in_db < nlogs_len;
+    let stored_in_storage_len = if extra_split {
+        nlogs_len - to_keep_in_db
+    } else {
+        0
+    };
+    let extra_to_newline = nlogs
+        .chars()
+        .skip(stored_in_storage_len)
+        .find_position(|x| x.is_line_break())
+        .map(|(i, _)| i)
+        .unwrap_or(to_keep_in_db);
+    let stored_in_storage_to_newline = stored_in_storage_len + extra_to_newline;
+
+    let (append_to_storage, stored_in_db) = if extra_split {
+        if stored_in_storage_to_newline == nlogs.len() {
+            (nlogs.as_ref(), "".to_string())
+        } else {
+            let split_idx = nlogs
+                .char_indices()
+                .nth(stored_in_storage_to_newline)
+                .map(|(i, _)| i)
+                .unwrap_or(0);
+            let (append_to_storage, stored_in_db) = nlogs.split_at(split_idx);
+            // tracing::error!("{append_to_storage} ||||| {stored_in_db}");
+            // tracing::error!(
+            //     "{:?} {:?} {} {}",
+            //     excess_prev_logs.lines().last(),
+            //     current_logs.lines().next(),
+            //     split_idx,
+            //     excess_size_modulo
+            // );
+            (append_to_storage, stored_in_db.to_string())
+        }
+    } else {
+        // tracing::error!("{:?}", nlogs.lines().last());
+        ("", nlogs.to_string())
+    };
+
+    let new_size_with_excess = size + stored_in_storage_to_newline as i32;
+
+    let new_size = total_size.fetch_add(
+        new_size_with_excess as u32,
+        std::sync::atomic::Ordering::SeqCst,
+    ) + new_size_with_excess as u32;
+
+    let path = format!(
+        "logs/{job_id}/{}_{new_size}.txt",
+        chrono::Utc::now().timestamp_millis()
+    );
+
+    let mut new_current_logs = match compact_kind {
+        CompactLogs::NoS3 => format!("\n[windmill] No object storage set in instance settings. Previous logs have been saved to disk at {path}"),
+        CompactLogs::S3 => format!("\n[windmill] Previous logs have been saved to object storage at {path}"),
+        #[cfg(not(all(feature = "enterprise", feature = "parquet")))]
+        CompactLogs::NotEE => format!("\n[windmill] Previous logs have been saved to disk at {path}"),
+    };
+    new_current_logs.push_str(&stored_in_db);
+
+    sqlx::query!(
+        "UPDATE job_logs SET logs = $1, log_offset = $2, 
+        log_file_index = array_append(coalesce(log_file_index, array[]::text[]), $3) 
+        WHERE workspace_id = $4 AND job_id = $5",
+        new_current_logs,
+        new_size as i32,
+        path,
+        w_id,
+        job_id
+    )
+    .execute(db)
+    .await?;
+    prev_logs.push_str(&append_to_storage);
+
+    return Ok((prev_logs, path));
+}
+
+async fn default_disk_log_storage(
+    job_id: Uuid,
+    w_id: &str,
+    db: &DB,
+    nlogs: String,
+    total_size: Arc<AtomicU32>,
+    compact_kind: CompactLogs,
+    worker_name: &str,
+) {
+    match compact_logs(
+        job_id,
+        &w_id,
+        &db,
+        nlogs,
+        total_size,
+        compact_kind,
+        worker_name,
+    )
+    .await
+    {
+        Err(e) => tracing::error!("Could not compact logs for job {job_id}: {e:?}",),
+        Ok((prev_logs, path)) => {
+            let path = format!("{}/{}", TMP_DIR, path);
+            let splitted = &path.split("/").collect_vec();
+            tokio::fs::create_dir_all(splitted.into_iter().take(splitted.len() - 1).join("/"))
+                .await
+                .map_err(|e| {
+                    tracing::error!("Could not create logs directory: {e:?}",);
+                    e
+                })
+                .ok();
+            let created = tokio::fs::File::create(&path).await;
+            if let Err(e) = created {
+                tracing::error!("Could not create logs file {path}: {e:?}",);
+                return;
+            }
+            if let Err(e) = tokio::fs::write(&path, prev_logs).await {
+                tracing::error!("Could not write to logs file {path}: {e:?}");
+            } else {
+                tracing::info!("Logs length of {job_id} has exceeded a threshold. Previous logs have been saved to disk at {path}");
+            }
+        }
+    }
+}
+
+pub(crate) async fn append_job_logs(
+    job_id: Uuid,
+    w_id: String,
+    logs: String,
+    db: DB,
+    must_compact_logs: bool,
+    total_size: Arc<AtomicU32>,
+    worker_name: String,
+) -> () {
+    if must_compact_logs {
+        #[cfg(all(feature = "enterprise", feature = "parquet"))]
+        if let Some(os) = OBJECT_STORE_CACHE_SETTINGS.read().await.clone() {
+            match compact_logs(
+                job_id,
+                &w_id,
+                &db,
+                logs,
+                total_size,
+                CompactLogs::S3,
+                &worker_name,
+            )
+            .await
+            {
+                Err(e) => tracing::error!("Could not compact logs for job {job_id}: {e:?}",),
+                Ok((prev_logs, path)) => {
+                    tracing::info!("Logs length of {job_id} has exceeded a threshold. Previous logs have been saved to object storage at {path}");
+                    let path2 = path.clone();
+                    if let Err(e) = os
+                        .put(&Path::from(path), prev_logs.to_string().into_bytes().into())
+                        .await
+                    {
+                        tracing::error!("Could not save logs to s3: {e:?}");
+                    }
+                    tracing::info!("Logs of {job_id} saved to object storage at {path2}");
+                }
+            }
+        } else {
+            default_disk_log_storage(
+                job_id,
+                &w_id,
+                &db,
+                logs,
+                total_size,
+                CompactLogs::NoS3,
+                &worker_name,
+            )
+            .await;
+        }
+
+        #[cfg(not(all(feature = "enterprise", feature = "parquet")))]
+        {
+            default_disk_log_storage(
+                job_id,
+                &w_id,
+                &db,
+                logs,
+                total_size,
+                CompactLogs::NotEE,
+                &worker_name,
+            )
+            .await;
+        }
+    } else {
+        append_logs(&job_id, w_id, logs, db).await;
+    }
+}
+
+pub const LARGE_LOG_THRESHOLD_SIZE: usize = 9000;
+
+lazy_static::lazy_static! {
+    static ref RE_00: Regex = Regex::new('\u{00}'.to_string().as_str()).unwrap();
+    pub static ref NO_LOGS_AT_ALL: bool = std::env::var("NO_LOGS_AT_ALL").ok().is_some_and(|x| x == "1" || x == "true");
+}
+// as a detail, `BufReader::lines()` removes \n and \r\n from the strings it yields,
+// so this pushes \n to thd destination string in each call
+pub fn append_with_limit(dst: &mut String, src: &str, limit: &mut usize) {
+    if *NO_LOGS_AT_ALL {
+        return;
+    }
+    let src_str;
+    let src = {
+        src_str = RE_00.replace_all(src, "");
+        src_str.as_ref()
+    };
+    if !*CLOUD_HOSTED {
+        dst.push('\n');
+        dst.push_str(&src);
+        return;
+    } else {
+        if *limit > 0 {
+            dst.push('\n');
+        }
+        *limit -= 1;
+    }
+
+    let src_len = src.chars().count();
+    if src_len <= *limit {
+        dst.push_str(&src);
+        *limit -= src_len;
+    } else {
+        let byte_pos = src
+            .char_indices()
+            .skip(*limit)
+            .next()
+            .map(|(byte_pos, _)| byte_pos)
+            .unwrap_or(0);
+        dst.push_str(&src[0..byte_pos]);
+        *limit = 0;
+    }
+}

backend/windmill-worker/src/js_eval.rs

@@ -40,7 +40,8 @@ use windmill_common::{error::Error, flow_status::JobResult, DB};
 use windmill_queue::CanceledBy;
 
 use crate::{
-    common::{run_future_with_polling_update_job_poller, unsafe_raw},
+    common::{unsafe_raw, OccupancyMetrics},
+    handle_child::run_future_with_polling_update_job_poller,
     AuthedClient,
 };
 
@@ -688,6 +689,7 @@ pub async fn eval_fetch_timeout(
     worker_name: &str,
     w_id: &str,
     load_client: bool,
+    occupation_metrics: &mut OccupancyMetrics,
 ) -> anyhow::Result<(Box<RawValue>, String)> {
     let (sender, mut receiver) = oneshot::channel::<IsolateHandle>();
 
@@ -836,6 +838,7 @@ pub async fn eval_fetch_timeout(
         async { result_f.await? },
         worker_name,
         w_id,
+        &mut Some(occupation_metrics),
     )
     .await
     .map_err(|e| {