pg_embed

backend/Cargo.lock

@@ -1268,7 +1268,7 @@ dependencies = [
  "hyper 1.8.1",
  "hyper-util",
  "itoa",
- "matchit",
+ "matchit 0.7.3",
  "memchr",
  "mime",
  "multer",
@@ -2823,7 +2823,7 @@ dependencies = [
  "hashbrown 0.14.5",
  "lock_api",
  "once_cell",
- "parking_lot_core",
+ "parking_lot_core 0.9.12",
 ]
 
 [[package]]
@@ -2837,7 +2837,7 @@ dependencies = [
  "hashbrown 0.14.5",
  "lock_api",
  "once_cell",
- "parking_lot_core",
+ "parking_lot_core 0.9.12",
 ]
 
 [[package]]
@@ -2894,7 +2894,7 @@ dependencies = [
  "itertools 0.14.0",
  "log",
  "object_store",
- "parking_lot",
+ "parking_lot 0.12.5",
  "parquet",
  "rand 0.8.5",
  "regex",
@@ -2929,7 +2929,7 @@ dependencies = [
  "itertools 0.14.0",
  "log",
  "object_store",
- "parking_lot",
+ "parking_lot 0.12.5",
  "tokio",
 ]
 
@@ -3102,7 +3102,7 @@ dependencies = [
  "itertools 0.14.0",
  "log",
  "object_store",
- "parking_lot",
+ "parking_lot 0.12.5",
  "parquet",
  "rand 0.8.5",
  "tokio",
@@ -3127,7 +3127,7 @@ dependencies = [
  "futures",
  "log",
  "object_store",
- "parking_lot",
+ "parking_lot 0.12.5",
  "rand 0.8.5",
  "tempfile",
  "url",
@@ -3263,7 +3263,7 @@ dependencies = [
  "datafusion-common",
  "datafusion-expr",
  "datafusion-physical-plan",
- "parking_lot",
+ "parking_lot 0.12.5",
  "paste",
 ]
 
@@ -3404,7 +3404,7 @@ dependencies = [
  "indexmap 2.11.1",
  "itertools 0.14.0",
  "log",
- "parking_lot",
+ "parking_lot 0.12.5",
  "pin-project-lite",
  "tokio",
 ]
@@ -3429,7 +3429,7 @@ dependencies = [
  "itertools 0.14.0",
  "log",
  "object_store",
- "parking_lot",
+ "parking_lot 0.12.5",
  "tokio",
 ]
 
@@ -3558,7 +3558,7 @@ dependencies = [
  "indexmap 2.11.1",
  "log",
  "once_cell",
- "parking_lot",
+ "parking_lot 0.12.5",
  "serde",
  "serde_json",
  "sha2 0.10.9",
@@ -3640,7 +3640,7 @@ dependencies = [
  "indexmap 2.11.1",
  "libc",
  "memoffset",
- "parking_lot",
+ "parking_lot 0.12.5",
  "percent-encoding",
  "pin-project",
  "serde",
@@ -3885,7 +3885,7 @@ dependencies = [
  "log",
  "once_cell",
  "os_pipe",
- "parking_lot",
+ "parking_lot 0.12.5",
  "pin-project",
  "rand 0.8.5",
  "tokio",
@@ -4265,7 +4265,7 @@ dependencies = [
  "log",
  "node_resolver",
  "once_cell",
- "parking_lot",
+ "parking_lot 0.12.5",
  "sys_traits",
  "thiserror 2.0.17",
  "url",
@@ -4422,7 +4422,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6742a724e8becb372a74c650a1aefb8924a5b8107f7d75b3848763ea24b27a87"
 dependencies = [
  "futures-util",
- "parking_lot",
+ "parking_lot 0.12.5",
  "tokio",
 ]
 
@@ -5667,7 +5667,7 @@ checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f"
 dependencies = [
  "futures-core",
  "lock_api",
- "parking_lot",
+ "parking_lot 0.12.5",
 ]
 
 [[package]]
@@ -6565,7 +6565,7 @@ dependencies = [
  "ipconfig",
  "moka",
  "once_cell",
- "parking_lot",
+ "parking_lot 0.12.5",
  "rand 0.9.0",
  "resolv-conf",
  "serde",
@@ -7162,6 +7162,18 @@ dependencies = [
  "generic-array",
 ]
 
+[[package]]
+name = "instant"
+version = "0.1.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222"
+dependencies = [
+ "cfg-if",
+ "js-sys",
+ "wasm-bindgen",
+ "web-sys",
+]
+
 [[package]]
 name = "integer-encoding"
 version = "3.0.4"
@@ -7621,7 +7633,7 @@ dependencies = [
  "json-patch",
  "k8s-openapi",
  "kube-client",
- "parking_lot",
+ "parking_lot 0.12.5",
  "pin-project",
  "serde",
  "serde_json",
@@ -8135,6 +8147,12 @@ version = "0.7.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94"
 
+[[package]]
+name = "matchit"
+version = "0.8.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2f926ade0c4e170215ae43342bf13b9310a437609c81f29f86c5df6657582ef9"
+
 [[package]]
 name = "md-5"
 version = "0.9.1"
@@ -8341,7 +8359,7 @@ dependencies = [
  "crossbeam-epoch",
  "crossbeam-utils",
  "equivalent",
- "parking_lot",
+ "parking_lot 0.12.5",
  "portable-atomic",
  "smallvec",
  "tagptr",
@@ -9059,7 +9077,7 @@ dependencies = [
  "hyper 1.8.1",
  "itertools 0.14.0",
  "md-5 0.10.6",
- "parking_lot",
+ "parking_lot 0.12.5",
  "percent-encoding",
  "quick-xml 0.37.5",
  "rand 0.9.0",
@@ -9473,6 +9491,17 @@ version = "2.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba"
 
+[[package]]
+name = "parking_lot"
+version = "0.11.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99"
+dependencies = [
+ "instant",
+ "lock_api",
+ "parking_lot_core 0.8.6",
+]
+
 [[package]]
 name = "parking_lot"
 version = "0.12.5"
@@ -9480,7 +9509,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a"
 dependencies = [
  "lock_api",
- "parking_lot_core",
+ "parking_lot_core 0.9.12",
+]
+
+[[package]]
+name = "parking_lot_core"
+version = "0.8.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc"
+dependencies = [
+ "cfg-if",
+ "instant",
+ "libc",
+ "redox_syscall 0.2.16",
+ "smallvec",
+ "winapi",
 ]
 
 [[package]]
@@ -9959,6 +10002,64 @@ dependencies = [
  "uuid",
 ]
 
+[[package]]
+name = "postgresql_archive"
+version = "0.18.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6c1e2f490f528e4eb06be6e9494ad28ea2b3f376bac7a9ad7c99b9062a7358e"
+dependencies = [
+ "async-trait",
+ "flate2",
+ "futures-util",
+ "hex",
+ "num-format",
+ "regex-lite",
+ "reqwest 0.12.28",
+ "reqwest-middleware 0.4.2",
+ "reqwest-retry 0.7.0",
+ "reqwest-tracing",
+ "semver 1.0.27",
+ "serde",
+ "serde_json",
+ "sha2 0.10.9",
+ "tar",
+ "target-triple",
+ "tempfile",
+ "thiserror 2.0.17",
+ "tracing",
+ "url",
+]
+
+[[package]]
+name = "postgresql_commands"
+version = "0.18.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "56eae883251fb55f23ff8e9f16121cfb2b00a56ebe62c56183c67904af26fa1d"
+dependencies = [
+ "thiserror 2.0.17",
+ "tracing",
+]
+
+[[package]]
+name = "postgresql_embedded"
+version = "0.18.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "47d56d311cb5868a3d91ba40dab5f87050dfd462ee81c39616735eb1f992cb17"
+dependencies = [
+ "anyhow",
+ "postgresql_archive",
+ "postgresql_commands",
+ "rand 0.9.0",
+ "semver 1.0.27",
+ "sqlx",
+ "target-triple",
+ "tempfile",
+ "thiserror 2.0.17",
+ "tokio",
+ "tracing",
+ "url",
+]
+
 [[package]]
 name = "potential_utf"
 version = "0.1.4"
@@ -10150,7 +10251,7 @@ dependencies = [
  "fnv",
  "lazy_static",
  "memchr",
- "parking_lot",
+ "parking_lot 0.12.5",
  "thiserror 2.0.17",
 ]
 
@@ -10319,7 +10420,7 @@ dependencies = [
  "ahash 0.8.12",
  "equivalent",
  "hashbrown 0.16.1",
- "parking_lot",
+ "parking_lot 0.12.5",
 ]
 
 [[package]]
@@ -10608,6 +10709,15 @@ dependencies = [
  "syn 2.0.114",
 ]
 
+[[package]]
+name = "redox_syscall"
+version = "0.2.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
+dependencies = [
+ "bitflags 1.3.2",
+]
+
 [[package]]
 name = "redox_syscall"
 version = "0.5.18"
@@ -10811,6 +10921,21 @@ dependencies = [
  "web-sys",
 ]
 
+[[package]]
+name = "reqwest-middleware"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "57f17d28a6e6acfe1733fe24bcd30774d13bffa4b8a22535b4c8c98423088d4e"
+dependencies = [
+ "anyhow",
+ "async-trait",
+ "http 1.4.0",
+ "reqwest 0.12.28",
+ "serde",
+ "thiserror 1.0.69",
+ "tower-service",
+]
+
 [[package]]
 name = "reqwest-middleware"
 version = "0.5.0"
@@ -10826,6 +10951,28 @@ dependencies = [
  "tower-service",
 ]
 
+[[package]]
+name = "reqwest-retry"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "29c73e4195a6bfbcb174b790d9b3407ab90646976c55de58a6515da25d851178"
+dependencies = [
+ "anyhow",
+ "async-trait",
+ "futures",
+ "getrandom 0.2.16",
+ "http 1.4.0",
+ "hyper 1.8.1",
+ "parking_lot 0.11.2",
+ "reqwest 0.12.28",
+ "reqwest-middleware 0.4.2",
+ "retry-policies 0.4.0",
+ "thiserror 1.0.69",
+ "tokio",
+ "tracing",
+ "wasm-timer",
+]
+
 [[package]]
 name = "reqwest-retry"
 version = "0.9.0"
@@ -10839,20 +10986,45 @@ dependencies = [
  "http 1.4.0",
  "hyper 1.8.1",
  "reqwest 0.13.1",
- "reqwest-middleware",
+ "reqwest-middleware 0.5.0",
- "retry-policies",
+ "retry-policies 0.5.0",
  "thiserror 2.0.17",
  "tokio",
  "tracing",
  "wasmtimer",
 ]
 
+[[package]]
+name = "reqwest-tracing"
+version = "0.5.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d70ea85f131b2ee9874f0b160ac5976f8af75f3c9badfe0d955880257d10bd83"
+dependencies = [
+ "anyhow",
+ "async-trait",
+ "getrandom 0.2.16",
+ "http 1.4.0",
+ "matchit 0.8.6",
+ "reqwest 0.12.28",
+ "reqwest-middleware 0.4.2",
+ "tracing",
+]
+
 [[package]]
 name = "resolv-conf"
 version = "0.7.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7"
 
+[[package]]
+name = "retry-policies"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5875471e6cab2871bc150ecb8c727db5113c9338cc3354dc5ee3425b6aa40a1c"
+dependencies = [
+ "rand 0.8.5",
+]
+
 [[package]]
 name = "retry-policies"
 version = "0.5.0"
@@ -13401,6 +13573,12 @@ dependencies = [
  "xattr",
 ]
 
+[[package]]
+name = "target-triple"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1ac9aa371f599d22256307c24a9d748c041e548cbf599f35d890f9d365361790"
+
 [[package]]
 name = "tempfile"
 version = "3.24.0"
@@ -13707,7 +13885,7 @@ dependencies = [
  "io-uring",
  "libc",
  "mio 1.1.1",
- "parking_lot",
+ "parking_lot 0.12.5",
  "pin-project-lite",
  "signal-hook-registry",
  "slab",
@@ -13772,7 +13950,7 @@ dependencies = [
  "futures-channel",
  "futures-util",
  "log",
- "parking_lot",
+ "parking_lot 0.12.5",
  "percent-encoding",
  "phf 0.11.3",
  "pin-project-lite",
@@ -13798,7 +13976,7 @@ dependencies = [
  "futures-channel",
  "futures-util",
  "log",
- "parking_lot",
+ "parking_lot 0.12.5",
  "percent-encoding",
  "phf 0.11.3",
  "pin-project-lite",
@@ -14074,7 +14252,7 @@ dependencies = [
  "cookie 0.18.1",
  "futures-util",
  "http 1.4.0",
- "parking_lot",
+ "parking_lot 0.12.5",
  "pin-project-lite",
  "tower-layer",
  "tower-service",
@@ -14952,6 +15130,21 @@ dependencies = [
  "web-sys",
 ]
 
+[[package]]
+name = "wasm-timer"
+version = "0.2.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "be0ecb0db480561e9a7642b5d3e4187c128914e58aa84330b9493e3eb68c5e7f"
+dependencies = [
+ "futures",
+ "js-sys",
+ "parking_lot 0.11.2",
+ "pin-utils",
+ "wasm-bindgen",
+ "wasm-bindgen-futures",
+ "web-sys",
+]
+
 [[package]]
 name = "wasm_dep_analyzer"
 version = "0.2.0"
@@ -14970,7 +15163,7 @@ checksum = "1c598d6b99ea013e35844697fc4670d08339d5cda15588f193c6beedd12f644b"
 dependencies = [
  "futures",
  "js-sys",
- "parking_lot",
+ "parking_lot 0.12.5",
  "pin-utils",
  "slab",
  "wasm-bindgen",
@@ -15048,7 +15241,7 @@ dependencies = [
  "log",
  "naga",
  "once_cell",
- "parking_lot",
+ "parking_lot 0.12.5",
  "profiling",
  "raw-window-handle",
  "ron",
@@ -15090,7 +15283,7 @@ dependencies = [
  "ndk-sys",
  "objc",
  "once_cell",
- "parking_lot",
+ "parking_lot 0.12.5",
  "profiling",
  "range-alloc",
  "raw-window-handle",
@@ -15213,6 +15406,7 @@ dependencies = [
  "object_store",
  "once_cell",
  "pep440_rs",
+ "postgresql_embedded",
  "prometheus",
  "quote",
  "rand 0.9.0",
@@ -15304,7 +15498,7 @@ dependencies = [
  "libxml",
  "magic-crypt",
  "mail-parser",
- "matchit",
+ "matchit 0.7.3",
  "mime_guess",
  "native-tls",
  "nkeys",
@@ -15468,13 +15662,14 @@ dependencies = [
  "phf 0.11.3",
  "pin-project-lite",
  "postgres-native-tls 0.5.1",
+ "postgresql_embedded",
  "prometheus",
  "quick_cache",
  "rand 0.9.0",
  "regex",
  "reqwest 0.13.1",
- "reqwest-middleware",
+ "reqwest-middleware 0.5.0",
- "reqwest-retry",
+ "reqwest-retry 0.9.0",
  "semver 1.0.27",
  "serde",
  "serde_json",
@@ -15916,7 +16111,7 @@ dependencies = [
  "rand 0.9.0",
  "regex",
  "reqwest 0.13.1",
- "reqwest-middleware",
+ "reqwest-middleware 0.5.0",
  "rust_decimal",
  "serde",
  "serde_json",

backend/Cargo.toml

@@ -86,6 +86,7 @@ oauth2 = ["windmill-api/oauth2"]
 zip = ["windmill-api/zip"]
 static_frontend = ["windmill-api/static_frontend"]
 scoped_cache = ["windmill-common/scoped_cache"]
+pg_embed = ["windmill-common/pg_embed", "dep:postgresql_embedded"]
 test_job_debouncing = []
 # Languages
 python = ["windmill-worker/python", "windmill-api/python"]
@@ -161,7 +162,7 @@ k8s-openapi.workspace = true
 libloading.workspace = true
 bitflags.workspace = true
 globset.workspace = true
-
+postgresql_embedded = { version = "0.18.1", optional = true, features = ["theseus"], default-features = false }
 
 [target.'cfg(windows)'.dependencies]
 windows-service = "0.7"

backend/README.md

@@ -14,6 +14,55 @@ contains files used to build the "root" binary.
 | [windmill-worker](./windmill-worker/) | The worker. Used to process and execute flows & jobs.                                                     |
 | [parsers](./parsers/)                 | Contains code to parse signatures in different langauges.                                                 |
 
+## Features
+
+### Embedded PostgreSQL (`pg_embed`)
+
+The `pg_embed` feature allows Windmill to run with an embedded PostgreSQL database, eliminating the need for a separate PostgreSQL instance. This is useful for local development, testing, or single-machine deployments.
+
+**Building with pg_embed:**
+```bash
+cargo build --features pg_embed
+```
+
+**System Dependencies:**
+
+The embedded PostgreSQL requires certain system libraries to be installed:
+
+- **Arch Linux:**
+  ```bash
+  sudo pacman -S libxml2 icu openssl
+  ```
+
+- **Ubuntu/Debian:**
+  ```bash
+  sudo apt-get install libxml2 libicu-dev libssl-dev
+  ```
+
+- **RHEL/Fedora:**
+  ```bash
+  sudo dnf install libxml2 libicu openssl-libs
+  ```
+
+**Usage:**
+
+When the `pg_embed` feature is enabled, if `DATABASE_URL` is not set, Windmill will automatically start an embedded PostgreSQL instance.
+
+You can customize the embedded PostgreSQL behavior with these environment variables:
+- `PG_EMBED_DATA_DIR`: Directory for PostgreSQL data (default: `./postgresql_data`)
+- `PG_EMBED_PORT`: Port for PostgreSQL (default: `5432`)
+- `PG_EMBED_DATABASE`: Database name (default: `windmill`)
+- `PG_EMBED`: Set to any value to force embedded PostgreSQL even if DATABASE_URL exists
+
+**Example:**
+```bash
+# Run with default embedded PostgreSQL
+./windmill
+
+# Or with custom settings
+PG_EMBED_DATA_DIR=/my/data PG_EMBED_PORT=5433 ./windmill
+```
+
 ### Compile sqlx for offline ci
 
 ```

backend/src/main.rs

@@ -332,6 +332,9 @@ fn print_help() {
 	println!("  cache [hubPaths.json]  Pre-cache hub scripts (default: ./hubPaths.json)");
 	println!();
 	println!("Environment variables (name = default):");
+	#[cfg(feature = "pg_embed")]
+	println!("  DATABASE_URL = <optional>              The Postgres database url (auto-generated with pg_embed if not set).");
+	#[cfg(not(feature = "pg_embed"))]
 	println!("  DATABASE_URL = <required>              The Postgres database url.");
 	println!("  MODE = standalone                      Mode: standalone | worker | server | agent");
 	println!("  BASE_URL = http://localhost:8000       Public base URL of your instance (overridden by instance settings)");
@@ -339,16 +342,33 @@ fn print_help() {
 	println!("  SERVER_BIND_ADDR = {}                  IP to bind the server to", DEFAULT_SERVER_BIND_ADDR);
 	println!("  NUM_WORKERS = {}                       Number of workers (standalone/worker modes)", DEFAULT_NUM_WORKERS);
 	println!("  WORKER_GROUP = default                 Worker group this worker belongs to",);
+	println!("  QUIT_AFTER_FIRST_PING = false          Exit worker after first successful ping (useful for testing)");
 	println!("  JSON_FMT = false                       Output logs in JSON instead of logfmt");
 	println!("  METRICS_ADDR = None                    (EE only) Prometheus metrics addr at /metrics; set \"true\" to use :8001");
 	println!("  SUPERADMIN_SECRET = None               Virtual superadmin token (server)");
 	println!("  LICENSE_KEY = None                     (EE only) Enterprise license key (workers require valid key)");
 	println!("  RUN_UPDATE_CA_CERTIFICATE_AT_START = false  Run system CA update at startup");
 	println!("  RUN_UPDATE_CA_CERTIFICATE_PATH = /usr/sbin/update-ca-certificates  Path to CA update tool");
+	#[cfg(feature = "pg_embed")]
+	{
+		println!();
+		println!("Embedded PostgreSQL settings (pg_embed feature):");
+		println!("  PG_EMBED = <unset>                     Force embedded PostgreSQL even if DATABASE_URL exists");
+		println!("  PG_EMBED_DATA_DIR = ./postgresql_data  Directory for embedded PostgreSQL data");
+		println!("  PG_EMBED_PORT = 5432                   Port for embedded PostgreSQL");
+		println!("  PG_EMBED_DATABASE = windmill           Database name for embedded PostgreSQL");
+		println!();
+		println!("System dependencies required for pg_embed:");
+		println!("  Arch: sudo pacman -S libxml2 icu openssl");
+		println!("  Ubuntu/Debian: sudo apt-get install libxml2 libicu-dev libssl-dev");
+		println!("  RHEL/Fedora: sudo dnf install libxml2 libicu openssl-libs");
+	}
 	println!();
 	println!("Notes:");
 	println!("- Advanced and less commonly used settings are managed via the database and are omitted here.");
 	println!("- At startup, Windmill logs currently set configuration keys for visibility.");
+	#[cfg(feature = "pg_embed")]
+	println!("- With pg_embed feature enabled, if DATABASE_URL is not set, an embedded PostgreSQL instance will be started automatically.");
 }
 
 async fn windmill_main() -> anyhow::Result<()> {
@@ -470,6 +490,15 @@ async fn windmill_main() -> anyhow::Result<()> {
         IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1))
     };
 
+    // Initialize embedded PostgreSQL if pg_embed feature is enabled
+    #[cfg(feature = "pg_embed")]
+    let _embedded_pg = if std::env::var("PG_EMBED").is_ok() || (mode != Mode::Agent && std::env::var("DATABASE_URL").is_err() && std::env::var("DATABASE_URL_FILE").is_err()) {
+        println!("DATABASE_URL not set, starting embedded PostgreSQL...");
+        Some(windmill_common::pg_embed::init_embedded_postgres().await?)
+    } else {
+        None
+    };
+
     let (conn, first_suffix) = if mode == Mode::Agent {
         tracing::info!(
             "Creating http client for cluster using base internal url {}",

backend/windmill-common/Cargo.toml

@@ -20,6 +20,7 @@ smtp = ["dep:mail-send"]
 scoped_cache = []
 cloud = []
 openidconnect = ["dep:openidconnect"]
+pg_embed = ["dep:postgresql_embedded"]
 [lib]
 name = "windmill_common"
 path = "src/lib.rs"
@@ -114,6 +115,7 @@ opentelemetry = { workspace = true, optional = true }
 tracing-opentelemetry = { workspace = true, optional = true }
 opentelemetry-appender-tracing = { workspace = true, optional = true }
 tonic = { workspace = true, optional = true }
+postgresql_embedded = { version = "0.18.1", optional = true, features = ["theseus"], default-features = false }
 
 [target.'cfg(not(target_env = "msvc"))'.dependencies]
 tikv-jemalloc-ctl = { optional = true, workspace = true }

backend/windmill-common/src/lib.rs

@@ -73,6 +73,8 @@ pub mod oidc_oss;
 #[cfg(feature = "private")]
 pub mod otel_ee;
 pub mod otel_oss;
+#[cfg(feature = "pg_embed")]
+pub mod pg_embed;
 pub mod queue;
 pub mod result_stream;
 pub mod runnable_settings;

backend/windmill-common/src/pg_embed.rs

@@ -0,0 +1,144 @@
+/*
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
+ * This file and its contents are licensed under the AGPLv3 License.
+ * Please see the included NOTICE for copyright information and
+ * LICENSE-AGPL for a copy of the license.
+ */
+
+use crate::error::Error;
+use postgresql_embedded::{PostgreSQL, Settings};
+use std::sync::Arc;
+use tokio::sync::RwLock;
+
+/// Manages an embedded PostgreSQL instance
+pub struct EmbeddedPostgres {
+    postgresql: Arc<RwLock<PostgreSQL>>,
+    database_url: String,
+}
+
+impl EmbeddedPostgres {
+    /// Initialize and start an embedded PostgreSQL instance
+    pub async fn new() -> Result<Self, Error> {
+        tracing::info!("Initializing embedded PostgreSQL instance...");
+
+        // Configure the embedded PostgreSQL settings
+        let mut settings = Settings::default();
+
+        // Use environment variables if provided for customization
+        if let Ok(data_dir) = std::env::var("PG_EMBED_DATA_DIR") {
+            settings.installation_dir = data_dir.into();
+        }
+
+        if let Ok(port) = std::env::var("PG_EMBED_PORT") {
+            if let Ok(port_num) = port.parse::<u16>() {
+                settings.port = port_num;
+            }
+        }
+
+        let mut postgresql = PostgreSQL::new(settings);
+
+        // Setup the PostgreSQL instance
+        postgresql
+            .setup()
+            .await
+            .map_err(|e| {
+                let err_msg = format!("Failed to setup embedded PostgreSQL: {}", e);
+                if err_msg.contains("libxml2") || err_msg.contains("shared libraries") {
+                    Error::InternalErr(format!(
+                        "{}\n\n\
+                        System dependencies are required for embedded PostgreSQL.\n\
+                        On Arch Linux, install: sudo pacman -S libxml2 icu openssl\n\
+                        On Ubuntu/Debian: sudo apt-get install libxml2 libicu-dev libssl-dev\n\
+                        On RHEL/Fedora: sudo dnf install libxml2 libicu openssl-libs\n\n\
+                        Alternatively, set DATABASE_URL to use an external PostgreSQL instance.",
+                        err_msg
+                    ))
+                } else {
+                    Error::InternalErr(err_msg)
+                }
+            })?;
+
+        tracing::info!("Starting embedded PostgreSQL...");
+
+        // Start the PostgreSQL instance
+        postgresql
+            .start()
+            .await
+            .map_err(|e| Error::InternalErr(format!("Failed to start embedded PostgreSQL: {}", e)))?;
+
+        tracing::info!("Embedded PostgreSQL started successfully");
+
+        // Get the database settings
+        let settings = postgresql.settings();
+
+        // Create the windmill database
+        let database_name = std::env::var("PG_EMBED_DATABASE")
+            .unwrap_or_else(|_| "windmill".to_string());
+
+        tracing::info!("Creating database: {}", database_name);
+
+        postgresql
+            .create_database(&database_name)
+            .await
+            .map_err(|e| {
+                Error::InternalErr(format!("Failed to create database '{}': {}", database_name, e))
+            })?;
+
+        // Build the connection string
+        let database_url = format!(
+            "postgres://{}:{}@{}:{}/{}",
+            settings.username,
+            settings.password,
+            settings.host,
+            settings.port,
+            database_name
+        );
+
+        tracing::info!("Embedded PostgreSQL ready at: postgres://{}:{}@{}:{}/{}",
+            settings.username,
+            "***",  // Don't log password
+            settings.host,
+            settings.port,
+            database_name
+        );
+
+        Ok(Self {
+            postgresql: Arc::new(RwLock::new(postgresql)),
+            database_url,
+        })
+    }
+
+    /// Get the database connection URL
+    pub fn database_url(&self) -> &str {
+        &self.database_url
+    }
+
+    /// Stop the embedded PostgreSQL instance
+    pub async fn stop(&self) -> Result<(), Error> {
+        tracing::info!("Stopping embedded PostgreSQL...");
+
+        let pg = self.postgresql.write().await;
+        pg.stop()
+            .await
+            .map_err(|e| Error::InternalErr(format!("Failed to stop embedded PostgreSQL: {}", e)))?;
+
+        tracing::info!("Embedded PostgreSQL stopped");
+        Ok(())
+    }
+}
+
+/// Initialize an embedded PostgreSQL instance and set the DATABASE_URL environment variable
+pub async fn init_embedded_postgres() -> Result<EmbeddedPostgres, Error> {
+    let embedded_pg = EmbeddedPostgres::new().await?;
+
+    // Set the DATABASE_URL environment variable so that the rest of the application
+    // can use it transparently
+    unsafe {
+        std::env::set_var("DATABASE_URL", embedded_pg.database_url());
+    }
+
+    tracing::info!("DATABASE_URL set to embedded PostgreSQL instance");
+
+    Ok(embedded_pg)
+}

backend/windmill-worker/src/worker.rs

@@ -316,6 +316,11 @@ lazy_static::lazy_static! {
         .and_then(|x| x.parse::<bool>().ok())
         .unwrap_or(false);
 
+    pub static ref QUIT_AFTER_FIRST_PING: bool = std::env::var("QUIT_AFTER_FIRST_PING")
+        .ok()
+        .and_then(|x| x.parse::<bool>().ok())
+        .unwrap_or(false);
+
     pub static ref UNSHARE_TINI_PATH: String = {
         std::env::var("UNSHARE_TINI_PATH").unwrap_or_else(|_| "tini".to_string())
     };
@@ -1265,6 +1270,14 @@ pub async fn run_worker(
         .await
         .expect("initial ping could be sent");
 
+    if *QUIT_AFTER_FIRST_PING {
+        tracing::info!(
+            worker = %worker_name, hostname = %hostname,
+            "QUIT_AFTER_FIRST_PING is set, exiting after successful initial ping"
+        );
+        return;
+    }
+
     #[cfg(feature = "prometheus")]
     let uptime_metric = if METRICS_ENABLED.load(Ordering::Relaxed) {
         Some(