From 3d4f4c6c38155396e9b2236a6a7a7ad4e02da877 Mon Sep 17 00:00:00 2001 From: Diego Imbert <70353967+diegoimbert@users.noreply.github.com> Date: Tue, 7 Apr 2026 23:03:06 +0200 Subject: [PATCH] feat: Fork datatables (#8339) * export_datatable_schema * Propose to fork the datatable on ws fork * dump datatable * Dockerfile * Fix import_datatable_dump * datatable schema fork works! * Option to copy both schema and data * Datatable fork behavior * nit ui * use psql instead * remove fork_datatable route * feat: add fork_pg_database and export_pg_schema routes with DB Manager UI Co-Authored-By: Claude Opus 4.6 * style: pluralize "schema" to "schemas" in DB Manager export/import UI Co-Authored-By: Claude Opus 4.6 * feat: add import mode select (schema only vs schema + data) to DB Manager import Co-Authored-By: Claude Opus 4.6 * Select schema or schema+data when important database * fix: prepend $res: prefix to resource paths in DB Manager import/export Co-Authored-By: Claude Opus 4.6 * style: dynamic import button label based on selected mode Co-Authored-By: Claude Opus 4.6 * nits * feat: add warning alert when schema+data import mode is selected Co-Authored-By: Claude Opus 4.6 * nit hide on cloud hosted * refactor: remove fork_behavior from datatable settings Co-Authored-By: Claude Opus 4.6 * refactor: split CreateWorkspace into layout wrapper and CreateWorkspaceInner Co-Authored-By: Claude Opus 4.6 * feat: instantiate CreateWorkspaceInner in globalForkModal Co-Authored-By: Claude Opus 4.6 * nit icons * Data table fork UI * feat: pass per-datatable fork behaviors from UI to backend during workspace fork Co-Authored-By: Claude Opus 4.6 * Fix fork overwriting all datatables * UI nits * custom instance db refactor * custom instance db wizard btn for all in dropdown * nit * Delete custom instance database button * Disable forking for resource datatables * Big import buttons when db empty * Revert "Disable forking for resource datatables" This reverts commit 9561cc8fd4564586e7dfe6c2edd9bb32ec2993b1. * feat: add non_diffable flag to resource table Co-Authored-By: Claude Opus 4.5 * feat: add resource-type datatable fork with CREATE DATABASE Co-Authored-By: Claude Opus 4.5 * feat: tag forked datatables with nonDiffable and forkedFrom Co-Authored-By: Claude Opus 4.5 * feat: diff datatable and ducklake settings individually on workspace merge Co-Authored-By: Claude Opus 4.5 * feat: skip non_diffable resources and datatables in workspace diff Co-Authored-By: Claude Opus 4.5 * feat: default datatable fork behavior to keep_original Co-Authored-By: Claude Opus 4.5 * fix: make grant permissions non-fatal in instance datatable fork Co-Authored-By: Claude Opus 4.5 * fix: make datatable and ducklake diffs visible in workspace comparison Co-Authored-By: Claude Opus 4.5 * refactor: remove datatable fork logic from workspace fork route Co-Authored-By: Claude Opus 4.5 * fix: correct ahead/behind logic for datatable and ducklake diffs Co-Authored-By: Claude Opus 4.5 * Revert "fix: correct ahead/behind logic for datatable and ducklake diffs" This reverts commit 6b50884dc60ba094c0468f26307fea50e80855b9. * revert: remove datatable and ducklake settings diffing logic Co-Authored-By: Claude Opus 4.5 * feat: add datatable clone UI with step-by-step confirmation modal Co-Authored-By: Claude Opus 4.5 * refactor: extract datatable fork UI into ForkDatatableSection component Co-Authored-By: Claude Opus 4.5 * nit * fix: run datatable cloning before workspace fork creation Co-Authored-By: Claude Opus 4.5 * nit disable fork admins * nit fix switching workspace prematurely * fix: use source workspace for forkPgDatabase calls during fork Co-Authored-By: Claude Opus 4.5 * feat: update forked workspace datatable settings after fork creation Co-Authored-By: Claude Opus 4.5 * feat: add forked_from field to DataTable and set it for instance forks Co-Authored-By: Claude Opus 4.5 * nit onFinish * fix: add forked_from to DataTableSettings OpenAPI schema Co-Authored-By: Claude Opus 4.5 * feat: track datatable table DDL changes in workspace_diff Co-Authored-By: Claude Opus 4.5 * Revert "feat: track datatable table DDL changes in workspace_diff" This reverts commit 7526dd68b9598aafe1b912c2491ce28504fc5585. * feat: add get_datatable_full_schema endpoint and snapshot schema on fork Co-Authored-By: Claude Opus 4.5 * fix duplicate migration key * fix: set forked_from on datatable config for both instance and resource types Co-Authored-By: Claude Opus 4.5 * nits * feat: drop forked databases on workspace deletion with confirmation UI Co-Authored-By: Claude Opus 4.5 * refactor: extract drop_forked_datatable_databases from delete_workspace Co-Authored-By: Claude Opus 4.5 * fix: cast pg char columns to text in FK schema query Co-Authored-By: Claude Opus 4.5 * fix: show dbname instead of resource type in fork deletion modal Co-Authored-By: Claude Opus 4.5 * ui nit * refactor: extract drop_custom_instance_database into windmill-common Co-Authored-By: Claude Opus 4.5 * feat: add datatable schema diff section to merge UI Co-Authored-By: Claude Opus 4.5 * UI * feat: add review drawer with YAML diff and SQL migration runner Co-Authored-By: Claude Opus 4.5 * refactor: use Monaco DiffEditor for YAML diff in review drawer Co-Authored-By: Claude Opus 4.5 * nit * Revert "refactor: use Monaco DiffEditor for YAML diff in review drawer" This reverts commit a86008ba4c504c222478e294c53b0cd4ca64c770. * Revert "feat: add review drawer with YAML diff and SQL migration runner" This reverts commit 0a0deb5ddb7e88cd398f3908e325b029ccc05941. * feat: add review drawer with DiffEditor and SQL migration runner Co-Authored-By: Claude Opus 4.5 * ui nits * fix: show diff between forked_from schema and changed side Co-Authored-By: Claude Opus 4.5 * fix: re-fetch target live schema after migration for correct baseline Co-Authored-By: Claude Opus 4.5 * revert * nit auto next * feat: add confirmation modal before deploying migration to parent Co-Authored-By: Claude Opus 4.5 * fix: handle missing columns/foreignKeys in schema conversion Co-Authored-By: Claude Opus 4.5 * nits * refactor: use temp file on disk for pg_dump instead of in-memory string Co-Authored-By: Claude Opus 4.5 * Don't replace postgres dbname * fix: add validation to drop_custom_instance_database and use source db for CREATE/DROP Co-Authored-By: Claude Opus 4.5 * refactor: type DataTable.forked_from as DataTableForkedFrom struct Co-Authored-By: Claude Opus 4.5 * refactor: simplify fork_pg_database to take source + target_dbname Co-Authored-By: Claude Opus 4.5 * dead code * feat: enforce schema_and_data admin-only and extract create_custom_instance_database Co-Authored-By: Claude Opus 4.5 * refactor: rename fork_pg_database to import_pg_database with source/target/override params Co-Authored-By: Claude Opus 4.5 * nit * refactor: remove original_dbname/original_resource from forked_from, resolve from parent Co-Authored-By: Claude Opus 4.5 * nit * fix: resolve forked dbname from fork workspace when dropping resource databases Co-Authored-By: Claude Opus 4.5 * nits * fix: always clean up global_settings even if database doesn't exist Co-Authored-By: Claude Opus 4.5 * fix: check datatable resource_type from config instead of URL prefix Co-Authored-By: Claude Opus 4.5 * fix: wrap PG default value expressions in braces to prevent CAST quoting Co-Authored-By: Claude Opus 4.5 * Revert "fix: wrap PG default value expressions in braces to prevent CAST quoting" This reverts commit 77f5a2c4e881d06d00deef7d8220f8aef254d228. * refactor: reuse columnDefToTableEditorValuesColumn for default value handling Co-Authored-By: Claude Opus 4.5 * fix: store raw API schema in forked_from to avoid double transformation Co-Authored-By: Claude Opus 4.5 * Revert "fix: store raw API schema in forked_from to avoid double transformation" This reverts commit e326197a20a871e8d1c2c204eb6f536cc9e9d66f. * Revert "refactor: reuse columnDefToTableEditorValuesColumn for default value handling" This reverts commit bd8f071d9f224215672f25d1187bfec05ac63d2f. * fix: validate dbname with strict regex to prevent SQL injection Co-Authored-By: Claude Opus 4.5 * fix default value * always validate dbname * refactor: move get_datatable_full_schema structs and logic to query_builders.rs Co-Authored-By: Claude Opus 4.5 * refactor: split import_pg_database into create_pg_database + import_pg_database Co-Authored-By: Claude Opus 4.5 * refactor: extract drop_forked_datatable_databases into its own route Co-Authored-By: Claude Opus 4.5 * fix: require admin when using $res: resource paths in import_pg_database Co-Authored-By: Claude Opus 4.5 * fix: use UserDB for $res: resource access and restrict dbname creation - resolve_pg_source_checked uses UserDB (row-level security) for $res: paths - transform_json_unchecked is now pub(crate) to prevent misuse - Non-superadmins can only create databases with wm_fork_ prefix - datatable:// remains accessible to everyone Co-Authored-By: Claude Opus 4.5 * fix: refuse to drop forked databases unless name starts with wm_fork_ Co-Authored-By: Claude Opus 4.5 * refactor: remove resolve_pg_source, use resolve_pg_source_checked everywhere Co-Authored-By: Claude Opus 4.5 * Fix permissions * sqlx prepare * compilation nits * sqlx prepare * sqlx prepare * wrong route syntax * fix: allow workspace owner to edit datatable config for fork setup Co-Authored-By: Claude Opus 4.5 * Revert "fix: allow workspace owner to edit datatable config for fork setup" This reverts commit ab683e637bcb31eedc15d69e3f57ae8d17a4a238. * refactor: move datatable fork setup into create_workspace_fork backend Instead of updating datatable settings from the frontend after fork creation (which required admin/owner access), pass forked_datatables info to create_workspace_fork and handle it atomically in the same transaction. Removes applyPostForkDatatableUpdates from frontend. Co-Authored-By: Claude Opus 4.5 * fix: snapshot schema in backend during fork instead of frontend The schema snapshot is now taken by the backend in apply_forked_datatable via snapshot_datatable_schema, which connects to the parent workspace's datatable and runs pg_get_full_schema. This removes the need for the frontend to call getDatatableFullSchema and pass the schema through. Co-Authored-By: Claude Opus 4.5 * fix: use get_resource_value_interpolated_internal for $res: to resolve $var: references Co-Authored-By: Claude Opus 4.5 * nit * sqlx prepare * fix: add permission check to drop_forked_datatable_databases, validate dbnames, restrict temp file perms - drop_forked_datatable_databases: same permission as delete_workspace (fork owner or super admin) - validate_dbname on target_dbname_override and ForkedDatatableInfo.new_dbname - Enforce wm_fork_ prefix on forked datatable new_dbname - DumpFile: set /tmp/windmill/ to 0700 and create files with 0600 Co-Authored-By: Claude Opus 4.5 * nit CLI * Rename to ws_specific * sqlx prepare * nit always validate dbname * fix: include foreign keys in CREATE TABLE migration for added tables Co-Authored-By: Claude Opus 4.5 * fix: detect nextval defaults and use SERIAL/BIGSERIAL types in CREATE TABLE Co-Authored-By: Claude Opus 4.5 * Update frontend/src/lib/components/DBManagerDrawer.svelte Co-authored-by: cubic-dev-ai[bot] <191113872+cubic-dev-ai[bot]@users.noreply.github.com> * Update backend/windmill-common/src/lib.rs Co-authored-by: cubic-dev-ai[bot] <191113872+cubic-dev-ai[bot]@users.noreply.github.com> * Update backend/windmill-common/src/lib.rs Co-authored-by: cubic-dev-ai[bot] <191113872+cubic-dev-ai[bot]@users.noreply.github.com> * fix: sort foreign keys by constraint name for deterministic schema output Co-Authored-By: Claude Opus 4.5 * sqlx prepare * rename migration to update timestamp --------- Co-authored-by: Claude Opus 4.6 Co-authored-by: cubic-dev-ai[bot] <191113872+cubic-dev-ai[bot]@users.noreply.github.com> --- Dockerfile | 10 +- ...1526adcecdd8729f28c15e0db8cd28eaa8cf0.json | 16 +- ...8159555b652869a036b006f4a151e999b17b7.json | 8 +- ...537caaf6d9177d72f1deeab4b5afcb83c0547.json | 21 + ...64f375f82e8904920d51454f10e04fc24b62a.json | 22 + ...23919cee3cc444725ac6b7906922554bae800.json | 23 + ...8b88c922d3442418701edd6b97e5fa50bd739.json | 20 + ...ce67c1281e16025aa3cf559a4f7a6a034d734.json | 41 + ...9138047b78f32d270a91a433cd6a04ecb48ea.json | 16 + ...0b047e71a171cf067fcd1e812681d8bd15005.json | 16 + ...c5e9aebca35a0245598000c5d741dc6ede2d5.json | 17 + ...0b0f7c9ed4f2aa5b2d54817e7630ca1d37753.json | 14 + ...75737a5855560a7f7c4a105461af73778a078.json | 23 + backend/Cargo.lock | 1 + ...260407000001_resource_ws_specific.down.sql | 1 + ...20260407000001_resource_ws_specific.up.sql | 3 + backend/windmill-api-settings/src/lib.rs | 16 + backend/windmill-api-workspaces/Cargo.toml | 1 + .../windmill-api-workspaces/src/workspaces.rs | 591 +++++++++++++- .../src/workspaces_extra.rs | 153 ++++ backend/windmill-api/openapi.yaml | 274 ++++++- backend/windmill-common/src/lib.rs | 176 +++- backend/windmill-common/src/query_builders.rs | 221 +++++ backend/windmill-common/src/workspaces.rs | 23 + backend/windmill-store/src/resources.rs | 18 +- cli/src/commands/resource/resource.ts | 1 + docker/DockerfileSlim | 10 +- docker/DockerfileSlimEe | 10 +- .../lib/components/CompareWorkspaces.svelte | 745 ++++++++--------- frontend/src/lib/components/DBManager.svelte | 51 +- .../lib/components/DBManagerContent.svelte | 6 +- .../src/lib/components/DBManagerDrawer.svelte | 159 +++- .../src/lib/components/DatatablePicker.svelte | 10 +- .../lib/components/DatatableSchemaDiff.svelte | 582 ++++++++++++++ frontend/src/lib/components/Editor.svelte | 4 +- frontend/src/lib/components/EditorBar.svelte | 2 +- .../lib/components/ForkWorkspaceBanner.svelte | 16 +- .../src/lib/components/ResourcePicker.svelte | 8 +- .../components/WorkspaceDeployLayout.svelte | 6 +- .../assets/AssetsDropdownButton.svelte | 4 +- .../ConfirmationModal.svelte | 2 +- .../raw_apps/RawAppDataTableDrawer.svelte | 4 +- .../raw_apps/datatableUtils.svelte.ts | 2 +- .../src/lib/components/select/Select.svelte | 3 + .../components/sidebar/SidebarContent.svelte | 68 +- .../components/sidebar/WorkspaceMenu.svelte | 13 +- .../workspaceSettings/CreateWorkspace.svelte | 753 +----------------- .../CreateWorkspaceInner.svelte | 662 +++++++++++++++ .../CustomInstanceDbSelect.svelte | 67 +- .../CustomInstanceDbWizardModal.svelte | 54 +- .../DataTableSettings.svelte | 49 +- .../workspaceSettings/DucklakeSettings.svelte | 2 +- .../ForkDatatableSection.svelte | 267 +++++++ frontend/src/lib/stores.ts | 5 + .../src/routes/(root)/(logged)/+layout.svelte | 18 +- .../(root)/(logged)/assets/+page.svelte | 2 +- .../(logged)/forks/compare/+page.svelte | 35 +- rust-client/src/client.rs | 1 + 58 files changed, 4088 insertions(+), 1258 deletions(-) create mode 100644 backend/.sqlx/query-4a1947f86ab5d2ff63a3f307c6f537caaf6d9177d72f1deeab4b5afcb83c0547.json create mode 100644 backend/.sqlx/query-4d03f08c86588d3fba178340a3b64f375f82e8904920d51454f10e04fc24b62a.json create mode 100644 backend/.sqlx/query-63c16a4277983aaed0aed54972923919cee3cc444725ac6b7906922554bae800.json create mode 100644 backend/.sqlx/query-7929fa087a28949906ffb3f508d8b88c922d3442418701edd6b97e5fa50bd739.json create mode 100644 backend/.sqlx/query-83d537bb8a4bb0121b6c7c2c6f7ce67c1281e16025aa3cf559a4f7a6a034d734.json create mode 100644 backend/.sqlx/query-b2903da4186589d9df7d4e245f29138047b78f32d270a91a433cd6a04ecb48ea.json create mode 100644 backend/.sqlx/query-ba09bbba6bd34ef52c702eddc7b0b047e71a171cf067fcd1e812681d8bd15005.json create mode 100644 backend/.sqlx/query-da6f37f91f2a24cf9ce8a1759adc5e9aebca35a0245598000c5d741dc6ede2d5.json create mode 100644 backend/.sqlx/query-df13aa5740f5475165b934f26d90b0f7c9ed4f2aa5b2d54817e7630ca1d37753.json create mode 100644 backend/.sqlx/query-f85208a1f14536125db503d5cf375737a5855560a7f7c4a105461af73778a078.json create mode 100644 backend/migrations/20260407000001_resource_ws_specific.down.sql create mode 100644 backend/migrations/20260407000001_resource_ws_specific.up.sql create mode 100644 frontend/src/lib/components/DatatableSchemaDiff.svelte create mode 100644 frontend/src/lib/components/workspaceSettings/CreateWorkspaceInner.svelte create mode 100644 frontend/src/lib/components/workspaceSettings/ForkDatatableSection.svelte diff --git a/Dockerfile b/Dockerfile index d312d3a823..e11cf9cecd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -162,11 +162,19 @@ ENV PATH /usr/local/bin:/root/.local/bin:/tmp/.local/bin:$PATH RUN apt-get update \ - && apt-get install -y --no-install-recommends netbase tzdata ca-certificates wget curl jq unzip build-essential unixodbc xmlsec1 software-properties-common tini \ + && apt-get install -y --no-install-recommends netbase tzdata ca-certificates wget curl jq unzip build-essential unixodbc xmlsec1 software-properties-common tini gnupg lsb-release \ && if echo "$features" | grep -q "ee"; then apt-get install -y --no-install-recommends libsasl2-modules-gssapi-mit krb5-user; fi \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* +# Install latest PostgreSQL client (pg_dump) from official PostgreSQL apt repository +RUN curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor -o /usr/share/keyrings/postgresql-archive-keyring.gpg \ + && echo "deb [signed-by=/usr/share/keyrings/postgresql-archive-keyring.gpg] https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list \ + && apt-get update \ + && apt-get install -y --no-install-recommends postgresql-client \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + RUN if [ "$WITH_GIT" = "true" ]; then \ apt-get update -y \ && apt-get install -y git \ diff --git a/backend/.sqlx/query-41f2c271514ee254739c3a097871526adcecdd8729f28c15e0db8cd28eaa8cf0.json b/backend/.sqlx/query-41f2c271514ee254739c3a097871526adcecdd8729f28c15e0db8cd28eaa8cf0.json index 78efa6ccbc..ba0fed760d 100644 --- a/backend/.sqlx/query-41f2c271514ee254739c3a097871526adcecdd8729f28c15e0db8cd28eaa8cf0.json +++ b/backend/.sqlx/query-41f2c271514ee254739c3a097871526adcecdd8729f28c15e0db8cd28eaa8cf0.json @@ -50,31 +50,36 @@ }, { "ordinal": 9, - "name": "is_expired", + "name": "ws_specific", "type_info": "Bool" }, { "ordinal": 10, - "name": "is_refreshed", + "name": "is_expired", "type_info": "Bool" }, { "ordinal": 11, + "name": "is_refreshed", + "type_info": "Bool" + }, + { + "ordinal": 12, "name": "refresh_error", "type_info": "Text" }, { - "ordinal": 12, + "ordinal": 13, "name": "is_linked", "type_info": "Bool" }, { - "ordinal": 13, + "ordinal": 14, "name": "is_oauth?", "type_info": "Bool" }, { - "ordinal": 14, + "ordinal": 15, "name": "account", "type_info": "Int4" } @@ -95,6 +100,7 @@ true, true, true, + false, null, null, true, diff --git a/backend/.sqlx/query-45e4d13f5806122faecdb1d9ab18159555b652869a036b006f4a151e999b17b7.json b/backend/.sqlx/query-45e4d13f5806122faecdb1d9ab18159555b652869a036b006f4a151e999b17b7.json index 12b8201ebb..7bb9a0dc66 100644 --- a/backend/.sqlx/query-45e4d13f5806122faecdb1d9ab18159555b652869a036b006f4a151e999b17b7.json +++ b/backend/.sqlx/query-45e4d13f5806122faecdb1d9ab18159555b652869a036b006f4a151e999b17b7.json @@ -47,6 +47,11 @@ "ordinal": 8, "name": "labels", "type_info": "TextArray" + }, + { + "ordinal": 9, + "name": "ws_specific", + "type_info": "Bool" } ], "parameters": { @@ -63,7 +68,8 @@ false, true, true, - true + true, + false ] }, "hash": "45e4d13f5806122faecdb1d9ab18159555b652869a036b006f4a151e999b17b7" diff --git a/backend/.sqlx/query-4a1947f86ab5d2ff63a3f307c6f537caaf6d9177d72f1deeab4b5afcb83c0547.json b/backend/.sqlx/query-4a1947f86ab5d2ff63a3f307c6f537caaf6d9177d72f1deeab4b5afcb83c0547.json new file mode 100644 index 0000000000..ae9a909e3e --- /dev/null +++ b/backend/.sqlx/query-4a1947f86ab5d2ff63a3f307c6f537caaf6d9177d72f1deeab4b5afcb83c0547.json @@ -0,0 +1,21 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO resource\n (workspace_id, path, value, description, resource_type, created_by, edited_at, ws_specific, labels)\n VALUES ($1, $2, $3, $4, $5, $6, now(), $7, $8) ON CONFLICT (workspace_id, path)\n DO UPDATE SET value = EXCLUDED.value, description = EXCLUDED.description, resource_type = EXCLUDED.resource_type, edited_at = now(), ws_specific = EXCLUDED.ws_specific, labels = EXCLUDED.labels", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Jsonb", + "Text", + "Varchar", + "Varchar", + "Bool", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "4a1947f86ab5d2ff63a3f307c6f537caaf6d9177d72f1deeab4b5afcb83c0547" +} diff --git a/backend/.sqlx/query-4d03f08c86588d3fba178340a3b64f375f82e8904920d51454f10e04fc24b62a.json b/backend/.sqlx/query-4d03f08c86588d3fba178340a3b64f375f82e8904920d51454f10e04fc24b62a.json new file mode 100644 index 0000000000..99b8051298 --- /dev/null +++ b/backend/.sqlx/query-4d03f08c86588d3fba178340a3b64f375f82e8904920d51454f10e04fc24b62a.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT datatable->'datatables' FROM workspace_settings WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "4d03f08c86588d3fba178340a3b64f375f82e8904920d51454f10e04fc24b62a" +} diff --git a/backend/.sqlx/query-63c16a4277983aaed0aed54972923919cee3cc444725ac6b7906922554bae800.json b/backend/.sqlx/query-63c16a4277983aaed0aed54972923919cee3cc444725ac6b7906922554bae800.json new file mode 100644 index 0000000000..4abea8d7e5 --- /dev/null +++ b/backend/.sqlx/query-63c16a4277983aaed0aed54972923919cee3cc444725ac6b7906922554bae800.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT value FROM resource WHERE path = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "value", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "63c16a4277983aaed0aed54972923919cee3cc444725ac6b7906922554bae800" +} diff --git a/backend/.sqlx/query-7929fa087a28949906ffb3f508d8b88c922d3442418701edd6b97e5fa50bd739.json b/backend/.sqlx/query-7929fa087a28949906ffb3f508d8b88c922d3442418701edd6b97e5fa50bd739.json new file mode 100644 index 0000000000..148f2ab664 --- /dev/null +++ b/backend/.sqlx/query-7929fa087a28949906ffb3f508d8b88c922d3442418701edd6b97e5fa50bd739.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO resource\n (workspace_id, path, value, description, resource_type, created_by, edited_at, ws_specific)\n VALUES ($1, $2, $3, $4, $5, $6, now(), $7) ON CONFLICT (workspace_id, path)\n DO UPDATE SET value = EXCLUDED.value, description = EXCLUDED.description, resource_type = EXCLUDED.resource_type, edited_at = now(), ws_specific = EXCLUDED.ws_specific", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Jsonb", + "Text", + "Varchar", + "Varchar", + "Bool" + ] + }, + "nullable": [] + }, + "hash": "7929fa087a28949906ffb3f508d8b88c922d3442418701edd6b97e5fa50bd739" +} diff --git a/backend/.sqlx/query-83d537bb8a4bb0121b6c7c2c6f7ce67c1281e16025aa3cf559a4f7a6a034d734.json b/backend/.sqlx/query-83d537bb8a4bb0121b6c7c2c6f7ce67c1281e16025aa3cf559a4f7a6a034d734.json new file mode 100644 index 0000000000..d293414791 --- /dev/null +++ b/backend/.sqlx/query-83d537bb8a4bb0121b6c7c2c6f7ce67c1281e16025aa3cf559a4f7a6a034d734.json @@ -0,0 +1,41 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT value, description, resource_type, ws_specific\n FROM resource\n WHERE workspace_id = $1 AND path = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "value", + "type_info": "Jsonb" + }, + { + "ordinal": 1, + "name": "description", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "resource_type", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "ws_specific", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + true, + true, + false, + false + ] + }, + "hash": "83d537bb8a4bb0121b6c7c2c6f7ce67c1281e16025aa3cf559a4f7a6a034d734" +} diff --git a/backend/.sqlx/query-b2903da4186589d9df7d4e245f29138047b78f32d270a91a433cd6a04ecb48ea.json b/backend/.sqlx/query-b2903da4186589d9df7d4e245f29138047b78f32d270a91a433cd6a04ecb48ea.json new file mode 100644 index 0000000000..d6aa0a4360 --- /dev/null +++ b/backend/.sqlx/query-b2903da4186589d9df7d4e245f29138047b78f32d270a91a433cd6a04ecb48ea.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings\n SET datatable = jsonb_set(datatable, ARRAY['datatables', $2, 'forked_from'], $3::jsonb)\n WHERE workspace_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "b2903da4186589d9df7d4e245f29138047b78f32d270a91a433cd6a04ecb48ea" +} diff --git a/backend/.sqlx/query-ba09bbba6bd34ef52c702eddc7b0b047e71a171cf067fcd1e812681d8bd15005.json b/backend/.sqlx/query-ba09bbba6bd34ef52c702eddc7b0b047e71a171cf067fcd1e812681d8bd15005.json new file mode 100644 index 0000000000..68697492b9 --- /dev/null +++ b/backend/.sqlx/query-ba09bbba6bd34ef52c702eddc7b0b047e71a171cf067fcd1e812681d8bd15005.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE resource\n SET value = jsonb_set(value, '{dbname}', to_jsonb($3::text)),\n ws_specific = true\n WHERE workspace_id = $1 AND path = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "ba09bbba6bd34ef52c702eddc7b0b047e71a171cf067fcd1e812681d8bd15005" +} diff --git a/backend/.sqlx/query-da6f37f91f2a24cf9ce8a1759adc5e9aebca35a0245598000c5d741dc6ede2d5.json b/backend/.sqlx/query-da6f37f91f2a24cf9ce8a1759adc5e9aebca35a0245598000c5d741dc6ede2d5.json new file mode 100644 index 0000000000..3abe38f3cb --- /dev/null +++ b/backend/.sqlx/query-da6f37f91f2a24cf9ce8a1759adc5e9aebca35a0245598000c5d741dc6ede2d5.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings\n SET datatable = jsonb_set(\n jsonb_set(datatable, ARRAY['datatables', $2, 'database', 'resource_path'], to_jsonb($3::text)),\n ARRAY['datatables', $2, 'forked_from'], $4::jsonb\n )\n WHERE workspace_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "da6f37f91f2a24cf9ce8a1759adc5e9aebca35a0245598000c5d741dc6ede2d5" +} diff --git a/backend/.sqlx/query-df13aa5740f5475165b934f26d90b0f7c9ed4f2aa5b2d54817e7630ca1d37753.json b/backend/.sqlx/query-df13aa5740f5475165b934f26d90b0f7c9ed4f2aa5b2d54817e7630ca1d37753.json new file mode 100644 index 0000000000..c2eb36edb4 --- /dev/null +++ b/backend/.sqlx/query-df13aa5740f5475165b934f26d90b0f7c9ed4f2aa5b2d54817e7630ca1d37753.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE global_settings SET value = value #- ARRAY['databases', $1] WHERE name = 'custom_instance_pg_databases'", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "df13aa5740f5475165b934f26d90b0f7c9ed4f2aa5b2d54817e7630ca1d37753" +} diff --git a/backend/.sqlx/query-f85208a1f14536125db503d5cf375737a5855560a7f7c4a105461af73778a078.json b/backend/.sqlx/query-f85208a1f14536125db503d5cf375737a5855560a7f7c4a105461af73778a078.json new file mode 100644 index 0000000000..f0080ed07c --- /dev/null +++ b/backend/.sqlx/query-f85208a1f14536125db503d5cf375737a5855560a7f7c4a105461af73778a078.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT datatable->'datatables'->$2 FROM workspace_settings WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "f85208a1f14536125db503d5cf375737a5855560a7f7c4a105461af73778a078" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 81ae56010f..055277d9ec 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -16696,6 +16696,7 @@ dependencies = [ "windmill-git-sync", "windmill-object-store", "windmill-queue", + "windmill-store", "windmill-types", ] diff --git a/backend/migrations/20260407000001_resource_ws_specific.down.sql b/backend/migrations/20260407000001_resource_ws_specific.down.sql new file mode 100644 index 0000000000..94a25ecb6f --- /dev/null +++ b/backend/migrations/20260407000001_resource_ws_specific.down.sql @@ -0,0 +1 @@ +ALTER TABLE resource DROP COLUMN IF EXISTS ws_specific; diff --git a/backend/migrations/20260407000001_resource_ws_specific.up.sql b/backend/migrations/20260407000001_resource_ws_specific.up.sql new file mode 100644 index 0000000000..67cfcb037b --- /dev/null +++ b/backend/migrations/20260407000001_resource_ws_specific.up.sql @@ -0,0 +1,3 @@ +-- Add ws_specific flag to resource table. +-- When true, the resource is excluded from workspace diff comparisons (e.g. auto-created resources during fork). +ALTER TABLE resource ADD COLUMN IF NOT EXISTS ws_specific BOOLEAN NOT NULL DEFAULT false; diff --git a/backend/windmill-api-settings/src/lib.rs b/backend/windmill-api-settings/src/lib.rs index 1ca22b98b7..0e5defe07a 100644 --- a/backend/windmill-api-settings/src/lib.rs +++ b/backend/windmill-api-settings/src/lib.rs @@ -101,6 +101,10 @@ pub fn global_service() -> Router { "/setup_custom_instance_pg_database/{name}", post(setup_custom_instance_pg_database), ) + .route( + "/drop_custom_instance_pg_database/{name}", + post(drop_custom_instance_pg_database), + ) .route( "/critical_alerts/acknowledge_all", post(acknowledge_all_critical_alerts), @@ -1179,6 +1183,18 @@ async fn setup_custom_instance_pg_database_inner( Ok(()) } +async fn drop_custom_instance_pg_database( + authed: ApiAuthed, + Extension(db): Extension, + Path(dbname): Path, +) -> Result { + require_super_admin(&db, &authed.email).await?; + + windmill_common::drop_custom_instance_database(&db, &dbname).await?; + + Ok(format!("Database '{}' dropped successfully", dbname)) +} + // ============================================================================ // Secret Backend Settings (HashiCorp Vault Integration) - Enterprise Edition // ============================================================================ diff --git a/backend/windmill-api-workspaces/Cargo.toml b/backend/windmill-api-workspaces/Cargo.toml index 86f0649c73..4a9e04c417 100644 --- a/backend/windmill-api-workspaces/Cargo.toml +++ b/backend/windmill-api-workspaces/Cargo.toml @@ -27,6 +27,7 @@ windmill-git-sync.workspace = true windmill-queue.workspace = true windmill-api-jobs.workspace = true windmill-dep-map.workspace = true +windmill-store.workspace = true axum.workspace = true chrono.workspace = true diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 64c07754e9..80c7d3dcec 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -40,8 +40,8 @@ use windmill_common::workspaces::GitRepositorySettings; use windmill_common::workspaces::WorkspaceDeploymentUISettings; use windmill_common::workspaces::{ check_user_against_rule, get_datatable_resource_from_db_unchecked, DataTable, - DataTableCatalogResourceType, ProtectionRuleKind, ProtectionRules, ProtectionRuleset, - RuleCheckResult, WorkspaceGitSyncSettings, + DataTableCatalogResourceType, DataTableForkBehavior, ProtectionRuleKind, ProtectionRules, + ProtectionRuleset, RuleCheckResult, WorkspaceGitSyncSettings, }; use windmill_common::workspaces::{Ducklake, DucklakeCatalogResourceType}; use windmill_common::PgDatabase; @@ -157,6 +157,17 @@ pub fn workspaced_service() -> Router { post(reset_workspace_diffs), ) .route("/compare/{target_workspace_id}", get(compare_workspaces)) + .route("/create_pg_database", post(create_pg_database)) + .route("/import_pg_database", post(import_pg_database)) + .route("/export_pg_schema", post(export_pg_schema)) + .route( + "/drop_forked_datatable_databases", + post(crate::workspaces_extra::drop_forked_datatable_databases), + ) + .route( + "/get_datatable_full_schema", + post(get_datatable_full_schema), + ) .route("/protection_rules", get(list_protection_rules)) .route("/protection_rules", post(create_protection_rule)) .route( @@ -381,6 +392,16 @@ struct CreateWorkspaceFork { id: String, name: String, color: Option, + /// Datatable names that were forked. For each, the backend will update the + /// forked workspace's datatable config to point to the new database. + #[serde(default)] + forked_datatables: Vec, +} + +#[derive(Deserialize)] +struct ForkedDatatableInfo { + name: String, + new_dbname: String, } #[derive(Deserialize)] @@ -1273,26 +1294,40 @@ async fn list_ducklakes( Ok(Json(ducklakes)) } +#[derive(Serialize)] +struct DataTableListItem { + name: String, + resource_type: String, + resource_path: String, +} + async fn list_datatables( _authed: ApiAuthed, Extension(db): Extension, Path(w_id): Path, -) -> JsonResult> { - let datatables = sqlx::query_scalar!( - r#" - SELECT jsonb_object_keys(ws.datatable->'datatables') AS datatable_name - FROM workspace_settings ws - WHERE ws.workspace_id = $1 - "#, +) -> JsonResult> { + let config = sqlx::query_scalar!( + "SELECT datatable->'datatables' FROM workspace_settings WHERE workspace_id = $1", &w_id ) - .fetch_all(&db) - .await? - .into_iter() - .filter_map(|s| s) - .collect(); + .fetch_one(&db) + .await?; - Ok(Json(datatables)) + let items: Vec = match config { + Some(val) => { + let map: HashMap = serde_json::from_value(val).unwrap_or_default(); + map.into_iter() + .map(|(name, dt)| DataTableListItem { + name, + resource_type: dt.database.resource_type.as_ref().to_string(), + resource_path: dt.database.resource_path, + }) + .collect() + } + None => vec![], + }; + + Ok(Json(items)) } /// Compact column representation: "type" or "type?" for nullable, with "=default" suffix if has default @@ -1455,6 +1490,397 @@ async fn get_datatable_schema(db: &DB, w_id: &str, datatable_name: &str) -> Resu Ok(schema_map) } +/// Resolve a source string to PgDatabase credentials with user-scoped permission checks. +/// For `datatable://name`: accessible to everyone (variables are resolved internally). +/// For `$res:path`: uses UserDB (row-level security) to verify the user can see the resource, +/// then interpolates `$var:` references in the resource value. +pub(crate) async fn resolve_pg_source_checked( + db: &DB, + user_db: &UserDB, + authed: &ApiAuthed, + w_id: &str, + source: &str, +) -> Result { + let db_resource = if let Some(name) = source.strip_prefix("datatable://") { + get_datatable_resource_from_db_unchecked(db, w_id, name).await? + } else if let Some(path) = source.strip_prefix("$res:") { + let db_with_authed = windmill_common::db::DbWithOptAuthed::from_authed( + authed, + db.clone(), + Some(user_db.clone()), + ); + let value = windmill_store::resources::get_resource_value_interpolated_internal( + &db_with_authed, + w_id, + path, + None, + None, + false, + ) + .await?; + + match value { + Some(v) => v, + None => { + return Err(Error::NotAuthorized(format!( + "Resource '{}' not found or you do not have access to it", + path + ))); + } + } + } else { + return Err(Error::BadRequest(format!( + "Invalid source format: '{}'. Expected 'datatable://name' or '$res:path'", + source + ))); + }; + + serde_json::from_value(db_resource) + .map_err(|e| Error::internal_err(format!("Failed to parse database credentials: {}", e))) +} + +/// A temporary file for pg_dump output that is automatically deleted when dropped. +struct DumpFile { + path: std::path::PathBuf, +} + +impl DumpFile { + fn new() -> Result { + let dir = std::path::Path::new("/tmp/windmill"); + std::fs::create_dir_all(dir) + .map_err(|e| Error::internal_err(format!("Failed to create /tmp/windmill: {}", e)))?; + // Set directory permissions to owner-only + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let _ = std::fs::set_permissions(dir, std::fs::Permissions::from_mode(0o700)); + } + let path = dir.join(format!("datatable_dump_{}", uuid::Uuid::new_v4())); + // Create the file with restrictive permissions before pg_dump writes to it + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + std::fs::OpenOptions::new() + .write(true) + .create(true) + .mode(0o600) + .open(&path) + .map_err(|e| Error::internal_err(format!("Failed to create dump file: {}", e)))?; + } + #[cfg(not(unix))] + { + std::fs::File::create(&path) + .map_err(|e| Error::internal_err(format!("Failed to create dump file: {}", e)))?; + } + Ok(Self { path }) + } +} + +impl Drop for DumpFile { + fn drop(&mut self) { + if self.path.exists() { + if let Err(e) = std::fs::remove_file(&self.path) { + tracing::warn!("Failed to remove dump file {:?}: {}", self.path, e); + } + } + } +} + +/// Run pg_dump against a PgDatabase, writing output to a temp file on disk. +/// Returns a DumpFile handle; the file is deleted when the handle is dropped. +async fn pg_dump_database(pg_db: &PgDatabase, schema_only: bool) -> Result { + let dump_file = DumpFile::new()?; + + let host = &pg_db.host; + let port = pg_db.port.unwrap_or(5432).to_string(); + let user = pg_db.user.as_deref().unwrap_or("postgres"); + let dbname = &pg_db.dbname; + + let mut cmd = tokio::process::Command::new("pg_dump"); + cmd.arg("--format=plain").arg("--file").arg(&dump_file.path); + if schema_only { + cmd.arg("--schema-only"); + } + cmd.arg("--host") + .arg(host) + .arg("--port") + .arg(&port) + .arg("--username") + .arg(user) + .arg(dbname); + + if let Some(ref password) = pg_db.password { + cmd.env("PGPASSWORD", password); + } + + if let Some(ref sslmode) = pg_db.sslmode { + cmd.env("PGSSLMODE", sslmode); + } + + let output = cmd + .output() + .await + .map_err(|e| Error::internal_err(format!("Failed to execute pg_dump: {}", e)))?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(Error::internal_err(format!("pg_dump failed: {}", stderr))); + } + + Ok(dump_file) +} + +/// Import a pg_dump file into a target database using psql. +async fn pg_import_dump(target_db: &PgDatabase, dump_file: &DumpFile) -> Result<()> { + let host = &target_db.host; + let port = target_db.port.unwrap_or(5432).to_string(); + let user = target_db.user.as_deref().unwrap_or("postgres"); + let dbname = &target_db.dbname; + + let mut cmd = tokio::process::Command::new("psql"); + cmd.arg("--host") + .arg(host) + .arg("--port") + .arg(&port) + .arg("--username") + .arg(user) + .arg("--dbname") + .arg(dbname) + .arg("--no-psqlrc") + .arg("--file") + .arg(&dump_file.path) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()); + + if let Some(ref password) = target_db.password { + cmd.env("PGPASSWORD", password); + } + + if let Some(ref sslmode) = target_db.sslmode { + cmd.env("PGSSLMODE", sslmode); + } + + let output = cmd + .output() + .await + .map_err(|e| Error::internal_err(format!("Failed to execute psql: {}", e)))?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(Error::internal_err(format!( + "psql import failed: {}", + stderr + ))); + } + + Ok(()) +} + +#[derive(Deserialize)] +struct CreatePgDatabaseRequest { + /// The datatable source to determine connection info: 'datatable://name' or '$res:path' + source: String, + /// Name for the new database + target_dbname: String, +} + +/// Create a new PostgreSQL database. For instance datatables, creates on the Windmill PG instance. +/// For resource datatables, creates on the same server as the source. +async fn create_pg_database( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Path(w_id): Path, + Json(req): Json, +) -> Result { + windmill_common::validate_dbname(&req.target_dbname)?; + + // Non-superadmin: restrict dbname to wm_fork_ prefix + if !windmill_common::auth::is_super_admin_email(&db, &authed.email).await? { + if !req.target_dbname.starts_with("wm_fork_") { + return Err(Error::BadRequest( + "Non-superadmin users can only create databases with names starting with 'wm_fork_'" + .to_string(), + )); + } + } + + // Determine if this is an instance or resource-backed datatable + let is_instance_datatable = if let Some(dt_name) = req.source.strip_prefix("datatable://") { + let config = sqlx::query_scalar!( + "SELECT datatable->'datatables'->$2 FROM workspace_settings WHERE workspace_id = $1", + &w_id, + dt_name + ) + .fetch_optional(&db) + .await? + .flatten(); + config + .and_then(|v| { + v.get("database") + .and_then(|d| d.get("resource_type")) + .and_then(|r| r.as_str()) + .map(|s| s == "instance") + }) + .unwrap_or(false) + } else { + false + }; + + if is_instance_datatable { + windmill_common::create_custom_instance_database(&db, &req.target_dbname, "datatable") + .await?; + } else { + let source_pg = + resolve_pg_source_checked(&db, &user_db, &authed, &w_id, &req.source).await?; + let (client, connection) = source_pg.connect().await?; + let join_handle = tokio::spawn(async move { connection.await }); + + let row = client + .query_one( + "SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_database WHERE datname = $1)", + &[&req.target_dbname], + ) + .await + .map_err(|e| { + Error::internal_err(format!("Failed to check database existence: {}", e)) + })?; + let db_exists: bool = row.get(0); + + if db_exists { + drop(client); + let _ = join_handle.await; + return Err(Error::BadRequest(format!( + "Database '{}' already exists on the resource server", + req.target_dbname + ))); + } + + client + .execute(&format!("CREATE DATABASE \"{}\"", &req.target_dbname), &[]) + .await + .map_err(|e| { + Error::internal_err(format!( + "Failed to create database '{}': {}", + req.target_dbname, e + )) + })?; + + drop(client); + join_handle + .await + .map_err(|e| Error::internal_err(format!("join error: {}", e)))? + .map_err(|e| Error::internal_err(format!("tokio_postgres error: {}", e)))?; + } + + Ok(format!("Created database '{}'", req.target_dbname)) +} + +#[derive(Deserialize)] +struct ImportPgDatabaseRequest { + source: String, + target: String, + #[serde(default)] + target_dbname_override: Option, + fork_behavior: DataTableForkBehavior, +} + +/// Import (pg_dump/pg_import) from source to target +async fn import_pg_database( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Path(w_id): Path, + Json(req): Json, +) -> Result { + if req.fork_behavior == DataTableForkBehavior::KeepOriginal { + return Ok("No action needed for KeepOriginal behavior".to_string()); + } + + if req.fork_behavior == DataTableForkBehavior::SchemaAndData { + require_admin(authed.is_admin, &authed.username)?; + if *CLOUD_HOSTED { + return Err(Error::BadRequest( + "Importing schema and data is not available on cloud".to_string(), + )); + } + } + + let schema_only = req.fork_behavior == DataTableForkBehavior::SchemaOnly; + let source_pg = resolve_pg_source_checked(&db, &user_db, &authed, &w_id, &req.source).await?; + let mut target_pg = + resolve_pg_source_checked(&db, &user_db, &authed, &w_id, &req.target).await?; + + if let Some(ref override_dbname) = req.target_dbname_override { + if !windmill_common::auth::is_super_admin_email(&db, &authed.email).await? { + if !override_dbname.starts_with("wm_fork_") { + return Err(Error::BadRequest( + "Non-superadmin users can only override target dbname with names starting with 'wm_fork_'" + .to_string(), + )); + } + } + target_pg.dbname = override_dbname.clone(); + } + windmill_common::validate_dbname(&target_pg.dbname)?; + + let dump_file = pg_dump_database(&source_pg, schema_only).await?; + pg_import_dump(&target_pg, &dump_file).await?; + + Ok(format!( + "Imported from '{}' into '{}'", + req.source, target_pg.dbname + )) +} + +#[derive(Deserialize)] +struct ExportPgSchemaRequest { + source: String, +} + +async fn export_pg_schema( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Path(w_id): Path, + Json(req): Json, +) -> Result { + let pg = resolve_pg_source_checked(&db, &user_db, &authed, &w_id, &req.source).await?; + let dump_file = pg_dump_database(&pg, true).await?; + tokio::fs::read_to_string(&dump_file.path) + .await + .map_err(|e| Error::internal_err(format!("Failed to read dump file: {}", e))) +} + +#[derive(Deserialize)] +struct GetDatatableFullSchemaRequest { + source: String, +} + +async fn get_datatable_full_schema( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Path(w_id): Path, + Json(req): Json, +) -> JsonResult { + let pg = resolve_pg_source_checked(&db, &user_db, &authed, &w_id, &req.source).await?; + let (client, connection) = pg.connect().await?; + let join_handle = tokio::spawn(async move { connection.await }); + + let result = windmill_common::query_builders::pg_get_full_schema(&client) + .await + .map_err(Error::internal_err)?; + + drop(client); + join_handle + .await + .map_err(|e| Error::internal_err(format!("join error: {}", e)))? + .map_err(|e| Error::internal_err(format!("tokio_postgres error: {}", e)))?; + + Ok(Json(result)) +} + async fn edit_ducklake_config( authed: ApiAuthed, Extension(db): Extension, @@ -3732,6 +4158,120 @@ async fn create_workspace_fork_branch( )) } +/// Update a forked workspace's datatable config to point to the new database. +/// For instance datatables: updates resource_path in the datatable config. +/// For resource datatables: updates the resource's dbname and sets ws_specific. +/// Snapshot the schema from the source datatable by connecting to its database. +async fn snapshot_datatable_schema( + db: &DB, + parent_w_id: &str, + dt_name: &str, +) -> Result { + let pg = get_datatable_resource_from_db_unchecked(db, parent_w_id, dt_name).await?; + let pg: PgDatabase = serde_json::from_value(pg) + .map_err(|e| Error::internal_err(format!("Failed to parse db credentials: {}", e)))?; + let (client, connection) = pg.connect().await?; + let join_handle = tokio::spawn(async move { connection.await }); + + let schema = windmill_common::query_builders::pg_get_full_schema(&client) + .await + .map_err(Error::internal_err)?; + + drop(client); + join_handle + .await + .map_err(|e| Error::internal_err(format!("join error: {}", e)))? + .map_err(|e| Error::internal_err(format!("tokio_postgres error: {}", e)))?; + + serde_json::to_value(schema) + .map_err(|e| Error::internal_err(format!("Failed to serialize schema: {}", e))) +} + +async fn apply_forked_datatable( + db: &DB, + tx: &mut Transaction<'_, Postgres>, + parent_w_id: &str, + forked_w_id: &str, + fdt: &ForkedDatatableInfo, +) -> Result<()> { + windmill_common::validate_dbname(&fdt.new_dbname)?; + if !fdt.new_dbname.starts_with("wm_fork_") { + return Err(Error::BadRequest(format!( + "Forked datatable database name '{}' must start with 'wm_fork_'", + fdt.new_dbname + ))); + } + + // Snapshot the schema from the source (parent) datatable + let schema = snapshot_datatable_schema(db, parent_w_id, &fdt.name).await?; + let forked_from = serde_json::json!({ "schema": schema }); + + // Read the datatable config from the forked workspace + let config_val = sqlx::query_scalar!( + "SELECT datatable->'datatables'->$2 FROM workspace_settings WHERE workspace_id = $1", + forked_w_id, + &fdt.name + ) + .fetch_optional(&mut **tx) + .await? + .flatten() + .ok_or_else(|| { + Error::NotFound(format!( + "Datatable '{}' not found in workspace '{}'", + fdt.name, forked_w_id + )) + })?; + + let dt: DataTable = serde_json::from_value(config_val) + .map_err(|e| Error::internal_err(format!("Failed to parse datatable config: {}", e)))?; + + if dt.database.resource_type == DataTableCatalogResourceType::Instance { + // Instance: update resource_path to the new dbname + sqlx::query!( + r#"UPDATE workspace_settings + SET datatable = jsonb_set( + jsonb_set(datatable, ARRAY['datatables', $2, 'database', 'resource_path'], to_jsonb($3::text)), + ARRAY['datatables', $2, 'forked_from'], $4::jsonb + ) + WHERE workspace_id = $1"#, + forked_w_id, + &fdt.name, + &fdt.new_dbname, + forked_from, + ) + .execute(&mut **tx) + .await?; + } else { + // Resource: update the resource's dbname and set ws_specific + let resource_path = &dt.database.resource_path; + sqlx::query!( + r#"UPDATE resource + SET value = jsonb_set(value, '{dbname}', to_jsonb($3::text)), + ws_specific = true + WHERE workspace_id = $1 AND path = $2"#, + forked_w_id, + resource_path, + &fdt.new_dbname, + ) + .execute(&mut **tx) + .await?; + + // Set forked_from on the datatable config + sqlx::query!( + r#"UPDATE workspace_settings + SET datatable = jsonb_set(datatable, ARRAY['datatables', $2, 'forked_from'], $3::jsonb) + WHERE workspace_id = $1"#, + forked_w_id, + &fdt.name, + forked_from, + ) + .execute(&mut **tx) + .await?; + } + + Ok(()) +} + async fn create_workspace_fork( authed: ApiAuthed, Extension(db): Extension, @@ -3813,6 +4353,11 @@ async fn create_workspace_fork( // Clone all data from the parent workspace using Rust implementation clone_workspace_data(&mut tx, &parent_workspace_id, &forked_id).await?; + // Update forked datatable settings to point to new databases + for fdt in &nw.forked_datatables { + apply_forked_datatable(&db, &mut tx, &parent_workspace_id, &forked_id, fdt).await?; + } + audit_log( &mut *tx, &authed, @@ -3824,6 +4369,7 @@ async fn create_workspace_fork( ) .await?; tx.commit().await?; + Ok(format!("Created forked workspace {}", &forked_id)) } @@ -5535,7 +6081,7 @@ async fn compare_two_resources( ) -> Result { // Get resource from each workspace let source_resource = sqlx::query!( - "SELECT value, description, resource_type + "SELECT value, description, resource_type, ws_specific FROM resource WHERE workspace_id = $1 AND path = $2", source_workspace_id, @@ -5545,7 +6091,7 @@ async fn compare_two_resources( .await?; let target_resource = sqlx::query!( - "SELECT value, description, resource_type + "SELECT value, description, resource_type, ws_specific FROM resource WHERE workspace_id = $1 AND path = $2", fork_workspace_id, @@ -5554,6 +6100,17 @@ async fn compare_two_resources( .fetch_optional(db) .await?; + // If either side is ws_specific, consider unchanged + let source_ws_specific = source_resource.as_ref().map_or(false, |r| r.ws_specific); + let target_ws_specific = target_resource.as_ref().map_or(false, |r| r.ws_specific); + if source_ws_specific || target_ws_specific { + return Ok(ItemComparison { + has_changes: false, + exists_in_source: source_resource.is_some(), + exists_in_fork: target_resource.is_some(), + }); + } + let mut has_changes = false; // Check metadata differences diff --git a/backend/windmill-api-workspaces/src/workspaces_extra.rs b/backend/windmill-api-workspaces/src/workspaces_extra.rs index edd3e2fb37..231f7cb429 100644 --- a/backend/windmill-api-workspaces/src/workspaces_extra.rs +++ b/backend/windmill-api-workspaces/src/workspaces_extra.rs @@ -23,8 +23,10 @@ use windmill_common::worker::CLOUD_HOSTED; use windmill_common::{ auth::is_super_admin_email, + db::UserDB, error::{Error, Result}, utils::require_admin, + workspaces::DataTable, }; use windmill_queue::schedule::{get_schedule_opt, push_scheduled_job}; @@ -858,6 +860,157 @@ pub(crate) async fn delete_workspace( Ok(format!("Deleted workspace {}", &w_id)) } +#[derive(Deserialize)] +pub struct DropForkedDatatableDatabasesRequest { + datatable_names: Vec, +} + +/// Drop forked datatable databases. Returns errors per datatable that failed. +/// Same permission as delete_workspace: fork owner or super admin. +pub async fn drop_forked_datatable_databases( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Path(w_id): Path, + Json(req): Json, +) -> Result>> { + // Same permission check as delete_workspace: fork owner or super admin + let mut tx = db.begin().await?; + if !(w_id.starts_with(WM_FORK_PREFIX) && is_workspace_owner(&authed, &w_id, &mut tx).await?) { + require_super_admin(&db, &authed.email).await?; + } + tx.commit().await?; + + let parent_w_id = sqlx::query_scalar!( + "SELECT parent_workspace_id FROM workspace WHERE id = $1", + &w_id + ) + .fetch_optional(&db) + .await? + .flatten() + .ok_or_else(|| Error::BadRequest("No parent workspace found".to_string()))?; + + let datatable_config = sqlx::query_scalar!( + "SELECT datatable->'datatables' FROM workspace_settings WHERE workspace_id = $1", + &w_id + ) + .fetch_optional(&db) + .await? + .flatten() + .unwrap_or(serde_json::json!({})); + + let datatables: HashMap = + serde_json::from_value(datatable_config).unwrap_or_default(); + + let mut errors: Vec = Vec::new(); + + for dt_name in &req.datatable_names { + let dt = match datatables.get(dt_name) { + Some(dt) if dt.forked_from.is_some() => dt, + _ => continue, + }; + + if dt.database.resource_type + == windmill_common::workspaces::DataTableCatalogResourceType::Instance + { + let db_to_drop = &dt.database.resource_path; + if !db_to_drop.starts_with("wm_fork_") { + errors.push(format!( + "Refusing to drop instance database '{}' for datatable://{}: name does not start with 'wm_fork_'", + db_to_drop, dt_name + )); + continue; + } + if let Err(e) = windmill_common::drop_custom_instance_database(&db, db_to_drop).await { + errors.push(format!( + "Could not drop instance database '{}' for datatable://{}: {}", + db_to_drop, dt_name, e + )); + } + } else { + let fork_pg = match crate::workspaces::resolve_pg_source_checked( + &db, + &user_db, + &authed, + &w_id, + &format!("datatable://{}", dt_name), + ) + .await + { + Ok(pg) => pg, + Err(e) => { + errors.push(format!( + "Could not resolve fork resource for datatable://{}: {}", + dt_name, e + )); + continue; + } + }; + // We cannot drop the current database, so we connect to the parent's version to run DROP DATABASE on + // the forked version + let parent_pg = match crate::workspaces::resolve_pg_source_checked( + &db, + &user_db, + &authed, + &parent_w_id, + &format!("datatable://{}", dt_name), + ) + .await + { + Ok(pg) => pg, + Err(e) => { + errors.push(format!( + "Could not resolve parent resource for datatable://{}: {}", + dt_name, e + )); + continue; + } + }; + + let db_to_drop = &fork_pg.dbname; + if let Err(e) = windmill_common::validate_dbname(db_to_drop) { + errors.push(format!( + "Invalid database name '{}' for datatable://{}: {}", + db_to_drop, dt_name, e + )); + continue; + } + if !db_to_drop.starts_with("wm_fork_") { + errors.push(format!( + "Refusing to drop resource database '{}' for datatable://{}: name does not start with 'wm_fork_'", + db_to_drop, dt_name + )); + continue; + } + + match parent_pg.connect().await { + Ok((client, connection)) => { + let join_handle = tokio::spawn(async move { connection.await }); + if let Err(e) = client + .execute(&format!("DROP DATABASE \"{}\"", db_to_drop), &[]) + .await + { + errors.push(format!( + "Could not drop database '{}' for datatable://{}: {}", + db_to_drop, dt_name, e + )); + } + drop(client); + let _ = join_handle.await; + } + Err(e) => { + errors.push(format!( + "Could not connect to drop database for datatable://{}: {}", + dt_name, e + )); + } + } + } + } + + Ok(Json(errors)) +} + async fn is_workspace_owner( authed: &ApiAuthed, w_id: &str, diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 10b1e470a6..bc87b6c7ec 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1079,6 +1079,27 @@ paths: schema: $ref: "#/components/schemas/CustomInstanceDb" + /settings/drop_custom_instance_pg_database/{name}: + post: + summary: Drops a custom instance database (superadmin only, irreversible) + operationId: dropCustomInstanceDb + tags: + - setting + parameters: + - in: path + name: name + required: true + schema: + type: string + description: The name of the database to drop + responses: + "200": + description: status + content: + text/plain: + schema: + type: string + /settings/global/{key}: get: summary: get global settings @@ -3755,7 +3776,16 @@ paths: schema: type: array items: - type: string + type: object + required: [name, resource_type, resource_path] + properties: + name: + type: string + resource_type: + type: string + enum: [postgres, instance] + resource_path: + type: string /w/{workspace}/workspaces/list_datatable_schemas: get: @@ -3828,6 +3858,211 @@ paths: application/json: schema: {} + /w/{workspace}/workspaces/create_pg_database: + post: + summary: create a new PostgreSQL database for a datatable + operationId: createPgDatabase + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + description: Create pg database request + required: true + content: + application/json: + schema: + type: object + required: [source, target_dbname] + properties: + source: + type: string + description: "Datatable source to determine connection info: 'datatable://name' or '$res:path'" + target_dbname: + type: string + description: "Name for the new database" + responses: + "200": + description: status + content: + text/plain: + schema: + type: string + + /w/{workspace}/workspaces/drop_forked_datatable_databases: + post: + summary: drop forked datatable databases + operationId: dropForkedDatatableDatabases + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [datatable_names] + properties: + datatable_names: + type: array + items: + type: string + responses: + "200": + description: list of errors (empty if all succeeded) + content: + application/json: + schema: + type: array + items: + type: string + + /w/{workspace}/workspaces/import_pg_database: + post: + summary: import a PostgreSQL database from source to target via pg_dump + operationId: importPgDatabase + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + description: Import pg database request + required: true + content: + application/json: + schema: + type: object + required: [source, target, fork_behavior] + properties: + source: + type: string + description: "Source database: 'datatable://name' or '$res:path'" + target: + type: string + description: "Target database: 'datatable://name' or '$res:path'" + target_dbname_override: + type: string + description: "Override the target database name" + fork_behavior: + type: string + enum: + - schema_only + - schema_and_data + - keep_original + responses: + "200": + description: status + content: + text/plain: + schema: + type: string + + /w/{workspace}/workspaces/export_pg_schema: + post: + summary: export the schema of a PostgreSQL database + operationId: exportPgSchema + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + description: Export pg schema request + required: true + content: + application/json: + schema: + type: object + required: [source] + properties: + source: + type: string + description: "Source database: 'datatable://name' or '$res:path'" + responses: + "200": + description: schema dump + content: + text/plain: + schema: + type: string + + /w/{workspace}/workspaces/get_datatable_full_schema: + post: + summary: get the full schema of a datatable database as TableEditorValues + operationId: getDatatableFullSchema + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [source] + properties: + source: + type: string + description: "Source datatable, e.g. 'datatable://main'" + responses: + "200": + description: "Schema as { schema_name: { table_name: TableEditorValues } }" + content: + application/json: + schema: + type: object + additionalProperties: + type: object + additionalProperties: + type: object + required: [name, columns, foreign_keys] + properties: + name: + type: string + columns: + type: array + items: + type: object + required: [name, datatype] + properties: + name: + type: string + datatype: + type: string + primary_key: + type: boolean + default_value: + type: string + nullable: + type: boolean + foreign_keys: + type: array + items: + type: object + required: [columns, on_delete, on_update] + properties: + target_table: + type: string + columns: + type: array + items: + type: object + properties: + source_column: + type: string + target_column: + type: string + on_delete: + type: string + on_update: + type: string + fk_constraint_name: + type: string + pk_constraint_name: + type: string + /w/{workspace}/workspaces/git_sync_enabled: get: summary: Check if git sync is available for this workspace @@ -18372,7 +18607,7 @@ paths: type: object description: The columns used (for tables) additionalProperties: - $ref: "#/components/schemas/AssetUsageAccessType" + $ref: "#/components/schemas/AssetUsageAccessType" created_at: type: string format: date-time @@ -18476,7 +18711,6 @@ paths: type: string description: The asset path - /w/{workspace}/volumes/list: get: summary: List all volumes in the workspace @@ -21423,6 +21657,9 @@ components: resource_type: type: string description: The new resource_type to be associated with the resource + ws_specific: + type: boolean + description: When true, the resource is excluded from workspace diff comparisons labels: type: array items: @@ -24220,6 +24457,18 @@ components: type: string color: type: string + forked_datatables: + type: array + items: + type: object + required: [name, new_dbname] + properties: + name: + type: string + description: "Datatable name" + new_dbname: + type: string + description: "New database name for the fork" required: - id - name @@ -24871,7 +25120,14 @@ components: type: string required: - resource_type - + forked_from: + type: object + description: Fork origin info with schema snapshot + properties: + schema: + type: object + description: Schema snapshot at fork time + additionalProperties: true DataTableSchema: type: object required: [datatable_name, schemas] @@ -25453,7 +25709,15 @@ components: kind: type: string enum: - ["script", "flow", "app", "raw_app", "resource", "variable", "resource_type"] + [ + "script", + "flow", + "app", + "raw_app", + "resource", + "variable", + "resource_type", + ] description: Type of the item path: type: string diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index 42151553bb..dcc87b416a 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -401,7 +401,7 @@ pub struct PrepareQueryResult { pub error: Option, } -#[derive(Deserialize, Serialize)] +#[derive(Deserialize, Serialize, Clone)] pub struct PgDatabase { pub host: String, pub user: Option, @@ -632,6 +632,180 @@ impl PgDatabase { } } +/// Validate a database name to prevent SQL injection. +/// Must start with a letter, contain only alphanumeric characters or underscores, and be <= 63 chars. +pub fn validate_dbname(dbname: &str) -> error::Result<()> { + let dbname = dbname.trim(); + if dbname.is_empty() { + return Err(error::Error::BadRequest( + "Database name cannot be empty".to_string(), + )); + } + if dbname.len() > 63 { + return Err(error::Error::BadRequest( + "Database name cannot exceed 63 characters".to_string(), + )); + } + if !dbname + .chars() + .next() + .map_or(false, |c| c.is_ascii_alphabetic()) + { + return Err(error::Error::BadRequest( + "Database name must start with a letter".to_string(), + )); + } + if !dbname + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '_') + { + return Err(error::Error::BadRequest( + "Database name must contain only alphanumeric characters or underscores".to_string(), + )); + } + Ok(()) +} + +/// Drop a custom instance database: validate, terminate connections, DROP DATABASE, remove from global_settings. +pub async fn drop_custom_instance_database(db: &DB, dbname: &str) -> error::Result<()> { + let dbname = dbname.trim(); + validate_dbname(dbname)?; + + let wmill_pg_creds = PgDatabase::parse_uri(&get_database_url().await?.as_str().await)?; + if wmill_pg_creds.dbname.trim().eq_ignore_ascii_case(dbname) { + return Err(error::Error::BadRequest( + "Cannot drop the main Windmill database".to_string(), + )); + } + + let db_exists = sqlx::query_scalar!( + "SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_database WHERE datname = $1)", + dbname + ) + .fetch_one(db) + .await? + .unwrap_or(false); + + if db_exists { + // Terminate active connections + if let Err(e) = sqlx::query(&format!( + "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '{}' AND pid <> pg_backend_pid()", + dbname.replace('\'', "''") + )) + .execute(db) + .await + { + tracing::warn!("Failed to terminate connections to '{}': {}", dbname, e); + } + + // Drop the database + sqlx::query(&format!("DROP DATABASE IF EXISTS \"{}\"", dbname)) + .execute(db) + .await + .map_err(|e| { + error::Error::internal_err(format!("Failed to drop database '{}': {}", dbname, e)) + })?; + + tracing::info!("Dropped instance database '{}'", dbname); + } else { + tracing::info!("Database '{}' does not exist, skipping drop", dbname); + } + + // Always remove from global_settings + sqlx::query!( + r#"UPDATE global_settings SET value = value #- ARRAY['databases', $1] WHERE name = 'custom_instance_pg_databases'"#, + dbname + ) + .execute(db) + .await?; + + Ok(()) +} + +/// Create a custom instance database: CREATE DATABASE, grant permissions, register in global_settings. +/// The `tag` is stored in global_settings metadata (e.g. "datatable" or "ducklake"). +pub async fn create_custom_instance_database( + db: &DB, + dbname: &str, + tag: &str, +) -> error::Result<()> { + let dbname = dbname.trim(); + validate_dbname(dbname)?; + + let db_exists = sqlx::query_scalar!( + "SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_database WHERE datname = $1)", + dbname + ) + .fetch_one(db) + .await? + .unwrap_or(false); + + if db_exists { + return Err(error::Error::BadRequest(format!( + "Database '{}' already exists", + dbname + ))); + } + + sqlx::query(&format!("CREATE DATABASE \"{}\"", dbname)) + .execute(db) + .await + .map_err(|e| { + error::Error::internal_err(format!("Failed to create database '{}': {}", dbname, e)) + })?; + + // Grant permissions to custom_instance_user + let wmill_pg_creds = PgDatabase::parse_uri(&get_database_url().await?.as_str().await)?; + let new_pg_creds = PgDatabase { dbname: dbname.to_string(), ..wmill_pg_creds }; + let (client, connection) = new_pg_creds.connect().await?; + let join_handle = tokio::spawn(async move { connection.await }); + + if let Err(e) = client + .batch_execute(&format!( + "GRANT CONNECT ON DATABASE \"{dbname}\" TO custom_instance_user; + GRANT USAGE ON SCHEMA public TO custom_instance_user; + GRANT CREATE ON SCHEMA public TO custom_instance_user; + GRANT CREATE ON DATABASE \"{dbname}\" TO custom_instance_user; + ALTER DEFAULT PRIVILEGES IN SCHEMA public + GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO custom_instance_user;" + )) + .await + { + tracing::warn!( + "Failed to grant permissions on '{}': {}. Continuing.", + dbname, + e + ); + } + + drop(client); + join_handle + .await + .map_err(|e| error::Error::internal_err(format!("join error: {}", e)))? + .map_err(|e| error::Error::internal_err(format!("tokio_postgres error: {}", e)))?; + + // Register in global_settings + let status_json = serde_json::json!({ + "logs": { + "created_database": "OK", + "db_connect": "OK", + "grant_permissions": "OK" + }, + "success": true, + "error": null, + "tag": tag + }); + sqlx::query!( + r#"UPDATE global_settings SET value = jsonb_set(value, '{databases}', (COALESCE(value->'databases', '{}'::jsonb) || to_jsonb($1::json))) WHERE name = 'custom_instance_pg_databases'"#, + serde_json::json!({ (dbname): status_json }) + ) + .execute(db) + .await?; + + tracing::info!("Created custom instance database '{}'", dbname); + Ok(()) +} + #[derive(Clone)] pub enum DatabaseUrl { #[cfg(all(feature = "enterprise", feature = "private"))] diff --git a/backend/windmill-common/src/query_builders.rs b/backend/windmill-common/src/query_builders.rs index d28afb22ba..4e8c93b1f8 100644 --- a/backend/windmill-common/src/query_builders.rs +++ b/backend/windmill-common/src/query_builders.rs @@ -4471,3 +4471,224 @@ mod tests { assert_eq!(sql, "SHOW PRIMARY KEYS IN ACCOUNT"); } } + +// ============================================================================ +// Full schema introspection types and logic (used by get_datatable_full_schema) +// ============================================================================ + +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct FullSchemaColumn { + pub name: String, + pub datatype: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub primary_key: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub default_value: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub nullable: Option, +} + +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct FullSchemaForeignKeyColumn { + #[serde(skip_serializing_if = "Option::is_none")] + pub source_column: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub target_column: Option, +} + +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct FullSchemaForeignKey { + #[serde(skip_serializing_if = "Option::is_none")] + pub target_table: Option, + pub columns: Vec, + pub on_delete: String, + pub on_update: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub fk_constraint_name: Option, +} + +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct FullSchemaTable { + pub name: String, + pub columns: Vec, + pub foreign_keys: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub pk_constraint_name: Option, +} + +/// Full database schema: { schema_name: { table_name: FullSchemaTable } } +pub type FullDatabaseSchema = + std::collections::HashMap>; + +fn pg_action_to_string(action: &str) -> String { + match action { + "a" => "NO ACTION".to_string(), + "r" => "RESTRICT".to_string(), + "c" => "CASCADE".to_string(), + "n" => "SET NULL".to_string(), + "d" => "SET DEFAULT".to_string(), + _ => "NO ACTION".to_string(), + } +} + +/// Introspect a PostgreSQL database and return the full schema. +/// Takes a connected tokio_postgres Client. +pub async fn pg_get_full_schema( + client: &tokio_postgres::Client, +) -> Result { + let column_rows = client + .query( + "SELECT + ns.nspname AS schema_name, + c.relname AS table_name, + a.attname AS column_name, + pg_catalog.format_type(a.atttypid, a.atttypmod) AS datatype, + (SELECT substring(pg_catalog.pg_get_expr(d.adbin, d.adrelid, true) for 128) + FROM pg_catalog.pg_attrdef d + WHERE d.adrelid = a.attrelid AND d.adnum = a.attnum AND a.atthasdef) AS default_value, + CASE a.attnotnull WHEN false THEN true ELSE false END AS nullable, + EXISTS ( + SELECT 1 FROM pg_catalog.pg_index i + WHERE i.indrelid = c.oid AND i.indisprimary AND a.attnum = ANY(i.indkey) + ) AS is_primary_key, + (SELECT con.conname FROM pg_catalog.pg_constraint con + WHERE con.conrelid = c.oid AND con.contype = 'p' LIMIT 1) AS pk_constraint_name + FROM pg_catalog.pg_attribute a + JOIN pg_catalog.pg_class c ON a.attrelid = c.oid + JOIN pg_catalog.pg_namespace ns ON c.relnamespace = ns.oid + WHERE c.relkind = 'r' + AND a.attnum > 0 + AND NOT a.attisdropped + AND ns.nspname NOT IN ('pg_catalog', 'information_schema') + ORDER BY ns.nspname, c.relname, a.attnum", + &[], + ) + .await + .map_err(|e| format!("Failed to query columns: {}", e))?; + + let fk_rows = client + .query( + "SELECT + ns.nspname AS schema_name, + c.relname AS table_name, + con.conname AS fk_constraint_name, + att_src.attname AS source_column, + ns_ref.nspname AS ref_schema, + c_ref.relname AS ref_table, + att_ref.attname AS ref_column, + con.confdeltype::text AS on_delete, + con.confupdtype::text AS on_update + FROM pg_catalog.pg_constraint con + JOIN pg_catalog.pg_class c ON con.conrelid = c.oid + JOIN pg_catalog.pg_namespace ns ON c.relnamespace = ns.oid + JOIN pg_catalog.pg_class c_ref ON con.confrelid = c_ref.oid + JOIN pg_catalog.pg_namespace ns_ref ON c_ref.relnamespace = ns_ref.oid + CROSS JOIN LATERAL unnest(con.conkey, con.confkey) WITH ORDINALITY AS u(src_attnum, ref_attnum, ord) + JOIN pg_catalog.pg_attribute att_src ON att_src.attrelid = c.oid AND att_src.attnum = u.src_attnum + JOIN pg_catalog.pg_attribute att_ref ON att_ref.attrelid = c_ref.oid AND att_ref.attnum = u.ref_attnum + WHERE con.contype = 'f' + AND ns.nspname NOT IN ('pg_catalog', 'information_schema') + ORDER BY ns.nspname, c.relname, con.conname, u.ord", + &[], + ) + .await + .map_err(|e| format!("Failed to query foreign keys: {}", e))?; + + let mut result: FullDatabaseSchema = std::collections::HashMap::new(); + + for row in &column_rows { + let schema_name: &str = row.get("schema_name"); + let table_name: &str = row.get("table_name"); + let column_name: &str = row.get("column_name"); + let datatype: &str = row.get("datatype"); + let default_value: Option<&str> = row.get("default_value"); + let nullable: bool = row.get("nullable"); + let is_primary_key: bool = row.get("is_primary_key"); + let pk_constraint_name: Option<&str> = row.get("pk_constraint_name"); + + let schema_tables = result.entry(schema_name.to_string()).or_default(); + let table = schema_tables + .entry(table_name.to_string()) + .or_insert_with(|| FullSchemaTable { + name: table_name.to_string(), + columns: vec![], + foreign_keys: vec![], + pk_constraint_name: pk_constraint_name.map(|s| s.to_string()), + }); + + table.columns.push(FullSchemaColumn { + name: column_name.to_string(), + datatype: datatype.to_string(), + primary_key: if is_primary_key { Some(true) } else { None }, + default_value: default_value.map(|s| s.to_string()), + nullable: Some(nullable), + }); + } + + let mut fk_map: std::collections::HashMap< + (String, String, String), + ( + Option, + Vec, + String, + String, + ), + > = std::collections::HashMap::new(); + + for row in &fk_rows { + let schema_name: &str = row.get("schema_name"); + let table_name: &str = row.get("table_name"); + let fk_name: &str = row.get("fk_constraint_name"); + let source_column: &str = row.get("source_column"); + let ref_schema: &str = row.get("ref_schema"); + let ref_table: &str = row.get("ref_table"); + let ref_column: &str = row.get("ref_column"); + let on_delete: &str = row.get("on_delete"); + let on_update: &str = row.get("on_update"); + + let target_table = if ref_schema == schema_name { + ref_table.to_string() + } else { + format!("{}.{}", ref_schema, ref_table) + }; + + let key = ( + schema_name.to_string(), + table_name.to_string(), + fk_name.to_string(), + ); + let entry = fk_map.entry(key).or_insert_with(|| { + ( + Some(target_table.clone()), + vec![], + pg_action_to_string(on_delete), + pg_action_to_string(on_update), + ) + }); + entry.1.push(FullSchemaForeignKeyColumn { + source_column: Some(source_column.to_string()), + target_column: Some(ref_column.to_string()), + }); + } + + let mut fk_entries: Vec<_> = fk_map.into_iter().collect(); + fk_entries.sort_by(|a, b| a.0.cmp(&b.0)); + + for ((schema_name, table_name, fk_name), (target_table, columns, on_delete, on_update)) in + fk_entries + { + if let Some(schema_tables) = result.get_mut(&schema_name) { + if let Some(table) = schema_tables.get_mut(&table_name) { + table.foreign_keys.push(FullSchemaForeignKey { + target_table, + columns, + on_delete, + on_update, + fk_constraint_name: Some(fk_name), + }); + } + } + } + + Ok(result) +} diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index 87d2b8917d..4f1e1fbea2 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -389,9 +389,32 @@ pub async fn check_user_against_rule( Ok(RuleCheckResult::Allowed) } +#[derive(Deserialize, Serialize, Debug, Clone, Copy, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum DataTableForkBehavior { + SchemaOnly, + SchemaAndData, + KeepOriginal, +} + +impl Default for DataTableForkBehavior { + fn default() -> Self { + DataTableForkBehavior::KeepOriginal + } +} + #[derive(Deserialize, Serialize, Debug)] pub struct DataTable { pub database: DataTableDatabase, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub forked_from: Option, +} + +#[derive(Deserialize, Serialize, Debug)] +pub struct DataTableForkedFrom { + /// Schema snapshot at fork time + #[serde(default, skip_serializing_if = "Option::is_none")] + pub schema: Option, } #[derive(Deserialize, Serialize, Debug)] diff --git a/backend/windmill-store/src/resources.rs b/backend/windmill-store/src/resources.rs index 42649129d0..c19ba8a834 100644 --- a/backend/windmill-store/src/resources.rs +++ b/backend/windmill-store/src/resources.rs @@ -125,6 +125,8 @@ pub struct Resource { pub extra_perms: serde_json::Value, pub created_by: Option, pub edited_at: Option>, + #[serde(default)] + pub ws_specific: bool, #[serde(skip_serializing_if = "Option::is_none")] pub labels: Option>, } @@ -145,6 +147,8 @@ pub struct ListableResource { pub is_expired: Option, pub refresh_error: Option, pub account: Option, + #[serde(default)] + pub ws_specific: bool, #[serde(skip_serializing_if = "Option::is_none")] pub labels: Option>, } @@ -156,6 +160,7 @@ pub struct CreateResource { pub description: Option, pub resource_type: String, #[serde(default)] + pub ws_specific: Option, pub labels: Option>, } #[derive(Deserialize)] @@ -163,6 +168,7 @@ struct EditResource { path: Option, description: Option, value: Option>, + ws_specific: Option, labels: Option>, } @@ -260,6 +266,7 @@ async fn list_resources( "account.refresh_error", "resource.created_by", "resource.edited_at", + "resource.ws_specific", "resource.labels", ]) .left() @@ -836,15 +843,16 @@ async fn create_resource( } sqlx::query!( "INSERT INTO resource - (workspace_id, path, value, description, resource_type, created_by, edited_at, labels) - VALUES ($1, $2, $3, $4, $5, $6, now(), $7) ON CONFLICT (workspace_id, path) - DO UPDATE SET value = EXCLUDED.value, description = EXCLUDED.description, resource_type = EXCLUDED.resource_type, edited_at = now(), labels = EXCLUDED.labels", + (workspace_id, path, value, description, resource_type, created_by, edited_at, ws_specific, labels) + VALUES ($1, $2, $3, $4, $5, $6, now(), $7, $8) ON CONFLICT (workspace_id, path) + DO UPDATE SET value = EXCLUDED.value, description = EXCLUDED.description, resource_type = EXCLUDED.resource_type, edited_at = now(), ws_specific = EXCLUDED.ws_specific, labels = EXCLUDED.labels", w_id, resource.path, raw_json as sqlx::types::Json<&RawValue>, resource.description, resource.resource_type, authed.username, + resource.ws_specific.unwrap_or(false), resource.labels.as_deref() as Option<&[String]> ) .execute(&mut *tx) @@ -1212,6 +1220,10 @@ async fn update_resource( if let Some(ndesc) = ns.description { sqlb.set_str("description", ndesc); } + if let Some(nd) = ns.ws_specific { + sqlb.set_str("ws_specific", if nd { "true" } else { "false" }); + } + sqlb.set_str("edited_at", "now()"); sqlb.returning("path"); diff --git a/cli/src/commands/resource/resource.ts b/cli/src/commands/resource/resource.ts index e1c9052261..aefc8edeb7 100644 --- a/cli/src/commands/resource/resource.ts +++ b/cli/src/commands/resource/resource.ts @@ -26,6 +26,7 @@ export interface ResourceFile { description?: string; resource_type: string; is_oauth?: boolean; // deprecated + ws_specific?: boolean; } async function readFilesetDirectory(dirPath: string): Promise> { diff --git a/docker/DockerfileSlim b/docker/DockerfileSlim index 61a365e496..181d87b380 100644 --- a/docker/DockerfileSlim +++ b/docker/DockerfileSlim @@ -36,7 +36,15 @@ ENV PATH=/usr/local/bin:/root/.local/bin:/tmp/.local/bin:$PATH # Install system dependencies RUN apt-get update \ - && apt-get install -y ca-certificates wget curl git jq unzip unixodbc xmlsec1 \ + && apt-get install -y ca-certificates wget curl git jq unzip unixodbc xmlsec1 gnupg lsb-release \ + && rm -rf /var/lib/apt/lists/* + +# Install latest PostgreSQL client (pg_dump) from official PostgreSQL apt repository +RUN curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor -o /usr/share/keyrings/postgresql-archive-keyring.gpg \ + && echo "deb [signed-by=/usr/share/keyrings/postgresql-archive-keyring.gpg] https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list \ + && apt-get update \ + && apt-get install -y --no-install-recommends postgresql-client \ + && apt-get clean \ && rm -rf /var/lib/apt/lists/* ENV TZ=Etc/UTC diff --git a/docker/DockerfileSlimEe b/docker/DockerfileSlimEe index 7ba85a0358..cc74d14996 100644 --- a/docker/DockerfileSlimEe +++ b/docker/DockerfileSlimEe @@ -36,7 +36,15 @@ ENV PATH=/usr/local/bin:/root/.local/bin:/tmp/.local/bin:$PATH # Install system dependencies RUN apt-get update \ - && apt-get install -y ca-certificates wget curl git jq unzip unixodbc xmlsec1 \ + && apt-get install -y ca-certificates wget curl git jq unzip unixodbc xmlsec1 gnupg lsb-release \ + && rm -rf /var/lib/apt/lists/* + +# Install latest PostgreSQL client (pg_dump) from official PostgreSQL apt repository +RUN curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor -o /usr/share/keyrings/postgresql-archive-keyring.gpg \ + && echo "deb [signed-by=/usr/share/keyrings/postgresql-archive-keyring.gpg] https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list \ + && apt-get update \ + && apt-get install -y --no-install-recommends postgresql-client \ + && apt-get clean \ && rm -rf /var/lib/apt/lists/* ENV TZ=Etc/UTC diff --git a/frontend/src/lib/components/CompareWorkspaces.svelte b/frontend/src/lib/components/CompareWorkspaces.svelte index 4f9184c93c..b8da7576bb 100644 --- a/frontend/src/lib/components/CompareWorkspaces.svelte +++ b/frontend/src/lib/components/CompareWorkspaces.svelte @@ -60,6 +60,7 @@ import { base } from '$lib/base' import ToggleButtonGroup from './common/toggleButton-v2/ToggleButtonGroup.svelte' import ToggleButton from './common/toggleButton-v2/ToggleButton.svelte' + import DatatableSchemaDiff from './DatatableSchemaDiff.svelte' interface Props { currentWorkspaceId: string @@ -716,382 +717,408 @@ {@const selectedConflicts = conflictingDiffs.filter((e) => selectedItems.includes(getItemKey(e)) ).length} - - selectableDiffs.some((d) => getItemKey(d) === item.key)} - {allSelected} - onToggleItem={(item) => { - const diff = comparison?.diffs.find((d) => getItemKey(d) === item.key) - if (diff) toggleItem(diff) - }} - onSelectAll={selectAll} - onDeselectAll={deselectAll} - emptyMessage="No comparison data available" - > - {#snippet header()} -
-
-
- - {#snippet children({ item })} - - - {/snippet} - - {#if currentWorkspaceInfo && parentWorkspaceInfo} - - merge: - {#if mergeIntoParent} - - {currentWorkspaceInfo.id} - {:else} - - {parentWorkspaceInfo.id} +
+
+ selectableDiffs.some((d) => getItemKey(d) === item.key)} + {allSelected} + onToggleItem={(item) => { + const diff = comparison?.diffs.find((d) => getItemKey(d) === item.key) + if (diff) toggleItem(diff) + }} + onSelectAll={selectAll} + onDeselectAll={deselectAll} + emptyMessage="No comparison data available" + > + {#snippet header()} +
+
+
+ + {#snippet children({ item })} + + + {/snippet} + + {#if currentWorkspaceInfo && parentWorkspaceInfo} +
+ + merge: + {#if mergeIntoParent} + + {currentWorkspaceInfo.id} + {:else} + + {parentWorkspaceInfo.id} + {/if} + + + + into: + {#if !mergeIntoParent} + + {currentWorkspaceInfo.id} + {:else} + + {parentWorkspaceInfo.id} + {/if} + +
{/if} - - - - into: - {#if !mergeIntoParent} - - {currentWorkspaceInfo.id} - {:else} - - {parentWorkspaceInfo.id} - {/if} - - {/if} +
+ + {comparison.summary.total_diffs} total items + + + {selectableDiffs.length} + {mergeIntoParent ? 'deployable' : 'updateable'} + + {#if conflictingDiffs.length > 0} + + + {conflictingDiffs.length} conflicts + + {/if} +
+
+
-
- - {comparison.summary.total_diffs} total items - - - {selectableDiffs.length} - {mergeIntoParent ? 'deployable' : 'updateable'} - - {#if conflictingDiffs.length > 0} - - - {conflictingDiffs.length} conflicts - - {/if} -
-
-
- {/snippet} + {/snippet} - {#snippet alerts()} - {#if mergeIntoParent} - { - canDeployToParent = canDeploy - }} - /> - {/if} - {#if conflictingDiffs.length > 0} - - - {conflictingDiffs.length} item{conflictingDiffs.length !== 1 ? 's have' : ' has'} conflicting - changes, it was modified on the original workspace while changes were made on this fork. - Make sure to resolve these before merging. - - - {/if} - {#if hasBehindChanges && hasAheadChanges && !(mergeIntoParent && !canDeployToParent)} - - You have items behind '{parentWorkspaceId}'. You need to update and test your changes - before being able to deploy. - - { + canDeployToParent = canDeploy + }} /> - Override: Deploy despite {itemsWithBehindChanges.length} outdated item{itemsWithBehindChanges.length !== - 1 - ? 's' - : ''} - - - {/if} - {#if !comparison.all_ahead_items_visible || !comparison.all_behind_items_visible} - - {#if !comparison.all_ahead_items_visible && !comparison.all_behind_items_visible} - This fork is ahead and behind its parent - {:else if !comparison.all_behind_items_visible} - This fork is behind of its parent - {:else if !comparison.all_ahead_items_visible} - This fork is ahead of its parent {/if} - and some of the changes are not visible by you. Only a user with access to the whole context - may deploy or update this fork. You can share the link to this page to someone with proper - permissions to get it deployed. - - {/if} - {/snippet} - - {#snippet itemSummary(item)} - {@const diff = item.diff as WorkspaceItemDiff} - {@const key = item.key} - {@const isSelectable = selectableDiffs.includes(diff)} - {@const oldSummary = mergeIntoParent ? summaryCache[key]?.parent : summaryCache[key]?.current} - {@const newSummary = mergeIntoParent ? summaryCache[key]?.current : summaryCache[key]?.parent} - {@const existsInBothWorkspaces = !( - (diff.exists_in_fork && !diff.exists_in_source) || - (!diff.exists_in_fork && diff.exists_in_source) - )} - {#if oldSummary != newSummary && isSelectable && existsInBothWorkspaces} - {oldSummary || diff.path} - {newSummary || diff.path} - {:else if !existsInBothWorkspaces} - {newSummary || oldSummary || diff.path} - {:else} - {newSummary || diff.path} - {/if} - {/snippet} - - {#snippet itemActions(item)} - {@const diff = item.diff as WorkspaceItemDiff} - {@const key = item.key} - {@const targetOnBehalfOf = getTargetOnBehalfOf(key)} - {@const isConflict = diff.ahead > 0 && diff.behind > 0} - {@const existsInBothWorkspaces = !( - (diff.exists_in_fork && !diff.exists_in_source) || - (!diff.exists_in_fork && diff.exists_in_source) - )} - - {#if itemNeedsOnBehalfOfSelection(key, diff.kind)} - { - onBehalfOfChoice[key] = choice - if (details) customOnBehalfOf[key] = details - }} - kind={diff.kind} - canPreserve={canPreserveOnBehalfOf} - customValue={customOnBehalfOf[key]?.permissionedAs} - /> - {/if} - {#if diff.kind === 'raw_app'} - Raw - {/if} - - {#if !diff.exists_in_fork && diff.exists_in_source && diff.ahead == 0 && diff.behind > 0} - New - {/if} - {#if !diff.exists_in_fork && diff.exists_in_source && diff.ahead > 0} - Deleted - {/if} - {#if diff.exists_in_fork && !diff.exists_in_source && diff.behind > 0} - Deleted - {/if} - {#if diff.exists_in_fork && !diff.exists_in_source && diff.ahead > 0 && diff.behind == 0} - New - {/if} - {#if !deploymentStatus[key] || deploymentStatus[key].status != 'deployed'} -
- {#if isConflict || existsInBothWorkspaces} - {#if diff.ahead > 0} - - - {diff.ahead} ahead - - {/if} - {#if diff.behind > 0} - - - {diff.behind} behind - - {/if} - {#if isConflict} - - - Conflict - - {/if} - {/if} -
-
- -
- {/if} - {/snippet} - - {#snippet footer()} -
-
- -
- {#if comparison.all_behind_items_visible && comparison.all_ahead_items_visible} - {#if !(mergeIntoParent && !canDeployToParent)} - - {#if hasUnselectedOnBehalfOf} - - You must set the "on behalf of" user for all items before deploying - - The "run on behalf of" field defines which user's permissions will be applied - during execution. Make sure this is set to an appropriate user before deploying. - - - {/if} - {/if} - {/if} - - {#if deploymentErrorMessage != ''} - + {#if conflictingDiffs.length > 0} + - {deploymentErrorMessage} + {conflictingDiffs.length} item{conflictingDiffs.length !== 1 ? 's have' : ' has'} conflicting + changes, it was modified on the original workspace while changes were made on this fork. + Make sure to resolve these before merging. {/if} -
-
- {/snippet} - - - -
-
-

Triggers created in this fork

- {#if !loadingTriggers} - {forkTriggers.length} trigger{forkTriggers.length !== 1 ? 's' : ''} - {/if} -
- - - When forking a workspace, triggers are not forked to avoid unnecessary executions or - collisions. If you created this triggers with the intention of deploying them to the parent - workspace, you can do so here. Otherwise it is recommended to delete them or disable them. - - - {#if loadingTriggers} -
- - Loading triggers... -
- {:else if forkTriggers.length === 0} -
- No triggers in this fork workspace. -
- {:else} -
- {#each forkTriggers as trigger (trigger.triggerKind + ':' + trigger.path)} - - {#snippet customSummary()} - {getTriggerDisplayName(trigger.triggerKind)} - → - {trigger.scriptPath} - {#if trigger.isFlow} - flow + {#if hasBehindChanges && hasAheadChanges && !(mergeIntoParent && !canDeployToParent)} + + You have items behind '{parentWorkspaceId}'. You need to update and test your changes + before being able to deploy. + + + Override: Deploy despite {itemsWithBehindChanges.length} outdated item{itemsWithBehindChanges.length !== + 1 + ? 's' + : ''} + + + {/if} + {#if !comparison.all_ahead_items_visible || !comparison.all_behind_items_visible} + + {#if !comparison.all_ahead_items_visible && !comparison.all_behind_items_visible} + This fork is ahead and behind its parent + {:else if !comparison.all_behind_items_visible} + This fork is behind of its parent + {:else if !comparison.all_ahead_items_visible} + This fork is ahead of its parent {/if} - {#if trigger.extraLabel} - ({trigger.extraLabel}) - {/if} - {/snippet} - {#snippet actions()} - {#if trigger.enabled != null} - - {trigger.enabled ? 'Enabled' : 'Disabled'} - + and some of the changes are not visible by you. Only a user with access to the whole context + may deploy or update this fork. You can share the link to this page to someone with proper + permissions to get it deployed. + + {/if} + {/snippet} + + {#snippet itemSummary(item)} + {@const diff = item.diff as WorkspaceItemDiff} + {@const key = item.key} + {@const isSelectable = selectableDiffs.includes(diff)} + {@const oldSummary = mergeIntoParent + ? summaryCache[key]?.parent + : summaryCache[key]?.current} + {@const newSummary = mergeIntoParent + ? summaryCache[key]?.current + : summaryCache[key]?.parent} + {@const existsInBothWorkspaces = !( + (diff.exists_in_fork && !diff.exists_in_source) || + (!diff.exists_in_fork && diff.exists_in_source) + )} + {#if oldSummary != newSummary && isSelectable && existsInBothWorkspaces} + {oldSummary || diff.path} + {newSummary || diff.path} + {:else if !existsInBothWorkspaces} + {newSummary || oldSummary || diff.path} + {:else} + {newSummary || diff.path} + {/if} + {/snippet} + + {#snippet itemActions(item)} + {@const diff = item.diff as WorkspaceItemDiff} + {@const key = item.key} + {@const targetOnBehalfOf = getTargetOnBehalfOf(key)} + {@const isConflict = diff.ahead > 0 && diff.behind > 0} + {@const existsInBothWorkspaces = !( + (diff.exists_in_fork && !diff.exists_in_source) || + (!diff.exists_in_fork && diff.exists_in_source) + )} + + {#if itemNeedsOnBehalfOfSelection(key, diff.kind)} + { + onBehalfOfChoice[key] = choice + if (details) customOnBehalfOf[key] = details + }} + kind={diff.kind} + canPreserve={canPreserveOnBehalfOf} + customValue={customOnBehalfOf[key]?.permissionedAs} + /> + {/if} + {#if diff.kind === 'raw_app'} + Raw + {/if} + + {#if !diff.exists_in_fork && diff.exists_in_source && diff.ahead == 0 && diff.behind > 0} + New + {/if} + {#if !diff.exists_in_fork && diff.exists_in_source && diff.ahead > 0} + Deleted + {/if} + {#if diff.exists_in_fork && !diff.exists_in_source && diff.behind > 0} + Deleted + {/if} + {#if diff.exists_in_fork && !diff.exists_in_source && diff.ahead > 0 && diff.behind == 0} + New + {/if} + {#if !deploymentStatus[key] || deploymentStatus[key].status != 'deployed'} +
+ {#if isConflict || existsInBothWorkspaces} + {#if diff.ahead > 0} + + + {diff.ahead} ahead + + {/if} + {#if diff.behind > 0} + + + {diff.behind} behind + + {/if} + {#if isConflict} + + + Conflict + + {/if} {/if} +
+
- - {/snippet} - - {/each} +
+ {/if} + {/snippet} + + {#snippet footer()} +
+
+ +
+ {#if comparison.all_behind_items_visible && comparison.all_ahead_items_visible} + {#if !(mergeIntoParent && !canDeployToParent)} + + {#if hasUnselectedOnBehalfOf} + + You must set the "on behalf of" user for all items before deploying + + The "run on behalf of" field defines which user's permissions will be + applied during execution. Make sure this is set to an appropriate user + before deploying. + + + {/if} + {/if} + {/if} + + {#if deploymentErrorMessage != ''} + + + {deploymentErrorMessage} + + + {/if} +
+
+ {/snippet} + +
+ + +
+
+

Triggers created in this fork

+ {#if !loadingTriggers} + {forkTriggers.length} trigger{forkTriggers.length !== 1 ? 's' : ''} + {/if}
- {/if} + + + When forking a workspace, triggers are not forked to avoid unnecessary executions or + collisions. If you created this triggers with the intention of deploying them to the parent + workspace, you can do so here. Otherwise it is recommended to delete them or disable them. + + + {#if loadingTriggers} +
+ + Loading triggers... +
+ {:else if forkTriggers.length === 0} +
+ No triggers in this fork workspace. +
+ {:else} +
+ {#each forkTriggers as trigger (trigger.triggerKind + ':' + trigger.path)} + + {#snippet customSummary()} + {getTriggerDisplayName(trigger.triggerKind)} + → + {trigger.scriptPath} + {#if trigger.isFlow} + flow + {/if} + {#if trigger.extraLabel} + ({trigger.extraLabel}) + {/if} + {/snippet} + {#snippet actions()} + {#if trigger.enabled != null} + + {trigger.enabled ? 'Enabled' : 'Disabled'} + + {/if} + + + {/snippet} + + {/each} +
+ {/if} +
+ +
+ +
diff --git a/frontend/src/lib/components/DBManager.svelte b/frontend/src/lib/components/DBManager.svelte index 2b7e643451..46177bc494 100644 --- a/frontend/src/lib/components/DBManager.svelte +++ b/frontend/src/lib/components/DBManager.svelte @@ -1,6 +1,14 @@ @@ -96,6 +175,9 @@ bind:hasReplResult bind:selectedSchemaKey={uriState.selectedSchema} bind:selectedTableKey={uriState.selectedTable} + onImport={enableImportExport + ? (mode) => ((importDrawerOpen = true), (importBehavior = mode)) + : undefined} > {#snippet dbSelector()} {#if uriState.isDatatableInput} @@ -119,6 +201,12 @@ {/key} {/if} {#snippet actions()} + {#if enableImportExport} + + + {/if} +
+ {/if} + + + + + (importDrawerOpen = false)}> +
+ + This will import the schemas from the selected source into the current database. Existing + tables with the same names may be affected. + +
+ Source database + +
+
+ Import mode + {:else}
-
{emptyMessage}
+
{emptyMessage}
{/if} {#if footer} -
+
{@render footer()}
{/if} diff --git a/frontend/src/lib/components/assets/AssetsDropdownButton.svelte b/frontend/src/lib/components/assets/AssetsDropdownButton.svelte index 351ee68331..3ed8690079 100644 --- a/frontend/src/lib/components/assets/AssetsDropdownButton.svelte +++ b/frontend/src/lib/components/assets/AssetsDropdownButton.svelte @@ -61,7 +61,9 @@ }) let datatables = resource([], () => - WorkspaceService.listDataTables({ workspace: $workspaceStore ?? '' }) + WorkspaceService.listDataTables({ workspace: $workspaceStore ?? '' }).then((d) => + d.map((d) => d.name) + ) ) let ducklakes = resource([], () => WorkspaceService.listDucklakes({ workspace: $workspaceStore ?? '' }) diff --git a/frontend/src/lib/components/common/confirmationModal/ConfirmationModal.svelte b/frontend/src/lib/components/common/confirmationModal/ConfirmationModal.svelte index f7205b2531..9dee3b6e87 100644 --- a/frontend/src/lib/components/common/confirmationModal/ConfirmationModal.svelte +++ b/frontend/src/lib/components/common/confirmationModal/ConfirmationModal.svelte @@ -86,7 +86,7 @@ {#if open} {/if} - {#if !strictWorkspaceSelect && !isCloudHosted() && !isRuleActive('DisableWorkspaceForking')} + {#if !strictWorkspaceSelect && !isCloudHosted() && !isRuleActive('DisableWorkspaceForking') && $workspaceStore !== 'admins'}
- + (globalForkModal.val = { opened: true })} + > Fork current workspace diff --git a/frontend/src/lib/components/workspaceSettings/CreateWorkspace.svelte b/frontend/src/lib/components/workspaceSettings/CreateWorkspace.svelte index e02facb38e..03deff091e 100644 --- a/frontend/src/lib/components/workspaceSettings/CreateWorkspace.svelte +++ b/frontend/src/lib/components/workspaceSettings/CreateWorkspace.svelte @@ -1,749 +1,28 @@ - -
- {#if isFork} -
- - Forking - - {$workspaceStore} - -
- {/if} - {#if errorMsgs.length != 0} - -
    - {#each errorMsgs as errorMsg} - 
  • - {errorMsg}
    • - {/each} -
- {#if failedSyncJobs.length != 0} - More details on the jobs that failed: - {#await fetchFailedSyncJobs(failedSyncJobs)} - - {:then failedJobs} -
    - {#each failedJobs as job} -
  • - - - - {job.id} - -
    • - - {#if isPathVersionLessThan(job.script_path, 28073)} -
    - This job was not running the latest version of the git sync script available on - the hub. You might be able to solve this issue by going to `Workspace Settings` - -> `Git Sync` and updating the script. -
    - {/if} - {/each} -
- {:catch error} - Tried to fetch jobs to get more information, but failed: {error}. Here are the failed - job ids: -
    - {#each failedSyncJobs as jobId} -
  • - - - - {jobId} - -
    • - {/each} -
- {/await} - {/if} - - {/if} - - {#if isFork || creationStep === 'details'} - - - - {#if !automateUsernameCreation} - - {/if} - {#if !isFork} -
- - - {#if isCloudHosted() && isDomainAllowed == false} -
{domain} domain not allowed for auto-invite
- {/if} - - {#if auto_invite} -
- - {#if isCloudHosted()} - - {/if} - - -
- {/if} -
- {/if} - -
- - {#if !forkCreationLoading} - - {:else} - - {/if} -
- {:else} -
-
- - - Windmill AI powers the chat, code generation, flow creation, and code completion. Set - it up now or configure it later in Workspace settings. - - Learn more - - - - - {#snippet children({ item })} - - - - - {/snippet} - -
- - -
-
- - {#if aiKey} -
- -
- {/if} -
- -
- - -
- {/if} -
+ + {#if isFork} +
+ + Forking + + {$workspaceStore} + +
+ {/if} + goto(rd ?? '/')} /> diff --git a/frontend/src/lib/components/workspaceSettings/CreateWorkspaceInner.svelte b/frontend/src/lib/components/workspaceSettings/CreateWorkspaceInner.svelte new file mode 100644 index 0000000000..e1f5d125bc --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/CreateWorkspaceInner.svelte @@ -0,0 +1,662 @@ + + +
+
+
+ {#if errorMsgs.length != 0} + +
    + {#each errorMsgs as errorMsg} + 
  • - {errorMsg}
    • + {/each} +
+ {#if failedSyncJobs.length != 0} + More details on the jobs that failed: + {#await fetchFailedSyncJobs(failedSyncJobs)} + + {:then failedJobs} +
    + {#each failedJobs as job} +
  • + - + + {job.id} + +
    • + + {#if isPathVersionLessThan(job.script_path, 28073)} +
    + This job was not running the latest version of the git sync script available + on the hub. You might be able to solve this issue by going to `Workspace + Settings` -> `Git Sync` and updating the script. +
    + {/if} + {/each} +
+ {:catch error} + Tried to fetch jobs to get more information, but failed: {error}. Here are the failed + job ids: +
    + {#each failedSyncJobs as jobId} +
  • + - + + {jobId} + +
    • + {/each} +
+ {/await} + {/if} + + {/if} + + + + {#if isFork} + { + completeFork(`${WM_FORK_PREFIX}${id}`) + }} + onCanceled={() => { + forkCreationLoading = false + }} + /> + {/if} + {#if !automateUsernameCreation} + + {/if} + {#if !isFork} +
+
+ + + + {#snippet children({ item })} + + + + + {/snippet} + +
+ + +
+
+ + {#if aiKey} +
+ +
+ {/if} +
+
+ + + {#if isCloudHosted() && isDomainAllowed == false} +
{domain} domain not allowed for auto-invite
+ {/if} + + {#if auto_invite} +
+ + {#if isCloudHosted()} + + {/if} + + +
+ {/if} +
+ {/if} +
+
+
+ + {#if !forkCreationLoading} + + {:else} + + {/if} +
+
diff --git a/frontend/src/lib/components/workspaceSettings/CustomInstanceDbSelect.svelte b/frontend/src/lib/components/workspaceSettings/CustomInstanceDbSelect.svelte index 44a4dcc6bf..9fccaa8b35 100644 --- a/frontend/src/lib/components/workspaceSettings/CustomInstanceDbSelect.svelte +++ b/frontend/src/lib/components/workspaceSettings/CustomInstanceDbSelect.svelte @@ -27,9 +27,7 @@ tag }: Props = $props() - let openedDbNameWizard = $state(false) - - let status = $derived(customInstanceDbs.current?.[value ?? '']) + let openedDbNameWizard: string | undefined = $state(undefined) let onlySelectedTags = $derived( safeSelectItems( @@ -38,12 +36,13 @@ .map(([name, _]) => name) ) ) - let currentIsAlreadyUsedElsewhere = $derived(tag && status && status.tag !== tag) + let open = $state(false)
- - + {#if value} + {@render customInstanceDbWizardButton(value, 'absolute right-1.5')} + {/if}
(openedDbNameWizard ? { dbname: value ?? '', status: status! } : undefined), - (v) => !v && (openedDbNameWizard = false) + () => + openedDbNameWizard + ? { dbname: openedDbNameWizard, status: customInstanceDbs.current?.[openedDbNameWizard] } + : undefined, + (v) => !v && (openedDbNameWizard = undefined) } /> + +{#snippet customInstanceDbWizardButton(dbname: string, clazz: string = '')} + {@const status = customInstanceDbs.current?.[dbname]} + +{/snippet} diff --git a/frontend/src/lib/components/workspaceSettings/CustomInstanceDbWizardModal.svelte b/frontend/src/lib/components/workspaceSettings/CustomInstanceDbWizardModal.svelte index c2e608e759..eab008fef6 100644 --- a/frontend/src/lib/components/workspaceSettings/CustomInstanceDbWizardModal.svelte +++ b/frontend/src/lib/components/workspaceSettings/CustomInstanceDbWizardModal.svelte @@ -15,7 +15,7 @@ import type { ResourceReturn } from 'runed' import type { ConfirmationModalHandle } from '../common/confirmationModal/asyncConfirmationModal.svelte' import ExploreAssetButton from '../ExploreAssetButton.svelte' - import { ArrowRight, InfoIcon } from 'lucide-svelte' + import { ArrowRight, InfoIcon, Trash2 } from 'lucide-svelte' import type { Snippet } from 'svelte' import { truncate } from '$lib/utils' import Tooltip from '../meltComponents/Tooltip.svelte' @@ -38,6 +38,7 @@ }: Props = $props() let customInstanceDbSetupIsRunning = $state(false) + let dropIsRunning = $state(false) let preventClose = false @@ -70,13 +71,50 @@ {@render bottomHint()}
{/if} - (opened = undefined)} - /> +
+ (opened = undefined)} + /> + {#if $superadmin} + + {/if} +
diff --git a/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte b/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte index 86e646b753..60794c6923 100644 --- a/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte @@ -15,7 +15,10 @@ const s: DataTableSettingsType = { dataTables: [] } if (settings?.datatables) { for (const [name, rest] of Object.entries(settings.datatables)) { - s.dataTables.push({ name, ...rest }) + s.dataTables.push({ + name, + ...rest + }) } } return s @@ -107,7 +110,7 @@ }) } - const customInstanceDbs = resource([], SettingService.listCustomInstanceDbs) + const customInstanceDbs = resource([() => $workspaceStore], SettingService.listCustomInstanceDbs) async function onSave() { try { @@ -252,27 +255,29 @@
+ - {#if dirtyMap[dataTable.name]} - - {#snippet trigger()} - - {/snippet} - {#snippet content()} - Please save settings first - {/snippet} - - {:else} - - {/if} +
+ {#if dirtyMap[dataTable.name]} + + {#snippet trigger()} + + {/snippet} + {#snippet content()} + Please save settings first + {/snippet} + + {:else} + + {/if} +
removeDataTable(dataTableIndex)} /> diff --git a/frontend/src/lib/components/workspaceSettings/DucklakeSettings.svelte b/frontend/src/lib/components/workspaceSettings/DucklakeSettings.svelte index 641b56bb04..0f02b1667f 100644 --- a/frontend/src/lib/components/workspaceSettings/DucklakeSettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/DucklakeSettings.svelte @@ -135,7 +135,7 @@ !Object.values(ducklakeIsDirty).every((v) => v === false) ) - const customInstanceDbs = resource([], SettingService.listCustomInstanceDbs) + const customInstanceDbs = resource([() => $workspaceStore], SettingService.listCustomInstanceDbs) async function onSave() { try { diff --git a/frontend/src/lib/components/workspaceSettings/ForkDatatableSection.svelte b/frontend/src/lib/components/workspaceSettings/ForkDatatableSection.svelte new file mode 100644 index 0000000000..87bcce5fbf --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/ForkDatatableSection.svelte @@ -0,0 +1,267 @@ + + + + +{#if allDatatables.current && allDatatables.current.length > 0} +