From 9bbab3321e064dc00c5734990949ce9515f46de7 Mon Sep 17 00:00:00 2001 From: Diego Imbert <70353967+diegoimbert@users.noreply.github.com> Date: Sat, 6 Dec 2025 00:08:58 +0100 Subject: [PATCH] feat: Data tables (#7226) * data tables settings ui * install runed * zod 4 fixes * use new toJSONSchema * Migrate ducklake catalogs to more generic custom instance databases * fix compilation * Safety conversion for old duckdb ffi * data tables settings * ts client basis * inline run works * datatables work * Revert "datatables work" This reverts commit 6e1588d59e7ed8d4e1b2ed631b4c2a7fe67e5540. * datatables work (without leaking pg credentials) * println * separate sqlUtils.ts * nit * Separate custom instance db Select and Wizard components * nit * nit wording * add tags to custom instance dbs * error when trying to use ducklake as datatable or opposite * show status in dropdown * data table instance setup works * sqk function for ducklake * factorize logic * fix temp reactivity * Data table assetexplore * Migrate S3 permissions to modal * Revert "Migrate S3 permissions to modal" This reverts commit 0631d03cb07269a762a7409ea33f6aa47d7e0fab. * nit query -> fetch * Custom instance setup new look * run_language_executor separate fn * run_inline param * nit wording * Better typed client * Data tables display as assets in frontend * asset db icon * nit * cleaner errors * nit * Fix sed calls in mac * run_inline_script_preview in python client * basic python datatable client * datatable and datalake parser in python * ducklake client python * nit fix * Fix migration producing NULL instead of {} when no custom databases * merge conflict fail * python ducklake client arg fix * parse or infer sql types in ts client * ts asset parser, detect datatable & ducklake R/W * fix sql repl for other read ops than select * export type SqlTemplateFunction * rename list_custom_instance_pg_databases * typecheck datatable and ducklake name in Typescript * Fix typecheck datatable and ducklake in TS * declare module overriding instead of extending * infer_sql_type in python client * SqlQuery object in python * fix merge conflicts * update const_format * CI fix * factor out to var_identifiers * sqlx prepare * unnecessary security (admin is required) * clearer comment * ee repo ref * nit snake case * claude step 1: detect var declarations * move detect_sql_access_type to common mod * claude step 2: detect when saved vars are queried * Revert "claude step 2: detect when saved vars are queried" This reverts commit 1e1f930568e4e2982b612f6d3e0eea2222b64947. * Revert "claude step 1: detect var declarations" This reverts commit f866f4819d98d072d1df4d4ed24b162824889836. * remove ducklake/datatable and default * detect data table assigns in var_identifiers * Python parser successfully infers R/W/RW from ducklake / datatable * still register ducklake/datatable if not used as unknown R/W * Go to settings button in Assets Dropdown on not found * nit * sqlx prepare fail * manual fix, somehow sqlx prepare won't do it * fix frontend ci * ee repo ref * ducklake_user doesnt exist in unit tests * nit fix * ui nit * nit * nit missing clone * fork ducklakes and datatables * fix surface hover bug * stupid mistake * better deeply reactive mutable derived * Ducklake picker * Editor bar data tables * DuckDB supports datatables * datatable in duckdb asset parser * duckdb asset parser var_identifiers * Revert "duckdb asset parser var_identifiers" This reverts commit 88068b1a777e5c3073056431a2d738472d86ef4a. * sqlx prepare * Box pin in test_workflow_as_code to fix stack overflow * go to settings button * ee repo ref * fix compilation * wording nit --- ...8234ca7d1efeee9661f3901f298da375e73f7.json | 6 + ...f299ef549e11a017016924c62c1cab179412c.json | 15 + ...857807d20645561c178d822ecad779c4f7bf4.json | 20 + ...974f2ce577b78decd6b821096c9f2f252ae8b.json | 22 + ...c61296a3ff7489ae12f52a19f9543173ac597.json | 5 + ...77ecc71394957d6f87ae52f91c6e31525beef.json | 20 + ...a247468969e135d604cb50147942bb857ecb0.json | 14 + ...a9740dcd95424df17059da709b7df2d033db2.json | 22 + ...5475568b02f4f4472dacf3d61e57b56335b69.json | 15 + ...91f4e90d237799f6817f292eb1391694164c7.json | 220 ++++++++ ...47763e0d674e8a1d9f3512629cb5a9e5a3439.json | 23 + backend/Cargo.lock | 1 + backend/Cargo.toml | 2 +- backend/ee-repo-ref.txt | 2 +- .../20251121144144_datatables.down.sql | 11 + .../20251121144144_datatables.up.sql | 4 + ...igrate_instance_ducklake_catalogs.down.sql | 23 + ..._migrate_instance_ducklake_catalogs.up.sql | 39 ++ .../windmill-parser-py/src/asset_parser.rs | 183 ++++++- .../windmill-parser-sql/src/asset_parser.rs | 15 + .../windmill-parser-ts/src/asset_parser.rs | 134 ++++- .../windmill-parser/src/asset_parser.rs | 36 ++ backend/src/main.rs | 17 +- backend/tests/worker.rs | 16 +- backend/windmill-api/openapi.yaml | 147 ++++- backend/windmill-api/src/jobs.rs | 45 +- backend/windmill-api/src/resources.rs | 10 +- backend/windmill-api/src/settings.rs | 64 ++- backend/windmill-api/src/workspaces.rs | 153 ++++-- backend/windmill-common/Cargo.toml | 1 + backend/windmill-common/src/assets.rs | 1 + backend/windmill-common/src/jobs.rs | 40 +- backend/windmill-common/src/utils.rs | 13 + backend/windmill-common/src/workspaces.rs | 82 ++- backend/windmill-worker/src/agent_workers.rs | 14 + backend/windmill-worker/src/common.rs | 2 +- .../windmill-worker/src/duckdb_executor.rs | 167 ++++-- backend/windmill-worker/src/pg_executor.rs | 50 +- backend/windmill-worker/src/worker.rs | 510 +++++++++++++----- frontend/package-lock.json | 128 +++-- frontend/package.json | 6 +- .../lib/components/AIAgentLogViewer.svelte | 2 +- .../src/lib/components/DBManagerDrawer.svelte | 4 +- frontend/src/lib/components/Editor.svelte | 48 +- frontend/src/lib/components/EditorBar.svelte | 102 +++- .../lib/components/ExploreAssetButton.svelte | 14 +- frontend/src/lib/components/SqlRepl.svelte | 11 +- .../components/display/dbtable/metadata.ts | 9 +- .../lib/components/assets/AssetButtons.svelte | 43 +- .../assets/AssetsDropdownButton.svelte | 22 +- frontend/src/lib/components/assets/lib.ts | 4 + .../asyncConfirmationModal.svelte.ts | 10 +- .../lib/components/common/modal/Modal2.svelte | 34 +- .../src/lib/components/copilot/chat/shared.ts | 13 +- frontend/src/lib/components/custom_ui.ts | 1 + frontend/src/lib/components/dbOps.ts | 21 +- .../components/icons/AssetDatabaseIcon.svelte | 30 ++ .../components/icons/AssetGenericIcon.svelte | 3 + .../sidebar/CriticalAlertModal.svelte | 8 +- .../wizards/LoggedWizardResult.svelte | 4 +- .../CustomInstanceDbSelect.svelte | 99 ++++ .../CustomInstanceDbWizardModal.svelte | 199 +++++++ .../DataTableSettings.svelte | 288 ++++++++++ .../workspaceSettings/DucklakeSettings.svelte | 229 +------- .../workspaceSettings/utils.svelte.ts | 9 + frontend/src/lib/svelte5Utils.svelte.ts | 5 +- .../(logged)/workspace_settings/+page.svelte | 193 ++++--- openflow.openapi.yaml | 1 + python-client/wmill/wmill/client.py | 117 ++++ typescript-client/build.jsr.sh | 3 +- typescript-client/build.sh | 3 +- typescript-client/client.d.ts | 92 +++- typescript-client/client.ts | 26 +- typescript-client/jsr.json | 2 +- typescript-client/publish.sh | 2 + typescript-client/sqlUtils.d.ts | 53 ++ typescript-client/sqlUtils.ts | 218 ++++++++ 77 files changed, 3470 insertions(+), 750 deletions(-) create mode 100644 backend/.sqlx/query-0a3ee1329fb4f705c0006480d03f299ef549e11a017016924c62c1cab179412c.json create mode 100644 backend/.sqlx/query-0b5103497ab09affbdf3793d7d7857807d20645561c178d822ecad779c4f7bf4.json create mode 100644 backend/.sqlx/query-16a67b92dbd32024838983184e6974f2ce577b78decd6b821096c9f2f252ae8b.json create mode 100644 backend/.sqlx/query-2767ea3f8fa1dc2c7285671560f77ecc71394957d6f87ae52f91c6e31525beef.json create mode 100644 backend/.sqlx/query-5556d7002bf613ba405bc1b8bc0a247468969e135d604cb50147942bb857ecb0.json create mode 100644 backend/.sqlx/query-800e04197578631bd75b2f8e511a9740dcd95424df17059da709b7df2d033db2.json create mode 100644 backend/.sqlx/query-897a5b32e0f5890d68017c0c1225475568b02f4f4472dacf3d61e57b56335b69.json create mode 100644 backend/.sqlx/query-95fa60eb45228ff289655fc676991f4e90d237799f6817f292eb1391694164c7.json create mode 100644 backend/.sqlx/query-caf4dc1046769f410d1277cdd3747763e0d674e8a1d9f3512629cb5a9e5a3439.json create mode 100644 backend/migrations/20251121144144_datatables.down.sql create mode 100644 backend/migrations/20251121144144_datatables.up.sql create mode 100644 backend/migrations/20251121144314_migrate_instance_ducklake_catalogs.down.sql create mode 100644 backend/migrations/20251121144314_migrate_instance_ducklake_catalogs.up.sql create mode 100644 frontend/src/lib/components/icons/AssetDatabaseIcon.svelte create mode 100644 frontend/src/lib/components/workspaceSettings/CustomInstanceDbSelect.svelte create mode 100644 frontend/src/lib/components/workspaceSettings/CustomInstanceDbWizardModal.svelte create mode 100644 frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte create mode 100644 frontend/src/lib/components/workspaceSettings/utils.svelte.ts create mode 100644 typescript-client/sqlUtils.d.ts create mode 100644 typescript-client/sqlUtils.ts diff --git a/backend/.sqlx/query-08f288d2781d823e109a9e5b8848234ca7d1efeee9661f3901f298da375e73f7.json b/backend/.sqlx/query-08f288d2781d823e109a9e5b8848234ca7d1efeee9661f3901f298da375e73f7.json index d266d7db3c..62da985ed8 100644 --- a/backend/.sqlx/query-08f288d2781d823e109a9e5b8848234ca7d1efeee9661f3901f298da375e73f7.json +++ b/backend/.sqlx/query-08f288d2781d823e109a9e5b8848234ca7d1efeee9661f3901f298da375e73f7.json @@ -167,6 +167,11 @@ "ordinal": 32, "name": "slack_oauth_client_secret", "type_info": "Varchar" + }, + { + "ordinal": 33, + "name": "datatable", + "type_info": "Jsonb" } ], "parameters": { @@ -207,6 +212,7 @@ true, true, true, + true, true ] }, diff --git a/backend/.sqlx/query-0a3ee1329fb4f705c0006480d03f299ef549e11a017016924c62c1cab179412c.json b/backend/.sqlx/query-0a3ee1329fb4f705c0006480d03f299ef549e11a017016924c62c1cab179412c.json new file mode 100644 index 0000000000..1caf124240 --- /dev/null +++ b/backend/.sqlx/query-0a3ee1329fb4f705c0006480d03f299ef549e11a017016924c62c1cab179412c.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings SET datatable = $1 WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Text" + ] + }, + "nullable": [] + }, + "hash": "0a3ee1329fb4f705c0006480d03f299ef549e11a017016924c62c1cab179412c" +} diff --git a/backend/.sqlx/query-0b5103497ab09affbdf3793d7d7857807d20645561c178d822ecad779c4f7bf4.json b/backend/.sqlx/query-0b5103497ab09affbdf3793d7d7857807d20645561c178d822ecad779c4f7bf4.json new file mode 100644 index 0000000000..3088609214 --- /dev/null +++ b/backend/.sqlx/query-0b5103497ab09affbdf3793d7d7857807d20645561c178d822ecad779c4f7bf4.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT value->>'user_pwd' FROM global_settings WHERE name = 'custom_instance_pg_databases';", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Text" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "0b5103497ab09affbdf3793d7d7857807d20645561c178d822ecad779c4f7bf4" +} diff --git a/backend/.sqlx/query-16a67b92dbd32024838983184e6974f2ce577b78decd6b821096c9f2f252ae8b.json b/backend/.sqlx/query-16a67b92dbd32024838983184e6974f2ce577b78decd6b821096c9f2f252ae8b.json new file mode 100644 index 0000000000..d099d97bd3 --- /dev/null +++ b/backend/.sqlx/query-16a67b92dbd32024838983184e6974f2ce577b78decd6b821096c9f2f252ae8b.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT ws.datatable->'datatables' AS datatable_name\n FROM workspace_settings ws\n WHERE ws.workspace_id = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "datatable_name", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "16a67b92dbd32024838983184e6974f2ce577b78decd6b821096c9f2f252ae8b" +} diff --git a/backend/.sqlx/query-1730f39fd1793d45fbb41b21389c61296a3ff7489ae12f52a19f9543173ac597.json b/backend/.sqlx/query-1730f39fd1793d45fbb41b21389c61296a3ff7489ae12f52a19f9543173ac597.json index 918967e934..6a94914de9 100644 --- a/backend/.sqlx/query-1730f39fd1793d45fbb41b21389c61296a3ff7489ae12f52a19f9543173ac597.json +++ b/backend/.sqlx/query-1730f39fd1793d45fbb41b21389c61296a3ff7489ae12f52a19f9543173ac597.json @@ -167,6 +167,11 @@ "ordinal": 32, "name": "slack_oauth_client_secret", "type_info": "Varchar" + }, + { + "ordinal": 33, + "name": "datatable", + "type_info": "Jsonb" } ], "parameters": { diff --git a/backend/.sqlx/query-2767ea3f8fa1dc2c7285671560f77ecc71394957d6f87ae52f91c6e31525beef.json b/backend/.sqlx/query-2767ea3f8fa1dc2c7285671560f77ecc71394957d6f87ae52f91c6e31525beef.json new file mode 100644 index 0000000000..86ab18b7b1 --- /dev/null +++ b/backend/.sqlx/query-2767ea3f8fa1dc2c7285671560f77ecc71394957d6f87ae52f91c6e31525beef.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT value->'databases' FROM global_settings WHERE name = 'custom_instance_pg_databases'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "2767ea3f8fa1dc2c7285671560f77ecc71394957d6f87ae52f91c6e31525beef" +} diff --git a/backend/.sqlx/query-5556d7002bf613ba405bc1b8bc0a247468969e135d604cb50147942bb857ecb0.json b/backend/.sqlx/query-5556d7002bf613ba405bc1b8bc0a247468969e135d604cb50147942bb857ecb0.json new file mode 100644 index 0000000000..0e7fbc099c --- /dev/null +++ b/backend/.sqlx/query-5556d7002bf613ba405bc1b8bc0a247468969e135d604cb50147942bb857ecb0.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE global_settings SET value = jsonb_set(value, '{databases}', (COALESCE(value->'databases', '{}'::jsonb) || to_jsonb($1::json))) WHERE name = 'custom_instance_pg_databases'", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Json" + ] + }, + "nullable": [] + }, + "hash": "5556d7002bf613ba405bc1b8bc0a247468969e135d604cb50147942bb857ecb0" +} diff --git a/backend/.sqlx/query-800e04197578631bd75b2f8e511a9740dcd95424df17059da709b7df2d033db2.json b/backend/.sqlx/query-800e04197578631bd75b2f8e511a9740dcd95424df17059da709b7df2d033db2.json new file mode 100644 index 0000000000..bfc791daf5 --- /dev/null +++ b/backend/.sqlx/query-800e04197578631bd75b2f8e511a9740dcd95424df17059da709b7df2d033db2.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT jsonb_object_keys(ws.datatable->'datatables') AS datatable_name\n FROM workspace_settings ws\n WHERE ws.workspace_id = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "datatable_name", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "800e04197578631bd75b2f8e511a9740dcd95424df17059da709b7df2d033db2" +} diff --git a/backend/.sqlx/query-897a5b32e0f5890d68017c0c1225475568b02f4f4472dacf3d61e57b56335b69.json b/backend/.sqlx/query-897a5b32e0f5890d68017c0c1225475568b02f4f4472dacf3d61e57b56335b69.json new file mode 100644 index 0000000000..bc7de546fe --- /dev/null +++ b/backend/.sqlx/query-897a5b32e0f5890d68017c0c1225475568b02f4f4472dacf3d61e57b56335b69.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE workspace_settings\n SET\n deploy_to = $1,\n ai_config = source_ws.ai_config,\n large_file_storage = source_ws.large_file_storage,\n ducklake = source_ws.ducklake,\n datatable = source_ws.datatable,\n git_app_installations = source_ws.git_app_installations\n FROM workspace_settings source_ws\n WHERE source_ws.workspace_id = $1\n AND workspace_settings.workspace_id = $2\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "897a5b32e0f5890d68017c0c1225475568b02f4f4472dacf3d61e57b56335b69" +} diff --git a/backend/.sqlx/query-95fa60eb45228ff289655fc676991f4e90d237799f6817f292eb1391694164c7.json b/backend/.sqlx/query-95fa60eb45228ff289655fc676991f4e90d237799f6817f292eb1391694164c7.json new file mode 100644 index 0000000000..99a1b41c90 --- /dev/null +++ b/backend/.sqlx/query-95fa60eb45228ff289655fc676991f4e90d237799f6817f292eb1391694164c7.json @@ -0,0 +1,220 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n workspace_id,\n slack_team_id,\n teams_team_id,\n teams_team_name,\n slack_name,\n slack_command_script,\n teams_command_script,\n slack_email,\n slack_oauth_client_id,\n slack_oauth_client_secret,\n auto_invite_domain,\n auto_invite_operator,\n auto_add,\n customer_id,\n plan,\n webhook,\n deploy_to,\n ai_config,\n error_handler,\n error_handler_extra_args,\n error_handler_muted_on_cancel,\n large_file_storage,\n datatable,\n ducklake,\n git_sync,\n deploy_ui,\n default_app,\n default_scripts,\n mute_critical_alerts,\n color,\n operator_settings,\n git_app_installations,\n auto_add_instance_groups,\n auto_add_instance_groups_roles\n FROM\n workspace_settings\n WHERE\n workspace_id = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "slack_team_id", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "teams_team_id", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "teams_team_name", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "slack_name", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "slack_command_script", + "type_info": "Varchar" + }, + { + "ordinal": 6, + "name": "teams_command_script", + "type_info": "Text" + }, + { + "ordinal": 7, + "name": "slack_email", + "type_info": "Varchar" + }, + { + "ordinal": 8, + "name": "slack_oauth_client_id", + "type_info": "Varchar" + }, + { + "ordinal": 9, + "name": "slack_oauth_client_secret", + "type_info": "Varchar" + }, + { + "ordinal": 10, + "name": "auto_invite_domain", + "type_info": "Varchar" + }, + { + "ordinal": 11, + "name": "auto_invite_operator", + "type_info": "Bool" + }, + { + "ordinal": 12, + "name": "auto_add", + "type_info": "Bool" + }, + { + "ordinal": 13, + "name": "customer_id", + "type_info": "Varchar" + }, + { + "ordinal": 14, + "name": "plan", + "type_info": "Varchar" + }, + { + "ordinal": 15, + "name": "webhook", + "type_info": "Text" + }, + { + "ordinal": 16, + "name": "deploy_to", + "type_info": "Varchar" + }, + { + "ordinal": 17, + "name": "ai_config", + "type_info": "Jsonb" + }, + { + "ordinal": 18, + "name": "error_handler", + "type_info": "Varchar" + }, + { + "ordinal": 19, + "name": "error_handler_extra_args", + "type_info": "Json" + }, + { + "ordinal": 20, + "name": "error_handler_muted_on_cancel", + "type_info": "Bool" + }, + { + "ordinal": 21, + "name": "large_file_storage", + "type_info": "Jsonb" + }, + { + "ordinal": 22, + "name": "datatable", + "type_info": "Jsonb" + }, + { + "ordinal": 23, + "name": "ducklake", + "type_info": "Jsonb" + }, + { + "ordinal": 24, + "name": "git_sync", + "type_info": "Jsonb" + }, + { + "ordinal": 25, + "name": "deploy_ui", + "type_info": "Jsonb" + }, + { + "ordinal": 26, + "name": "default_app", + "type_info": "Varchar" + }, + { + "ordinal": 27, + "name": "default_scripts", + "type_info": "Jsonb" + }, + { + "ordinal": 28, + "name": "mute_critical_alerts", + "type_info": "Bool" + }, + { + "ordinal": 29, + "name": "color", + "type_info": "Varchar" + }, + { + "ordinal": 30, + "name": "operator_settings", + "type_info": "Jsonb" + }, + { + "ordinal": 31, + "name": "git_app_installations", + "type_info": "Jsonb" + }, + { + "ordinal": 32, + "name": "auto_add_instance_groups", + "type_info": "TextArray" + }, + { + "ordinal": 33, + "name": "auto_add_instance_groups_roles", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + true, + true, + true, + true, + true, + true, + false, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + false, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + false, + true, + true + ] + }, + "hash": "95fa60eb45228ff289655fc676991f4e90d237799f6817f292eb1391694164c7" +} diff --git a/backend/.sqlx/query-caf4dc1046769f410d1277cdd3747763e0d674e8a1d9f3512629cb5a9e5a3439.json b/backend/.sqlx/query-caf4dc1046769f410d1277cdd3747763e0d674e8a1d9f3512629cb5a9e5a3439.json new file mode 100644 index 0000000000..f2398bf143 --- /dev/null +++ b/backend/.sqlx/query-caf4dc1046769f410d1277cdd3747763e0d674e8a1d9f3512629cb5a9e5a3439.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT ws.datatable->'datatables'->$2 AS config\n FROM workspace_settings ws\n WHERE ws.workspace_id = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "config", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "caf4dc1046769f410d1277cdd3747763e0d674e8a1d9f3512629cb5a9e5a3439" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index f0ff1f8d7b..5e91858d78 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -15425,6 +15425,7 @@ dependencies = [ "magic-crypt", "mail-send", "object_store", + "once_cell", "openidconnect", "opentelemetry", "opentelemetry-appender-tracing", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 6d07a9a9ea..1cf7bdc653 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -328,7 +328,7 @@ futures = "^0" futures-core = "^0" lazy_static = "1.4.0" serde_derive = "1.0.147" -const_format = { version = "0.2", features = ["rust_1_64", "rust_1_51"] } +const_format = { version = "0.2.35", features = ["rust_1_64", "rust_1_51"] } constant_time_eq = "0.3.1" dyn-iter = "0.2.0" rsa = "^0" diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 20e0fa2c17..2b19bb4da9 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -f03dfeccbe7610c1e3029fb67b47faeba8fac918 \ No newline at end of file +9501f6dc030f4235984286cf00762ab4bb54c846 \ No newline at end of file diff --git a/backend/migrations/20251121144144_datatables.down.sql b/backend/migrations/20251121144144_datatables.down.sql new file mode 100644 index 0000000000..dd2485660e --- /dev/null +++ b/backend/migrations/20251121144144_datatables.down.sql @@ -0,0 +1,11 @@ +-- Add down migration script here + +ALTER TABLE workspace_settings +DROP COLUMN datatable; + +-- Remove 'datatable' kind +DELETE FROM asset WHERE kind = 'datatable'; +ALTER TABLE asset ALTER column kind TYPE VARCHAR; +DROP TYPE asset_kind; +CREATE TYPE ASSET_KIND AS ENUM ('s3object', 'resource', 'variable', 'ducklake'); +ALTER TABLE asset ALTER column kind TYPE ASSET_KIND using kind::ASSET_KIND; \ No newline at end of file diff --git a/backend/migrations/20251121144144_datatables.up.sql b/backend/migrations/20251121144144_datatables.up.sql new file mode 100644 index 0000000000..7c58c263c6 --- /dev/null +++ b/backend/migrations/20251121144144_datatables.up.sql @@ -0,0 +1,4 @@ +ALTER TYPE asset_kind ADD VALUE 'datatable'; + +ALTER TABLE workspace_settings +ADD COLUMN datatable JSONB; \ No newline at end of file diff --git a/backend/migrations/20251121144314_migrate_instance_ducklake_catalogs.down.sql b/backend/migrations/20251121144314_migrate_instance_ducklake_catalogs.down.sql new file mode 100644 index 0000000000..ef701f2817 --- /dev/null +++ b/backend/migrations/20251121144314_migrate_instance_ducklake_catalogs.down.sql @@ -0,0 +1,23 @@ +UPDATE global_settings +SET value = jsonb_build_object( + 'status', ( + SELECT COALESCE( + jsonb_object_agg(key, value - 'tag'), + '{}'::jsonb + ) + FROM jsonb_each(value->'databases') + ), + 'user_pwd', value->'user_pwd' +) +WHERE name = 'custom_instance_pg_databases'; + + +UPDATE global_settings +SET name = 'ducklake_settings', + value = jsonb_build_object( + 'ducklake_user_pg_pwd', value->'user_pwd', + 'instance_catalog_db_status', value->'status' + ) +WHERE name = 'custom_instance_pg_databases'; + +ALTER ROLE custom_instance_user RENAME TO ducklake_user; \ No newline at end of file diff --git a/backend/migrations/20251121144314_migrate_instance_ducklake_catalogs.up.sql b/backend/migrations/20251121144314_migrate_instance_ducklake_catalogs.up.sql new file mode 100644 index 0000000000..505a099583 --- /dev/null +++ b/backend/migrations/20251121144314_migrate_instance_ducklake_catalogs.up.sql @@ -0,0 +1,39 @@ +-- Superadmins have the ability to create databases in the Windmill Postgres instance +-- for use as Ducklake catalogs or Data tables. These databases can be accessed by +-- the 'custom_instance_user'. The setting below stores the password and logs +-- about the creation status of these databases. + +DO $$ +DECLARE + new_settings_value text; + old_setting_value text; +BEGIN + ALTER ROLE ducklake_user RENAME TO custom_instance_user; +EXCEPTION + WHEN others THEN + RAISE NOTICE 'ducklake_user migration error, skipping.'; +END +$$; + +-- Rename to more generic names +UPDATE global_settings +SET name = 'custom_instance_pg_databases', + value = jsonb_build_object( + 'user_pwd', value->'ducklake_user_pg_pwd', + 'databases', value->'instance_catalog_db_status' + ) +WHERE name = 'ducklake_settings'; + +-- Add ducklake tag to existing databases +UPDATE global_settings +SET value = jsonb_build_object( + 'databases', ( + SELECT COALESCE( + jsonb_object_agg(key, value || jsonb_build_object('tag', 'ducklake')), + '{}'::jsonb + ) + FROM jsonb_each(value->'databases') + ), + 'user_pwd', value->'user_pwd' +) +WHERE name = 'custom_instance_pg_databases'; diff --git a/backend/parsers/windmill-parser-py/src/asset_parser.rs b/backend/parsers/windmill-parser-py/src/asset_parser.rs index c218182a2a..17a898655b 100644 --- a/backend/parsers/windmill-parser-py/src/asset_parser.rs +++ b/backend/parsers/windmill-parser-py/src/asset_parser.rs @@ -1,7 +1,9 @@ use rustpython_ast::{Constant, Expr, ExprConstant, Visitor}; use rustpython_parser::{ast::Suite, Parse}; +use std::collections::HashMap; use windmill_parser::asset_parser::{ - merge_assets, parse_asset_syntax, AssetKind, AssetUsageAccessType, ParseAssetsResult, + detect_sql_access_type, merge_assets, parse_asset_syntax, AssetKind, AssetUsageAccessType, + ParseAssetsResult, }; use AssetUsageAccessType::*; @@ -9,16 +11,77 @@ pub fn parse_assets(input: &str) -> anyhow::Result let ast = Suite::parse(input, "main.py") .map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))?; - let mut assets_finder = AssetsFinder { assets: vec![] }; + let mut assets_finder = AssetsFinder { assets: vec![], var_identifiers: HashMap::new() }; ast.into_iter() .for_each(|stmt| assets_finder.visit_stmt(stmt)); + + for (kind, name) in assets_finder.var_identifiers.into_values() { + // if a db = wmill.datatable() was never used (e.g db.query(...)), + // we still want to register the asset as unknown access type + if assets_finder + .assets + .iter() + .all(|a| !(a.kind == kind && a.path == name)) + { + assets_finder + .assets + .push(ParseAssetsResult { kind, access_type: None, path: name }); + } + } + Ok(merge_assets(assets_finder.assets)) } struct AssetsFinder { assets: Vec>, + var_identifiers: HashMap, } + impl Visitor for AssetsFinder { + // Handle assignment statements like: x = wmill.datatable('name') + fn visit_stmt_assign(&mut self, node: rustpython_ast::StmtAssign) { + // Check if the value is a call to a tracked function + if let Some((kind, name)) = self.extract_asset_from_call(&node.value) { + // Track all target variables + for target in &node.targets { + if let Expr::Name(name_expr) = target { + let Ok(var_name) = name_expr.id.parse::(); + self.var_identifiers + .insert(var_name, (kind.clone(), name.clone())); + } + } + } else { + // If not wmill.datatable or similar, remove any tracked variables + // It means the identifier is no longer refering to an asset + for target in &node.targets { + if let Expr::Name(name_expr) = target { + let Ok(var_name) = name_expr.id.parse::(); + let removed = self.var_identifiers.remove(&var_name); + // if a db = wmill.datatable() or similar was removed, but never used (e.g db.query(...)), + // we still want to register the asset as unknown access type + match removed { + Some((kind, name)) => { + if self + .assets + .iter() + .all(|a| !(a.kind == kind && a.path == name)) + { + self.assets.push(ParseAssetsResult { + kind, + access_type: None, + path: name, + }); + } + } + None => {} + } + } + } + } + // Continue with generic visit to catch any other assets in the expression + self.generic_visit_stmt_assign(node); + } + // visit_call_expr will not recurse if it detects an asset, // so this will only be called when no further context was found fn visit_expr_constant(&mut self, node: ExprConstant) { @@ -42,7 +105,9 @@ impl Visitor for AssetsFinder { Err(_) => { // Check keyword arguments for assets before falling back to generic visit for keyword in &node.keywords { - if let Expr::Constant(ExprConstant { value: Constant::Str(s), .. }) = &keyword.value { + if let Expr::Constant(ExprConstant { value: Constant::Str(s), .. }) = + &keyword.value + { if let Some((kind, path)) = parse_asset_syntax(s) { self.assets.push(ParseAssetsResult { kind, @@ -59,6 +124,49 @@ impl Visitor for AssetsFinder { } impl AssetsFinder { + /// Extract asset info from calls like wmill.datatable('name'), wmill.ducklake('name'), etc. + fn extract_asset_from_call(&self, expr: &Expr) -> Option<(AssetKind, String)> { + let call = expr.as_call_expr()?; + + // Check for wmill.datatable, wmill.ducklake pattern + let attr = call.func.as_attribute_expr()?; + + // Verify the object is 'wmill' + let obj_name = attr.value.as_name_expr()?; + if obj_name.id.as_str() != "wmill" { + return None; + } + + let method_name = attr.attr.as_str(); + let kind = match method_name { + "datatable" => AssetKind::DataTable, + "ducklake" => AssetKind::Ducklake, + _ => return None, + }; + + // Get the first argument (the asset name) + let name = call + .args + .first() + .or_else(|| { + // Try keyword argument 'name' or 'path' + call.keywords + .iter() + .find(|kw| matches!(kw.arg.as_deref(), Some("name") | Some("path"))) + .map(|kw| &kw.value) + }) + .and_then(|first_arg| { + if let Expr::Constant(ExprConstant { value: Constant::Str(name), .. }) = first_arg { + Some(name.clone()) + } else { + None + } + }) + .unwrap_or_else(|| "main".to_string()); + + Some((kind, name)) + } + fn visit_expr_call_inner(&mut self, node: &rustpython_ast::ExprCall) -> Result<(), ()> { let ident: String = node .func @@ -71,21 +179,66 @@ impl AssetsFinder { }) .ok_or(())?; + // `obj_name` is the object (receiver) - i.e., `obj` in `obj.method()` + let obj_name: String = if let Expr::Attribute(rustpython_ast::ExprAttribute { + value, .. + }) = node.func.as_ref() + { + if let Expr::Name(name_expr) = value.as_ref() { + name_expr.id.parse().map_err(|_| ())? + } else { + return Err(()); + } + } else { + return Err(()); + }; + + if obj_name == "wmill" { + // Continue + } else if let Some((kind, ref path)) = self.var_identifiers.get(&obj_name) { + if ident == "query" { + let name_expr = node.args.get(0).or_else(|| { + node.keywords + .iter() + .find(|kw| kw.arg.as_deref() == Some("name")) + .map(|kw| &kw.value) + }); + match name_expr { + Some(Expr::Constant(ExprConstant { + value: Constant::Str(sql_query), .. + })) => { + let access_type = detect_sql_access_type(&sql_query); + self.assets.push(ParseAssetsResult { + kind: *kind, + path: path.to_string(), + access_type, + }); + return Ok(()); + } + _ => return Err(()), + }; + } else { + return Err(()); + } + } else { + return Err(()); + } + use AssetKind::*; let (kind, access_type, arg) = match ident.as_str() { - "load_s3_file" => (S3Object, Some(R), Arg::Pos(0, "s3object")), - "load_s3_file_reader" => (S3Object, Some(R), Arg::Pos(0, "s3object")), - "write_s3_file" => (S3Object, Some(W), Arg::Pos(0, "s3object")), - "get_resource" => (Resource, None, Arg::Pos(0, "path")), - "set_resource" => (Resource, Some(W), Arg::Pos(0, "path")), - "get_boto3_connection_settings" => (Resource, None, Arg::Pos(0, "s3_resource_path")), - "get_polars_connection_settings" => (Resource, None, Arg::Pos(0, "s3_resource_path")), - "get_duckdb_connection_settings" => (Resource, None, Arg::Pos(0, "s3_resource_path")), + "load_s3_file" => (S3Object, Some(R), Arg(0, "s3object")), + "load_s3_file_reader" => (S3Object, Some(R), Arg(0, "s3object")), + "write_s3_file" => (S3Object, Some(W), Arg(0, "s3object")), + "get_resource" => (Resource, None, Arg(0, "path")), + "set_resource" => (Resource, Some(W), Arg(0, "path")), + "get_boto3_connection_settings" => (Resource, None, Arg(0, "s3_resource_path")), + "get_polars_connection_settings" => (Resource, None, Arg(0, "s3_resource_path")), + "get_duckdb_connection_settings" => (Resource, None, Arg(0, "s3_resource_path")), _ => return Err(()), }; let arg_val = match arg { - Arg::Pos(i, name) => node.args.get(i).or_else(|| { + Arg(i, name) => node.args.get(i).or_else(|| { // Get arg by name node.keywords .iter() @@ -106,7 +259,5 @@ impl AssetsFinder { } } -enum Arg { - // Positional arguments in python can also be used by their name - Pos(usize, &'static str), -} +struct Arg(usize, &'static str); +// Positional arguments in python can also be used by their name diff --git a/backend/parsers/windmill-parser-sql/src/asset_parser.rs b/backend/parsers/windmill-parser-sql/src/asset_parser.rs index a6579798d3..0323bcb062 100644 --- a/backend/parsers/windmill-parser-sql/src/asset_parser.rs +++ b/backend/parsers/windmill-parser-sql/src/asset_parser.rs @@ -59,6 +59,11 @@ fn parse_asset(input: &str) -> IResult<&str, ParseAssetsResult<&str>> { kind: AssetKind::Ducklake, access_type: None, }), + parse_datatable_lit.map(|path| ParseAssetsResult { + path, + kind: AssetKind::DataTable, + access_type: None, + }), )) .parse(input) } @@ -129,6 +134,16 @@ fn parse_ducklake_lit(input: &str) -> IResult<&str, &str> { let (input, _) = quote(input)?; Ok((input, path.unwrap_or("main"))) } + +fn parse_datatable_lit(input: &str) -> IResult<&str, &str> { + let (input, _) = quote(input)?; + let (input, _) = tag("datatable").parse(input)?; + let (input, path) = + opt(preceded(tag("://"), take_while(|c| c != '\'' && c != '"'))).parse(input)?; + let (input, _) = quote(input)?; + Ok((input, path.unwrap_or("main"))) +} + fn parse_comment(input: &str) -> IResult<&str, &str> { let (input, _) = tag("--").parse(input)?; let (input, comment) = take_while(|c| c != '\n')(input)?; diff --git a/backend/parsers/windmill-parser-ts/src/asset_parser.rs b/backend/parsers/windmill-parser-ts/src/asset_parser.rs index 6b81272ec1..6ac4c4dd17 100644 --- a/backend/parsers/windmill-parser-ts/src/asset_parser.rs +++ b/backend/parsers/windmill-parser-ts/src/asset_parser.rs @@ -1,9 +1,12 @@ +use std::collections::HashMap; + use swc_common::{sync::Lrc, FileName, SourceMap}; use swc_ecma_ast::{CallExpr, Expr, Lit, MemberExpr, MemberProp, Str}; use swc_ecma_parser::{lexer::Lexer, Parser, StringInput, Syntax, TsSyntax}; use swc_ecma_visit::{Visit, VisitWith}; use windmill_parser::asset_parser::{ - merge_assets, parse_asset_syntax, AssetKind, AssetUsageAccessType, ParseAssetsResult, + detect_sql_access_type, merge_assets, parse_asset_syntax, AssetKind, AssetUsageAccessType, + ParseAssetsResult, }; use AssetUsageAccessType::*; @@ -32,13 +35,21 @@ pub fn parse_assets(code: &str) -> anyhow::Result> anyhow::anyhow!("Error while parsing code, it is invalid TypeScript: {err_s}, {e:?}") })? .body; - let mut assets_finder = AssetsFinder { assets: vec![] }; + let mut assets_finder = AssetsFinder { assets: vec![], var_identifiers: HashMap::new() }; assets_finder.visit_module_items(&ast); Ok(merge_assets(assets_finder.assets)) } struct AssetsFinder { assets: Vec>, + + // The user will write code like: + // let sql = wmill.datatable('main') + // return await sql`SELECT * FROM friends WHERE age = ${21}`.fetch() + // The goal is to remember that the identifier "sql" corresponds to the datatable "main" + // so that when we see a tagged template expression with tag "sql" we know which datatable it + // corresponds to. This allows us to infer if a datatable is Read or Write based on the SQL query. + var_identifiers: HashMap, } impl Visit for AssetsFinder { @@ -65,6 +76,125 @@ impl Visit for AssetsFinder { Err(_) => >::visit_children_with(node, self), } } + + fn visit_block_stmt(&mut self, node: &swc_ecma_ast::BlockStmt) { + // Save current state before entering the block + let saved_var_identifiers = self.var_identifiers.clone(); + + // Visit children (this may add new identifiers) + node.visit_children_with(self); + + // If we find 'let sql = wmill.datatable(...)', + // but no sql`` tagged templates were used, we add + // the asset with unknown access type + for var in self.var_identifiers.keys() { + if saved_var_identifiers.contains_key(var) { + continue; + } + let (kind, ref path) = self.var_identifiers[var]; + if self + .assets + .iter() + .any(|a| a.kind == kind && &a.path == path) + { + continue; + } + self.assets + .push(ParseAssetsResult { kind, access_type: None, path: path.clone() }); + } + + // Restore state - identifiers declared in this block go out of scope + self.var_identifiers = saved_var_identifiers; + } + + fn visit_var_declarator(&mut self, node: &swc_ecma_ast::VarDeclarator) { + // Extract the variable name (name1) + let var_name = match &node.name { + swc_ecma_ast::Pat::Ident(ident) => ident.sym.as_str().to_string(), + _ => { + node.visit_children_with(self); + return; + } + }; + + // Check if init is a call to wmill.datatable(...) or wmill.ducklake(...) + if let Some(init) = &node.init { + if let Expr::Call(call_expr) = init.as_ref() { + if let Some(Expr::Member(member)) = call_expr.callee.as_expr().map(AsRef::as_ref) { + // Check if object is "wmill" + let is_wmill = matches!( + member.obj.as_ref(), + Expr::Ident(ident) if ident.sym.as_str() == "wmill" + ); + + if is_wmill { + if let MemberProp::Ident(prop) = &member.prop { + // Get the asset name from first arg, default to "main" + let asset_name = call_expr + .args + .first() + .and_then(|arg| match arg.expr.as_ref() { + Expr::Lit(Lit::Str(s)) => Some(s.value.to_string()), + _ => None, + }) + .unwrap_or_else(|| "main".to_string()); + + match prop.sym.as_str() { + "datatable" => { + self.var_identifiers + .insert(var_name, (AssetKind::DataTable, asset_name)); + return; + } + "ducklake" => { + self.var_identifiers + .insert(var_name, (AssetKind::Ducklake, asset_name)); + return; + } + _ => {} + } + } + } + } + } + } + + // Default: visit children + node.visit_children_with(self); + } + + fn visit_tagged_tpl(&mut self, node: &swc_ecma_ast::TaggedTpl) { + // Get the tag identifier + let tag_name = match node.tag.as_ref() { + Expr::Ident(ident) => ident.sym.as_str(), + _ => { + node.visit_children_with(self); + return; + } + }; + + // Check if it's a known identifier + let (kind, asset_name) = if let Some((kind, name)) = self.var_identifiers.get(tag_name) { + (*kind, name.clone()) + } else { + node.visit_children_with(self); + return; + }; + + // Extract the SQL query from the template quasis (string parts) + let sql: String = node + .tpl + .quasis + .iter() + .map(|quasi| quasi.raw.as_str()) + .collect::>() + .join(" "); + + // Determine access type based on SQL keywords + let access_type = detect_sql_access_type(&sql); + + self.assets + .push(ParseAssetsResult { kind, path: asset_name, access_type }); + } } impl AssetsFinder { diff --git a/backend/parsers/windmill-parser/src/asset_parser.rs b/backend/parsers/windmill-parser/src/asset_parser.rs index a721590ae9..26a9203c8e 100644 --- a/backend/parsers/windmill-parser/src/asset_parser.rs +++ b/backend/parsers/windmill-parser/src/asset_parser.rs @@ -16,6 +16,7 @@ pub enum AssetKind { S3Object, Resource, Ducklake, + DataTable, } #[derive(Serialize)] @@ -66,7 +67,42 @@ pub fn parse_asset_syntax(s: &str) -> Option<(AssetKind, &str)> { Some((AssetKind::Resource, &s[5..])) } else if s.starts_with("ducklake://") { Some((AssetKind::Ducklake, &s[11..])) + } else if s.starts_with("datatable://") { + Some((AssetKind::DataTable, &s[11..])) } else { None } } + +pub fn detect_sql_access_type(sql: &str) -> Option { + let first_kw = sql + .trim() + .split_whitespace() + .next() + .unwrap_or("") + .to_lowercase(); + + // Check for write operations + let has_write = first_kw.starts_with("insert") + || first_kw.starts_with("update") + || first_kw.starts_with("delete") + || first_kw.starts_with("drop") + || first_kw.starts_with("create") + || first_kw.starts_with("alter") + || first_kw.starts_with("truncate") + || first_kw.starts_with("merge"); + + // Check for read operations + let has_read = first_kw.starts_with("select") + || first_kw.starts_with("with") // CTEs, usually for reads + || first_kw.starts_with("show") + || first_kw.starts_with("describe") + || first_kw.starts_with("explain"); + + match (has_read, has_write) { + (true, true) => Some(RW), + (true, false) => Some(R), + (false, true) => Some(W), + (false, false) => None, // Unknown - couldn't determine + } +} diff --git a/backend/src/main.rs b/backend/src/main.rs index 6810d2826d..3002ac5ae5 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -82,11 +82,11 @@ static GLOBAL: Jemalloc = Jemalloc; use windmill_common::global_settings::OBJECT_STORE_CONFIG_SETTING; use windmill_worker::{ - get_hub_script_content_and_requirements, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, CSHARP_CACHE_DIR, - DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM, GO_BIN_CACHE_DIR, GO_CACHE_DIR, - JAVA_CACHE_DIR, NU_CACHE_DIR, POWERSHELL_CACHE_DIR, PY310_CACHE_DIR, PY311_CACHE_DIR, - PY312_CACHE_DIR, PY313_CACHE_DIR, RUBY_CACHE_DIR, RUST_CACHE_DIR, TAR_JAVA_CACHE_DIR, - UV_CACHE_DIR, + get_hub_script_content_and_requirements, init_worker_internal_server_inline_utils, + BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, CSHARP_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, + DENO_CACHE_DIR_NPM, GO_BIN_CACHE_DIR, GO_CACHE_DIR, JAVA_CACHE_DIR, NU_CACHE_DIR, + POWERSHELL_CACHE_DIR, PY310_CACHE_DIR, PY311_CACHE_DIR, PY312_CACHE_DIR, PY313_CACHE_DIR, + RUBY_CACHE_DIR, RUST_CACHE_DIR, TAR_JAVA_CACHE_DIR, UV_CACHE_DIR, }; use crate::monitor::{ @@ -750,9 +750,16 @@ Windmill Community Edition {GIT_VERSION} #[cfg(not(all(feature = "tantivy", feature = "parquet")))] let log_indexer_f = async { Ok(()) as anyhow::Result<()> }; + let worker_internal_server_killpill_rx = killpill_rx.resubscribe(); let server_f = async { if !is_agent { if let Some(db) = conn.as_sql() { + if worker_mode { + init_worker_internal_server_inline_utils( + worker_internal_server_killpill_rx, + base_internal_url.clone(), + )?; + } windmill_api::run_server( db.clone(), index_reader, diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index 59c5f32941..b7a46b8c78 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -2874,13 +2874,15 @@ async fn test_workflow_as_code(db: Pool) -> anyhow::Result<()> { in_test_worker( db, async move { - let job = RunJob::from(JobPayload::Code(RawCode { - language: ScriptLang::Python3, - content: WORKFLOW_AS_CODE.into(), - ..RawCode::default() - })) - .arg("n", json!(3)) - .run_until_complete(db, false, port) + let job = Box::pin( + RunJob::from(JobPayload::Code(RawCode { + language: ScriptLang::Python3, + content: WORKFLOW_AS_CODE.into(), + ..RawCode::default() + })) + .arg("n", json!(3)) + .run_until_complete(db, false, port), + ) .await; assert_eq!(job.json_result().unwrap(), json!(["OK", 3])); diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 8875aab7a2..44bdee8d11 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -847,28 +847,37 @@ paths: schema: type: boolean - /settings/get_ducklake_instance_catalog_db_status: + /settings/list_custom_instance_pg_databases: post: - summary: Returns the set-up statuses of ducklake instance catalog dbs - operationId: getDucklakeInstanceCatalogDbStatus + summary: Returns the set-up statuses of custom instance pg databases + operationId: listCustomInstanceDbs tags: - setting responses: "200": - description: Statuses of all ducklake instance catalog dbs + description: Statuses of all custom instance dbs content: application/json: schema: type: object additionalProperties: - $ref: "#/components/schemas/DucklakeInstanceCatalogDbStatus" + $ref: "#/components/schemas/CustomInstanceDb" - /settings/setup_ducklake_catalog_db/{name}: + /settings/setup_custom_instance_pg_database/{name}: post: - summary: Runs CREATE DATABASE on the Windmill Postgres and grants access to the ducklake_user - operationId: setupDucklakeCatalogDb + summary: Runs CREATE DATABASE on the Windmill Postgres and grants access to the custom_instance_user + operationId: setupCustomInstanceDb tags: - setting + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + tag: + $ref: "#/components/schemas/CustomInstanceDbTag" parameters: - in: path name: name @@ -882,7 +891,7 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/DucklakeInstanceCatalogDbStatus" + $ref: "#/components/schemas/CustomInstanceDb" /settings/global/{key}: get: @@ -2093,6 +2102,8 @@ paths: $ref: "#/components/schemas/LargeFileStorage" ducklake: $ref: "#/components/schemas/DucklakeSettings" + datatable: + $ref: "#/components/schemas/DataTableSettings" git_sync: $ref: "#/components/schemas/WorkspaceGitSyncSettings" deploy_ui: @@ -2828,6 +2839,24 @@ paths: items: type: string + /w/{workspace}/workspaces/list_datatables: + get: + summary: list Datatables + operationId: listDataTables + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: status + content: + application/json: + schema: + type: array + items: + type: string + /w/{workspace}/workspaces/edit_ducklake_config: post: summary: edit ducklake settings @@ -2855,6 +2884,32 @@ paths: application/json: schema: {} + /w/{workspace}/workspaces/edit_datatable_config: + post: + summary: edit datatable settings + operationId: editDataTableConfig + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + description: DataTable settings + required: true + content: + application/json: + schema: + type: object + required: [settings] + properties: + settings: + $ref: "#/components/schemas/DataTableSettings" + responses: + "200": + description: status + content: + application/json: + schema: {} + /w/{workspace}/workspaces/edit_git_sync_config: post: summary: edit workspace git sync settings @@ -8301,6 +8356,29 @@ paths: type: string format: uuid + /w/{workspace}/jobs/run_inline/preview: + post: + summary: run script preview without starting a new job + operationId: runScriptPreviewInline + tags: + - job + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + description: preview + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/PreviewInline" + + responses: + "200": + description: script result + content: + application/json: + schema: {} + /w/{workspace}/jobs/run_wait_result/preview: post: summary: run script preview and wait for result @@ -17508,6 +17586,18 @@ components: required: - args + PreviewInline: + type: object + properties: + content: + type: string + description: The code to run + args: + $ref: "#/components/schemas/ScriptArgs" + language: + $ref: "#/components/schemas/ScriptLang" + required: [content, args, language] + WorkflowTask: type: object properties: @@ -18795,7 +18885,7 @@ components: - SKIP - FAIL - DucklakeInstanceCatalogDbStatusLogs: + CustomInstanceDbLogs: type: object properties: super_admin: @@ -18806,20 +18896,25 @@ components: $ref: "#/components/schemas/LoggedWizardStatus" created_database: $ref: "#/components/schemas/LoggedWizardStatus" - description: Created database status log db_connect: $ref: "#/components/schemas/LoggedWizardStatus" grant_permissions: $ref: "#/components/schemas/LoggedWizardStatus" - DucklakeInstanceCatalogDbStatus: + CustomInstanceDbTag: + type: string + enum: + - ducklake + - datatable + + CustomInstanceDb: type: object required: - logs - success properties: logs: - $ref: "#/components/schemas/DucklakeInstanceCatalogDbStatusLogs" + $ref: "#/components/schemas/CustomInstanceDbLogs" success: type: boolean description: Whether the operation completed successfully @@ -18829,6 +18924,8 @@ components: nullable: true description: Error message if the operation failed example: "Connection timeout" + tag: + $ref: "#/components/schemas/CustomInstanceDbTag" NewSqsTrigger: type: object @@ -20137,6 +20234,29 @@ components: required: - path + DataTableSettings: + type: object + required: [datatables] + properties: + datatables: + type: object + additionalProperties: + type: object + required: [database] + properties: + database: + type: object + properties: + resource_type: + type: string + enum: + - postgresql + - instance + resource_path: + type: string + required: + - resource_type + DynamicInputData: type: object properties: @@ -20777,6 +20897,7 @@ components: - s3object - resource - ducklake + - datatable Asset: type: object properties: diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 39edaade1d..f3a0cc7e11 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -29,6 +29,7 @@ use url::Url; #[cfg(all(feature = "enterprise", feature = "smtp"))] use windmill_common::auth::is_super_admin_email; use windmill_common::auth::TOKEN_PREFIX_LEN; +use windmill_common::client::AuthedClient; use windmill_common::db::UserDbWithAuthed; use windmill_common::error::JsonResult; use windmill_common::flow_conversations::add_message_to_conversation_tx; @@ -36,7 +37,7 @@ use windmill_common::flow_status::{JobResult, RestartedFrom}; use windmill_common::jobs::{ check_tag_available_for_workspace_internal, format_completed_job_result, format_result, ConcurrencySettings, ConcurrencySettingsWithCustom, DebouncingSettings, DynamicInput, - JobTriggerKind, ENTRYPOINT_OVERRIDE, + JobTriggerKind, RunInlinePreviewScriptFnParams, ENTRYPOINT_OVERRIDE, }; use windmill_common::s3_helpers::{upload_artifact_to_store, BundleFormat}; use windmill_common::utils::{RunnableKind, WarnAfterExt}; @@ -47,6 +48,7 @@ use windmill_common::workspace_dependencies::{ use windmill_common::DYNAMIC_INPUT_CACHE; #[cfg(all(feature = "enterprise", feature = "smtp"))] use windmill_common::{email_oss::send_email_html, server::load_smtp_config}; +use windmill_worker::get_worker_internal_server_inline_utils; use windmill_common::variables::get_workspace_key; @@ -237,6 +239,7 @@ pub fn workspaced_service() -> Router { .layer(ce_headers.clone()), ) .route("/run/preview", post(run_preview_script)) + .route("/run_inline/preview", post(run_inline_preview_script)) .route( "/run_wait_result/preview", post(run_wait_result_preview_script), @@ -3577,6 +3580,13 @@ struct Preview { format: Option, } +#[derive(Debug, Deserialize)] +struct PreviewInline { + content: String, + args: Option>>, + language: ScriptLang, +} + #[derive(Deserialize)] pub struct WorkflowTask { pub args: Option>>, @@ -5983,6 +5993,39 @@ async fn run_preview_script( Ok((StatusCode::CREATED, uuid.to_string())) } +async fn run_inline_preview_script( + authed: ApiAuthed, + Tokened { token }: Tokened, + Extension(db): Extension, + Path(w_id): Path, + Json(preview): Json, +) -> error::Result { + let utils = get_worker_internal_server_inline_utils()?; + let result = utils.run_inline_preview_script.as_ref()(RunInlinePreviewScriptFnParams { + content: preview.content, + args: preview.args, + workspace_id: w_id.clone(), + base_internal_url: utils.base_internal_url.clone(), + killpill_rx: utils.killpill_rx.resubscribe(), + created_by: authed.display_username().to_string(), + permissioned_as: username_to_permissioned_as(&authed.username), + permissioned_as_email: authed.email.clone(), + lang: preview.language, + job_dir: "".to_string(), + worker_name: "".to_string(), + worker_dir: "".to_string(), + client: AuthedClient { + base_internal_url: utils.base_internal_url.clone(), + force_client: None, + token, + workspace: w_id, + }, + conn: windmill_common::worker::Connection::Sql(db), + }) + .await?; + Ok(Json(to_raw_value(&result)).into_response()) +} + async fn run_wait_result_preview_script( authed: ApiAuthed, Extension(db): Extension, diff --git a/backend/windmill-api/src/resources.rs b/backend/windmill-api/src/resources.rs index 503c60e1f8..14c7054c3a 100644 --- a/backend/windmill-api/src/resources.rs +++ b/backend/windmill-api/src/resources.rs @@ -38,10 +38,10 @@ use windmill_common::{ db::{UserDB, UserDbWithAuthed, UserDbWithOptAuthed}, error::{self, Error, JsonResult, Result}, get_database_url, parse_postgres_url, + utils::get_custom_pg_instance_password, utils::{not_found_if_none, paginate, require_admin, Pagination, StripPath}, variables, worker::{CLOUD_HOSTED, TMP_DIR}, - workspaces::get_ducklake_instance_pg_catalog_password, }; pub fn workspaced_service() -> Router { @@ -469,17 +469,17 @@ pub async fn get_resource_value_interpolated_internal( token: &str, allow_cache: bool, ) -> Result> { - // This is a special syntax to help debugging ducklake catalogs stored in the instance - if let Some(dbname) = path.strip_prefix("INSTANCE_DUCKLAKE_CATALOG/") { + // This is a special syntax to help debugging custom instance databases + if let Some(dbname) = path.strip_prefix("CUSTOM_INSTANCE_DB/") { require_super_admin(db, &authed.email).await?; let pg_creds = parse_postgres_url(&get_database_url().await?.as_str().await)?; return Ok(Some(serde_json::json!({ "dbname": dbname, "host": pg_creds.host, "port": pg_creds.port, - "user": "ducklake_user", + "user": "custom_instance_user", "sslmode": pg_creds.ssl_mode, - "password": get_ducklake_instance_pg_catalog_password(&db).await?, + "password": get_custom_pg_instance_password(&db).await?, }))); } diff --git a/backend/windmill-api/src/settings.rs b/backend/windmill-api/src/settings.rs index bff2b282b6..ec0fe5061e 100644 --- a/backend/windmill-api/src/settings.rs +++ b/backend/windmill-api/src/settings.rs @@ -70,12 +70,12 @@ pub fn global_service() -> Router { post(acknowledge_critical_alert), ) .route( - "/get_ducklake_instance_catalog_db_status", - post(get_ducklake_instance_catalog_db_status), + "/list_custom_instance_pg_databases", + post(list_custom_instance_pg_databases), ) .route( - "/setup_ducklake_catalog_db/:name", - post(setup_ducklake_catalog_db), + "/setup_custom_instance_pg_database/:name", + post(setup_custom_instance_pg_database), ) .route( "/critical_alerts/acknowledge_all", @@ -218,7 +218,12 @@ pub struct Value { } pub async fn delete_global_setting(db: &DB, key: &str) -> error::Result<()> { - if key == "ducklake_user_pg_pwd" || key == "ducklake_settings" { + // ducklake_user_pg_pwd and ducklake_settings were old names stored as standalone global settings. + // Leave them for backward compatibility (CLI will try to delete them if not present in the yaml) + if key == "ducklake_user_pg_pwd" + || key == "ducklake_settings" + || key == "custom_instance_pg_databases" + { tracing::error!("Tried to unset global setting {}, ignored", key); return Ok(()); } @@ -578,15 +583,16 @@ pub async fn acknowledge_all_critical_alerts() -> error::Error { } #[derive(Deserialize, Debug, Serialize)] -struct DucklakeInstanceCatalogDbStatus { - logs: DucklakeInstanceCatalogDbStatusLogs, // (Step, Message)[] +struct CustomInstanceDb { + logs: CustomInstanceDbLogs, // (Step, Message)[] success: bool, error: Option, + tag: Option, } #[derive(Deserialize, Debug, Serialize, Default)] #[serde(default)] -struct DucklakeInstanceCatalogDbStatusLogs { +struct CustomInstanceDbLogs { super_admin: String, #[serde(skip_serializing_if = "String::is_empty")] database_credentials: String, @@ -600,50 +606,56 @@ struct DucklakeInstanceCatalogDbStatusLogs { grant_permissions: String, } -async fn get_ducklake_instance_catalog_db_status( +async fn list_custom_instance_pg_databases( _authed: ApiAuthed, Extension(db): Extension, -) -> JsonResult> { +) -> JsonResult> { let result = sqlx::query_scalar!( - r#"SELECT value->'instance_catalog_db_status' FROM global_settings WHERE name = 'ducklake_settings'"#, + r#"SELECT value->'databases' FROM global_settings WHERE name = 'custom_instance_pg_databases'"#, ) .fetch_one(&db) .await? - .ok_or_else(|| error::Error::ExecutionErr("Couldn't find ducklake_settings".to_string()))?; + .ok_or_else(|| error::Error::ExecutionErr("Couldn't find custom_instance_pg_databases".to_string()))?; let result = serde_json::from_value(result).map_err(|e| { error::Error::ExecutionErr(format!( - "couldn't parse instance_catalog_db_status : {}", + "couldn't parse custom_instance_pg_databases.databases : {}", e.to_string() )) })?; return Ok(Json(result)); } -async fn setup_ducklake_catalog_db( +#[derive(Deserialize)] +struct SetupCustomInstanceDbBody { + tag: Option, +} + +async fn setup_custom_instance_pg_database( authed: ApiAuthed, Extension(db): Extension, Path(dbname): Path, -) -> JsonResult { - let mut logs = DucklakeInstanceCatalogDbStatusLogs::default(); - let result = setup_ducklake_catalog_db_inner(authed, &db, &dbname, &mut logs).await; + Json(body): Json, +) -> JsonResult { + let mut logs = CustomInstanceDbLogs::default(); + let result = setup_custom_instance_pg_database_inner(authed, &db, &dbname, &mut logs).await; let success = result.is_ok(); let error = result.err().map(|e| e.to_string()); - let status = DucklakeInstanceCatalogDbStatus { logs, success, error }; + let status = CustomInstanceDb { logs, success, error, tag: body.tag }; let status_json = serde_json::to_value(&status).map_err(to_anyhow)?; // Save that the database was setup successfully sqlx::query!( - r#"UPDATE global_settings SET value = jsonb_set(value, '{instance_catalog_db_status}', (COALESCE(value->'instance_catalog_db_status', '{}'::jsonb) || to_jsonb($1::json))) WHERE name = 'ducklake_settings'"#, + r#"UPDATE global_settings SET value = jsonb_set(value, '{databases}', (COALESCE(value->'databases', '{}'::jsonb) || to_jsonb($1::json))) WHERE name = 'custom_instance_pg_databases'"#, json!({ dbname: status_json }) ).execute(&db).await?; Ok(Json(status)) } -async fn setup_ducklake_catalog_db_inner( +async fn setup_custom_instance_pg_database_inner( authed: ApiAuthed, db: &DB, dbname: &str, - logs: &mut DucklakeInstanceCatalogDbStatusLogs, + logs: &mut CustomInstanceDbLogs, ) -> Result<()> { require_super_admin(db, &authed.email).await?; logs.super_admin = "OK".to_string(); @@ -738,16 +750,16 @@ async fn setup_ducklake_catalog_db_inner( client .batch_execute(&format!( - "GRANT CONNECT ON DATABASE \"{dbname}\" TO ducklake_user; - GRANT USAGE ON SCHEMA public TO ducklake_user; - GRANT CREATE ON SCHEMA public TO ducklake_user; + "GRANT CONNECT ON DATABASE \"{dbname}\" TO custom_instance_user; + GRANT USAGE ON SCHEMA public TO custom_instance_user; + GRANT CREATE ON SCHEMA public TO custom_instance_user; ALTER DEFAULT PRIVILEGES IN SCHEMA public - GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO ducklake_user;" + GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO custom_instance_user;" )) .await .map_err(|e| { error::Error::ExecutionErr(format!( - "Failed to grant permissions to ducklake_user: {}", + "Failed to grant permissions to custom_instance_user: {}", e.to_string(), )) })?; diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index effe576d55..8c1a83ce13 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -42,7 +42,9 @@ use windmill_common::worker::{to_raw_value, CLOUD_HOSTED}; use windmill_common::workspaces::GitRepositorySettings; #[cfg(feature = "enterprise")] use windmill_common::workspaces::WorkspaceDeploymentUISettings; -use windmill_common::workspaces::WorkspaceGitSyncSettings; +use windmill_common::workspaces::{ + DataTable, DataTableCatalogResourceType, WorkspaceGitSyncSettings, +}; use windmill_common::workspaces::{Ducklake, DucklakeCatalogResourceType}; use windmill_common::{ error::{Error, JsonResult, Result}, @@ -129,6 +131,8 @@ pub fn workspaced_service() -> Router { ) .route("/edit_ducklake_config", post(edit_ducklake_config)) .route("/list_ducklakes", get(list_ducklakes)) + .route("/list_datatables", get(list_datatables)) + .route("/edit_datatable_config", post(edit_datatable_config)) .route("/edit_git_sync_config", post(edit_git_sync_config)) .route("/edit_git_sync_repository", post(edit_git_sync_repository)) .route( @@ -260,6 +264,8 @@ pub struct WorkspaceSettings { #[serde(skip_serializing_if = "Option::is_none")] pub ducklake: Option, #[serde(skip_serializing_if = "Option::is_none")] + pub datatable: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub git_sync: Option, // effectively: WorkspaceGitSyncSettings #[serde(skip_serializing_if = "Option::is_none")] pub deploy_ui: Option, // effectively: WorkspaceDeploymentUISettings @@ -330,12 +336,6 @@ struct LargeFileStorageWithSecondary { #[serde(default)] secondary_storage: HashMap, } - -#[derive(Deserialize, Serialize, Debug)] -pub struct DucklakeSettings { - pub ducklakes: HashMap, -} - #[derive(Deserialize, Debug)] struct EditLargeFileStorageConfig { large_file_storage: Option, @@ -346,6 +346,21 @@ struct EditDucklakeConfig { settings: DucklakeSettings, } +#[derive(Deserialize, Serialize, Debug)] +pub struct DucklakeSettings { + pub ducklakes: HashMap, +} + +#[derive(Deserialize, Debug)] +struct EditDataTableConfig { + settings: DataTableSettings, +} + +#[derive(Deserialize, Serialize, Debug)] +pub struct DataTableSettings { + pub datatables: HashMap, +} + #[derive(Deserialize)] struct CreateWorkspace { id: String, @@ -530,6 +545,7 @@ async fn get_settings( error_handler_extra_args, error_handler_muted_on_cancel, large_file_storage, + datatable, ducklake, git_sync, deploy_ui, @@ -1154,6 +1170,28 @@ async fn list_ducklakes( Ok(Json(ducklakes)) } +async fn list_datatables( + _authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, +) -> JsonResult> { + let datatables = sqlx::query_scalar!( + r#" + SELECT jsonb_object_keys(ws.datatable->'datatables') AS datatable_name + FROM workspace_settings ws + WHERE ws.workspace_id = $1 + "#, + &w_id + ) + .fetch_all(&db) + .await? + .into_iter() + .filter_map(|s| s) + .collect(); + + Ok(Json(datatables)) +} + async fn edit_ducklake_config( authed: ApiAuthed, Extension(db): Extension, @@ -1178,30 +1216,7 @@ async fn edit_ducklake_config( ) .await?; - // Check that all ducklake catalog resources exist to prevent - // exploiting the shared property to see any resource - for dl in new_config.settings.ducklakes.values() { - if dl.catalog.resource_type == DucklakeCatalogResourceType::Instance { - continue; - } - let catalog_res = sqlx::query_scalar!( - "SELECT 1 FROM resource WHERE workspace_id = $1 AND path = $2", - &w_id, - &dl.catalog.resource_path - ) - .fetch_optional(&mut *tx) - .await? - .flatten(); - - if catalog_res.is_none() { - return Err(Error::BadRequest(format!( - "Ducklake catalog resource {} not found in workspace {}", - dl.catalog.resource_path, &w_id - ))); - } - } - - // Check that non-superadmins are not abusing Instance catalogs + // Check that non-superadmins are not abusing Instance databases if !is_superadmin { let old_ducklakes = sqlx::query_scalar!( r#" @@ -1225,7 +1240,7 @@ async fn edit_ducklake_config( || old_dl.unwrap().catalog.resource_path != dl.catalog.resource_path { return Err(Error::BadRequest( - "Only superadmins can create or modify ducklakes with Instance catalogs" + "Only superadmins can create or modify ducklakes with Instance databases" .to_string(), )); } @@ -1249,6 +1264,78 @@ async fn edit_ducklake_config( Ok(format!("Edit ducklake config for workspace {}", &w_id)) } +async fn edit_datatable_config( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + ApiAuthed { is_admin, username, email, .. }: ApiAuthed, + Json(new_config): Json, +) -> Result { + require_admin(is_admin, &username)?; + let is_superadmin = require_super_admin(&db, &email).await.is_ok(); + + let mut tx = db.begin().await?; + + let args_for_audit = format!("{:?}", new_config.settings); + audit_log( + &mut *tx, + &authed, + "workspaces.edit_datatable_config", + ActionKind::Update, + &w_id, + Some(&authed.email), + Some([("datatable", args_for_audit.as_str())].into()), + ) + .await?; + + // Check that non-superadmins are not abusing Instance databases + if !is_superadmin { + let old_datatables = sqlx::query_scalar!( + r#" + SELECT ws.datatable->'datatables' AS datatable_name + FROM workspace_settings ws + WHERE ws.workspace_id = $1 + "#, + &w_id + ) + .fetch_one(&db) + .await? + .unwrap_or(serde_json::Value::Null); + let old_datatables: HashMap = + serde_json::from_value(old_datatables).unwrap_or_default(); + for (name, dt) in new_config.settings.datatables.iter() { + if dt.database.resource_type == DataTableCatalogResourceType::Instance { + let old_dt = old_datatables.get(name); + if old_dt.is_none() + || old_dt.unwrap().database.resource_type + != DataTableCatalogResourceType::Instance + || old_dt.unwrap().database.resource_path != dt.database.resource_path + { + return Err(Error::BadRequest( + "Only superadmins can create or modify data tables with Instance databases" + .to_string(), + )); + } + } + } + } + + let config: serde_json::Value = serde_json::to_value(new_config.settings) + .map_err(|err| Error::internal_err(err.to_string()))?; + + sqlx::query!( + "UPDATE workspace_settings SET datatable = $1 WHERE workspace_id = $2", + config, + &w_id + ) + .execute(&mut *tx) + .await?; + + tx.commit().await?; + + Ok(format!("Edit datatable config for workspace {}", &w_id)) +} + #[derive(Deserialize)] pub struct EditGitSyncConfig { #[cfg(feature = "enterprise")] @@ -2586,6 +2673,8 @@ async fn update_workspace_settings( deploy_to = $1, ai_config = source_ws.ai_config, large_file_storage = source_ws.large_file_storage, + ducklake = source_ws.ducklake, + datatable = source_ws.datatable, git_app_installations = source_ws.git_app_installations FROM workspace_settings source_ws WHERE source_ws.workspace_id = $1 diff --git a/backend/windmill-common/Cargo.toml b/backend/windmill-common/Cargo.toml index 0ba041e1ec..2ffb3d5a85 100644 --- a/backend/windmill-common/Cargo.toml +++ b/backend/windmill-common/Cargo.toml @@ -69,6 +69,7 @@ aws-credential-types.workspace = true aws-smithy-types.workspace = true base64.workspace = true bitflags.workspace = true +once_cell.workspace = true phf.workspace = true aws-smithy-types-convert = { workspace = true, optional = true } diff --git a/backend/windmill-common/src/assets.rs b/backend/windmill-common/src/assets.rs index 80804a3544..c76824a5cb 100644 --- a/backend/windmill-common/src/assets.rs +++ b/backend/windmill-common/src/assets.rs @@ -12,6 +12,7 @@ pub enum AssetKind { // Avoid unnexpected crashes when deserializing old assets Variable, // Deprecated Ducklake, + DataTable, } #[derive(Serialize, Deserialize, Debug, PartialEq, Copy, Clone, Hash, Eq, sqlx::Type)] diff --git a/backend/windmill-common/src/jobs.rs b/backend/windmill-common/src/jobs.rs index 693343e11b..e88fb9f25c 100644 --- a/backend/windmill-common/src/jobs.rs +++ b/backend/windmill-common/src/jobs.rs @@ -1,8 +1,9 @@ -use std::collections::HashMap; +use std::{collections::HashMap, future::Future, pin::Pin, sync::Arc}; use bytes::Bytes; use futures_core::Stream; use indexmap::IndexMap; +use once_cell::sync::OnceCell; use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; use sqlx::types::Json; @@ -17,6 +18,7 @@ pub const EMAIL_ERROR_HANDLER_USER_EMAIL: &str = "email_error_handler@windmill.d use crate::{ apps::AppScriptId, auth::is_super_admin_email, + client::AuthedClient, db::{AuthedRef, UserDbWithAuthed, DB}, error::{self, to_anyhow, Error}, flow_status::{FlowStatus, RestartedFrom}, @@ -993,3 +995,39 @@ pub async fn lock_debounce_key<'c>( .await .map_err(error::Error::from) } + +pub struct RunInlinePreviewScriptFnParams { + pub workspace_id: String, + pub content: String, + pub lang: ScriptLang, + pub args: Option>>, + pub created_by: String, + pub permissioned_as: String, + pub permissioned_as_email: String, + pub base_internal_url: String, + pub worker_name: String, + pub conn: crate::worker::Connection, + pub client: AuthedClient, + pub job_dir: String, + pub worker_dir: String, + pub killpill_rx: tokio::sync::broadcast::Receiver<()>, +} + +#[derive(Clone)] +pub struct WorkerInternalServerInlineUtils { + pub killpill_rx: Arc>, + pub base_internal_url: String, + pub run_inline_preview_script: Arc< + dyn Fn( + RunInlinePreviewScriptFnParams, + ) -> Pin>> + Send>> + + Send + + Sync, + >, +} +// To run a script inline, bypassing the db and job queue, windmill-api uses these functions. +// They should only be called by the internal server of a worker. +// main() sets the global on startup. +// The server cannot call the worker functions directly because they are independent crates +pub static WORKER_INTERNAL_SERVER_INLINE_UTILS: OnceCell = + OnceCell::new(); diff --git a/backend/windmill-common/src/utils.rs b/backend/windmill-common/src/utils.rs index 623486d19c..dc5189eff1 100644 --- a/backend/windmill-common/src/utils.rs +++ b/backend/windmill-common/src/utils.rs @@ -953,3 +953,16 @@ impl ExpiringCacheEntry { self.expiry < std::time::Instant::now() } } + +pub async fn get_custom_pg_instance_password(db: &DB) -> Result { + sqlx::query_scalar!( + "SELECT value->>'user_pwd' FROM global_settings WHERE name = 'custom_instance_pg_databases';" + ) + .fetch_optional(db) + .await? + .flatten().ok_or_else(|| + Error::BadRequest(format!( + "Custom instance db password not found, did you run migrations ?" + )) + ) +} diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index 0fa8769ffd..a1eafa0549 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -9,6 +9,7 @@ use strum::AsRefStr; use crate::{ error::{to_anyhow, Error, Result}, get_database_url, parse_postgres_url, + utils::get_custom_pg_instance_password, variables::{build_crypt, decrypt}, DB, }; @@ -148,6 +149,70 @@ pub async fn get_team_plan_status(_db: &crate::DB, _w_id: &str) -> Result Result { + let datatable = sqlx::query_scalar!( + r#" + SELECT ws.datatable->'datatables'->$2 AS config + FROM workspace_settings ws + WHERE ws.workspace_id = $1 + "#, + &w_id, + name + ) + .fetch_one(db) + .await + .map_err(|err| Error::internal_err(format!("getting datatable {name}: {err}")))? + .ok_or_else(|| Error::internal_err(format!("datatable {name} not found")))?; + let datatable = serde_json::from_value::(datatable)?; + + let db_resource = if datatable.database.resource_type == DataTableCatalogResourceType::Instance + { + let pg_creds = parse_postgres_url(&get_database_url().await?.as_str().await)?; + json!({ + "dbname": datatable.database.resource_path, + "host": pg_creds.host, + "port": pg_creds.port, + "user": "custom_instance_user", + "sslmode": pg_creds.ssl_mode, + "password": get_custom_pg_instance_password(&db).await?, + }) + } else { + transform_json_unchecked( + &serde_json::Value::String(format!("$res:{}", datatable.database.resource_path)), + w_id, + db, + ) + .await? + }; + + Ok(db_resource) +} + #[derive(Deserialize, Serialize, Debug)] pub struct Ducklake { pub catalog: DucklakeCatalog, @@ -213,9 +278,9 @@ pub async fn get_ducklake_from_db_unchecked( "dbname": ducklake.catalog.resource_path, "host": pg_creds.host, "port": pg_creds.port, - "user": "ducklake_user", + "user": "custom_instance_user", "sslmode": pg_creds.ssl_mode, - "password": get_ducklake_instance_pg_catalog_password(&db).await?, + "password": get_custom_pg_instance_password(&db).await?, }) } else { transform_json_unchecked( @@ -233,19 +298,6 @@ pub async fn get_ducklake_from_db_unchecked( Ok(ducklake) } -pub async fn get_ducklake_instance_pg_catalog_password(db: &DB) -> Result { - sqlx::query_scalar!( - "SELECT value->>'ducklake_user_pg_pwd' FROM global_settings WHERE name = 'ducklake_settings';" - ) - .fetch_optional(db) - .await? - .flatten().ok_or_else(|| - Error::BadRequest(format!( - "Ducklake instance catalog password not found, did you run migrations ?" - )) - ) -} - // This does not check for any permission. Should never be displayed to a user. #[async_recursion] async fn transform_json_unchecked( diff --git a/backend/windmill-worker/src/agent_workers.rs b/backend/windmill-worker/src/agent_workers.rs index 0aaecdbe1e..9bf0ea1834 100644 --- a/backend/windmill-worker/src/agent_workers.rs +++ b/backend/windmill-worker/src/agent_workers.rs @@ -64,4 +64,18 @@ pub async fn get_ducklake_from_agent_http( .await } +#[allow(dead_code)] +pub async fn get_datatable_resource_from_agent_http( + client: &HttpClient, + name: &str, + w_id: &str, +) -> anyhow::Result { + client + .get(&format!( + "/api/w/{}/agent_workers/get_datatable_resource/{}", + w_id, &name + )) + .await +} + pub const UPDATE_PING_URL: &str = "/api/agent_workers/update_ping"; diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index ed4cfccd26..ea1f6a763d 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -238,7 +238,7 @@ pub async fn transform_json_value( Value::String(y) if y.starts_with("$res:") => { let path = y.strip_prefix("$res:").unwrap(); - if path.split("/").count() < 2 && !path.starts_with("INSTANCE_DUCKLAKE_CATALOG/") { + if path.split("/").count() < 2 && !path.starts_with("CUSTOM_INSTANCE_DB/") { return Err(Error::internal_err(format!( "Argument `{name}` is an invalid resource path: {path}", ))); diff --git a/backend/windmill-worker/src/duckdb_executor.rs b/backend/windmill-worker/src/duckdb_executor.rs index f8c7febad4..8cd0380df6 100644 --- a/backend/windmill-worker/src/duckdb_executor.rs +++ b/backend/windmill-worker/src/duckdb_executor.rs @@ -13,11 +13,14 @@ use windmill_common::error::{to_anyhow, Error, Result}; use windmill_common::s3_helpers::{S3Object, S3_PROXY_LAST_ERRORS_CACHE}; use windmill_common::utils::sanitize_string_from_password; use windmill_common::worker::{Connection, SqlResultCollectionStrategy}; -use windmill_common::workspaces::{get_ducklake_from_db_unchecked, DucklakeCatalogResourceType}; +use windmill_common::workspaces::{ + get_datatable_resource_from_db_unchecked, get_ducklake_from_db_unchecked, + DucklakeCatalogResourceType, +}; use windmill_parser_sql::{parse_duckdb_sig, parse_sql_blocks}; use windmill_queue::{CanceledBy, MiniPulledJob}; -use crate::agent_workers::get_ducklake_from_agent_http; +use crate::agent_workers::{get_datatable_resource_from_agent_http, get_ducklake_from_agent_http}; use crate::common::{build_args_values, get_reserved_variables, OccupancyMetrics}; use crate::handle_child::run_future_with_polling_update_job_poller; #[cfg(feature = "mysql")] @@ -38,6 +41,7 @@ pub async fn do_duckdb( #[allow(unused_variables)] column_order_ref: &mut Option>, occupancy_metrics: &mut OccupancyMetrics, parent_runnable_path: Option, + run_inline: bool, ) -> Result> { let annotations = windmill_common::worker::SqlAnnotations::parse(query); let collection_strategy = @@ -105,41 +109,48 @@ pub async fn do_duckdb( let query_block_list = parse_sql_blocks(&query); - // Replace windmill resource ATTACH statements with the real instructions + // Replace custom ATTACH statements with the real instructions let query_block_list = { let mut v = vec![]; for query_block in query_block_list.iter() { let query_block = remove_comments(&query_block); - match parse_attach_db_resource(query_block) { - Some(parsed) => { - v.extend( - transform_attach_db_resource_query( - &parsed, - &job.id, - client, - &mut hidden_passwords, - ) - .await?, - ); - if parsed.db_type == "bigquery" { - bigquery_credentials = Some(UseBigQueryCredentialsFile::new( - job.id, - parsed.resource_path, - )?); - } + if let Some(parsed) = parse_attach_db_resource(query_block) { + v.extend( + transform_attach_db_resource_query( + &parsed, + &job.id, + client, + &mut hidden_passwords, + ) + .await?, + ); + if parsed.db_type == "bigquery" { + bigquery_credentials = Some(UseBigQueryCredentialsFile::new( + job.id, + parsed.resource_path, + )?); } - None => match transform_attach_ducklake( - &query_block, - conn, - &mut hidden_passwords, - &job.workspace_id, - ) - .await? - { - Some(ducklake_query) => v.extend(ducklake_query), - None => v.push(query_block.to_string()), - }, - }; + } else if let Some(ducklake_query) = transform_attach_ducklake( + &query_block, + conn, + &mut hidden_passwords, + &job.workspace_id, + ) + .await? + { + v.extend(ducklake_query); + } else if let Some(datatable_query) = transform_attach_datatable( + &query_block, + conn, + &mut hidden_passwords, + &job.workspace_id, + ) + .await? + { + v.extend(datatable_query); + } else { + v.push(query_block.to_string()); + } } v }; @@ -182,19 +193,23 @@ pub async fn do_duckdb( Ok(result) }; - let result = run_future_with_polling_update_job_poller( - job.id, - job.timeout, - conn, - mem_peak, - canceled_by, - result_f, - worker_name, - &job.workspace_id, - &mut Some(occupancy_metrics), - Box::pin(futures::stream::once(async { 0 })), - ) - .await; + let result = if run_inline { + result_f.await + } else { + run_future_with_polling_update_job_poller( + job.id, + job.timeout, + conn, + mem_peak, + canceled_by, + result_f, + worker_name, + &job.workspace_id, + &mut Some(occupancy_metrics), + Box::pin(futures::stream::once(async { 0 })), + ) + .await + }; match result { Ok(result) => Ok(result), @@ -492,18 +507,28 @@ async fn transform_attach_db_resource_query( if let Some(pwd) = db_resource.get("password").and_then(|p| p.as_str()) { hidden_passwords.lock().unwrap().push(pwd.to_string()); } + db_resource_to_attach_statements(db_resource, parsed.name, parsed.db_type, parsed.extra_args) + .await +} + +async fn db_resource_to_attach_statements( + db_resource: Value, + ident_name: &str, + db_type: &str, + extra_args: Option<&str>, +) -> Result> { let attach_str = format!( "ATTACH '{}' as {} (TYPE {}{});", - format_attach_db_conn_str(db_resource, parsed.db_type)?, - parsed.name, - parsed.db_type, - parsed.extra_args.unwrap_or("") + format_attach_db_conn_str(db_resource, db_type)?, + ident_name, + db_type, + extra_args.unwrap_or("") ) .to_string(); Ok(vec![ - get_attach_db_install_str(parsed.db_type)?.to_string(), - format!("LOAD {};", parsed.db_type), + get_attach_db_install_str(db_type)?.to_string(), + format!("LOAD {};", db_type), attach_str, ]) } @@ -569,6 +594,38 @@ async fn transform_attach_ducklake( ])) } +async fn transform_attach_datatable( + query: &str, + conn: &Connection, + hidden_passwords: &mut Arc>>, + w_id: &str, +) -> Result>> { + lazy_static::lazy_static! { + static ref RE: regex::Regex = regex::Regex::new(r"(?i)ATTACH\s*'datatable(://[^':]+)?'\s*AS\s+([^ ;]+)").unwrap(); + } + let Some(cap) = RE.captures(query) else { + return Ok(None); + }; + let name = cap.get(1).map(|m| &m.as_str()[3..]).unwrap_or("main"); + let alias_name = cap.get(2).map(|m| m.as_str()).unwrap_or(""); + + let db_resource = match conn { + Connection::Http(client) => { + get_datatable_resource_from_agent_http(client, name, w_id).await? + } + Connection::Sql(db) => get_datatable_resource_from_db_unchecked(db, w_id, name).await?, + }; + let db_type = "postgres"; + + if let Some(pwd) = db_resource.get("password").and_then(|p| p.as_str()) { + hidden_passwords.lock().unwrap().push(pwd.to_string()); + } + + Ok(Some( + db_resource_to_attach_statements(db_resource, alias_name, db_type, None).await?, + )) +} + async fn transform_s3_uris(query: &str) -> Result { let mut transformed_query = None; lazy_static::lazy_static! { @@ -608,7 +665,9 @@ pub struct UseBigQueryCredentialsFile { impl UseBigQueryCredentialsFile { fn new(job_id: Uuid, bigquery_resource: &str) -> Result { let path = format!("/tmp/service-account-credentials-{}.json", job_id); - env::set_var("GOOGLE_APPLICATION_CREDENTIALS", &path); + unsafe { + env::set_var("GOOGLE_APPLICATION_CREDENTIALS", &path); + } std::fs::write(&path, bigquery_resource) .map_err(|e| Error::ExecutionErr(format!("Failed to write BigQuery creds: {e}")))?; Ok(Self { path }) @@ -616,7 +675,9 @@ impl UseBigQueryCredentialsFile { } impl Drop for UseBigQueryCredentialsFile { fn drop(&mut self) { - env::remove_var("GOOGLE_APPLICATION_CREDENTIALS"); + unsafe { + env::remove_var("GOOGLE_APPLICATION_CREDENTIALS"); + } if matches!(std::fs::exists(&self.path), Ok(true)) { let _ = std::fs::remove_file(&self.path); } diff --git a/backend/windmill-worker/src/pg_executor.rs b/backend/windmill-worker/src/pg_executor.rs index 3b3eec71f3..08ff7a08e9 100644 --- a/backend/windmill-worker/src/pg_executor.rs +++ b/backend/windmill-worker/src/pg_executor.rs @@ -31,6 +31,7 @@ use windmill_common::s3_helpers::convert_json_line_stream; use windmill_common::worker::{ to_raw_value, Connection, SqlResultCollectionStrategy, CLOUD_HOSTED, }; +use windmill_common::workspaces::get_datatable_resource_from_db_unchecked; use windmill_parser::{Arg, Typ}; use windmill_parser_sql::{ parse_db_resource, parse_pg_statement_arg_indices, parse_pgsql_sig, parse_s3_mode, @@ -38,6 +39,7 @@ use windmill_parser_sql::{ }; use windmill_queue::{CanceledBy, MiniPulledJob}; +use crate::agent_workers::get_datatable_resource_from_agent_http; use crate::common::{ build_args_values, get_reserved_variables, s3_mode_args_to_worker_data, sizeof_val, OccupancyMetrics, S3ModeWorkerData, @@ -197,6 +199,7 @@ pub async fn do_postgresql( column_order: &mut Option>, occupancy_metrics: &mut OccupancyMetrics, parent_runnable_path: Option, + run_inline: bool, ) -> error::Result> { let pg_args = build_args_values(job, client, conn).await?; @@ -214,7 +217,22 @@ pub async fn do_postgresql( .await?, ) } else { - pg_args.get("database").cloned() + match pg_args.get("database").cloned() { + Some(Value::String(db_str)) if db_str.starts_with("datatable://") => { + let db_str = db_str.trim_start_matches("datatable://"); + Some(match conn { + Connection::Http(client) => { + get_datatable_resource_from_agent_http(client, &db_str, &job.workspace_id) + .await? + } + Connection::Sql(db) => { + get_datatable_resource_from_db_unchecked(db, &job.workspace_id, &db_str) + .await? + } + }) + } + database => database, + } }; let database = if let Some(db) = db_arg { @@ -380,19 +398,23 @@ pub async fn do_postgresql( collection_strategy.collect(results) }; - let result = run_future_with_polling_update_job_poller( - job.id, - job.timeout, - conn, - mem_peak, - canceled_by, - result_f, - worker_name, - &job.workspace_id, - &mut Some(occupancy_metrics), - Box::pin(futures::stream::once(async { 0 })), - ) - .await?; + let result = if run_inline { + result_f.await? + } else { + run_future_with_polling_update_job_poller( + job.id, + job.timeout, + conn, + mem_peak, + canceled_by, + result_f, + worker_name, + &job.workspace_id, + &mut Some(occupancy_metrics), + Box::pin(futures::stream::once(async { 0 })), + ) + .await? + }; // drop the mtex to avoid holding the lock for too long, result has been returned drop(mtex); diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 4d8d7506fc..1ffb0f11d9 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -15,6 +15,8 @@ use tokio::sync::Mutex; use tokio::time::sleep; use tokio::time::timeout; use windmill_common::client::AuthedClient; +use windmill_common::jobs::WorkerInternalServerInlineUtils; +use windmill_common::jobs::WORKER_INTERNAL_SERVER_INLINE_UTILS; use windmill_common::scripts::hash_to_codebase_id; use windmill_common::scripts::is_special_codebase_hash; use windmill_common::utils::report_critical_error; @@ -3364,6 +3366,57 @@ async fn handle_code_execution_job( .await?; let language = language.clone(); + run_language_executor( + job, + conn, + client, + parent_runnable_path, + job_dir, + worker_dir, + mem_peak, + canceled_by, + base_internal_url, + worker_name, + column_order, + new_args, + occupancy_metrics, + killpill_rx, + precomputed_agent_info, + has_stream, + language, + code, + envs, + codebase, + lock, + false, + ) + .await +} + +pub async fn run_language_executor( + job: &MiniPulledJob, + conn: &Connection, + client: &AuthedClient, + parent_runnable_path: Option, + job_dir: &str, + #[allow(unused_variables)] worker_dir: &str, + mem_peak: &mut i32, + canceled_by: &mut Option, + base_internal_url: &str, + worker_name: &str, + column_order: &mut Option>, + new_args: &mut Option>>, + occupancy_metrics: &mut OccupancyMetrics, + killpill_rx: &mut tokio::sync::broadcast::Receiver<()>, + precomputed_agent_info: Option, + has_stream: &mut bool, + language: Option, + code: &String, + envs: &Option>, + codebase: &Option, + lock: &Option, + run_inline: bool, +) -> error::Result> { if language == Some(ScriptLang::Postgresql) { return do_postgresql( job, @@ -3376,6 +3429,7 @@ async fn handle_code_execution_job( column_order, occupancy_metrics, parent_runnable_path, + run_inline, ) .await; } else if language == Some(ScriptLang::Mysql) { @@ -3385,19 +3439,26 @@ async fn handle_code_execution_job( )); #[cfg(feature = "mysql")] - return do_mysql( - job, - &client, - &code, - conn, - mem_peak, - canceled_by, - worker_name, - column_order, - occupancy_metrics, - parent_runnable_path, - ) - .await; + { + if run_inline { + return Err(Error::internal_err( + "Inline execution is not yet supported for this language".to_string(), + )); + } + return do_mysql( + job, + &client, + &code, + conn, + mem_peak, + canceled_by, + worker_name, + column_order, + occupancy_metrics, + parent_runnable_path, + ) + .await; + } } else if language == Some(ScriptLang::Bigquery) { #[cfg(not(feature = "enterprise"))] { @@ -3416,6 +3477,11 @@ async fn handle_code_execution_job( #[cfg(all(feature = "enterprise", feature = "bigquery"))] { + if run_inline { + return Err(Error::internal_err( + "Inline execution is not yet supported for this language".to_string(), + )); + } return do_bigquery( job, &client, @@ -3440,6 +3506,11 @@ async fn handle_code_execution_job( #[cfg(feature = "enterprise")] { + if run_inline { + return Err(Error::internal_err( + "Inline execution is not yet supported for this language".to_string(), + )); + } return do_snowflake( job, &client, @@ -3472,6 +3543,11 @@ async fn handle_code_execution_job( #[cfg(all(feature = "enterprise", feature = "mssql"))] { + if run_inline { + return Err(Error::internal_err( + "Inline execution is not yet supported for this language".to_string(), + )); + } return do_mssql( job, &client, @@ -3504,6 +3580,11 @@ async fn handle_code_execution_job( #[cfg(all(feature = "enterprise", feature = "oracledb"))] { + if run_inline { + return Err(Error::internal_err( + "Inline execution is not yet supported for this language".to_string(), + )); + } return do_oracledb( job, &client, @@ -3540,10 +3621,16 @@ async fn handle_code_execution_job( column_order, occupancy_metrics, parent_runnable_path, + run_inline, ) .await; } } else if language == Some(ScriptLang::Graphql) { + if run_inline { + return Err(Error::internal_err( + "Inline execution is not yet supported for this language".to_string(), + )); + } return do_graphql( job, &client, @@ -3556,6 +3643,11 @@ async fn handle_code_execution_job( ) .await; } else if language == Some(ScriptLang::Nativets) { + if run_inline { + return Err(Error::internal_err( + "Inline execution is not yet supported for this language".to_string(), + )); + } append_logs( &job.id, &job.workspace_id, @@ -3671,29 +3763,41 @@ mount {{ )); #[cfg(feature = "python")] - Box::pin(handle_python_job( - lock.as_ref(), - job_dir, - worker_dir, - worker_name, - job, - mem_peak, - canceled_by, - conn, - client, - parent_runnable_path, - &code, - &shared_mount, - base_internal_url, - envs, - new_args, - occupancy_metrics, - precomputed_agent_info, - has_stream, - )) - .await + { + if run_inline { + return Err(Error::internal_err( + "Inline execution is not yet supported for this language".to_string(), + )); + } + Box::pin(handle_python_job( + lock.as_ref(), + job_dir, + worker_dir, + worker_name, + job, + mem_peak, + canceled_by, + conn, + client, + parent_runnable_path, + &code, + &shared_mount, + base_internal_url, + envs, + new_args, + occupancy_metrics, + precomputed_agent_info, + has_stream, + )) + .await + } } ScriptLang::Deno => { + if run_inline { + return Err(Error::internal_err( + "Inline execution is not yet supported for this language".to_string(), + )); + } Box::pin(handle_deno_job( lock.as_ref(), mem_peak, @@ -3714,6 +3818,11 @@ mount {{ .await } ScriptLang::Bun | ScriptLang::Bunnative => { + if run_inline { + return Err(Error::internal_err( + "Inline execution is not yet supported for this language".to_string(), + )); + } let maybe_lock = resolve_maybe_lock( &lock, &code, @@ -3746,6 +3855,11 @@ mount {{ .await } ScriptLang::Go => { + if run_inline { + return Err(Error::internal_err( + "Inline execution is not yet supported for this language".to_string(), + )); + } let maybe_lock = resolve_maybe_lock( &lock, &code, @@ -3774,6 +3888,11 @@ mount {{ .await } ScriptLang::Bash => { + if run_inline { + return Err(Error::internal_err( + "Inline execution is not yet supported for this language".to_string(), + )); + } Box::pin(handle_bash_job( mem_peak, canceled_by, @@ -3793,6 +3912,11 @@ mount {{ .await } ScriptLang::Powershell => { + if run_inline { + return Err(Error::internal_err( + "Inline execution is not yet supported for this language".to_string(), + )); + } Box::pin(handle_powershell_job( mem_peak, canceled_by, @@ -3818,6 +3942,11 @@ mount {{ #[cfg(feature = "php")] { + if run_inline { + return Err(Error::internal_err( + "Inline execution is not yet supported for this language".to_string(), + )); + } let maybe_lock = resolve_maybe_lock( &lock, &code, @@ -3853,23 +3982,30 @@ mount {{ )); #[cfg(feature = "rust")] - Box::pin(handle_rust_job( - mem_peak, - canceled_by, - job, - conn, - client, - parent_runnable_path, - &code, - job_dir, - lock.as_ref(), - &shared_mount, - base_internal_url, - worker_name, - envs, - occupancy_metrics, - )) - .await + { + if run_inline { + return Err(Error::internal_err( + "Inline execution is not yet supported for this language".to_string(), + )); + } + Box::pin(handle_rust_job( + mem_peak, + canceled_by, + job, + conn, + client, + parent_runnable_path, + &code, + job_dir, + lock.as_ref(), + &shared_mount, + base_internal_url, + worker_name, + envs, + occupancy_metrics, + )) + .await + } } ScriptLang::Ansible => { #[cfg(not(feature = "python"))] @@ -3878,26 +4014,38 @@ mount {{ )); #[cfg(feature = "python")] - Box::pin(handle_ansible_job( - lock.as_ref(), - job_dir, - worker_dir, - worker_name, - job, - mem_peak, - canceled_by, - conn, - client, - parent_runnable_path, - &code, - &shared_mount, - base_internal_url, - envs, - occupancy_metrics, - )) - .await + { + if run_inline { + return Err(Error::internal_err( + "Inline execution is not yet supported for this language".to_string(), + )); + } + Box::pin(handle_ansible_job( + lock.as_ref(), + job_dir, + worker_dir, + worker_name, + job, + mem_peak, + canceled_by, + conn, + client, + parent_runnable_path, + &code, + &shared_mount, + base_internal_url, + envs, + occupancy_metrics, + )) + .await + } } ScriptLang::CSharp => { + if run_inline { + return Err(Error::internal_err( + "Inline execution is not yet supported for this language".to_string(), + )); + } Box::pin(handle_csharp_job( mem_peak, canceled_by, @@ -3923,23 +4071,30 @@ mount {{ ); #[cfg(feature = "nu")] - Box::pin(handle_nu_job(JobHandlerInputNu { - mem_peak, - canceled_by, - job, - conn, - client, - parent_runnable_path, - inner_content: &code, - job_dir, - requirements_o: lock.as_ref(), - shared_mount: &shared_mount, - base_internal_url, - worker_name, - envs, - occupancy_metrics, - })) - .await + { + if run_inline { + return Err(Error::internal_err( + "Inline execution is not yet supported for this language".to_string(), + )); + } + Box::pin(handle_nu_job(JobHandlerInputNu { + mem_peak, + canceled_by, + job, + conn, + client, + parent_runnable_path, + inner_content: &code, + job_dir, + requirements_o: lock.as_ref(), + shared_mount: &shared_mount, + base_internal_url, + worker_name, + envs, + occupancy_metrics, + })) + .await + } } ScriptLang::Java => { #[cfg(not(feature = "java"))] @@ -3949,23 +4104,30 @@ mount {{ .into()); #[cfg(feature = "java")] - Box::pin(handle_java_job(JobHandlerInputJava { - mem_peak, - canceled_by, - job, - conn, - client, - parent_runnable_path, - inner_content: &code, - job_dir, - requirements_o: lock.as_ref(), - shared_mount: &shared_mount, - base_internal_url, - worker_name, - envs, - occupancy_metrics, - })) - .await + { + if run_inline { + return Err(Error::internal_err( + "Inline execution is not yet supported for this language".to_string(), + )); + } + Box::pin(handle_java_job(JobHandlerInputJava { + mem_peak, + canceled_by, + job, + conn, + client, + parent_runnable_path, + inner_content: &code, + job_dir, + requirements_o: lock.as_ref(), + shared_mount: &shared_mount, + base_internal_url, + worker_name, + envs, + occupancy_metrics, + })) + .await + } } ScriptLang::Ruby => { #[cfg(not(feature = "ruby"))] @@ -3975,23 +4137,30 @@ mount {{ .into()); #[cfg(feature = "ruby")] - Box::pin(handle_ruby_job(JobHandlerInputRuby { - mem_peak, - canceled_by, - job, - conn, - client, - parent_runnable_path, - inner_content: &code, - job_dir, - requirements_o: lock.as_ref(), - shared_mount: &shared_mount, - base_internal_url, - worker_name, - envs, - occupancy_metrics, - })) - .await + { + if run_inline { + return Err(Error::internal_err( + "Inline execution is not yet supported for this language".to_string(), + )); + } + Box::pin(handle_ruby_job(JobHandlerInputRuby { + mem_peak, + canceled_by, + job, + conn, + client, + parent_runnable_path, + inner_content: &code, + job_dir, + requirements_o: lock.as_ref(), + shared_mount: &shared_mount, + base_internal_url, + worker_name, + envs, + occupancy_metrics, + })) + .await + } } // for related places search: ADD_NEW_LANG _ => panic!("unreachable, language is not supported: {language:#?}"), @@ -4076,3 +4245,104 @@ pub fn parse_sig_of_lang( None }) } + +pub fn init_worker_internal_server_inline_utils( + killpill_rx: tokio::sync::broadcast::Receiver<()>, + base_internal_url: String, +) -> windmill_common::error::Result<()> { + let utils = WorkerInternalServerInlineUtils { + base_internal_url, + killpill_rx: Arc::new(killpill_rx), + run_inline_preview_script: Arc::new(|params| { + let job = MiniPulledJob { + workspace_id: params.workspace_id, + id: Uuid::new_v4(), + args: params.args.map(Json), + parent_job: None, + created_by: params.created_by, + scheduled_for: chrono::Utc::now(), + started_at: None, + runnable_path: None, + kind: JobKind::Preview, + runnable_id: None, + canceled_reason: None, + canceled_by: None, + permissioned_as: params.permissioned_as, + permissioned_as_email: params.permissioned_as_email, + flow_status: None, + tag: "inline_preview".to_string(), + script_lang: Some(params.lang), + same_worker: true, + pre_run_error: None, + concurrent_limit: None, + concurrency_time_window_s: None, + flow_innermost_root_job: None, + root_job: None, + timeout: None, + flow_step_id: None, + cache_ttl: None, + cache_ignore_s3_path: None, + priority: None, + preprocessed: None, + script_entrypoint_override: None, + trigger: None, + trigger_kind: None, + visible_to_owner: false, + permissioned_as_end_user_email: None, + }; + Box::pin(async move { + let mut mem_peak: i32 = -1; + let mut canceled_by: Option = None; + let mut column_order: Option> = None; + let mut new_args: Option>> = None; + let mut occupancy_metrics = OccupancyMetrics::new(Instant::now()); + let mut has_stream: bool = false; + let mut killpill_rx = params.killpill_rx; + + run_language_executor( + &job, + ¶ms.conn, + ¶ms.client, + None, + ¶ms.job_dir, + ¶ms.worker_dir, + &mut mem_peak, + &mut canceled_by, + ¶ms.base_internal_url, + ¶ms.worker_name, + &mut column_order, + &mut new_args, + &mut occupancy_metrics, + &mut killpill_rx, + None, + &mut has_stream, + Some(params.lang), + ¶ms.content, + &None, + &None, + &None, + true, + ) + .await + }) + }), + }; + WORKER_INTERNAL_SERVER_INLINE_UTILS + .set(utils) + .map_err(|_| { + error::Error::InternalErr( + "Couldn't set WorkerInternalServerInlineUtils OnceCell".to_string(), + ) + })?; + Ok(()) +} + +pub fn get_worker_internal_server_inline_utils( +) -> windmill_common::error::Result<&'static WorkerInternalServerInlineUtils> { + match WORKER_INTERNAL_SERVER_INLINE_UTILS.get() { + Some(utils) => Ok(utils), + None => Err(error::Error::internal_err( + "worker inline functions are meant to be called from a worker's internal server", + )), + } +} diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 7b1af937fd..018107154f 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -55,7 +55,7 @@ "monaco-languageclient": "10.1.0", "monaco-vim": "^0.4.1", "ol": "^7.4.0", - "openai": "^6.9.0", + "openai": "^6.9.1", "openapi-types": "^12.1.3", "p-limit": "^6.1.0", "panzoom": "^9.4.3", @@ -65,6 +65,7 @@ "rehype-github-alerts": "^3.0.0", "rehype-raw": "^7.0.0", "rfc4648": "^1.5.3", + "runed": "^0.36.0", "svelte-carousel": "^1.0.25", "svelte-exmarkdown": "^5.0.0", "svelte-infinite-loading": "^1.4.0", @@ -92,8 +93,7 @@ "y-websocket": "^1.5.4", "yaml": "^2.8.0", "yjs": "^13.6.7", - "zod": "^3.24.2", - "zod-to-json-schema": "^3.24.5" + "zod": "^4.1.10" }, "devDependencies": { "@floating-ui/core": "^1.3.1", @@ -2432,7 +2432,7 @@ "version": "0.99.0", "resolved": "https://registry.npmjs.org/@oxc-project/runtime/-/runtime-0.99.0.tgz", "integrity": "sha512-8iE5/4OK0SLHqWzRxSvI1gjFPmIH6718s8iwkuco95rBZsCZIHq+5wy4lYsASxnH+8FOhbGndiUrcwsVG5i2zw==", - "dev": true, + "devOptional": true, "license": "MIT", "engines": { "node": "^20.19.0 || >=22.12.0" @@ -2442,7 +2442,7 @@ "version": "0.99.0", "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.99.0.tgz", "integrity": "sha512-LLDEhXB7g1m5J+woRSgfKsFPS3LhR9xRhTeIoEBm5WrkwMxn6eZ0Ld0c0K5eHB57ChZX6I3uSmmLjZ8pcjlRcw==", - "dev": true, + "devOptional": true, "license": "MIT", "funding": { "url": "https://github.com/sponsors/Boshen" @@ -2485,7 +2485,7 @@ "version": "1.0.0-next.29", "resolved": "https://registry.npmjs.org/@polka/url/-/url-1.0.0-next.29.tgz", "integrity": "sha512-wwQAWhWSuHaag8c4q/KN/vCoeOJYshAIvMQwD4GpSb3OiZklFfvAgmj0VCBBImRpuF/aFgIRzllXlVX93Jevww==", - "dev": true, + "devOptional": true, "license": "MIT" }, "node_modules/@popperjs/core": { @@ -2746,7 +2746,7 @@ "version": "1.0.0-beta.52", "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-beta.52.tgz", "integrity": "sha512-/L0htLJZbaZFL1g9OHOblTxbCYIGefErJjtYOwgl9ZqNx27P3L0SDfjhhHIss32gu5NWgnxuT2a2Hnnv6QGHKA==", - "dev": true, + "devOptional": true, "license": "MIT" }, "node_modules/@rollup/rollup-linux-x64-gnu": { @@ -2795,7 +2795,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.0.0.tgz", "integrity": "sha512-m2bOd0f2RT9k8QJx1JN85cZYyH1RqFBdlwtkSlf4tBDYLCiiZnv1fIIwacK6cqwXavOydf0NPToMQgpKq+dVlA==", - "dev": true, + "devOptional": true, "license": "MIT" }, "node_modules/@streamparser/json": { @@ -2836,7 +2836,7 @@ "version": "2.46.5", "resolved": "https://registry.npmjs.org/@sveltejs/kit/-/kit-2.46.5.tgz", "integrity": "sha512-7TSvMrCdmig5TMyYDW876C5FljhA0wlGixtvASCiqUqtLfmyEEpaysXjC7GhR5mWcGRrCGF+L2Bl1eEaW1wTCA==", - "dev": true, + "devOptional": true, "license": "MIT", "dependencies": { "@standard-schema/spec": "^1.0.0", @@ -2913,7 +2913,7 @@ "version": "6.2.1", "resolved": "https://registry.npmjs.org/@sveltejs/vite-plugin-svelte/-/vite-plugin-svelte-6.2.1.tgz", "integrity": "sha512-YZs/OSKOQAQCnJvM/P+F1URotNnYNeU3P2s4oIpzm1uFaqUEqRxUB0g5ejMjEb5Gjb9/PiBI5Ktrq4rUUF8UVQ==", - "dev": true, + "devOptional": true, "license": "MIT", "dependencies": { "@sveltejs/vite-plugin-svelte-inspector": "^5.0.0", @@ -2934,7 +2934,7 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/@sveltejs/vite-plugin-svelte-inspector/-/vite-plugin-svelte-inspector-5.0.1.tgz", "integrity": "sha512-ubWshlMk4bc8mkwWbg6vNvCeT7lGQojE3ijDh3QTR6Zr/R+GXxsGbyH4PExEPpiFmqPhYiVSVmHBjUcVc1JIrA==", - "dev": true, + "devOptional": true, "license": "MIT", "dependencies": { "debug": "^4.4.1" @@ -3050,7 +3050,7 @@ "version": "0.6.0", "resolved": "https://registry.npmjs.org/@types/cookie/-/cookie-0.6.0.tgz", "integrity": "sha512-4Kh9a6B2bQciAhf7FSuMRRkUWecJgJu9nPnx3yzpsfXX/c50REIqpHY4C82bXP90qrLtXtkDxTZosYO3UpOwlA==", - "dev": true, + "devOptional": true, "license": "MIT" }, "node_modules/@types/d3": { @@ -4740,7 +4740,7 @@ "version": "0.6.0", "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.6.0.tgz", "integrity": "sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw==", - "dev": true, + "devOptional": true, "license": "MIT", "engines": { "node": ">= 0.6" @@ -5301,7 +5301,7 @@ "version": "4.3.1", "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz", "integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==", - "dev": true, + "devOptional": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -5403,7 +5403,7 @@ "version": "5.3.2", "resolved": "https://registry.npmjs.org/devalue/-/devalue-5.3.2.tgz", "integrity": "sha512-UDsjUbpQn9kvm68slnrs+mfxwFkIflOhkanmyabZ8zOYk8SMEIbJ3TK+88g70hSIeytu4y18f0z/hYHMTrXIWw==", - "dev": true, + "devOptional": true, "license": "MIT" }, "node_modules/devlop": { @@ -7669,7 +7669,7 @@ "version": "4.1.5", "resolved": "https://registry.npmjs.org/kleur/-/kleur-4.1.5.tgz", "integrity": "sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==", - "dev": true, + "devOptional": true, "license": "MIT", "engines": { "node": ">=6" @@ -7889,7 +7889,7 @@ "version": "1.30.2", "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.30.2.tgz", "integrity": "sha512-utfs7Pr5uJyyvDETitgsaqSyjCb2qNRAtuqUeWIAKztsOYdcACf2KtARYXg2pSvhkt+9NfoaNY7fxjl6nuMjIQ==", - "dev": true, + "devOptional": true, "license": "MPL-2.0", "dependencies": { "detect-libc": "^2.0.3" @@ -8294,6 +8294,15 @@ "svelte": "^3 || ^4 || ^5.0.0-next.42" } }, + "node_modules/lz-string": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/lz-string/-/lz-string-1.5.0.tgz", + "integrity": "sha512-h5bgJWpxJNswbU7qCrV0tIKQCaS3blPDrqKWx+QxzuzL1zGUzij9XCWLrSLsJPu5t+eWA/ycetzYAO5IOMcWAQ==", + "license": "MIT", + "bin": { + "lz-string": "bin/bin.js" + } + }, "node_modules/magic-string": { "version": "0.30.19", "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.19.tgz", @@ -9457,7 +9466,7 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/mri/-/mri-1.2.0.tgz", "integrity": "sha512-tzzskb3bG8LvYGFF/mDTpq3jpI6Q9wc3LEmBaghu+DdCssd1FakN7Bc0hVNmEyGq1bq3RgfkCb3cmQLpNPOroA==", - "dev": true, + "devOptional": true, "license": "MIT", "engines": { "node": ">=4" @@ -9467,7 +9476,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/mrmime/-/mrmime-2.0.1.tgz", "integrity": "sha512-Y3wQdFg2Va6etvQ5I82yUhGdsKrcYox6p7FfL1LbK2J4V01F9TGlepTIhnK24t7koZibmg82KGglhA1XK5IsLQ==", - "dev": true, + "devOptional": true, "license": "MIT", "engines": { "node": ">=10" @@ -9803,9 +9812,9 @@ } }, "node_modules/openai": { - "version": "6.9.0", - "resolved": "https://registry.npmjs.org/openai/-/openai-6.9.0.tgz", - "integrity": "sha512-n2sJRYmM+xfJ0l3OfH8eNnIyv3nQY7L08gZQu3dw6wSdfPtKAk92L83M2NIP5SS8Cl/bsBBG3yKzEOjkx0O+7A==", + "version": "6.9.1", + "resolved": "https://registry.npmjs.org/openai/-/openai-6.9.1.tgz", + "integrity": "sha512-vQ5Rlt0ZgB3/BNmTa7bIijYFhz3YBceAA3Z4JuoMSBftBF9YqFHIEhZakSs+O/Ad7EaoEimZvHxD5ylRjN11Lg==", "license": "Apache-2.0", "bin": { "openai": "bin/cli" @@ -10131,7 +10140,7 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", - "dev": true, + "devOptional": true, "license": "ISC" }, "node_modules/picomatch": { @@ -10258,7 +10267,7 @@ "version": "8.5.6", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.6.tgz", "integrity": "sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg==", - "dev": true, + "devOptional": true, "funding": [ { "type": "opencollective", @@ -10970,7 +10979,7 @@ "version": "3.3.11", "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", "integrity": "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==", - "dev": true, + "devOptional": true, "funding": [ { "type": "github", @@ -11632,7 +11641,7 @@ "version": "1.0.0-beta.52", "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.0.0-beta.52.tgz", "integrity": "sha512-Hbnpljue+JhMJrlOjQ1ixp9me7sUec7OjFvS+A1Qm8k8Xyxmw3ZhxFu7LlSXW1s9AX3POE9W9o2oqCEeR5uDmg==", - "dev": true, + "devOptional": true, "license": "MIT", "dependencies": { "@oxc-project/types": "=0.99.0", @@ -11685,6 +11694,34 @@ "queue-microtask": "^1.2.2" } }, + "node_modules/runed": { + "version": "0.36.0", + "resolved": "https://registry.npmjs.org/runed/-/runed-0.36.0.tgz", + "integrity": "sha512-CK84KPwAausPQEyWF9t6miCuNW5isAKPMswDsz7jhdueiZZ9du/UrgWc/aggLts8QuppT8KucryrHDFBAqk9Ww==", + "funding": [ + "https://github.com/sponsors/huntabyte", + "https://github.com/sponsors/tglide" + ], + "license": "MIT", + "dependencies": { + "dequal": "^2.0.3", + "esm-env": "^1.0.0", + "lz-string": "^1.5.0" + }, + "peerDependencies": { + "@sveltejs/kit": "^2.21.0", + "svelte": "^5.7.0", + "zod": "^4.1.0" + }, + "peerDependenciesMeta": { + "@sveltejs/kit": { + "optional": true + }, + "zod": { + "optional": true + } + } + }, "node_modules/rw": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/rw/-/rw-1.3.3.tgz", @@ -11695,7 +11732,7 @@ "version": "1.8.1", "resolved": "https://registry.npmjs.org/sade/-/sade-1.8.1.tgz", "integrity": "sha512-xal3CZX1Xlo/k4ApwCFrHVACi9fBqJ7V+mwhBsuf/1IOKbBy098Fex+Wa/5QMubw09pSZ/u8EY8PWgevJsXp1A==", - "dev": true, + "devOptional": true, "license": "MIT", "dependencies": { "mri": "^1.1.0" @@ -11747,7 +11784,7 @@ "version": "2.7.1", "resolved": "https://registry.npmjs.org/set-cookie-parser/-/set-cookie-parser-2.7.1.tgz", "integrity": "sha512-IOc8uWeOZgnb3ptbCURJWNjWUPcO3ZnTTdzsurqERrP6nPyv+paC55vJM0LpOlT2ne+Ix+9+CRG1MNLlyZ4GjQ==", - "dev": true, + "devOptional": true, "license": "MIT" }, "node_modules/set-function-length": { @@ -11869,7 +11906,7 @@ "version": "3.0.2", "resolved": "https://registry.npmjs.org/sirv/-/sirv-3.0.2.tgz", "integrity": "sha512-2wcC/oGxHis/BoHkkPwldgiPSYcpZK3JU28WoMVv55yHJgcZ8rlXvuG9iZggz+sU1d4bRgIGASwyWqjxu3FM0g==", - "dev": true, + "devOptional": true, "license": "MIT", "dependencies": { "@polka/url": "^1.0.0-next.24", @@ -11951,7 +11988,7 @@ "version": "1.2.1", "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", - "dev": true, + "devOptional": true, "license": "BSD-3-Clause", "engines": { "node": ">=0.10.0" @@ -13019,7 +13056,7 @@ "version": "0.2.15", "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.15.tgz", "integrity": "sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==", - "dev": true, + "devOptional": true, "license": "MIT", "dependencies": { "fdir": "^6.5.0", @@ -13036,7 +13073,7 @@ "version": "6.5.0", "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", - "dev": true, + "devOptional": true, "license": "MIT", "engines": { "node": ">=12.0.0" @@ -13054,7 +13091,7 @@ "version": "4.0.3", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", - "dev": true, + "devOptional": true, "license": "MIT", "engines": { "node": ">=12" @@ -13080,7 +13117,7 @@ "version": "3.0.1", "resolved": "https://registry.npmjs.org/totalist/-/totalist-3.0.1.tgz", "integrity": "sha512-sf4i37nQ2LBx4m3wB74y+ubopq6W/dIzXg0FDGjsYnZHVa1Da8FH853wlL2gtUhg+xJXjfk3kUZS3BRoQeoQBQ==", - "dev": true, + "devOptional": true, "license": "MIT", "engines": { "node": ">=6" @@ -13474,7 +13511,7 @@ "version": "7.2.8", "resolved": "https://registry.npmjs.org/rolldown-vite/-/rolldown-vite-7.2.8.tgz", "integrity": "sha512-8wKihlF6EDF8grimwd7GPOhLkQkSIgj6Hlcp0CXhtO3HAXeUUqhgZmJmn07OF8e4PbTusMX6Yxmy1BptVRZsdw==", - "dev": true, + "devOptional": true, "license": "MIT", "dependencies": { "@oxc-project/runtime": "0.99.0", @@ -13568,7 +13605,7 @@ "version": "6.5.0", "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", - "dev": true, + "devOptional": true, "license": "MIT", "engines": { "node": ">=12.0.0" @@ -13586,7 +13623,7 @@ "version": "4.0.3", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", - "dev": true, + "devOptional": true, "license": "MIT", "engines": { "node": ">=12" @@ -13599,7 +13636,7 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/vitefu/-/vitefu-1.1.1.tgz", "integrity": "sha512-B/Fegf3i8zh0yFbpzZ21amWzHmuNlLlmJT6n7bu5e+pCHUKQIfXSYokrqOBGEMMe9UG2sostKQF9mml/vYaWJQ==", - "dev": true, + "devOptional": true, "license": "MIT", "workspaces": [ "tests/deps/*", @@ -14241,23 +14278,14 @@ "license": "MIT" }, "node_modules/zod": { - "version": "3.25.76", - "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", - "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.1.12.tgz", + "integrity": "sha512-JInaHOamG8pt5+Ey8kGmdcAcg3OL9reK8ltczgHTAwNhMys/6ThXHityHxVV2p3fkw/c+MAvBHFVYHFZDmjMCQ==", "license": "MIT", "funding": { "url": "https://github.com/sponsors/colinhacks" } }, - "node_modules/zod-to-json-schema": { - "version": "3.24.6", - "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.24.6.tgz", - "integrity": "sha512-h/z3PKvcTcTetyjl1fkj79MHNEjm+HpD6NXheWjzOekY7kV+lwDYnHw+ivHkijnCSMz1yJaWBD9vu/Fcmk+vEg==", - "license": "ISC", - "peerDependencies": { - "zod": "^3.24.1" - } - }, "node_modules/zstddec": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/zstddec/-/zstddec-0.1.0.tgz", diff --git a/frontend/package.json b/frontend/package.json index a6d0487146..54672c3367 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -122,7 +122,7 @@ "monaco-languageclient": "10.1.0", "monaco-vim": "^0.4.1", "ol": "^7.4.0", - "openai": "^6.9.0", + "openai": "^6.9.1", "openapi-types": "^12.1.3", "p-limit": "^6.1.0", "panzoom": "^9.4.3", @@ -132,6 +132,7 @@ "rehype-github-alerts": "^3.0.0", "rehype-raw": "^7.0.0", "rfc4648": "^1.5.3", + "runed": "^0.36.0", "svelte-carousel": "^1.0.25", "svelte-exmarkdown": "^5.0.0", "svelte-infinite-loading": "^1.4.0", @@ -159,8 +160,7 @@ "y-websocket": "^1.5.4", "yaml": "^2.8.0", "yjs": "^13.6.7", - "zod": "^3.24.2", - "zod-to-json-schema": "^3.24.5" + "zod": "^4.1.10" }, "peerDependencies": { "svelte": "^5.0.0" diff --git a/frontend/src/lib/components/AIAgentLogViewer.svelte b/frontend/src/lib/components/AIAgentLogViewer.svelte index 1671fe7b77..7833975ed4 100644 --- a/frontend/src/lib/components/AIAgentLogViewer.svelte +++ b/frontend/src/lib/components/AIAgentLogViewer.svelte @@ -35,7 +35,7 @@ call_id: z.string(), function_name: z.string(), resource_path: z.string(), - arguments: z.record(z.unknown()).optional() + arguments: z.record(z.any(), z.any()).optional() }), z.object({ type: z.literal('message') diff --git a/frontend/src/lib/components/DBManagerDrawer.svelte b/frontend/src/lib/components/DBManagerDrawer.svelte index 39b1f331c8..11ab468c9b 100644 --- a/frontend/src/lib/components/DBManagerDrawer.svelte +++ b/frontend/src/lib/components/DBManagerDrawer.svelte @@ -14,6 +14,7 @@ import { dbDeleteTableActionWithPreviewScript, dbTableOpsWithPreviewScripts, + getDatabaseArg, getDucklakeSchema } from './dbOps' import { makeCreateTableQuery } from './apps/components/display/dbtable/queries/createTable' @@ -204,8 +205,7 @@ dbType, previewSql: (values) => makeCreateTableQuery(values, dbType, selectedSchemaKey), async onConfirm(values) { - const dbArg = - input?.type === 'database' ? { database: '$res:' + input.resourcePath } : {} + const dbArg = getDatabaseArg(input) const language = getLanguageByResourceType(dbType) let query = makeCreateTableQuery(values, dbType, selectedSchemaKey) if (input?.type === 'ducklake') query = wrapDucklakeQuery(query, input.ducklake) diff --git a/frontend/src/lib/components/Editor.svelte b/frontend/src/lib/components/Editor.svelte index 9eb37b15ff..92718fe175 100644 --- a/frontend/src/lib/components/Editor.svelte +++ b/frontend/src/lib/components/Editor.svelte @@ -40,7 +40,13 @@ import { editorConfig, updateOptions } from '$lib/editorUtils' import { createHash as randomHash } from '$lib/editorLangUtils' import { workspaceStore } from '$lib/stores' - import { type Preview, ResourceService, type ScriptLang, UserService } from '$lib/gen' + import { + type Preview, + ResourceService, + type ScriptLang, + UserService, + WorkspaceService + } from '$lib/gen' import type { Text } from 'yjs' import { initializeVscode, @@ -95,6 +101,7 @@ import { setMonacoTypescriptOptions } from './monacoLanguagesOptions' import { copilotInfo } from '$lib/aiStore' import { getDbSchemas } from './apps/components/display/dbtable/metadata' + import { resource, watch } from 'runed' // import EditorTheme from './EditorTheme.svelte' let divEl: HTMLDivElement | null = $state(null) @@ -1457,6 +1464,42 @@ ata?.(deps) } + let customTsTypesData = resource([() => lang], async () => { + if (lang !== 'typescript') return undefined + let datatables = await WorkspaceService.listDataTables({ workspace: $workspaceStore ?? '' }) + let ducklakes = await WorkspaceService.listDucklakes({ workspace: $workspaceStore ?? '' }) + return { datatables, ducklakes } + }) + function setTypescriptCustomTypes() { + if (!customTsTypesData.current) return + if (lang !== 'typescript') return + + const ducklakeNames = customTsTypesData.current.ducklakes + const datatableNames = customTsTypesData.current.datatables + + const ducklakeNameType = ducklakeNames.length + ? ducklakeNames.map((name) => JSON.stringify(name)).join(' | ') + : 'string' + const datatableNameType = datatableNames.length + ? datatableNames.map((name) => JSON.stringify(name)).join(' | ') + : 'string' + const isDucklakeOptional = ducklakeNames.includes('main') + const isDataTableOptional = datatableNames.includes('main') + + let disposeTs = languages.typescript.typescriptDefaults.addExtraLib( + `export {}; + declare module 'windmill-client' { + import { type SqlTemplateFunction } from 'windmill-client'; + export function ducklake(name${isDucklakeOptional ? '?' : ''}: ${ducklakeNameType}): SqlTemplateFunction; + export function datatable(name${isDataTableOptional ? '?' : ''}: ${datatableNameType}): SqlTemplateFunction; + }`, + 'file:///custom_wmill_types.d.ts' + ) + return () => { + disposeTs.dispose() + } + } + async function setTypescriptRTNamespace() { if ( scriptLang && @@ -1483,6 +1526,7 @@ const uri = mUri.parse('file:///extraLib.d.ts') languages.typescript.typescriptDefaults.addExtraLib(extraLib, uri.toString()) } + if ( lang === 'typescript' && (scriptLang == 'bun' || scriptLang == 'tsx' || scriptLang == 'bunnative') && @@ -1763,6 +1807,8 @@ lineNumbers: $relativeLineNumbers ? 'relative' : 'on' }) }) + + watch([() => customTsTypesData.current], setTypescriptCustomTypes) diff --git a/frontend/src/lib/components/EditorBar.svelte b/frontend/src/lib/components/EditorBar.svelte index 315bfeba75..9718e3f652 100644 --- a/frontend/src/lib/components/EditorBar.svelte +++ b/frontend/src/lib/components/EditorBar.svelte @@ -1,5 +1,5 @@ @@ -30,20 +34,39 @@ {#if asset.kind === 'resource' && resourceDataCache[asset.path] !== undefined} + {:else if datatableNotFound} + + {:else if asset.kind === 'resource' && resourceDataCache[asset.path] === undefined} + + {/if} + + + {:else if assetCanBeExplored(asset, { resource_type: resourceDataCache[asset.path] })} + WorkspaceService.listDataTables({ workspace: $workspaceStore ?? '' }) + ) + let ducklakes = resource([], () => + WorkspaceService.listDucklakes({ workspace: $workspaceStore ?? '' }) + ) + $effect(() => { assets untrack(() => { @@ -97,7 +105,7 @@ class={twMerge( size === '3xs' ? 'h-[1.6rem]' : 'py-1.5', 'text-xs flex items-center gap-1.5 px-2 rounded-md relative', - 'border border-tertiary/30', + 'border', 'bg-surface hover:bg-surface-hover active:bg-surface', 'transition-all hover:text-primary cursor-pointer' )} @@ -117,6 +125,14 @@
    {#each assets as asset} + {@const ducklakeNotFound = + asset.kind === 'ducklake' && + ducklakes.current && + !ducklakes.current.find((name) => name === asset.path)} + {@const datatableNotFound = + asset.kind === 'datatable' && + datatables.current && + !datatables.current.find((name) => name === asset.path)}
  • onHoverLi?.(asset, 'enter')} @@ -187,6 +203,8 @@ {dbManagerDrawer} {resourceEditorDrawer} {s3FilePicker} + {ducklakeNotFound} + {datatableNotFound} />
    • {/each} diff --git a/frontend/src/lib/components/assets/lib.ts b/frontend/src/lib/components/assets/lib.ts index f002beed15..0d96e67c39 100644 --- a/frontend/src/lib/components/assets/lib.ts +++ b/frontend/src/lib/components/assets/lib.ts @@ -22,6 +22,8 @@ export function formatAsset(asset: Asset): string { return `s3://${asset.path}` case 'ducklake': return `ducklake://${asset.path}` + case 'datatable': + return `datatable://${asset.path}` } } @@ -72,6 +74,8 @@ export function formatAssetKind(asset: { return 'S3 Object' case 'ducklake': return 'Ducklake' + case 'datatable': + return 'Data table' } } diff --git a/frontend/src/lib/components/common/confirmationModal/asyncConfirmationModal.svelte.ts b/frontend/src/lib/components/common/confirmationModal/asyncConfirmationModal.svelte.ts index 2790784ea9..508f111426 100644 --- a/frontend/src/lib/components/common/confirmationModal/asyncConfirmationModal.svelte.ts +++ b/frontend/src/lib/components/common/confirmationModal/asyncConfirmationModal.svelte.ts @@ -2,6 +2,11 @@ import { createRawSnippet, type ComponentProps } from 'svelte' import type ConfirmationModal from './ConfirmationModal.svelte' import { sendUserToast } from '$lib/toast' +export type ConfirmationModalHandle = { + props: ComponentProps + ask: (props: Params) => Promise +} + /** * This allows asking for confirmation while maintaining a linear imperative flow, * and avoiding unnecessary states and callback hopping @@ -18,10 +23,7 @@ import { sendUserToast } from '$lib/toast' * // ... * */ -export function createAsyncConfirmationModal(): { - props: ComponentProps - ask: (props: Params) => Promise -} { +export function createAsyncConfirmationModal(): ConfirmationModalHandle { // Create a new instance of ConfirmationModal with the provided props const o: ReturnType = $state({ props: { diff --git a/frontend/src/lib/components/common/modal/Modal2.svelte b/frontend/src/lib/components/common/modal/Modal2.svelte index cecadb18b3..bb388452fd 100644 --- a/frontend/src/lib/components/common/modal/Modal2.svelte +++ b/frontend/src/lib/components/common/modal/Modal2.svelte @@ -12,16 +12,25 @@ export let css: any = {} export let target: string = '' export let isOpen = false - export let fixedSize: 'xs' | 'sm' | 'md' | 'lg' | 'xl' | 'xxl' = 'md' + export let fixedWidth: 'xs' | 'sm' | 'md' | 'lg' | 'xl' | 'xxl' = 'md' + export let fixedHeight: 'xs' | 'sm' | 'md' | 'lg' | 'xl' | 'xxl' = 'md' + export let contentClasses: string = '' - // Add size mapping with custom pixel values - const sizeStyles = { - xs: { width: '400px', height: '250px' }, - sm: { width: '600px', height: '400px' }, - md: { width: '800px', height: '500px' }, - lg: { width: '1400px', height: '720px' }, - xl: { width: '1600px', height: '800px' }, - xxl: { width: '1600px', height: '1000px' } + const widthMap = { + xs: '400px', + sm: '600px', + md: '800px', + lg: '1400px', + xl: '1600px', + xxl: '1600px' + } + const heightMap = { + xs: '250px', + sm: '400px', + md: '500px', + lg: '720px', + xl: '800px', + xxl: '1000px' } export function close() { @@ -55,7 +64,7 @@ >
    -
    {}}> +
    {}} + >
    diff --git a/frontend/src/lib/components/copilot/chat/shared.ts b/frontend/src/lib/components/copilot/chat/shared.ts index 0277794632..ab473fc0a5 100644 --- a/frontend/src/lib/components/copilot/chat/shared.ts +++ b/frontend/src/lib/components/copilot/chat/shared.ts @@ -8,7 +8,6 @@ import type { CodePieceElement, ContextElement, FlowModuleCodePieceElement } fro import { workspaceStore } from '$lib/stores' import type { ExtendedOpenFlow } from '$lib/components/flows/types' import type { FunctionParameters } from 'openai/resources/shared.mjs' -import { zodToJsonSchema } from 'zod-to-json-schema' import { z } from 'zod' import { ScriptService, JobService, type CompletedJob, type FlowModule } from '$lib/gen' import { scriptLangToEditorLang } from '$lib/scripts' @@ -412,15 +411,9 @@ export function createToolDef( name: string, description: string ): ChatCompletionFunctionTool { - const schema = zodToJsonSchema(zodSchema, { - name, - target: 'openAi' - }) - let parameters = schema.definitions![name] as FunctionParameters - parameters = { - ...parameters, - required: parameters.required ?? [] - } + let parameters = z.toJSONSchema(zodSchema) + delete parameters.$schema + if (!parameters.required) parameters.required = [] return { type: 'function', diff --git a/frontend/src/lib/components/custom_ui.ts b/frontend/src/lib/components/custom_ui.ts index 731e14c692..d21b7ae9b6 100644 --- a/frontend/src/lib/components/custom_ui.ts +++ b/frontend/src/lib/components/custom_ui.ts @@ -77,6 +77,7 @@ export type EditorBarUi = { s3object?: boolean database?: boolean ducklake?: boolean + dataTable?: boolean } export type EditableSchemaFormUi = { diff --git a/frontend/src/lib/components/dbOps.ts b/frontend/src/lib/components/dbOps.ts index d002a75cc6..404d8bbf73 100644 --- a/frontend/src/lib/components/dbOps.ts +++ b/frontend/src/lib/components/dbOps.ts @@ -11,6 +11,7 @@ import type { DBSchema, SQLSchema } from '$lib/stores' import { stringifySchema } from './copilot/lib' import type { DbInput, DbType } from './dbTypes' import { wrapDucklakeQuery } from './ducklake' +import { assert } from '$lib/utils' export type IDbTableOps = { dbType: DbType @@ -47,7 +48,7 @@ export function dbTableOpsWithPreviewScripts({ }): IDbTableOps { const dbType = getDbType(input) const language = getLanguageByResourceType(dbType) - const dbArg = input?.type === 'database' ? { database: '$res:' + input.resourcePath } : {} + const dbArg = getDatabaseArg(input) return { dbType, tableKey, @@ -128,7 +129,7 @@ export function dbDeleteTableActionWithPreviewScript({ workspace: string input: DbInput }): DbTableActionFactory { - const dbArg = input?.type === 'database' ? { database: '$res:' + input.resourcePath } : {} + const dbArg = getDatabaseArg(input) return ({ tableKey, refresh }) => ({ confirmTitle: `Are you sure you want to delete '${tableKey}' ? This action is irreversible`, @@ -169,9 +170,12 @@ export async function getDucklakeSchema({ args: {} } }) - const mainSchema = Array.isArray(result) && result.length && (result?.[0]?.['result'] ?? '[]') + let mainSchema = Array.isArray(result) && result.length && (result?.[0]?.['result'] ?? []) + // Safety for agent workers (duckdb ffi lib used to return JSON as stringified json) + if (typeof mainSchema === 'string') mainSchema = JSON.parse(mainSchema) if (!mainSchema) throw new Error('Failed to get Ducklake schema: ' + JSON.stringify(result)) + assert('mainSchema is an object', typeof mainSchema === 'object') let schema: Omit = { schema: { main: mainSchema }, publicOnly: true, @@ -205,3 +209,14 @@ export function getDbType(input: DbInput): DbType { return 'duckdb' } } + +export function getDatabaseArg(input: DbInput | undefined) { + if (input?.type === 'database') { + if (input.resourcePath.startsWith('datatable://')) { + return { database: 'datatable://' + input.resourcePath } + } else { + return { database: '$res:' + input.resourcePath } + } + } + return {} +} diff --git a/frontend/src/lib/components/icons/AssetDatabaseIcon.svelte b/frontend/src/lib/components/icons/AssetDatabaseIcon.svelte new file mode 100644 index 0000000000..b4d019e4b6 --- /dev/null +++ b/frontend/src/lib/components/icons/AssetDatabaseIcon.svelte @@ -0,0 +1,30 @@ + + + + + + diff --git a/frontend/src/lib/components/icons/AssetGenericIcon.svelte b/frontend/src/lib/components/icons/AssetGenericIcon.svelte index 16263f6344..2f0cb28a5c 100644 --- a/frontend/src/lib/components/icons/AssetGenericIcon.svelte +++ b/frontend/src/lib/components/icons/AssetGenericIcon.svelte @@ -4,6 +4,7 @@ import AssetResIcon from './AssetResIcon.svelte' import AssetS3Icon from './AssetS3Icon.svelte' import AssetDucklakeIcon from './AssetDucklakeIcon.svelte' + import AssetDatabaseIcon from './AssetDatabaseIcon.svelte' interface Props { size?: string @@ -19,6 +20,8 @@ {:else if assetKind == 'resource'} +{:else if assetKind == 'datatable'} + {:else if assetKind == 'ducklake'} {:else} diff --git a/frontend/src/lib/components/sidebar/CriticalAlertModal.svelte b/frontend/src/lib/components/sidebar/CriticalAlertModal.svelte index 948381f76c..6aa2a013fb 100644 --- a/frontend/src/lib/components/sidebar/CriticalAlertModal.svelte +++ b/frontend/src/lib/components/sidebar/CriticalAlertModal.svelte @@ -171,7 +171,13 @@ } - + diff --git a/frontend/src/lib/components/wizards/LoggedWizardResult.svelte b/frontend/src/lib/components/wizards/LoggedWizardResult.svelte index be82f0b82e..e924ae4ea7 100644 --- a/frontend/src/lib/components/wizards/LoggedWizardResult.svelte +++ b/frontend/src/lib/components/wizards/LoggedWizardResult.svelte @@ -50,7 +50,7 @@
    { if (step.description) { @@ -60,7 +60,7 @@ }} >
    - + {#if step.status == 'SKIP'} {:else if step.status == 'FAIL'} diff --git a/frontend/src/lib/components/workspaceSettings/CustomInstanceDbSelect.svelte b/frontend/src/lib/components/workspaceSettings/CustomInstanceDbSelect.svelte new file mode 100644 index 0000000000..fe7ecefb09 --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/CustomInstanceDbSelect.svelte @@ -0,0 +1,99 @@ + + +
    + + + +
    + + (openedDbNameWizard ? { dbname: value ?? '', status: status! } : undefined), + (v) => !v && (openedDbNameWizard = false) + } +/> diff --git a/frontend/src/lib/components/workspaceSettings/CustomInstanceDbWizardModal.svelte b/frontend/src/lib/components/workspaceSettings/CustomInstanceDbWizardModal.svelte new file mode 100644 index 0000000000..9a8a2d443f --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/CustomInstanceDbWizardModal.svelte @@ -0,0 +1,199 @@ + + + !!opened, (v) => !v && !preventClose && (opened = undefined)} + target="#content" + title={'Custom Instance Database Setup'} + contentClasses="flex flex-col" + fixedWidth="md" + fixedHeight="md" +> + {#if opened} + {@const status = opened?.status} + {@const dbname = opened?.dbname} + {@const enableManageButton = + status?.logs.created_database === 'OK' || status?.logs.created_database === 'SKIP'} +
    +
    +
    + {dbname} + + Custom instance databases are databases created in the Windmill PostgreSQL instance. + Their credentials are automatically managed by Windmill and are never exposed to users. + Only super admins can create them. + +
    +
    + {#if bottomHint} +
    + {@render bottomHint()} +
    + {/if} + (opened = undefined)} + /> +
    +
    +
    +
    + {#if status?.error} +
    + + {status.error} + +
    + {/if} + + +
    + +
    +
    + {/if} +     diff --git a/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte b/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte new file mode 100644 index 0000000000..51e95bc7eb --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte @@ -0,0 +1,288 @@ + + + + +
    +
    +
    Data tables
    + + Store relational data out of the box. Interact with a fully managed PostgreSQL database + directly from the Windmill SDK. + +
    +
    + + + + + {#each tableHeadNames as name, i} + + {name} + {#if tableHeadTooltips[name]} + + {@html tableHeadTooltips[name]} + + {/if} + + {/each} + + + + {#if tempSettings.dataTables.length == 0} + + + No data table in this workspace yet + + + {/if} + {#each tempSettings.dataTables as dataTable, dataTableIndex} + + + + + +
    +
    + {#if dataTable.database.resource_type === 'instance'} + + Use Windmill's PostgreSQL instance + + {/if} + (ducklake.catalog.resource_path = i)} - placeholder="PostgreSQL database name" - items={safeSelectItems(Object.keys(instanceCatalogStatuses.value ?? {}))} - disabled={!isInstanceCatalogEnabled} - /> - - - - - - - {@render instanceCatalogWizard(status, ducklake.catalog.resource_path ?? '')} - - + {#snippet wizardBottomHint()} + Note: this is different from the Manage Ducklake button. This will show you the + content of the PostgreSQL database used as a catalog, while the other button + shows you the content of the ducklake (the parquet files). + {/snippet} + {/if}
    @@ -399,142 +367,3 @@ - -{#snippet instanceCatalogWizard( - status: DucklakeInstanceCatalogDbStatus | undefined, - dbname: string -)} - {@const showManageCatalogButton = - status?.logs.created_database === 'OK' || status?.logs.created_database === 'SKIP'} - {#if !status} -
    - {dbname} needs to be configured in the Windmill postgres instance -
    - {/if} - - {#if status?.error} -
    - - {status.error} - -
    - {/if} - - - {#if showManageCatalogButton} -
    - Note: the 'Manage catalog' button below is different from the Manage Ducklake button. This - will show you the content of the PostgreSQL database used as a catalog, while the other button - shows you the actual content of the ducklake (the parquet files). -
    - {/if} -
    - - {#if showManageCatalogButton} - instanceCatalogPopover?.close()} - /> - {/if} -
    -{/snippet} diff --git a/frontend/src/lib/components/workspaceSettings/utils.svelte.ts b/frontend/src/lib/components/workspaceSettings/utils.svelte.ts new file mode 100644 index 0000000000..3d5d70afec --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/utils.svelte.ts @@ -0,0 +1,9 @@ +import { isCloudHosted } from '$lib/cloud' + +import { superadmin } from '$lib/stores' +import { derived } from 'svelte/store' + +export let isCustomInstanceDbEnabled = derived( + [superadmin], + ([superadmin_]) => superadmin_ && !isCloudHosted() +) diff --git a/frontend/src/lib/svelte5Utils.svelte.ts b/frontend/src/lib/svelte5Utils.svelte.ts index adb09a4809..90de70ba6d 100644 --- a/frontend/src/lib/svelte5Utils.svelte.ts +++ b/frontend/src/lib/svelte5Utils.svelte.ts @@ -2,7 +2,7 @@ import { untrack } from 'svelte' import { deepEqual } from 'fast-equals' -import type { StateStore } from './utils' +import { type StateStore } from './utils' export function withProps(component: Component, props: Props) { const ret = $state({ @@ -39,6 +39,9 @@ export type UsePromiseOptions = { clearValueOnRefresh?: boolean } +/** + * @deprecated Use `resource` from `runed` instead + */ export function usePromise( createPromise: () => Promise, { loadInit = true, clearValueOnRefresh = true }: UsePromiseOptions = {} diff --git a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte index 9294ddbf63..465cd78183 100644 --- a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte @@ -59,6 +59,10 @@ import UnsavedConfirmationModal from '$lib/components/common/confirmationModal/UnsavedConfirmationModal.svelte' import TextInput from '$lib/components/text_input/TextInput.svelte' import CollapseLink from '$lib/components/CollapseLink.svelte' + import DataTableSettings, { + convertDataTableSettingsFromBackend, + type DataTableSettingsType + } from '$lib/components/workspaceSettings/DataTableSettings.svelte' import WorkspaceDependenciesSettings from '$lib/components/workspaceSettings/WorkspaceDependenciesSettings.svelte' let slackInitialPath: string = $state('') @@ -111,9 +115,9 @@ secondaryStorage: undefined }) - let ducklakeSettings: DucklakeSettingsType = $state({ - ducklakes: [] - }) + let dataTableSettings: DataTableSettingsType = $state({ dataTables: [] }) + + let ducklakeSettings: DucklakeSettingsType = $state({ ducklakes: [] }) let ducklakeSavedSettings: DucklakeSettingsType = $state(untrack(() => ducklakeSettings)) let workspaceDefaultAppPath: string | undefined = $state(undefined) @@ -132,6 +136,7 @@ | 'deploy_to' | 'error_handler' | 'ai' + | 'windmill_data_tables' | 'windmill_lfs' | 'git_sync' | 'default_app' @@ -307,6 +312,7 @@ !!$enterpriseLicense ) initialS3ResourceSettings = clone(s3ResourceSettings) + dataTableSettings = convertDataTableSettingsFromBackend(settings.datatable) ducklakeSettings = convertDucklakeSettingsFromBackend(settings.ducklake) ducklakeSavedSettings = clone(ducklakeSettings) @@ -341,7 +347,9 @@ if (!$workspaceStore) return try { - const config = await WorkspaceService.getWorkspaceSlackOauthConfig({ workspace: $workspaceStore }) + const config = await WorkspaceService.getWorkspaceSlackOauthConfig({ + workspace: $workspaceStore + }) useCustomSlackApp = !!config.slack_oauth_client_id slackOAuthClientId = config.slack_oauth_client_id || '' slackOAuthClientSecret = config.slack_oauth_client_secret || '' @@ -664,6 +672,13 @@ aiDescription="Windmill AI workspace settings" label="Windmill AI" /> + {#snippet workspaceConfig()} -
    - {#if slackOAuthConfigLoaded} - -
    -
    Workspace specific Slack app
    -
    -
    -
    - Client ID: - {slackOAuthClientId} -
    - +
    +
    + Client ID: + {slackOAuthClientId} +
    + +
    +
    -
    + {:else} + + + + {#if useCustomSlackApp} +
    + + + + + +
    + Create a Slack app at{' '} + + Slack API + . Set the redirect URI to:{' '} + + {window.location.origin}{base}/oauth/callback_slack + +
    +     + +
    + +
    +
    + {/if} + {/if}
    - {:else} - - - - {#if useCustomSlackApp} -
    - - - - - -
    - Create a Slack app at{' '} - - Slack API - . Set the redirect URI to:{' '} - - {window.location.origin}{base}/oauth/callback_slack - -
    -     - -
    - -
    -
    - {/if} - {/if} -
    - {/snippet} - - {:else if slack_tabs === 'teams_commands'} + {/snippet} + + {:else if slack_tabs === 'teams_commands'} {#if !$enterpriseLicense}
    @@ -1111,6 +1126,8 @@ initialMaxTokensPerModel = clone(maxTokensPerModel) }} /> + {:else if tab == 'windmill_data_tables'} + {:else if tab == 'windmill_lfs'} Any: + """Run a script on the current worker without creating a job""" + endpoint = f"/w/{self.workspace}/jobs/run_inline/preview" + body = { + "content": content, + "language": language, + "args": args or {}, + } + return self.post(endpoint, json=body).text + def wait_job( self, job_id, @@ -1000,6 +1015,13 @@ class Windmill: }, ) + def datatable(self, name: str = "main"): + return DataTableClient(self, name) + + def ducklake(self, name: str = "main"): + return DucklakeClient(self, name) + + def init_global_client(f): @functools.wraps(f) @@ -1527,6 +1549,18 @@ def run_script_by_hash( timeout=timeout, ) +@init_global_client +def run_inline_script_preview( + content: str, + language: str, + args: dict = None, +) -> Any: + """Run a script on the current worker without creating a job""" + return _client.run_inline_script_preview( + content=content, + language=language, + args=args, + ) @init_global_client def username_to_email(username: str) -> str: @@ -1538,6 +1572,14 @@ def username_to_email(username: str) -> str: return _client.username_to_email(username) +@init_global_client +def datatable(name: str = "main") -> DataTableClient: + return _client.datatable(name) + +@init_global_client +def ducklake(name: str = "main") -> DucklakeClient: + return _client.ducklake(name) + def task(*args, **kwargs): from inspect import signature @@ -1635,3 +1677,78 @@ def stream_result(stream) -> None: """ for text in stream: append_to_result_stream(text) + +class DataTableClient: + def __init__(self, client: Windmill, name: str): + self.client = client + self.name = name + def query(self, sql: str, *args): + args_dict = {} + args_def = "" + for i, arg in enumerate(args): + args_dict[f"arg{i+1}"] = arg + args_def += f"-- ${i+1} arg{i+1}\n" + sql = args_def + sql + return SqlQuery( + sql, + lambda sql: self.client.run_inline_script_preview( + content=sql, + language="postgresql", + args={"database": f"datatable://{self.name}", **args_dict}, + ) + ) + +class DucklakeClient: + def __init__(self, client: Windmill, name: str): + self.client = client + self.name = name + def query(self, sql: str, **kwargs): + args_dict = {} + args_def = "" + for key, value in kwargs.items(): + args_dict[key] = value + args_def += f"-- ${key} ({infer_sql_type(value)})\n" + attach = f"ATTACH 'ducklake://{self.name}' AS dl;USE dl;\n" + sql = args_def + attach + sql + return SqlQuery( + sql, + lambda sql: self.client.run_inline_script_preview( + content=sql, + language="duckdb", + args=args_dict, + ) + ) + +class SqlQuery: + def __init__(self, sql: str, fetch_fn): + self.sql = sql + self.fetch_fn = fetch_fn + def fetch(self, result_collection: str | None = None): + sql = self.sql + if result_collection is not None: + sql = f'-- result_collection={result_collection}\n{sql}' + return self.fetch_fn(sql) + def fetch_one(self): + return self.fetch(result_collection="last_statement_first_row") + +def infer_sql_type(value) -> str: + """ + DuckDB executor requires explicit argument types at declaration + These types exist in both DuckDB and Postgres + Check that the types exist if you plan to extend this function for other SQL engines. + """ + if isinstance(value, bool): + # Check bool before int since bool is a subclass of int in Python + return "BOOLEAN" + elif isinstance(value, int): + return "BIGINT" + elif isinstance(value, float): + return "DOUBLE PRECISION" + elif value is None: + return "TEXT" + elif isinstance(value, str): + return "TEXT" + elif isinstance(value, dict) or isinstance(value, list): + return "JSON" + else: + return "TEXT" diff --git a/typescript-client/build.jsr.sh b/typescript-client/build.jsr.sh index 26fcd52ffe..4937c2b069 100755 --- a/typescript-client/build.jsr.sh +++ b/typescript-client/build.jsr.sh @@ -11,9 +11,10 @@ sed -i 's/get \[Symbol\.toStringTag\]() {/get \[Symbol\.toStringTag\]() : string cp "${script_dirpath}/client.ts" "${script_dirpath}/src/" cp "${script_dirpath}/s3Types.ts" "${script_dirpath}/src/" +cp "${script_dirpath}/sqlUtils.ts" "${script_dirpath}/src/" echo "" >> "${script_dirpath}/src/index.ts" echo 'export type { S3Object, DenoS3LightClientSettings } from "./s3Types";' >> "${script_dirpath}/src/index.ts" echo "" >> "${script_dirpath}/src/index.ts" -echo 'export { type Base64, setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, setProgress, getProgress, getState, getIdToken, denoS3LightClientSettings, loadS3FileStream, loadS3File, writeS3File, signS3Objects, signS3Object, task, runScript, runScriptAsync, runScriptByPath, runScriptByHash, runScriptByPathAsync, runScriptByHashAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, Sql, requestInteractiveTeamsApproval, appendToResultStream, streamResult } from "./client";' >> "${script_dirpath}/src/index.ts" +echo 'export { type Base64, setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, setProgress, getProgress, getState, getIdToken, denoS3LightClientSettings, loadS3FileStream, loadS3File, writeS3File, signS3Objects, signS3Object, task, runScript, runScriptAsync, runScriptByPath, runScriptByHash, runScriptByPathAsync, runScriptByHashAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, Sql, requestInteractiveTeamsApproval, appendToResultStream, streamResult, datatable, ducklake, type SqlTemplateFunction } from "./client";' >> "${script_dirpath}/src/index.ts" diff --git a/typescript-client/build.sh b/typescript-client/build.sh index c17fb2a8c9..f8db442962 100755 --- a/typescript-client/build.sh +++ b/typescript-client/build.sh @@ -36,7 +36,8 @@ fi cp "${script_dirpath}/client.ts" "${script_dirpath}/src/" cp "${script_dirpath}/s3Types.ts" "${script_dirpath}/src/" +cp "${script_dirpath}/sqlUtils.ts" "${script_dirpath}/src/" echo "" >> "${script_dirpath}/src/index.ts" echo 'export type { S3Object, DenoS3LightClientSettings } from "./s3Types";' >> "${script_dirpath}/src/index.ts" echo "" >> "${script_dirpath}/src/index.ts" -echo 'export { type Base64, setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, setProgress, getProgress, getState, getIdToken, denoS3LightClientSettings, loadS3FileStream, loadS3File, writeS3File, signS3Objects, signS3Object, task, runScript, runScriptAsync, runScriptByPath, runScriptByHash, runScriptByPathAsync, runScriptByHashAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, Sql, requestInteractiveTeamsApproval, appendToResultStream, streamResult } from "./client";' >> "${script_dirpath}/src/index.ts" +echo 'export { type Base64, setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, setProgress, getProgress, getState, getIdToken, denoS3LightClientSettings, loadS3FileStream, loadS3File, writeS3File, signS3Objects, signS3Object, task, runScript, runScriptAsync, runScriptByPath, runScriptByHash, runScriptByPathAsync, runScriptByHashAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, Sql, requestInteractiveTeamsApproval, appendToResultStream, streamResult, datatable, ducklake, type SqlTemplateFunction } from "./client";' >> "${script_dirpath}/src/index.ts" diff --git a/typescript-client/client.d.ts b/typescript-client/client.d.ts index 41238ff787..1ad02efbbe 100644 --- a/typescript-client/client.d.ts +++ b/typescript-client/client.d.ts @@ -1,5 +1,20 @@ import { DenoS3LightClientSettings, type S3Object } from "./s3Types"; -export { AdminService, AuditService, FlowService, GranularAclService, GroupService, JobService, ResourceService, VariableService, ScriptService, ScheduleService, SettingsService, UserService, WorkspaceService, } from "./index"; +export { + AdminService, + AuditService, + FlowService, + GranularAclService, + GroupService, + JobService, + ResourceService, + VariableService, + ScriptService, + ScheduleService, + SettingsService, + UserService, + WorkspaceService, +} from "./index"; +export { datatable, ducklake, type SqlTemplateFunction } from "./sqlUtils"; export type Sql = string; export type Email = string; export type Base64 = string; @@ -17,19 +32,32 @@ export declare function getWorkspace(): string; * @param undefinedIfEmpty if the resource does not exist, return undefined instead of throwing an error * @returns resource value */ -export declare function getResource(path?: string, undefinedIfEmpty?: boolean): Promise; +export declare function getResource( + path?: string, + undefinedIfEmpty?: boolean +): Promise; /** * Get a resource value by path * @param jobId job id to get the root job id from (default to current job) * @returns root job id */ export declare function getRootJobId(jobId?: string): Promise; -export declare function runScript(path?: string | null, hash_?: string | null, args?: Record | null, verbose?: boolean): Promise; +export declare function runScript( + path?: string | null, + hash_?: string | null, + args?: Record | null, + verbose?: boolean +): Promise; export declare function waitJob(jobId: string, verbose?: boolean): Promise; export declare function getResult(jobId: string): Promise; export declare function getResultMaybe(jobId: string): Promise; export declare function task(f: (_: P) => T): (_: P) => Promise; -export declare function runScriptAsync(path: string | null, hash_: string | null, args: Record | null, scheduledInSeconds?: number | null): Promise; +export declare function runScriptAsync( + path: string | null, + hash_: string | null, + args: Record | null, + scheduledInSeconds?: number | null +): Promise; /** * Resolve a resource value in case the default value was picked because the input payload was undefined * @param obj resource value or path of the resource under the format `$res:path` @@ -43,7 +71,11 @@ export declare function getStatePath(): string; * @param value new value of the resource to set * @param initializeToTypeIfNotExist if the resource does not exist, initialize it with this type */ -export declare function setResource(value: any, path?: string, initializeToTypeIfNotExist?: string): Promise; +export declare function setResource( + value: any, + path?: string, + initializeToTypeIfNotExist?: string +): Promise; /** * Set the state * @param state state to set @@ -61,13 +93,20 @@ export declare function setState(state: any): Promise; * @param value value of the state */ -export declare function setFlowUserState(key: string, value: any, errorIfNotPossible?: boolean): Promise; +export declare function setFlowUserState( + key: string, + value: any, + errorIfNotPossible?: boolean +): Promise; /** * Get a flow user state * @param path path of the variable */ -export declare function getFlowUserState(key: string, errorIfNotPossible?: boolean): Promise; +export declare function getFlowUserState( + key: string, + errorIfNotPossible?: boolean +): Promise; /** * Get the internal state * @deprecated use getState instead @@ -90,9 +129,16 @@ export declare function getVariable(path: string): Promise; * @param isSecretIfNotExist if the variable does not exist, create it as secret or not (default: false) * @param descriptionIfNotExist if the variable does not exist, create it with this description (default: "") */ -export declare function setVariable(path: string, value: string, isSecretIfNotExist?: boolean, descriptionIfNotExist?: string): Promise; +export declare function setVariable( + path: string, + value: string, + isSecretIfNotExist?: boolean, + descriptionIfNotExist?: string +): Promise; export declare function databaseUrlFromResource(path: string): Promise; -export declare function denoS3LightClientSettings(s3_resource_path: string | undefined): Promise; +export declare function denoS3LightClientSettings( + s3_resource_path: string | undefined +): Promise; /** * Load the content of a file stored in S3. If the s3ResourcePath is undefined, it will default to the workspace S3 resource. * @@ -103,7 +149,10 @@ export declare function denoS3LightClientSettings(s3_resource_path: string | und * console.log(text); * ``` */ -export declare function loadS3File(s3object: S3Object, s3ResourcePath?: string | undefined): Promise; +export declare function loadS3File( + s3object: S3Object, + s3ResourcePath?: string | undefined +): Promise; /** * Load the content of a file stored in S3 as a stream. If the s3ResourcePath is undefined, it will default to the workspace S3 resource. * @@ -113,7 +162,10 @@ export declare function loadS3File(s3object: S3Object, s3ResourcePath?: string | * console.log(await fileContentBlob.text()); * ``` */ -export declare function loadS3FileStream(s3object: S3Object, s3ResourcePath?: string | undefined): Promise; +export declare function loadS3FileStream( + s3object: S3Object, + s3ResourcePath?: string | undefined +): Promise; /** * Persist a file to the S3 bucket. If the s3ResourcePath is undefined, it will default to the workspace S3 resource. * @@ -123,24 +175,28 @@ export declare function loadS3FileStream(s3object: S3Object, s3ResourcePath?: st * console.log(fileContentAsUtf8Str) * ``` */ -export declare function writeS3File(s3object: S3Object | undefined, fileContent: string | Blob, s3ResourcePath?: string | undefined): Promise; +export declare function writeS3File( + s3object: S3Object | undefined, + fileContent: string | Blob, + s3ResourcePath?: string | undefined +): Promise; /** * Get URLs needed for resuming a flow after this step * @param approver approver name * @returns approval page UI URL, resume and cancel API URLs for resuming the flow */ export declare function getResumeUrls(approver?: string): Promise<{ - approvalPage: string; - resume: string; - cancel: string; + approvalPage: string; + resume: string; + cancel: string; }>; /** * @deprecated use getResumeUrls instead */ export declare function getResumeEndpoints(approver?: string): Promise<{ - approvalPage: string; - resume: string; - cancel: string; + approvalPage: string; + resume: string; + cancel: string; }>; /** * Get an OIDC jwt token for auth to external services (e.g: Vault, AWS) (ee only) diff --git a/typescript-client/client.ts b/typescript-client/client.ts index 93fc66669d..ab1c232ca6 100644 --- a/typescript-client/client.ts +++ b/typescript-client/client.ts @@ -17,6 +17,8 @@ import { type S3Object, } from "./s3Types"; +export { datatable, ducklake, type SqlTemplateFunction } from "./sqlUtils"; + export { AdminService, AuditService, @@ -138,7 +140,9 @@ export async function runScript( args: Record | null = null, verbose: boolean = false ): Promise { - console.warn('runScript is deprecated. Use runScriptByPath or runScriptByHash instead.'); + console.warn( + "runScript is deprecated. Use runScriptByPath or runScriptByHash instead." + ); if (path && hash_) { throw new Error("path and hash_ are mutually exclusive"); } @@ -157,7 +161,10 @@ async function _runScriptInternal( if (path) { console.info(`running \`${path}\` synchronously with args:`, args); } else if (hash_) { - console.info(`running script with hash \`${hash_}\` synchronously with args:`, args); + console.info( + `running script with hash \`${hash_}\` synchronously with args:`, + args + ); } } @@ -185,9 +192,7 @@ export async function runScriptByHash( * Append a text to the result stream * @param text text to append to the result stream */ -export function appendToResultStream( - text: string -) { +export function appendToResultStream(text: string) { console.log("WM_STREAM: " + text.replace(/\n/g, "\\n")); } @@ -195,9 +200,7 @@ export function appendToResultStream( * Stream to the result stream * @param stream stream to stream to the result stream */ -export async function streamResult( - stream: AsyncIterable -) { +export async function streamResult(stream: AsyncIterable) { for await (const text of stream) { appendToResultStream(text); } @@ -310,7 +313,9 @@ export async function runScriptAsync( args: Record | null, scheduledInSeconds: number | null = null ): Promise { - console.warn('runScriptAsync is deprecated. Use runScriptByPathAsync or runScriptByHashAsync instead.'); + console.warn( + "runScriptAsync is deprecated. Use runScriptByPathAsync or runScriptByHashAsync instead." + ); // Create a script job and return its job id. if (path && hash_) { throw new Error("path and hash_ are mutually exclusive"); @@ -815,7 +820,8 @@ export async function loadS3FileStream( // We use raw fetch here b/c OpenAPI generated client doesn't handle Blobs nicely const response = await fetch( - `${OpenAPI.BASE + `${ + OpenAPI.BASE }/w/${getWorkspace()}/job_helpers/download_s3_file?${queryParams}`, { method: "GET", diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index 66ba76441d..0893075673 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -3,7 +3,7 @@ "version": "1.589.3", "exports": "./src/index.ts", "publish": { - "exclude": ["!src", "./s3Types.ts", "./client.ts"] + "exclude": ["!src", "./s3Types.ts", "./sqlUtils.ts", "./client.ts"] }, "license": "Apache-2.0" } diff --git a/typescript-client/publish.sh b/typescript-client/publish.sh index 99f88ab29a..ba5c09c82f 100755 --- a/typescript-client/publish.sh +++ b/typescript-client/publish.sh @@ -9,8 +9,10 @@ rm -rf "${script_dirpath}/dist" ${script_dirpath}/build.sh rm "${script_dirpath}/client.ts" rm "${script_dirpath}/s3Types.ts" +rm "${script_dirpath}/sqlUtils.ts" npm install tsc cp "${script_dirpath}/src/client.ts" ${script_dirpath} cp "${script_dirpath}/src/s3Types.ts" ${script_dirpath} +cp "${script_dirpath}/src/sqlUtils.ts" ${script_dirpath} npm publish ${args} diff --git a/typescript-client/sqlUtils.d.ts b/typescript-client/sqlUtils.d.ts new file mode 100644 index 0000000000..012ebcb38a --- /dev/null +++ b/typescript-client/sqlUtils.d.ts @@ -0,0 +1,53 @@ +type ResultCollection = + | "last_statement_all_rows" + | "last_statement_first_row" + | "last_statement_all_rows_scalar" + | "last_statement_first_row_scalar" + | "all_statements_all_rows" + | "all_statements_first_row" + | "all_statements_all_rows_scalar" + | "all_statements_first_row_scalar" + | "legacy"; + +type FetchParams = { + resultCollection?: ResultCollectionT; +}; + +type SqlResult = + ResultCollectionT extends "last_statement_first_row" + ? object + : ResultCollectionT extends "all_statements_first_row" + ? object[] + : ResultCollectionT extends "last_statement_all_rows" + ? object[] + : ResultCollectionT extends "all_statements_all_rows" + ? object[][] + : ResultCollectionT extends "last_statement_all_rows_scalar" + ? any[] + : ResultCollectionT extends "all_statements_all_rows_scalar" + ? any[][] + : ResultCollectionT extends "last_statement_first_row_scalar" + ? any + : ResultCollectionT extends "all_statements_first_row_scalar" + ? any[] + : unknown; +export type SqlStatement = { + content: string; + + args: Record; + + fetch( + params?: FetchParams // The union is for auto-completion + ): Promise>; + + fetchOne( + params?: Omit, "resultCollection"> + ): Promise>; +}; + +export interface SqlTemplateFunction { + (strings: TemplateStringsArray, ...values: any[]): SqlStatement; +} + +export declare function datatable(name: string): SqlTemplateFunction; +export declare function ducklake(name: string): SqlTemplateFunction; diff --git a/typescript-client/sqlUtils.ts b/typescript-client/sqlUtils.ts new file mode 100644 index 0000000000..a024992de6 --- /dev/null +++ b/typescript-client/sqlUtils.ts @@ -0,0 +1,218 @@ +import { getWorkspace, JobService } from "./client"; + +type ResultCollection = + | "last_statement_all_rows" + | "last_statement_first_row" + | "last_statement_all_rows_scalar" + | "last_statement_first_row_scalar" + | "all_statements_all_rows" + | "all_statements_first_row" + | "all_statements_all_rows_scalar" + | "all_statements_first_row_scalar" + | "legacy"; + +type FetchParams = { + resultCollection?: ResultCollectionT; +}; + +type SqlResult = + ResultCollectionT extends "last_statement_first_row" + ? object + : ResultCollectionT extends "all_statements_first_row" + ? object[] + : ResultCollectionT extends "last_statement_all_rows" + ? object[] + : ResultCollectionT extends "all_statements_all_rows" + ? object[][] + : ResultCollectionT extends "last_statement_all_rows_scalar" + ? any[] + : ResultCollectionT extends "all_statements_all_rows_scalar" + ? any[][] + : ResultCollectionT extends "last_statement_first_row_scalar" + ? any + : ResultCollectionT extends "all_statements_first_row_scalar" + ? any[] + : unknown; +export type SqlStatement = { + content: string; + + args: Record; + + fetch( + params?: FetchParams // The union is for auto-completion + ): Promise>; + + fetchOne( + params?: Omit, "resultCollection"> + ): Promise>; +}; + +export interface SqlTemplateFunction { + (strings: TemplateStringsArray, ...values: any[]): SqlStatement; +} + +/** + * @example + * let sql = wmill.datatable() + * let name = 'Robin' + * let age = 21 + * await sql` + * SELECT * FROM friends + * WHERE name = ${name} AND age = ${age}::int + * `.fetch() + */ +export function datatable(name: string = "main"): SqlTemplateFunction { + return sqlProviderImpl(name, "datatable"); +} + +/** + * @example + * let sql = wmill.ducklake() + * let name = 'Robin' + * let age = 21 + * await sql` + * SELECT * FROM friends + * WHERE name = ${name} AND age = ${age} + * `.fetch() + */ +export function ducklake(name: string = "main"): SqlTemplateFunction { + return sqlProviderImpl(name, "ducklake"); +} + +function sqlProviderImpl( + name: string, + provider: "datatable" | "ducklake" +): SqlTemplateFunction { + let sql: SqlTemplateFunction = ( + strings: TemplateStringsArray, + ...values: any[] + ) => { + let formatArgDecl = { + datatable: (i: number) => `-- $${i + 1} arg${i + 1}`, + ducklake: (i: number) => { + let argType = + parseTypeAnnotation(strings[i], strings[i + 1]) || + inferSqlType(values[i]); + return `-- $arg${i + 1} (${argType})`; + }, + }[provider]; + + let formatArgUsage = { + datatable: (i: number) => { + let argType = + parseTypeAnnotation(strings[i], strings[i + 1]) || + inferSqlType(values[i]); + return `$${i + 1}::${argType}`; + }, + ducklake: (i: number) => `$arg${i + 1}`, + }[provider]; + + let content = values.map((_, i) => formatArgDecl(i)).join("\n") + "\n"; + if (provider === "ducklake") + content += `ATTACH 'ducklake://${name}' AS dl;USE dl;\n`; + + let contentBody = ""; + for (let i = 0; i < strings.length; i++) { + contentBody += strings[i]; + if (i !== strings.length - 1) contentBody += formatArgUsage(i); + } + content += contentBody; + + const args = { + ...Object.fromEntries(values.map((v, i) => [`arg${i + 1}`, v])), + ...(provider === "datatable" ? { database: `datatable://${name}` } : {}), + }; + const language = { + datatable: "postgresql" as const, + ducklake: "duckdb" as const, + }[provider]; + + async function fetch({ + resultCollection, + }: FetchParams = {}) { + if (resultCollection) + content = `-- result_collection=${resultCollection}\n${content}`; + try { + let result = await JobService.runScriptPreviewInline({ + workspace: getWorkspace(), + requestBody: { args, content, language }, + }); + return result as SqlResult; + } catch (e: any) { + let err = e; + if ( + e && + typeof e.body == "string" && + e.statusText == "Internal Server Error" + ) { + let body = e.body; + if (body.startsWith("Internal:")) body = body.slice(9).trim(); + if (body.startsWith("Error:")) body = body.slice(6).trim(); + if (body.startsWith("datatable")) body = body.slice(9).trim(); + err = Error(`${provider} ${body}`); + err.query = contentBody; + err.request = e.request; + } + throw err; + } + } + + return { + content, + args, + fetch, + fetchOne: (params) => + fetch({ ...params, resultCollection: "last_statement_first_row" }), + } satisfies SqlStatement; + }; + return sql; +} + +// DuckDB executor requires explicit argument types at declaration +// And postgres at argument usage. +// These types exist in both DuckDB and Postgres +// Check that the types exist if you plan to extend this function for other SQL engines. +function inferSqlType(value: any): string { + if (typeof value === "number" || typeof value === "bigint") { + if (Number.isInteger(value)) return "BIGINT"; + return "DOUBLE PRECISION"; + } else if (value === null || value === undefined) { + return "TEXT"; + } else if (typeof value === "string") { + return "TEXT"; + } else if (typeof value === "object") { + return "JSON"; + } else if (typeof value === "boolean") { + return "BOOLEAN"; + } else { + return "TEXT"; + } +} + +// The goal is to detect if the user added a type annotation manually +// +// untyped : sql`SELECT ${x} = 0` => ['SELECT ', ' = 0'] +// typed : sql`SELECT ${x}::int = 0` => ['SELECT ', '::int = 0'] +// typed : sql`SELECT CAST ( ${x} AS int ) = 0` => ['SELECT CAST ( ', ' AS int ) = 0'] +function parseTypeAnnotation( + prevTemplateString: string | undefined, + nextTemplateString: string | undefined +): string | undefined { + if (!nextTemplateString) return; + nextTemplateString = nextTemplateString.trimStart(); + if (nextTemplateString.startsWith("::")) { + return nextTemplateString.substring(2).trimStart().split(/\s+/)[0]; + } + prevTemplateString = prevTemplateString?.trimEnd(); + if ( + prevTemplateString?.endsWith("(") && + prevTemplateString + .substring(0, prevTemplateString.length - 1) + .trim() + .toUpperCase() + .endsWith("CAST") && + nextTemplateString.toUpperCase().startsWith("AS ") + ) { + return nextTemplateString.substring(2).trimStart().split(/\s+/)[0]; + } +}