Merge branch 'main' into di/db-manager-backend-scripts

backend/windmill-api/src/apps.rs

@@ -1877,6 +1877,15 @@ pub struct ExecuteApp {
     pub run_query_params: Option<RunJobQuery>,
 }
 
+fn maybe_replace_internal_db_script(mut raw_code: RawCode) -> RawCode {
+    if let Some(replaced) =
+        crate::db_studio_scripts::maybe_replace_internal_script(&raw_code.content)
+    {
+        raw_code.content = replaced;
+    }
+    raw_code
+}
+
 fn digest(code: &str) -> String {
     let mut hasher = Sha256::new();
     hasher.update(code);
@@ -2138,13 +2147,30 @@ async fn execute_component(
         // flow or script:
         (Some(path), None, None) => get_payload_tag_from_prefixed_path(&path, &db, &w_id).await?,
         // inline script: in "preview" mode or without entry in the `app_script` table.
-        (None, Some(raw_code), None) => (JobPayload::Code(raw_code), None, None),
+        (None, Some(raw_code), None) => {
+            let raw_code = maybe_replace_internal_db_script(raw_code);
+            (JobPayload::Code(raw_code), None, None)
+        }
         // inline script: in "run" mode and with an entry in the `app_script` table.
-        (None, Some(RawCode { language, path, cache_ttl, .. }), Some(id)) => (
+        (None, Some(raw_code), Some(id)) => {
-            JobPayload::AppScript { id: AppScriptId(id), cache_ttl, language, path },
+            // Check if this is an internal DB script marker — if so, replace content and
-            None,
+            // execute as Code (preview-style) since the content is server-controlled.
-            None,
+            if raw_code
-        ),
+                .content
+                .trim_start()
+                .starts_with(crate::db_studio_scripts::WM_INTERNAL_PREFIX)
+            {
+                let raw_code = maybe_replace_internal_db_script(raw_code);
+                (JobPayload::Code(raw_code), None, None)
+            } else {
+                let RawCode { language, path, cache_ttl, .. } = raw_code;
+                (
+                    JobPayload::AppScript { id: AppScriptId(id), cache_ttl, language, path },
+                    None,
+                    None,
+                )
+            }
+        }
         _ => unreachable!(),
     };
     let tx = PushIsolationLevel::IsolatedRoot(db.clone());

backend/windmill-api/src/jobs.rs

@@ -4612,21 +4612,26 @@ async fn run_preview_script(
         match preview.kind {
             Some(PreviewKind::Identity) => JobPayload::Identity,
             Some(PreviewKind::Noop) => JobPayload::Noop,
-            _ => JobPayload::Code(RawCode {
+            _ => {
-                hash: preview
+                let content = preview.content.unwrap_or_default();
-                    .script_hash
+                let content = crate::db_studio_scripts::maybe_replace_internal_script(&content)
-                    .as_ref()
+                    .unwrap_or(content);
-                    .and_then(|s| windmill_common::scripts::to_i64(s).ok()),
+                JobPayload::Code(RawCode {
-                content: preview.content.unwrap_or_default(),
+                    hash: preview
-                path: preview.path,
+                        .script_hash
-                language: preview.language.unwrap_or(ScriptLang::Deno),
+                        .as_ref()
-                lock: preview.lock,
+                        .and_then(|s| windmill_common::scripts::to_i64(s).ok()),
-                concurrency_settings: ConcurrencySettingsWithCustom::default(), // TODO(gbouv): once I find out how to store limits in the content of a script, should be easy to plug limits here
+                    content,
-                debouncing_settings: DebouncingSettings::default(), // TODO(pyra): same as for concurrency limits.
+                    path: preview.path,
-                cache_ttl: None,
+                    language: preview.language.unwrap_or(ScriptLang::Deno),
-                cache_ignore_s3_path: None,
+                    lock: preview.lock,
-                dedicated_worker: preview.dedicated_worker,
+                    concurrency_settings: ConcurrencySettingsWithCustom::default(), // TODO(gbouv): once I find out how to store limits in the content of a script, should be easy to plug limits here
-            }),
+                    debouncing_settings: DebouncingSettings::default(), // TODO(pyra): same as for concurrency limits.
+                    cache_ttl: None,
+                    cache_ignore_s3_path: None,
+                    dedicated_worker: preview.dedicated_worker,
+                })
+            }
         },
         push_args,
         authed.display_username(),

backend/windmill-api/src/lib.rs

@@ -76,6 +76,7 @@ mod bedrock;
 mod capture;
 mod concurrency_groups;
 mod db;
+mod db_studio_scripts;
 
 mod drafts;
 #[cfg(feature = "private")]

frontend/src/lib/components/apps/components/display/dbtable/queries/count.ts

@@ -1,5 +1,4 @@
 import type { AppInput, RunnableByName } from '$lib/components/apps/inputType'
-import { wrapDucklakeQuery } from '../../../../../ducklake'
 import type { DbType, DbInput } from '$lib/components/dbTypes'
 import { buildParameters } from '../utils'
 import { getLanguageByResourceType, type ColumnDef, buildVisibleFieldList } from '../utils'
@@ -142,6 +141,28 @@ export function makeCountQuery(
 	return query
 }
 
+export function buildCountMarker(
+	table: string,
+	columnDefs: ColumnDef[],
+	whereClause: string | undefined,
+	dbType: DbType,
+	ducklake?: string
+): string {
+	const params: Record<string, unknown> = {
+		table,
+		column_defs: columnDefs.map((c) => ({
+			field: c.field,
+			datatype: c.datatype,
+			isprimarykey: c.isprimarykey,
+			ignored: c.ignored ?? false
+		})),
+		where_clause: whereClause ?? null,
+		db_type: dbType
+	}
+	if (ducklake) params.ducklake = ducklake
+	return `-- WM_INTERNAL_DB_COUNT_SCRIPT\n${JSON.stringify(params)}`
+}
+
 export function getCountInput(
 	dbInput: DbInput,
 	table: string,
@@ -157,8 +178,8 @@ export function getCountInput(
 		return undefined
 	}
 	const dbType = dbInput.type === 'ducklake' ? 'duckdb' : dbInput.resourceType
-	let query = makeCountQuery(dbType, table, whereClause, columnDefs)
+	const ducklake = dbInput.type === 'ducklake' ? dbInput.ducklake : undefined
-	if (dbInput.type === 'ducklake') query = wrapDucklakeQuery(query, dbInput.ducklake)
+	const query = buildCountMarker(table, columnDefs, whereClause, dbType, ducklake)
 
 	const updateRunnable: RunnableByName = {
 		name: 'AppDbExplorer',

frontend/src/lib/components/apps/components/display/dbtable/queries/delete.ts

@@ -1,6 +1,5 @@
 import type { AppInput, RunnableByName } from '$lib/components/apps/inputType'
 import type { DbType, DbInput } from '$lib/components/dbTypes'
-import { wrapDucklakeQuery } from '../../../../../ducklake'
 import { getLanguageByResourceType, type ColumnDef, buildParameters } from '../utils'
 
 export function makeDeleteQuery(table: string, columns: ColumnDef[], dbType: DbType) {
@@ -66,6 +65,21 @@ export function makeDeleteQuery(table: string, columns: ColumnDef[], dbType: DbT
 	}
 }
 
+export function buildDeleteMarker(
+	table: string,
+	columns: ColumnDef[],
+	dbType: DbType,
+	ducklake?: string
+): string {
+	const params: Record<string, unknown> = {
+		table,
+		columns: columns.map((c) => ({ field: c.field, datatype: c.datatype })),
+		db_type: dbType
+	}
+	if (ducklake) params.ducklake = ducklake
+	return `-- WM_INTERNAL_DB_DELETE_SCRIPT\n${JSON.stringify(params)}`
+}
+
 export function getDeleteInput(
 	dbInput: DbInput,
 	table: string,
@@ -79,8 +93,8 @@ export function getDeleteInput(
 		return undefined
 	}
 	const dbType = dbInput.type === 'ducklake' ? 'duckdb' : dbInput.resourceType
-	let query = makeDeleteQuery(table, columns, dbType)
+	const ducklake = dbInput.type === 'ducklake' ? dbInput.ducklake : undefined
-	if (dbInput.type === 'ducklake') query = wrapDucklakeQuery(query, dbInput.ducklake)
+	const query = buildDeleteMarker(table, columns, dbType, ducklake)
 	const deleteRunnable: RunnableByName = {
 		name: 'AppDbExplorer',
 		type: 'inline',

frontend/src/lib/components/apps/components/display/dbtable/queries/insert.ts

@@ -1,5 +1,4 @@
 import type { AppInput } from '$lib/components/apps/inputType'
-import { wrapDucklakeQuery } from '../../../../../ducklake'
 import type { DbType, DbInput } from '$lib/components/dbTypes'
 import { buildParameters, ColumnIdentity } from '../utils'
 import { getLanguageByResourceType, type ColumnDef } from '../utils'
@@ -106,10 +105,37 @@ export function makeInsertQuery(table: string, columns: ColumnDef[], dbType: DbT
 	return query
 }
 
+export function buildInsertMarker(
+	table: string,
+	columns: ColumnDef[],
+	dbType: DbType,
+	ducklake?: string
+): string {
+	const params: Record<string, unknown> = {
+		table,
+		columns: columns.map((c) => ({
+			field: c.field,
+			datatype: c.datatype,
+			isprimarykey: c.isprimarykey,
+			ignored: c.ignored ?? false,
+			isnullable: c.isnullable ?? 'YES',
+			isidentity: c.isidentity ?? 'No',
+			defaultvalue: c.defaultvalue ?? null,
+			hideInsert: c.hideInsert ?? false,
+			overrideDefaultValue: c.overrideDefaultValue ?? false,
+			defaultUserValue: c.defaultUserValue ?? null,
+			defaultValueNull: c.defaultValueNull ?? false
+		})),
+		db_type: dbType
+	}
+	if (ducklake) params.ducklake = ducklake
+	return `-- WM_INTERNAL_DB_INSERT_SCRIPT\n${JSON.stringify(params)}`
+}
+
 export function getInsertInput(dbInput: DbInput, table: string, columns: ColumnDef[]): AppInput {
 	const dbType = dbInput.type === 'ducklake' ? 'duckdb' : dbInput.resourceType
-	let query = makeInsertQuery(table, columns, dbType)
+	const ducklake = dbInput.type === 'ducklake' ? dbInput.ducklake : undefined
-	if (dbInput.type === 'ducklake') query = wrapDucklakeQuery(query, dbInput.ducklake)
+	const query = buildInsertMarker(table, columns, dbType, ducklake)
 	return {
 		runnable: {
 			name: 'AppDbExplorer',

frontend/src/lib/components/apps/components/display/dbtable/queries/select.ts

@@ -1,5 +1,4 @@
 import type { AppInput, RunnableByName } from '$lib/components/apps/inputType'
-import { wrapDucklakeQuery } from '../../../../../ducklake'
 import type { DbType, DbInput } from '$lib/components/dbTypes'
 import { buildParameters } from '../utils'
 import { getLanguageByResourceType, type ColumnDef, buildVisibleFieldList } from '../utils'
@@ -318,6 +317,33 @@ function coerceToNumber(value: any): number {
 	return 0
 }
 
+export function buildSelectMarker(
+	table: string,
+	columnDefs: ColumnDef[],
+	whereClause: string | undefined,
+	dbType: DbType,
+	ducklake?: string
+): string {
+	const params: Record<string, unknown> = {
+		table,
+		column_defs: columnDefs.map((c) => ({
+			field: c.field,
+			datatype: c.datatype,
+			isprimarykey: c.isprimarykey,
+			ignored: c.ignored ?? false,
+			editable: c.editable ?? false,
+			isnullable: c.isnullable ?? 'YES',
+			isidentity: c.isidentity ?? 'No',
+			defaultvalue: c.defaultvalue ?? null,
+			hideInsert: c.hideInsert ?? false
+		})),
+		where_clause: whereClause ?? null,
+		db_type: dbType
+	}
+	if (ducklake) params.ducklake = ducklake
+	return `-- WM_INTERNAL_DB_SELECT_SCRIPT\n${JSON.stringify(params)}`
+}
+
 export function getSelectInput(
 	dbInput: DbInput,
 	table: string | undefined,
@@ -335,8 +361,8 @@ export function getSelectInput(
 	}
 
 	const dbType = dbInput.type === 'ducklake' ? 'duckdb' : dbInput.resourceType
-	let content = makeSelectQuery(table, columnDefs, whereClause, dbType, options)
+	const ducklake = dbInput.type === 'ducklake' ? dbInput.ducklake : undefined
-	if (dbInput.type === 'ducklake') content = wrapDucklakeQuery(content, dbInput.ducklake)
+	const content = buildSelectMarker(table, columnDefs, whereClause, dbType, ducklake)
 	const getRunnable: RunnableByName = {
 		name: 'AppDbExplorer',
 		type: 'inline',

frontend/src/lib/components/apps/components/display/dbtable/queries/update.ts

@@ -1,5 +1,4 @@
 import type { AppInput, RunnableByName } from '$lib/components/apps/inputType'
-import { wrapDucklakeQuery } from '../../../../../ducklake'
 import type { DbInput, DbType } from '$lib/components/dbTypes'
 import { getLanguageByResourceType, type ColumnDef, buildParameters } from '../utils'
 
@@ -76,6 +75,23 @@ export function makeUpdateQuery(
 	}
 }
 
+export function buildUpdateMarker(
+	table: string,
+	column: { datatype: string; field: string },
+	columns: { datatype: string; field: string }[],
+	dbType: DbType,
+	ducklake?: string
+): string {
+	const params: Record<string, unknown> = {
+		table,
+		column: { field: column.field, datatype: column.datatype },
+		columns: columns.map((c) => ({ field: c.field, datatype: c.datatype })),
+		db_type: dbType
+	}
+	if (ducklake) params.ducklake = ducklake
+	return `-- WM_INTERNAL_DB_UPDATE_SCRIPT\n${JSON.stringify(params)}`
+}
+
 export function getUpdateInput(
 	dbInput: DbInput,
 	table: string,
@@ -90,8 +106,8 @@ export function getUpdateInput(
 		return undefined
 	}
 	const dbType = dbInput.type === 'ducklake' ? 'duckdb' : dbInput.resourceType
-	let query = makeUpdateQuery(table, column, columns, dbType)
+	const ducklake = dbInput.type === 'ducklake' ? dbInput.ducklake : undefined
-	if (dbInput.type === 'ducklake') query = wrapDucklakeQuery(query, dbInput.ducklake)
+	const query = buildUpdateMarker(table, column, columns, dbType, ducklake)
 
 	const updateRunnable: RunnableByName = {
 		name: 'AppDbExplorer',

frontend/src/lib/components/dbOps.ts

@@ -3,12 +3,12 @@ import {
 	type ColumnDef,
 	type TableMetadata
 } from './apps/components/display/dbtable/utils'
-import { makeSelectQuery } from './apps/components/display/dbtable/queries/select'
+import { buildSelectMarker } from './apps/components/display/dbtable/queries/select'
 import { runScriptAndPollResult } from './jobs/utils'
-import { makeCountQuery } from './apps/components/display/dbtable/queries/count'
+import { buildCountMarker } from './apps/components/display/dbtable/queries/count'
-import { makeUpdateQuery } from './apps/components/display/dbtable/queries/update'
+import { buildUpdateMarker } from './apps/components/display/dbtable/queries/update'
-import { makeDeleteQuery } from './apps/components/display/dbtable/queries/delete'
+import { buildDeleteMarker } from './apps/components/display/dbtable/queries/delete'
-import { makeInsertQuery } from './apps/components/display/dbtable/queries/insert'
+import { buildInsertMarker } from './apps/components/display/dbtable/queries/insert'
 import { makeDeleteTableQuery } from './apps/components/display/dbtable/queries/deleteTable'
 import type { DBSchema, SQLSchema } from '$lib/stores'
 import { stringifySchema } from './copilot/lib'
@@ -68,8 +68,8 @@ export function dbTableOpsWithPreviewScripts({
 		tableKey,
 		colDefs,
 		getCount: async ({ quicksearch }) => {
-			let countQuery = makeCountQuery(dbType, tableKey, undefined, colDefs)
+			const ducklake = input.type === 'ducklake' ? input.ducklake : undefined
-			if (input.type === 'ducklake') countQuery = wrapDucklakeQuery(countQuery, input.ducklake)
+			const countQuery = buildCountMarker(tableKey, colDefs, undefined, dbType, ducklake)
 			const result = await runScriptAndPollResult({
 				workspace,
 				requestBody: { args: { ...dbArg, quicksearch }, language, content: countQuery }
@@ -78,10 +78,8 @@ export function dbTableOpsWithPreviewScripts({
 			return count
 		},
 		getRows: async (params) => {
-			let query = makeSelectQuery(tableKey, colDefs, undefined, dbType, undefined, {
+			const ducklake = input.type === 'ducklake' ? input.ducklake : undefined
-				fixPgIntTypes: true
+			const query = buildSelectMarker(tableKey, colDefs, undefined, dbType, ducklake)
-			})
-			if (input.type === 'ducklake') query = wrapDucklakeQuery(query, input.ducklake)
 			let items = (await runScriptAndPollResult({
 				workspace,
 				requestBody: { args: { ...dbArg, ...params }, language, content: query }
@@ -92,8 +90,8 @@ export function dbTableOpsWithPreviewScripts({
 			return items
 		},
 		onUpdate: async ({ values }, colDef, newValue) => {
-			let updateQuery = makeUpdateQuery(tableKey, colDef, colDefs, dbType)
+			const ducklake = input.type === 'ducklake' ? input.ducklake : undefined
-			if (input.type === 'ducklake') updateQuery = wrapDucklakeQuery(updateQuery, input.ducklake)
+			const updateQuery = buildUpdateMarker(tableKey, colDef, colDefs, dbType, ducklake)
 			await runScriptAndPollResult({
 				workspace,
 				requestBody: {
@@ -104,16 +102,16 @@ export function dbTableOpsWithPreviewScripts({
 			})
 		},
 		onDelete: async ({ values }) => {
-			let deleteQuery = makeDeleteQuery(tableKey, colDefs, dbType)
+			const ducklake = input.type === 'ducklake' ? input.ducklake : undefined
-			if (input.type === 'ducklake') deleteQuery = wrapDucklakeQuery(deleteQuery, input.ducklake)
+			const deleteQuery = buildDeleteMarker(tableKey, colDefs, dbType, ducklake)
 			await runScriptAndPollResult({
 				workspace,
 				requestBody: { args: { ...dbArg, ...values }, language, content: deleteQuery }
 			})
 		},
 		onInsert: async ({ values }) => {
-			let insertQuery = makeInsertQuery(tableKey, colDefs, dbType)
+			const ducklake = input.type === 'ducklake' ? input.ducklake : undefined
-			if (input.type === 'ducklake') insertQuery = wrapDucklakeQuery(insertQuery, input.ducklake)
+			const insertQuery = buildInsertMarker(tableKey, colDefs, dbType, ducklake)
 			await runScriptAndPollResult({
 				workspace,
 				requestBody: { args: { ...dbArg, ...values }, language, content: insertQuery }