Files
windmill/frontend/src/lib/components/dbOps.ts
Diego Imbert e6f1211d31 feat: Ducklake native support (#6268)
* upgrade duckdb

* basic ducklake works

* ducklake works with custom db catalogs

* fix: pwsh skip already installed modules outside of cache (#6037)

* improve query performance of user stats

* separate ducklake_catalog db

* ducklake settings

* DucklakeSettings frontend

* Ducklake ws settings saved in backend

* fetch ducklake catalog resource

* Ducklake works with configured s3 storage

* Ducklake as asset

* ducklake asset icon

* Fix duckdb array and object args not working properly (#6254)

* Fix bug with comments in duckdb

* Avoid multiple queries when doing ATTACH ducklake

* trunc sig no longer needed now that comments are trimmed

* cache DuckdbConnectionSettingsResponse

* duplicated code

* transform_attach_ducklake contributes to duckdb_connection_settings_cache

* eliminate the need for used_storages

* nit

* cleaner management of the bigquery credentials file

* DBManagerDrawer refactor to prepare for Ducklake

* get ducklake schema

* implement delete for ducklake

* load column metadata for ducklake

* Select query works for ducklake, basic db explorer works !

* duckdb count query

* Support all db ops for ducklake

* clean migrations

* SQL repl for Ducklake

* fix broken database studio

* nit

* assert function

* Ducklake in Editor Bar

* default ducklake syntax + allow extra args

* DucklakeCatalogWizard UI

* nit + remove extra $

* modal when databases do not exist

* cannot be windmill

* Ducklake works safely with instance database

* Avoid sending instance db credentials on network

* resource leak security

* remove fetch_attach_db_conn_str

* prevent instance pg password leak

* hide asset usage count when not available

* case unsensitivity duckdb

* warnings

* disable instance catalog

* use shorthand syntax when inserting with EditorBar

* Instance ducklake catalog is now safe to use

* use safer argon2 pwd

* update package json parsers

* update package json

* better msgs

* tooltips

* disable explore button until saved

* nit

* fix warnings

* better ducklake_user password management

* nit

* Sanitize passwords from errors in ducklake

* DisplayResult broken in job result

* remove superadmin requirement to check databases_exist

* duckdb_connection_settings_v2_inner

* Ducklake works on agent worker (finally)

* ci

* #[allow(dead_code)]

* fix openapi missing response

* Separate +Database button for DuckDB in EditorBar

* Fix dropdown in ducklake settings

* Attempt to fix migration race condition in CI

* update sqlx failing for some offline queries

* avoid temp password for ducklake_user

* nits

* ducklake settings nits

* update duckdb default script

* fix sql repl resetting text on refresh

* avoid pgcrypto extension

---------

Co-authored-by: HugoCasa <hugo@casademont.ch>
Co-authored-by: Ruben Fiszel <ruben@windmill.dev>
2025-08-06 13:55:36 +00:00

223 lines
6.6 KiB
TypeScript

import {
getLanguageByResourceType,
type ColumnDef,
type DbType
} from './apps/components/display/dbtable/utils'
import { makeSelectQuery } from './apps/components/display/dbtable/queries/select'
import { runScriptAndPollResult } from './jobs/utils'
import { makeCountQuery } from './apps/components/display/dbtable/queries/count'
import { makeUpdateQuery } from './apps/components/display/dbtable/queries/update'
import { makeDeleteQuery } from './apps/components/display/dbtable/queries/delete'
import { makeInsertQuery } from './apps/components/display/dbtable/queries/insert'
import { Trash2 } from 'lucide-svelte'
import { makeDeleteTableQuery } from './apps/components/display/dbtable/queries/deleteTable'
import type { DBSchema, SQLSchema } from '$lib/stores'
import { stringifySchema } from './copilot/lib'
export type DbInput =
| {
type: 'database'
resourceType: DbType
resourcePath: string
}
| { type: 'ducklake'; ducklake: string }
export type IDbTableOps = {
dbType: DbType
tableKey: string
colDefs: ColumnDef[]
getRows: (params: {
offset: number
limit: number
quicksearch: string
order_by: string
is_desc: boolean
}) => Promise<unknown[]>
getCount: (params: { quicksearch: string }) => Promise<number>
onUpdate?: (
row: { values: object },
colDef: { field: string; datatype: string },
newValue: string
) => Promise<void>
onDelete?: (row: { values: object }) => Promise<void>
onInsert?: (row: { values: object }) => Promise<void>
}
export function dbTableOpsWithPreviewScripts({
input,
tableKey,
colDefs,
workspace
}: {
input: DbInput
tableKey: string
colDefs: ColumnDef[]
workspace: string
}): IDbTableOps {
const dbType = getDbType(input)
const language = getLanguageByResourceType(dbType)
const dbArg = input?.type === 'database' ? { database: '$res:' + input.resourcePath } : {}
return {
dbType,
tableKey,
colDefs,
getCount: async ({ quicksearch }) => {
let countQuery = makeCountQuery(dbType, tableKey, undefined, colDefs)
if (input.type === 'ducklake') countQuery = wrapDucklakeQuery(countQuery, input.ducklake)
const result = await runScriptAndPollResult({
workspace,
requestBody: { args: { ...dbArg, quicksearch }, language, content: countQuery }
})
const count = result?.[0].count as number
return count
},
getRows: async (params) => {
let query = makeSelectQuery(tableKey, colDefs, undefined, dbType)
if (input.type === 'ducklake') query = wrapDucklakeQuery(query, input.ducklake)
let items = (await runScriptAndPollResult({
workspace,
requestBody: { args: { ...dbArg, ...params }, language, content: query }
})) as unknown[]
if (input.type === 'database' && input.resourceType === 'ms_sql_server')
items = items?.[0] as unknown[]
if (!items || !Array.isArray(items)) {
throw 'items is not an array'
}
return items
},
onUpdate: async ({ values }, colDef, newValue) => {
let updateQuery = makeUpdateQuery(tableKey, colDef, colDefs, dbType)
if (input.type === 'ducklake') updateQuery = wrapDucklakeQuery(updateQuery, input.ducklake)
await runScriptAndPollResult({
workspace,
requestBody: {
args: { ...dbArg, value_to_update: newValue, ...values },
language,
content: updateQuery
}
})
},
onDelete: async ({ values }) => {
let deleteQuery = makeDeleteQuery(tableKey, colDefs, dbType)
if (input.type === 'ducklake') deleteQuery = wrapDucklakeQuery(deleteQuery, input.ducklake)
await runScriptAndPollResult({
workspace,
requestBody: { args: { ...dbArg, ...values }, language, content: deleteQuery }
})
},
onInsert: async ({ values }) => {
let insertQuery = makeInsertQuery(tableKey, colDefs, dbType)
if (input.type === 'ducklake') insertQuery = wrapDucklakeQuery(insertQuery, input.ducklake)
await runScriptAndPollResult({
workspace,
requestBody: { args: { ...dbArg, ...values }, language, content: insertQuery }
})
}
}
}
export type DbTableAction = {
action: () => void | Promise<void>
displayName: string
confirmTitle?: string
confirmBtnText?: string
icon?: any
successText?: string
}
export type DbTableActionFactory = (params: {
tableKey: string
refresh: () => void
}) => DbTableAction
export function dbDeleteTableActionWithPreviewScript({
workspace,
input
}: {
workspace: string
input: DbInput
}): DbTableActionFactory {
const dbArg = input?.type === 'database' ? { database: '$res:' + input.resourcePath } : {}
return ({ tableKey, refresh }) => ({
confirmTitle: `Are you sure you want to delete '${tableKey}' ? This action is irreversible`,
displayName: 'Delete',
confirmBtnText: `Delete permanently`,
icon: Trash2,
successText: `Table '${tableKey}' deleted successfully`,
action: async () => {
const dbType = getDbType(input)
const language = getLanguageByResourceType(dbType)
let deleteQuery = makeDeleteTableQuery(tableKey, dbType)
if (input.type === 'ducklake') deleteQuery = wrapDucklakeQuery(deleteQuery, input.ducklake)
await runScriptAndPollResult({
workspace,
requestBody: {
args: { ...dbArg },
language,
content: deleteQuery
}
})
refresh()
}
})
}
export async function getDucklakeSchema({
workspace,
ducklake
}: {
workspace: string
ducklake: string
}): Promise<DBSchema> {
let result = await runScriptAndPollResult({
workspace,
requestBody: {
language: 'duckdb',
content: `ATTACH 'ducklake://${ducklake}' AS __ducklake__; ${DUCKLAKE_GET_SCHEMA_QUERY}`,
args: {}
}
})
const stringified = Array.isArray(result) && result.length && (result?.[0]?.['result'] ?? '[]')
if (!stringified) throw new Error('Failed to get Ducklake schema: ' + JSON.stringify(result))
let schema: Omit<SQLSchema, 'stringified'> = {
schema: { main: JSON.parse(stringified) },
publicOnly: true,
lang: 'ducklake'
}
return { ...schema, stringified: stringifySchema(schema) }
}
const DUCKLAKE_GET_SCHEMA_QUERY = `
SELECT json_group_object(table_name, table_data) AS result FROM (
SELECT
table_name,
json_group_object(
c.column_name,
json_object(
'type', c.data_type,
'default', c.column_default,
'required', c.is_nullable == 'NO'
)
) AS table_data
FROM information_schema.columns c
WHERE table_catalog = '__ducklake__' AND table_schema = current_schema()
GROUP BY c.table_name
)`
export function getDbType(input: DbInput): DbType {
switch (input.type) {
case 'database':
return input.resourceType
case 'ducklake':
return 'duckdb'
}
}
export function wrapDucklakeQuery(query: string, ducklake: string): string {
let attach = `ATTACH 'ducklake://${ducklake}' AS dl;USE dl;\n`
return query.replace(/^(--.*\n)*/, (match) => match + attach)
}