* upgrade duckdb * basic ducklake works * ducklake works with custom db catalogs * fix: pwsh skip already installed modules outside of cache (#6037) * improve query performance of user stats * separate ducklake_catalog db * ducklake settings * DucklakeSettings frontend * Ducklake ws settings saved in backend * fetch ducklake catalog resource * Ducklake works with configured s3 storage * Ducklake as asset * ducklake asset icon * Fix duckdb array and object args not working properly (#6254) * Fix bug with comments in duckdb * Avoid multiple queries when doing ATTACH ducklake * trunc sig no longer needed now that comments are trimmed * cache DuckdbConnectionSettingsResponse * duplicated code * transform_attach_ducklake contributes to duckdb_connection_settings_cache * eliminate the need for used_storages * nit * cleaner management of the bigquery credentials file * DBManagerDrawer refactor to prepare for Ducklake * get ducklake schema * implement delete for ducklake * load column metadata for ducklake * Select query works for ducklake, basic db explorer works ! * duckdb count query * Support all db ops for ducklake * clean migrations * SQL repl for Ducklake * fix broken database studio * nit * assert function * Ducklake in Editor Bar * default ducklake syntax + allow extra args * DucklakeCatalogWizard UI * nit + remove extra $ * modal when databases do not exist * cannot be windmill * Ducklake works safely with instance database * Avoid sending instance db credentials on network * resource leak security * remove fetch_attach_db_conn_str * prevent instance pg password leak * hide asset usage count when not available * case unsensitivity duckdb * warnings * disable instance catalog * use shorthand syntax when inserting with EditorBar * Instance ducklake catalog is now safe to use * use safer argon2 pwd * update package json parsers * update package json * better msgs * tooltips * disable explore button until saved * nit * fix warnings * better ducklake_user password management * nit * Sanitize passwords from errors in ducklake * DisplayResult broken in job result * remove superadmin requirement to check databases_exist * duckdb_connection_settings_v2_inner * Ducklake works on agent worker (finally) * ci * #[allow(dead_code)] * fix openapi missing response * Separate +Database button for DuckDB in EditorBar * Fix dropdown in ducklake settings * Attempt to fix migration race condition in CI * update sqlx failing for some offline queries * avoid temp password for ducklake_user * nits * ducklake settings nits * update duckdb default script * fix sql repl resetting text on refresh * avoid pgcrypto extension --------- Co-authored-by: HugoCasa <hugo@casademont.ch> Co-authored-by: Ruben Fiszel <ruben@windmill.dev>
223 lines
6.6 KiB
TypeScript
223 lines
6.6 KiB
TypeScript
import {
|
|
getLanguageByResourceType,
|
|
type ColumnDef,
|
|
type DbType
|
|
} from './apps/components/display/dbtable/utils'
|
|
import { makeSelectQuery } from './apps/components/display/dbtable/queries/select'
|
|
import { runScriptAndPollResult } from './jobs/utils'
|
|
import { makeCountQuery } from './apps/components/display/dbtable/queries/count'
|
|
import { makeUpdateQuery } from './apps/components/display/dbtable/queries/update'
|
|
import { makeDeleteQuery } from './apps/components/display/dbtable/queries/delete'
|
|
import { makeInsertQuery } from './apps/components/display/dbtable/queries/insert'
|
|
import { Trash2 } from 'lucide-svelte'
|
|
import { makeDeleteTableQuery } from './apps/components/display/dbtable/queries/deleteTable'
|
|
import type { DBSchema, SQLSchema } from '$lib/stores'
|
|
import { stringifySchema } from './copilot/lib'
|
|
|
|
export type DbInput =
|
|
| {
|
|
type: 'database'
|
|
resourceType: DbType
|
|
resourcePath: string
|
|
}
|
|
| { type: 'ducklake'; ducklake: string }
|
|
|
|
export type IDbTableOps = {
|
|
dbType: DbType
|
|
tableKey: string
|
|
colDefs: ColumnDef[]
|
|
|
|
getRows: (params: {
|
|
offset: number
|
|
limit: number
|
|
quicksearch: string
|
|
order_by: string
|
|
is_desc: boolean
|
|
}) => Promise<unknown[]>
|
|
getCount: (params: { quicksearch: string }) => Promise<number>
|
|
onUpdate?: (
|
|
row: { values: object },
|
|
colDef: { field: string; datatype: string },
|
|
newValue: string
|
|
) => Promise<void>
|
|
onDelete?: (row: { values: object }) => Promise<void>
|
|
onInsert?: (row: { values: object }) => Promise<void>
|
|
}
|
|
|
|
export function dbTableOpsWithPreviewScripts({
|
|
input,
|
|
tableKey,
|
|
colDefs,
|
|
workspace
|
|
}: {
|
|
input: DbInput
|
|
tableKey: string
|
|
colDefs: ColumnDef[]
|
|
workspace: string
|
|
}): IDbTableOps {
|
|
const dbType = getDbType(input)
|
|
const language = getLanguageByResourceType(dbType)
|
|
const dbArg = input?.type === 'database' ? { database: '$res:' + input.resourcePath } : {}
|
|
return {
|
|
dbType,
|
|
tableKey,
|
|
colDefs,
|
|
getCount: async ({ quicksearch }) => {
|
|
let countQuery = makeCountQuery(dbType, tableKey, undefined, colDefs)
|
|
if (input.type === 'ducklake') countQuery = wrapDucklakeQuery(countQuery, input.ducklake)
|
|
const result = await runScriptAndPollResult({
|
|
workspace,
|
|
requestBody: { args: { ...dbArg, quicksearch }, language, content: countQuery }
|
|
})
|
|
const count = result?.[0].count as number
|
|
return count
|
|
},
|
|
getRows: async (params) => {
|
|
let query = makeSelectQuery(tableKey, colDefs, undefined, dbType)
|
|
if (input.type === 'ducklake') query = wrapDucklakeQuery(query, input.ducklake)
|
|
let items = (await runScriptAndPollResult({
|
|
workspace,
|
|
requestBody: { args: { ...dbArg, ...params }, language, content: query }
|
|
})) as unknown[]
|
|
if (input.type === 'database' && input.resourceType === 'ms_sql_server')
|
|
items = items?.[0] as unknown[]
|
|
if (!items || !Array.isArray(items)) {
|
|
throw 'items is not an array'
|
|
}
|
|
return items
|
|
},
|
|
onUpdate: async ({ values }, colDef, newValue) => {
|
|
let updateQuery = makeUpdateQuery(tableKey, colDef, colDefs, dbType)
|
|
if (input.type === 'ducklake') updateQuery = wrapDucklakeQuery(updateQuery, input.ducklake)
|
|
await runScriptAndPollResult({
|
|
workspace,
|
|
requestBody: {
|
|
args: { ...dbArg, value_to_update: newValue, ...values },
|
|
language,
|
|
content: updateQuery
|
|
}
|
|
})
|
|
},
|
|
onDelete: async ({ values }) => {
|
|
let deleteQuery = makeDeleteQuery(tableKey, colDefs, dbType)
|
|
if (input.type === 'ducklake') deleteQuery = wrapDucklakeQuery(deleteQuery, input.ducklake)
|
|
await runScriptAndPollResult({
|
|
workspace,
|
|
requestBody: { args: { ...dbArg, ...values }, language, content: deleteQuery }
|
|
})
|
|
},
|
|
onInsert: async ({ values }) => {
|
|
let insertQuery = makeInsertQuery(tableKey, colDefs, dbType)
|
|
if (input.type === 'ducklake') insertQuery = wrapDucklakeQuery(insertQuery, input.ducklake)
|
|
await runScriptAndPollResult({
|
|
workspace,
|
|
requestBody: { args: { ...dbArg, ...values }, language, content: insertQuery }
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
export type DbTableAction = {
|
|
action: () => void | Promise<void>
|
|
displayName: string
|
|
confirmTitle?: string
|
|
confirmBtnText?: string
|
|
icon?: any
|
|
successText?: string
|
|
}
|
|
|
|
export type DbTableActionFactory = (params: {
|
|
tableKey: string
|
|
refresh: () => void
|
|
}) => DbTableAction
|
|
|
|
export function dbDeleteTableActionWithPreviewScript({
|
|
workspace,
|
|
input
|
|
}: {
|
|
workspace: string
|
|
input: DbInput
|
|
}): DbTableActionFactory {
|
|
const dbArg = input?.type === 'database' ? { database: '$res:' + input.resourcePath } : {}
|
|
|
|
return ({ tableKey, refresh }) => ({
|
|
confirmTitle: `Are you sure you want to delete '${tableKey}' ? This action is irreversible`,
|
|
displayName: 'Delete',
|
|
confirmBtnText: `Delete permanently`,
|
|
icon: Trash2,
|
|
successText: `Table '${tableKey}' deleted successfully`,
|
|
action: async () => {
|
|
const dbType = getDbType(input)
|
|
const language = getLanguageByResourceType(dbType)
|
|
let deleteQuery = makeDeleteTableQuery(tableKey, dbType)
|
|
if (input.type === 'ducklake') deleteQuery = wrapDucklakeQuery(deleteQuery, input.ducklake)
|
|
await runScriptAndPollResult({
|
|
workspace,
|
|
requestBody: {
|
|
args: { ...dbArg },
|
|
language,
|
|
content: deleteQuery
|
|
}
|
|
})
|
|
refresh()
|
|
}
|
|
})
|
|
}
|
|
|
|
export async function getDucklakeSchema({
|
|
workspace,
|
|
ducklake
|
|
}: {
|
|
workspace: string
|
|
ducklake: string
|
|
}): Promise<DBSchema> {
|
|
let result = await runScriptAndPollResult({
|
|
workspace,
|
|
requestBody: {
|
|
language: 'duckdb',
|
|
content: `ATTACH 'ducklake://${ducklake}' AS __ducklake__; ${DUCKLAKE_GET_SCHEMA_QUERY}`,
|
|
args: {}
|
|
}
|
|
})
|
|
const stringified = Array.isArray(result) && result.length && (result?.[0]?.['result'] ?? '[]')
|
|
|
|
if (!stringified) throw new Error('Failed to get Ducklake schema: ' + JSON.stringify(result))
|
|
let schema: Omit<SQLSchema, 'stringified'> = {
|
|
schema: { main: JSON.parse(stringified) },
|
|
publicOnly: true,
|
|
lang: 'ducklake'
|
|
}
|
|
return { ...schema, stringified: stringifySchema(schema) }
|
|
}
|
|
|
|
const DUCKLAKE_GET_SCHEMA_QUERY = `
|
|
SELECT json_group_object(table_name, table_data) AS result FROM (
|
|
SELECT
|
|
table_name,
|
|
json_group_object(
|
|
c.column_name,
|
|
json_object(
|
|
'type', c.data_type,
|
|
'default', c.column_default,
|
|
'required', c.is_nullable == 'NO'
|
|
)
|
|
) AS table_data
|
|
FROM information_schema.columns c
|
|
WHERE table_catalog = '__ducklake__' AND table_schema = current_schema()
|
|
GROUP BY c.table_name
|
|
)`
|
|
|
|
export function getDbType(input: DbInput): DbType {
|
|
switch (input.type) {
|
|
case 'database':
|
|
return input.resourceType
|
|
case 'ducklake':
|
|
return 'duckdb'
|
|
}
|
|
}
|
|
|
|
export function wrapDucklakeQuery(query: string, ducklake: string): string {
|
|
let attach = `ATTACH 'ducklake://${ducklake}' AS dl;USE dl;\n`
|
|
return query.replace(/^(--.*\n)*/, (match) => match + attach)
|
|
}
|