From 62b0925db9767c843c7a8ece4ed339880e7c5997 Mon Sep 17 00:00:00 2001 From: HugoCasa Date: Fri, 11 Apr 2025 15:00:16 +0200 Subject: [PATCH] b --- backend/windmill-api/openapi.yaml | 2 + backend/windmill-api/src/apps.rs | 157 +++++------ .../src/lib/components/DisplayResult.svelte | 18 +- .../apps/components/display/AppImage.svelte | 12 +- .../lib/components/apps/editor/appUtilsS3.ts | 36 ++- .../settingsPanel/InputsSpecEditor.svelte | 5 - .../inputEditor/StaticInputEditor.svelte | 74 ++++- .../common/fileDownload/FileDownload.svelte | 2 +- .../components/multiselect/MultiSelect.svelte | 2 +- .../multiselect/MultiSelectWrapper.svelte | 5 +- .../workspaceSettings/AISettings.svelte | 52 ++-- .../workspaceSettings/StorageSettings.svelte | 252 ++++++++++++++++++ .../(logged)/workspace_settings/+page.svelte | 218 +-------------- python-client/wmill/wmill/client.py | 16 +- python-client/wmill/wmill/s3_types.py | 1 + typescript-client/build.jsr.sh | 2 +- typescript-client/build.sh | 2 +- typescript-client/client.ts | 16 +- typescript-client/s3Types.ts | 1 + 19 files changed, 503 insertions(+), 370 deletions(-) create mode 100644 frontend/src/lib/components/workspaceSettings/StorageSettings.svelte diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 04996d62b2..cda6eb6b2f 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -6327,6 +6327,8 @@ paths: type: array items: $ref: "#/components/schemas/S3Object" + required: + - s3_objects responses: "200": description: signed s3 tokens diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index 7bc321a9f8..b3ffdb896b 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -20,8 +20,7 @@ use crate::{ use crate::{ job_helpers_ee::{ download_s3_file_internal, get_random_file_name, get_s3_resource, - get_workspace_s3_resource, load_image_preview_internal, upload_file_from_req, - DownloadFileQuery, LoadImagePreviewQuery, + get_workspace_s3_resource, upload_file_from_req, DownloadFileQuery, }, users::fetch_api_authed_from_permissioned_as, }; @@ -110,10 +109,6 @@ pub fn unauthed_service() -> Router { .route("/upload_s3_file/*path", post(upload_s3_file_from_app)) .route("/delete_s3_file", delete(delete_s3_file_from_app)) .route("/download_s3_file/*path", get(download_s3_file_from_app)) - .route( - "/load_image_preview/*path", - get(load_s3_file_image_preview_from_app), - ) .route("/public_app/:secret", get(get_public_app_by_secret)) .route("/public_resource/*path", get(get_public_resource)) } @@ -1610,27 +1605,24 @@ async fn sign_s3_objects( } #[cfg(feature = "parquet")] -async fn validate_s3_signature(s3_object: &S3Object, w_id: &str, db: &DB) -> Result<()> { - use url::form_urlencoded; - +async fn validate_s3_signature(file_query: &AppS3FileQuery, w_id: &str, db: &DB) -> Result<()> { let workspace_key = get_workspace_key(w_id, &db).await?; - let params: HashMap<_, _> = - form_urlencoded::parse(s3_object.presigned.as_ref().unwrap().as_bytes()) - .into_owned() - .collect(); + let Some(exp) = file_query + .exp + .as_ref() + .map(|e| e.parse::().unwrap_or_default()) + else { + return Err(Error::BadRequest("Missing exp".to_string())); + }; - let exp = params - .get("exp") - .map(|s| s.parse::().unwrap_or_default()) - .ok_or_else(|| Error::NotAuthorized("Missing exp".to_string()))?; + let Some(ref sig) = file_query.sig else { + return Err(Error::BadRequest("Missing signature".to_string())); + }; - let signature = params - .get("sig") - .ok_or_else(|| Error::NotAuthorized("Missing signature".to_string()))?; + let mut message = format!("file_key={}&exp={}", file_query.s3, exp); - let mut message = format!("file_key={}&exp={}", s3_object.s3.clone(), exp); - if let Some(ref storage) = s3_object.storage { + if let Some(ref storage) = file_query.storage { message = format!("{}&storage={}", message, storage); } @@ -1639,12 +1631,12 @@ async fn validate_s3_signature(s3_object: &S3Object, w_id: &str, db: &DB) -> Res mac.update(message.as_bytes()); - let signature_bytes = hex::decode(signature)?; - mac.verify_slice(&signature_bytes) - .map_err(|err| Error::NotAuthorized(format!("Invalid signature: {}", err)))?; + let sig_bytes = hex::decode(sig)?; + mac.verify_slice(&sig_bytes) + .map_err(|err| Error::BadRequest(format!("Invalid signature: {}", err)))?; if exp < chrono::Utc::now().timestamp() { - return Err(Error::NotAuthorized("Signature expired".to_string())); + return Err(Error::BadRequest("Signature expired".to_string())); } Ok(()) @@ -2058,7 +2050,7 @@ async fn get_on_behalf_authed_from_app( async fn check_if_allowed_to_access_s3_file_from_app( db: &DB, opt_authed: &Option, - file_query: &mut S3Object, + file_query: &AppS3FileQuery, w_id: &str, path: &str, policy: &Policy, @@ -2066,49 +2058,59 @@ async fn check_if_allowed_to_access_s3_file_from_app( // if anonymous, check that the file was the result of an app script ran by an anonymous user in the last 3 hours // otherwise, if logged in, allow any file (TODO: change that when we implement better s3 policy) - if file_query.presigned.is_some() { - validate_s3_signature(file_query, w_id, &db).await?; - } - - let allowed = opt_authed.is_some() - || policy + if file_query.sig.is_some() { + validate_s3_signature(file_query, w_id, &db).await + } else if opt_authed.is_some() { + Ok(()) + } else { + let allowed = policy .allowed_s3_keys .as_ref() .unwrap() .iter() .any(|key| key.s3_path == file_query.s3 && key.storage == file_query.storage) - || { - sqlx::query_scalar!( - r#"SELECT EXISTS ( - SELECT 1 FROM v2_as_completed_job - WHERE workspace_id = $2 + || { + sqlx::query_scalar!( + r#"SELECT EXISTS ( + SELECT 1 FROM v2_as_completed_job + WHERE workspace_id = $2 AND (job_kind = 'appscript' OR job_kind = 'preview') - AND created_by = 'anonymous' + AND created_by = 'anonymous' AND started_at > now() - interval '3 hours' - AND script_path LIKE $3 || '/%' - AND result @> ('{"s3":"' || $1 || '"}')::jsonb + AND script_path LIKE $3 || '/%' + AND result @> ('{"s3":"' || $1 || '"}')::jsonb )"#, - file_query.s3, - w_id, - path, - ) - .fetch_one(db) - .await? - .unwrap_or(false) - }; + file_query.s3, + w_id, + path, + ) + .fetch_one(db) + .await? + .unwrap_or(false) + }; - if !allowed { - Err(Error::BadRequest("File restricted".to_string())) - } else { - Ok(()) + if !allowed { + Err(Error::BadRequest("File restricted".to_string())) + } else { + Ok(()) + } } } #[cfg(feature = "parquet")] -#[derive(Deserialize)] -pub struct S3ObjectWithForceViewerAllowedS3Keys { +#[derive(Deserialize, Debug)] +struct AppS3FileQuery { + s3: String, + storage: Option, + sig: Option, + exp: Option, +} + +#[cfg(feature = "parquet")] +#[derive(Deserialize, Debug)] +struct AppS3FileQueryWithForceViewerAllowedS3Keys { #[serde(flatten)] - pub file_query: S3Object, + pub file_query: AppS3FileQuery, pub force_viewer_allowed_s3_keys: Option, } @@ -2117,7 +2119,7 @@ async fn download_s3_file_from_app( OptAuthed(opt_authed): OptAuthed, Extension(db): Extension, Path((w_id, path)): Path<(String, StripPath)>, - Query(mut query): Query, + Query(query): Query, ) -> Result { let path = path.to_path(); @@ -2136,7 +2138,7 @@ async fn download_s3_file_from_app( check_if_allowed_to_access_s3_file_from_app( &db, &opt_authed, - &mut query.file_query, + &query.file_query, &w_id, &path, &policy, @@ -2158,45 +2160,6 @@ async fn download_s3_file_from_app( .await } -#[cfg(not(feature = "parquet"))] -async fn load_s3_file_image_preview_from_app() -> Result<()> { - return Err(Error::BadRequest( - "This endpoint requires the parquet feature to be enabled".to_string(), - )); -} - -#[cfg(feature = "parquet")] -async fn load_s3_file_image_preview_from_app( - OptAuthed(opt_authed): OptAuthed, - Extension(db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, - Query(mut query): Query, -) -> Result { - let path = path.to_path(); - - let (on_behalf_authed, policy) = - get_on_behalf_authed_from_app(&db, &path, &w_id, &opt_authed, None).await?; - - check_if_allowed_to_access_s3_file_from_app( - &db, - &opt_authed, - &mut query, - &w_id, - &path, - &policy, - ) - .await?; - - load_image_preview_internal( - on_behalf_authed, - &db, - "", - &w_id, - LoadImagePreviewQuery { file_key: query.s3, storage: query.storage }, - ) - .await -} - fn get_on_behalf_of(policy: &Policy) -> Result<(String, String)> { let permissioned_as = policy .on_behalf_of diff --git a/frontend/src/lib/components/DisplayResult.svelte b/frontend/src/lib/components/DisplayResult.svelte index d9eecafc4d..a789b70bea 100644 --- a/frontend/src/lib/components/DisplayResult.svelte +++ b/frontend/src/lib/components/DisplayResult.svelte @@ -772,24 +772,30 @@ preview rendered {:else if result?.s3?.endsWith('.pdf')}
{/if} diff --git a/frontend/src/lib/components/apps/components/display/AppImage.svelte b/frontend/src/lib/components/apps/components/display/AppImage.svelte index c38c6fb1d2..1ac67dd8e6 100644 --- a/frontend/src/lib/components/apps/components/display/AppImage.svelte +++ b/frontend/src/lib/components/apps/components/display/AppImage.svelte @@ -46,7 +46,7 @@ let imageUrl: string | undefined = undefined - async function getS3Image(source: string | undefined, storage?: string) { + async function getS3Image(source: string | undefined, storage?: string, presigned?: string) { if (!source) return '' const appPathOrUser = defaultIfEmptyString( $appPath, @@ -63,14 +63,18 @@ params.append('force_viewer_allowed_s3_keys', JSON.stringify([forceViewerPolicies])) } - return `/api/w/${workspace}/apps_u/download_s3_file/${appPathOrUser}?${params.toString()}` + return `/api/w/${workspace}/apps_u/download_s3_file/${appPathOrUser}?${params.toString()}${presigned ? `&${presigned}` : ''}` } async function loadImage() { if (isPartialS3Object(resolvedConfig.source)) { - imageUrl = await getS3Image(resolvedConfig.source.s3, resolvedConfig.source.storage) + imageUrl = await getS3Image( + resolvedConfig.source.s3, + resolvedConfig.source.storage, + resolvedConfig.source.presigned + ) } else if (resolvedConfig.source && typeof resolvedConfig.source !== 'string') { - throw new Error('Invalid s3 object' + typeof resolvedConfig.source) + throw new Error('Invalid image object' + typeof resolvedConfig.source) } else if ( resolvedConfig.sourceKind === 's3 (workspace storage)' || resolvedConfig.source?.startsWith('s3://') diff --git a/frontend/src/lib/components/apps/editor/appUtilsS3.ts b/frontend/src/lib/components/apps/editor/appUtilsS3.ts index 60ca4f1e6b..4bef5ef6b7 100644 --- a/frontend/src/lib/components/apps/editor/appUtilsS3.ts +++ b/frontend/src/lib/components/apps/editor/appUtilsS3.ts @@ -83,7 +83,8 @@ export function computeS3FileInputPolicy(s3Config: any, app: App) { const partialS3ObjectSchema = z.object({ s3: z.string(), - storage: z.string().optional() + storage: z.string().optional(), + presigned: z.string().optional() }) export function isPartialS3Object(input: unknown): input is z.infer { @@ -91,24 +92,21 @@ export function isPartialS3Object(input: unknown): input is z.infer {#if !(resourceOnly && (fieldType !== 'object' || !format?.startsWith('resource-')))} @@ -258,9 +256,6 @@ {:then Module} { - dispatch('focus') - }} code={JSON.stringify(componentInput.value ?? { s3: '' }, null, 2)} bind:value={componentInput.value} /> diff --git a/frontend/src/lib/components/apps/editor/settingsPanel/inputEditor/StaticInputEditor.svelte b/frontend/src/lib/components/apps/editor/settingsPanel/inputEditor/StaticInputEditor.svelte index 1cbb90abd3..da0f6fbf88 100644 --- a/frontend/src/lib/components/apps/editor/settingsPanel/inputEditor/StaticInputEditor.svelte +++ b/frontend/src/lib/components/apps/editor/settingsPanel/inputEditor/StaticInputEditor.svelte @@ -10,7 +10,7 @@ import Toggle from '$lib/components/Toggle.svelte' import autosize from '$lib/autosize' import Button from '$lib/components/common/button/Button.svelte' - import { Settings } from 'lucide-svelte' + import { Loader2, Pipette, Settings } from 'lucide-svelte' import AgGridWizard from '$lib/components/wizards/AgGridWizard.svelte' import TableColumnWizard from '$lib/components/wizards/TableColumnWizard.svelte' import PlotlyWizard from '$lib/components/wizards/PlotlyWizard.svelte' @@ -23,6 +23,8 @@ import EditableSchemaDrawer from '$lib/components/schema/EditableSchemaDrawer.svelte' import AppPicker from '$lib/components/wizards/AppPicker.svelte' import JsonEditor from '$lib/components/JsonEditor.svelte' + import S3FilePicker from '$lib/components/S3FilePicker.svelte' + import FileUpload from '$lib/components/common/fileUpload/FileUpload.svelte' export let componentInput: StaticInput | undefined export let fieldType: InputType | undefined = undefined @@ -34,6 +36,9 @@ const { onchange } = getContext('AppViewerContext') + let s3FileUploadRawMode = false + let s3FilePicker: S3FilePicker | undefined = undefined + $: componentInput && onchange?.() @@ -136,7 +141,72 @@ {:else if fieldType === 'color'} {:else if fieldType === 'object' || fieldType == 'labeledselect'} - {#if format?.startsWith('resource-') && (componentInput.value == undefined || typeof componentInput.value == 'string')} + {#if format && format.split('-').length > 1 && format + .replace('resource-', '') + .replace('_', '') + .toLowerCase() == 's3object'} +
+ + {#if s3FileUploadRawMode} + {#await import('$lib/components/JsonEditor.svelte')} + + {:then Module} + + {/await} + {:else} + { + if (componentInput) { + componentInput.value = { + s3: evt.detail?.path ?? '', + filename: evt.detail?.filename ?? '' + } + s3FileUploadRawMode = true + } + }} + on:deletion={(evt) => { + if (componentInput) { + componentInput.value = { + s3: '' + } + } + }} + /> + {/if} + +
+ { + if (componentInput?.value?.s3) { + s3FileUploadRawMode = true + } + }} + bind:selectedFileKey={componentInput.value} + /> + {:else if format?.startsWith('resource-') && (componentInput.value == undefined || typeof componentInput.value == 'string')} { diff --git a/frontend/src/lib/components/common/fileDownload/FileDownload.svelte b/frontend/src/lib/components/common/fileDownload/FileDownload.svelte index afaaf9befd..ec94243179 100644 --- a/frontend/src/lib/components/common/fileDownload/FileDownload.svelte +++ b/frontend/src/lib/components/common/fileDownload/FileDownload.svelte @@ -17,7 +17,7 @@ duration-200 rounded-lg p-1 gap-2" appPath ? `/apps_u/download_s3_file/${appPath}` : '/job_helpers/download_s3_file' }?${appPath ? 's3' : 'file_key'}=${encodeURIComponent(s3object?.s3 ?? '')}${ s3object?.storage ? `&storage=${s3object.storage}` : '' - }`} + }${appPath && s3object?.presigned ? `&${s3object.presigned}` : ''}`} download={s3object?.s3.split('/').pop() ?? 'unnamed_download.file'} > diff --git a/frontend/src/lib/components/multiselect/MultiSelect.svelte b/frontend/src/lib/components/multiselect/MultiSelect.svelte index 5bf1a96ed3..1e4f00c66d 100644 --- a/frontend/src/lib/components/multiselect/MultiSelect.svelte +++ b/frontend/src/lib/components/multiselect/MultiSelect.svelte @@ -572,7 +572,7 @@ {/if} {/if} - {#if (searchText && noMatchingOptionsMsg) || options?.length > 0} + {#if allowUserOptions || (searchText && noMatchingOptionsMsg) || options?.length > 0}
0) { + $: if (portalRef && outerDiv && (allowUserOptions || items?.length > 0)) { tick().then(() => { moveOptionsToPortal() }) @@ -53,6 +53,7 @@ {#if !value || Array.isArray(value)}
key !== provider) ) + if (defaultModel) { + const currentDefaultModel = Object.values(aiProviders).find( + (p) => defaultModel && p.models.includes(defaultModel) + ) + if (!currentDefaultModel) { + defaultModel = undefined + } + } + if (codeCompletionModel) { + const currentCodeCompletionModel = Object.values(aiProviders).find( + (p) => codeCompletionModel && p.models.includes(codeCompletionModel) + ) + if (!currentCodeCompletionModel) { + codeCompletionModel = undefined + } + } } }} /> @@ -139,8 +155,8 @@ bind:value={aiProviders[provider].resource_path} on:change={() => { if ( - aiProviders[provider].resource_path && - aiProviders[provider].models.length === 0 && + aiProviders[provider]?.resource_path && + aiProviders[provider]?.models.length === 0 && AI_DEFAULT_MODELS[provider].length > 0 ) { aiProviders[provider].models = AI_DEFAULT_MODELS[provider].slice(0, 1) @@ -158,11 +174,11 @@
@@ -177,16 +193,18 @@

Settings

diff --git a/frontend/src/lib/components/workspaceSettings/StorageSettings.svelte b/frontend/src/lib/components/workspaceSettings/StorageSettings.svelte new file mode 100644 index 0000000000..51caaa1a42 --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/StorageSettings.svelte @@ -0,0 +1,252 @@ + + + + + + +
+
+
Workspace Object Storage (S3/Azure Blob)
+ + Connect your Windmill workspace to your S3 bucket or your Azure Blob storage to enable users + to read and write from S3 without having to have access to the credentials. + +
+
+{#if !$enterpriseLicense} + + Windmill S3 bucket browser will not work for buckets containing more than 20 files and uploads + are limited to files {'<'} 50MB. Consider upgrading to Windmill EE to use this feature with large + buckets. + +{:else} + + This setting is only for storage of large files allowing to upload files directly to object + storage using S3Object and use the wmill sdk to read and write large files backed by an object + storage. Large-scale log management and distributed dependency caching is under Instance object storage, set by the superadmins in the instance settings UI. + +{/if} +{#if s3ResourceSettings} +
+
+ + + + S3 + Azure Blob + AWS OIDC + Azure Workload Identity + +
+
+ + + + +
+
+ {#if s3ResourceSettings.resourceType == 's3'} +
+ + + + {#if s3ResourceSettings.publicResource === true} +
+ + + S3 resource public access is ON, which means that the entire content of the S3 bucket will + be accessible to all the users of this workspace regardless of whether they have access + the resource or not. Similarly, certain Windmill SDK endpoints can be used in scripts to + access the resource details, including public and private keys. + + {/if} +
+ {:else} +
+ + + + {#if s3ResourceSettings.publicResource === true} +
+ + object public access is ON, which means that the entire content of the object store will + be accessible to all the users of this workspace regardless of whether they have access + the resource or not. + + {/if} +
+ {/if} +
+
+ {#each s3ResourceSettings.secondaryStorage ?? [] as _, idx} +
+ s3ResourceSettings.secondaryStorage?.[idx]?.[0] || '', + (v) => { + if (s3ResourceSettings.secondaryStorage?.[idx]) { + s3ResourceSettings.secondaryStorage[idx][0] = v + } + } + } + placeholder="Storage name" + /> + + + + s3ResourceSettings.secondaryStorage?.[idx]?.[1].resourcePath || '', + (v) => { + if (s3ResourceSettings.secondaryStorage?.[idx]) { + s3ResourceSettings.secondaryStorage[idx][1].resourcePath = v + } + } + } + /> + + +
+ {/each} +
+ + + Secondary storage is a feature that allows you to read and write from storage that isn't + your main storage by specifying it in the s3 object as "secondary_storage" with the name + of it + +
+
+
+
+ +
+{/if} diff --git a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte index 2809592688..a672288ff2 100644 --- a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte @@ -10,7 +10,6 @@ import PageHeader from '$lib/components/PageHeader.svelte' import ResourcePicker from '$lib/components/ResourcePicker.svelte' import ScriptPicker from '$lib/components/ScriptPicker.svelte' - import S3FilePicker from '$lib/components/S3FilePicker.svelte' import Tooltip from '$lib/components/Tooltip.svelte' import WorkspaceUserSettings from '$lib/components/settings/WorkspaceUserSettings.svelte' @@ -46,7 +45,6 @@ import PremiumInfo from '$lib/components/settings/PremiumInfo.svelte' import Toggle from '$lib/components/Toggle.svelte' - import Portal from '$lib/components/Portal.svelte' import { fade } from 'svelte/transition' import ChangeWorkspaceName from '$lib/components/settings/ChangeWorkspaceName.svelte' @@ -54,14 +52,14 @@ import ChangeWorkspaceColor from '$lib/components/settings/ChangeWorkspaceColor.svelte' import { convertBackendSettingsToFrontendSettings, - convertFrontendToBackendSetting, type S3ResourceSettings } from '$lib/workspace_settings' import { base } from '$lib/base' import { hubPaths } from '$lib/hub' import Description from '$lib/components/Description.svelte' import ConnectionSection from '$lib/components/ConnectionSection.svelte' - import AiSettings from '$lib/components/workspaceSettings/AISettings.svelte' + import AISettings from '$lib/components/workspaceSettings/AISettings.svelte' + import StorageSettings from '$lib/components/workspaceSettings/StorageSettings.svelte' type GitSyncTypeMap = { scripts: boolean @@ -89,8 +87,6 @@ | 'user' | 'group' - let s3FileViewer: S3FilePicker - let slackInitialPath: string let slackScriptPath: string let teamsInitialPath: string @@ -212,18 +208,6 @@ } } - async function editWindmillLFSSettings(): Promise { - const large_file_storage = convertFrontendToBackendSetting(s3ResourceSettings) - await WorkspaceService.editLargeFileStorageConfig({ - workspace: $workspaceStore!, - requestBody: { - large_file_storage: large_file_storage - } - }) - console.log('Large file storage settings changed', large_file_storage) - sendUserToast(`Large file storage settings changed`) - } - async function editWindmillGitSyncSettings(): Promise { let alreadySeenResource: string[] = [] let repositories = gitSyncSettings.repositories.map((elmt) => { @@ -629,10 +613,6 @@ $: updateFromSearchTab($page.url.searchParams.get('tab')) - - - - {#if $userStore?.is_admin || $superadmin}
{:else if tab == 'ai'} - {:else if tab == 'windmill_lfs'} -
-
-
Workspace Object Storage (S3/Azure Blob)
- - Connect your Windmill workspace to your S3 bucket or your Azure Blob storage to enable - users to read and write from S3 without having to have access to the credentials. - -
-
- {#if !$enterpriseLicense} - - Windmill S3 bucket browser will not work for buckets containing more than 20 files and - uploads are limited to files {'<'} 50MB. Consider upgrading to Windmill EE to use this feature - with large buckets. - - {:else} - - This setting is only for storage of large files allowing to upload files directly to - object storage using S3Object and use the wmill sdk to read and write large files backed - by an object storage. Large-scale log management and distributed dependency caching is - under Instance object storage, set by the superadmins in the instance settings UI. - - {/if} - {#if s3ResourceSettings} -
-
- - S3 - Azure Blob - AWS OIDC - Azure Workload Identity - -
-
- - -
-
- {#if s3ResourceSettings.resourceType == 's3'} -
- - {#if s3ResourceSettings.publicResource === true} -
- - - S3 resource public access is ON, which means that the entire content of the S3 - bucket will be accessible to all the users of this workspace regardless of whether - they have access the resource or not. Similarly, certain Windmill SDK endpoints can - be used in scripts to access the resource details, including public and private - keys. - - {/if} -
- {:else} -
- - {#if s3ResourceSettings.publicResource === true} -
- - object public access is ON, which means that the entire content of the object store - will be accessible to all the users of this workspace regardless of whether they - have access the resource or not. - - {/if} -
- {/if} -
-
- {#each s3ResourceSettings.secondaryStorage ?? [] as secondaryStorage, idx} -
- - - - - -
- {/each} -
- - - Secondary storage is a feature that allows you to read and write from storage that - isn't your main storage by specifying it in the s3 object as "secondary_storage" - with the name of it - -
-
-
-
- -
- {/if} + {:else if tab == 'git_sync'}
diff --git a/python-client/wmill/wmill/client.py b/python-client/wmill/wmill/client.py index 6d29dc186f..bca007b438 100644 --- a/python-client/wmill/wmill/client.py +++ b/python-client/wmill/wmill/client.py @@ -586,9 +586,12 @@ class Windmill: raise Exception("Could not write file to S3") from e return S3Object(s3=response["file_key"]) - def sign_s3_objects(self, s3_objects: list[S3Object]) -> list[str]: + def sign_s3_objects(self, s3_objects: list[S3Object]) -> list[S3Object]: return self.post(f"/w/{self.workspace}/apps/sign_s3_objects", json={"s3_objects": s3_objects}).json() + def sign_s3_object(self, s3_object: S3Object) -> S3Object: + return self.post(f"/w/{self.workspace}/apps/sign_s3_objects", json={"s3_objects": [s3_object]}).json()[0] + def __boto3_connection_settings(self, s3_resource) -> Boto3ConnectionSettings: endpoint_url_prefix = "https://" if s3_resource["useSSL"] else "http://" return Boto3ConnectionSettings( @@ -978,7 +981,7 @@ def write_s3_file( @init_global_client -def sign_s3_objects(s3_objects: list[S3Object]) -> list[str]: +def sign_s3_objects(s3_objects: list[S3Object]) -> list[S3Object]: """ Sign S3 objects to be used by anonymous users in public apps Returns a list of signed s3 tokens @@ -986,6 +989,15 @@ def sign_s3_objects(s3_objects: list[S3Object]) -> list[str]: return _client.sign_s3_objects(s3_objects) +@init_global_client +def sign_s3_object(s3_object: S3Object) -> S3Object: + """ + Sign S3 object to be used by anonymous users in public apps + Returns a signed s3 object + """ + return _client.sign_s3_object(s3_object) + + @init_global_client def whoami() -> dict: """ diff --git a/python-client/wmill/wmill/s3_types.py b/python-client/wmill/wmill/s3_types.py index f633532591..b1736db9e9 100644 --- a/python-client/wmill/wmill/s3_types.py +++ b/python-client/wmill/wmill/s3_types.py @@ -1,6 +1,7 @@ class S3Object(dict): s3: str storage: str | None + presigned: str | None def __getattr__(self, attr): return self[attr] diff --git a/typescript-client/build.jsr.sh b/typescript-client/build.jsr.sh index 9da32e945e..06dae6605b 100755 --- a/typescript-client/build.jsr.sh +++ b/typescript-client/build.jsr.sh @@ -14,5 +14,5 @@ cp "${script_dirpath}/s3Types.ts" "${script_dirpath}/src/" echo "" >> "${script_dirpath}/src/index.ts" echo 'export type { S3Object, DenoS3LightClientSettings } from "./s3Types";' >> "${script_dirpath}/src/index.ts" echo "" >> "${script_dirpath}/src/index.ts" -echo 'export { type Base64, setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, getState, getIdToken, denoS3LightClientSettings, loadS3FileStream, loadS3File, writeS3File, signS3Objects, task, runScript, runScriptAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, Sql } from "./client";' >> "${script_dirpath}/src/index.ts" +echo 'export { type Base64, setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, getState, getIdToken, denoS3LightClientSettings, loadS3FileStream, loadS3File, writeS3File, signS3Objects, signS3Object, task, runScript, runScriptAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, Sql } from "./client";' >> "${script_dirpath}/src/index.ts" diff --git a/typescript-client/build.sh b/typescript-client/build.sh index 9f16e0f950..5f9cdbc4ec 100755 --- a/typescript-client/build.sh +++ b/typescript-client/build.sh @@ -39,4 +39,4 @@ cp "${script_dirpath}/s3Types.ts" "${script_dirpath}/src/" echo "" >> "${script_dirpath}/src/index.ts" echo 'export type { S3Object, DenoS3LightClientSettings } from "./s3Types";' >> "${script_dirpath}/src/index.ts" echo "" >> "${script_dirpath}/src/index.ts" -echo 'export { type Base64, setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, setProgress, getProgress, getState, getIdToken, denoS3LightClientSettings, loadS3FileStream, loadS3File, writeS3File, signS3Objects, task, runScript, runScriptAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, Sql } from "./client";' >> "${script_dirpath}/src/index.ts" +echo 'export { type Base64, setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, setProgress, getProgress, getState, getIdToken, denoS3LightClientSettings, loadS3FileStream, loadS3File, writeS3File, signS3Objects, signS3Object, task, runScript, runScriptAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, Sql } from "./client";' >> "${script_dirpath}/src/index.ts" diff --git a/typescript-client/client.ts b/typescript-client/client.ts index bd256e8186..258ea76215 100644 --- a/typescript-client/client.ts +++ b/typescript-client/client.ts @@ -774,9 +774,11 @@ export async function writeS3File( /** * Sign S3 objects to be used by anonymous users in public apps * @param s3objects s3 objects to sign - * @returns signed s3 tokens + * @returns signed s3 objects */ -export async function signS3Objects(s3objects: S3Object[]): Promise { +export async function signS3Objects( + s3objects: S3Object[] +): Promise { const signedKeys = await AppService.signS3Objects({ workspace: getWorkspace(), requestBody: { @@ -786,6 +788,16 @@ export async function signS3Objects(s3objects: S3Object[]): Promise { return signedKeys; } +/** + * Sign S3 object to be used by anonymous users in public apps + * @param s3object s3 object to sign + * @returns signed s3 object + */ +export async function signS3Object(s3object: S3Object): Promise { + const [signedObject] = await signS3Objects([s3object]); + return signedObject; +} + /** * Get URLs needed for resuming a flow after this step * @param approver approver name diff --git a/typescript-client/s3Types.ts b/typescript-client/s3Types.ts index 16641477d4..a46248d778 100644 --- a/typescript-client/s3Types.ts +++ b/typescript-client/s3Types.ts @@ -1,6 +1,7 @@ export type S3Object = { s3: string; storage?: string; + presigned?: string; }; export type DenoS3LightClientSettings = {