From cd48fdce2e1930639da20abe997a5e7cd0b3ed5f Mon Sep 17 00:00:00 2001 From: Guillaume Bouvignies Date: Fri, 22 Dec 2023 15:02:16 +0100 Subject: [PATCH] feat: move S3 file within bucket (#2913) --- backend/windmill-api/openapi-deref.yaml | 27 +++++++ backend/windmill-api/openapi.yaml | 25 ++++++ backend/windmill-api/src/job_helpers.rs | 49 ++++++++++++ .../src/lib/components/S3FilePicker.svelte | 79 +++++++++++++++++-- .../ConfirmationModal.svelte | 27 +++++-- 5 files changed, 195 insertions(+), 12 deletions(-) diff --git a/backend/windmill-api/openapi-deref.yaml b/backend/windmill-api/openapi-deref.yaml index 500b96e459..34f42f59b1 100644 --- a/backend/windmill-api/openapi-deref.yaml +++ b/backend/windmill-api/openapi-deref.yaml @@ -10211,6 +10211,33 @@ paths: content: application/json: schema: {} + /w/{workspace}/job_helpers/move_s3_file: + get: + summary: Move a S3 file from one path to the other within the same bucket + operationId: moveS3File + tags: + - helpers + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: src_file_key + in: query + required: true + schema: + type: string + - name: dest_file_key + in: query + required: true + schema: + type: string + responses: + '200': + description: Confirmation + content: + application/json: + schema: {} /w/{workspace}/job_helpers/multipart_upload_s3_file: post: summary: Upload file to S3 bucket using multipart upload diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 6e5cf03567..a2418b943c 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -6833,6 +6833,31 @@ paths: application/json: schema: {} + /w/{workspace}/job_helpers/move_s3_file: + get: + summary: Move a S3 file from one path to the other within the same bucket + operationId: moveS3File + tags: + - helpers + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: src_file_key + in: query + required: true + schema: + type: string + - name: dest_file_key + in: query + required: true + schema: + type: string + responses: + "200": + description: Confirmation + content: + application/json: + schema: {} + /w/{workspace}/job_helpers/multipart_upload_s3_file: post: summary: Upload file to S3 bucket using multipart upload diff --git a/backend/windmill-api/src/job_helpers.rs b/backend/windmill-api/src/job_helpers.rs index e914299e53..f0cc5023b3 100644 --- a/backend/windmill-api/src/job_helpers.rs +++ b/backend/windmill-api/src/job_helpers.rs @@ -80,6 +80,7 @@ pub fn workspaced_service() -> Router { "/delete_s3_file", delete(delete_s3_file).layer(cors.clone()), ) + .route("/move_s3_file", get(move_s3_file).layer(cors.clone())) .route( "/multipart_upload_s3_file", post(multipart_upload_s3_file).layer(cors.clone()), @@ -710,6 +711,54 @@ async fn delete_s3_file( return Ok(Json(())); } +#[derive(Deserialize)] +struct MoveS3FileQuery { + pub src_file_key: String, + pub dest_file_key: String, +} + +async fn move_s3_file( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Tokened { token }: Tokened, + Path(w_id): Path, + Query(query): Query, +) -> error::JsonResult<()> { + let s3_resource_opt = get_workspace_s3_resource(&authed, &user_db, &db, &token, &w_id).await?; + + let s3_resource = s3_resource_opt.ok_or(error::Error::InternalErr( + "No files storage resource defined at the workspace level".to_string(), + ))?; + let s3_client = build_s3_client(&s3_resource); + + let s3_bucket = s3_resource.bucket.clone(); + let source_uri = format!("{}/{}", s3_bucket, query.src_file_key); + s3_client + .copy_object() + .copy_source(&source_uri) + .bucket(&s3_bucket) + .key(&query.dest_file_key) + .send() + .await + .map_err(|err| { + tracing::error!("{:?}", err); + error::Error::InternalErr(err.to_string()) + })?; + + s3_client + .delete_object() + .bucket(&s3_bucket) + .key(&query.src_file_key) + .send() + .await + .map_err(|err| { + tracing::error!("{:?}", err); + error::Error::InternalErr(err.to_string()) + })?; + return Ok(Json(())); +} + #[derive(Deserialize)] struct UploadFileQuery { pub file_key: String, diff --git a/frontend/src/lib/components/S3FilePicker.svelte b/frontend/src/lib/components/S3FilePicker.svelte index 18b23a0327..be5b2c6c9d 100644 --- a/frontend/src/lib/components/S3FilePicker.svelte +++ b/frontend/src/lib/components/S3FilePicker.svelte @@ -7,7 +7,8 @@ Loader2, Download, Trash, - FileUp + FileUp, + MoveRight } from 'lucide-svelte' import { workspaceStore } from '$lib/stores' import { HelpersService, type UploadFilePart } from '$lib/gen' @@ -24,6 +25,10 @@ let deletionModalOpen = false let fileDeletionInProgress = false + let moveModalOpen = false + let moveDestKey: string | undefined = undefined + let fileMoveInProgress = false + let uploadModalOpen = false let fileToUpload: File | undefined = undefined let fileToUploadKey: string | undefined = undefined @@ -89,6 +94,8 @@ maxKeys: 1000, // fixed pages of 1000 files for now marker: paginationMarker }) + allFilesByKey = {} + displayedFileKeys = [] for (let file_path of availableFiles.windmill_large_files) { let split_path = file_path.s3.split('/') let parent_path: string | undefined = undefined @@ -210,8 +217,10 @@ }) } finally { fileDeletionInProgress = false + deletionModalOpen = false } sendUserToast(`${fileKey} deleted from S3 bucket`) + selectedFileKey = { s3: '' } const idx = displayedFileKeys.indexOf(fileKey) if (idx >= 0) { displayedFileKeys.splice(idx, 1) @@ -220,6 +229,27 @@ delete allFilesByKey[fileKey] } + async function moveS3File(srcFileKey: string | undefined, destFileKey: string | undefined) { + fileMoveInProgress = true + if (srcFileKey === undefined || emptyString(destFileKey)) { + return + } + try { + await HelpersService.moveS3File({ + workspace: $workspaceStore!, + srcFileKey: srcFileKey, + destFileKey: destFileKey! + }) + } finally { + fileMoveInProgress = false + moveModalOpen = false + } + sendUserToast(`${srcFileKey} moved to ${destFileKey}`) + selectedFileKey = { s3: destFileKey! } + await loadFiles() + await loadFileMetadataPlusPreviewAsync(selectedFileKey.s3) + } + async function uploadFileToS3() { fileUploadErrorMsg = undefined if (fileToUpload === undefined || fileToUploadKey === undefined) { @@ -497,6 +527,17 @@ startIcon={{ icon: Download }} iconOnly={true} /> + Upload File -