fix: S3 SDK nits + Presigned S3 Public URL function (#7342)

* export S3Object + URI / Record in TS SDK

* stash getS3SignedPublicUrls

* getPresignedS3PublicUrls in TS client

* update python client for get_presigned_s3_public_urls

python-client/wmill/wmill/client.py

@@ -826,6 +826,81 @@ class Windmill:
             json={"s3_objects": [s3_object]},
         ).json()[0]
 
+    def get_presigned_s3_public_urls(
+        self,
+        s3_objects: list[S3Object | str],
+        base_url: str | None = None,
+    ) -> list[str]:
+        """
+        Generate presigned public URLs for an array of S3 objects.
+        If an S3 object is not signed yet, it will be signed first.
+
+        Args:
+            s3_objects: List of S3 objects to sign
+            base_url: Optional base URL for the presigned URLs (defaults to WM_BASE_URL)
+
+        Returns:
+            List of signed public URLs
+
+        Example:
+            >>> s3_objs = [S3Object(s3="/path/to/file1.txt"), S3Object(s3="/path/to/file2.txt")]
+            >>> urls = client.get_presigned_s3_public_urls(s3_objs)
+        """
+        base_url = base_url or self._get_public_base_url()
+
+        s3_objs = [parse_s3_object(s3_obj) for s3_obj in s3_objects]
+
+        # Sign all S3 objects that need to be signed in one go
+        s3_objs_to_sign: list[tuple[S3Object, int]] = [
+            (s3_obj, index)
+            for index, s3_obj in enumerate(s3_objs)
+            if s3_obj.get("presigned") is None
+        ]
+
+        if s3_objs_to_sign:
+            signed_s3_objs = self.sign_s3_objects(
+                [s3_obj for s3_obj, _ in s3_objs_to_sign]
+            )
+            for i, (_, original_index) in enumerate(s3_objs_to_sign):
+                s3_objs[original_index] = parse_s3_object(signed_s3_objs[i])
+
+        signed_urls: list[str] = []
+        for s3_obj in s3_objs:
+            s3 = s3_obj.get("s3", "")
+            presigned = s3_obj.get("presigned", "")
+            storage = s3_obj.get("storage", "_default_")
+            signed_url = f"{base_url}/api/w/{self.workspace}/s3_proxy/{storage}/{s3}?{presigned}"
+            signed_urls.append(signed_url)
+
+        return signed_urls
+
+    def get_presigned_s3_public_url(
+        self,
+        s3_object: S3Object | str,
+        base_url: str | None = None,
+    ) -> str:
+        """
+        Generate a presigned public URL for an S3 object.
+        If the S3 object is not signed yet, it will be signed first.
+
+        Args:
+            s3_object: S3 object to sign
+            base_url: Optional base URL for the presigned URL (defaults to WM_BASE_URL)
+
+        Returns:
+            Signed public URL
+
+        Example:
+            >>> s3_obj = S3Object(s3="/path/to/file.txt")
+            >>> url = client.get_presigned_s3_public_url(s3_obj)
+        """
+        urls = self.get_presigned_s3_public_urls([s3_object], base_url)
+        return urls[0]
+
+    def _get_public_base_url(self) -> str:
+        """Get the public base URL from environment or default to localhost"""
+        return os.environ.get("WM_BASE_URL", "http://localhost:3000")
+
     def __boto3_connection_settings(self, s3_resource) -> Boto3ConnectionSettings:
         endpoint_url_prefix = "https://" if s3_resource["useSSL"] else "http://"
         return Boto3ConnectionSettings(
@@ -1283,6 +1358,56 @@ def sign_s3_object(s3_object: S3Object| str) -> S3Object:
     return _client.sign_s3_object(s3_object)
 
 
+@init_global_client
+def get_presigned_s3_public_urls(
+    s3_objects: list[S3Object | str],
+    base_url: str | None = None,
+) -> list[str]:
+    """
+    Generate presigned public URLs for an array of S3 objects.
+    If an S3 object is not signed yet, it will be signed first.
+
+    Args:
+        s3_objects: List of S3 objects to sign
+        base_url: Optional base URL for the presigned URLs (defaults to WM_BASE_URL)
+
+    Returns:
+        List of signed public URLs
+
+    Example:
+        >>> import wmill
+        >>> from wmill import S3Object
+        >>> s3_objs = [S3Object(s3="/path/to/file1.txt"), S3Object(s3="/path/to/file2.txt")]
+        >>> urls = wmill.get_presigned_s3_public_urls(s3_objs)
+    """
+    return _client.get_presigned_s3_public_urls(s3_objects, base_url)
+
+
+@init_global_client
+def get_presigned_s3_public_url(
+    s3_object: S3Object | str,
+    base_url: str | None = None,
+) -> str:
+    """
+    Generate a presigned public URL for an S3 object.
+    If the S3 object is not signed yet, it will be signed first.
+
+    Args:
+        s3_object: S3 object to sign
+        base_url: Optional base URL for the presigned URL (defaults to WM_BASE_URL)
+
+    Returns:
+        Signed public URL
+
+    Example:
+        >>> import wmill
+        >>> from wmill import S3Object
+        >>> s3_obj = S3Object(s3="/path/to/file.txt")
+        >>> url = wmill.get_presigned_s3_public_url(s3_obj)
+    """
+    return _client.get_presigned_s3_public_url(s3_object, base_url)
+
+
 @init_global_client
 def whoami() -> dict:
     """

typescript-client/build.jsr.sh

@@ -13,8 +13,8 @@ cp "${script_dirpath}/client.ts" "${script_dirpath}/src/"
 cp "${script_dirpath}/s3Types.ts" "${script_dirpath}/src/"
 cp "${script_dirpath}/sqlUtils.ts" "${script_dirpath}/src/"
 echo "" >> "${script_dirpath}/src/index.ts"
-echo 'export type { S3Object, DenoS3LightClientSettings } from "./s3Types";' >> "${script_dirpath}/src/index.ts"
+echo 'export type { DenoS3LightClientSettings } from "./s3Types";' >> "${script_dirpath}/src/index.ts"
 echo "" >> "${script_dirpath}/src/index.ts"
-echo 'export { type Base64, setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, setProgress, getProgress, getState, getIdToken, denoS3LightClientSettings, loadS3FileStream, loadS3File, writeS3File, signS3Objects, signS3Object, task, runScript, runScriptAsync, runScriptByPath, runScriptByHash, runScriptByPathAsync, runScriptByHashAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, Sql, requestInteractiveTeamsApproval, appendToResultStream, streamResult, datatable, ducklake, type SqlTemplateFunction } from "./client";' >> "${script_dirpath}/src/index.ts"
+echo 'export { type Base64, setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, setProgress, getProgress, getState, getIdToken, denoS3LightClientSettings, loadS3FileStream, loadS3File, writeS3File, signS3Objects, signS3Object, getPresignedS3PublicUrls, getPresignedS3PublicUrl, task, runScript, runScriptAsync, runScriptByPath, runScriptByHash, runScriptByPathAsync, runScriptByHashAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, Sql, requestInteractiveTeamsApproval, appendToResultStream, streamResult, datatable, ducklake, type SqlTemplateFunction, type S3Object, type S3ObjectRecord, type S3ObjectURI } from "./client";' >> "${script_dirpath}/src/index.ts"
 
 

typescript-client/build.sh

@@ -38,6 +38,6 @@ cp "${script_dirpath}/client.ts" "${script_dirpath}/src/"
 cp "${script_dirpath}/s3Types.ts" "${script_dirpath}/src/"
 cp "${script_dirpath}/sqlUtils.ts" "${script_dirpath}/src/"
 echo "" >> "${script_dirpath}/src/index.ts"
-echo 'export type { S3Object, DenoS3LightClientSettings } from "./s3Types";' >> "${script_dirpath}/src/index.ts"
+echo 'export type { DenoS3LightClientSettings } from "./s3Types";' >> "${script_dirpath}/src/index.ts"
 echo "" >> "${script_dirpath}/src/index.ts"
-echo 'export { type Base64, setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, setProgress, getProgress, getState, getIdToken, denoS3LightClientSettings, loadS3FileStream, loadS3File, writeS3File, signS3Objects, signS3Object, task, runScript, runScriptAsync, runScriptByPath, runScriptByHash, runScriptByPathAsync, runScriptByHashAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, Sql, requestInteractiveTeamsApproval, appendToResultStream, streamResult, datatable, ducklake, type SqlTemplateFunction } from "./client";' >> "${script_dirpath}/src/index.ts"
+echo 'export { type Base64, setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, setProgress, getProgress, getState, getIdToken, denoS3LightClientSettings, loadS3FileStream, loadS3File, writeS3File, signS3Objects, signS3Object, getPresignedS3PublicUrls, getPresignedS3PublicUrl, task, runScript, runScriptAsync, runScriptByPath, runScriptByHash, runScriptByPathAsync, runScriptByHashAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, Sql, requestInteractiveTeamsApproval, appendToResultStream, streamResult, datatable, ducklake, type SqlTemplateFunction, type S3Object, type S3ObjectRecord, type S3ObjectURI } from "./client";' >> "${script_dirpath}/src/index.ts"

typescript-client/client.d.ts

@@ -14,6 +14,11 @@ export {
   UserService,
   WorkspaceService,
 } from "./index";
+export {
+  type S3Object,
+  type S3ObjectRecord,
+  type S3ObjectURI,
+} from "./s3Types";
 export { datatable, ducklake, type SqlTemplateFunction } from "./sqlUtils";
 export type Sql = string;
 export type Email = string;
@@ -180,6 +185,43 @@ export declare function writeS3File(
   fileContent: string | Blob,
   s3ResourcePath?: string | undefined
 ): Promise<S3Object>;
+
+/**
+ * Sign S3 objects to be used by anonymous users in public apps
+ * @param s3objects s3 objects to sign
+ * @returns signed s3 objects
+ */
+export declare function signS3Objects(
+  s3objects: S3Object[]
+): Promise<S3Object[]>;
+/**
+ * Sign S3 object to be used by anonymous users in public apps
+ * @param s3object s3 object to sign
+ * @returns signed s3 object
+ */
+export declare function signS3Object(s3object: S3Object): Promise<S3Object>;
+
+/**
+ * Generate a presigned public URL for an array of S3 objects.
+ * If an S3 object is not signed yet, it will be signed first.
+ * @param s3Objects s3 objects to sign
+ * @returns list of signed public URLs
+ */
+export declare function getPresignedS3PublicUrls(
+  s3Objects: S3Object[],
+  { baseUrl }: { baseUrl?: string }
+): Promise<string[]>;
+
+/**
+ * Generate a presigned public URL for an S3 object. If the S3 object is not signed yet, it will be signed first.
+ * @param s3Object s3 object to sign
+ * @returns signed public URL
+ */
+export declare function getPresignedS3PublicUrl(
+  s3Objects: S3Object,
+  { baseUrl }: { baseUrl?: string }
+): Promise<string>;
+
 /**
  * Get URLs needed for resuming a flow after this step
  * @param approver approver name

typescript-client/client.ts

@@ -17,6 +17,11 @@ import {
   type S3Object,
 } from "./s3Types";
 
+export {
+  type S3Object,
+  type S3ObjectRecord,
+  type S3ObjectURI,
+} from "./s3Types";
 export { datatable, ducklake, type SqlTemplateFunction } from "./sqlUtils";
 
 export {
@@ -60,6 +65,10 @@ export function setClient(token?: string, baseUrl?: string) {
   OpenAPI.BASE = baseUrl + "/api";
 }
 
+function getPublicBaseUrl(): string {
+  return getEnv("WM_BASE_URL") ?? "http://localhost:3000";
+}
+
 const getEnv = (key: string) => {
   if (typeof window === "undefined") {
     // node
@@ -900,7 +909,6 @@ export async function signS3Objects(
   });
   return signedKeys;
 }
-
 /**
  * Sign S3 object to be used by anonymous users in public apps
  * @param s3object s3 object to sign
@@ -911,6 +919,56 @@ export async function signS3Object(s3object: S3Object): Promise<S3Object> {
   return signedObject;
 }
 
+/**
+ * Generate a presigned public URL for an array of S3 objects.
+ * If an S3 object is not signed yet, it will be signed first.
+ * @param s3Objects s3 objects to sign
+ * @returns list of signed public URLs
+ */
+export async function getPresignedS3PublicUrls(
+  s3Objects: S3Object[],
+  { baseUrl }: { baseUrl?: string } = {}
+): Promise<string[]> {
+  baseUrl ??= getPublicBaseUrl();
+
+  const s3Objs = s3Objects.map(parseS3Object);
+
+  // Sign all S3 objects that need to be signed in one go
+  const s3ObjsToSign: (readonly [S3ObjectRecord, number])[] = s3Objs
+    .map((s3Obj, index) => [s3Obj, index] as const)
+    .filter(([s3Obj, _]) => s3Obj.presigned === undefined);
+  if (s3ObjsToSign.length > 0) {
+    const signedS3Objs = await signS3Objects(
+      s3ObjsToSign.map(([s3Obj, _]) => s3Obj)
+    );
+    for (let i = 0; i < s3ObjsToSign.length; i++) {
+      const [_, originalIndex] = s3ObjsToSign[i];
+      s3Objs[originalIndex] = parseS3Object(signedS3Objs[i]);
+    }
+  }
+
+  const signedUrls: string[] = [];
+  for (const s3Obj of s3Objs) {
+    const { s3, presigned, storage = "_default_" } = s3Obj;
+    const signedUrl = `${baseUrl}/api/w/${getWorkspace()}/s3_proxy/${storage}/${s3}?${presigned}`;
+    signedUrls.push(signedUrl);
+  }
+  return signedUrls;
+}
+
+/**
+ * Generate a presigned public URL for an S3 object. If the S3 object is not signed yet, it will be signed first.
+ * @param s3Object s3 object to sign
+ * @returns signed public URL
+ */
+export async function getPresignedS3PublicUrl(
+  s3Objects: S3Object,
+  { baseUrl }: { baseUrl?: string } = {}
+): Promise<string> {
+  const [s3Object] = await getPresignedS3PublicUrls([s3Objects], { baseUrl });
+  return s3Object;
+}
+
 /**
  * Get URLs needed for resuming a flow after this step
  * @param approver approver name
@@ -948,7 +1006,10 @@ export function getResumeEndpoints(approver?: string): Promise<{
  * @param expiresIn Optional number of seconds until the token expires
  * @returns jwt token
  */
-export async function getIdToken(audience: string, expiresIn?: number): Promise<string> {
+export async function getIdToken(
+  audience: string,
+  expiresIn?: number
+): Promise<string> {
   const workspace = getWorkspace();
   return await OidcService.getOidcToken({
     workspace,