feat: S3 objects are now typed in Python and TS SDK (#2878)

* feat: S3 objects are now typed in Python and TS SDK

* fix ts and python SDK after testing

* cleanup

python-client/wmill/wmill/__init__.py

@@ -1 +1,2 @@
 from .client import *
+from .s3_types import *

python-client/wmill/wmill/client.py

@@ -13,6 +13,8 @@ from typing import Dict, Any, Union, Literal
 
 import httpx
 
+from .s3_types import Boto3ConnectionSettings, DuckDbConnectionSettings, PolarsConnectionSettings
+
 _client: "Windmill | None" = None
 
 logger = logging.getLogger("windmill_client")
@@ -318,16 +320,17 @@ class Windmill:
         self,
         s3_resource_path: str = "",
         none_if_undefined: bool = False,
-    ) -> Union[str, None]:
+    ) -> DuckDbConnectionSettings | None:
         """
         Convenient helpers that takes an S3 resource as input and returns the settings necessary to
         initiate an S3 connection from DuckDB
         """
         try:
-            return self.post(
+            raw_obj = self.post(
                 f"/w/{self.workspace}/job_helpers/v2/duckdb_connection_settings",
                 json={} if s3_resource_path == "" else {"s3_resource_path": s3_resource_path},
             ).json()
+            return DuckDbConnectionSettings(raw_obj)
         except JSONDecodeError as e:
             if none_if_undefined:
                 return None
@@ -337,16 +340,17 @@ class Windmill:
         self,
         s3_resource_path: str = "",
         none_if_undefined: bool = False,
-    ) -> Any:
+    ) -> PolarsConnectionSettings | None:
         """
         Convenient helpers that takes an S3 resource as input and returns the settings necessary to
         initiate an S3 connection from Polars
         """
         try:
-            return self.post(
+            raw_obj = self.post(
                 f"/w/{self.workspace}/job_helpers/v2/polars_connection_settings",
                 json={} if s3_resource_path == "" else {"s3_resource_path": s3_resource_path},
             ).json()
+            return PolarsConnectionSettings(raw_obj)
         except JSONDecodeError as e:
             if none_if_undefined:
                 return None
@@ -356,16 +360,28 @@ class Windmill:
         self,
         s3_resource_path: str = "",
         none_if_undefined: bool = False,
-    ) -> Any:
+    ) -> Boto3ConnectionSettings | None:
         """
         Convenient helpers that takes an S3 resource as input and returns the settings necessary to
         initiate an S3 connection using boto3
         """
         try:
-            return self.post(
-                f"/w/{self.workspace}/job_helpers/v2/boto3_connection_settings",
+            s3_resource = self.post(
+                f"/w/{self.workspace}/job_helpers/v2/s3_resource_info",
                 json={} if s3_resource_path == "" else {"s3_resource_path": s3_resource_path},
             ).json()
+            endpoint_url_prefix = "https://" if s3_resource["useSSL"] else "http://"
+            boto3_settings = Boto3ConnectionSettings(
+                {
+                    "endpoint_url": "{}{}".format(endpoint_url_prefix, s3_resource["endPoint"]),
+                    "region_name": s3_resource["region"],
+                    "use_ssl": s3_resource["useSSL"],
+                    "aws_access_key_id": s3_resource["accessKey"],
+                    "aws_secret_access_key": s3_resource["secretKey"],
+                    # no need for path_style here as boto3 is clever enough to determine which one to use
+                }
+            )
+            return boto3_settings
         except JSONDecodeError as e:
             if none_if_undefined:
                 return None
@@ -574,7 +590,9 @@ def get_result(job_id: str, assert_result_is_not_none=True) -> Dict[str, Any]:
 
 
 @init_global_client
-def duckdb_connection_settings(s3_resource_path: str = "", none_if_undefined: bool = False) -> Union[str, None]:
+def duckdb_connection_settings(
+    s3_resource_path: str = "", none_if_undefined: bool = False
+) -> DuckDbConnectionSettings | None:
     """
     Convenient helpers that takes an S3 resource as input and returns the settings necessary to
     initiate an S3 connection from DuckDB
@@ -585,7 +603,9 @@ def duckdb_connection_settings(s3_resource_path: str = "", none_if_undefined: bo
 
 
 @init_global_client
-def polars_connection_settings(s3_resource_path: str = "", none_if_undefined: bool = False) -> Any:
+def polars_connection_settings(
+    s3_resource_path: str = "", none_if_undefined: bool = False
+) -> PolarsConnectionSettings | None:
     """
     Convenient helpers that takes an S3 resource as input and returns the settings necessary to
     initiate an S3 connection from Polars
@@ -594,7 +614,9 @@ def polars_connection_settings(s3_resource_path: str = "", none_if_undefined: bo
 
 
 @init_global_client
-def boto3_connection_settings(s3_resource_path: str = "", none_if_undefined: bool = False) -> Any:
+def boto3_connection_settings(
+    s3_resource_path: str = "", none_if_undefined: bool = False
+) -> Boto3ConnectionSettings | None:
     """
     Convenient helpers that takes an S3 resource as input and returns the settings necessary to
     initiate an S3 connection using boto3

python-client/wmill/wmill/s3_types.py

@@ -0,0 +1,61 @@
+class S3Object(dict):
+    s3: str
+
+    def __getattr__(self, attr):
+        return self[attr]
+
+
+class S3FsClientKwargs(dict):
+    region_name: str
+
+    def __getattr__(self, attr):
+        return self[attr]
+
+
+class S3FsArgs(dict):
+    endpoint_url: str
+    key: str
+    secret: str
+    use_ssl: bool
+    cache_regions: bool
+    client_kwargs: S3FsClientKwargs
+
+    def __getattr__(self, attr):
+        return self[attr]
+
+
+class PolarsCloudOptions(dict):
+    aws_endpoint_url: str
+    aws_access_key_id: str
+    aws_secret_access_key: str
+    aws_region: bool
+    aws_allow_http: bool
+
+    def __getattr__(self, attr):
+        return self[attr]
+
+
+class PolarsConnectionSettings(dict):
+    s3fs_args: S3FsArgs
+    polars_cloud_options: PolarsCloudOptions
+
+    def __getattr__(self, attr):
+        return self[attr]
+
+
+class Boto3ConnectionSettings(dict):
+    endpoint_url: str
+    region_name: str
+    use_ssl: bool
+    aws_access_key_id: str
+    aws_secret_access_key: str
+
+    def __getattr__(self, attr):
+        return self[attr]
+
+
+class DuckDbConnectionSettings(dict):
+    connection_settings_str: str
+
+    def __getattr__(self, attr):
+        return self[attr]