kalugin66/windmill
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: migrate s3 client to object_store (#3116)

Browse Source 
* feat: migrate s3 client to object_store

* remove multipart from open API

* Remove multipart in favor of a single stream

* progress report

* add progress reader on server side

* small nit fix

* fix read chunk

* Fix TS and python SDK

* Fix download button

* Fix download button object viewer

* fix list

* Better errors

* export loadS3FileContent

* revert changes SDK

* fix browser

* small file list unavailable fix

* Old endpoints throws informative error messages

* Typescript SDK uses raw fetch

* update python SDK

* Error if uploaded file > 50Mb

* revert python SDL changes

* Update python SDK method docs

---------

Co-authored-by: Ruben Fiszel <ruben@windmill.dev>
Co-authored-by: Ruben Fiszel <ruben@rubenfiszel.com>
This commit is contained in:
Guillaume Bouvignies
2024-02-01 16:03:30 +01:00
committed by GitHub
parent 8a8a30c5ec
commit 5dabe22935
18 changed files with 1162 additions and 2086 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
python-client/tests/wmill_client_test.py
View File

@@ -15,6 +15,7 @@ class TestStringMethods(unittest.TestCase):
os.environ["WM_TOKEN"] = self._token
os.environ["BASE_INTERNAL_URL"] = self._host
@unittest.skip("skipping")
def test_duckdb_connection_settings(self):
settings = wmill.duckdb_connection_settings(self._resource_path)
self.assertIsNotNone(settings)
@@ -35,6 +36,7 @@ SET s3_secret_access_key='80yMndIMcyXwEujxVNINQbf0tBlIzRaLPyM2m1n4';
settings = wmill.polars_connection_settings(self._resource_path)
print(settings)
@unittest.skip("skipping")
def test_polars_connection_settings(self):
settings = wmill.polars_connection_settings(self._resource_path)
s3fs_args_expected = {
@@ -59,6 +61,7 @@ SET s3_secret_access_key='80yMndIMcyXwEujxVNINQbf0tBlIzRaLPyM2m1n4';
)
self.assertEqual(settings.polars_cloud_options, polars_cloud_options_expected)
@unittest.skip("skipping")
def test_boto3_connection_settings(self):
settings = wmill.boto3_connection_settings(self._resource_path)
expected_settings = {
@@ -72,6 +75,24 @@ SET s3_secret_access_key='80yMndIMcyXwEujxVNINQbf0tBlIzRaLPyM2m1n4';
self.assertEqual(settings["endpoint_url"], "http://localhost:9000")
self.assertEqual(settings.endpoint_url, "http://localhost:9000")
@unittest.skip("skipping")
def test_download_s3_file(self):
with wmill.load_s3_file_reader(S3Object(s3="region.csv")) as file_content, open(
"region.csv", "wb"
) as output_file:
output_file.write(file_content.read())
@unittest.skip("skipping")
def test_download_s3_file_content(self):
file_content = wmill.load_s3_file(S3Object(s3="region.csv"))
print(file_content)
@unittest.skip("skipping")
def test_upload_s3_file(self):
with open("region.csv", "rb") as file_content:
file_key = wmill.write_s3_file(S3Object(s3="region.csv"), file_content)
print(file_key)
if __name__ == "__main__":
unittest.main()