feat: assets as a primary concept (#6125)

* assets migration

* parse assets (duckdb)

* iterate on assets

* S3 object Preview

* remove pagination

* filterText

* better occurence list

* tweak

* assets in JobPreview

* clone impl

* AssetsDetectedBadge

* improve DbManagerButton + asset dropdown button

* edit resource btn

* warning when incorrect resource

* +Resource in DuckDB

* +S3 Object editor bar

* nit fix rename

* flow asset badge

* More Generic OnChange

* Highlight assets used in modules

* Show occurence count in flow

* Better UX, avoid moving parts

* nit

* Asset nodes

* move to dedicated Asset ctx

* fix layoutNodes not handling first assetsMap

* explore asset btn in flow asset node

* correct offset

* single computeAssetNodes function

* Fix y positioning of nodes with assets

* resource editor

* write mode node (ui)

* accessType in ctx + fix insert button positioning

* right positioning when mixing read and write nodes

* right positioning when mixing R and W assets

* Better layout fix algorithm

* listAssetsByUsage and asset nodes on transitive usages

* refactor + remove linkAssets

* Refactor to allow for custom R/W modes

* AssetsDropdownButton in flow script editor

* R/W/RW selection and changes node pos in flow

* layoutNodes doesnt need recompute now

* fix wrong assumption that nodes recompute when assets change

* r/w/rw multi toggle

* MultiToggle cool animation + clearable

* rename + 1px nit

* remove mini toggle button group, use ToggleButtonGroup

* Combinator parser that detects R / W asset context

* nit fix missing flex-1

* missing order by

* better ui indication for access type

* special x offset case when only one asset node for clarity

* parse getResource in TS with swc ecma parser

* support load and write s3 detection in TS

* Python asset parser

* support wmill api calls without special $res: or s3:// syntax

* detect out of context asset uris python

* do not use access type override when not ambiguous in flow graph

* parse_assets match case in rust

* AsRef<str> refactor

* From impl

* Save flow assets

* Save script asset usages + fixes + save fallback access types

* asset sub icon

* max total asset node width to avoid overlap

* small refactor

* don't parse comments in duckdb assets

* fix assets clearing on parse error

* fix script asset save in wrong place

* load initial asset fallback access types

* support variables

* ui fixes

* Support S3Object as URI in TS client

* support new syntax in python client

* Support +S3Object in EditorBar for TS and python

* Reduce resource requests in assets page

* import windmill client when necessary

* update s3Types.d.ts

* nit fix

* Show input resources and s3 objects as assets

* improve asset icons

* DarkModeObserver refactor

* asset page tabs

* Moved resource variables and s3object pages to assets tabs

* fetch resource usages

* Get variables usages

* move assets usage dropdown to component

* Revert "move assets usage dropdown to component"

This reverts commit 622ea4ab12.

* Revert "Get variables usages"

This reverts commit b11ced4e29.

* Revert "fetch resource usages"

This reverts commit aa5187ad4b.

* Revert "Moved resource variables and s3object pages to assets tabs"

This reverts commit 4430487be4.

* Revert "asset page tabs"

This reverts commit dacc2f0da5.

* move assets usage dropdown to component

* asset icon in asset pages

* tooltip

* details

* Storage selector in S3 File Picker

* make edge less opaque

* Refactor computeAssetNodes to separate in and out nodes

* AssetsOverflowedNode

* nits

* fix assets not being parsed in flows sometimes

* show asset kind and resource_type

* ui nits

* support res:// in duckdb

* add banner for old deployments

* Fix permissionning

* fix broken disable /enable all

* assets page view permission for operators

* Disable ExploreAssetButton for operators

* asset kind as subtitle

* do not spam getResource in assets page. prob. revert fail

* update assets page on workspace change

* reload storage names on ws change

* delete assets on archive / deletion

* sqlx prepare

* missing update when updating user

* add indexes on asset

* better message

* missing loadInit: false

* dead code

* use transaction

* typo

* update package.json

* update package.json

---------

Co-authored-by: Ruben Fiszel <ruben@windmill.dev>

python-client/wmill/wmill/client.py

@@ -11,7 +11,8 @@ import time
 import warnings
 import json
 from json import JSONDecodeError
-from typing import Dict, Any, Union, Literal
+from typing import Dict, Any, Union, Literal, Optional
+import re
 
 import httpx
 
@@ -312,6 +313,7 @@ class Windmill:
             return result_text
 
     def get_variable(self, path: str) -> str:
+        path = parse_variable_syntax(path) or path
         if self.mocked_api is not None:
             variables = self.mocked_api["variables"]
             try:
@@ -326,6 +328,7 @@ class Windmill:
         return self.get(f"/w/{self.workspace}/variables/get_value/{path}").json()
 
     def set_variable(self, path: str, value: str, is_secret: bool = False) -> None:
+        path = parse_variable_syntax(path) or path
         if self.mocked_api is not None:
             self.mocked_api["variables"][path] = value
             return
@@ -358,6 +361,7 @@ class Windmill:
         path: str,
         none_if_undefined: bool = False,
     ) -> dict | None:
+        path = parse_resource_syntax(path) or path
         if self.mocked_api is not None:
             resources = self.mocked_api["resources"]
             try:
@@ -391,6 +395,7 @@ class Windmill:
         path: str,
         resource_type: str,
     ):
+        path = parse_resource_syntax(path) or path
         if self.mocked_api is not None:
             self.mocked_api["resources"][path] = value
             return
@@ -485,6 +490,7 @@ class Windmill:
         Convenient helpers that takes an S3 resource as input and returns the settings necessary to
         initiate an S3 connection from DuckDB
         """
+        s3_resource_path = parse_resource_syntax(s3_resource_path) or s3_resource_path
         try:
             raw_obj = self.post(
                 f"/w/{self.workspace}/job_helpers/v2/duckdb_connection_settings",
@@ -506,6 +512,7 @@ class Windmill:
         Convenient helpers that takes an S3 resource as input and returns the settings necessary to
         initiate an S3 connection from Polars
         """
+        s3_resource_path = parse_resource_syntax(s3_resource_path) or s3_resource_path
         try:
             raw_obj = self.post(
                 f"/w/{self.workspace}/job_helpers/v2/polars_connection_settings",
@@ -527,6 +534,7 @@ class Windmill:
         Convenient helpers that takes an S3 resource as input and returns the settings necessary to
         initiate an S3 connection using boto3
         """
+        s3_resource_path = parse_resource_syntax(s3_resource_path) or s3_resource_path
         try:
             s3_resource = self.post(
                 f"/w/{self.workspace}/job_helpers/v2/s3_resource_info",
@@ -540,7 +548,7 @@ class Windmill:
                 "Could not generate Boto3 S3 connection settings from the provided resource"
             ) from e
 
-    def load_s3_file(self, s3object: S3Object, s3_resource_path: str | None) -> bytes:
+    def load_s3_file(self, s3object: S3Object | str, s3_resource_path: str | None) -> bytes:
         """
         Load a file from the workspace s3 bucket and returns its content as bytes.
 
@@ -552,11 +560,12 @@ class Windmill:
         file_content = my_obj_content.decode("utf-8")
         '''
         """
+        s3object = parse_s3_object(s3object)
         with self.load_s3_file_reader(s3object, s3_resource_path) as file_reader:
             return file_reader.read()
 
     def load_s3_file_reader(
-        self, s3object: S3Object, s3_resource_path: str | None
+        self, s3object: S3Object | str, s3_resource_path: str | None
     ) -> BufferedReader:
         """
         Load a file from the workspace s3 bucket and returns the bytes stream.
@@ -569,6 +578,7 @@ class Windmill:
             print(file_reader.read())
         '''
         """
+        s3object = parse_s3_object(s3object)
         reader = S3BufferedReader(
             f"{self.workspace}",
             self.client,
@@ -580,7 +590,7 @@ class Windmill:
 
     def write_s3_file(
         self,
-        s3object: S3Object | None,
+        s3object: S3Object | str | None,
         file_content: BufferedReader | bytes,
         s3_resource_path: str | None,
         content_type: str | None = None,
@@ -603,6 +613,7 @@ class Windmill:
             client.write_s3_file(s3_obj, my_file)
         '''
         """
+        s3object = parse_s3_object(s3object)
         # httpx accepts either bytes or "a bytes generator" as content. If it's a BufferedReader, we need to convert it to a generator
         if isinstance(file_content, BufferedReader):
             content_payload = bytes_generator(file_content)
@@ -644,12 +655,12 @@ class Windmill:
             raise Exception("Could not write file to S3") from e
         return S3Object(s3=response["file_key"])
 
-    def sign_s3_objects(self, s3_objects: list[S3Object]) -> list[S3Object]:
+    def sign_s3_objects(self, s3_objects: list[S3Object | str]) -> list[S3Object]:
         return self.post(
-            f"/w/{self.workspace}/apps/sign_s3_objects", json={"s3_objects": s3_objects}
+            f"/w/{self.workspace}/apps/sign_s3_objects", json={"s3_objects": list(map(parse_s3_object, s3_objects))}
         ).json()
 
-    def sign_s3_object(self, s3_object: S3Object) -> S3Object:
+    def sign_s3_object(self, s3_object: S3Object | str) -> S3Object:
         return self.post(
             f"/w/{self.workspace}/apps/sign_s3_objects",
             json={"s3_objects": [s3_object]},
@@ -1027,7 +1038,7 @@ def boto3_connection_settings(s3_resource_path: str = "") -> Boto3ConnectionSett
 
 
 @init_global_client
-def load_s3_file(s3object: S3Object, s3_resource_path: str | None = None) -> bytes:
+def load_s3_file(s3object: S3Object | str, s3_resource_path: str | None = None) -> bytes:
     """
     Load the entire content of a file stored in S3 as bytes
     """
@@ -1038,7 +1049,7 @@ def load_s3_file(s3object: S3Object, s3_resource_path: str | None = None) -> byt
 
 @init_global_client
 def load_s3_file_reader(
-    s3object: S3Object, s3_resource_path: str | None = None
+    s3object: S3Object | str, s3_resource_path: str | None = None
 ) -> BufferedReader:
     """
     Load the content of a file stored in S3
@@ -1050,7 +1061,7 @@ def load_s3_file_reader(
 
 @init_global_client
 def write_s3_file(
-    s3object: S3Object | None,
+    s3object: S3Object | str | None,
     file_content: BufferedReader | bytes,
     s3_resource_path: str | None = None,
     content_type: str | None = None,
@@ -1075,7 +1086,7 @@ def write_s3_file(
 
 
 @init_global_client
-def sign_s3_objects(s3_objects: list[S3Object]) -> list[S3Object]:
+def sign_s3_objects(s3_objects: list[S3Object | str]) -> list[S3Object]:
     """
     Sign S3 objects to be used by anonymous users in public apps
     Returns a list of signed s3 tokens
@@ -1084,7 +1095,7 @@ def sign_s3_objects(s3_objects: list[S3Object]) -> list[S3Object]:
 
 
 @init_global_client
-def sign_s3_object(s3_object: S3Object) -> S3Object:
+def sign_s3_object(s3_object: S3Object| str) -> S3Object:
     """
     Sign S3 object to be used by anonymous users in public apps
     Returns a signed s3 object
@@ -1336,3 +1347,31 @@ def task(*args, **kwargs):
         return f(args[0], None)
     else:
         return lambda x: f(x, kwargs.get("tag"))
+
+def parse_resource_syntax(s: str) -> Optional[str]:
+    """Parse resource syntax from string."""
+    if s is None:
+        return None
+    if s.startswith("$res:"):
+        return s[5:]
+    if s.startswith("res://"):
+        return s[6:]
+    return None
+
+def parse_s3_object(s3_object: S3Object | str) -> S3Object:
+    """Parse S3 object from string or S3Object format."""
+    if isinstance(s3_object, str):
+        match = re.match(r'^s3://([^/]*)/(.*)$', s3_object)
+        if match:
+            return S3Object(s3=match.group(2) or "", storage=match.group(1) or None)
+        return S3Object(s3="")
+    else:
+        return s3_object
+
+    
+
+def parse_variable_syntax(s: str) -> Optional[str]:
+    """Parse variable syntax from string."""
+    if s.startswith("var://"):
+        return s[6:]
+    return None