feat: Add Python SDK capabilities to generate Polars and DuckDB connection settings to S3 (#2625)
* feat: Add Python SDK capabilities to generate Polars and DuckDB connection settings to S3 * Add endpoint to list S3 objects * Add FE to set the workspace S3 resource * Fix openapi * sqlx prepare * Hide Windmil LFS tab
This commit is contained in:
Guillaume Bouvignies
committed by
GitHub
GitHub
parent
b649fae0b0
commit
f932e4c916
@@ -42,7 +42,6 @@
|
||||
"bash",
|
||||
"postgresql",
|
||||
"nativets",
|
||||
"Nativets",
|
||||
"bun",
|
||||
"mysql",
|
||||
"bigquery",
|
||||
|
||||
@@ -82,6 +82,11 @@
|
||||
"ordinal": 15,
|
||||
"name": "error_handler_muted_on_cancel",
|
||||
"type_info": "Bool"
|
||||
},
|
||||
{
|
||||
"ordinal": 16,
|
||||
"name": "large_file_storage",
|
||||
"type_info": "Jsonb"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
@@ -105,7 +110,8 @@
|
||||
true,
|
||||
false,
|
||||
true,
|
||||
false
|
||||
false,
|
||||
true
|
||||
]
|
||||
},
|
||||
"hash": "1730f39fd1793d45fbb41b21389c61296a3ff7489ae12f52a19f9543173ac597"
|
||||
|
||||
@@ -67,7 +67,6 @@
|
||||
"bash",
|
||||
"postgresql",
|
||||
"nativets",
|
||||
"Nativets",
|
||||
"bun",
|
||||
"mysql",
|
||||
"bigquery",
|
||||
|
||||
14
backend/.sqlx/query-2ff397ad0aeabdc15776025f75c2d1390ce5ac700b0bf71481d31747c8f2e870.json
generated
Normal file
14
backend/.sqlx/query-2ff397ad0aeabdc15776025f75c2d1390ce5ac700b0bf71481d31747c8f2e870.json
generated
Normal file
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "UPDATE workspace_settings SET large_file_storage = NULL WHERE workspace_id = $1",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "2ff397ad0aeabdc15776025f75c2d1390ce5ac700b0bf71481d31747c8f2e870"
|
||||
}
|
||||
@@ -28,7 +28,6 @@
|
||||
"bash",
|
||||
"postgresql",
|
||||
"nativets",
|
||||
"Nativets",
|
||||
"bun",
|
||||
"mysql",
|
||||
"bigquery",
|
||||
|
||||
@@ -82,6 +82,11 @@
|
||||
"ordinal": 15,
|
||||
"name": "error_handler_muted_on_cancel",
|
||||
"type_info": "Bool"
|
||||
},
|
||||
{
|
||||
"ordinal": 16,
|
||||
"name": "large_file_storage",
|
||||
"type_info": "Jsonb"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
@@ -105,7 +110,8 @@
|
||||
true,
|
||||
false,
|
||||
true,
|
||||
false
|
||||
false,
|
||||
true
|
||||
]
|
||||
},
|
||||
"hash": "5445083864b2b092b012e894bff7630a1d7b9deb8d33e9f909061f351f96844e"
|
||||
|
||||
22
backend/.sqlx/query-5e4e0740e1cff4b2a7c620da3758604a47f81b505fa9a713efa3519b050b546a.json
generated
Normal file
22
backend/.sqlx/query-5e4e0740e1cff4b2a7c620da3758604a47f81b505fa9a713efa3519b050b546a.json
generated
Normal file
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT large_file_storage FROM workspace_settings WHERE workspace_id = $1",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "large_file_storage",
|
||||
"type_info": "Jsonb"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
true
|
||||
]
|
||||
},
|
||||
"hash": "5e4e0740e1cff4b2a7c620da3758604a47f81b505fa9a713efa3519b050b546a"
|
||||
}
|
||||
@@ -60,7 +60,6 @@
|
||||
"bash",
|
||||
"postgresql",
|
||||
"nativets",
|
||||
"Nativets",
|
||||
"bun",
|
||||
"mysql",
|
||||
"bigquery",
|
||||
|
||||
@@ -40,7 +40,6 @@
|
||||
"bash",
|
||||
"postgresql",
|
||||
"nativets",
|
||||
"Nativets",
|
||||
"bun",
|
||||
"mysql",
|
||||
"bigquery",
|
||||
|
||||
@@ -46,7 +46,6 @@
|
||||
"bash",
|
||||
"postgresql",
|
||||
"nativets",
|
||||
"Nativets",
|
||||
"bun",
|
||||
"mysql",
|
||||
"bigquery",
|
||||
|
||||
@@ -42,7 +42,6 @@
|
||||
"bash",
|
||||
"postgresql",
|
||||
"nativets",
|
||||
"Nativets",
|
||||
"bun",
|
||||
"mysql",
|
||||
"bigquery",
|
||||
|
||||
15
backend/.sqlx/query-bf27083af3514e5fae868364518096d1cf4a042c23526dc134c3b2d27904e954.json
generated
Normal file
15
backend/.sqlx/query-bf27083af3514e5fae868364518096d1cf4a042c23526dc134c3b2d27904e954.json
generated
Normal file
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "UPDATE workspace_settings SET large_file_storage = $1 WHERE workspace_id = $2",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Jsonb",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "bf27083af3514e5fae868364518096d1cf4a042c23526dc134c3b2d27904e954"
|
||||
}
|
||||
@@ -42,7 +42,6 @@
|
||||
"bash",
|
||||
"postgresql",
|
||||
"nativets",
|
||||
"Nativets",
|
||||
"bun",
|
||||
"mysql",
|
||||
"bigquery",
|
||||
|
||||
@@ -37,7 +37,6 @@
|
||||
"bash",
|
||||
"postgresql",
|
||||
"nativets",
|
||||
"Nativets",
|
||||
"bun",
|
||||
"mysql",
|
||||
"bigquery",
|
||||
|
||||
511
backend/Cargo.lock
generated
511
backend/Cargo.lock
generated
@@ -503,6 +503,382 @@ version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
|
||||
|
||||
[[package]]
|
||||
name = "aws-config"
|
||||
version = "0.57.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f2bf00cb9416daab4ce4927c54ebe63c08b9caf4d7b9314b6d7a4a2c5a1afb09"
|
||||
dependencies = [
|
||||
"aws-credential-types",
|
||||
"aws-http",
|
||||
"aws-runtime",
|
||||
"aws-sdk-sso",
|
||||
"aws-sdk-ssooidc",
|
||||
"aws-sdk-sts",
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-http",
|
||||
"aws-smithy-json",
|
||||
"aws-smithy-runtime",
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-types",
|
||||
"aws-types",
|
||||
"bytes",
|
||||
"fastrand 2.0.1",
|
||||
"hex",
|
||||
"http",
|
||||
"hyper",
|
||||
"ring 0.17.5",
|
||||
"time",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-credential-types"
|
||||
version = "0.57.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cb9073c88dbf12f68ce7d0e149f989627a1d1ae3d2b680459f04ccc29d1cbd0f"
|
||||
dependencies = [
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-types",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-http"
|
||||
version = "0.57.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "24067106d09620cf02d088166cdaedeaca7146d4d499c41b37accecbea11b246"
|
||||
dependencies = [
|
||||
"aws-smithy-http",
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-types",
|
||||
"aws-types",
|
||||
"bytes",
|
||||
"http",
|
||||
"http-body",
|
||||
"pin-project-lite",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-runtime"
|
||||
version = "0.57.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fc6ee0152c06d073602236a4e94a8c52a327d310c1ecd596570ce795af8777ff"
|
||||
dependencies = [
|
||||
"aws-credential-types",
|
||||
"aws-http",
|
||||
"aws-sigv4",
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-eventstream",
|
||||
"aws-smithy-http",
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-types",
|
||||
"aws-types",
|
||||
"fastrand 2.0.1",
|
||||
"http",
|
||||
"percent-encoding",
|
||||
"tracing",
|
||||
"uuid 1.5.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-sdk-s3"
|
||||
version = "0.36.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf188e8af89adb0b41ae2f21b6c666c618959e40242e79aaa43fafa9c0c90a8a"
|
||||
dependencies = [
|
||||
"aws-credential-types",
|
||||
"aws-http",
|
||||
"aws-runtime",
|
||||
"aws-sigv4",
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-checksums",
|
||||
"aws-smithy-eventstream",
|
||||
"aws-smithy-http",
|
||||
"aws-smithy-json",
|
||||
"aws-smithy-runtime",
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-types",
|
||||
"aws-smithy-xml",
|
||||
"aws-types",
|
||||
"bytes",
|
||||
"http",
|
||||
"http-body",
|
||||
"once_cell",
|
||||
"percent-encoding",
|
||||
"regex",
|
||||
"tracing",
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-sdk-sso"
|
||||
version = "0.36.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2eb8158015232b4596ccef74a205600398e152d704b40b7ec9f486092474d7fa"
|
||||
dependencies = [
|
||||
"aws-credential-types",
|
||||
"aws-http",
|
||||
"aws-runtime",
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-http",
|
||||
"aws-smithy-json",
|
||||
"aws-smithy-runtime",
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-types",
|
||||
"aws-types",
|
||||
"bytes",
|
||||
"http",
|
||||
"regex",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-sdk-ssooidc"
|
||||
version = "0.36.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "36a1493e1c57f173e53621935bfb5b6217376168dbdb4cd459aebcf645924a48"
|
||||
dependencies = [
|
||||
"aws-credential-types",
|
||||
"aws-http",
|
||||
"aws-runtime",
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-http",
|
||||
"aws-smithy-json",
|
||||
"aws-smithy-runtime",
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-types",
|
||||
"aws-types",
|
||||
"bytes",
|
||||
"http",
|
||||
"regex",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-sdk-sts"
|
||||
version = "0.36.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e032b77f5cd1dd3669d777a38ac08cbf8ec68e29460d4ef5d3e50cffa74ec75a"
|
||||
dependencies = [
|
||||
"aws-credential-types",
|
||||
"aws-http",
|
||||
"aws-runtime",
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-http",
|
||||
"aws-smithy-json",
|
||||
"aws-smithy-query",
|
||||
"aws-smithy-runtime",
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-types",
|
||||
"aws-smithy-xml",
|
||||
"aws-types",
|
||||
"http",
|
||||
"regex",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-sigv4"
|
||||
version = "0.57.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "64f81a6abc4daab06b53cabf27c54189928893283093e37164ca53aa47488a5b"
|
||||
dependencies = [
|
||||
"aws-credential-types",
|
||||
"aws-smithy-eventstream",
|
||||
"aws-smithy-http",
|
||||
"aws-smithy-runtime-api",
|
||||
"bytes",
|
||||
"form_urlencoded",
|
||||
"hex",
|
||||
"hmac",
|
||||
"http",
|
||||
"num-bigint",
|
||||
"once_cell",
|
||||
"p256",
|
||||
"percent-encoding",
|
||||
"regex",
|
||||
"ring 0.17.5",
|
||||
"sha2 0.10.8",
|
||||
"time",
|
||||
"tracing",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-async"
|
||||
version = "0.57.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dbe53fccd3b10414b9cae63767a15a2789b34e6c6727b6e32b33e8c7998a3e80"
|
||||
dependencies = [
|
||||
"futures-util",
|
||||
"pin-project-lite",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-checksums"
|
||||
version = "0.57.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9fb5701fbfb40600cc0fa547f318552dfd4e632b2099bd75d95fb0faae70675d"
|
||||
dependencies = [
|
||||
"aws-smithy-http",
|
||||
"aws-smithy-types",
|
||||
"bytes",
|
||||
"crc32c",
|
||||
"crc32fast",
|
||||
"hex",
|
||||
"http",
|
||||
"http-body",
|
||||
"md-5 0.10.6",
|
||||
"pin-project-lite",
|
||||
"sha1",
|
||||
"sha2 0.10.8",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-eventstream"
|
||||
version = "0.57.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6b33fa99f928a5815b94ee07e1377901bcf51aa749034a2c802dc38f9dcfacf5"
|
||||
dependencies = [
|
||||
"aws-smithy-types",
|
||||
"bytes",
|
||||
"crc32fast",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-http"
|
||||
version = "0.57.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f7972373213d1d6e619c0edc9dda2d6634154e4ed75c5e0b2bf065cd5ec9f0d1"
|
||||
dependencies = [
|
||||
"aws-smithy-eventstream",
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-types",
|
||||
"bytes",
|
||||
"bytes-utils",
|
||||
"futures-core",
|
||||
"http",
|
||||
"http-body",
|
||||
"once_cell",
|
||||
"percent-encoding",
|
||||
"pin-project-lite",
|
||||
"pin-utils",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-json"
|
||||
version = "0.57.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6d64d5af16dd585de9ff6c606423c1aaad47c6baa38de41c2beb32ef21c6645"
|
||||
dependencies = [
|
||||
"aws-smithy-types",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-query"
|
||||
version = "0.57.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7527bf5335154ba1b285479c50b630e44e93d1b4a759eaceb8d0bf9fbc82caa5"
|
||||
dependencies = [
|
||||
"aws-smithy-types",
|
||||
"urlencoding",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-runtime"
|
||||
version = "0.57.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "839b363adf3b2bdab2742a1f540fec23039ea8bc9ec0f9f61df48470cfe5527b"
|
||||
dependencies = [
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-http",
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-types",
|
||||
"bytes",
|
||||
"fastrand 2.0.1",
|
||||
"http",
|
||||
"http-body",
|
||||
"hyper",
|
||||
"hyper-rustls",
|
||||
"once_cell",
|
||||
"pin-project-lite",
|
||||
"pin-utils",
|
||||
"rustls 0.21.8",
|
||||
"tokio",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-runtime-api"
|
||||
version = "0.57.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f24ecc446e62c3924539e7c18dec8038dba4fdf8718d5c2de62f9d2fecca8ba9"
|
||||
dependencies = [
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-types",
|
||||
"bytes",
|
||||
"http",
|
||||
"pin-project-lite",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-types"
|
||||
version = "0.57.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "051de910296522a21178a2ea402ea59027eef4b63f1cef04a0be2bb5e25dea03"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"bytes",
|
||||
"bytes-utils",
|
||||
"futures-core",
|
||||
"http",
|
||||
"http-body",
|
||||
"itoa",
|
||||
"num-integer",
|
||||
"pin-project-lite",
|
||||
"pin-utils",
|
||||
"ryu",
|
||||
"serde",
|
||||
"time",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-xml"
|
||||
version = "0.57.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cb1e3ac22c652662096c8e37a6f9af80c6f3520cab5610b2fe76c725bce18eac"
|
||||
dependencies = [
|
||||
"xmlparser",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-types"
|
||||
version = "0.57.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "048bbf1c24cdf4eb1efcdc243388a93a90ebf63979e25fc1c7b8cbd9cb6beb38"
|
||||
dependencies = [
|
||||
"aws-credential-types",
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-types",
|
||||
"http",
|
||||
"rustc_version 0.4.0",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "axum"
|
||||
version = "0.6.20"
|
||||
@@ -568,6 +944,12 @@ dependencies = [
|
||||
"rustc-demangle",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "base16ct"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "349a06037c7bf932dd7e7d1f653678b2038b9ad46a74102f1fc7bd7872678cce"
|
||||
|
||||
[[package]]
|
||||
name = "base64"
|
||||
version = "0.13.1"
|
||||
@@ -911,6 +1293,16 @@ version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223"
|
||||
|
||||
[[package]]
|
||||
name = "bytes-utils"
|
||||
version = "0.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e47d3a8076e283f3acd27400535992edb3ba4b5bb72f8891ad8fbe7932a7d4b9"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"either",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bzip2"
|
||||
version = "0.4.4"
|
||||
@@ -1312,6 +1704,15 @@ version = "2.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5"
|
||||
|
||||
[[package]]
|
||||
name = "crc32c"
|
||||
version = "0.6.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d8f48d60e5b4d2c53d5c2b1d8a58c849a70ae5e5509b08a48d047e3b65714a74"
|
||||
dependencies = [
|
||||
"rustc_version 0.4.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crc32fast"
|
||||
version = "1.3.2"
|
||||
@@ -1405,6 +1806,18 @@ version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7"
|
||||
|
||||
[[package]]
|
||||
name = "crypto-bigint"
|
||||
version = "0.4.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ef2b4b23cddf68b89b8f8069890e8c270d54e2d5fe1b143820234805e4cb17ef"
|
||||
dependencies = [
|
||||
"generic-array",
|
||||
"rand_core 0.6.4",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crypto-common"
|
||||
version = "0.1.6"
|
||||
@@ -1949,6 +2362,18 @@ dependencies = [
|
||||
"reborrow",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ecdsa"
|
||||
version = "0.14.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "413301934810f597c1d19ca71c8710e99a3f1ba28a0d2ebc01551a2daeea3c5c"
|
||||
dependencies = [
|
||||
"der 0.6.1",
|
||||
"elliptic-curve",
|
||||
"rfc6979",
|
||||
"signature 1.6.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "either"
|
||||
version = "1.9.0"
|
||||
@@ -1958,6 +2383,26 @@ dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "elliptic-curve"
|
||||
version = "0.12.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e7bb888ab5300a19b8e5bceef25ac745ad065f3c9f7efc6de1b91958110891d3"
|
||||
dependencies = [
|
||||
"base16ct",
|
||||
"crypto-bigint",
|
||||
"der 0.6.1",
|
||||
"digest 0.10.7",
|
||||
"ff",
|
||||
"generic-array",
|
||||
"group",
|
||||
"pkcs8 0.9.0",
|
||||
"rand_core 0.6.4",
|
||||
"sec1",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "embed-doc-image"
|
||||
version = "0.1.4"
|
||||
@@ -2153,6 +2598,16 @@ version = "2.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5"
|
||||
|
||||
[[package]]
|
||||
name = "ff"
|
||||
version = "0.12.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d013fc25338cc558c5c2cfbad646908fb23591e2404481826742b651c9af7160"
|
||||
dependencies = [
|
||||
"rand_core 0.6.4",
|
||||
"subtle",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "filetime"
|
||||
version = "0.2.22"
|
||||
@@ -2706,6 +3161,17 @@ dependencies = [
|
||||
"unic-ucd-category",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "group"
|
||||
version = "0.12.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5dfbfb3a6cfbd390d5c9564ab283a0349b9b9fcd46a706c1eb10e0db70bfbac7"
|
||||
dependencies = [
|
||||
"ff",
|
||||
"rand_core 0.6.4",
|
||||
"subtle",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "h2"
|
||||
version = "0.3.21"
|
||||
@@ -2959,6 +3425,7 @@ dependencies = [
|
||||
"futures-util",
|
||||
"http",
|
||||
"hyper",
|
||||
"log",
|
||||
"rustls 0.21.8",
|
||||
"rustls-native-certs",
|
||||
"tokio",
|
||||
@@ -4010,6 +4477,17 @@ version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"
|
||||
|
||||
[[package]]
|
||||
name = "p256"
|
||||
version = "0.11.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "51f44edd08f51e2ade572f141051021c5af22677e42b7dd28a88155151c33594"
|
||||
dependencies = [
|
||||
"ecdsa",
|
||||
"elliptic-curve",
|
||||
"sha2 0.10.8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "parking"
|
||||
version = "2.2.0"
|
||||
@@ -5093,6 +5571,17 @@ dependencies = [
|
||||
"winreg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rfc6979"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7743f17af12fa0b03b803ba12cd6a8d9483a587e89c69445e3909655c0b9fabb"
|
||||
dependencies = [
|
||||
"crypto-bigint",
|
||||
"hmac",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "riff"
|
||||
version = "1.0.1"
|
||||
@@ -5572,6 +6061,20 @@ version = "4.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b"
|
||||
|
||||
[[package]]
|
||||
name = "sec1"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3be24c1842290c45df0a7bf069e0c268a747ad05a192f2fd7dcfdbc1cba40928"
|
||||
dependencies = [
|
||||
"base16ct",
|
||||
"der 0.6.1",
|
||||
"generic-array",
|
||||
"pkcs8 0.9.0",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "security-framework"
|
||||
version = "2.9.2"
|
||||
@@ -8221,6 +8724,8 @@ dependencies = [
|
||||
"async-recursion",
|
||||
"async-stripe",
|
||||
"async_zip",
|
||||
"aws-config",
|
||||
"aws-sdk-s3",
|
||||
"axum",
|
||||
"base64 0.21.5",
|
||||
"bytes",
|
||||
@@ -8727,6 +9232,12 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "xmlparser"
|
||||
version = "0.13.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4"
|
||||
|
||||
[[package]]
|
||||
name = "xz2"
|
||||
version = "0.1.7"
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
-- Add down migration script here
|
||||
ALTER TABLE workspace_settings DROP COLUMN large_file_storage;
|
||||
@@ -0,0 +1,2 @@
|
||||
-- Add up migration script here
|
||||
ALTER TABLE workspace_settings ADD COLUMN IF NOT EXISTS large_file_storage JSONB;
|
||||
@@ -81,4 +81,6 @@ hf-hub.workspace = true
|
||||
tokenizers.workspace = true
|
||||
candle-core.workspace = true
|
||||
candle-transformers.workspace = true
|
||||
candle-nn.workspace = true
|
||||
candle-nn.workspace = true
|
||||
aws-sdk-s3 = "0.36.0"
|
||||
aws-config = "0.57.2"
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1176,6 +1176,8 @@ paths:
|
||||
$ref: "#/components/schemas/ScriptArgs"
|
||||
error_handler_muted_on_cancel:
|
||||
type: boolean
|
||||
large_file_storage:
|
||||
$ref: "#/components/schemas/LargeFileStorage"
|
||||
required:
|
||||
- code_completion_enabled
|
||||
|
||||
@@ -1469,6 +1471,49 @@ paths:
|
||||
schema:
|
||||
type: string
|
||||
|
||||
/w/{workspace}/workspaces/edit_large_file_storage_config:
|
||||
post:
|
||||
summary: edit large file storage settings
|
||||
operationId: editLargeFileStorageConfig
|
||||
tags:
|
||||
- workspace
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/WorkspaceId"
|
||||
requestBody:
|
||||
description: LargeFileStorage info
|
||||
required: true
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
properties:
|
||||
large_file_storage:
|
||||
$ref: "#/components/schemas/LargeFileStorage"
|
||||
|
||||
responses:
|
||||
"200":
|
||||
description: status
|
||||
content:
|
||||
application/json:
|
||||
schema: {}
|
||||
|
||||
/w/{workspace}/workspaces/get_large_file_storage_config:
|
||||
get:
|
||||
summary: get large file storage config
|
||||
operationId: getLargeFileStorageConfig
|
||||
tags:
|
||||
- workspace
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/WorkspaceId"
|
||||
|
||||
responses:
|
||||
"200":
|
||||
description: status
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/LargeFileStorage"
|
||||
|
||||
/w/{workspace}/users/list:
|
||||
get:
|
||||
summary: list users
|
||||
@@ -6231,6 +6276,69 @@ paths:
|
||||
type: string
|
||||
format: uuid
|
||||
|
||||
/w/{workspace}/job_helpers/duckdb_connection_settings:
|
||||
post:
|
||||
summary: Converts an S3 resource to the set of instructions necessary to connect DuckDB to an S3 bucket
|
||||
operationId: duckdbConnectionSettings
|
||||
tags:
|
||||
- helpers
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/WorkspaceId"
|
||||
requestBody:
|
||||
description: S3 resource to connect to
|
||||
required: true
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
s3_resource:
|
||||
$ref: "#/components/schemas/S3Resource"
|
||||
responses:
|
||||
"200":
|
||||
description: Connection settings
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
properties:
|
||||
connection_settings_str:
|
||||
type: string
|
||||
|
||||
/w/{workspace}/job_helpers/test_connection:
|
||||
get:
|
||||
summary: Test connection to the workspace datasets storage
|
||||
operationId: datasetStorageTestConnection
|
||||
tags:
|
||||
- helpers
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/WorkspaceId"
|
||||
responses:
|
||||
"200":
|
||||
description: Connection settings
|
||||
content:
|
||||
application/json:
|
||||
schema: {}
|
||||
|
||||
/w/{workspace}/job_helpers/list_stored_datasets:
|
||||
post:
|
||||
summary: List the dataset keys available in the worspace datasets storage
|
||||
operationId: polarsConnectionSettings
|
||||
tags:
|
||||
- helpers
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/WorkspaceId"
|
||||
responses:
|
||||
"200":
|
||||
description: Connection settings
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
properties:
|
||||
dataset_keys:
|
||||
type: array
|
||||
items:
|
||||
$ref: "#/components/schemas/WindmillLargeFile"
|
||||
|
||||
components:
|
||||
securitySchemes:
|
||||
bearerAuth:
|
||||
@@ -8206,3 +8314,46 @@ components:
|
||||
schema:
|
||||
type: string
|
||||
enum: [script, failure, trigger, approval]
|
||||
|
||||
LargeFileStorage:
|
||||
type: object
|
||||
properties:
|
||||
type:
|
||||
type: string
|
||||
enum: ["S3Storage"]
|
||||
s3_resource_path:
|
||||
type: string
|
||||
|
||||
WindmillLargeFile:
|
||||
type: object
|
||||
properties:
|
||||
s3:
|
||||
type: string
|
||||
s3_bucket:
|
||||
type: string
|
||||
required:
|
||||
- s3
|
||||
|
||||
S3Resource:
|
||||
type: object
|
||||
properties:
|
||||
bucket:
|
||||
type: string
|
||||
region:
|
||||
type: string
|
||||
endPoint:
|
||||
type: string
|
||||
useSSL:
|
||||
type: boolean
|
||||
accessKey:
|
||||
type: string
|
||||
secretKey:
|
||||
type: string
|
||||
pathStyle:
|
||||
type: boolean
|
||||
required:
|
||||
- bucket
|
||||
- region
|
||||
- endPoint
|
||||
- useSSL
|
||||
- pathStyle
|
||||
|
||||
306
backend/windmill-api/src/job_helpers.rs
Normal file
306
backend/windmill-api/src/job_helpers.rs
Normal file
@@ -0,0 +1,306 @@
|
||||
use crate::{resources::transform_json_value, users::Tokened, workspaces::LargeFileStorage};
|
||||
use aws_sdk_s3::config::{Credentials, Region};
|
||||
use axum::{
|
||||
extract::Path,
|
||||
routing::{get, post},
|
||||
Extension, Json, Router,
|
||||
};
|
||||
use hyper::http;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tower_http::cors::{Any, CorsLayer};
|
||||
use windmill_common::{db::UserDB, error};
|
||||
|
||||
use crate::db::ApiAuthed;
|
||||
|
||||
pub fn workspaced_service() -> Router {
|
||||
let cors = CorsLayer::new()
|
||||
.allow_methods([http::Method::GET, http::Method::POST])
|
||||
.allow_headers([http::header::CONTENT_TYPE, http::header::AUTHORIZATION])
|
||||
.allow_origin(Any);
|
||||
|
||||
Router::new()
|
||||
.route(
|
||||
"/duckdb_connection_settings",
|
||||
post(duckdb_connection_settings).layer(cors.clone()),
|
||||
)
|
||||
.route(
|
||||
"/polars_connection_settings",
|
||||
post(polars_connection_settings).layer(cors.clone()),
|
||||
)
|
||||
.route("/test_connection", get(test_connection).layer(cors.clone()))
|
||||
.route(
|
||||
"/list_stored_datasets",
|
||||
get(list_stored_datasets).layer(cors.clone()),
|
||||
)
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Clone)]
|
||||
struct S3Resource {
|
||||
#[serde(rename = "bucket")]
|
||||
bucket: String,
|
||||
region: String,
|
||||
#[serde(rename = "endPoint")]
|
||||
endpoint: String,
|
||||
#[serde(rename = "useSSL")]
|
||||
use_ssl: bool,
|
||||
#[serde(rename = "accessKey")]
|
||||
access_key: Option<String>,
|
||||
#[serde(rename = "secretKey")]
|
||||
secret_key: Option<String>,
|
||||
#[serde(rename = "pathStyle")]
|
||||
path_style: bool,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct DuckdbConnectionSettingsQuery {
|
||||
s3_resource: S3Resource,
|
||||
}
|
||||
#[derive(Serialize)]
|
||||
struct DuckdbConnectionSettingsResponse {
|
||||
connection_settings_str: String,
|
||||
}
|
||||
|
||||
async fn duckdb_connection_settings(
|
||||
Path(_w_id): Path<String>,
|
||||
Json(query): Json<DuckdbConnectionSettingsQuery>,
|
||||
) -> error::JsonResult<DuckdbConnectionSettingsResponse> {
|
||||
let mut duckdb_settings: String = String::new();
|
||||
|
||||
let s3_resource = query.s3_resource;
|
||||
duckdb_settings.push_str(format!("SET home_directory='./';\n").as_str()); // TODO: make this configurable maybe, or point to a temporary folder
|
||||
duckdb_settings.push_str(format!("INSTALL 'httpfs';\n").as_str());
|
||||
if s3_resource.path_style {
|
||||
duckdb_settings.push_str(format!("SET s3_url_style='path';\n").as_str());
|
||||
}
|
||||
duckdb_settings.push_str(format!("SET s3_region='{}';\n", s3_resource.region).as_str());
|
||||
duckdb_settings.push_str(format!("SET s3_endpoint='{}';\n", s3_resource.endpoint).as_str());
|
||||
if !s3_resource.use_ssl {
|
||||
duckdb_settings.push_str(format!("SET s3_use_ssl=0;\n").as_str()); // default is true for DuckDB
|
||||
}
|
||||
if let Some(access_key_id) = s3_resource.access_key {
|
||||
duckdb_settings.push_str(format!("SET s3_access_key_id='{}';\n", access_key_id).as_str());
|
||||
}
|
||||
if let Some(secret_access_key) = s3_resource.secret_key {
|
||||
duckdb_settings
|
||||
.push_str(format!("SET s3_secret_access_key='{}';\n", secret_access_key).as_str());
|
||||
}
|
||||
|
||||
let response = DuckdbConnectionSettingsResponse { connection_settings_str: duckdb_settings };
|
||||
return Ok(Json(response));
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct PolarsConnectionSettingsQuery {
|
||||
s3_resource: S3Resource,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct PolarsConnectionSettingsResponse {
|
||||
endpoint_url: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
key: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
secret: Option<String>,
|
||||
use_ssl: bool,
|
||||
cache_regions: bool,
|
||||
client_kwargs: PolarsConnectionSettings,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct PolarsConnectionSettings {
|
||||
pub region_name: String,
|
||||
}
|
||||
|
||||
async fn polars_connection_settings(
|
||||
Path(_w_id): Path<String>,
|
||||
Json(query): Json<PolarsConnectionSettingsQuery>,
|
||||
) -> error::JsonResult<PolarsConnectionSettingsResponse> {
|
||||
let s3_resource = query.s3_resource;
|
||||
|
||||
let response = PolarsConnectionSettingsResponse {
|
||||
endpoint_url: s3_resource.endpoint,
|
||||
key: s3_resource.access_key,
|
||||
secret: s3_resource.secret_key,
|
||||
use_ssl: s3_resource.use_ssl,
|
||||
cache_regions: false,
|
||||
client_kwargs: PolarsConnectionSettings { region_name: s3_resource.region },
|
||||
};
|
||||
|
||||
return Ok(Json(response));
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct ListStoredDatasetsResponse {
|
||||
windmill_large_files: Vec<WindmillLargeFile>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Clone)]
|
||||
struct WindmillLargeFile {
|
||||
s3: String,
|
||||
s3_bucket: Option<String>,
|
||||
}
|
||||
|
||||
async fn test_connection(
|
||||
authed: ApiAuthed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Tokened { token }: Tokened,
|
||||
Path(w_id): Path<String>,
|
||||
) -> error::JsonResult<()> {
|
||||
let s3_resource_opt = get_workspace_s3_resource(&authed, &user_db, &token, &w_id).await?;
|
||||
if s3_resource_opt.is_none() {
|
||||
return Err(error::Error::NotFound(
|
||||
"No datasets storage resource defined at the workspace level".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
let s3_resource = s3_resource_opt.unwrap();
|
||||
let s3_client = build_s3_client(&s3_resource);
|
||||
s3_client
|
||||
.list_objects()
|
||||
.bucket(s3_resource.bucket)
|
||||
.max_keys(1)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|err| error::Error::InternalErr(err.to_string()))?;
|
||||
return Ok(Json(()));
|
||||
}
|
||||
|
||||
async fn list_stored_datasets(
|
||||
authed: ApiAuthed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Tokened { token }: Tokened,
|
||||
Path(w_id): Path<String>,
|
||||
) -> error::JsonResult<ListStoredDatasetsResponse> {
|
||||
let s3_resource_opt = get_workspace_s3_resource(&authed, &user_db, &token, &w_id).await?;
|
||||
|
||||
let s3_resource = s3_resource_opt.unwrap();
|
||||
let s3_client = build_s3_client(&s3_resource);
|
||||
|
||||
let mut stored_datasets = Vec::<WindmillLargeFile>::new();
|
||||
let s3_bucket = s3_resource.bucket;
|
||||
loop {
|
||||
let mut list_object_query = s3_client
|
||||
.list_objects()
|
||||
.bucket(s3_bucket.clone())
|
||||
.max_keys(32);
|
||||
|
||||
if let Some(last_dataset_name) = stored_datasets.last() {
|
||||
// if stored_datasets already has some elements, it means we're looping through pages
|
||||
// according to AWS SDK docs, we can set the marker to the last returned dataset
|
||||
list_object_query = list_object_query.set_marker(Some(last_dataset_name.clone().s3))
|
||||
}
|
||||
|
||||
let bucket_objects = list_object_query
|
||||
.send()
|
||||
.await
|
||||
.map_err(|err| error::Error::InternalErr(err.to_string()))?;
|
||||
|
||||
let page_datasets = bucket_objects
|
||||
.contents()
|
||||
.iter()
|
||||
.filter(|object| object.key().is_some())
|
||||
.map(|object| object.key())
|
||||
.map(Option::unwrap)
|
||||
.map(&str::to_string)
|
||||
.map(|object_key| WindmillLargeFile {
|
||||
s3: object_key.clone(),
|
||||
s3_bucket: Some(s3_bucket.clone()),
|
||||
})
|
||||
.collect::<Vec<WindmillLargeFile>>();
|
||||
|
||||
stored_datasets.extend(page_datasets.clone());
|
||||
|
||||
if !bucket_objects.is_truncated() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return Ok(Json(ListStoredDatasetsResponse {
|
||||
windmill_large_files: stored_datasets,
|
||||
}));
|
||||
}
|
||||
|
||||
async fn get_workspace_s3_resource<'c>(
|
||||
authed: &ApiAuthed,
|
||||
user_db: &UserDB,
|
||||
token: &str,
|
||||
w_id: &str,
|
||||
) -> error::Result<Option<S3Resource>> {
|
||||
let mut tx = user_db.clone().begin(authed).await?;
|
||||
let raw_lfs_opt = sqlx::query_scalar!(
|
||||
"SELECT large_file_storage FROM workspace_settings WHERE workspace_id = $1",
|
||||
w_id
|
||||
)
|
||||
.fetch_optional(&mut *tx)
|
||||
.await?
|
||||
.flatten();
|
||||
tx.commit().await?;
|
||||
|
||||
if raw_lfs_opt.is_none() {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let large_file_storage = serde_json::from_value::<LargeFileStorage>(
|
||||
raw_lfs_opt.unwrap(),
|
||||
)
|
||||
.map_err(|err| {
|
||||
tracing::error!(
|
||||
"Value stored in large_file_storage column is invalid and could not be deserialized: {}",
|
||||
err
|
||||
);
|
||||
error::Error::InternalErr(
|
||||
"Could not deserialize LargeFileStorage value found in database".to_string(),
|
||||
)
|
||||
})?;
|
||||
let s3_lfs = match large_file_storage {
|
||||
LargeFileStorage::S3Storage(s3_lfs) => s3_lfs,
|
||||
};
|
||||
|
||||
let resource_path_json_value = serde_json::to_value(s3_lfs.s3_resource_path)
|
||||
.map_err(|err| error::Error::InternalErr(err.to_string()))?;
|
||||
let interpolated_value = transform_json_value(
|
||||
authed,
|
||||
user_db,
|
||||
&w_id,
|
||||
resource_path_json_value,
|
||||
&Option::None,
|
||||
token,
|
||||
)
|
||||
.await?;
|
||||
|
||||
let s3_resource = serde_json::from_value::<S3Resource>(interpolated_value)
|
||||
.map_err(|err| error::Error::InternalErr(err.to_string()))?;
|
||||
return Ok(Some(s3_resource));
|
||||
}
|
||||
|
||||
fn build_s3_client(s3_resource_ref: &S3Resource) -> aws_sdk_s3::Client {
|
||||
let s3_resource = s3_resource_ref.clone();
|
||||
|
||||
let endpoint_with_prefix = if s3_resource.endpoint.starts_with("http://")
|
||||
|| s3_resource.endpoint.starts_with("https://")
|
||||
{
|
||||
s3_resource.endpoint.clone()
|
||||
} else if s3_resource.use_ssl {
|
||||
format!("https://{}", s3_resource.endpoint)
|
||||
} else {
|
||||
format!("http://{}", s3_resource.endpoint)
|
||||
};
|
||||
|
||||
let mut s3_config_builder = aws_sdk_s3::Config::builder()
|
||||
.endpoint_url(endpoint_with_prefix)
|
||||
.region(Region::new(s3_resource.region));
|
||||
if s3_resource.access_key.is_some() {
|
||||
s3_config_builder = s3_config_builder.credentials_provider(Credentials::new(
|
||||
s3_resource.access_key.unwrap_or_default(),
|
||||
s3_resource.secret_key.unwrap_or_default(),
|
||||
None,
|
||||
None,
|
||||
"s3_storage",
|
||||
));
|
||||
}
|
||||
if s3_resource.path_style {
|
||||
s3_config_builder = s3_config_builder.force_path_style(true);
|
||||
}
|
||||
let s3_config = s3_config_builder.build();
|
||||
return aws_sdk_s3::Client::from_conf(s3_config);
|
||||
}
|
||||
@@ -57,6 +57,7 @@ mod granular_acls;
|
||||
mod groups;
|
||||
mod inputs;
|
||||
mod integration;
|
||||
pub mod job_helpers;
|
||||
pub mod jobs;
|
||||
pub mod oauth2;
|
||||
mod openai;
|
||||
@@ -179,31 +180,32 @@ pub async fn run_server(
|
||||
// Reordered alphabetically
|
||||
.nest("/acls", granular_acls::workspaced_service())
|
||||
.nest("/apps", apps::workspaced_service())
|
||||
.nest("/raw_apps", raw_apps::workspaced_service())
|
||||
.nest("/audit", audit::workspaced_service())
|
||||
.nest("/capture", capture::workspaced_service())
|
||||
.nest(
|
||||
"/embeddings",
|
||||
embeddings::workspaced_service(embeddings_db.clone()),
|
||||
)
|
||||
.nest("/drafts", drafts::workspaced_service())
|
||||
.nest("/favorites", favorite::workspaced_service())
|
||||
.nest("/flows", flows::workspaced_service())
|
||||
.nest("/folders", folders::workspaced_service())
|
||||
.nest("/groups", groups::workspaced_service())
|
||||
.nest("/inputs", inputs::workspaced_service())
|
||||
.nest("/job_helpers", job_helpers::workspaced_service())
|
||||
.nest("/jobs", jobs::workspaced_service())
|
||||
.nest("/oauth", oauth2::workspaced_service())
|
||||
.nest("/openai", openai::workspaced_service())
|
||||
.nest("/raw_apps", raw_apps::workspaced_service())
|
||||
.nest("/resources", resources::workspaced_service())
|
||||
.nest("/schedules", schedule::workspaced_service())
|
||||
.nest("/scripts", scripts::workspaced_service())
|
||||
.nest("/drafts", drafts::workspaced_service())
|
||||
.nest(
|
||||
"/users",
|
||||
users::workspaced_service().layer(Extension(argon2.clone())),
|
||||
)
|
||||
.nest("/variables", variables::workspaced_service())
|
||||
.nest("/workspaces", workspaces::workspaced_service())
|
||||
.nest("/openai", openai::workspaced_service())
|
||||
.nest(
|
||||
"/embeddings",
|
||||
embeddings::workspaced_service(embeddings_db.clone()),
|
||||
),
|
||||
.nest("/workspaces", workspaces::workspaced_service()),
|
||||
)
|
||||
.nest("/workspaces", workspaces::global_service())
|
||||
.nest(
|
||||
|
||||
@@ -79,7 +79,8 @@ pub fn workspaced_service() -> Router {
|
||||
.route("/premium_info", get(premium_info))
|
||||
.route("/edit_copilot_config", post(edit_copilot_config))
|
||||
.route("/get_copilot_info", get(get_copilot_info) )
|
||||
.route("/edit_error_handler", post(edit_error_handler));
|
||||
.route("/edit_error_handler", post(edit_error_handler))
|
||||
.route("/edit_large_file_storage_config", post(edit_large_file_storage_config));
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
{
|
||||
@@ -143,6 +144,7 @@ pub struct WorkspaceSettings {
|
||||
pub error_handler: Option<String>,
|
||||
pub error_handler_extra_args: Option<serde_json::Value>,
|
||||
pub error_handler_muted_on_cancel: Option<bool>,
|
||||
pub large_file_storage: Option<serde_json::Value>, // effectively: DatasetsStorage
|
||||
}
|
||||
|
||||
#[derive(FromRow, Serialize, Debug)]
|
||||
@@ -199,6 +201,11 @@ struct EditCopilotConfig {
|
||||
code_completion_enabled: bool,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct EditLargeFileStorageConfig {
|
||||
large_file_storage: Option<LargeFileStorage>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct CreateWorkspace {
|
||||
id: String,
|
||||
@@ -258,6 +265,18 @@ pub struct EditErrorHandler {
|
||||
pub error_handler_muted_on_cancel: Option<bool>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug)]
|
||||
#[serde(tag = "type")]
|
||||
pub enum LargeFileStorage {
|
||||
S3Storage(S3Storage),
|
||||
// TODO: Add a filesystem type here in the future if needed
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug)]
|
||||
pub struct S3Storage {
|
||||
pub s3_resource_path: String,
|
||||
}
|
||||
|
||||
async fn list_pending_invites(
|
||||
authed: ApiAuthed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
@@ -885,6 +904,52 @@ async fn get_copilot_info(
|
||||
}))
|
||||
}
|
||||
|
||||
async fn edit_large_file_storage_config(
|
||||
authed: ApiAuthed,
|
||||
Extension(db): Extension<DB>,
|
||||
Path(w_id): Path<String>,
|
||||
ApiAuthed { is_admin, username, .. }: ApiAuthed,
|
||||
Json(new_config): Json<EditLargeFileStorageConfig>,
|
||||
) -> Result<String> {
|
||||
require_admin(is_admin, &username)?;
|
||||
|
||||
let mut tx = db.begin().await?;
|
||||
|
||||
let args_for_audit = format!("{:?}", new_config.large_file_storage);
|
||||
audit_log(
|
||||
&mut *tx,
|
||||
&authed.username,
|
||||
"workspaces.edit_large_file_storage_config",
|
||||
ActionKind::Update,
|
||||
&w_id,
|
||||
Some(&authed.email),
|
||||
Some([("large_file_storage", args_for_audit.as_str())].into()),
|
||||
)
|
||||
.await?;
|
||||
|
||||
if let Some(lfs_config) = new_config.large_file_storage {
|
||||
let serialized_lfs_config = serde_json::to_value::<LargeFileStorage>(lfs_config)
|
||||
.map_err(|err| Error::InternalErr(err.to_string()))?;
|
||||
|
||||
sqlx::query!(
|
||||
"UPDATE workspace_settings SET large_file_storage = $1 WHERE workspace_id = $2",
|
||||
serialized_lfs_config,
|
||||
&w_id
|
||||
)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
} else {
|
||||
sqlx::query!(
|
||||
"UPDATE workspace_settings SET large_file_storage = NULL WHERE workspace_id = $1",
|
||||
&w_id,
|
||||
)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
}
|
||||
tx.commit().await?;
|
||||
|
||||
Ok(format!("Edit copilot config for workspace {}", &w_id))
|
||||
}
|
||||
|
||||
async fn edit_error_handler(
|
||||
authed: ApiAuthed,
|
||||
|
||||
@@ -14,7 +14,13 @@
|
||||
import Tooltip from '$lib/components/Tooltip.svelte'
|
||||
import WorkspaceUserSettings from '$lib/components/settings/WorkspaceUserSettings.svelte'
|
||||
import { WORKSPACE_SHOW_SLACK_CMD, WORKSPACE_SHOW_WEBHOOK_CLI_SYNC } from '$lib/consts'
|
||||
import { OauthService, Script, WorkspaceService } from '$lib/gen'
|
||||
import {
|
||||
LargeFileStorage,
|
||||
OauthService,
|
||||
Script,
|
||||
WorkspaceService,
|
||||
HelpersService
|
||||
} from '$lib/gen'
|
||||
import {
|
||||
enterpriseLicense,
|
||||
copilotInfo,
|
||||
@@ -47,6 +53,7 @@
|
||||
let errorHandlerExtraArgs: Record<string, any> = {}
|
||||
let errorHandlerMutedOnCancel: boolean | undefined = undefined
|
||||
let openaiResourceInitialPath: string | undefined = undefined
|
||||
let s3ResourceInitialPath: string | undefined = undefined
|
||||
let codeCompletionEnabled: boolean = false
|
||||
let tab =
|
||||
($page.url.searchParams.get('tab') as
|
||||
@@ -148,6 +155,31 @@
|
||||
sendUserToast(`Copilot settings updated`)
|
||||
}
|
||||
|
||||
async function editWindmillLFSSettings(s3ResourcePath: string): Promise<void> {
|
||||
s3ResourceInitialPath = s3ResourcePath
|
||||
if (s3ResourcePath) {
|
||||
let resourcePathWithPrefix = `$res:${s3ResourcePath}`
|
||||
await WorkspaceService.editLargeFileStorageConfig({
|
||||
workspace: $workspaceStore!,
|
||||
requestBody: {
|
||||
large_file_storage: {
|
||||
type: LargeFileStorage.type.S3STORAGE,
|
||||
s3_resource_path: resourcePathWithPrefix
|
||||
}
|
||||
}
|
||||
})
|
||||
sendUserToast(`Large file storage settings updated`)
|
||||
} else {
|
||||
await WorkspaceService.editLargeFileStorageConfig({
|
||||
workspace: $workspaceStore!,
|
||||
requestBody: {
|
||||
large_file_storage: undefined
|
||||
}
|
||||
})
|
||||
sendUserToast(`Large file storage settings reset`)
|
||||
}
|
||||
}
|
||||
|
||||
async function loadSettings(): Promise<void> {
|
||||
const settings = await WorkspaceService.getSettings({ workspace: $workspaceStore! })
|
||||
team_name = settings.slack_name
|
||||
@@ -177,6 +209,11 @@
|
||||
}
|
||||
errorHandlerExtraArgs = settings.error_handler_extra_args ?? {}
|
||||
codeCompletionEnabled = settings.code_completion_enabled
|
||||
console.log(settings)
|
||||
s3ResourceInitialPath =
|
||||
settings.large_file_storage?.type === LargeFileStorage.type.S3STORAGE
|
||||
? settings.large_file_storage?.s3_resource_path?.replace('$res:', '')
|
||||
: undefined
|
||||
}
|
||||
|
||||
$: {
|
||||
@@ -259,7 +296,6 @@
|
||||
<Tab size="xs" value="error_handler">
|
||||
<div class="flex gap-2 items-center my-1">Error Handler</div>
|
||||
</Tab>
|
||||
|
||||
<Tab size="xs" value="openai">
|
||||
<div class="flex gap-2 items-center my-1">Windmill AI</div>
|
||||
</Tab>
|
||||
@@ -543,6 +579,33 @@
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
{:else if tab == 'windmill_lfs'}
|
||||
<PageHeader title="Windmill Large File Storage" primary={false} />
|
||||
<div class="mt-5 flex gap-1">
|
||||
{#key s3ResourceInitialPath}
|
||||
<ResourcePicker
|
||||
resourceType="s3"
|
||||
initialValue={s3ResourceInitialPath}
|
||||
on:change={(ev) => {
|
||||
editWindmillLFSSettings(ev.detail)
|
||||
}}
|
||||
/>
|
||||
{/key}
|
||||
<Button
|
||||
size="sm"
|
||||
variant="contained"
|
||||
color="dark"
|
||||
disabled={!s3ResourceInitialPath}
|
||||
on:click={async () => {
|
||||
if ($workspaceStore) {
|
||||
await HelpersService.datasetStorageTestConnection({
|
||||
workspace: $workspaceStore
|
||||
})
|
||||
sendUserToast('Connection successful')
|
||||
}
|
||||
}}>Test Connection</Button
|
||||
>
|
||||
</div>
|
||||
{/if}
|
||||
{:else}
|
||||
<div class="bg-red-100 border-l-4 border-red-600 text-orange-700 p-4 m-4" role="alert">
|
||||
|
||||
1
python-client/.gitignore
vendored
1
python-client/.gitignore
vendored
@@ -1 +0,0 @@
|
||||
windmill-api/
|
||||
|
||||
@@ -40,4 +40,4 @@ mv windmill-api/README.md.tmp windmill-api/README.md
|
||||
cd windmill-api && poetry build
|
||||
cd ../wmill && poetry build
|
||||
cd ../wmill_pg && poetry build
|
||||
cd .. && echo "windmill-api/" >> .gitignore
|
||||
cd .. && echo "windmill-api/" >> .gitignore
|
||||
|
||||
16
python-client/tests/README.md
Normal file
16
python-client/tests/README.md
Normal file
@@ -0,0 +1,16 @@
|
||||
Testing pythong wmill client
|
||||
============================
|
||||
|
||||
Make sure you have a local windmill BE running and listening on localhost:8000 (either using cargo run or via the docker compose).
|
||||
|
||||
Install the local package to your virtual env:
|
||||
```bash
|
||||
# this is necessary only if you made a change to the BE API via the openapi.yml file
|
||||
cd ./windmill-api # you can generate it using the build.sh script at the root of this repo
|
||||
pip3 install .
|
||||
|
||||
cd ./wmill
|
||||
pip3 install .
|
||||
```
|
||||
|
||||
Then you can run go to `wmill_client_test.py` and add a Token and Workspace if necessary. You can then implement your own test calling any function in the wmill client and test its output.
|
||||
69
python-client/tests/wmill_client_test.py
Normal file
69
python-client/tests/wmill_client_test.py
Normal file
@@ -0,0 +1,69 @@
|
||||
import unittest
|
||||
import wmill
|
||||
import os
|
||||
|
||||
|
||||
class TestStringMethods(unittest.TestCase):
|
||||
_token = "<TOKEN>"
|
||||
_workspace = "<WORKSPACE>"
|
||||
|
||||
def setUp(self):
|
||||
os.environ["WM_WORKSPACE"] = self._workspace
|
||||
os.environ["WM_TOKEN"] = self._token
|
||||
|
||||
def test_duckdb_connection_settings(self):
|
||||
s3_resource = {
|
||||
"port": 9000,
|
||||
"bucket": "windmill",
|
||||
"region": "fr-paris",
|
||||
"useSSL": False,
|
||||
"endPoint": "localhost:9000",
|
||||
"accessKey": "ACCESS_KEY",
|
||||
"pathStyle": True,
|
||||
"secretKey": "SECRET_KEY",
|
||||
}
|
||||
|
||||
settings = wmill.duckdb_connection_settings(s3_resource)
|
||||
self.assertIsNotNone(settings)
|
||||
|
||||
expected_settings_str = """SET home_directory='./shared/';
|
||||
INSTALL 'httpfs';
|
||||
SET s3_url_style='path';
|
||||
SET s3_region='fr-paris';
|
||||
SET s3_endpoint='localhost:9000';
|
||||
SET s3_use_ssl=0;
|
||||
SET s3_access_key_id='ACCESS_KEY';
|
||||
SET s3_secret_access_key='SECRET_KEY';
|
||||
"""
|
||||
self.assertEqual(settings, {"connection_settings_str": expected_settings_str})
|
||||
|
||||
settings = wmill.polars_connection_settings(s3_resource)
|
||||
print(settings)
|
||||
|
||||
def test_polars_connection_settings(self):
|
||||
s3_resource = {
|
||||
"port": 9000,
|
||||
"bucket": "windmill",
|
||||
"region": "fr-paris",
|
||||
"useSSL": False,
|
||||
"endPoint": "localhost:9000",
|
||||
"accessKey": "ACCESS_KEY",
|
||||
"pathStyle": True,
|
||||
"secretKey": "SECRET_KEY",
|
||||
}
|
||||
|
||||
settings = wmill.polars_connection_settings(s3_resource)
|
||||
print(settings)
|
||||
expected_settings = {
|
||||
"cache_regions": False,
|
||||
"client_kwargs": {"region_name": "fr-paris"},
|
||||
"endpoint_url": "localhost:9000",
|
||||
"key": "ACCESS_KEY",
|
||||
"secret": "SECRET_KEY",
|
||||
"use_ssl": False,
|
||||
}
|
||||
self.assertEqual(settings, expected_settings)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -34,9 +34,7 @@ class JobStatus(Enum):
|
||||
_client: "AuthenticatedClient | None" = None
|
||||
|
||||
|
||||
def create_client(
|
||||
base_url: "str | None" = None, token: "str | None" = None
|
||||
) -> AuthenticatedClient:
|
||||
def create_client(base_url: "str | None" = None, token: "str | None" = None) -> AuthenticatedClient:
|
||||
env_base_url = os.environ.get("BASE_INTERNAL_URL")
|
||||
|
||||
if env_base_url is not None:
|
||||
@@ -46,9 +44,7 @@ def create_client(
|
||||
token_: str = token or os.environ.get("WM_TOKEN") or ""
|
||||
global _client
|
||||
if _client is None:
|
||||
_client = AuthenticatedClient(
|
||||
base_url=base_url_, token=token_, timeout=30, verify_ssl=False
|
||||
)
|
||||
_client = AuthenticatedClient(base_url=base_url_, token=token_, timeout=30, verify_ssl=False)
|
||||
return _client
|
||||
|
||||
|
||||
@@ -65,9 +61,7 @@ def get_version() -> str:
|
||||
"""
|
||||
from windmill_api.api.settings import backend_version
|
||||
|
||||
return backend_version.sync_detailed(client=create_client()).content.decode(
|
||||
"us-ascii"
|
||||
)
|
||||
return backend_version.sync_detailed(client=create_client()).content.decode("us-ascii")
|
||||
|
||||
|
||||
def run_script_async(
|
||||
@@ -92,9 +86,7 @@ def run_script_async(
|
||||
).content.decode("us-ascii")
|
||||
|
||||
|
||||
def run_script_sync(
|
||||
hash: str, args: Dict[str, Any] = {}, verbose: bool = False
|
||||
) -> Dict[str, Any]:
|
||||
def run_script_sync(hash: str, args: Dict[str, Any] = {}, verbose: bool = False) -> Dict[str, Any]:
|
||||
"""
|
||||
Run a script, wait for it to complete and return the result of the launched script
|
||||
"""
|
||||
@@ -133,9 +125,7 @@ def run_script_by_path_async(
|
||||
).content.decode("us-ascii")
|
||||
|
||||
|
||||
def run_script_by_path_sync(
|
||||
path: str, args: Dict[str, Any] = {}, verbose: bool = False
|
||||
) -> Dict[str, Any]:
|
||||
def run_script_by_path_sync(path: str, args: Dict[str, Any] = {}, verbose: bool = False) -> Dict[str, Any]:
|
||||
"""
|
||||
Run a script, wait for it to complete and return the result of the launched script
|
||||
"""
|
||||
@@ -159,9 +149,7 @@ def get_job_status(job_id: str) -> JobStatus:
|
||||
from windmill_api.models.get_job_response_200_type import GetJobResponse200Type
|
||||
from windmill_api.api.job import get_job
|
||||
|
||||
res = get_job.sync_detailed(
|
||||
client=create_client(), workspace=get_workspace(), id=job_id
|
||||
).parsed
|
||||
res = get_job.sync_detailed(client=create_client(), workspace=get_workspace(), id=job_id).parsed
|
||||
if not res:
|
||||
raise Exception(f"Job {job_id} not found")
|
||||
elif not res.type:
|
||||
@@ -183,9 +171,7 @@ def get_result(job_id: str) -> Dict[str, Any]:
|
||||
"""
|
||||
from windmill_api.api.job import get_completed_job
|
||||
|
||||
res = get_completed_job.sync_detailed(
|
||||
client=create_client(), workspace=get_workspace(), id=job_id
|
||||
).parsed
|
||||
res = get_completed_job.sync_detailed(client=create_client(), workspace=get_workspace(), id=job_id).parsed
|
||||
if not res:
|
||||
raise Exception(f"Job {job_id} not found")
|
||||
if not res.result:
|
||||
@@ -203,9 +189,7 @@ def get_resource(path: str | None = None, none_if_undefined: bool = False) -> An
|
||||
)
|
||||
|
||||
path = path or get_state_path()
|
||||
parsed = get_resource_api.sync_detailed(
|
||||
workspace=get_workspace(), path=path, client=create_client()
|
||||
)
|
||||
parsed = get_resource_api.sync_detailed(workspace=get_workspace(), path=path, client=create_client())
|
||||
try:
|
||||
content = parsed.content.decode("utf-8")
|
||||
parsed = json.loads(content)
|
||||
@@ -223,6 +207,63 @@ def get_resource(path: str | None = None, none_if_undefined: bool = False) -> An
|
||||
return parsed
|
||||
|
||||
|
||||
def duckdb_connection_settings(s3_resource: Any, none_if_undefined: bool = False) -> str | None:
|
||||
"""
|
||||
Convenient helpers that takes an S3 resource as input and returns the settings necessary to
|
||||
initiate an S3 connection from DuckDB
|
||||
"""
|
||||
from windmill_api.api.helpers import duckdb_connection_settings
|
||||
from windmill_api.models.s3_resource import S3Resource
|
||||
|
||||
parsed = duckdb_connection_settings.sync_detailed(
|
||||
workspace=get_workspace(),
|
||||
client=create_client(),
|
||||
json_body={
|
||||
"s3_resource": s3_resource,
|
||||
},
|
||||
)
|
||||
try:
|
||||
content = parsed.content.decode("utf-8")
|
||||
parsed = json.loads(content)
|
||||
except:
|
||||
parsed = None
|
||||
|
||||
if parsed is None:
|
||||
if none_if_undefined:
|
||||
return None
|
||||
else:
|
||||
raise Exception(f"Could not generate DuckDB S3 connection settings from the provided resource")
|
||||
return parsed
|
||||
|
||||
|
||||
def polars_connection_settings(s3_resource: Any, none_if_undefined: bool = False) -> Any:
|
||||
"""
|
||||
Convenient helpers that takes an S3 resource as input and returns the settings necessary to
|
||||
initiate an S3 connection from Polars
|
||||
"""
|
||||
from windmill_api.api.helpers import polars_connection_settings
|
||||
|
||||
parsed = polars_connection_settings.sync_detailed(
|
||||
workspace=get_workspace(),
|
||||
client=create_client(),
|
||||
json_body={
|
||||
"s3_resource": s3_resource,
|
||||
},
|
||||
)
|
||||
try:
|
||||
content = parsed.content.decode("utf-8")
|
||||
parsed = json.loads(content)
|
||||
except:
|
||||
parsed = None
|
||||
|
||||
if parsed is None:
|
||||
if none_if_undefined:
|
||||
return None
|
||||
else:
|
||||
raise Exception(f"Could not generate Polars S3 connection settings from the provided resource")
|
||||
return parsed
|
||||
|
||||
|
||||
def whoami() -> WhoamiResponse200 | None:
|
||||
"""
|
||||
Returns the current user
|
||||
@@ -239,9 +280,7 @@ def get_state() -> Any:
|
||||
return get_resource(None, True)
|
||||
|
||||
|
||||
def set_resource(
|
||||
value: Any, path: str | None = None, resource_type: str = "state"
|
||||
) -> None:
|
||||
def set_resource(value: Any, path: str | None = None, resource_type: str = "state") -> None:
|
||||
"""
|
||||
Set the resource at a given path as a string, creating it if it does not exist
|
||||
"""
|
||||
@@ -258,15 +297,11 @@ def set_resource(
|
||||
path = path or get_state_path()
|
||||
workspace = get_workspace()
|
||||
client = create_client()
|
||||
if not exists_resource.sync_detailed(
|
||||
workspace=workspace, path=path, client=client
|
||||
).parsed:
|
||||
if not exists_resource.sync_detailed(workspace=workspace, path=path, client=client).parsed:
|
||||
create_resource.sync_detailed(
|
||||
workspace=workspace,
|
||||
client=client,
|
||||
json_body=CreateResourceJsonBody(
|
||||
path=path, value=value, resource_type=resource_type
|
||||
),
|
||||
json_body=CreateResourceJsonBody(path=path, value=value, resource_type=resource_type),
|
||||
)
|
||||
else:
|
||||
update_resource_value.sync_detailed(
|
||||
@@ -330,9 +365,7 @@ def get_variable(path: str) -> str:
|
||||
"""
|
||||
from windmill_api.api.variable import get_variable_value as get_variable_api
|
||||
|
||||
res = get_variable_api.sync_detailed(
|
||||
workspace=get_workspace(), path=path, client=create_client()
|
||||
)
|
||||
res = get_variable_api.sync_detailed(workspace=get_workspace(), path=path, client=create_client())
|
||||
parsed = res.parsed
|
||||
if parsed is None:
|
||||
raise Exception(
|
||||
@@ -357,15 +390,11 @@ def set_variable(path: str, value: str) -> None:
|
||||
|
||||
workspace = get_workspace()
|
||||
client = create_client()
|
||||
if not exists_variable.sync_detailed(
|
||||
workspace=workspace, path=path, client=client
|
||||
).parsed:
|
||||
if not exists_variable.sync_detailed(workspace=workspace, path=path, client=client).parsed:
|
||||
create_variable.sync_detailed(
|
||||
workspace=workspace,
|
||||
client=client,
|
||||
json_body=CreateVariableJsonBody(
|
||||
path=path, value=value, is_secret=False, description=""
|
||||
),
|
||||
json_body=CreateVariableJsonBody(path=path, value=value, is_secret=False, description=""),
|
||||
)
|
||||
else:
|
||||
update_variable.sync_detailed(
|
||||
@@ -392,9 +421,7 @@ def get_resume_urls(approver: str | None = None) -> Dict:
|
||||
import random
|
||||
|
||||
nonce = random.randint(0, 1000000000)
|
||||
res = get_resume_urls_api.sync_detailed(
|
||||
workspace, job_id, nonce, client=client, approver=approver
|
||||
)
|
||||
res = get_resume_urls_api.sync_detailed(workspace, job_id, nonce, client=client, approver=approver)
|
||||
if res.parsed is not None:
|
||||
return res.parsed.to_dict()
|
||||
else:
|
||||
|
||||
Reference in New Issue
Block a user