From 90093656b49f78d550012ff2e8627fc43b8e26ea Mon Sep 17 00:00:00 2001 From: Guillaume Bouvignies Date: Wed, 15 Nov 2023 19:37:02 +0100 Subject: [PATCH] feat: Add Python SDK capabilities to generate Polars and DuckDB connection settings to S3 (#2625) * feat: Add Python SDK capabilities to generate Polars and DuckDB connection settings to S3 * Add endpoint to list S3 objects * Add FE to set the workspace S3 resource * Fix openapi * sqlx prepare * Hide Windmil LFS tab --- ...e73766ae3330319619a68383b508a76f2d19c.json | 1 - ...c61296a3ff7489ae12f52a19f9543173ac597.json | 8 +- ...3ee6b27c836197df6454d3bb5c59441c34f44.json | 1 - ...2d1390ce5ac700b0bf71481d31747c8f2e870.json | 14 + ...8f051f27d3f21b615c468b85aa75b64d943db.json | 1 - ...7630a1d7b9deb8d33e9f909061f351f96844e.json | 8 +- ...8604a47f81b505fa9a713efa3519b050b546a.json | 22 + ...0bccb19beeb9ddfa308ca97f254cd5ba8157e.json | 1 - ...eda837cc63e4d8be912c0b5bfeea4a0c8db2e.json | 1 - ...a4fc993507e255004107345b7151322e063de.json | 1 - ...fbc44f1688035e1928b4c3649d875187b1e1d.json | 1 - ...096d1cf4a042c23526dc134c3b2d27904e954.json | 15 + ...ceb38751a6527827525a582d9f8804981f552.json | 1 - ...5ab9f0e98340cd33c6e29824f6030f0089071.json | 1 - backend/Cargo.lock | 511 +++++++ ...1254_workspace_s3_bucket_settings.down.sql | 2 + ...101254_workspace_s3_bucket_settings.up.sql | 2 + backend/windmill-api/Cargo.toml | 4 +- backend/windmill-api/openapi-deref.yaml | 1356 ++++++++++------- backend/windmill-api/openapi.yaml | 151 ++ backend/windmill-api/src/job_helpers.rs | 306 ++++ backend/windmill-api/src/lib.rs | 18 +- backend/windmill-api/src/workspaces.rs | 67 +- .../(logged)/workspace_settings/+page.svelte | 67 +- python-client/.gitignore | 1 - python-client/build.sh | 2 +- python-client/tests/README.md | 16 + python-client/tests/wmill_client_test.py | 69 + python-client/wmill/wmill/client.py | 117 +- 29 files changed, 2102 insertions(+), 663 deletions(-) create mode 100644 backend/.sqlx/query-2ff397ad0aeabdc15776025f75c2d1390ce5ac700b0bf71481d31747c8f2e870.json create mode 100644 backend/.sqlx/query-5e4e0740e1cff4b2a7c620da3758604a47f81b505fa9a713efa3519b050b546a.json create mode 100644 backend/.sqlx/query-bf27083af3514e5fae868364518096d1cf4a042c23526dc134c3b2d27904e954.json create mode 100644 backend/migrations/20231115101254_workspace_s3_bucket_settings.down.sql create mode 100644 backend/migrations/20231115101254_workspace_s3_bucket_settings.up.sql create mode 100644 backend/windmill-api/src/job_helpers.rs create mode 100644 python-client/tests/README.md create mode 100644 python-client/tests/wmill_client_test.py diff --git a/backend/.sqlx/query-11ae5e1aade2bc9a78af2d956ece73766ae3330319619a68383b508a76f2d19c.json b/backend/.sqlx/query-11ae5e1aade2bc9a78af2d956ece73766ae3330319619a68383b508a76f2d19c.json index d54ce25120..2cb08ba2d4 100644 --- a/backend/.sqlx/query-11ae5e1aade2bc9a78af2d956ece73766ae3330319619a68383b508a76f2d19c.json +++ b/backend/.sqlx/query-11ae5e1aade2bc9a78af2d956ece73766ae3330319619a68383b508a76f2d19c.json @@ -42,7 +42,6 @@ "bash", "postgresql", "nativets", - "Nativets", "bun", "mysql", "bigquery", diff --git a/backend/.sqlx/query-1730f39fd1793d45fbb41b21389c61296a3ff7489ae12f52a19f9543173ac597.json b/backend/.sqlx/query-1730f39fd1793d45fbb41b21389c61296a3ff7489ae12f52a19f9543173ac597.json index 05d67add73..adf3055d35 100644 --- a/backend/.sqlx/query-1730f39fd1793d45fbb41b21389c61296a3ff7489ae12f52a19f9543173ac597.json +++ b/backend/.sqlx/query-1730f39fd1793d45fbb41b21389c61296a3ff7489ae12f52a19f9543173ac597.json @@ -82,6 +82,11 @@ "ordinal": 15, "name": "error_handler_muted_on_cancel", "type_info": "Bool" + }, + { + "ordinal": 16, + "name": "large_file_storage", + "type_info": "Jsonb" } ], "parameters": { @@ -105,7 +110,8 @@ true, false, true, - false + false, + true ] }, "hash": "1730f39fd1793d45fbb41b21389c61296a3ff7489ae12f52a19f9543173ac597" diff --git a/backend/.sqlx/query-254776959f60455a00a2c29a1783ee6b27c836197df6454d3bb5c59441c34f44.json b/backend/.sqlx/query-254776959f60455a00a2c29a1783ee6b27c836197df6454d3bb5c59441c34f44.json index 297c78d0e7..5aed763be0 100644 --- a/backend/.sqlx/query-254776959f60455a00a2c29a1783ee6b27c836197df6454d3bb5c59441c34f44.json +++ b/backend/.sqlx/query-254776959f60455a00a2c29a1783ee6b27c836197df6454d3bb5c59441c34f44.json @@ -67,7 +67,6 @@ "bash", "postgresql", "nativets", - "Nativets", "bun", "mysql", "bigquery", diff --git a/backend/.sqlx/query-2ff397ad0aeabdc15776025f75c2d1390ce5ac700b0bf71481d31747c8f2e870.json b/backend/.sqlx/query-2ff397ad0aeabdc15776025f75c2d1390ce5ac700b0bf71481d31747c8f2e870.json new file mode 100644 index 0000000000..2bb714a885 --- /dev/null +++ b/backend/.sqlx/query-2ff397ad0aeabdc15776025f75c2d1390ce5ac700b0bf71481d31747c8f2e870.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings SET large_file_storage = NULL WHERE workspace_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "2ff397ad0aeabdc15776025f75c2d1390ce5ac700b0bf71481d31747c8f2e870" +} diff --git a/backend/.sqlx/query-52ad1773a9f5a1b6be1bb16af408f051f27d3f21b615c468b85aa75b64d943db.json b/backend/.sqlx/query-52ad1773a9f5a1b6be1bb16af408f051f27d3f21b615c468b85aa75b64d943db.json index f383776194..bf25140825 100644 --- a/backend/.sqlx/query-52ad1773a9f5a1b6be1bb16af408f051f27d3f21b615c468b85aa75b64d943db.json +++ b/backend/.sqlx/query-52ad1773a9f5a1b6be1bb16af408f051f27d3f21b615c468b85aa75b64d943db.json @@ -28,7 +28,6 @@ "bash", "postgresql", "nativets", - "Nativets", "bun", "mysql", "bigquery", diff --git a/backend/.sqlx/query-5445083864b2b092b012e894bff7630a1d7b9deb8d33e9f909061f351f96844e.json b/backend/.sqlx/query-5445083864b2b092b012e894bff7630a1d7b9deb8d33e9f909061f351f96844e.json index c7b7991492..525891e500 100644 --- a/backend/.sqlx/query-5445083864b2b092b012e894bff7630a1d7b9deb8d33e9f909061f351f96844e.json +++ b/backend/.sqlx/query-5445083864b2b092b012e894bff7630a1d7b9deb8d33e9f909061f351f96844e.json @@ -82,6 +82,11 @@ "ordinal": 15, "name": "error_handler_muted_on_cancel", "type_info": "Bool" + }, + { + "ordinal": 16, + "name": "large_file_storage", + "type_info": "Jsonb" } ], "parameters": { @@ -105,7 +110,8 @@ true, false, true, - false + false, + true ] }, "hash": "5445083864b2b092b012e894bff7630a1d7b9deb8d33e9f909061f351f96844e" diff --git a/backend/.sqlx/query-5e4e0740e1cff4b2a7c620da3758604a47f81b505fa9a713efa3519b050b546a.json b/backend/.sqlx/query-5e4e0740e1cff4b2a7c620da3758604a47f81b505fa9a713efa3519b050b546a.json new file mode 100644 index 0000000000..129074f3fb --- /dev/null +++ b/backend/.sqlx/query-5e4e0740e1cff4b2a7c620da3758604a47f81b505fa9a713efa3519b050b546a.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT large_file_storage FROM workspace_settings WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "large_file_storage", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "5e4e0740e1cff4b2a7c620da3758604a47f81b505fa9a713efa3519b050b546a" +} diff --git a/backend/.sqlx/query-620ddf29c5e867079df4c2aa6e80bccb19beeb9ddfa308ca97f254cd5ba8157e.json b/backend/.sqlx/query-620ddf29c5e867079df4c2aa6e80bccb19beeb9ddfa308ca97f254cd5ba8157e.json index bc67744b37..e706b0c58f 100644 --- a/backend/.sqlx/query-620ddf29c5e867079df4c2aa6e80bccb19beeb9ddfa308ca97f254cd5ba8157e.json +++ b/backend/.sqlx/query-620ddf29c5e867079df4c2aa6e80bccb19beeb9ddfa308ca97f254cd5ba8157e.json @@ -60,7 +60,6 @@ "bash", "postgresql", "nativets", - "Nativets", "bun", "mysql", "bigquery", diff --git a/backend/.sqlx/query-9d3556319411a27a875bf6cf0e5eda837cc63e4d8be912c0b5bfeea4a0c8db2e.json b/backend/.sqlx/query-9d3556319411a27a875bf6cf0e5eda837cc63e4d8be912c0b5bfeea4a0c8db2e.json index 6008792091..f33a341566 100644 --- a/backend/.sqlx/query-9d3556319411a27a875bf6cf0e5eda837cc63e4d8be912c0b5bfeea4a0c8db2e.json +++ b/backend/.sqlx/query-9d3556319411a27a875bf6cf0e5eda837cc63e4d8be912c0b5bfeea4a0c8db2e.json @@ -40,7 +40,6 @@ "bash", "postgresql", "nativets", - "Nativets", "bun", "mysql", "bigquery", diff --git a/backend/.sqlx/query-a9915d3c72a278ff08fbe6571c2a4fc993507e255004107345b7151322e063de.json b/backend/.sqlx/query-a9915d3c72a278ff08fbe6571c2a4fc993507e255004107345b7151322e063de.json index 54d56094f2..01d0629ea2 100644 --- a/backend/.sqlx/query-a9915d3c72a278ff08fbe6571c2a4fc993507e255004107345b7151322e063de.json +++ b/backend/.sqlx/query-a9915d3c72a278ff08fbe6571c2a4fc993507e255004107345b7151322e063de.json @@ -46,7 +46,6 @@ "bash", "postgresql", "nativets", - "Nativets", "bun", "mysql", "bigquery", diff --git a/backend/.sqlx/query-b6074eb9369de071f79165bf506fbc44f1688035e1928b4c3649d875187b1e1d.json b/backend/.sqlx/query-b6074eb9369de071f79165bf506fbc44f1688035e1928b4c3649d875187b1e1d.json index ac27ab1d70..f5122a7f3c 100644 --- a/backend/.sqlx/query-b6074eb9369de071f79165bf506fbc44f1688035e1928b4c3649d875187b1e1d.json +++ b/backend/.sqlx/query-b6074eb9369de071f79165bf506fbc44f1688035e1928b4c3649d875187b1e1d.json @@ -42,7 +42,6 @@ "bash", "postgresql", "nativets", - "Nativets", "bun", "mysql", "bigquery", diff --git a/backend/.sqlx/query-bf27083af3514e5fae868364518096d1cf4a042c23526dc134c3b2d27904e954.json b/backend/.sqlx/query-bf27083af3514e5fae868364518096d1cf4a042c23526dc134c3b2d27904e954.json new file mode 100644 index 0000000000..065f940888 --- /dev/null +++ b/backend/.sqlx/query-bf27083af3514e5fae868364518096d1cf4a042c23526dc134c3b2d27904e954.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings SET large_file_storage = $1 WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Text" + ] + }, + "nullable": [] + }, + "hash": "bf27083af3514e5fae868364518096d1cf4a042c23526dc134c3b2d27904e954" +} diff --git a/backend/.sqlx/query-c7febee7146a28481ce36f3570bceb38751a6527827525a582d9f8804981f552.json b/backend/.sqlx/query-c7febee7146a28481ce36f3570bceb38751a6527827525a582d9f8804981f552.json index 18342de0fe..603bea689f 100644 --- a/backend/.sqlx/query-c7febee7146a28481ce36f3570bceb38751a6527827525a582d9f8804981f552.json +++ b/backend/.sqlx/query-c7febee7146a28481ce36f3570bceb38751a6527827525a582d9f8804981f552.json @@ -42,7 +42,6 @@ "bash", "postgresql", "nativets", - "Nativets", "bun", "mysql", "bigquery", diff --git a/backend/.sqlx/query-d48070ef889723f408ed6c1a97b5ab9f0e98340cd33c6e29824f6030f0089071.json b/backend/.sqlx/query-d48070ef889723f408ed6c1a97b5ab9f0e98340cd33c6e29824f6030f0089071.json index 5b24664a2f..53974ad40e 100644 --- a/backend/.sqlx/query-d48070ef889723f408ed6c1a97b5ab9f0e98340cd33c6e29824f6030f0089071.json +++ b/backend/.sqlx/query-d48070ef889723f408ed6c1a97b5ab9f0e98340cd33c6e29824f6030f0089071.json @@ -37,7 +37,6 @@ "bash", "postgresql", "nativets", - "Nativets", "bun", "mysql", "bigquery", diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 706688ee84..bf50e0bf42 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -503,6 +503,382 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "aws-config" +version = "0.57.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2bf00cb9416daab4ce4927c54ebe63c08b9caf4d7b9314b6d7a4a2c5a1afb09" +dependencies = [ + "aws-credential-types", + "aws-http", + "aws-runtime", + "aws-sdk-sso", + "aws-sdk-ssooidc", + "aws-sdk-sts", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand 2.0.1", + "hex", + "http", + "hyper", + "ring 0.17.5", + "time", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-credential-types" +version = "0.57.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb9073c88dbf12f68ce7d0e149f989627a1d1ae3d2b680459f04ccc29d1cbd0f" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "zeroize", +] + +[[package]] +name = "aws-http" +version = "0.57.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24067106d09620cf02d088166cdaedeaca7146d4d499c41b37accecbea11b246" +dependencies = [ + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "http", + "http-body", + "pin-project-lite", + "tracing", +] + +[[package]] +name = "aws-runtime" +version = "0.57.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc6ee0152c06d073602236a4e94a8c52a327d310c1ecd596570ce795af8777ff" +dependencies = [ + "aws-credential-types", + "aws-http", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "fastrand 2.0.1", + "http", + "percent-encoding", + "tracing", + "uuid 1.5.0", +] + +[[package]] +name = "aws-sdk-s3" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf188e8af89adb0b41ae2f21b6c666c618959e40242e79aaa43fafa9c0c90a8a" +dependencies = [ + "aws-credential-types", + "aws-http", + "aws-runtime", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-checksums", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "bytes", + "http", + "http-body", + "once_cell", + "percent-encoding", + "regex", + "tracing", + "url", +] + +[[package]] +name = "aws-sdk-sso" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2eb8158015232b4596ccef74a205600398e152d704b40b7ec9f486092474d7fa" +dependencies = [ + "aws-credential-types", + "aws-http", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "http", + "regex", + "tracing", +] + +[[package]] +name = "aws-sdk-ssooidc" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36a1493e1c57f173e53621935bfb5b6217376168dbdb4cd459aebcf645924a48" +dependencies = [ + "aws-credential-types", + "aws-http", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "http", + "regex", + "tracing", +] + +[[package]] +name = "aws-sdk-sts" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e032b77f5cd1dd3669d777a38ac08cbf8ec68e29460d4ef5d3e50cffa74ec75a" +dependencies = [ + "aws-credential-types", + "aws-http", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "http", + "regex", + "tracing", +] + +[[package]] +name = "aws-sigv4" +version = "0.57.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64f81a6abc4daab06b53cabf27c54189928893283093e37164ca53aa47488a5b" +dependencies = [ + "aws-credential-types", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-runtime-api", + "bytes", + "form_urlencoded", + "hex", + "hmac", + "http", + "num-bigint", + "once_cell", + "p256", + "percent-encoding", + "regex", + "ring 0.17.5", + "sha2 0.10.8", + "time", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-async" +version = "0.57.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbe53fccd3b10414b9cae63767a15a2789b34e6c6727b6e32b33e8c7998a3e80" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "aws-smithy-checksums" +version = "0.57.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb5701fbfb40600cc0fa547f318552dfd4e632b2099bd75d95fb0faae70675d" +dependencies = [ + "aws-smithy-http", + "aws-smithy-types", + "bytes", + "crc32c", + "crc32fast", + "hex", + "http", + "http-body", + "md-5 0.10.6", + "pin-project-lite", + "sha1", + "sha2 0.10.8", + "tracing", +] + +[[package]] +name = "aws-smithy-eventstream" +version = "0.57.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b33fa99f928a5815b94ee07e1377901bcf51aa749034a2c802dc38f9dcfacf5" +dependencies = [ + "aws-smithy-types", + "bytes", + "crc32fast", +] + +[[package]] +name = "aws-smithy-http" +version = "0.57.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7972373213d1d6e619c0edc9dda2d6634154e4ed75c5e0b2bf065cd5ec9f0d1" +dependencies = [ + "aws-smithy-eventstream", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "http", + "http-body", + "once_cell", + "percent-encoding", + "pin-project-lite", + "pin-utils", + "tracing", +] + +[[package]] +name = "aws-smithy-json" +version = "0.57.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d64d5af16dd585de9ff6c606423c1aaad47c6baa38de41c2beb32ef21c6645" +dependencies = [ + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-query" +version = "0.57.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7527bf5335154ba1b285479c50b630e44e93d1b4a759eaceb8d0bf9fbc82caa5" +dependencies = [ + "aws-smithy-types", + "urlencoding", +] + +[[package]] +name = "aws-smithy-runtime" +version = "0.57.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "839b363adf3b2bdab2742a1f540fec23039ea8bc9ec0f9f61df48470cfe5527b" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "fastrand 2.0.1", + "http", + "http-body", + "hyper", + "hyper-rustls", + "once_cell", + "pin-project-lite", + "pin-utils", + "rustls 0.21.8", + "tokio", + "tracing", +] + +[[package]] +name = "aws-smithy-runtime-api" +version = "0.57.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f24ecc446e62c3924539e7c18dec8038dba4fdf8718d5c2de62f9d2fecca8ba9" +dependencies = [ + "aws-smithy-async", + "aws-smithy-types", + "bytes", + "http", + "pin-project-lite", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-types" +version = "0.57.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "051de910296522a21178a2ea402ea59027eef4b63f1cef04a0be2bb5e25dea03" +dependencies = [ + "base64-simd", + "bytes", + "bytes-utils", + "futures-core", + "http", + "http-body", + "itoa", + "num-integer", + "pin-project-lite", + "pin-utils", + "ryu", + "serde", + "time", + "tokio", + "tokio-util", +] + +[[package]] +name = "aws-smithy-xml" +version = "0.57.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb1e3ac22c652662096c8e37a6f9af80c6f3520cab5610b2fe76c725bce18eac" +dependencies = [ + "xmlparser", +] + +[[package]] +name = "aws-types" +version = "0.57.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "048bbf1c24cdf4eb1efcdc243388a93a90ebf63979e25fc1c7b8cbd9cb6beb38" +dependencies = [ + "aws-credential-types", + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "http", + "rustc_version 0.4.0", + "tracing", +] + [[package]] name = "axum" version = "0.6.20" @@ -568,6 +944,12 @@ dependencies = [ "rustc-demangle", ] +[[package]] +name = "base16ct" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349a06037c7bf932dd7e7d1f653678b2038b9ad46a74102f1fc7bd7872678cce" + [[package]] name = "base64" version = "0.13.1" @@ -911,6 +1293,16 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" +[[package]] +name = "bytes-utils" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e47d3a8076e283f3acd27400535992edb3ba4b5bb72f8891ad8fbe7932a7d4b9" +dependencies = [ + "bytes", + "either", +] + [[package]] name = "bzip2" version = "0.4.4" @@ -1312,6 +1704,15 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" +[[package]] +name = "crc32c" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8f48d60e5b4d2c53d5c2b1d8a58c849a70ae5e5509b08a48d047e3b65714a74" +dependencies = [ + "rustc_version 0.4.0", +] + [[package]] name = "crc32fast" version = "1.3.2" @@ -1405,6 +1806,18 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" +[[package]] +name = "crypto-bigint" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef2b4b23cddf68b89b8f8069890e8c270d54e2d5fe1b143820234805e4cb17ef" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "subtle", + "zeroize", +] + [[package]] name = "crypto-common" version = "0.1.6" @@ -1949,6 +2362,18 @@ dependencies = [ "reborrow", ] +[[package]] +name = "ecdsa" +version = "0.14.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413301934810f597c1d19ca71c8710e99a3f1ba28a0d2ebc01551a2daeea3c5c" +dependencies = [ + "der 0.6.1", + "elliptic-curve", + "rfc6979", + "signature 1.6.4", +] + [[package]] name = "either" version = "1.9.0" @@ -1958,6 +2383,26 @@ dependencies = [ "serde", ] +[[package]] +name = "elliptic-curve" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7bb888ab5300a19b8e5bceef25ac745ad065f3c9f7efc6de1b91958110891d3" +dependencies = [ + "base16ct", + "crypto-bigint", + "der 0.6.1", + "digest 0.10.7", + "ff", + "generic-array", + "group", + "pkcs8 0.9.0", + "rand_core 0.6.4", + "sec1", + "subtle", + "zeroize", +] + [[package]] name = "embed-doc-image" version = "0.1.4" @@ -2153,6 +2598,16 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" +[[package]] +name = "ff" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d013fc25338cc558c5c2cfbad646908fb23591e2404481826742b651c9af7160" +dependencies = [ + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "filetime" version = "0.2.22" @@ -2706,6 +3161,17 @@ dependencies = [ "unic-ucd-category", ] +[[package]] +name = "group" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfbfb3a6cfbd390d5c9564ab283a0349b9b9fcd46a706c1eb10e0db70bfbac7" +dependencies = [ + "ff", + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "h2" version = "0.3.21" @@ -2959,6 +3425,7 @@ dependencies = [ "futures-util", "http", "hyper", + "log", "rustls 0.21.8", "rustls-native-certs", "tokio", @@ -4010,6 +4477,17 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" +[[package]] +name = "p256" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51f44edd08f51e2ade572f141051021c5af22677e42b7dd28a88155151c33594" +dependencies = [ + "ecdsa", + "elliptic-curve", + "sha2 0.10.8", +] + [[package]] name = "parking" version = "2.2.0" @@ -5093,6 +5571,17 @@ dependencies = [ "winreg", ] +[[package]] +name = "rfc6979" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7743f17af12fa0b03b803ba12cd6a8d9483a587e89c69445e3909655c0b9fabb" +dependencies = [ + "crypto-bigint", + "hmac", + "zeroize", +] + [[package]] name = "riff" version = "1.0.1" @@ -5572,6 +6061,20 @@ version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" +[[package]] +name = "sec1" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be24c1842290c45df0a7bf069e0c268a747ad05a192f2fd7dcfdbc1cba40928" +dependencies = [ + "base16ct", + "der 0.6.1", + "generic-array", + "pkcs8 0.9.0", + "subtle", + "zeroize", +] + [[package]] name = "security-framework" version = "2.9.2" @@ -8221,6 +8724,8 @@ dependencies = [ "async-recursion", "async-stripe", "async_zip", + "aws-config", + "aws-sdk-s3", "axum", "base64 0.21.5", "bytes", @@ -8727,6 +9232,12 @@ dependencies = [ "libc", ] +[[package]] +name = "xmlparser" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" + [[package]] name = "xz2" version = "0.1.7" diff --git a/backend/migrations/20231115101254_workspace_s3_bucket_settings.down.sql b/backend/migrations/20231115101254_workspace_s3_bucket_settings.down.sql new file mode 100644 index 0000000000..9d206397d2 --- /dev/null +++ b/backend/migrations/20231115101254_workspace_s3_bucket_settings.down.sql @@ -0,0 +1,2 @@ +-- Add down migration script here +ALTER TABLE workspace_settings DROP COLUMN large_file_storage; \ No newline at end of file diff --git a/backend/migrations/20231115101254_workspace_s3_bucket_settings.up.sql b/backend/migrations/20231115101254_workspace_s3_bucket_settings.up.sql new file mode 100644 index 0000000000..638039bb15 --- /dev/null +++ b/backend/migrations/20231115101254_workspace_s3_bucket_settings.up.sql @@ -0,0 +1,2 @@ +-- Add up migration script here +ALTER TABLE workspace_settings ADD COLUMN IF NOT EXISTS large_file_storage JSONB; \ No newline at end of file diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index 993a336411..7184312b9b 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -81,4 +81,6 @@ hf-hub.workspace = true tokenizers.workspace = true candle-core.workspace = true candle-transformers.workspace = true -candle-nn.workspace = true \ No newline at end of file +candle-nn.workspace = true +aws-sdk-s3 = "0.36.0" +aws-config = "0.57.2" diff --git a/backend/windmill-api/openapi-deref.yaml b/backend/windmill-api/openapi-deref.yaml index 3bb976bca8..4c9d1c209f 100644 --- a/backend/windmill-api/openapi-deref.yaml +++ b/backend/windmill-api/openapi-deref.yaml @@ -1,6 +1,6 @@ openapi: 3.0.3 info: - version: 1.202.1 + version: 1.206.0 title: Windmill API contact: name: Windmill Team @@ -87,7 +87,7 @@ paths: - name: id in: path required: true - schema: &ref_22 + schema: &ref_23 type: integer responses: '200': @@ -231,34 +231,34 @@ paths: - name: before description: filter on created before (exclusive) timestamp in: query - schema: &ref_92 + schema: &ref_93 type: string format: date-time - name: after description: filter on created after (exclusive) timestamp in: query - schema: &ref_91 + schema: &ref_92 type: string format: date-time - name: username description: filter on exact username of user in: query - schema: &ref_93 + schema: &ref_94 type: string - name: operation description: filter on exact or prefix name of operation in: query - schema: &ref_94 + schema: &ref_95 type: string - name: resource description: filter on exact or prefix name of resource in: query - schema: &ref_95 + schema: &ref_96 type: string - name: action_kind description: filter on type of operation in: query - schema: &ref_96 + schema: &ref_97 type: string enum: - Create @@ -290,12 +290,12 @@ paths: application/json: schema: type: object - properties: &ref_106 + properties: &ref_107 email: type: string password: type: string - required: &ref_107 + required: &ref_108 - email - password responses: @@ -351,14 +351,14 @@ paths: application/json: schema: type: object - properties: &ref_108 + properties: &ref_109 email: type: string username: type: string is_admin: type: boolean - required: &ref_109 + required: &ref_110 - email - username - is_admin @@ -393,7 +393,7 @@ paths: application/json: schema: type: object - properties: &ref_110 + properties: &ref_111 is_admin: type: boolean operator: @@ -421,7 +421,7 @@ paths: - name: path in: path required: true - schema: &ref_15 + schema: &ref_16 type: string responses: '200': @@ -616,7 +616,7 @@ paths: application/json: schema: type: object - properties: &ref_138 + properties: &ref_139 email: type: string workspaces: @@ -634,7 +634,7 @@ paths: - id - name - username - required: &ref_139 + required: &ref_140 - email - workspaces /workspaces/list_as_superadmin: @@ -676,14 +676,14 @@ paths: application/json: schema: type: object - properties: &ref_140 + properties: &ref_141 id: type: string name: type: string username: type: string - required: &ref_141 + required: &ref_142 - id - name - username @@ -1111,7 +1111,7 @@ paths: type: string usage: type: object - properties: &ref_105 + properties: &ref_106 executions: type: number required: &ref_9 @@ -1502,6 +1502,15 @@ paths: additionalProperties: &ref_14 {} error_handler_muted_on_cancel: type: boolean + large_file_storage: + type: object + properties: &ref_15 + type: + type: string + enum: + - S3Storage + s3_resource_path: + type: string required: - code_completion_enabled /w/{workspace}/workspaces/get_deploy_to: @@ -1787,6 +1796,53 @@ paths: text/plain: schema: type: string + /w/{workspace}/workspaces/edit_large_file_storage_config: + post: + summary: edit large file storage settings + operationId: editLargeFileStorageConfig + tags: + - workspace + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + requestBody: + description: LargeFileStorage info + required: true + content: + application/json: + schema: + type: object + properties: + large_file_storage: + type: object + properties: *ref_15 + responses: + '200': + description: status + content: + application/json: + schema: {} + /w/{workspace}/workspaces/get_large_file_storage_config: + get: + summary: get large file storage config + operationId: getLargeFileStorageConfig + tags: + - workspace + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + responses: + '200': + description: status + content: + application/json: + schema: + type: object + properties: *ref_15 /w/{workspace}/users/list: get: summary: list users @@ -1842,7 +1898,7 @@ paths: application/json: schema: type: object - properties: &ref_113 + properties: &ref_114 label: type: string expiration: @@ -1872,7 +1928,7 @@ paths: application/json: schema: type: object - properties: &ref_114 + properties: &ref_115 label: type: string expiration: @@ -1880,7 +1936,7 @@ paths: format: date-time impersonate_email: type: string - required: &ref_115 + required: &ref_116 - impersonate_email responses: '201': @@ -1923,7 +1979,7 @@ paths: type: array items: type: object - properties: &ref_111 + properties: &ref_112 label: type: string expiration: @@ -1941,7 +1997,7 @@ paths: type: array items: type: string - required: &ref_112 + required: &ref_113 - token_prefix - created_at - last_used_at @@ -1967,7 +2023,7 @@ paths: application/json: schema: type: object - properties: &ref_118 + properties: &ref_119 path: type: string value: @@ -1980,7 +2036,7 @@ paths: type: integer is_oauth: type: boolean - required: &ref_119 + required: &ref_120 - path - value - is_secret @@ -2031,7 +2087,7 @@ paths: - name: path in: path required: true - schema: *ref_15 + schema: *ref_16 responses: '200': description: variable deleted @@ -2053,7 +2109,7 @@ paths: - name: path in: path required: true - schema: *ref_15 + schema: *ref_16 - name: already_encrypted in: query schema: @@ -2065,7 +2121,7 @@ paths: application/json: schema: type: object - properties: &ref_120 + properties: &ref_121 path: type: string value: @@ -2095,7 +2151,7 @@ paths: - name: path in: path required: true - schema: *ref_15 + schema: *ref_16 - name: decrypt_secret description: | ask to decrypt secret if this variable is secret @@ -2110,7 +2166,7 @@ paths: application/json: schema: type: object - properties: &ref_16 + properties: &ref_17 workspace_id: type: string path: @@ -2137,7 +2193,7 @@ paths: type: boolean is_refreshed: type: boolean - required: &ref_17 + required: &ref_18 - workspace_id - path - is_secret @@ -2156,7 +2212,7 @@ paths: - name: path in: path required: true - schema: *ref_15 + schema: *ref_16 responses: '200': description: variable @@ -2178,7 +2234,7 @@ paths: - name: path in: path required: true - schema: *ref_15 + schema: *ref_16 responses: '200': description: variable @@ -2206,8 +2262,8 @@ paths: type: array items: type: object - properties: *ref_16 - required: *ref_17 + properties: *ref_17 + required: *ref_18 /w/{workspace}/variables/list_contextual: get: summary: list contextual variables @@ -2228,14 +2284,14 @@ paths: type: array items: type: object - properties: &ref_116 + properties: &ref_117 name: type: string value: type: string description: type: string - required: &ref_117 + required: &ref_118 - name - value - description @@ -2250,7 +2306,7 @@ paths: - name: client_name in: path required: true - schema: &ref_18 + schema: &ref_19 type: string requestBody: description: Partially filled script @@ -2323,7 +2379,7 @@ paths: - name: client_name in: path required: true - schema: *ref_18 + schema: *ref_19 requestBody: description: code endpoint required: true @@ -2346,7 +2402,7 @@ paths: application/json: schema: type: object - properties: &ref_152 + properties: &ref_153 access_token: type: string expires_in: @@ -2357,7 +2413,7 @@ paths: type: array items: type: string - required: &ref_153 + required: &ref_154 - access_token /w/{workspace}/oauth/create_account: post: @@ -2411,7 +2467,7 @@ paths: - name: id in: path required: true - schema: &ref_19 + schema: &ref_20 type: integer requestBody: description: variable path @@ -2446,7 +2502,7 @@ paths: - name: id in: path required: true - schema: *ref_19 + schema: *ref_20 responses: '200': description: disconnected client @@ -2538,7 +2594,7 @@ paths: application/json: schema: type: object - properties: &ref_123 + properties: &ref_124 path: type: string value: {} @@ -2546,7 +2602,7 @@ paths: type: string resource_type: type: string - required: &ref_124 + required: &ref_125 - path - value - resource_type @@ -2571,7 +2627,7 @@ paths: - name: path in: path required: true - schema: *ref_15 + schema: *ref_16 responses: '200': description: resource deleted @@ -2593,7 +2649,7 @@ paths: - name: path in: path required: true - schema: *ref_15 + schema: *ref_16 requestBody: description: updated resource required: true @@ -2601,7 +2657,7 @@ paths: application/json: schema: type: object - properties: &ref_125 + properties: &ref_126 path: type: string description: @@ -2628,7 +2684,7 @@ paths: - name: path in: path required: true - schema: *ref_15 + schema: *ref_16 requestBody: description: updated resource required: true @@ -2659,7 +2715,7 @@ paths: - name: path in: path required: true - schema: *ref_15 + schema: *ref_16 responses: '200': description: resource @@ -2667,7 +2723,7 @@ paths: application/json: schema: type: object - properties: &ref_126 + properties: &ref_127 workspace_id: type: string path: @@ -2683,7 +2739,7 @@ paths: type: object additionalProperties: type: boolean - required: &ref_127 + required: &ref_128 - path - resource_type - is_oauth @@ -2701,7 +2757,7 @@ paths: - name: path in: path required: true - schema: *ref_15 + schema: *ref_16 - name: job_id description: job id in: query @@ -2728,7 +2784,7 @@ paths: - name: path in: path required: true - schema: *ref_15 + schema: *ref_16 responses: '200': description: resource value @@ -2749,7 +2805,7 @@ paths: - name: path in: path required: true - schema: *ref_15 + schema: *ref_16 responses: '200': description: does resource exists @@ -2795,7 +2851,7 @@ paths: type: array items: type: object - properties: &ref_128 + properties: &ref_129 workspace_id: type: string path: @@ -2821,7 +2877,7 @@ paths: type: boolean account: type: number - required: &ref_129 + required: &ref_130 - path - resource_type - is_oauth @@ -2868,7 +2924,7 @@ paths: - name: name in: path required: true - schema: &ref_75 + schema: &ref_76 type: string responses: '200': @@ -2905,7 +2961,7 @@ paths: application/json: schema: type: object - properties: &ref_20 + properties: &ref_21 workspace_id: type: string name: @@ -2913,7 +2969,7 @@ paths: schema: {} description: type: string - required: &ref_21 + required: &ref_22 - name responses: '201': @@ -2936,7 +2992,7 @@ paths: - name: path in: path required: true - schema: *ref_15 + schema: *ref_16 responses: '200': description: resource_type deleted @@ -2958,7 +3014,7 @@ paths: - name: path in: path required: true - schema: *ref_15 + schema: *ref_16 requestBody: description: updated resource_type required: true @@ -2966,7 +3022,7 @@ paths: application/json: schema: type: object - properties: &ref_130 + properties: &ref_131 schema: {} description: type: string @@ -2991,7 +3047,7 @@ paths: - name: path in: path required: true - schema: *ref_15 + schema: *ref_16 responses: '200': description: resource_type deleted @@ -2999,8 +3055,8 @@ paths: application/json: schema: type: object - properties: *ref_20 - required: *ref_21 + properties: *ref_21 + required: *ref_22 /w/{workspace}/resources/type/exists/{path}: get: summary: does resource_type exists @@ -3015,7 +3071,7 @@ paths: - name: path in: path required: true - schema: *ref_15 + schema: *ref_16 responses: '200': description: does resource_type exist @@ -3043,8 +3099,8 @@ paths: type: array items: type: object - properties: *ref_20 - required: *ref_21 + properties: *ref_21 + required: *ref_22 /w/{workspace}/resources/type/listnames: get: summary: list resource_types names @@ -3183,7 +3239,7 @@ paths: - name: id in: path required: true - schema: *ref_22 + schema: *ref_23 responses: '200': description: flow @@ -3194,51 +3250,51 @@ paths: properties: flow: type: object - properties: &ref_40 + properties: &ref_41 summary: type: string description: type: string value: type: object - properties: &ref_47 + properties: &ref_48 modules: type: array items: type: object - properties: &ref_25 + properties: &ref_26 id: type: string value: - oneOf: &ref_174 + oneOf: &ref_177 - type: object - properties: &ref_158 + properties: &ref_161 input_transforms: type: object additionalProperties: - oneOf: &ref_23 + oneOf: &ref_24 - type: object - properties: &ref_154 + properties: &ref_157 value: {} type: type: string enum: - javascript - required: &ref_155 + required: &ref_158 - expr - type - type: object - properties: &ref_156 + properties: &ref_159 expr: type: string type: type: string enum: - javascript - required: &ref_157 + required: &ref_160 - expr - type - discriminator: &ref_24 + discriminator: &ref_25 propertyName: type mapping: static: '#/components/schemas/StaticTransform' @@ -3258,6 +3314,7 @@ paths: - mysql - bigquery - snowflake + - mssql - graphql - nativets path: @@ -3274,18 +3331,18 @@ paths: type: number concurrency_time_window_s: type: number - required: &ref_159 + required: &ref_162 - type - content - language - input_transforms - type: object - properties: &ref_160 + properties: &ref_163 input_transforms: type: object additionalProperties: - oneOf: *ref_23 - discriminator: *ref_24 + oneOf: *ref_24 + discriminator: *ref_25 path: type: string hash: @@ -3294,40 +3351,40 @@ paths: type: string enum: - script - required: &ref_161 + required: &ref_164 - type - path - input_transforms - type: object - properties: &ref_162 + properties: &ref_165 input_transforms: type: object additionalProperties: - oneOf: *ref_23 - discriminator: *ref_24 + oneOf: *ref_24 + discriminator: *ref_25 path: type: string type: type: string enum: - flow - required: &ref_163 + required: &ref_166 - type - path - input_transforms - type: object - properties: &ref_164 + properties: &ref_167 modules: type: array items: type: object - properties: *ref_25 - required: &ref_26 + properties: *ref_26 + required: &ref_27 - value - id iterator: - oneOf: *ref_23 - discriminator: *ref_24 + oneOf: *ref_24 + discriminator: *ref_25 skip_failures: type: boolean type: @@ -3338,13 +3395,13 @@ paths: type: boolean parallelism: type: integer - required: &ref_165 + required: &ref_168 - modules - iterator - skip_failures - type - type: object - properties: &ref_166 + properties: &ref_169 branches: type: array items: @@ -3358,8 +3415,8 @@ paths: type: array items: type: object - properties: *ref_25 - required: *ref_26 + properties: *ref_26 + required: *ref_27 required: - modules - expr @@ -3367,20 +3424,20 @@ paths: type: array items: type: object - properties: *ref_25 - required: *ref_26 + properties: *ref_26 + required: *ref_27 required: - modules type: type: string enum: - branchone - required: &ref_167 + required: &ref_170 - branches - default - type - type: object - properties: &ref_168 + properties: &ref_171 branches: type: array items: @@ -3394,8 +3451,8 @@ paths: type: array items: type: object - properties: *ref_25 - required: *ref_26 + properties: *ref_26 + required: *ref_27 required: - modules - expr @@ -3405,28 +3462,28 @@ paths: - branchall parallel: type: boolean - required: &ref_169 + required: &ref_172 - branches - type - type: object - properties: &ref_170 + properties: &ref_173 type: type: string enum: - identity flow: type: boolean - required: &ref_171 + required: &ref_174 - type - type: object - properties: &ref_172 + properties: &ref_175 type: type: string enum: - graphql - required: &ref_173 + required: &ref_176 - type - discriminator: &ref_175 + discriminator: &ref_178 propertyName: type mapping: rawscript: '#/components/schemas/RawScript' @@ -3447,8 +3504,8 @@ paths: required: - expr sleep: - oneOf: *ref_23 - discriminator: *ref_24 + oneOf: *ref_24 + discriminator: *ref_25 cache_ttl: type: number timeout: @@ -3476,13 +3533,13 @@ paths: user_auth_required: type: boolean user_groups_required: - oneOf: *ref_23 - discriminator: *ref_24 + oneOf: *ref_24 + discriminator: *ref_25 priority: type: number retry: type: object - properties: &ref_176 + properties: &ref_179 constant: type: object properties: @@ -3499,11 +3556,11 @@ paths: type: integer seconds: type: integer - required: *ref_26 + required: *ref_27 failure_module: type: object - properties: *ref_25 - required: *ref_26 + properties: *ref_26 + required: *ref_27 same_worker: type: boolean concurrent_limit: @@ -3516,11 +3573,11 @@ paths: type: number priority: type: number - required: &ref_48 + required: &ref_49 - modules schema: type: object - required: &ref_41 + required: &ref_42 - summary - value /apps/hub/list: @@ -3573,7 +3630,7 @@ paths: - name: id in: path required: true - schema: *ref_22 + schema: *ref_23 responses: '200': description: app @@ -3603,7 +3660,7 @@ paths: - name: path in: path required: true - schema: &ref_27 + schema: &ref_28 type: string responses: '200': @@ -3622,7 +3679,7 @@ paths: - name: path in: path required: true - schema: *ref_27 + schema: *ref_28 responses: '200': description: script details @@ -3693,7 +3750,7 @@ paths: type: number kind: name: kind - schema: &ref_28 + schema: &ref_29 type: string enum: - script @@ -3766,7 +3823,7 @@ paths: type: string kind: name: kind - schema: *ref_28 + schema: *ref_29 score: type: number required: @@ -3827,12 +3884,12 @@ paths: - name: order_desc description: order by desc order (default true) in: query - schema: &ref_38 + schema: &ref_39 type: boolean - name: created_by description: mask to filter exact matching user creator in: query - schema: &ref_39 + schema: &ref_40 type: string - name: path_start description: mask to filter matching starting path @@ -3925,7 +3982,7 @@ paths: type: array items: type: object - properties: &ref_29 + properties: &ref_30 workspace_id: type: string hash: @@ -3978,6 +4035,7 @@ paths: - mysql - bigquery - snowflake + - mssql - graphql - nativets - bun @@ -4013,7 +4071,7 @@ paths: type: boolean priority: type: integer - required: &ref_30 + required: &ref_31 - hash - path - summary @@ -4109,7 +4167,7 @@ paths: - name: path in: path required: true - schema: *ref_27 + schema: *ref_28 responses: '200': description: draft deleted @@ -4135,7 +4193,7 @@ paths: application/json: schema: type: object - properties: &ref_32 + properties: &ref_33 path: type: string parent_hash: @@ -4166,6 +4224,7 @@ paths: - mysql - bigquery - snowflake + - mssql - graphql - nativets - bun @@ -4197,7 +4256,7 @@ paths: type: boolean priority: type: integer - required: &ref_33 + required: &ref_34 - path - summary - description @@ -4224,7 +4283,7 @@ paths: - name: path in: path required: true - schema: *ref_27 + schema: *ref_28 requestBody: description: Workspace error handler enabled required: true @@ -4273,7 +4332,7 @@ paths: - name: path in: path required: true - schema: *ref_27 + schema: *ref_28 responses: '200': description: script archived @@ -4295,7 +4354,7 @@ paths: - name: hash in: path required: true - schema: &ref_31 + schema: &ref_32 type: string responses: '200': @@ -4304,8 +4363,8 @@ paths: application/json: schema: type: object - properties: *ref_29 - required: *ref_30 + properties: *ref_30 + required: *ref_31 /w/{workspace}/scripts/delete/h/{hash}: post: summary: delete script by hash (erase content but keep hash, require admin) @@ -4320,7 +4379,7 @@ paths: - name: hash in: path required: true - schema: *ref_31 + schema: *ref_32 responses: '200': description: script details @@ -4328,8 +4387,8 @@ paths: application/json: schema: type: object - properties: *ref_29 - required: *ref_30 + properties: *ref_30 + required: *ref_31 /w/{workspace}/scripts/delete/p/{path}: post: summary: delete all scripts at a given path (require admin) @@ -4344,7 +4403,7 @@ paths: - name: path in: path required: true - schema: *ref_27 + schema: *ref_28 responses: '200': description: script path @@ -4366,7 +4425,7 @@ paths: - name: path in: path required: true - schema: *ref_27 + schema: *ref_28 responses: '200': description: script details @@ -4374,8 +4433,8 @@ paths: application/json: schema: type: object - properties: *ref_29 - required: *ref_30 + properties: *ref_30 + required: *ref_31 /w/{workspace}/scripts/get/draft/{path}: get: summary: get script by path with draft @@ -4390,23 +4449,23 @@ paths: - name: path in: path required: true - schema: *ref_27 + schema: *ref_28 responses: '200': description: script details content: application/json: schema: - allOf: &ref_99 + allOf: &ref_100 - type: object - properties: *ref_32 - required: *ref_33 + properties: *ref_33 + required: *ref_34 - type: object properties: draft: type: object - properties: *ref_32 - required: *ref_33 + properties: *ref_33 + required: *ref_34 hash: type: string required: @@ -4425,7 +4484,7 @@ paths: - name: path in: path required: true - schema: *ref_27 + schema: *ref_28 responses: '200': description: script content @@ -4449,12 +4508,12 @@ paths: - name: token in: path required: true - schema: &ref_87 + schema: &ref_88 type: string - name: path in: path required: true - schema: *ref_27 + schema: *ref_28 responses: '200': description: script content @@ -4476,7 +4535,7 @@ paths: - name: path in: path required: true - schema: *ref_27 + schema: *ref_28 responses: '200': description: does it exists @@ -4498,7 +4557,7 @@ paths: - name: hash in: path required: true - schema: *ref_31 + schema: *ref_32 responses: '200': description: script details @@ -4506,8 +4565,8 @@ paths: application/json: schema: type: object - properties: *ref_29 - required: *ref_30 + properties: *ref_30 + required: *ref_31 /w/{workspace}/scripts/raw/h/{path}: get: summary: raw script by hash @@ -4522,7 +4581,7 @@ paths: - name: path in: path required: true - schema: *ref_27 + schema: *ref_28 responses: '200': description: script content @@ -4544,7 +4603,7 @@ paths: - name: hash in: path required: true - schema: *ref_31 + schema: *ref_32 responses: '200': description: script details @@ -4571,7 +4630,7 @@ paths: - name: path in: path required: true - schema: *ref_27 + schema: *ref_28 - name: scheduled_for description: when to schedule this job (leave empty for immediate run) in: query @@ -4588,7 +4647,7 @@ paths: The parent job that is at the origin and responsible for the execution of this script if any in: query - schema: &ref_34 + schema: &ref_35 type: string format: uuid - name: job_id @@ -4598,7 +4657,7 @@ paths: queue or as a completed job, the request to create one will fail (Bad Request) in: query - schema: &ref_35 + schema: &ref_36 type: string format: uuid - name: invisible_to_owner @@ -4636,13 +4695,13 @@ paths: - name: path in: path required: true - schema: *ref_27 + schema: *ref_28 - name: parent_job description: >- The parent job that is at the origin and responsible for the execution of this script if any in: query - schema: *ref_34 + schema: *ref_35 - name: job_id description: >- The job id to assign to the created job. if missing, job is chosen @@ -4650,7 +4709,7 @@ paths: queue or as a completed job, the request to create one will fail (Bad Request) in: query - schema: *ref_35 + schema: *ref_36 - name: include_header description: > List of headers's keys (separated with ',') whove value are added to @@ -4659,14 +4718,14 @@ paths: Header's key lowercased and '-'' replaced to '_' such that 'Content-Type' becomes the 'content_type' arg key in: query - schema: &ref_36 + schema: &ref_37 type: string - name: queue_limit description: > The maximum size of the queue for which the request would get rejected if that job would push it above that limit in: query - schema: &ref_37 + schema: &ref_38 type: string requestBody: description: script args @@ -4696,13 +4755,13 @@ paths: - name: path in: path required: true - schema: *ref_27 + schema: *ref_28 - name: parent_job description: >- The parent job that is at the origin and responsible for the execution of this script if any in: query - schema: *ref_34 + schema: *ref_35 - name: job_id description: >- The job id to assign to the created job. if missing, job is chosen @@ -4710,7 +4769,7 @@ paths: queue or as a completed job, the request to create one will fail (Bad Request) in: query - schema: *ref_35 + schema: *ref_36 - name: include_header description: > List of headers's keys (separated with ',') whove value are added to @@ -4719,13 +4778,13 @@ paths: Header's key lowercased and '-'' replaced to '_' such that 'Content-Type' becomes the 'content_type' arg key in: query - schema: *ref_36 + schema: *ref_37 - name: queue_limit description: > The maximum size of the queue for which the request would get rejected if that job would push it above that limit in: query - schema: *ref_37 + schema: *ref_38 requestBody: description: script args required: true @@ -4753,13 +4812,13 @@ paths: - name: path in: path required: true - schema: *ref_27 + schema: *ref_28 - name: parent_job description: >- The parent job that is at the origin and responsible for the execution of this script if any in: query - schema: *ref_34 + schema: *ref_35 - name: job_id description: >- The job id to assign to the created job. if missing, job is chosen @@ -4767,7 +4826,7 @@ paths: queue or as a completed job, the request to create one will fail (Bad Request) in: query - schema: *ref_35 + schema: *ref_36 - name: include_header description: > List of headers's keys (separated with ',') whove value are added to @@ -4776,13 +4835,13 @@ paths: Header's key lowercased and '-'' replaced to '_' such that 'Content-Type' becomes the 'content_type' arg key in: query - schema: *ref_36 + schema: *ref_37 - name: queue_limit description: > The maximum size of the queue for which the request would get rejected if that job would push it above that limit in: query - schema: *ref_37 + schema: *ref_38 - name: payload description: > The base64 encoded payload that has been encoded as a JSON. e.g how @@ -4790,7 +4849,7 @@ paths: `encodeURIComponent(btoa(JSON.stringify({a: 2})))` in: query - schema: &ref_72 + schema: &ref_73 type: string responses: '200': @@ -4812,7 +4871,7 @@ paths: - name: path in: path required: true - schema: *ref_27 + schema: *ref_28 - name: include_header description: > List of headers's keys (separated with ',') whove value are added to @@ -4821,13 +4880,13 @@ paths: Header's key lowercased and '-'' replaced to '_' such that 'Content-Type' becomes the 'content_type' arg key in: query - schema: *ref_36 + schema: *ref_37 - name: queue_limit description: > The maximum size of the queue for which the request would get rejected if that job would push it above that limit in: query - schema: *ref_37 + schema: *ref_38 - name: job_id description: >- The job id to assign to the created job. if missing, job is chosen @@ -4835,7 +4894,7 @@ paths: queue or as a completed job, the request to create one will fail (Bad Request) in: query - schema: *ref_35 + schema: *ref_36 requestBody: description: script args required: true @@ -4864,7 +4923,7 @@ paths: - name: path in: path required: true - schema: *ref_27 + schema: *ref_28 - name: include_header description: > List of headers's keys (separated with ',') whove value are added to @@ -4873,13 +4932,13 @@ paths: Header's key lowercased and '-'' replaced to '_' such that 'Content-Type' becomes the 'content_type' arg key in: query - schema: *ref_36 + schema: *ref_37 - name: queue_limit description: > The maximum size of the queue for which the request would get rejected if that job would push it above that limit in: query - schema: *ref_37 + schema: *ref_38 - name: job_id description: >- The job id to assign to the created job. if missing, job is chosen @@ -4887,7 +4946,7 @@ paths: queue or as a completed job, the request to create one will fail (Bad Request) in: query - schema: *ref_35 + schema: *ref_36 requestBody: description: script args required: true @@ -4998,11 +5057,11 @@ paths: - name: order_desc description: order by desc order (default true) in: query - schema: *ref_38 + schema: *ref_39 - name: created_by description: mask to filter exact matching user creator in: query - schema: *ref_39 + schema: *ref_40 - name: path_start description: mask to filter matching starting path in: query @@ -5042,12 +5101,12 @@ paths: type: array items: allOf: - - allOf: &ref_42 + - allOf: &ref_43 - type: object - properties: *ref_40 - required: *ref_41 + properties: *ref_41 + required: *ref_42 - type: object - properties: &ref_142 + properties: &ref_143 workspace_id: type: string path: @@ -5073,7 +5132,7 @@ paths: type: boolean priority: type: integer - required: &ref_143 + required: &ref_144 - path - edited_by - edited_at @@ -5099,14 +5158,14 @@ paths: - name: path in: path required: true - schema: *ref_27 + schema: *ref_28 responses: '200': description: flow details content: application/json: schema: - allOf: *ref_42 + allOf: *ref_43 /w/{workspace}/flows/toggle_workspace_error_handler/{path}: post: summary: Toggle ON and OFF the workspace error handler for a given flow @@ -5121,7 +5180,7 @@ paths: - name: path in: path required: true - schema: *ref_27 + schema: *ref_28 requestBody: description: Workspace error handler enabled required: true @@ -5153,7 +5212,7 @@ paths: - name: path in: path required: true - schema: *ref_27 + schema: *ref_28 responses: '200': description: flow details with draft @@ -5161,11 +5220,11 @@ paths: application/json: schema: allOf: - - allOf: *ref_42 + - allOf: *ref_43 - type: object properties: draft: - allOf: *ref_42 + allOf: *ref_43 /w/{workspace}/flows/exists/{path}: get: summary: exists flow by path @@ -5180,7 +5239,7 @@ paths: - name: path in: path required: true - schema: *ref_27 + schema: *ref_28 responses: '200': description: flow details @@ -5206,10 +5265,10 @@ paths: application/json: schema: allOf: - - allOf: &ref_43 + - allOf: &ref_44 - type: object - properties: *ref_40 - required: *ref_41 + properties: *ref_41 + required: *ref_42 - type: object properties: path: @@ -5247,14 +5306,14 @@ paths: - name: path in: path required: true - schema: *ref_27 + schema: *ref_28 requestBody: description: Partially filled flow required: true content: application/json: schema: - allOf: *ref_43 + allOf: *ref_44 responses: '200': description: flow updated @@ -5276,7 +5335,7 @@ paths: - name: path in: path required: true - schema: *ref_27 + schema: *ref_28 requestBody: description: archiveFlow required: true @@ -5308,7 +5367,7 @@ paths: - name: path in: path required: true - schema: *ref_27 + schema: *ref_28 responses: '200': description: flow delete @@ -5330,7 +5389,7 @@ paths: - name: path in: path required: true - schema: *ref_27 + schema: *ref_28 - name: page description: which page to return (start at 1, default 1) in: query @@ -5348,7 +5407,7 @@ paths: type: array items: type: object - properties: &ref_82 + properties: &ref_83 id: type: string name: @@ -5364,7 +5423,7 @@ paths: type: boolean success: type: boolean - required: &ref_83 + required: &ref_84 - id - name - args @@ -5393,11 +5452,11 @@ paths: - name: order_desc description: order by desc order (default true) in: query - schema: *ref_38 + schema: *ref_39 - name: created_by description: mask to filter exact matching user creator in: query - schema: *ref_39 + schema: *ref_40 - name: path_start description: mask to filter matching starting path in: query @@ -5424,7 +5483,7 @@ paths: type: array items: type: object - properties: &ref_149 + properties: &ref_150 workspace_id: type: string path: @@ -5442,7 +5501,7 @@ paths: edited_at: type: string format: date-time - required: &ref_150 + required: &ref_151 - workspace_id - path - summary @@ -5463,7 +5522,7 @@ paths: - name: path in: path required: true - schema: *ref_15 + schema: *ref_16 responses: '200': description: app exists @@ -5485,12 +5544,12 @@ paths: - name: version in: path required: true - schema: &ref_86 + schema: &ref_87 type: number - name: path in: path required: true - schema: *ref_27 + schema: *ref_28 responses: '200': description: app details @@ -5547,11 +5606,11 @@ paths: - name: order_desc description: order by desc order (default true) in: query - schema: *ref_38 + schema: *ref_39 - name: created_by description: mask to filter exact matching user creator in: query - schema: *ref_39 + schema: *ref_40 - name: path_start description: mask to filter matching starting path in: query @@ -5578,7 +5637,7 @@ paths: type: array items: type: object - properties: &ref_147 + properties: &ref_148 id: type: integer workspace_id: @@ -5604,7 +5663,7 @@ paths: - viewer - publisher - anonymous - required: &ref_148 + required: &ref_149 - id - workspace_id - path @@ -5639,7 +5698,7 @@ paths: type: string policy: type: object - properties: &ref_44 + properties: &ref_45 triggerables: type: object additionalProperties: @@ -5682,7 +5741,7 @@ paths: - name: path in: path required: true - schema: *ref_15 + schema: *ref_16 responses: '200': description: app exists @@ -5704,7 +5763,7 @@ paths: - name: path in: path required: true - schema: *ref_27 + schema: *ref_28 responses: '200': description: app details @@ -5712,7 +5771,7 @@ paths: application/json: schema: type: object - properties: &ref_45 + properties: &ref_46 id: type: integer workspace_id: @@ -5733,7 +5792,7 @@ paths: value: {} policy: type: object - properties: *ref_44 + properties: *ref_45 execution_mode: type: string enum: @@ -5744,7 +5803,7 @@ paths: type: object additionalProperties: type: boolean - required: &ref_46 + required: &ref_47 - id - workspace_id - path @@ -5770,17 +5829,17 @@ paths: - name: path in: path required: true - schema: *ref_27 + schema: *ref_28 responses: '200': description: app details with draft content: application/json: schema: - allOf: &ref_151 + allOf: &ref_152 - type: object - properties: *ref_45 - required: *ref_46 + properties: *ref_46 + required: *ref_47 - type: object properties: draft_only: @@ -5800,7 +5859,7 @@ paths: - name: path in: path required: true - schema: *ref_15 + schema: *ref_16 responses: '200': description: app details @@ -5808,8 +5867,8 @@ paths: application/json: schema: type: object - properties: *ref_45 - required: *ref_46 + properties: *ref_46 + required: *ref_47 /w/{workspace}/apps_u/public_resource/{path}: get: summary: get public resource @@ -5824,7 +5883,7 @@ paths: - name: path in: path required: true - schema: *ref_15 + schema: *ref_16 responses: '200': description: resource value @@ -5845,7 +5904,7 @@ paths: - name: path in: path required: true - schema: *ref_15 + schema: *ref_16 responses: '200': description: app secret @@ -5867,7 +5926,7 @@ paths: - name: id in: path required: true - schema: *ref_22 + schema: *ref_23 responses: '200': description: app details @@ -5875,8 +5934,8 @@ paths: application/json: schema: type: object - properties: *ref_45 - required: *ref_46 + properties: *ref_46 + required: *ref_47 /w/{workspace}/raw_apps/create: post: summary: create raw app @@ -5927,7 +5986,7 @@ paths: - name: path in: path required: true - schema: *ref_27 + schema: *ref_28 requestBody: description: updateraw app required: true @@ -5963,7 +6022,7 @@ paths: - name: path in: path required: true - schema: *ref_15 + schema: *ref_16 responses: '200': description: app deleted @@ -5985,7 +6044,7 @@ paths: - name: path in: path required: true - schema: *ref_15 + schema: *ref_16 responses: '200': description: app deleted @@ -6007,7 +6066,7 @@ paths: - name: path in: path required: true - schema: *ref_27 + schema: *ref_28 requestBody: description: update app required: true @@ -6023,7 +6082,7 @@ paths: value: {} policy: type: object - properties: *ref_44 + properties: *ref_45 responses: '200': description: app updated @@ -6045,7 +6104,7 @@ paths: - name: path in: path required: true - schema: *ref_27 + schema: *ref_28 requestBody: description: update app required: true @@ -6099,7 +6158,7 @@ paths: - name: path in: path required: true - schema: *ref_27 + schema: *ref_28 - name: scheduled_for description: when to schedule this job (leave empty for immediate run) in: query @@ -6116,7 +6175,7 @@ paths: The parent job that is at the origin and responsible for the execution of this script if any in: query - schema: *ref_34 + schema: *ref_35 - name: job_id description: >- The job id to assign to the created job. if missing, job is chosen @@ -6124,7 +6183,7 @@ paths: queue or as a completed job, the request to create one will fail (Bad Request) in: query - schema: *ref_35 + schema: *ref_36 - name: include_header description: > List of headers's keys (separated with ',') whove value are added to @@ -6133,7 +6192,7 @@ paths: Header's key lowercased and '-'' replaced to '_' such that 'Content-Type' becomes the 'content_type' arg key in: query - schema: *ref_36 + schema: *ref_37 - name: invisible_to_owner description: make the run invisible to the the flow owner (default false) in: query @@ -6169,7 +6228,7 @@ paths: - name: id in: path required: true - schema: &ref_69 + schema: &ref_70 type: string format: uuid - name: step_id @@ -6202,7 +6261,7 @@ paths: The parent job that is at the origin and responsible for the execution of this script if any in: query - schema: *ref_34 + schema: *ref_35 - name: job_id description: >- The job id to assign to the created job. if missing, job is chosen @@ -6210,7 +6269,7 @@ paths: queue or as a completed job, the request to create one will fail (Bad Request) in: query - schema: *ref_35 + schema: *ref_36 - name: include_header description: > List of headers's keys (separated with ',') whove value are added to @@ -6219,7 +6278,7 @@ paths: Header's key lowercased and '-'' replaced to '_' such that 'Content-Type' becomes the 'content_type' arg key in: query - schema: *ref_36 + schema: *ref_37 - name: invisible_to_owner description: make the run invisible to the the flow owner (default false) in: query @@ -6255,7 +6314,7 @@ paths: - name: hash in: path required: true - schema: *ref_31 + schema: *ref_32 - name: scheduled_for description: when to schedule this job (leave empty for immediate run) in: query @@ -6272,7 +6331,7 @@ paths: The parent job that is at the origin and responsible for the execution of this script if any in: query - schema: *ref_34 + schema: *ref_35 - name: job_id description: >- The job id to assign to the created job. if missing, job is chosen @@ -6280,7 +6339,7 @@ paths: queue or as a completed job, the request to create one will fail (Bad Request) in: query - schema: *ref_35 + schema: *ref_36 - name: include_header description: > List of headers's keys (separated with ',') whove value are added to @@ -6289,7 +6348,7 @@ paths: Header's key lowercased and '-'' replaced to '_' such that 'Content-Type' becomes the 'content_type' arg key in: query - schema: *ref_36 + schema: *ref_37 - name: invisible_to_owner description: make the run invisible to the the script owner (default false) in: query @@ -6329,7 +6388,7 @@ paths: Header's key lowercased and '-'' replaced to '_' such that 'Content-Type' becomes the 'content_type' arg key in: query - schema: *ref_36 + schema: *ref_37 - name: invisible_to_owner description: make the run invisible to the the script owner (default false) in: query @@ -6342,7 +6401,7 @@ paths: queue or as a completed job, the request to create one will fail (Bad Request) in: query - schema: *ref_35 + schema: *ref_36 requestBody: description: preview required: true @@ -6350,7 +6409,7 @@ paths: application/json: schema: type: object - properties: &ref_121 + properties: &ref_122 content: type: string path: @@ -6370,6 +6429,7 @@ paths: - mysql - bigquery - snowflake + - mssql - graphql - nativets - bun @@ -6383,7 +6443,7 @@ paths: - http dedicated_worker: type: boolean - required: &ref_122 + required: &ref_123 - args responses: '201': @@ -6412,7 +6472,7 @@ paths: Header's key lowercased and '-'' replaced to '_' such that 'Content-Type' becomes the 'content_type' arg key in: query - schema: *ref_36 + schema: *ref_37 - name: invisible_to_owner description: make the run invisible to the the script owner (default false) in: query @@ -6425,7 +6485,7 @@ paths: queue or as a completed job, the request to create one will fail (Bad Request) in: query - schema: *ref_35 + schema: *ref_36 requestBody: description: preview required: true @@ -6433,11 +6493,11 @@ paths: application/json: schema: type: object - properties: &ref_144 + properties: &ref_145 value: type: object - properties: *ref_47 - required: *ref_48 + properties: *ref_48 + required: *ref_49 path: type: string args: @@ -6447,7 +6507,7 @@ paths: type: string restarted_from: type: object - properties: &ref_146 + properties: &ref_147 flow_job_id: type: string format: uuid @@ -6455,7 +6515,7 @@ paths: type: string branch_or_iteration_n: type: integer - required: &ref_145 + required: &ref_146 - value - content - args @@ -6481,89 +6541,89 @@ paths: - name: order_desc description: order by desc order (default true) in: query - schema: *ref_38 + schema: *ref_39 - name: created_by description: mask to filter exact matching user creator in: query - schema: *ref_39 + schema: *ref_40 - name: parent_job description: >- The parent job that is at the origin and responsible for the execution of this script if any in: query - schema: *ref_34 + schema: *ref_35 - name: script_path_exact description: mask to filter exact matching path in: query - schema: &ref_51 + schema: &ref_52 type: string - name: script_path_start description: mask to filter matching starting path in: query - schema: &ref_52 + schema: &ref_53 type: string - name: schedule_path description: mask to filter by schedule path in: query - schema: &ref_53 + schema: &ref_54 type: string - name: script_hash description: mask to filter exact matching path in: query - schema: &ref_54 + schema: &ref_55 type: string - name: started_before description: filter on started before (inclusive) timestamp in: query - schema: &ref_55 + schema: &ref_56 type: string format: date-time - name: started_after description: filter on started after (exclusive) timestamp in: query - schema: &ref_56 + schema: &ref_57 type: string format: date-time - name: success description: filter on successful jobs in: query - schema: &ref_57 + schema: &ref_58 type: boolean - name: job_kinds description: >- filter on job kind (values 'preview', 'script', 'dependencies', 'flow') separated by, in: query - schema: &ref_58 + schema: &ref_59 type: string - name: suspended description: filter on suspended jobs in: query - schema: &ref_90 + schema: &ref_91 type: boolean - name: running description: filter on running jobs in: query - schema: &ref_64 + schema: &ref_65 type: boolean - name: args description: >- filter on jobs containing those args as a json subset (@> in postgres) in: query - schema: &ref_59 + schema: &ref_60 type: string - name: result description: >- filter on jobs containing those result as a json subset (@> in postgres) in: query - schema: &ref_60 + schema: &ref_61 type: string - name: tag description: filter on jobs with a given tag/worker group in: query - schema: &ref_61 + schema: &ref_62 type: string responses: '200': @@ -6574,7 +6634,7 @@ paths: type: array items: type: object - properties: &ref_67 + properties: &ref_68 workspace_id: type: string id: @@ -6639,14 +6699,14 @@ paths: by extension its DT_TOKEN. flow_status: type: object - properties: &ref_62 + properties: &ref_63 step: type: integer modules: type: array items: type: object - properties: &ref_49 + properties: &ref_50 type: type: string enum: @@ -6710,13 +6770,13 @@ paths: required: - resume_id - approver - required: &ref_50 + required: &ref_51 - type failure_module: allOf: - type: object - properties: *ref_49 - required: *ref_50 + properties: *ref_50 + required: *ref_51 - type: object properties: parent_module: @@ -6731,14 +6791,14 @@ paths: items: type: string format: uuid - required: &ref_63 + required: &ref_64 - step - modules - failure_module raw_flow: type: object - properties: *ref_47 - required: *ref_48 + properties: *ref_48 + required: *ref_49 is_flow_step: type: boolean language: @@ -6753,6 +6813,7 @@ paths: - mysql - bigquery - snowflake + - mssql - graphql - nativets - bun @@ -6766,7 +6827,7 @@ paths: type: string priority: type: integer - required: &ref_68 + required: &ref_69 - id - running - canceled @@ -6856,67 +6917,67 @@ paths: - name: order_desc description: order by desc order (default true) in: query - schema: *ref_38 + schema: *ref_39 - name: created_by description: mask to filter exact matching user creator in: query - schema: *ref_39 + schema: *ref_40 - name: parent_job description: >- The parent job that is at the origin and responsible for the execution of this script if any in: query - schema: *ref_34 + schema: *ref_35 - name: script_path_exact description: mask to filter exact matching path in: query - schema: *ref_51 + schema: *ref_52 - name: script_path_start description: mask to filter matching starting path in: query - schema: *ref_52 + schema: *ref_53 - name: schedule_path description: mask to filter by schedule path in: query - schema: *ref_53 + schema: *ref_54 - name: script_hash description: mask to filter exact matching path in: query - schema: *ref_54 + schema: *ref_55 - name: started_before description: filter on started before (inclusive) timestamp in: query - schema: *ref_55 + schema: *ref_56 - name: started_after description: filter on started after (exclusive) timestamp in: query - schema: *ref_56 + schema: *ref_57 - name: success description: filter on successful jobs in: query - schema: *ref_57 + schema: *ref_58 - name: job_kinds description: >- filter on job kind (values 'preview', 'script', 'dependencies', 'flow') separated by, in: query - schema: *ref_58 + schema: *ref_59 - name: args description: >- filter on jobs containing those args as a json subset (@> in postgres) in: query - schema: *ref_59 + schema: *ref_60 - name: result description: >- filter on jobs containing those result as a json subset (@> in postgres) in: query - schema: *ref_60 + schema: *ref_61 - name: tag description: filter on jobs with a given tag/worker group in: query - schema: *ref_61 + schema: *ref_62 - name: is_skipped description: is the job skipped in: query @@ -6936,7 +6997,7 @@ paths: type: array items: type: object - properties: &ref_65 + properties: &ref_66 workspace_id: type: string id: @@ -7000,12 +7061,12 @@ paths: by extension its DT_TOKEN. flow_status: type: object - properties: *ref_62 - required: *ref_63 + properties: *ref_63 + required: *ref_64 raw_flow: type: object - properties: *ref_47 - required: *ref_48 + properties: *ref_48 + required: *ref_49 is_flow_step: type: boolean language: @@ -7020,6 +7081,7 @@ paths: - mysql - bigquery - snowflake + - mssql - graphql - nativets - bun @@ -7035,7 +7097,7 @@ paths: type: string priority: type: integer - required: &ref_66 + required: &ref_67 - id - created_by - duration_ms @@ -7064,55 +7126,55 @@ paths: - name: created_by description: mask to filter exact matching user creator in: query - schema: *ref_39 + schema: *ref_40 - name: parent_job description: >- The parent job that is at the origin and responsible for the execution of this script if any in: query - schema: *ref_34 + schema: *ref_35 - name: script_path_exact description: mask to filter exact matching path in: query - schema: *ref_51 + schema: *ref_52 - name: script_path_start description: mask to filter matching starting path in: query - schema: *ref_52 + schema: *ref_53 - name: schedule_path description: mask to filter by schedule path in: query - schema: *ref_53 + schema: *ref_54 - name: script_hash description: mask to filter exact matching path in: query - schema: *ref_54 + schema: *ref_55 - name: started_before description: filter on started before (inclusive) timestamp in: query - schema: *ref_55 + schema: *ref_56 - name: started_after description: filter on started after (exclusive) timestamp in: query - schema: *ref_56 + schema: *ref_57 - name: created_or_started_before description: >- filter on created_at for non non started job and started_at otherwise before (inclusive) timestamp in: query - schema: &ref_89 + schema: &ref_90 type: string format: date-time - name: running description: filter on running jobs in: query - schema: *ref_64 + schema: *ref_65 - name: created_or_started_after description: >- filter on created_at for non non started job and started_at otherwise after (exclusive) timestamp in: query - schema: &ref_88 + schema: &ref_89 type: string format: date-time - name: job_kinds @@ -7120,23 +7182,23 @@ paths: filter on job kind (values 'preview', 'script', 'dependencies', 'flow') separated by, in: query - schema: *ref_58 + schema: *ref_59 - name: args description: >- filter on jobs containing those args as a json subset (@> in postgres) in: query - schema: *ref_59 + schema: *ref_60 - name: tag description: filter on jobs with a given tag/worker group in: query - schema: *ref_61 + schema: *ref_62 - name: result description: >- filter on jobs containing those result as a json subset (@> in postgres) in: query - schema: *ref_60 + schema: *ref_61 - name: is_skipped description: is the job skipped in: query @@ -7160,14 +7222,14 @@ paths: schema: type: array items: - allOf: &ref_70 + allOf: &ref_71 - oneOf: - type: object - properties: *ref_65 - required: *ref_66 + properties: *ref_66 + required: *ref_67 - type: object - properties: *ref_67 - required: *ref_68 + properties: *ref_68 + required: *ref_69 - type: object properties: type: @@ -7175,7 +7237,7 @@ paths: enum: - CompletedJob - QueuedJob - discriminator: &ref_71 + discriminator: &ref_72 propertyName: type /jobs/db_clock: get: @@ -7204,15 +7266,15 @@ paths: - name: id in: path required: true - schema: *ref_69 + schema: *ref_70 responses: '200': description: job details content: application/json: schema: - allOf: *ref_70 - discriminator: *ref_71 + allOf: *ref_71 + discriminator: *ref_72 /w/{workspace}/jobs_u/get_logs/{id}: get: summary: get job logs @@ -7227,7 +7289,7 @@ paths: - name: id in: path required: true - schema: *ref_69 + schema: *ref_70 responses: '200': description: job details @@ -7249,7 +7311,7 @@ paths: - name: id in: path required: true - schema: *ref_69 + schema: *ref_70 - name: running in: query schema: @@ -7288,7 +7350,7 @@ paths: - name: id in: path required: true - schema: *ref_69 + schema: *ref_70 responses: '200': description: job details @@ -7296,8 +7358,8 @@ paths: application/json: schema: type: object - properties: *ref_65 - required: *ref_66 + properties: *ref_66 + required: *ref_67 /w/{workspace}/jobs_u/completed/get_result/{id}: get: summary: get completed job result @@ -7312,7 +7374,7 @@ paths: - name: id in: path required: true - schema: *ref_69 + schema: *ref_70 responses: '200': description: result @@ -7333,10 +7395,10 @@ paths: - name: id in: path required: true - schema: *ref_69 + schema: *ref_70 - name: get_started in: query - schema: &ref_98 + schema: &ref_99 type: boolean responses: '200': @@ -7368,7 +7430,7 @@ paths: - name: id in: path required: true - schema: *ref_69 + schema: *ref_70 responses: '200': description: job details @@ -7376,8 +7438,8 @@ paths: application/json: schema: type: object - properties: *ref_65 - required: *ref_66 + properties: *ref_66 + required: *ref_67 /w/{workspace}/jobs_u/queue/cancel/{id}: post: summary: cancel queued job @@ -7392,7 +7454,7 @@ paths: - name: id in: path required: true - schema: *ref_69 + schema: *ref_70 requestBody: description: reason required: true @@ -7424,7 +7486,7 @@ paths: - name: id in: path required: true - schema: *ref_69 + schema: *ref_70 requestBody: description: reason required: true @@ -7456,7 +7518,7 @@ paths: - name: id in: path required: true - schema: *ref_69 + schema: *ref_70 - name: resume_id in: path required: true @@ -7487,7 +7549,7 @@ paths: - name: id in: path required: true - schema: *ref_69 + schema: *ref_70 - name: resume_id in: path required: true @@ -7529,7 +7591,7 @@ paths: - name: id in: path required: true - schema: *ref_69 + schema: *ref_70 - name: payload description: > The base64 encoded payload that has been encoded as a JSON. e.g how @@ -7537,7 +7599,7 @@ paths: `encodeURIComponent(btoa(JSON.stringify({a: 2})))` in: query - schema: *ref_72 + schema: *ref_73 - name: resume_id in: path required: true @@ -7572,7 +7634,7 @@ paths: - name: id in: path required: true - schema: *ref_69 + schema: *ref_70 - name: resume_id in: path required: true @@ -7614,7 +7676,7 @@ paths: - name: id in: path required: true - schema: *ref_69 + schema: *ref_70 requestBody: required: true content: @@ -7642,7 +7704,7 @@ paths: - name: id in: path required: true - schema: *ref_69 + schema: *ref_70 - name: resume_id in: path required: true @@ -7677,7 +7739,7 @@ paths: - name: id in: path required: true - schema: *ref_69 + schema: *ref_70 - name: resume_id in: path required: true @@ -7719,7 +7781,7 @@ paths: - name: id in: path required: true - schema: *ref_69 + schema: *ref_70 - name: resume_id in: path required: true @@ -7743,8 +7805,8 @@ paths: type: object properties: job: - allOf: *ref_70 - discriminator: *ref_71 + allOf: *ref_71 + discriminator: *ref_72 approvers: type: array items: @@ -7809,7 +7871,7 @@ paths: application/json: schema: type: object - properties: &ref_132 + properties: &ref_133 path: type: string schedule: @@ -7843,7 +7905,7 @@ paths: additionalProperties: *ref_14 ws_error_handler_muted: type: boolean - required: &ref_133 + required: &ref_134 - path - schedule - timezone @@ -7871,7 +7933,7 @@ paths: - name: path in: path required: true - schema: *ref_15 + schema: *ref_16 requestBody: description: updated schedule required: true @@ -7879,7 +7941,7 @@ paths: application/json: schema: type: object - properties: &ref_134 + properties: &ref_135 schedule: type: string timezone: @@ -7905,7 +7967,7 @@ paths: additionalProperties: *ref_14 ws_error_handler_muted: type: boolean - required: &ref_135 + required: &ref_136 - schedule - timezone - script_path @@ -7932,7 +7994,7 @@ paths: - name: path in: path required: true - schema: *ref_15 + schema: *ref_16 requestBody: description: updated schedule enable required: true @@ -7966,7 +8028,7 @@ paths: - name: path in: path required: true - schema: *ref_15 + schema: *ref_16 responses: '200': description: schedule deleted @@ -7988,7 +8050,7 @@ paths: - name: path in: path required: true - schema: *ref_15 + schema: *ref_16 responses: '200': description: schedule deleted @@ -7996,7 +8058,7 @@ paths: application/json: schema: type: object - properties: &ref_73 + properties: &ref_74 path: type: string edited_by: @@ -8043,7 +8105,7 @@ paths: additionalProperties: *ref_14 ws_error_handler_muted: type: boolean - required: &ref_74 + required: &ref_75 - path - edited_by - edited_at @@ -8068,7 +8130,7 @@ paths: - name: path in: path required: true - schema: *ref_15 + schema: *ref_16 responses: '200': description: schedule exists @@ -8113,8 +8175,8 @@ paths: type: array items: type: object - properties: *ref_73 - required: *ref_74 + properties: *ref_74 + required: *ref_75 /w/{workspace}/schedules/list_with_jobs: get: summary: list schedules with last 20 jobs @@ -8142,10 +8204,10 @@ paths: schema: type: array items: - allOf: &ref_131 + allOf: &ref_132 - type: object - properties: *ref_73 - required: *ref_74 + properties: *ref_74 + required: *ref_75 - type: object properties: jobs: @@ -8220,14 +8282,14 @@ paths: type: array items: type: object - properties: &ref_76 + properties: &ref_77 name: type: string emails: type: array items: type: string - required: &ref_77 + required: &ref_78 - name /groups/get/{name}: get: @@ -8239,7 +8301,7 @@ paths: - name: name in: path required: true - schema: *ref_75 + schema: *ref_76 responses: '200': description: instance group @@ -8247,8 +8309,8 @@ paths: application/json: schema: type: object - properties: *ref_76 - required: *ref_77 + properties: *ref_77 + required: *ref_78 /w/{workspace}/groups/list: get: summary: list groups @@ -8277,7 +8339,7 @@ paths: type: array items: type: object - properties: &ref_78 + properties: &ref_79 name: type: string summary: @@ -8290,7 +8352,7 @@ paths: type: object additionalProperties: type: boolean - required: &ref_79 + required: &ref_80 - name /w/{workspace}/groups/listnames: get: @@ -8363,7 +8425,7 @@ paths: - name: name in: path required: true - schema: *ref_75 + schema: *ref_76 requestBody: description: updated group required: true @@ -8395,7 +8457,7 @@ paths: - name: name in: path required: true - schema: *ref_75 + schema: *ref_76 responses: '200': description: group deleted @@ -8417,7 +8479,7 @@ paths: - name: name in: path required: true - schema: *ref_75 + schema: *ref_76 responses: '200': description: group @@ -8425,8 +8487,8 @@ paths: application/json: schema: type: object - properties: *ref_78 - required: *ref_79 + properties: *ref_79 + required: *ref_80 /w/{workspace}/groups/adduser/{name}: post: summary: add user to group @@ -8441,7 +8503,7 @@ paths: - name: name in: path required: true - schema: *ref_75 + schema: *ref_76 requestBody: description: added user to group required: true @@ -8473,7 +8535,7 @@ paths: - name: name in: path required: true - schema: *ref_75 + schema: *ref_76 requestBody: description: added user to group required: true @@ -8519,7 +8581,7 @@ paths: type: array items: type: object - properties: &ref_80 + properties: &ref_81 name: type: string owners: @@ -8530,7 +8592,7 @@ paths: type: object additionalProperties: type: boolean - required: &ref_81 + required: &ref_82 - name - owners - extra_perms @@ -8610,7 +8672,7 @@ paths: - name: name in: path required: true - schema: *ref_75 + schema: *ref_76 requestBody: description: update folder required: true @@ -8647,7 +8709,7 @@ paths: - name: name in: path required: true - schema: *ref_75 + schema: *ref_76 responses: '200': description: folder deleted @@ -8669,7 +8731,7 @@ paths: - name: name in: path required: true - schema: *ref_75 + schema: *ref_76 responses: '200': description: folder @@ -8677,8 +8739,8 @@ paths: application/json: schema: type: object - properties: *ref_80 - required: *ref_81 + properties: *ref_81 + required: *ref_82 /w/{workspace}/folders/getusage/{name}: get: summary: get folder usage @@ -8693,7 +8755,7 @@ paths: - name: name in: path required: true - schema: *ref_75 + schema: *ref_76 responses: '200': description: folder @@ -8735,7 +8797,7 @@ paths: - name: name in: path required: true - schema: *ref_75 + schema: *ref_76 requestBody: description: owner user to folder required: true @@ -8767,7 +8829,7 @@ paths: - name: name in: path required: true - schema: *ref_75 + schema: *ref_76 requestBody: description: added owner to folder required: true @@ -8809,7 +8871,7 @@ paths: type: array items: type: object - properties: &ref_136 + properties: &ref_137 worker: type: string worker_instance: @@ -8831,7 +8893,7 @@ paths: type: string wm_version: type: string - required: &ref_137 + required: &ref_138 - worker - worker_instance - ping_at @@ -8872,7 +8934,7 @@ paths: - name: name in: path required: true - schema: *ref_75 + schema: *ref_76 responses: '200': description: a config @@ -8889,7 +8951,7 @@ paths: - name: name in: path required: true - schema: *ref_75 + schema: *ref_76 requestBody: description: worker group required: true @@ -8912,7 +8974,7 @@ paths: - name: name in: path required: true - schema: *ref_75 + schema: *ref_76 responses: '200': description: Delete config @@ -8934,7 +8996,7 @@ paths: - name: path in: path required: true - schema: *ref_15 + schema: *ref_16 - name: kind in: path required: true @@ -8973,7 +9035,7 @@ paths: - name: path in: path required: true - schema: *ref_15 + schema: *ref_16 - name: kind in: path required: true @@ -9024,7 +9086,7 @@ paths: - name: path in: path required: true - schema: *ref_15 + schema: *ref_16 - name: kind in: path required: true @@ -9073,7 +9135,7 @@ paths: - name: path in: path required: true - schema: *ref_15 + schema: *ref_16 responses: '204': description: flow preview captured @@ -9091,7 +9153,7 @@ paths: - name: path in: path required: true - schema: *ref_15 + schema: *ref_16 responses: '201': description: flow preview capture created @@ -9108,7 +9170,7 @@ paths: - name: path in: path required: true - schema: *ref_15 + schema: *ref_16 responses: '200': description: captured flow preview @@ -9188,13 +9250,13 @@ paths: schema: *ref_0 - name: runnable_id in: query - schema: &ref_84 + schema: &ref_85 type: string - name: runnable_type in: query - schema: &ref_85 + schema: &ref_86 type: string - enum: &ref_104 + enum: &ref_105 - ScriptHash - ScriptPath - FlowPath @@ -9215,8 +9277,8 @@ paths: type: array items: type: object - properties: *ref_82 - required: *ref_83 + properties: *ref_83 + required: *ref_84 /w/{workspace}/inputs/list: get: summary: List saved Inputs for a Runnable @@ -9230,10 +9292,10 @@ paths: schema: *ref_0 - name: runnable_id in: query - schema: *ref_84 + schema: *ref_85 - name: runnable_type in: query - schema: *ref_85 + schema: *ref_86 - name: page description: which page to return (start at 1, default 1) in: query @@ -9251,8 +9313,8 @@ paths: type: array items: type: object - properties: *ref_82 - required: *ref_83 + properties: *ref_83 + required: *ref_84 /w/{workspace}/inputs/create: post: summary: Create an Input for future use in a script or flow @@ -9266,10 +9328,10 @@ paths: schema: *ref_0 - name: runnable_id in: query - schema: *ref_84 + schema: *ref_85 - name: runnable_type in: query - schema: *ref_85 + schema: *ref_86 requestBody: description: Input required: true @@ -9277,12 +9339,12 @@ paths: application/json: schema: type: object - properties: &ref_100 + properties: &ref_101 name: type: string args: type: object - required: &ref_101 + required: &ref_102 - name - args - created_by @@ -9312,14 +9374,14 @@ paths: application/json: schema: type: object - properties: &ref_102 + properties: &ref_103 id: type: string name: type: string is_public: type: boolean - required: &ref_103 + required: &ref_104 - id - name - is_public @@ -9345,7 +9407,7 @@ paths: - name: input in: path required: true - schema: &ref_97 + schema: &ref_98 type: string responses: '200': @@ -9355,6 +9417,84 @@ paths: schema: type: string format: uuid + /w/{workspace}/job_helpers/duckdb_connection_settings: + post: + summary: >- + Converts an S3 resource to the set of instructions necessary to connect + DuckDB to an S3 bucket + operationId: duckdbConnectionSettings + tags: + - helpers + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + requestBody: + description: S3 resource to connect to + required: true + content: + application/json: + schema: + s3_resource: + $ref: '#/components/schemas/S3Resource' + responses: + '200': + description: Connection settings + content: + application/json: + schema: + type: object + properties: + connection_settings_str: + type: string + /w/{workspace}/job_helpers/test_connection: + get: + summary: Test connection to the workspace datasets storage + operationId: datasetStorageTestConnection + tags: + - helpers + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + responses: + '200': + description: Connection settings + content: + application/json: + schema: {} + /w/{workspace}/job_helpers/list_stored_datasets: + post: + summary: List the dataset keys available in the worspace datasets storage + operationId: polarsConnectionSettings + tags: + - helpers + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + responses: + '200': + description: Connection settings + content: + application/json: + schema: + type: object + properties: + dataset_keys: + type: array + items: + type: object + properties: &ref_155 + s3: + type: string + s3_bucket: + type: string + required: &ref_156 + - s3 components: securitySchemes: bearerAuth: @@ -9379,52 +9519,52 @@ components: name: version in: path required: true - schema: *ref_86 + schema: *ref_87 Token: name: token in: path required: true - schema: *ref_87 + schema: *ref_88 AccountId: name: id in: path required: true - schema: *ref_19 + schema: *ref_20 ClientName: name: client_name in: path required: true - schema: *ref_18 + schema: *ref_19 ScriptPath: name: path in: path required: true - schema: *ref_27 + schema: *ref_28 ScriptHash: name: hash in: path required: true - schema: *ref_31 + schema: *ref_32 JobId: name: id in: path required: true - schema: *ref_69 + schema: *ref_70 Path: name: path in: path required: true - schema: *ref_15 + schema: *ref_16 PathId: name: id in: path required: true - schema: *ref_22 + schema: *ref_23 Name: name: name in: path required: true - schema: *ref_75 + schema: *ref_76 Page: name: page description: which page to return (start at 1, default 1) @@ -9439,19 +9579,19 @@ components: name: order_desc description: order by desc order (default true) in: query - schema: *ref_38 + schema: *ref_39 CreatedBy: name: created_by description: mask to filter exact matching user creator in: query - schema: *ref_39 + schema: *ref_40 ParentJob: name: parent_job description: >- The parent job that is at the origin and responsible for the execution of this script if any in: query - schema: *ref_34 + schema: *ref_35 NewJobId: name: job_id description: >- @@ -9459,7 +9599,7 @@ components: randomly using the ULID scheme. If a job id already exists in the queue or as a completed job, the request to create one will fail (Bad Request) in: query - schema: *ref_35 + schema: *ref_36 IncludeHeader: name: include_header description: > @@ -9469,14 +9609,14 @@ components: Header's key lowercased and '-'' replaced to '_' such that 'Content-Type' becomes the 'content_type' arg key in: query - schema: *ref_36 + schema: *ref_37 QueueLimit: name: queue_limit description: > The maximum size of the queue for which the request would get rejected if that job would push it above that limit in: query - schema: *ref_37 + schema: *ref_38 Payload: name: payload description: > @@ -9485,219 +9625,219 @@ components: `encodeURIComponent(btoa(JSON.stringify({a: 2})))` in: query - schema: *ref_72 + schema: *ref_73 ScriptStartPath: name: script_path_start description: mask to filter matching starting path in: query - schema: *ref_52 + schema: *ref_53 SchedulePath: name: schedule_path description: mask to filter by schedule path in: query - schema: *ref_53 + schema: *ref_54 ScriptExactPath: name: script_path_exact description: mask to filter exact matching path in: query - schema: *ref_51 + schema: *ref_52 ScriptExactHash: name: script_hash description: mask to filter exact matching path in: query - schema: *ref_54 + schema: *ref_55 StartedBefore: name: started_before description: filter on started before (inclusive) timestamp in: query - schema: *ref_55 + schema: *ref_56 StartedAfter: name: started_after description: filter on started after (exclusive) timestamp in: query - schema: *ref_56 + schema: *ref_57 CreatedOrStartedAfter: name: created_or_started_after description: >- filter on created_at for non non started job and started_at otherwise after (exclusive) timestamp in: query - schema: *ref_88 + schema: *ref_89 CreatedOrStartedBefore: name: created_or_started_before description: >- filter on created_at for non non started job and started_at otherwise before (inclusive) timestamp in: query - schema: *ref_89 + schema: *ref_90 Success: name: success description: filter on successful jobs in: query - schema: *ref_57 + schema: *ref_58 Suspended: name: suspended description: filter on suspended jobs in: query - schema: *ref_90 + schema: *ref_91 Running: name: running description: filter on running jobs in: query - schema: *ref_64 + schema: *ref_65 ArgsFilter: name: args description: filter on jobs containing those args as a json subset (@> in postgres) in: query - schema: *ref_59 + schema: *ref_60 Tag: name: tag description: filter on jobs with a given tag/worker group in: query - schema: *ref_61 + schema: *ref_62 ResultFilter: name: result description: filter on jobs containing those result as a json subset (@> in postgres) in: query - schema: *ref_60 + schema: *ref_61 After: name: after description: filter on created after (exclusive) timestamp in: query - schema: *ref_91 + schema: *ref_92 Before: name: before description: filter on created before (exclusive) timestamp in: query - schema: *ref_92 + schema: *ref_93 Username: name: username description: filter on exact username of user in: query - schema: *ref_93 + schema: *ref_94 Operation: name: operation description: filter on exact or prefix name of operation in: query - schema: *ref_94 + schema: *ref_95 ResourceName: name: resource description: filter on exact or prefix name of resource in: query - schema: *ref_95 + schema: *ref_96 ActionKind: name: action_kind description: filter on type of operation in: query - schema: *ref_96 + schema: *ref_97 JobKinds: name: job_kinds description: >- filter on job kind (values 'preview', 'script', 'dependencies', 'flow') separated by, in: query - schema: *ref_58 + schema: *ref_59 RunnableId: name: runnable_id in: query - schema: *ref_84 + schema: *ref_85 RunnableTypeQuery: name: runnable_type in: query - schema: *ref_85 + schema: *ref_86 InputId: name: input in: path required: true - schema: *ref_97 + schema: *ref_98 GetStarted: name: get_started in: query - schema: *ref_98 + schema: *ref_99 schemas: Script: type: object - properties: *ref_29 - required: *ref_30 + properties: *ref_30 + required: *ref_31 NewScript: type: object - properties: *ref_32 - required: *ref_33 + properties: *ref_33 + required: *ref_34 NewScriptWithDraft: - allOf: *ref_99 + allOf: *ref_100 ScriptArgs: type: object additionalProperties: *ref_14 Input: type: object - properties: *ref_82 - required: *ref_83 + properties: *ref_83 + required: *ref_84 CreateInput: type: object - properties: *ref_100 - required: *ref_101 + properties: *ref_101 + required: *ref_102 UpdateInput: type: object - properties: *ref_102 - required: *ref_103 + properties: *ref_103 + required: *ref_104 RunnableType: type: string - enum: *ref_104 + enum: *ref_105 QueuedJob: type: object - properties: *ref_67 - required: *ref_68 + properties: *ref_68 + required: *ref_69 CompletedJob: type: object - properties: *ref_65 - required: *ref_66 + properties: *ref_66 + required: *ref_67 Job: - allOf: *ref_70 - discriminator: *ref_71 + allOf: *ref_71 + discriminator: *ref_72 User: type: object properties: *ref_8 required: *ref_9 Usage: type: object - properties: *ref_105 + properties: *ref_106 Login: type: object - properties: *ref_106 - required: *ref_107 + properties: *ref_107 + required: *ref_108 NewUser: type: object - properties: *ref_108 - required: *ref_109 + properties: *ref_109 + required: *ref_110 EditWorkspaceUser: type: object - properties: *ref_110 + properties: *ref_111 TruncatedToken: type: object - properties: *ref_111 - required: *ref_112 + properties: *ref_112 + required: *ref_113 NewToken: type: object - properties: *ref_113 + properties: *ref_114 NewTokenImpersonate: type: object - properties: *ref_114 - required: *ref_115 + properties: *ref_115 + required: *ref_116 ListableVariable: type: object - properties: *ref_16 - required: *ref_17 + properties: *ref_17 + required: *ref_18 ContextualVariable: type: object - properties: *ref_116 - required: *ref_117 + properties: *ref_117 + required: *ref_118 CreateVariable: type: object - properties: *ref_118 - required: *ref_119 + properties: *ref_119 + required: *ref_120 EditVariable: type: object - properties: *ref_120 + properties: *ref_121 AuditLog: type: object properties: *ref_1 @@ -9821,68 +9961,68 @@ components: - error Preview: type: object - properties: *ref_121 - required: *ref_122 + properties: *ref_122 + required: *ref_123 CreateResource: type: object - properties: *ref_123 - required: *ref_124 + properties: *ref_124 + required: *ref_125 EditResource: type: object - properties: *ref_125 + properties: *ref_126 Resource: type: object - properties: *ref_126 - required: *ref_127 + properties: *ref_127 + required: *ref_128 ListableResource: type: object - properties: *ref_128 - required: *ref_129 + properties: *ref_129 + required: *ref_130 ResourceType: type: object - properties: *ref_20 - required: *ref_21 + properties: *ref_21 + required: *ref_22 EditResourceType: type: object - properties: *ref_130 + properties: *ref_131 Schedule: type: object - properties: *ref_73 - required: *ref_74 + properties: *ref_74 + required: *ref_75 ScheduleWJobs: - allOf: *ref_131 + allOf: *ref_132 NewSchedule: type: object - properties: *ref_132 - required: *ref_133 + properties: *ref_133 + required: *ref_134 EditSchedule: type: object - properties: *ref_134 - required: *ref_135 + properties: *ref_135 + required: *ref_136 Group: type: object - properties: *ref_78 - required: *ref_79 + properties: *ref_79 + required: *ref_80 InstanceGroup: type: object - properties: *ref_76 - required: *ref_77 + properties: *ref_77 + required: *ref_78 Folder: type: object - properties: *ref_80 - required: *ref_81 + properties: *ref_81 + required: *ref_82 WorkerPing: type: object - properties: *ref_136 - required: *ref_137 + properties: *ref_137 + required: *ref_138 UserWorkspaceList: type: object - properties: *ref_138 - required: *ref_139 + properties: *ref_139 + required: *ref_140 CreateWorkspace: type: object - properties: *ref_140 - required: *ref_141 + properties: *ref_141 + required: *ref_142 Workspace: type: object properties: *ref_5 @@ -9896,37 +10036,37 @@ components: properties: *ref_10 required: *ref_11 Flow: - allOf: *ref_42 + allOf: *ref_43 FlowMetadata: type: object - properties: *ref_142 - required: *ref_143 + properties: *ref_143 + required: *ref_144 OpenFlowWPath: - allOf: *ref_43 + allOf: *ref_44 FlowPreview: type: object - properties: *ref_144 - required: *ref_145 + properties: *ref_145 + required: *ref_146 RestartedFrom: type: object - properties: *ref_146 + properties: *ref_147 Policy: type: object - properties: *ref_44 + properties: *ref_45 ListableApp: type: object - properties: *ref_147 - required: *ref_148 + properties: *ref_148 + required: *ref_149 ListableRawApp: type: object - properties: *ref_149 - required: *ref_150 + properties: *ref_150 + required: *ref_151 AppWithLastVersion: type: object - properties: *ref_45 - required: *ref_46 + properties: *ref_46 + required: *ref_47 AppWithLastVersionWDraft: - allOf: *ref_151 + allOf: *ref_152 SlackToken: type: object properties: @@ -9948,80 +10088,110 @@ components: - bot TokenResponse: type: object - properties: *ref_152 - required: *ref_153 + properties: *ref_153 + required: *ref_154 HubScriptKind: name: kind - schema: *ref_28 + schema: *ref_29 + LargeFileStorage: + type: object + properties: *ref_15 + WindmillLargeFile: + type: object + properties: *ref_155 + required: *ref_156 + S3Resource: + type: object + properties: + bucket: + type: string + region: + type: string + endPoint: + type: string + useSSL: + type: boolean + accessKey: + type: string + secretKey: + type: string + pathStyle: + type: boolean + required: + - bucket + - region + - endPoint + - useSSL + - pathStyle StaticTransform: type: object - properties: *ref_154 - required: *ref_155 + properties: *ref_157 + required: *ref_158 JavascriptTransform: type: object - properties: *ref_156 - required: *ref_157 + properties: *ref_159 + required: *ref_160 InputTransform: - oneOf: *ref_23 - discriminator: *ref_24 + oneOf: *ref_24 + discriminator: *ref_25 RawScript: type: object - properties: *ref_158 - required: *ref_159 + properties: *ref_161 + required: *ref_162 PathScript: type: object - properties: *ref_160 - required: *ref_161 + properties: *ref_163 + required: *ref_164 PathFlow: type: object - properties: *ref_162 - required: *ref_163 + properties: *ref_165 + required: *ref_166 FlowModule: type: object - properties: *ref_25 - required: *ref_26 + properties: *ref_26 + required: *ref_27 ForloopFlow: type: object - properties: *ref_164 - required: *ref_165 + properties: *ref_167 + required: *ref_168 BranchOne: type: object - properties: *ref_166 - required: *ref_167 + properties: *ref_169 + required: *ref_170 BranchAll: type: object - properties: *ref_168 - required: *ref_169 + properties: *ref_171 + required: *ref_172 Identity: type: object - properties: *ref_170 - required: *ref_171 + properties: *ref_173 + required: *ref_174 Graphql: type: object - properties: *ref_172 - required: *ref_173 + properties: *ref_175 + required: *ref_176 FlowModuleValue: - oneOf: *ref_174 - discriminator: *ref_175 + oneOf: *ref_177 + discriminator: *ref_178 Retry: type: object - properties: *ref_176 + properties: *ref_179 FlowValue: type: object - properties: *ref_47 - required: *ref_48 + properties: *ref_48 + required: *ref_49 OpenFlow: type: object - properties: *ref_40 - required: *ref_41 + properties: *ref_41 + required: *ref_42 FlowStatusModule: type: object - properties: *ref_49 - required: *ref_50 + properties: *ref_50 + required: *ref_51 FlowStatus: type: object - properties: *ref_62 - required: *ref_63 + properties: *ref_63 + required: *ref_64 Http: type: object properties: diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 4a1c85d072..6536b04333 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1176,6 +1176,8 @@ paths: $ref: "#/components/schemas/ScriptArgs" error_handler_muted_on_cancel: type: boolean + large_file_storage: + $ref: "#/components/schemas/LargeFileStorage" required: - code_completion_enabled @@ -1469,6 +1471,49 @@ paths: schema: type: string + /w/{workspace}/workspaces/edit_large_file_storage_config: + post: + summary: edit large file storage settings + operationId: editLargeFileStorageConfig + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + description: LargeFileStorage info + required: true + content: + application/json: + schema: + type: object + properties: + large_file_storage: + $ref: "#/components/schemas/LargeFileStorage" + + responses: + "200": + description: status + content: + application/json: + schema: {} + + /w/{workspace}/workspaces/get_large_file_storage_config: + get: + summary: get large file storage config + operationId: getLargeFileStorageConfig + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + + responses: + "200": + description: status + content: + application/json: + schema: + $ref: "#/components/schemas/LargeFileStorage" + /w/{workspace}/users/list: get: summary: list users @@ -6231,6 +6276,69 @@ paths: type: string format: uuid + /w/{workspace}/job_helpers/duckdb_connection_settings: + post: + summary: Converts an S3 resource to the set of instructions necessary to connect DuckDB to an S3 bucket + operationId: duckdbConnectionSettings + tags: + - helpers + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + description: S3 resource to connect to + required: true + content: + application/json: + schema: + s3_resource: + $ref: "#/components/schemas/S3Resource" + responses: + "200": + description: Connection settings + content: + application/json: + schema: + type: object + properties: + connection_settings_str: + type: string + + /w/{workspace}/job_helpers/test_connection: + get: + summary: Test connection to the workspace datasets storage + operationId: datasetStorageTestConnection + tags: + - helpers + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: Connection settings + content: + application/json: + schema: {} + + /w/{workspace}/job_helpers/list_stored_datasets: + post: + summary: List the dataset keys available in the worspace datasets storage + operationId: polarsConnectionSettings + tags: + - helpers + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: Connection settings + content: + application/json: + schema: + type: object + properties: + dataset_keys: + type: array + items: + $ref: "#/components/schemas/WindmillLargeFile" + components: securitySchemes: bearerAuth: @@ -8206,3 +8314,46 @@ components: schema: type: string enum: [script, failure, trigger, approval] + + LargeFileStorage: + type: object + properties: + type: + type: string + enum: ["S3Storage"] + s3_resource_path: + type: string + + WindmillLargeFile: + type: object + properties: + s3: + type: string + s3_bucket: + type: string + required: + - s3 + + S3Resource: + type: object + properties: + bucket: + type: string + region: + type: string + endPoint: + type: string + useSSL: + type: boolean + accessKey: + type: string + secretKey: + type: string + pathStyle: + type: boolean + required: + - bucket + - region + - endPoint + - useSSL + - pathStyle diff --git a/backend/windmill-api/src/job_helpers.rs b/backend/windmill-api/src/job_helpers.rs new file mode 100644 index 0000000000..2c20d97381 --- /dev/null +++ b/backend/windmill-api/src/job_helpers.rs @@ -0,0 +1,306 @@ +use crate::{resources::transform_json_value, users::Tokened, workspaces::LargeFileStorage}; +use aws_sdk_s3::config::{Credentials, Region}; +use axum::{ + extract::Path, + routing::{get, post}, + Extension, Json, Router, +}; +use hyper::http; +use serde::{Deserialize, Serialize}; +use tower_http::cors::{Any, CorsLayer}; +use windmill_common::{db::UserDB, error}; + +use crate::db::ApiAuthed; + +pub fn workspaced_service() -> Router { + let cors = CorsLayer::new() + .allow_methods([http::Method::GET, http::Method::POST]) + .allow_headers([http::header::CONTENT_TYPE, http::header::AUTHORIZATION]) + .allow_origin(Any); + + Router::new() + .route( + "/duckdb_connection_settings", + post(duckdb_connection_settings).layer(cors.clone()), + ) + .route( + "/polars_connection_settings", + post(polars_connection_settings).layer(cors.clone()), + ) + .route("/test_connection", get(test_connection).layer(cors.clone())) + .route( + "/list_stored_datasets", + get(list_stored_datasets).layer(cors.clone()), + ) +} + +#[derive(Debug, Deserialize, Clone)] +struct S3Resource { + #[serde(rename = "bucket")] + bucket: String, + region: String, + #[serde(rename = "endPoint")] + endpoint: String, + #[serde(rename = "useSSL")] + use_ssl: bool, + #[serde(rename = "accessKey")] + access_key: Option, + #[serde(rename = "secretKey")] + secret_key: Option, + #[serde(rename = "pathStyle")] + path_style: bool, +} + +#[derive(Deserialize)] +struct DuckdbConnectionSettingsQuery { + s3_resource: S3Resource, +} +#[derive(Serialize)] +struct DuckdbConnectionSettingsResponse { + connection_settings_str: String, +} + +async fn duckdb_connection_settings( + Path(_w_id): Path, + Json(query): Json, +) -> error::JsonResult { + let mut duckdb_settings: String = String::new(); + + let s3_resource = query.s3_resource; + duckdb_settings.push_str(format!("SET home_directory='./';\n").as_str()); // TODO: make this configurable maybe, or point to a temporary folder + duckdb_settings.push_str(format!("INSTALL 'httpfs';\n").as_str()); + if s3_resource.path_style { + duckdb_settings.push_str(format!("SET s3_url_style='path';\n").as_str()); + } + duckdb_settings.push_str(format!("SET s3_region='{}';\n", s3_resource.region).as_str()); + duckdb_settings.push_str(format!("SET s3_endpoint='{}';\n", s3_resource.endpoint).as_str()); + if !s3_resource.use_ssl { + duckdb_settings.push_str(format!("SET s3_use_ssl=0;\n").as_str()); // default is true for DuckDB + } + if let Some(access_key_id) = s3_resource.access_key { + duckdb_settings.push_str(format!("SET s3_access_key_id='{}';\n", access_key_id).as_str()); + } + if let Some(secret_access_key) = s3_resource.secret_key { + duckdb_settings + .push_str(format!("SET s3_secret_access_key='{}';\n", secret_access_key).as_str()); + } + + let response = DuckdbConnectionSettingsResponse { connection_settings_str: duckdb_settings }; + return Ok(Json(response)); +} + +#[derive(Deserialize)] +struct PolarsConnectionSettingsQuery { + s3_resource: S3Resource, +} + +#[derive(Serialize)] +struct PolarsConnectionSettingsResponse { + endpoint_url: String, + #[serde(skip_serializing_if = "Option::is_none")] + key: Option, + #[serde(skip_serializing_if = "Option::is_none")] + secret: Option, + use_ssl: bool, + cache_regions: bool, + client_kwargs: PolarsConnectionSettings, +} + +#[derive(Serialize)] +struct PolarsConnectionSettings { + pub region_name: String, +} + +async fn polars_connection_settings( + Path(_w_id): Path, + Json(query): Json, +) -> error::JsonResult { + let s3_resource = query.s3_resource; + + let response = PolarsConnectionSettingsResponse { + endpoint_url: s3_resource.endpoint, + key: s3_resource.access_key, + secret: s3_resource.secret_key, + use_ssl: s3_resource.use_ssl, + cache_regions: false, + client_kwargs: PolarsConnectionSettings { region_name: s3_resource.region }, + }; + + return Ok(Json(response)); +} + +#[derive(Serialize)] +struct ListStoredDatasetsResponse { + windmill_large_files: Vec, +} + +#[derive(Serialize, Clone)] +struct WindmillLargeFile { + s3: String, + s3_bucket: Option, +} + +async fn test_connection( + authed: ApiAuthed, + Extension(user_db): Extension, + Tokened { token }: Tokened, + Path(w_id): Path, +) -> error::JsonResult<()> { + let s3_resource_opt = get_workspace_s3_resource(&authed, &user_db, &token, &w_id).await?; + if s3_resource_opt.is_none() { + return Err(error::Error::NotFound( + "No datasets storage resource defined at the workspace level".to_string(), + )); + } + + let s3_resource = s3_resource_opt.unwrap(); + let s3_client = build_s3_client(&s3_resource); + s3_client + .list_objects() + .bucket(s3_resource.bucket) + .max_keys(1) + .send() + .await + .map_err(|err| error::Error::InternalErr(err.to_string()))?; + return Ok(Json(())); +} + +async fn list_stored_datasets( + authed: ApiAuthed, + Extension(user_db): Extension, + Tokened { token }: Tokened, + Path(w_id): Path, +) -> error::JsonResult { + let s3_resource_opt = get_workspace_s3_resource(&authed, &user_db, &token, &w_id).await?; + + let s3_resource = s3_resource_opt.unwrap(); + let s3_client = build_s3_client(&s3_resource); + + let mut stored_datasets = Vec::::new(); + let s3_bucket = s3_resource.bucket; + loop { + let mut list_object_query = s3_client + .list_objects() + .bucket(s3_bucket.clone()) + .max_keys(32); + + if let Some(last_dataset_name) = stored_datasets.last() { + // if stored_datasets already has some elements, it means we're looping through pages + // according to AWS SDK docs, we can set the marker to the last returned dataset + list_object_query = list_object_query.set_marker(Some(last_dataset_name.clone().s3)) + } + + let bucket_objects = list_object_query + .send() + .await + .map_err(|err| error::Error::InternalErr(err.to_string()))?; + + let page_datasets = bucket_objects + .contents() + .iter() + .filter(|object| object.key().is_some()) + .map(|object| object.key()) + .map(Option::unwrap) + .map(&str::to_string) + .map(|object_key| WindmillLargeFile { + s3: object_key.clone(), + s3_bucket: Some(s3_bucket.clone()), + }) + .collect::>(); + + stored_datasets.extend(page_datasets.clone()); + + if !bucket_objects.is_truncated() { + break; + } + } + + return Ok(Json(ListStoredDatasetsResponse { + windmill_large_files: stored_datasets, + })); +} + +async fn get_workspace_s3_resource<'c>( + authed: &ApiAuthed, + user_db: &UserDB, + token: &str, + w_id: &str, +) -> error::Result> { + let mut tx = user_db.clone().begin(authed).await?; + let raw_lfs_opt = sqlx::query_scalar!( + "SELECT large_file_storage FROM workspace_settings WHERE workspace_id = $1", + w_id + ) + .fetch_optional(&mut *tx) + .await? + .flatten(); + tx.commit().await?; + + if raw_lfs_opt.is_none() { + return Ok(None); + } + + let large_file_storage = serde_json::from_value::( + raw_lfs_opt.unwrap(), + ) + .map_err(|err| { + tracing::error!( + "Value stored in large_file_storage column is invalid and could not be deserialized: {}", + err + ); + error::Error::InternalErr( + "Could not deserialize LargeFileStorage value found in database".to_string(), + ) + })?; + let s3_lfs = match large_file_storage { + LargeFileStorage::S3Storage(s3_lfs) => s3_lfs, + }; + + let resource_path_json_value = serde_json::to_value(s3_lfs.s3_resource_path) + .map_err(|err| error::Error::InternalErr(err.to_string()))?; + let interpolated_value = transform_json_value( + authed, + user_db, + &w_id, + resource_path_json_value, + &Option::None, + token, + ) + .await?; + + let s3_resource = serde_json::from_value::(interpolated_value) + .map_err(|err| error::Error::InternalErr(err.to_string()))?; + return Ok(Some(s3_resource)); +} + +fn build_s3_client(s3_resource_ref: &S3Resource) -> aws_sdk_s3::Client { + let s3_resource = s3_resource_ref.clone(); + + let endpoint_with_prefix = if s3_resource.endpoint.starts_with("http://") + || s3_resource.endpoint.starts_with("https://") + { + s3_resource.endpoint.clone() + } else if s3_resource.use_ssl { + format!("https://{}", s3_resource.endpoint) + } else { + format!("http://{}", s3_resource.endpoint) + }; + + let mut s3_config_builder = aws_sdk_s3::Config::builder() + .endpoint_url(endpoint_with_prefix) + .region(Region::new(s3_resource.region)); + if s3_resource.access_key.is_some() { + s3_config_builder = s3_config_builder.credentials_provider(Credentials::new( + s3_resource.access_key.unwrap_or_default(), + s3_resource.secret_key.unwrap_or_default(), + None, + None, + "s3_storage", + )); + } + if s3_resource.path_style { + s3_config_builder = s3_config_builder.force_path_style(true); + } + let s3_config = s3_config_builder.build(); + return aws_sdk_s3::Client::from_conf(s3_config); +} diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index 15d80bbc16..fc5237fa9d 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -57,6 +57,7 @@ mod granular_acls; mod groups; mod inputs; mod integration; +pub mod job_helpers; pub mod jobs; pub mod oauth2; mod openai; @@ -179,31 +180,32 @@ pub async fn run_server( // Reordered alphabetically .nest("/acls", granular_acls::workspaced_service()) .nest("/apps", apps::workspaced_service()) - .nest("/raw_apps", raw_apps::workspaced_service()) .nest("/audit", audit::workspaced_service()) .nest("/capture", capture::workspaced_service()) + .nest( + "/embeddings", + embeddings::workspaced_service(embeddings_db.clone()), + ) + .nest("/drafts", drafts::workspaced_service()) .nest("/favorites", favorite::workspaced_service()) .nest("/flows", flows::workspaced_service()) .nest("/folders", folders::workspaced_service()) .nest("/groups", groups::workspaced_service()) .nest("/inputs", inputs::workspaced_service()) + .nest("/job_helpers", job_helpers::workspaced_service()) .nest("/jobs", jobs::workspaced_service()) .nest("/oauth", oauth2::workspaced_service()) + .nest("/openai", openai::workspaced_service()) + .nest("/raw_apps", raw_apps::workspaced_service()) .nest("/resources", resources::workspaced_service()) .nest("/schedules", schedule::workspaced_service()) .nest("/scripts", scripts::workspaced_service()) - .nest("/drafts", drafts::workspaced_service()) .nest( "/users", users::workspaced_service().layer(Extension(argon2.clone())), ) .nest("/variables", variables::workspaced_service()) - .nest("/workspaces", workspaces::workspaced_service()) - .nest("/openai", openai::workspaced_service()) - .nest( - "/embeddings", - embeddings::workspaced_service(embeddings_db.clone()), - ), + .nest("/workspaces", workspaces::workspaced_service()), ) .nest("/workspaces", workspaces::global_service()) .nest( diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index eaa8ed6649..854211f86b 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -79,7 +79,8 @@ pub fn workspaced_service() -> Router { .route("/premium_info", get(premium_info)) .route("/edit_copilot_config", post(edit_copilot_config)) .route("/get_copilot_info", get(get_copilot_info) ) - .route("/edit_error_handler", post(edit_error_handler)); + .route("/edit_error_handler", post(edit_error_handler)) + .route("/edit_large_file_storage_config", post(edit_large_file_storage_config)); #[cfg(feature = "enterprise")] { @@ -143,6 +144,7 @@ pub struct WorkspaceSettings { pub error_handler: Option, pub error_handler_extra_args: Option, pub error_handler_muted_on_cancel: Option, + pub large_file_storage: Option, // effectively: DatasetsStorage } #[derive(FromRow, Serialize, Debug)] @@ -199,6 +201,11 @@ struct EditCopilotConfig { code_completion_enabled: bool, } +#[derive(Deserialize)] +struct EditLargeFileStorageConfig { + large_file_storage: Option, +} + #[derive(Deserialize)] struct CreateWorkspace { id: String, @@ -258,6 +265,18 @@ pub struct EditErrorHandler { pub error_handler_muted_on_cancel: Option, } +#[derive(Serialize, Deserialize, Debug)] +#[serde(tag = "type")] +pub enum LargeFileStorage { + S3Storage(S3Storage), + // TODO: Add a filesystem type here in the future if needed +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct S3Storage { + pub s3_resource_path: String, +} + async fn list_pending_invites( authed: ApiAuthed, Extension(user_db): Extension, @@ -885,6 +904,52 @@ async fn get_copilot_info( })) } +async fn edit_large_file_storage_config( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + ApiAuthed { is_admin, username, .. }: ApiAuthed, + Json(new_config): Json, +) -> Result { + require_admin(is_admin, &username)?; + + let mut tx = db.begin().await?; + + let args_for_audit = format!("{:?}", new_config.large_file_storage); + audit_log( + &mut *tx, + &authed.username, + "workspaces.edit_large_file_storage_config", + ActionKind::Update, + &w_id, + Some(&authed.email), + Some([("large_file_storage", args_for_audit.as_str())].into()), + ) + .await?; + + if let Some(lfs_config) = new_config.large_file_storage { + let serialized_lfs_config = serde_json::to_value::(lfs_config) + .map_err(|err| Error::InternalErr(err.to_string()))?; + + sqlx::query!( + "UPDATE workspace_settings SET large_file_storage = $1 WHERE workspace_id = $2", + serialized_lfs_config, + &w_id + ) + .execute(&mut *tx) + .await?; + } else { + sqlx::query!( + "UPDATE workspace_settings SET large_file_storage = NULL WHERE workspace_id = $1", + &w_id, + ) + .execute(&mut *tx) + .await?; + } + tx.commit().await?; + + Ok(format!("Edit copilot config for workspace {}", &w_id)) +} async fn edit_error_handler( authed: ApiAuthed, diff --git a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte index a802de4263..07dec52ab7 100644 --- a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte @@ -14,7 +14,13 @@ import Tooltip from '$lib/components/Tooltip.svelte' import WorkspaceUserSettings from '$lib/components/settings/WorkspaceUserSettings.svelte' import { WORKSPACE_SHOW_SLACK_CMD, WORKSPACE_SHOW_WEBHOOK_CLI_SYNC } from '$lib/consts' - import { OauthService, Script, WorkspaceService } from '$lib/gen' + import { + LargeFileStorage, + OauthService, + Script, + WorkspaceService, + HelpersService + } from '$lib/gen' import { enterpriseLicense, copilotInfo, @@ -47,6 +53,7 @@ let errorHandlerExtraArgs: Record = {} let errorHandlerMutedOnCancel: boolean | undefined = undefined let openaiResourceInitialPath: string | undefined = undefined + let s3ResourceInitialPath: string | undefined = undefined let codeCompletionEnabled: boolean = false let tab = ($page.url.searchParams.get('tab') as @@ -148,6 +155,31 @@ sendUserToast(`Copilot settings updated`) } + async function editWindmillLFSSettings(s3ResourcePath: string): Promise { + s3ResourceInitialPath = s3ResourcePath + if (s3ResourcePath) { + let resourcePathWithPrefix = `$res:${s3ResourcePath}` + await WorkspaceService.editLargeFileStorageConfig({ + workspace: $workspaceStore!, + requestBody: { + large_file_storage: { + type: LargeFileStorage.type.S3STORAGE, + s3_resource_path: resourcePathWithPrefix + } + } + }) + sendUserToast(`Large file storage settings updated`) + } else { + await WorkspaceService.editLargeFileStorageConfig({ + workspace: $workspaceStore!, + requestBody: { + large_file_storage: undefined + } + }) + sendUserToast(`Large file storage settings reset`) + } + } + async function loadSettings(): Promise { const settings = await WorkspaceService.getSettings({ workspace: $workspaceStore! }) team_name = settings.slack_name @@ -177,6 +209,11 @@ } errorHandlerExtraArgs = settings.error_handler_extra_args ?? {} codeCompletionEnabled = settings.code_completion_enabled + console.log(settings) + s3ResourceInitialPath = + settings.large_file_storage?.type === LargeFileStorage.type.S3STORAGE + ? settings.large_file_storage?.s3_resource_path?.replace('$res:', '') + : undefined } $: { @@ -259,7 +296,6 @@
Error Handler
 -
Windmill AI
 @@ -543,6 +579,33 @@ }} /> + {:else if tab == 'windmill_lfs'} + +
+ {#key s3ResourceInitialPath} + { + editWindmillLFSSettings(ev.detail) + }} + /> + {/key} + +
{/if} {:else}